From 82c708f603269f9a880859c4b9da481e4b9e1103 Mon Sep 17 00:00:00 2001 From: qstokkink Date: Thu, 2 Nov 2023 16:05:45 +0100 Subject: [PATCH] Made TorrentDef.load async and threaded --- src/tribler/core/components/conftest.py | 6 +- .../gigachannel_manager.py | 7 +- .../tests/test_gigachannel_manager.py | 12 +- .../download_manager/download_manager.py | 28 ++- .../restapi/create_torrent_endpoint.py | 2 +- .../restapi/torrentinfo_endpoint.py | 2 +- .../libtorrent/tests/test_download_manager.py | 48 ++-- .../libtorrent/tests/test_seeding.py | 2 +- .../libtorrent/tests/test_torrent_def.py | 10 +- .../core/components/libtorrent/torrentdef.py | 16 +- .../db/orm_bindings/collection_node.py | 27 ++- .../db/tests/test_torrent_metadata.py | 31 +-- .../restapi/channels_endpoint.py | 2 +- .../restapi/tests/test_channels_endpoint.py | 8 +- .../metadata_store/tests/gen_test_data.py | 80 ------- .../tests/test_channel_download.py | 10 +- .../tests/test_channel_metadata.py | 205 +++++++++--------- .../tests/test_create_torrent_endpoint.py | 4 +- .../test_tunnel_community.py | 12 +- .../watch_folder/tests/test_watch_folder.py | 26 ++- .../components/watch_folder/watch_folder.py | 16 +- 21 files changed, 245 insertions(+), 309 deletions(-) delete mode 100644 src/tribler/core/components/metadata_store/tests/gen_test_data.py diff --git a/src/tribler/core/components/conftest.py b/src/tribler/core/components/conftest.py index c4b52ad9aa5..0eb9a89fcb6 100644 --- a/src/tribler/core/components/conftest.py +++ b/src/tribler/core/components/conftest.py @@ -69,8 +69,8 @@ def mock_dlmgr(state_dir): @pytest.fixture -def video_tdef(): - return TorrentDef.load(TESTS_DATA_DIR / 'video.avi.torrent') +async def video_tdef(): + return await TorrentDef.load(TESTS_DATA_DIR / 'video.avi.torrent') @pytest.fixture @@ -90,7 +90,7 @@ async def video_seeder(tmp_path_factory, video_tdef): dlmgr.initialize() dscfg_seed = DownloadConfig() dscfg_seed.set_dest_dir(TESTS_DATA_DIR) - upload = dlmgr.start_download(tdef=video_tdef, config=dscfg_seed) + upload = await dlmgr.start_download(tdef=video_tdef, config=dscfg_seed) await upload.wait_for_status(DownloadStatus.SEEDING) yield dlmgr await dlmgr.shutdown() diff --git a/src/tribler/core/components/gigachannel_manager/gigachannel_manager.py b/src/tribler/core/components/gigachannel_manager/gigachannel_manager.py index fa007d59d76..e2ef763d4d1 100644 --- a/src/tribler/core/components/gigachannel_manager/gigachannel_manager.py +++ b/src/tribler/core/components/gigachannel_manager/gigachannel_manager.py @@ -252,7 +252,7 @@ async def download_channel(self, channel): dcfg.set_channel_download(True) tdef = TorrentDef(metainfo=metainfo) - download = self.download_manager.start_download(tdef=tdef, config=dcfg, hidden=True) + download = await self.download_manager.start_download(tdef=tdef, config=dcfg, hidden=True) try: await download.future_finished except CancelledError: @@ -279,7 +279,8 @@ def _process_download(): if updated_channel: self.notifier[notifications.channel_entity_updated](channel_dict) - def updated_my_channel(self, tdef): + @task + async def updated_my_channel(self, tdef): """ Notify the core that we updated our channel. """ @@ -293,7 +294,7 @@ def updated_my_channel(self, tdef): dcfg = DownloadConfig(state_dir=self.state_dir) dcfg.set_dest_dir(self.mds.channels_dir) dcfg.set_channel_download(True) - return self.download_manager.start_download(tdef=tdef, config=dcfg) + return await self.download_manager.start_download(tdef=tdef, config=dcfg) @db_session def clean_unsubscribed_channels(self): diff --git a/src/tribler/core/components/gigachannel_manager/tests/test_gigachannel_manager.py b/src/tribler/core/components/gigachannel_manager/tests/test_gigachannel_manager.py index 863dd2541d9..d814fe26308 100644 --- a/src/tribler/core/components/gigachannel_manager/tests/test_gigachannel_manager.py +++ b/src/tribler/core/components/gigachannel_manager/tests/test_gigachannel_manager.py @@ -30,11 +30,11 @@ def torrent_template(): @pytest.fixture -def personal_channel(metadata_store): +async def personal_channel(metadata_store): global update_metainfo + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) with db_session: chan = metadata_store.ChannelMetadata.create_channel(title="my test chan", description="test") - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) chan.add_torrent_to_channel(tdef, None) update_metainfo = chan.commit_channel_torrent() return chan @@ -108,11 +108,11 @@ async def mock_remove_download(download_obj, **_): gigachannel_manager.updated_my_channel.assert_called_once() -def test_updated_my_channel(personal_channel, gigachannel_manager, tmpdir): +async def test_updated_my_channel(personal_channel, gigachannel_manager, tmpdir): tdef = TorrentDef.load_from_dict(update_metainfo) - gigachannel_manager.download_manager.start_download = MagicMock() + gigachannel_manager.download_manager.start_download = AsyncMock() gigachannel_manager.download_manager.download_exists = lambda *_: False - gigachannel_manager.updated_my_channel(tdef) + await gigachannel_manager.updated_my_channel(tdef) gigachannel_manager.download_manager.start_download.assert_called_once() @@ -388,7 +388,7 @@ def mock_get_metainfo_good(*args, **kwargs): initiated_download = False - def mock_download_from_tdef(*_, **__): + async def mock_download_from_tdef(*_, **__): global initiated_download initiated_download = True mock_dl = MockObject() diff --git a/src/tribler/core/components/libtorrent/download_manager/download_manager.py b/src/tribler/core/components/libtorrent/download_manager/download_manager.py index e0e7e2deac3..6946e3effbe 100644 --- a/src/tribler/core/components/libtorrent/download_manager/download_manager.py +++ b/src/tribler/core/components/libtorrent/download_manager/download_manager.py @@ -13,7 +13,7 @@ from shutil import rmtree from typing import Callable, Dict, List, Optional -from ipv8.taskmanager import TaskManager, task +from ipv8.taskmanager import TaskManager from tribler.core import notifications from tribler.core.components.libtorrent.download_manager.dht_health_manager import DHTHealthManager @@ -491,7 +491,7 @@ async def get_metainfo(self, infohash: bytes, timeout: float = 30, hops: Optiona dcfg.set_upload_mode(True) # Upload mode should prevent libtorrent from creating files dcfg.set_dest_dir(self.metadata_tmpdir) try: - download = self.start_download(tdef=tdef, config=dcfg, hidden=True, checkpoint_disabled=True) + download = await self.start_download(tdef=tdef, config=dcfg, hidden=True, checkpoint_disabled=True) except TypeError as e: self._logger.warning(e) if raise_errors: @@ -550,7 +550,7 @@ async def start_download_from_uri(self, uri, config=None): if scheme in (HTTP_SCHEME, HTTPS_SCHEME): tdef = await TorrentDef.load_from_url(uri) - return self.start_download(tdef=tdef, config=config) + return await self.start_download(tdef=tdef, config=config) if scheme == MAGNET_SCHEME: name, infohash, _ = parse_magnetlink(uri) if infohash is None: @@ -559,14 +559,14 @@ async def start_download_from_uri(self, uri, config=None): tdef = TorrentDef.load_from_dict(self.metainfo_cache[infohash]['meta_info']) else: tdef = TorrentDefNoMetainfo(infohash, "Unknown name" if name is None else name, url=uri) - return self.start_download(tdef=tdef, config=config) + return await self.start_download(tdef=tdef, config=config) if scheme == FILE_SCHEME: file = url_to_path(uri) - return self.start_download(torrent_file=file, config=config) + return await self.start_download(torrent_file=file, config=config) raise Exception("invalid uri") - def start_download(self, torrent_file=None, tdef=None, config: DownloadConfig = None, checkpoint_disabled=False, - hidden=False) -> Download: + async def start_download(self, torrent_file=None, tdef=None, config: DownloadConfig = None, + checkpoint_disabled=False, hidden=False) -> Download: self._logger.debug(f'Starting download: filename: {torrent_file}, torrent def: {tdef}') if config is None: config = DownloadConfig.from_defaults(self.download_defaults) @@ -578,7 +578,7 @@ def start_download(self, torrent_file=None, tdef=None, config: DownloadConfig = if torrent_file is None: raise ValueError("Torrent file must be provided if tdef is not given") # try to get the torrent from the given torrent file - tdef = TorrentDef.load(torrent_file) + tdef = await TorrentDef.load(torrent_file) assert tdef is not None, "tdef MUST not be None after loading torrent" @@ -617,10 +617,9 @@ def start_download(self, torrent_file=None, tdef=None, config: DownloadConfig = if infohash not in self.metainfo_requests or self.metainfo_requests[infohash][0] == download: self.downloads[infohash] = download if not self.dummy_mode: - self.start_handle(download, atp) + await self.start_handle(download, atp) return download - @task async def start_handle(self, download, atp): atp_resume_data_skipped = atp.copy() resume_data = atp.get('resume_data') @@ -662,7 +661,6 @@ async def start_handle(self, download, atp): except asyncio.TimeoutError: self._logger.warning("Timeout waiting for libtorrent DHT getting enough peers") ltsession.async_add_torrent(encode_atp(atp)) - return await download.future_added def get_libtorrent_version(self): try: @@ -768,7 +766,7 @@ async def update_hops(self, download, new_hops): # If the user wants to change the hop count to 0, don't automatically bump this up to 1 anymore config.set_safe_seeding(False) - self.start_download(tdef=download.tdef, config=config) + await self.start_download(tdef=download.tdef, config=config) def update_trackers(self, infohash, trackers): """ Update the trackers for a download. @@ -862,13 +860,13 @@ async def load_checkpoints(self): checkpoint_filenames = list(self.get_checkpoint_dir().glob('*.conf')) self.checkpoints_count = len(checkpoint_filenames) for i, filename in enumerate(checkpoint_filenames, start=1): - self.load_checkpoint(filename) + await self.load_checkpoint(filename) self.checkpoints_loaded = i await sleep(.01) self.all_checkpoints_are_loaded = True self._logger.info("Checkpoints are loaded") - def load_checkpoint(self, filename): + async def load_checkpoint(self, filename): try: config = DownloadConfig.load(filename) except Exception: @@ -910,7 +908,7 @@ def load_checkpoint(self, filename): if self.download_exists(tdef.get_infohash()): self._logger.info("Not resuming checkpoint because download has already been added") else: - self.start_download(tdef=tdef, config=config) + await self.start_download(tdef=tdef, config=config) except Exception: self._logger.exception("Not resume checkpoint due to exception while adding download") diff --git a/src/tribler/core/components/libtorrent/restapi/create_torrent_endpoint.py b/src/tribler/core/components/libtorrent/restapi/create_torrent_endpoint.py index bc8c5acd69f..e4cbc62b24b 100644 --- a/src/tribler/core/components/libtorrent/restapi/create_torrent_endpoint.py +++ b/src/tribler/core/components/libtorrent/restapi/create_torrent_endpoint.py @@ -113,6 +113,6 @@ async def create_torrent(self, request): download_config = DownloadConfig() download_config.set_dest_dir(result['base_dir']) download_config.set_hops(self.download_manager.download_defaults.number_hops) - self.download_manager.start_download(tdef=TorrentDef(metainfo_dict), config=download_config) + await self.download_manager.start_download(tdef=TorrentDef(metainfo_dict), config=download_config) return RESTResponse(json.dumps({"torrent": base64.b64encode(result['metainfo']).decode('utf-8')})) diff --git a/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py b/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py index 6b0bf4be4ef..53a8859b7bf 100644 --- a/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py +++ b/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py @@ -93,7 +93,7 @@ async def get_torrent_info(self, request): if scheme == FILE_SCHEME: file = url_to_path(uri) try: - tdef = TorrentDef.load(file) + tdef = await TorrentDef.load(file) metainfo = tdef.metainfo except (FileNotFoundError, TypeError, ValueError, RuntimeError): return RESTResponse({"error": f"error while decoding torrent file: {file}"}, diff --git a/src/tribler/core/components/libtorrent/tests/test_download_manager.py b/src/tribler/core/components/libtorrent/tests/test_download_manager.py index bf1befdbf99..409a82ec0fb 100644 --- a/src/tribler/core/components/libtorrent/tests/test_download_manager.py +++ b/src/tribler/core/components/libtorrent/tests/test_download_manager.py @@ -1,6 +1,6 @@ import asyncio from asyncio import Future, gather, get_event_loop, sleep -from unittest.mock import MagicMock +from unittest.mock import MagicMock, AsyncMock import pytest from ipv8.util import succeed @@ -65,7 +65,7 @@ async def test_get_metainfo_valid_metadata(fake_dlmgr): download_impl.future_metainfo = succeed(metainfo) fake_dlmgr.initialize() - fake_dlmgr.start_download = MagicMock(return_value=download_impl) + fake_dlmgr.start_download = AsyncMock(return_value=download_impl) fake_dlmgr.download_defaults.number_hops = 1 fake_dlmgr.remove_download = MagicMock(return_value=succeed(None)) @@ -86,7 +86,7 @@ async def test_get_metainfo_add_fail(fake_dlmgr): download_impl.tdef.get_metainfo = lambda: None fake_dlmgr.initialize() - fake_dlmgr.start_download = MagicMock() + fake_dlmgr.start_download = AsyncMock() fake_dlmgr.start_download.side_effect = TypeError fake_dlmgr.download_defaults.number_hops = 1 fake_dlmgr.remove = MagicMock(return_value=succeed(None)) @@ -109,7 +109,7 @@ async def test_get_metainfo_duplicate_request(fake_dlmgr): get_event_loop().call_later(0.1, download_impl.future_metainfo.set_result, metainfo) fake_dlmgr.initialize() - fake_dlmgr.start_download = MagicMock(return_value=download_impl) + fake_dlmgr.start_download = AsyncMock(return_value=download_impl) fake_dlmgr.download_defaults.number_hops = 1 fake_dlmgr.remove_download = MagicMock(return_value=succeed(None)) @@ -134,7 +134,7 @@ async def test_get_metainfo_with_already_added_torrent(fake_dlmgr): Testing metainfo fetching for a torrent which is already in session. """ sample_torrent = TESTS_DATA_DIR / "bak_single.torrent" - torrent_def = TorrentDef.load(sample_torrent) + torrent_def = await TorrentDef.load(sample_torrent) download_impl = MagicMock() download_impl.future_metainfo = succeed(bencode(torrent_def.get_metainfo())) @@ -164,10 +164,10 @@ async def test_start_download_while_getting_metainfo(fake_dlmgr): fake_dlmgr.get_session = lambda *_: metainfo_session fake_dlmgr.downloads[infohash] = metainfo_dl fake_dlmgr.metainfo_requests[infohash] = [metainfo_dl, 1] - fake_dlmgr.remove_download = MagicMock(return_value=succeed(None)) + fake_dlmgr.remove_download = AsyncMock(return_value=succeed(None)) tdef = TorrentDefNoMetainfo(infohash, 'name', f'magnet:?xt=urn:btih:{hexlify(infohash)}&') - download = fake_dlmgr.start_download(tdef=tdef, checkpoint_disabled=True) + download = await fake_dlmgr.start_download(tdef=tdef, checkpoint_disabled=True) assert metainfo_dl != download await sleep(.1) assert fake_dlmgr.downloads[infohash] == download @@ -199,7 +199,7 @@ async def test_start_download(fake_dlmgr): fake_dlmgr.get_session = lambda *_: mock_ltsession - download = fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, ''), checkpoint_disabled=True) + download = await fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, ''), checkpoint_disabled=True) handle = await download.get_handle() assert handle == mock_handle fake_dlmgr.downloads.clear() @@ -249,14 +249,14 @@ async def test_start_download_existing_handle(fake_dlmgr): fake_dlmgr.get_session = lambda *_: mock_ltsession - download = fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True) + download = await fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True) handle = await download.get_handle() assert handle == mock_handle fake_dlmgr.downloads.clear() await download.shutdown() -def test_start_download_existing_download(fake_dlmgr): +async def test_start_download_existing_download(fake_dlmgr): """ Testing the addition of a torrent to the libtorrent manager, if there is a pre-existing download. """ @@ -270,18 +270,18 @@ def test_start_download_existing_download(fake_dlmgr): fake_dlmgr.downloads[infohash] = mock_download fake_dlmgr.get_session = lambda *_: mock_ltsession - download = fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True) + download = await fake_dlmgr.start_download(tdef=TorrentDefNoMetainfo(infohash, 'name'), checkpoint_disabled=True) assert download == mock_download fake_dlmgr.downloads.clear() -def test_start_download_no_ti_url(fake_dlmgr): +async def test_start_download_no_ti_url(fake_dlmgr): """ Test whether a ValueError is raised if we try to add a torrent without infohash or url """ fake_dlmgr.initialize() with pytest.raises(ValueError): - fake_dlmgr.start_download() + await fake_dlmgr.start_download() def test_remove_unregistered_torrent(fake_dlmgr): @@ -349,27 +349,27 @@ def test_post_session_stats(fake_dlmgr): mock_lt_session.post_session_stats.assert_called_once() -def test_load_checkpoint(fake_dlmgr): +async def test_load_checkpoint(fake_dlmgr): good = [] - def mock_start_download(*_, **__): + async def mock_start_download(*_, **__): good.append(1) fake_dlmgr.start_download = mock_start_download # Try opening real state file state = TESTS_DATA_DIR / "config_files/13a25451c761b1482d3e85432f07c4be05ca8a56.conf" - fake_dlmgr.load_checkpoint(state) + await fake_dlmgr.load_checkpoint(state) assert good # Try opening nonexistent file good = [] - fake_dlmgr.load_checkpoint("nonexistent_file") + await fake_dlmgr.load_checkpoint("nonexistent_file") assert not good # Try opening corrupt file config_file_path = TESTS_DATA_DIR / "config_files/corrupt_session_config.conf" - fake_dlmgr.load_checkpoint(config_file_path) + await fake_dlmgr.load_checkpoint(config_file_path) assert not good @@ -380,19 +380,19 @@ async def test_download_manager_start(fake_dlmgr): assert fake_dlmgr.all_checkpoints_are_loaded -def test_load_empty_checkpoint(fake_dlmgr, tmpdir): +async def test_load_empty_checkpoint(fake_dlmgr, tmpdir): """ Test whether download resumes with faulty pstate file. """ fake_dlmgr.get_downloads_pstate_dir = lambda: tmpdir - fake_dlmgr.start_download = MagicMock() + fake_dlmgr.start_download = AsyncMock() # Empty pstate file pstate_filename = fake_dlmgr.get_downloads_pstate_dir() / 'abcd.state' with open(pstate_filename, 'wb') as state_file: state_file.write(b"") - fake_dlmgr.load_checkpoint(pstate_filename) + await fake_dlmgr.load_checkpoint(pstate_filename) fake_dlmgr.start_download.assert_not_called() @@ -401,7 +401,7 @@ async def test_load_checkpoints(fake_dlmgr, tmpdir): Test whether we are resuming downloads after loading checkpoints """ - def mocked_load_checkpoint(filename): + async def mocked_load_checkpoint(filename): assert str(filename).endswith('abcd.conf') mocked_load_checkpoint.called = True @@ -442,8 +442,8 @@ async def mocked_update_hops(*_): await readd_future -def test_get_downloads_by_name(fake_dlmgr): - dl = fake_dlmgr.start_download(torrent_file=TORRENT_UBUNTU_FILE, checkpoint_disabled=True) +async def test_get_downloads_by_name(fake_dlmgr): + dl = await fake_dlmgr.start_download(torrent_file=TORRENT_UBUNTU_FILE, checkpoint_disabled=True) assert fake_dlmgr.get_downloads_by_name("ubuntu-15.04-desktop-amd64.iso") assert not fake_dlmgr.get_downloads_by_name("ubuntu-15.04-desktop-amd64.iso", channels_only=True) assert not fake_dlmgr.get_downloads_by_name("bla") diff --git a/src/tribler/core/components/libtorrent/tests/test_seeding.py b/src/tribler/core/components/libtorrent/tests/test_seeding.py index 4764ffe39c3..7f1ef91124c 100644 --- a/src/tribler/core/components/libtorrent/tests/test_seeding.py +++ b/src/tribler/core/components/libtorrent/tests/test_seeding.py @@ -15,7 +15,7 @@ async def test_seeding(download_manager, video_seeder, video_tdef, tmp_path): """ dscfg = DownloadConfig() dscfg.set_dest_dir(tmp_path) - download = download_manager.start_download(tdef=video_tdef, config=dscfg) + download = await download_manager.start_download(tdef=video_tdef, config=dscfg) download.add_peer(("127.0.0.1", video_seeder.libtorrent_port)) await download.wait_for_status(DownloadStatus.SEEDING) diff --git a/src/tribler/core/components/libtorrent/tests/test_torrent_def.py b/src/tribler/core/components/libtorrent/tests/test_torrent_def.py index 009222abc06..9512d326fdc 100644 --- a/src/tribler/core/components/libtorrent/tests/test_torrent_def.py +++ b/src/tribler/core/components/libtorrent/tests/test_torrent_def.py @@ -107,15 +107,15 @@ def test_is_private(tdef): assert tdef.is_private() is False -def test_is_private_loaded_from_existing_torrent(): +async def test_is_private_loaded_from_existing_torrent(): """ Test whether the private field from an existing torrent is correctly read """ privatefn = TESTS_DATA_DIR / "private.torrent" publicfn = TESTS_DATA_DIR / "bak_single.torrent" - t1 = TorrentDef.load(privatefn) - t2 = TorrentDef.load(publicfn) + t1 = await TorrentDef.load(privatefn) + t2 = await TorrentDef.load(publicfn) assert t1.is_private() assert not t2.is_private() @@ -126,8 +126,8 @@ async def test_load_from_url(file_server, tmpdir): torrent_url = 'http://localhost:%d/ubuntu.torrent' % file_server torrent_def = await TorrentDef.load_from_url(torrent_url) - assert torrent_def.get_metainfo() == TorrentDef.load(TORRENT_UBUNTU_FILE).get_metainfo() - assert torrent_def.infohash == TorrentDef.load(TORRENT_UBUNTU_FILE).infohash + assert torrent_def.get_metainfo() == (await TorrentDef.load(TORRENT_UBUNTU_FILE)).get_metainfo() + assert torrent_def.infohash == (await TorrentDef.load(TORRENT_UBUNTU_FILE)).infohash async def test_load_from_url_404(file_server, tmpdir): diff --git a/src/tribler/core/components/libtorrent/torrentdef.py b/src/tribler/core/components/libtorrent/torrentdef.py index 952792bc1db..98c15a1e0d3 100644 --- a/src/tribler/core/components/libtorrent/torrentdef.py +++ b/src/tribler/core/components/libtorrent/torrentdef.py @@ -3,6 +3,7 @@ """ import itertools import logging +from asyncio import get_running_loop from hashlib import sha1 import aiohttp @@ -95,15 +96,24 @@ def copy_metainfo_to_torrent_parameters(self): self.torrent_parameters[key] = self.metainfo[b'info'][key] @staticmethod - def load(filepath): + def _threaded_load_job(filepath): """ - Create a TorrentDef object from a .torrent file - :param filepath: The path to the .torrent file + Perform the actual loading of the torrent. + + Called from a thread: don't call this directly! """ with open(filepath, "rb") as torrent_file: file_content = torrent_file.read() return TorrentDef.load_from_memory(file_content) + @staticmethod + async def load(filepath): + """ + Create a TorrentDef object from a .torrent file + :param filepath: The path to the .torrent file + """ + return await get_running_loop().run_in_executor(None, TorrentDef._threaded_load_job, filepath) + @staticmethod def load_from_memory(bencoded_data): """ diff --git a/src/tribler/core/components/metadata_store/db/orm_bindings/collection_node.py b/src/tribler/core/components/metadata_store/db/orm_bindings/collection_node.py index 2b02b4d0d9b..8c2b99fb461 100644 --- a/src/tribler/core/components/metadata_store/db/orm_bindings/collection_node.py +++ b/src/tribler/core/components/metadata_store/db/orm_bindings/collection_node.py @@ -193,8 +193,7 @@ def get_contents_recursive(self): results_stack.append(subnode) return results_stack - @db_session - def add_torrents_from_dir(self, torrents_dir, recursive=False): + async def add_torrents_from_dir(self, torrents_dir, recursive=False): torrents_list = [] errors_list = [] @@ -208,17 +207,21 @@ def rec_gen(dir_): torrents_list_generator = (Path(torrents_dir, f) for f in filename_generator) torrents_list = [f for f in torrents_list_generator if f.is_file() and f.suffix == ".torrent"] + torrent_defs = [] + for filename in torrents_list: + try: + torrent_defs.append(await TorrentDef.load(filename)) + except Exception: # pylint: disable=W0703 + # Have to use the broad exception clause because Py3 versions of libtorrent + # generate generic Exceptions + errors_list.append(filename) + # 100 is a reasonable chunk size for commits - for chunk in chunks(torrents_list, 100): - for f in chunk: - try: - self.add_torrent_to_channel(TorrentDef.load(f)) - except Exception: # pylint: disable=W0703 - # Have to use the broad exception clause because Py3 versions of libtorrent - # generate generic Exceptions - errors_list.append(f) - # Optimization to drop excess cache - orm.commit() + with db_session: + for chunk in chunks(torrent_defs, 100): + for tdef in chunk: + self.add_torrent_to_channel(tdef) + orm.commit() return torrents_list, errors_list diff --git a/src/tribler/core/components/metadata_store/db/tests/test_torrent_metadata.py b/src/tribler/core/components/metadata_store/db/tests/test_torrent_metadata.py index 7c8c3a2db27..2b56a00bc1f 100644 --- a/src/tribler/core/components/metadata_store/db/tests/test_torrent_metadata.py +++ b/src/tribler/core/components/metadata_store/db/tests/test_torrent_metadata.py @@ -34,29 +34,30 @@ def test_serialization(metadata_store): assert torrent_metadata.serialized() -@db_session -def test_create_ffa_from_dict(metadata_store): +async def test_create_ffa_from_dict(metadata_store): """ Test creating a free-for-all torrent entry """ - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) - # Make sure that FFA entry with the infohash that is already known to GigaChannel cannot be created - signed_entry = metadata_store.TorrentMetadata.from_dict(tdef_to_metadata_dict(tdef)) - metadata_store.TorrentMetadata.add_ffa_from_dict(tdef_to_metadata_dict(tdef)) - assert metadata_store.TorrentMetadata.select(lambda g: g.public_key == EMPTY_BLOB).count() == 0 + with db_session: + # Make sure that FFA entry with the infohash that is already known to GigaChannel cannot be created + signed_entry = metadata_store.TorrentMetadata.from_dict(tdef_to_metadata_dict(tdef)) + metadata_store.TorrentMetadata.add_ffa_from_dict(tdef_to_metadata_dict(tdef)) + assert metadata_store.TorrentMetadata.select(lambda g: g.public_key == EMPTY_BLOB).count() == 0 - signed_entry.delete() - # Create FFA entry - metadata_store.TorrentMetadata.add_ffa_from_dict(tdef_to_metadata_dict(tdef)) - assert metadata_store.TorrentMetadata.select(lambda g: g.public_key == EMPTY_BLOB).count() == 1 + signed_entry.delete() + # Create FFA entry + metadata_store.TorrentMetadata.add_ffa_from_dict(tdef_to_metadata_dict(tdef)) + assert metadata_store.TorrentMetadata.select(lambda g: g.public_key == EMPTY_BLOB).count() == 1 -@db_session -def test_sanitize_tdef(metadata_store): - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) +async def test_sanitize_tdef(metadata_store): + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) tdef.metainfo["creation date"] = -100000 - assert metadata_store.TorrentMetadata.from_dict(tdef_to_metadata_dict(tdef)) + + with db_session: + assert metadata_store.TorrentMetadata.from_dict(tdef_to_metadata_dict(tdef)) @db_session diff --git a/src/tribler/core/components/metadata_store/restapi/channels_endpoint.py b/src/tribler/core/components/metadata_store/restapi/channels_endpoint.py index 26c11325db0..2a25c6ab5b1 100644 --- a/src/tribler/core/components/metadata_store/restapi/channels_endpoint.py +++ b/src/tribler/core/components/metadata_store/restapi/channels_endpoint.py @@ -434,7 +434,7 @@ async def add_torrent_to_channel(self, request): ) if torrents_dir: - torrents_list, errors_list = channel.add_torrents_from_dir(torrents_dir, recursive) + torrents_list, errors_list = await channel.add_torrents_from_dir(torrents_dir, recursive) return RESTResponse({"added": len(torrents_list), "errors": errors_list}) if not parameters.get('torrent', None): diff --git a/src/tribler/core/components/metadata_store/restapi/tests/test_channels_endpoint.py b/src/tribler/core/components/metadata_store/restapi/tests/test_channels_endpoint.py index 9d342ca991c..8b198732e46 100644 --- a/src/tribler/core/components/metadata_store/restapi/tests/test_channels_endpoint.py +++ b/src/tribler/core/components/metadata_store/restapi/tests/test_channels_endpoint.py @@ -427,8 +427,8 @@ async def test_add_torrent_duplicate(my_channel, rest_api): """ Test that adding a duplicate torrent to you channel does not result in an error """ + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) with db_session: - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) my_channel.add_torrent_to_channel(tdef, {'description': 'blabla'}) with open(TORRENT_UBUNTU_FILE, "rb") as torrent_file: @@ -497,12 +497,6 @@ async def test_add_torrent_from_magnet(my_channel, mock_dlmgr, rest_api, metadat """ Test whether we can add a torrent to your channel from a magnet link """ - - def fake_get_metainfo(_, **__): - meta_info = TorrentDef.load(TORRENT_UBUNTU_FILE).get_metainfo() - return succeed(meta_info) - - mock_dlmgr.get_metainfo = fake_get_metainfo metadata_store.torrent_exists_in_personal_channel = Mock() post_params = {'uri': 'magnet:?xt=urn:btih:1111111111111111111111111111111111111111'} diff --git a/src/tribler/core/components/metadata_store/tests/gen_test_data.py b/src/tribler/core/components/metadata_store/tests/gen_test_data.py deleted file mode 100644 index e56001f463f..00000000000 --- a/src/tribler/core/components/metadata_store/tests/gen_test_data.py +++ /dev/null @@ -1,80 +0,0 @@ -import os -import random -from datetime import datetime - -from ipv8.keyvault.crypto import default_eccrypto -from pony.orm import db_session - -from tribler.core.components.libtorrent.torrentdef import TorrentDef -from tribler.core.components.metadata_store.db.orm_bindings.channel_node import NEW -from tribler.core.components.metadata_store.db.store import MetadataStore -from tribler.core.components.metadata_store.tests.test_channel_download import ( - CHANNEL_METADATA, - CHANNEL_METADATA_UPDATED, - CHANNEL_TORRENT, - CHANNEL_TORRENT_UPDATED, -) -from tribler.core.tests.tools.common import TORRENT_UBUNTU_FILE, TORRENT_VIDEO_FILE -from tribler.core.utilities.path_util import Path -from tribler.core.utilities.utilities import MEMORY_DB - -DATA_DIR = Path(__file__).parent / '..' / '..' / 'data' -SAMPLE_DIR = DATA_DIR / 'sample_channel' - -my_key = default_eccrypto.generate_key("curve25519") - -ALL_PRINTABLE_CHARS = ''.join(tuple(chr(i) for i in range(32, 0x110000) if chr(i).isprintable())) - - -def get_random_text_string(size=200): - return "".join(random.sample(ALL_PRINTABLE_CHARS, size)) - - -def gen_random_entry(): - return { - "title": "test entry " + str(random.randint(0, 1000000)), - "infohash": str(random.getrandbits(160)), - "torrent_date": datetime(1970, 1, 1), - "size": 100 + random.randint(0, 10000), - "tags": "video", - "status": NEW, - } - - -@db_session -def gen_sample_channel(mds): - my_channel = mds.ChannelMetadata.create_channel('test_channel', 'test description') - - my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_UBUNTU_FILE), None) - my_channel.commit_channel_torrent() - - t2 = my_channel.add_torrent_to_channel(TorrentDef.load(TORRENT_VIDEO_FILE), None) - mds.TorrentMetadata.from_dict(dict(origin_id=my_channel.id_, **gen_random_entry())) - mds.TorrentMetadata.from_dict(dict(origin_id=my_channel.id_, **gen_random_entry())) - coll = mds.CollectionNode(origin_id=my_channel.id_, title='internal collection') - mds.TorrentMetadata.from_dict(dict(origin_id=coll.id_, **gen_random_entry())) - mds.TorrentMetadata.from_dict(dict(origin_id=coll.id_, **gen_random_entry())) - my_channel.commit_channel_torrent() - - t2.soft_delete() - my_channel.commit_channel_torrent() - - # Rename files to stable names - mdblob_name = SAMPLE_DIR / (my_channel.dirname + ".mdblob") - torrent_name = SAMPLE_DIR / (my_channel.dirname + ".torrent") - - os.rename(mdblob_name, CHANNEL_METADATA) - os.rename(torrent_name, CHANNEL_TORRENT) - - # Update channel - mds.TorrentMetadata.from_dict(dict(origin_id=my_channel.id_, **gen_random_entry())) - my_channel.commit_channel_torrent() - - # Rename updated files to stable names - os.rename(mdblob_name, CHANNEL_METADATA_UPDATED) - os.rename(torrent_name, CHANNEL_TORRENT_UPDATED) - - -if __name__ == "__main__": - mds = MetadataStore(MEMORY_DB, SAMPLE_DIR, my_key) - gen_sample_channel(mds) diff --git a/src/tribler/core/components/metadata_store/tests/test_channel_download.py b/src/tribler/core/components/metadata_store/tests/test_channel_download.py index 29d3161b25e..9ab58b0eae8 100644 --- a/src/tribler/core/components/metadata_store/tests/test_channel_download.py +++ b/src/tribler/core/components/metadata_store/tests/test_channel_download.py @@ -23,8 +23,8 @@ # pylint: disable=redefined-outer-name @pytest.fixture -def channel_tdef(): - return TorrentDef.load(TESTS_DATA_DIR / 'sample_channel' / 'channel_upd.torrent') +async def channel_tdef(): + return await TorrentDef.load(TESTS_DATA_DIR / 'sample_channel' / 'channel_upd.torrent') @pytest.fixture @@ -40,7 +40,7 @@ async def channel_seeder(channel_tdef, tmp_path_factory): # pylint: disable=unu seeder_dlmgr.initialize() dscfg_seed = DownloadConfig() dscfg_seed.set_dest_dir(TESTS_DATA_DIR / 'sample_channel') - upload = seeder_dlmgr.start_download(tdef=channel_tdef, config=dscfg_seed) + upload = await seeder_dlmgr.start_download(tdef=channel_tdef, config=dscfg_seed) await upload.wait_for_status(DownloadStatus.SEEDING) yield seeder_dlmgr await seeder_dlmgr.shutdown() @@ -88,8 +88,8 @@ def fake_get_metainfo(*args, **kwargs): # and get_metainfo to provide the hint. original_start_download_from_tdef = download_manager.start_download - def hinted_start_download(tdef=None, config=None, hidden=False): - download = original_start_download_from_tdef(tdef=tdef, config=config, hidden=hidden) + async def hinted_start_download(tdef=None, config=None, hidden=False): + download = await original_start_download_from_tdef(tdef=tdef, config=config, hidden=hidden) download.add_peer(("127.0.0.1", channel_seeder.libtorrent_port)) return download diff --git a/src/tribler/core/components/metadata_store/tests/test_channel_metadata.py b/src/tribler/core/components/metadata_store/tests/test_channel_metadata.py index e0ad307c403..f1a6666072d 100644 --- a/src/tribler/core/components/metadata_store/tests/test_channel_metadata.py +++ b/src/tribler/core/components/metadata_store/tests/test_channel_metadata.py @@ -197,18 +197,20 @@ def test_add_metadata_to_channel(torrent_template, metadata_store): assert channel_metadata.num_entries == 1 -@db_session -def test_add_torrent_to_channel(metadata_store): +async def test_add_torrent_to_channel(metadata_store): """ Test adding a torrent to your channel """ - channel_metadata = metadata_store.ChannelMetadata.create_channel('test', 'test') - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) - channel_metadata.add_torrent_to_channel(tdef, {'description': 'blabla'}) - assert channel_metadata.contents_list + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) + + with db_session: + channel_metadata = metadata_store.ChannelMetadata.create_channel('test', 'test') + + channel_metadata.add_torrent_to_channel(tdef, {'description': 'blabla'}) + assert channel_metadata.contents_list - # Make sure trying to add a duplicate torrent does not result in an error - channel_metadata.add_torrent_to_channel(tdef, None) + # Make sure trying to add a duplicate torrent does not result in an error + channel_metadata.add_torrent_to_channel(tdef, None) @db_session @@ -247,56 +249,58 @@ def test_copy_to_channel(torrent_template, metadata_store): assert len(channel2.contents_list) == 1 -@db_session -def test_restore_torrent_in_channel(metadata_store): +async def test_restore_torrent_in_channel(metadata_store): """ Test if the torrent scheduled for deletion is restored/updated after the user tries to re-add it. """ - channel_metadata = metadata_store.ChannelMetadata.create_channel('test', 'test') - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) - md = channel_metadata.add_torrent_to_channel(tdef, None) - - # Check correct re-add - md.status = TODELETE - md_updated = channel_metadata.add_torrent_to_channel(tdef, None) - assert UPDATED == md.status - assert md_updated == md - assert md.has_valid_signature - - # Check update of torrent properties from a new tdef - md.status = TODELETE - new_tracker_address = 'http://tribler.org/announce' - tdef.torrent_parameters[b'announce'] = new_tracker_address.encode('utf-8') - md_updated = channel_metadata.add_torrent_to_channel(tdef, None) - assert md_updated == md - assert md.status == UPDATED - assert md.tracker_info == new_tracker_address - assert md.has_valid_signature - # In addition, check that the trackers table was properly updated - assert len(md.health.trackers) == 2 + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) - -@db_session -def test_delete_torrent_from_channel(metadata_store): + with db_session: + channel_metadata = metadata_store.ChannelMetadata.create_channel('test', 'test') + md = channel_metadata.add_torrent_to_channel(tdef, None) + + # Check correct re-add + md.status = TODELETE + md_updated = channel_metadata.add_torrent_to_channel(tdef, None) + assert UPDATED == md.status + assert md_updated == md + assert md.has_valid_signature + + # Check update of torrent properties from a new tdef + md.status = TODELETE + new_tracker_address = 'http://tribler.org/announce' + tdef.torrent_parameters[b'announce'] = new_tracker_address.encode('utf-8') + md_updated = channel_metadata.add_torrent_to_channel(tdef, None) + assert md_updated == md + assert md.status == UPDATED + assert md.tracker_info == new_tracker_address + assert md.has_valid_signature + # In addition, check that the trackers table was properly updated + assert len(md.health.trackers) == 2 + + +async def test_delete_torrent_from_channel(metadata_store): """ Test deleting a torrent from your channel """ - channel_metadata = metadata_store.ChannelMetadata.create_channel('test', 'test') - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) - # Check that nothing is committed when deleting uncommited torrent metadata - torrent = channel_metadata.add_torrent_to_channel(tdef, None) - torrent.soft_delete() - assert not channel_metadata.contents_list + with db_session: + channel_metadata = metadata_store.ChannelMetadata.create_channel('test', 'test') - # Check append-only deletion process - torrent = channel_metadata.add_torrent_to_channel(tdef, None) - channel_metadata.commit_channel_torrent() - assert len(channel_metadata.contents_list) == 1 + # Check that nothing is committed when deleting uncommited torrent metadata + torrent = channel_metadata.add_torrent_to_channel(tdef, None) + torrent.soft_delete() + assert not channel_metadata.contents_list - torrent.soft_delete() - channel_metadata.commit_channel_torrent() - assert not channel_metadata.contents_list + # Check append-only deletion process + torrent = channel_metadata.add_torrent_to_channel(tdef, None) + channel_metadata.commit_channel_torrent() + assert len(channel_metadata.contents_list) == 1 + + torrent.soft_delete() + channel_metadata.commit_channel_torrent() + assert not channel_metadata.contents_list @db_session @@ -369,24 +373,25 @@ def test_vsids(freezer, metadata_store): assert 2.0 < channel.votes < 2.5 -@db_session -def test_commit_channel_torrent(metadata_store): +async def test_commit_channel_torrent(metadata_store): """ Test committing a channel torrent """ - channel = metadata_store.ChannelMetadata.create_channel('test', 'test') - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) - channel.add_torrent_to_channel(tdef, None) - # The first run should return the infohash, the second should return None, because nothing was really done - assert channel.commit_channel_torrent() - assert not channel.commit_channel_torrent() + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) - # Test adding flags to channel torrent when adding thumbnail and description - metadata_store.ChannelThumbnail(public_key=channel.public_key, origin_id=channel.id_, status=NEW) - metadata_store.ChannelDescription(public_key=channel.public_key, origin_id=channel.id_, status=NEW) - assert channel.commit_channel_torrent() - assert channel.reserved_flags == 3 - assert not channel.commit_channel_torrent() + with db_session: + channel = metadata_store.ChannelMetadata.create_channel('test', 'test') + channel.add_torrent_to_channel(tdef, None) + # The first run should return the infohash, the second should return None, because nothing was really done + assert channel.commit_channel_torrent() + assert not channel.commit_channel_torrent() + + # Test adding flags to channel torrent when adding thumbnail and description + metadata_store.ChannelThumbnail(public_key=channel.public_key, origin_id=channel.id_, status=NEW) + metadata_store.ChannelDescription(public_key=channel.public_key, origin_id=channel.id_, status=NEW) + assert channel.commit_channel_torrent() + assert channel.reserved_flags == 3 + assert not channel.commit_channel_torrent() @db_session @@ -481,43 +486,45 @@ def generate_channel(recurse=False, status=NEW): assert chan.num_entries == 366 -@db_session -def test_consolidate_channel_torrent(torrent_template, metadata_store): +async def test_consolidate_channel_torrent(torrent_template, metadata_store): """ Test completely re-commit your channel """ - channel = metadata_store.ChannelMetadata.create_channel('test', 'test') - my_dir = Path(metadata_store.ChannelMetadata._channels_dir / channel.dirname).absolute() - tdef = TorrentDef.load(TORRENT_UBUNTU_FILE) + tdef = await TorrentDef.load(TORRENT_UBUNTU_FILE) - # 1st torrent - torrent_entry = channel.add_torrent_to_channel(tdef, None) - channel.commit_channel_torrent() + with db_session: + channel = metadata_store.ChannelMetadata.create_channel('test', 'test') + my_dir = Path(metadata_store.ChannelMetadata._channels_dir / channel.dirname).absolute() - # 2nd torrent - metadata_store.TorrentMetadata.from_dict( - dict(torrent_template, public_key=channel.public_key, origin_id=channel.id_, status=NEW) - ) - channel.commit_channel_torrent() - # Delete entry - torrent_entry.soft_delete() - channel.commit_channel_torrent() + # 1st torrent + torrent_entry = channel.add_torrent_to_channel(tdef, None) + channel.commit_channel_torrent() + + # 2nd torrent + metadata_store.TorrentMetadata.from_dict( + dict(torrent_template, public_key=channel.public_key, origin_id=channel.id_, status=NEW) + ) + channel.commit_channel_torrent() + # Delete entry + torrent_entry.soft_delete() + channel.commit_channel_torrent() - assert len(channel.contents_list) == 1 - assert len(os.listdir(my_dir)) == 3 + assert len(channel.contents_list) == 1 + assert len(os.listdir(my_dir)) == 3 - torrent3 = metadata_store.TorrentMetadata( - public_key=channel.public_key, origin_id=channel.id_, status=NEW, infohash=random_infohash() - ) - channel.commit_channel_torrent() - torrent3.soft_delete() + torrent3 = metadata_store.TorrentMetadata( + public_key=channel.public_key, origin_id=channel.id_, status=NEW, infohash=random_infohash() + ) + channel.commit_channel_torrent() + torrent3.soft_delete() - channel.consolidate_channel_torrent() - assert len(os.listdir(my_dir)) == 1 - metadata_store.TorrentMetadata.select(lambda g: g.metadata_type == REGULAR_TORRENT).delete() - channel.local_version = 0 - metadata_store.process_channel_dir(my_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False) - assert len(channel.contents[:]) == 1 + channel.consolidate_channel_torrent() + assert len(os.listdir(my_dir)) == 1 + metadata_store.TorrentMetadata.select(lambda g: g.metadata_type == REGULAR_TORRENT).delete() + channel.local_version = 0 + metadata_store.process_channel_dir(my_dir, channel.public_key, channel.id_, + skip_personal_metadata_payload=False) + assert len(channel.contents[:]) == 1 @db_session @@ -856,11 +863,11 @@ def test_get_channel_name(metadata_store): metadata_store.ChannelMetadata.get_channel_name.assert_not_called() -@db_session -def check_add(metadata_store, torrents_in_dir, errors, recursive): +async def check_add(metadata_store, torrents_in_dir, errors, recursive): TEST_TORRENTS_DIR = TESTS_DATA_DIR / 'linux_torrents' - chan = metadata_store.ChannelMetadata.create_channel(title='testchan') - torrents, e = chan.add_torrents_from_dir(TEST_TORRENTS_DIR, recursive) + with db_session: + chan = metadata_store.ChannelMetadata.create_channel(title='testchan') + torrents, e = await chan.add_torrents_from_dir(TEST_TORRENTS_DIR, recursive) assert torrents_in_dir == len(torrents) assert errors == len(e) with db_session: @@ -868,12 +875,12 @@ def check_add(metadata_store, torrents_in_dir, errors, recursive): assert torrents_in_dir - len(e) == q.count() -def test_add_torrents_from_dir(metadata_store): - check_add(metadata_store, 9, 0, recursive=False) +async def test_add_torrents_from_dir(metadata_store): + await check_add(metadata_store, 9, 0, recursive=False) -def test_add_torrents_from_dir_recursive(metadata_store): - check_add(metadata_store, 11, 1, recursive=True) +async def test_add_torrents_from_dir_recursive(metadata_store): + await check_add(metadata_store, 11, 1, recursive=True) @db_session diff --git a/src/tribler/core/components/restapi/rest/tests/test_create_torrent_endpoint.py b/src/tribler/core/components/restapi/rest/tests/test_create_torrent_endpoint.py index 1ea6abe504d..f2ee01c0610 100644 --- a/src/tribler/core/components/restapi/rest/tests/test_create_torrent_endpoint.py +++ b/src/tribler/core/components/restapi/rest/tests/test_create_torrent_endpoint.py @@ -1,6 +1,6 @@ import random import string -from unittest.mock import Mock +from unittest.mock import AsyncMock import pytest from ipv8.util import succeed @@ -36,7 +36,7 @@ def fake_create_torrent_file(*_, **__): download_manager.download_defaults = DownloadDefaultsSettings() download_manager.create_torrent_file = fake_create_torrent_file - download_manager.start_download = start_download = Mock() + download_manager.start_download = start_download = AsyncMock() torrent_path = tmp_path / "video.avi.torrent" post_data = { diff --git a/src/tribler/core/components/tunnel/tests/test_full_session/test_tunnel_community.py b/src/tribler/core/components/tunnel/tests/test_full_session/test_tunnel_community.py index 3cd6aae4eb9..f3f827bda1f 100644 --- a/src/tribler/core/components/tunnel/tests/test_full_session/test_tunnel_community.py +++ b/src/tribler/core/components/tunnel/tests/test_full_session/test_tunnel_community.py @@ -78,7 +78,7 @@ async def hidden_seeder_comm(proxy_factory: ProxyFactory, video_tdef: TorrentDef download_config = DownloadConfig() download_config.set_dest_dir(TESTS_DATA_DIR) download_config.set_hops(1) - upload = community.download_manager.start_download(tdef=video_tdef, config=download_config) + upload = await community.download_manager.start_download(tdef=video_tdef, config=download_config) def seeder_state_callback(download_state): """ @@ -153,7 +153,7 @@ async def create_tunnel_community(temp_path_factory: TempPathFactory, return tunnel_community -def start_anon_download(tunnel_community: TriblerTunnelCommunity, +async def start_anon_download(tunnel_community: TriblerTunnelCommunity, seeder_port: int, torrent_def: TorrentDef, hops: int = 1) -> Download: @@ -164,7 +164,7 @@ def start_anon_download(tunnel_community: TriblerTunnelCommunity, config = DownloadConfig() config.set_dest_dir(download_manager.state_dir) config.set_hops(hops) - download = download_manager.start_download(tdef=torrent_def, config=config) + download = await download_manager.start_download(tdef=torrent_def, config=config) tunnel_community.bittorrent_peers[download] = [("127.0.0.1", seeder_port)] return download @@ -233,7 +233,7 @@ async def test_anon_download(proxy_factory: ProxyFactory, video_seeder: Download await introduce_peers([tunnel_community] + relays + exit_nodes) download_manager = tunnel_community.download_manager - download = start_anon_download(tunnel_community, video_seeder.libtorrent_port, video_tdef) + download = await start_anon_download(tunnel_community, video_seeder.libtorrent_port, video_tdef) await download.wait_for_status(DownloadStatus.DOWNLOADING) download_manager.set_download_states_callback(download_manager.sesscb_states_callback, interval=.1) @@ -282,7 +282,7 @@ def __getitem__(self, key): for e in exit_nodes: e.exit_sockets = MockExitDict(e.exit_sockets) - download = start_anon_download(leecher_community, hidden_seeder_comm.download_manager.libtorrent_port, video_tdef, - hops=1) + download = await start_anon_download(leecher_community, hidden_seeder_comm.download_manager.libtorrent_port, + video_tdef, hops=1) download.set_state_callback(download_state_callback) await download_finished.wait() diff --git a/src/tribler/core/components/watch_folder/tests/test_watch_folder.py b/src/tribler/core/components/watch_folder/tests/test_watch_folder.py index 2a5a7e7ec4c..58dcf27d180 100644 --- a/src/tribler/core/components/watch_folder/tests/test_watch_folder.py +++ b/src/tribler/core/components/watch_folder/tests/test_watch_folder.py @@ -1,6 +1,6 @@ import asyncio import shutil -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, patch, AsyncMock import pytest @@ -18,13 +18,15 @@ @pytest.fixture async def watch_folder(tmp_path): + download_manager = MagicMock() + download_manager.start_download = AsyncMock() watch = WatchFolder( state_dir=tmp_path, settings=WatchFolderSettings( enabled=True, directory='' ), - download_manager=MagicMock(), + download_manager=download_manager, notifier=MagicMock(), check_interval=TEST_CHECK_INTERVAL ) @@ -32,31 +34,31 @@ async def watch_folder(tmp_path): await watch.stop() -def test_watch_folder_no_files(watch_folder): +async def test_watch_folder_no_files(watch_folder): # Test that in the case of an empty folder, downloads are not started - watch_folder._check_watch_folder() + await watch_folder._check_watch_folder() assert not watch_folder.download_manager.start_download.called -def test_watch_folder_no_torrent_file(watch_folder: WatchFolder): +async def test_watch_folder_no_torrent_file(watch_folder: WatchFolder): # Test that in the case of a folder without torrents, downloads are not started directory = watch_folder.settings.get_path_as_absolute('directory', watch_folder.state_dir) shutil.copyfile(TORRENT_UBUNTU_FILE, directory / "test.txt") - watch_folder._check_watch_folder() + await watch_folder._check_watch_folder() assert not watch_folder.download_manager.start_download.called -def test_watch_folder_utf8_dir(watch_folder, tmp_path): +async def test_watch_folder_utf8_dir(watch_folder, tmp_path): # Test that torrents with UTF characters in the path are processed correctly watch_folder.download_manager.download_exists = Mock(return_value=False) unicode_folder = tmp_path / "\xe2\x82\xac" unicode_folder.mkdir() shutil.copyfile(TORRENT_UBUNTU_FILE, unicode_folder / "\xe2\x82\xac.torrent") - watch_folder._check_watch_folder() + await watch_folder._check_watch_folder() assert watch_folder.download_manager.start_download.called @@ -73,22 +75,22 @@ async def test_watch_folder_torrent_file_corrupt(watch_folder: WatchFolder): @patch.object(TorrentDef, 'get_metainfo', Mock(return_value=None)) -def test_watch_folder_torrent_file_no_metainfo(watch_folder: WatchFolder): +async def test_watch_folder_torrent_file_no_metainfo(watch_folder: WatchFolder): # Test that in the case of missing metainfo, the torrent file will be skipped watch_folder.download_manager.download_exists = Mock(return_value=False) shutil.copyfile(TORRENT_UBUNTU_FILE, watch_folder.state_dir / "test.torrent") - watch_folder._check_watch_folder() + await watch_folder._check_watch_folder() assert not watch_folder.download_manager.start_download.called -def test_watch_folder_torrent_file_start_download(watch_folder: WatchFolder): +async def test_watch_folder_torrent_file_start_download(watch_folder: WatchFolder): # Test that in the case of presence of a torrent file, a download is started watch_folder.download_manager.download_exists = Mock(return_value=False) shutil.copyfile(TORRENT_VIDEO_FILE, watch_folder.state_dir / "test.torrent") - watch_folder._check_watch_folder() + await watch_folder._check_watch_folder() assert watch_folder.download_manager.start_download.call_count == 1 diff --git a/src/tribler/core/components/watch_folder/watch_folder.py b/src/tribler/core/components/watch_folder/watch_folder.py index e1f00b195ea..80da5935ddd 100644 --- a/src/tribler/core/components/watch_folder/watch_folder.py +++ b/src/tribler/core/components/watch_folder/watch_folder.py @@ -40,12 +40,12 @@ async def _run(self): async def _check_watch_folder_handle_exceptions(self): try: - self._check_watch_folder() + await self._check_watch_folder() except Exception as e: self._logger.exception(f'Failed download attempt: {e}') raise NoCrashException from e - def _check_watch_folder(self): + async def _check_watch_folder(self): self._logger.debug('Checking watch folder...') if not self.settings.enabled or not self.state_dir: self._logger.debug(f'Cancelled. Enabled: {self.settings.enabled}. State dir: {self.state_dir}.') @@ -60,18 +60,18 @@ def _check_watch_folder(self): for root, _, files in os.walk(str(directory)): for name in files: path = Path(root) / name - self._process_torrent_file(path) + await self._process_torrent_file(path) self._logger.debug('Checking watch folder completed.') - def _process_torrent_file(self, path: Path): + async def _process_torrent_file(self, path: Path): if not path.name.endswith(".torrent"): return self._logger.info(f'Torrent file found: {path}') exception = None try: - self._start_download(path) + await self._start_download(path) except Exception as e: # pylint: disable=broad-except self._logger.error(f'{e.__class__.__name__}: {e}') exception = e @@ -83,8 +83,8 @@ def _process_torrent_file(self, path: Path): except OSError as e: self._logger.warning(f'{e.__class__.__name__}: {e}') - def _start_download(self, path: Path): - tdef = TorrentDef.load(path) + async def _start_download(self, path: Path): + tdef = await TorrentDef.load(path) if not tdef.get_metainfo(): self._logger.warning(f'Missed metainfo: {path}') return @@ -97,4 +97,4 @@ def _start_download(self, path: Path): download_config = DownloadConfig.from_defaults(self.download_manager.download_defaults, state_dir=self.state_dir) - self.download_manager.start_download(torrent_file=path, config=download_config) + await self.download_manager.start_download(torrent_file=path, config=download_config)