From e7e0df2ff667880e4e8257b31983eb5bd4a38c3a Mon Sep 17 00:00:00 2001 From: Jules-WinnfieldX Date: Tue, 5 Dec 2023 15:42:35 -0700 Subject: [PATCH] add: --retry-failed --input-file --output-folder --config-file --appdata-folder fix: URLs.txt being touched inappropriately --- cyberdrop_dl/__init__.py | 2 +- cyberdrop_dl/managers/args_manager.py | 24 +++++++++++++++++-- cyberdrop_dl/managers/cache_manager.py | 15 ++++++++---- cyberdrop_dl/managers/config_manager.py | 4 ++++ cyberdrop_dl/managers/manager.py | 6 +++-- cyberdrop_dl/managers/path_manager.py | 10 +++++--- cyberdrop_dl/scraper/scraper.py | 6 ++++- cyberdrop_dl/utils/args/args.py | 8 +++++++ .../utils/dataclasses/supported_domains.py | 2 +- cyberdrop_dl/utils/utilities.py | 10 +++++--- pyproject.toml | 2 +- 11 files changed, 71 insertions(+), 18 deletions(-) diff --git a/cyberdrop_dl/__init__.py b/cyberdrop_dl/__init__.py index ba6d1add8..6f8ae5c30 100644 --- a/cyberdrop_dl/__init__.py +++ b/cyberdrop_dl/__init__.py @@ -1 +1 @@ -__version__ = "5.0.43" +__version__ = "5.0.44" diff --git a/cyberdrop_dl/managers/args_manager.py b/cyberdrop_dl/managers/args_manager.py index 483265593..92ed4fe2d 100644 --- a/cyberdrop_dl/managers/args_manager.py +++ b/cyberdrop_dl/managers/args_manager.py @@ -16,6 +16,12 @@ def __init__(self): self.other_links: list = [] + # Files + self.input_file = None + self.download_dir = None + self.config_file = None + self.appdata_dir = None + def startup(self) -> None: """Parses arguments and sets variables accordingly""" if self.parsed_args: @@ -26,12 +32,26 @@ def startup(self) -> None: self.immediate_download = self.parsed_args['download'] self.load_config_name = self.parsed_args['config'] + if self.load_config_name: + self.load_config_from_args = True + if self.parsed_args['download_all_configs']: self.all_configs = True self.immediate_download = True - if self.load_config_name: - self.load_config_from_args = True + if self.parsed_args['retry_failed']: + self.retry = True + self.immediate_download = True + + if self.parsed_args['input_file']: + self.input_file = self.parsed_args['input_file'] + if self.parsed_args['output_folder']: + self.download_dir = self.parsed_args['download_folder'] + if self.parsed_args['appdata_folder']: + self.appdata_dir = self.parsed_args['appdata_folder'] + if self.parsed_args['config_file']: + self.config_file = self.parsed_args['config_file'] + self.immediate_download = True self.other_links = self.parsed_args['links'] diff --git a/cyberdrop_dl/managers/cache_manager.py b/cyberdrop_dl/managers/cache_manager.py index eb95df9a3..6337b0278 100644 --- a/cyberdrop_dl/managers/cache_manager.py +++ b/cyberdrop_dl/managers/cache_manager.py @@ -2,10 +2,13 @@ from dataclasses import field from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, TYPE_CHECKING import yaml +if TYPE_CHECKING: + from cyberdrop_dl.managers.manager import Manager + def _save_yaml(file: Path, data: Dict) -> None: """Saves a dict to a yaml file""" @@ -21,7 +24,9 @@ def _load_yaml(file: Path) -> Dict: class CacheManager: - def __init__(self): + def __init__(self, manager: 'Manager'): + self.manager = manager + self.cache_file: Path = field(init=False) self._cache = {} @@ -30,8 +35,10 @@ def startup(self, cache_file: Path) -> None: self.cache_file = cache_file if not self.cache_file.is_file(): self.save('default_config', "Default") - else: - self.load() + + if self.manager.args_manager.appdata_dir: + self.save('first_startup_completed', True) + self.load() def load(self) -> None: """Loads the cache file into memory""" diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py index 98b098b08..c8d5a8632 100644 --- a/cyberdrop_dl/managers/config_manager.py +++ b/cyberdrop_dl/managers/config_manager.py @@ -79,6 +79,10 @@ def load_configs(self) -> None: self.global_settings_data = copy.deepcopy(global_settings) _save_yaml(self.global_settings, self.global_settings_data) + if self.manager.args_manager.config_file: + self.settings = Path(self.manager.args_manager.config_file) + self.loaded_config = "CLI-Arg Specified" + if self.settings.is_file(): self._verify_settings_config() else: diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py index bbbed61f8..ac3506421 100644 --- a/cyberdrop_dl/managers/manager.py +++ b/cyberdrop_dl/managers/manager.py @@ -23,7 +23,7 @@ class Manager: def __init__(self): self.args_manager: ArgsManager = ArgsManager() - self.cache_manager: CacheManager = CacheManager() + self.cache_manager: CacheManager = CacheManager(self) self.path_manager: PathManager = field(init=False) self.config_manager: ConfigManager = field(init=False) self.log_manager: LogManager = field(init=False) @@ -34,7 +34,6 @@ def __init__(self): self.progress_manager: ProgressManager = field(init=False) self.first_time_setup: TransitionManager = TransitionManager(self) - self.first_time_setup.startup() self._loaded_args_config: bool = False self._made_portable: bool = False @@ -43,6 +42,9 @@ def startup(self) -> None: """Startup process for the manager""" self.args_startup() + if not self.args_manager.appdata_dir: + self.first_time_setup.startup() + self.path_manager = PathManager(self) self.path_manager.pre_startup() diff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.py index d59fcf77c..b64853384 100644 --- a/cyberdrop_dl/managers/path_manager.py +++ b/cyberdrop_dl/managers/path_manager.py @@ -3,8 +3,6 @@ from pathlib import Path from typing import TYPE_CHECKING -import platformdirs - if TYPE_CHECKING: from cyberdrop_dl.managers.manager import Manager @@ -16,6 +14,7 @@ APP_STORAGE = Path("./AppData") DOWNLOAD_STORAGE = Path("./Downloads") + class PathManager: def __init__(self, manager: 'Manager'): self.manager = manager @@ -37,6 +36,10 @@ def __init__(self, manager: 'Manager'): self.scrape_error_log: Path = field(init=False) def pre_startup(self): + if self.manager.args_manager.appdata_dir: + global APP_STORAGE + APP_STORAGE = Path(self.manager.args_manager.appdata_dir) + self.cache_dir = APP_STORAGE / "Cache" self.config_dir = APP_STORAGE / "Configs" @@ -62,5 +65,6 @@ def startup(self) -> None: self.sorted_dir.mkdir(parents=True, exist_ok=True) self.log_dir.mkdir(parents=True, exist_ok=True) - self.input_file.touch(exist_ok=True) + if not self.input_file.is_file(): + self.input_file.touch(exist_ok=True) self.history_db.touch(exist_ok=True) diff --git a/cyberdrop_dl/scraper/scraper.py b/cyberdrop_dl/scraper/scraper.py index ed794ab1e..40bf871ea 100644 --- a/cyberdrop_dl/scraper/scraper.py +++ b/cyberdrop_dl/scraper/scraper.py @@ -271,8 +271,12 @@ async def regex_links(self, line: str) -> List: async def load_links(self) -> None: """Loads links from args / input file""" + input_file = self.manager.path_manager.input_file + if self.manager.args_manager.input_file: + input_file = Path(self.manager.args_manager.input_file) + links = [] - async with aiofiles.open(self.manager.path_manager.input_file, "r", encoding="utf8") as f: + async with aiofiles.open(input_file, "r", encoding="utf8") as f: async for line in f: assert isinstance(line, str) links.extend(await self.regex_links(line)) diff --git a/cyberdrop_dl/utils/args/args.py b/cyberdrop_dl/utils/args/args.py index b45cce40a..59c203f9c 100644 --- a/cyberdrop_dl/utils/args/args.py +++ b/cyberdrop_dl/utils/args/args.py @@ -12,6 +12,14 @@ def parse_args() -> argparse.Namespace: general.add_argument("--config", type=str, help="name of config to load", default="") general.add_argument("--download", action="store_true", help="Skip the UI and go straight to downloading", default=False) general.add_argument("--download-all-configs", action="store_true", help="Skip the UI and go straight to downloading (runs all configs sequentially)", default=False) + general.add_argument("--retry-failed", action="store_true", help="retry failed downloads", default=False) + + # File Paths + file_paths = parser.add_argument_group("File_Paths") + file_paths.add_argument("-i", "--input-file", type=str, help="path to txt file containing urls to download", default="") + file_paths.add_argument("-d", "--output-folder", type=str, help="path to download folder", default="") + file_paths.add_argument("--config-file", type=str, help="path to the CDL settings.yaml file to load", default="") + file_paths.add_argument("--appdata-folder", type=str, help="path to where you want CDL to store it's AppData folder", default="") # Settings download_options = parser.add_argument_group("Download_Options") diff --git a/cyberdrop_dl/utils/dataclasses/supported_domains.py b/cyberdrop_dl/utils/dataclasses/supported_domains.py index cc835d2d7..0d5b4f043 100644 --- a/cyberdrop_dl/utils/dataclasses/supported_domains.py +++ b/cyberdrop_dl/utils/dataclasses/supported_domains.py @@ -5,7 +5,7 @@ @dataclass class SupportedDomains: """The allows options for domains to skip when scraping and mappings""" - supported_hosts: ClassVar[Tuple[str, ...]] = ("bunkrr", "celebforum", "coomer", "cyberdrop", "cyberfile", + supported_hosts: ClassVar[Tuple[str, ...]] = ("bunkr", "bunkrr", "celebforum", "coomer", "cyberdrop", "cyberfile", "e-hentai", "erome", "fapello", "f95zone", "gofile", "hotpic", "ibb.co", "imageban", "imgbox", "imgur", "img.kiwi", "jpg.church", "jpg.homes", "jpg.fish", "jpg.fishing", "jpg.pet", "jpeg.pet", diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py index 743a3d655..416729477 100644 --- a/cyberdrop_dl/utils/utilities.py +++ b/cyberdrop_dl/utils/utilities.py @@ -139,15 +139,19 @@ async def get_filename_and_ext(filename: str, forum: bool = False) -> Tuple[str, async def get_download_path(manager: Manager, scrape_item: ScrapeItem, domain: str) -> Path: """Returns the path to the download folder""" + download_dir = manager.path_manager.download_dir + if manager.args_manager.download_dir: + download_dir = Path(manager.args_manager.download_dir) + if scrape_item.retry: return scrape_item.retry_path if scrape_item.parent_title and scrape_item.part_of_album: - return manager.path_manager.download_dir / scrape_item.parent_title + return download_dir / scrape_item.parent_title elif scrape_item.parent_title: - return manager.path_manager.download_dir / scrape_item.parent_title / f"Loose Files ({domain})" + return download_dir / scrape_item.parent_title / f"Loose Files ({domain})" else: - return manager.path_manager.download_dir / f"Loose Files ({domain})" + return download_dir / f"Loose Files ({domain})" async def remove_id(manager: Manager, filename: str, ext: str) -> Tuple[str, str]: diff --git a/pyproject.toml b/pyproject.toml index cf8a4aeb6..b1c7fba79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cyberdrop-dl" -version = "5.0.43" +version = "5.0.44" description = "Bulk downloader for multiple file hosts" authors = ["Jules Winnfield "] readme = "README.md"