From 8e103d58862ec65a59a874b97bfce73dd7b35d59 Mon Sep 17 00:00:00 2001 From: Arkadiy Illarionov Date: Sun, 4 Feb 2024 03:43:44 +0300 Subject: [PATCH] Fix some Ruff SIM warnings (#808) --- cyberdrop_dl/clients/download_client.py | 7 +++---- cyberdrop_dl/downloader/downloader.py | 20 ++++++++----------- cyberdrop_dl/main.py | 4 +--- cyberdrop_dl/managers/config_manager.py | 5 ++--- cyberdrop_dl/managers/download_manager.py | 11 +++++----- cyberdrop_dl/managers/manager.py | 8 ++++---- cyberdrop_dl/scraper/crawler.py | 10 ++++------ .../scraper/crawlers/imgur_crawler.py | 5 +---- .../scraper/crawlers/kemono_crawler.py | 7 +++---- .../scraper/crawlers/redgifs_crawler.py | 9 +++------ .../scraper/crawlers/simpcity_crawler.py | 5 ++--- .../crawlers/socialmediagirls_crawler.py | 5 ++--- cyberdrop_dl/scraper/scraper.py | 5 ++--- .../ui/progress/statistic_progress.py | 4 ++-- cyberdrop_dl/ui/prompts/general_prompts.py | 2 +- .../ui/prompts/settings_user_prompts.py | 10 +++++----- .../utils/args/browser_cookie_extraction.py | 2 +- .../utils/transfer/first_time_setup.py | 8 +++----- cyberdrop_dl/utils/utilities.py | 5 +---- 19 files changed, 53 insertions(+), 79 deletions(-) diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py index a543c8fd5..7cf42cee4 100644 --- a/cyberdrop_dl/clients/download_client.py +++ b/cyberdrop_dl/clients/download_client.py @@ -87,10 +87,9 @@ async def _download(self, domain: str, manager: Manager, media_item: MediaItem, if any(s in content_type.lower() for s in ('html', 'text')) and ext not in FILE_FORMATS['Text']: raise InvalidContentTypeFailure(message=f"Received {content_type}, was expecting other") - if resp.status != HTTPStatus.PARTIAL_CONTENT: - if file.is_file(): - await manager.progress_manager.file_progress.advance_file(file_task, -file.stat().st_size) - file.unlink() + if resp.status != HTTPStatus.PARTIAL_CONTENT and file.is_file(): + await manager.progress_manager.file_progress.advance_file(file_task, -file.stat().st_size) + file.unlink() await save_content(resp.content) diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py index 0bb5db822..002b55783 100644 --- a/cyberdrop_dl/downloader/downloader.py +++ b/cyberdrop_dl/downloader/downloader.py @@ -61,7 +61,7 @@ async def wrapper(self, *args, **kwargs): await self.manager.log_manager.write_download_error_log(media_item.url, f" {e.status}") else: await self.manager.progress_manager.download_stats_progress.add_failure("Unknown") - await self.manager.log_manager.write_download_error_log(media_item.url, f" See Log for Details") + await self.manager.log_manager.write_download_error_log(media_item.url, " See Log for Details") await log(f"Download Failed: {media_item.url} with error {e}", 40) await self.manager.progress_manager.download_progress.add_failed() break @@ -359,17 +359,13 @@ async def download(self, media_item: MediaItem) -> None: except (aiohttp.ServerDisconnectedError, asyncio.TimeoutError, aiohttp.ServerTimeoutError) as e: await self._file_lock.release_lock(FL_Filename) - if partial_file: - if partial_file.is_file(): - size = partial_file.stat().st_size - if partial_file.name not in self._current_attempt_filesize: - self._current_attempt_filesize[media_item.filename] = size - elif self._current_attempt_filesize[media_item.filename] < size: - self._current_attempt_filesize[media_item.filename] = size - else: - raise DownloadFailure(status=getattr(e, "status", 1), message="Download timeout reached, retrying") - media_item.current_attempt = 0 - raise DownloadFailure(status=999, message="Download timeout reached, retrying") + if partial_file and partial_file.is_file(): + size = partial_file.stat().st_size + if partial_file.name in self._current_attempt_filesize and self._current_attempt_filesize[media_item.filename] >= size: + raise DownloadFailure(status=getattr(e, "status", 1), message="Download timeout reached, retrying") + self._current_attempt_filesize[media_item.filename] = size + media_item.current_attempt = 0 + raise DownloadFailure(status=999, message="Download timeout reached, retrying") raise DownloadFailure(status=getattr(e, "status", 1), message=repr(e)) diff --git a/cyberdrop_dl/main.py b/cyberdrop_dl/main.py index d5ef8e8e6..ac0bdca30 100644 --- a/cyberdrop_dl/main.py +++ b/cyberdrop_dl/main.py @@ -129,10 +129,8 @@ def main(): asyncio.run(director(manager)) except KeyboardInterrupt: print("\nTrying to Exit...") - try: + with contextlib.suppress(Exception): asyncio.run(manager.close()) - except Exception: - pass exit(1) sys.exit(0) diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py index 7e7de2776..1029721b2 100644 --- a/cyberdrop_dl/managers/config_manager.py +++ b/cyberdrop_dl/managers/config_manager.py @@ -19,9 +19,8 @@ def _match_config_dicts(default: Dict, existing: Dict) -> Dict: """Matches the keys of two dicts and returns the default dict with the values of the existing dict""" for group in default: for key in default[group]: - if group in existing: - if key in existing[group]: - default[group][key] = existing[group][key] + if group in existing and key in existing[group]: + default[group][key] = existing[group][key] return default diff --git a/cyberdrop_dl/managers/download_manager.py b/cyberdrop_dl/managers/download_manager.py index 1ce274103..b3186bc91 100644 --- a/cyberdrop_dl/managers/download_manager.py +++ b/cyberdrop_dl/managers/download_manager.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import contextlib import shutil from base64 import b64encode from typing import TYPE_CHECKING @@ -29,10 +30,8 @@ async def check_lock(self, filename: str) -> None: async def release_lock(self, filename: str) -> None: """Releases the file lock""" - try: + with contextlib.suppress(KeyError, RuntimeError): self._locked_files[filename].release() - except (KeyError, RuntimeError): - pass class DownloadManager: @@ -70,11 +69,11 @@ async def check_allowed_filetype(self, media_item: MediaItem) -> bool: """Checks if the file type is allowed to download""" if media_item.ext in FILE_FORMATS['Images'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_images']: return False - elif media_item.ext in FILE_FORMATS['Videos'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_videos']: + if media_item.ext in FILE_FORMATS['Videos'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_videos']: return False - elif media_item.ext in FILE_FORMATS['Audio'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_audio']: + if media_item.ext in FILE_FORMATS['Audio'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_audio']: return False - elif (self.manager.config_manager.settings_data['Ignore_Options']['exclude_other'] and + if (self.manager.config_manager.settings_data['Ignore_Options']['exclude_other'] and media_item.ext not in FILE_FORMATS['Images'] and media_item.ext not in FILE_FORMATS['Videos'] and media_item.ext not in FILE_FORMATS['Audio']): return False diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py index ddcc1f33d..93340ced7 100644 --- a/cyberdrop_dl/managers/manager.py +++ b/cyberdrop_dl/managers/manager.py @@ -86,7 +86,7 @@ async def async_startup(self) -> None: async def args_consolidation(self) -> None: """Consolidates runtime arguments with config values""" - for arg in self.args_manager.parsed_args.keys(): + for arg in self.args_manager.parsed_args: if arg in config_definitions.settings['Download_Options']: if self.args_manager.parsed_args[arg] != config_definitions.settings['Download_Options'][arg]: self.config_manager.settings_data['Download_Options'][arg] = self.args_manager.parsed_args[arg] @@ -116,7 +116,7 @@ async def args_logging(self) -> None: else: forum_credentials_provided[f"{forum} Credentials Provided"] = False - gofile_credentials_provided = True if self.config_manager.authentication_data["GoFile"]['gofile_api_key'] else False + gofile_credentials_provided = bool(self.config_manager.authentication_data["GoFile"]["gofile_api_key"]) bunkr_ddg_credentials_provided = False coomer_ddg_credentials_provided = False kemono_ddg_credentials_provided = False @@ -128,13 +128,13 @@ async def args_logging(self) -> None: if self.config_manager.authentication_data["DDOS-Guard"]['kemono_ddg1']: kemono_ddg_credentials_provided = True - imgur_credentials_provided = True if self.config_manager.authentication_data["Imgur"]['imgur_client_id'] else False + imgur_credentials_provided = bool(self.config_manager.authentication_data["Imgur"]["imgur_client_id"]) jdownloader_credentials_provided = False if self.config_manager.authentication_data["JDownloader"]['jdownloader_username'] and self.config_manager.authentication_data["JDownloader"]['jdownloader_password'] and self.config_manager.authentication_data["JDownloader"]['jdownloader_device']: jdownloader_credentials_provided = True - pixeldrain_credentials_provided = True if self.config_manager.authentication_data["PixelDrain"]['pixeldrain_api_key'] else False + pixeldrain_credentials_provided = bool(self.config_manager.authentication_data["PixelDrain"]["pixeldrain_api_key"]) reddit_credentials_provided = False if self.config_manager.authentication_data["Reddit"]['reddit_personal_use_script'] and self.config_manager.authentication_data["Reddit"]['reddit_secret']: diff --git a/cyberdrop_dl/scraper/crawler.py b/cyberdrop_dl/scraper/crawler.py index 2b256b5c0..f4f56eea4 100644 --- a/cyberdrop_dl/scraper/crawler.py +++ b/cyberdrop_dl/scraper/crawler.py @@ -187,13 +187,11 @@ async def create_title(self, title: str, album_id: Optional[str], thread_id: Opt title = "Untitled" title = title.strip() - if self.manager.config_manager.settings_data['Download_Options']['include_album_id_in_folder_name']: - if album_id: - title = f"{title} {album_id}" + if self.manager.config_manager.settings_data['Download_Options']['include_album_id_in_folder_name'] and album_id: + title = f"{title} {album_id}" - if self.manager.config_manager.settings_data['Download_Options']['include_thread_id_in_folder_name']: - if thread_id: - title = f"{title} {thread_id}" + if self.manager.config_manager.settings_data['Download_Options']['include_thread_id_in_folder_name'] and thread_id: + title = f"{title} {thread_id}" if not self.manager.config_manager.settings_data['Download_Options']['remove_domains_from_folder_names']: title = f"{title} ({self.folder_domain})" diff --git a/cyberdrop_dl/scraper/crawlers/imgur_crawler.py b/cyberdrop_dl/scraper/crawlers/imgur_crawler.py index 80e659fba..3bd6aae72 100644 --- a/cyberdrop_dl/scraper/crawlers/imgur_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/imgur_crawler.py @@ -50,10 +50,7 @@ async def album(self, scrape_item: ScrapeItem) -> None: async with self.request_limiter: JSON_Obj = await self.client.get_json(self.domain, self.imgur_api / f"album/{album_id}", headers_inc=self.headers) - if "title" in JSON_Obj["data"].keys(): - title_part = JSON_Obj["data"]["title"] - else: - title_part = album_id + title_part = JSON_Obj["data"].get("title", album_id) title = await self.create_title(title_part, scrape_item.url.parts[2], None) async with self.request_limiter: diff --git a/cyberdrop_dl/scraper/crawlers/kemono_crawler.py b/cyberdrop_dl/scraper/crawlers/kemono_crawler.py index 6e430bc80..64c7d0761 100644 --- a/cyberdrop_dl/scraper/crawlers/kemono_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/kemono_crawler.py @@ -98,16 +98,15 @@ async def handle_post_content(self, scrape_item: ScrapeItem, post: Dict, user: s """Handles the content of a post""" date = post["published"].replace("T", " ") post_id = post["id"] - post_title = post["title"] if "title" in post else "" + post_title = post.get("title", "") async def handle_file(file_obj): link = self.primary_base_domain / ("data" + file_obj['path']) link = link.with_query({"f": file_obj['name']}) await self.create_new_scrape_item(link, scrape_item, user_str, post_title, post_id, date) - if "file" in post: - if post['file']: - await handle_file(post['file']) + if post.get('file'): + await handle_file(post['file']) for file in post['attachments']: await handle_file(file) diff --git a/cyberdrop_dl/scraper/crawlers/redgifs_crawler.py b/cyberdrop_dl/scraper/crawlers/redgifs_crawler.py index 510158a00..44e4684c2 100644 --- a/cyberdrop_dl/scraper/crawlers/redgifs_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/redgifs_crawler.py @@ -53,7 +53,7 @@ async def user(self, scrape_item: ScrapeItem) -> None: for gif in gifs: links = gif["urls"] date = gif["createDate"] - title_part = gif["title"] if "title" in gif else f"Loose Files" + title_part = gif.get("title", "Loose Files") title = await self.create_title(title_part, None, None) try: @@ -74,15 +74,12 @@ async def post(self, scrape_item: ScrapeItem) -> None: async with self.request_limiter: JSON_Resp = await self.client.get_json(self.domain, self.redgifs_api / "v2/gifs" / post_id, headers_inc=self.headers) - title_part = JSON_Resp["gif"]["title"] if "title" in JSON_Resp["gif"] else "Loose Files" + title_part = JSON_Resp["gif"].get("title", "Loose Files") title = await self.create_title(title_part, None, None) links = JSON_Resp["gif"]["urls"] date = JSON_Resp["gif"]["createDate"] - if "hd" in links: - link = URL(links["hd"]) - else: - link = URL(links["sd"]) + link = URL(links["hd"] if "hd" in links else links["sd"]) filename, ext = await get_filename_and_ext(link.name) new_scrape_item = await self.create_scrape_item(scrape_item, link, title, True, date) diff --git a/cyberdrop_dl/scraper/crawlers/simpcity_crawler.py b/cyberdrop_dl/scraper/crawlers/simpcity_crawler.py index 1823e2530..7b0f60b2a 100644 --- a/cyberdrop_dl/scraper/crawlers/simpcity_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/simpcity_crawler.py @@ -187,9 +187,8 @@ async def images(self, scrape_item: ScrapeItem, post_content: Tag) -> None: continue parent_simp_check = image.parent.get("data-simp") - if parent_simp_check: - if "init" in parent_simp_check: - continue + if parent_simp_check and "init" in parent_simp_check: + continue link = link.replace(".th.", ".").replace(".md.", ".") if link.endswith("/"): diff --git a/cyberdrop_dl/scraper/crawlers/socialmediagirls_crawler.py b/cyberdrop_dl/scraper/crawlers/socialmediagirls_crawler.py index 7539fd26f..3c4684f9e 100644 --- a/cyberdrop_dl/scraper/crawlers/socialmediagirls_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/socialmediagirls_crawler.py @@ -190,9 +190,8 @@ async def images(self, scrape_item: ScrapeItem, post_content: Tag) -> None: continue parent_simp_check = image.parent.get("data-simp") - if parent_simp_check: - if "init" in parent_simp_check: - continue + if parent_simp_check and "init" in parent_simp_check: + continue link = link.replace(".th.", ".").replace(".md.", ".") if link.endswith("/"): diff --git a/cyberdrop_dl/scraper/scraper.py b/cyberdrop_dl/scraper/scraper.py index b7070b162..62ffeed0a 100644 --- a/cyberdrop_dl/scraper/scraper.py +++ b/cyberdrop_dl/scraper/scraper.py @@ -251,9 +251,8 @@ async def start_scrapers(self) -> None: async def start_jdownloader(self) -> None: """Starts JDownloader""" - if self.jdownloader.enabled: - if isinstance(self.jdownloader.jdownloader_agent, Field): - await self.jdownloader.jdownloader_setup() + if self.jdownloader.enabled and isinstance(self.jdownloader.jdownloader_agent, Field): + await self.jdownloader.jdownloader_setup() async def start(self) -> None: """Starts the orchestra""" diff --git a/cyberdrop_dl/ui/progress/statistic_progress.py b/cyberdrop_dl/ui/progress/statistic_progress.py index a1a5d9aa2..6e6c43b79 100644 --- a/cyberdrop_dl/ui/progress/statistic_progress.py +++ b/cyberdrop_dl/ui/progress/statistic_progress.py @@ -24,7 +24,7 @@ async def get_progress(self) -> Panel: async def update_total(self, total: int) -> None: """Updates the total number of files to be downloaded""" - for key in self.failure_types.keys(): + for key in self.failure_types: self.progress.update(self.failure_types[key], total=total) async def add_failure(self, failure_type: [str, int]) -> None: @@ -66,7 +66,7 @@ async def get_progress(self) -> Panel: async def update_total(self, total: int) -> None: """Updates the total number of sites to be scraped""" - for key in self.failure_types.keys(): + for key in self.failure_types: self.progress.update(self.failure_types[key], total=total) async def add_failure(self, failure_type: [str, int]) -> None: diff --git a/cyberdrop_dl/ui/prompts/general_prompts.py b/cyberdrop_dl/ui/prompts/general_prompts.py index ba87e0c03..4ae14998c 100644 --- a/cyberdrop_dl/ui/prompts/general_prompts.py +++ b/cyberdrop_dl/ui/prompts/general_prompts.py @@ -82,7 +82,7 @@ def import_cyberdrop_v4_items_prompt(manager: Manager) -> None: """Import Cyberdrop_V4 Items""" while True: console.clear() - console.print(f"Editing Config Values") + console.print("Editing Config Values") action = inquirer.select( message="What would you like to do?", choices=[ diff --git a/cyberdrop_dl/ui/prompts/settings_user_prompts.py b/cyberdrop_dl/ui/prompts/settings_user_prompts.py index 1f1a12884..e63df7795 100644 --- a/cyberdrop_dl/ui/prompts/settings_user_prompts.py +++ b/cyberdrop_dl/ui/prompts/settings_user_prompts.py @@ -21,7 +21,7 @@ def create_new_config_prompt(manager: Manager) -> None: """Create a new config file""" console.clear() - console.print(f"Create a new config file") + console.print("Create a new config file") config_name = inquirer.text( message="Enter the name of the config:", validate=EmptyInputValidator("Input should not be empty") @@ -40,7 +40,7 @@ def edit_config_values_prompt(manager: Manager) -> None: while True: console.clear() - console.print(f"Editing Config Values") + console.print("Editing Config Values") action = inquirer.select( message="What would you like to do?", choices=[ @@ -129,7 +129,7 @@ def edit_download_options_prompt(config: Dict) -> None: ], long_instruction="ARROW KEYS: Navigate | TAB: Select | ENTER: Confirm" ).execute() - for key in config["Download_Options"].keys(): + for key in config["Download_Options"]: config["Download_Options"][key] = False for key in action: @@ -272,7 +272,7 @@ def edit_ignore_options_prompt(config: Dict) -> None: ], long_instruction="ARROW KEYS: Move | TAB: Select | ENTER: Confirm", ).execute() - for key in config["Ignore_Options"].keys(): + for key in config["Ignore_Options"]: config["Ignore_Options"][key] = False for key in action: @@ -335,7 +335,7 @@ def edit_runtime_options_prompt(config: Dict) -> None: long_instruction="10 is the default (uses pythons logging numerical levels)", ).execute() - for key in config["Runtime_Options"].keys(): + for key in config["Runtime_Options"]: config["Runtime_Options"][key] = False for key in action: diff --git a/cyberdrop_dl/utils/args/browser_cookie_extraction.py b/cyberdrop_dl/utils/args/browser_cookie_extraction.py index b7af735b4..9b1714313 100644 --- a/cyberdrop_dl/utils/args/browser_cookie_extraction.py +++ b/cyberdrop_dl/utils/args/browser_cookie_extraction.py @@ -24,7 +24,7 @@ def wrapper(self, *args, **kwargs): except PermissionError: console = Console() console.clear() - console.print(f"We've encountered a Permissions Error. Please close all browsers and try again.", style="bold red") + console.print("We've encountered a Permissions Error. Please close all browsers and try again.", style="bold red") console.print("If you are still having issues, make sure all browsers processes are closed in a Task Manager.", style="bold red") console.print("Nothing has been saved.", style="bold red") inquirer.confirm(message="Press enter to return menu.").execute() diff --git a/cyberdrop_dl/utils/transfer/first_time_setup.py b/cyberdrop_dl/utils/transfer/first_time_setup.py index d2d83bd1c..8ad596a4c 100644 --- a/cyberdrop_dl/utils/transfer/first_time_setup.py +++ b/cyberdrop_dl/utils/transfer/first_time_setup.py @@ -24,12 +24,10 @@ def startup(self) -> None: OLD_APP_STORAGE: Path = Path(platformdirs.user_config_dir("Cyberdrop-DL")) OLD_DOWNLOAD_STORAGE = Path(platformdirs.user_downloads_path()) / "Cyberdrop-DL Downloads" - check = False if APP_STORAGE.exists(): - if (APP_STORAGE / "Cache" / "cache.yaml").is_file(): - check = self.check_cache_for_moved(APP_STORAGE / "Cache" / "cache.yaml") - if check: - return + cache_file = APP_STORAGE / "Cache" / "cache.yaml" + if cache_file.is_file() and self.check_cache_for_moved(cache_file): + return OLD_FILES = Path("./Old Files") OLD_FILES.mkdir(parents=True, exist_ok=True) diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py index 56c2bd833..833321b30 100644 --- a/cyberdrop_dl/utils/utilities.py +++ b/cyberdrop_dl/utils/utilities.py @@ -53,10 +53,7 @@ def error_handling_wrapper(func): """Wrapper handles errors for url scraping""" @wraps(func) async def wrapper(self, *args, **kwargs): - if isinstance(args[0], URL): - link = args[0] - else: - link = args[0].url + link = args[0] if isinstance(args[0], URL) else args[0].url try: return await func(self, *args, **kwargs)