Skip to content
This repository has been archived by the owner on Jul 5, 2024. It is now read-only.

Commit

Permalink
Fix some Ruff SIM warnings (#808)
Browse files Browse the repository at this point in the history
  • Loading branch information
qarkai authored Feb 4, 2024
1 parent 6ac1242 commit 8e103d5
Show file tree
Hide file tree
Showing 19 changed files with 53 additions and 79 deletions.
7 changes: 3 additions & 4 deletions cyberdrop_dl/clients/download_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,9 @@ async def _download(self, domain: str, manager: Manager, media_item: MediaItem,
if any(s in content_type.lower() for s in ('html', 'text')) and ext not in FILE_FORMATS['Text']:
raise InvalidContentTypeFailure(message=f"Received {content_type}, was expecting other")

if resp.status != HTTPStatus.PARTIAL_CONTENT:
if file.is_file():
await manager.progress_manager.file_progress.advance_file(file_task, -file.stat().st_size)
file.unlink()
if resp.status != HTTPStatus.PARTIAL_CONTENT and file.is_file():
await manager.progress_manager.file_progress.advance_file(file_task, -file.stat().st_size)
file.unlink()

await save_content(resp.content)

Expand Down
20 changes: 8 additions & 12 deletions cyberdrop_dl/downloader/downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ async def wrapper(self, *args, **kwargs):
await self.manager.log_manager.write_download_error_log(media_item.url, f" {e.status}")
else:
await self.manager.progress_manager.download_stats_progress.add_failure("Unknown")
await self.manager.log_manager.write_download_error_log(media_item.url, f" See Log for Details")
await self.manager.log_manager.write_download_error_log(media_item.url, " See Log for Details")
await log(f"Download Failed: {media_item.url} with error {e}", 40)
await self.manager.progress_manager.download_progress.add_failed()
break
Expand Down Expand Up @@ -359,17 +359,13 @@ async def download(self, media_item: MediaItem) -> None:
except (aiohttp.ServerDisconnectedError, asyncio.TimeoutError, aiohttp.ServerTimeoutError) as e:
await self._file_lock.release_lock(FL_Filename)

if partial_file:
if partial_file.is_file():
size = partial_file.stat().st_size
if partial_file.name not in self._current_attempt_filesize:
self._current_attempt_filesize[media_item.filename] = size
elif self._current_attempt_filesize[media_item.filename] < size:
self._current_attempt_filesize[media_item.filename] = size
else:
raise DownloadFailure(status=getattr(e, "status", 1), message="Download timeout reached, retrying")
media_item.current_attempt = 0
raise DownloadFailure(status=999, message="Download timeout reached, retrying")
if partial_file and partial_file.is_file():
size = partial_file.stat().st_size
if partial_file.name in self._current_attempt_filesize and self._current_attempt_filesize[media_item.filename] >= size:
raise DownloadFailure(status=getattr(e, "status", 1), message="Download timeout reached, retrying")
self._current_attempt_filesize[media_item.filename] = size
media_item.current_attempt = 0
raise DownloadFailure(status=999, message="Download timeout reached, retrying")

raise DownloadFailure(status=getattr(e, "status", 1), message=repr(e))

Expand Down
4 changes: 1 addition & 3 deletions cyberdrop_dl/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,8 @@ def main():
asyncio.run(director(manager))
except KeyboardInterrupt:
print("\nTrying to Exit...")
try:
with contextlib.suppress(Exception):
asyncio.run(manager.close())
except Exception:
pass
exit(1)
sys.exit(0)

Expand Down
5 changes: 2 additions & 3 deletions cyberdrop_dl/managers/config_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ def _match_config_dicts(default: Dict, existing: Dict) -> Dict:
"""Matches the keys of two dicts and returns the default dict with the values of the existing dict"""
for group in default:
for key in default[group]:
if group in existing:
if key in existing[group]:
default[group][key] = existing[group][key]
if group in existing and key in existing[group]:
default[group][key] = existing[group][key]
return default


Expand Down
11 changes: 5 additions & 6 deletions cyberdrop_dl/managers/download_manager.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import asyncio
import contextlib
import shutil
from base64 import b64encode
from typing import TYPE_CHECKING
Expand Down Expand Up @@ -29,10 +30,8 @@ async def check_lock(self, filename: str) -> None:

async def release_lock(self, filename: str) -> None:
"""Releases the file lock"""
try:
with contextlib.suppress(KeyError, RuntimeError):
self._locked_files[filename].release()
except (KeyError, RuntimeError):
pass


class DownloadManager:
Expand Down Expand Up @@ -70,11 +69,11 @@ async def check_allowed_filetype(self, media_item: MediaItem) -> bool:
"""Checks if the file type is allowed to download"""
if media_item.ext in FILE_FORMATS['Images'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_images']:
return False
elif media_item.ext in FILE_FORMATS['Videos'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_videos']:
if media_item.ext in FILE_FORMATS['Videos'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_videos']:
return False
elif media_item.ext in FILE_FORMATS['Audio'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_audio']:
if media_item.ext in FILE_FORMATS['Audio'] and self.manager.config_manager.settings_data['Ignore_Options']['exclude_audio']:
return False
elif (self.manager.config_manager.settings_data['Ignore_Options']['exclude_other'] and
if (self.manager.config_manager.settings_data['Ignore_Options']['exclude_other'] and
media_item.ext not in FILE_FORMATS['Images'] and media_item.ext not in FILE_FORMATS['Videos'] and
media_item.ext not in FILE_FORMATS['Audio']):
return False
Expand Down
8 changes: 4 additions & 4 deletions cyberdrop_dl/managers/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ async def async_startup(self) -> None:

async def args_consolidation(self) -> None:
"""Consolidates runtime arguments with config values"""
for arg in self.args_manager.parsed_args.keys():
for arg in self.args_manager.parsed_args:
if arg in config_definitions.settings['Download_Options']:
if self.args_manager.parsed_args[arg] != config_definitions.settings['Download_Options'][arg]:
self.config_manager.settings_data['Download_Options'][arg] = self.args_manager.parsed_args[arg]
Expand Down Expand Up @@ -116,7 +116,7 @@ async def args_logging(self) -> None:
else:
forum_credentials_provided[f"{forum} Credentials Provided"] = False

gofile_credentials_provided = True if self.config_manager.authentication_data["GoFile"]['gofile_api_key'] else False
gofile_credentials_provided = bool(self.config_manager.authentication_data["GoFile"]["gofile_api_key"])
bunkr_ddg_credentials_provided = False
coomer_ddg_credentials_provided = False
kemono_ddg_credentials_provided = False
Expand All @@ -128,13 +128,13 @@ async def args_logging(self) -> None:
if self.config_manager.authentication_data["DDOS-Guard"]['kemono_ddg1']:
kemono_ddg_credentials_provided = True

imgur_credentials_provided = True if self.config_manager.authentication_data["Imgur"]['imgur_client_id'] else False
imgur_credentials_provided = bool(self.config_manager.authentication_data["Imgur"]["imgur_client_id"])
jdownloader_credentials_provided = False

if self.config_manager.authentication_data["JDownloader"]['jdownloader_username'] and self.config_manager.authentication_data["JDownloader"]['jdownloader_password'] and self.config_manager.authentication_data["JDownloader"]['jdownloader_device']:
jdownloader_credentials_provided = True

pixeldrain_credentials_provided = True if self.config_manager.authentication_data["PixelDrain"]['pixeldrain_api_key'] else False
pixeldrain_credentials_provided = bool(self.config_manager.authentication_data["PixelDrain"]["pixeldrain_api_key"])
reddit_credentials_provided = False

if self.config_manager.authentication_data["Reddit"]['reddit_personal_use_script'] and self.config_manager.authentication_data["Reddit"]['reddit_secret']:
Expand Down
10 changes: 4 additions & 6 deletions cyberdrop_dl/scraper/crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,13 +187,11 @@ async def create_title(self, title: str, album_id: Optional[str], thread_id: Opt
title = "Untitled"

title = title.strip()
if self.manager.config_manager.settings_data['Download_Options']['include_album_id_in_folder_name']:
if album_id:
title = f"{title} {album_id}"
if self.manager.config_manager.settings_data['Download_Options']['include_album_id_in_folder_name'] and album_id:
title = f"{title} {album_id}"

if self.manager.config_manager.settings_data['Download_Options']['include_thread_id_in_folder_name']:
if thread_id:
title = f"{title} {thread_id}"
if self.manager.config_manager.settings_data['Download_Options']['include_thread_id_in_folder_name'] and thread_id:
title = f"{title} {thread_id}"

if not self.manager.config_manager.settings_data['Download_Options']['remove_domains_from_folder_names']:
title = f"{title} ({self.folder_domain})"
Expand Down
5 changes: 1 addition & 4 deletions cyberdrop_dl/scraper/crawlers/imgur_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,7 @@ async def album(self, scrape_item: ScrapeItem) -> None:

async with self.request_limiter:
JSON_Obj = await self.client.get_json(self.domain, self.imgur_api / f"album/{album_id}", headers_inc=self.headers)
if "title" in JSON_Obj["data"].keys():
title_part = JSON_Obj["data"]["title"]
else:
title_part = album_id
title_part = JSON_Obj["data"].get("title", album_id)
title = await self.create_title(title_part, scrape_item.url.parts[2], None)

async with self.request_limiter:
Expand Down
7 changes: 3 additions & 4 deletions cyberdrop_dl/scraper/crawlers/kemono_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,16 +98,15 @@ async def handle_post_content(self, scrape_item: ScrapeItem, post: Dict, user: s
"""Handles the content of a post"""
date = post["published"].replace("T", " ")
post_id = post["id"]
post_title = post["title"] if "title" in post else ""
post_title = post.get("title", "")

async def handle_file(file_obj):
link = self.primary_base_domain / ("data" + file_obj['path'])
link = link.with_query({"f": file_obj['name']})
await self.create_new_scrape_item(link, scrape_item, user_str, post_title, post_id, date)

if "file" in post:
if post['file']:
await handle_file(post['file'])
if post.get('file'):
await handle_file(post['file'])

for file in post['attachments']:
await handle_file(file)
Expand Down
9 changes: 3 additions & 6 deletions cyberdrop_dl/scraper/crawlers/redgifs_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ async def user(self, scrape_item: ScrapeItem) -> None:
for gif in gifs:
links = gif["urls"]
date = gif["createDate"]
title_part = gif["title"] if "title" in gif else f"Loose Files"
title_part = gif.get("title", "Loose Files")
title = await self.create_title(title_part, None, None)

try:
Expand All @@ -74,15 +74,12 @@ async def post(self, scrape_item: ScrapeItem) -> None:
async with self.request_limiter:
JSON_Resp = await self.client.get_json(self.domain, self.redgifs_api / "v2/gifs" / post_id, headers_inc=self.headers)

title_part = JSON_Resp["gif"]["title"] if "title" in JSON_Resp["gif"] else "Loose Files"
title_part = JSON_Resp["gif"].get("title", "Loose Files")
title = await self.create_title(title_part, None, None)
links = JSON_Resp["gif"]["urls"]
date = JSON_Resp["gif"]["createDate"]

if "hd" in links:
link = URL(links["hd"])
else:
link = URL(links["sd"])
link = URL(links["hd"] if "hd" in links else links["sd"])

filename, ext = await get_filename_and_ext(link.name)
new_scrape_item = await self.create_scrape_item(scrape_item, link, title, True, date)
Expand Down
5 changes: 2 additions & 3 deletions cyberdrop_dl/scraper/crawlers/simpcity_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,8 @@ async def images(self, scrape_item: ScrapeItem, post_content: Tag) -> None:
continue

parent_simp_check = image.parent.get("data-simp")
if parent_simp_check:
if "init" in parent_simp_check:
continue
if parent_simp_check and "init" in parent_simp_check:
continue

link = link.replace(".th.", ".").replace(".md.", ".")
if link.endswith("/"):
Expand Down
5 changes: 2 additions & 3 deletions cyberdrop_dl/scraper/crawlers/socialmediagirls_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,8 @@ async def images(self, scrape_item: ScrapeItem, post_content: Tag) -> None:
continue

parent_simp_check = image.parent.get("data-simp")
if parent_simp_check:
if "init" in parent_simp_check:
continue
if parent_simp_check and "init" in parent_simp_check:
continue

link = link.replace(".th.", ".").replace(".md.", ".")
if link.endswith("/"):
Expand Down
5 changes: 2 additions & 3 deletions cyberdrop_dl/scraper/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,8 @@ async def start_scrapers(self) -> None:

async def start_jdownloader(self) -> None:
"""Starts JDownloader"""
if self.jdownloader.enabled:
if isinstance(self.jdownloader.jdownloader_agent, Field):
await self.jdownloader.jdownloader_setup()
if self.jdownloader.enabled and isinstance(self.jdownloader.jdownloader_agent, Field):
await self.jdownloader.jdownloader_setup()

async def start(self) -> None:
"""Starts the orchestra"""
Expand Down
4 changes: 2 additions & 2 deletions cyberdrop_dl/ui/progress/statistic_progress.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ async def get_progress(self) -> Panel:

async def update_total(self, total: int) -> None:
"""Updates the total number of files to be downloaded"""
for key in self.failure_types.keys():
for key in self.failure_types:
self.progress.update(self.failure_types[key], total=total)

async def add_failure(self, failure_type: [str, int]) -> None:
Expand Down Expand Up @@ -66,7 +66,7 @@ async def get_progress(self) -> Panel:

async def update_total(self, total: int) -> None:
"""Updates the total number of sites to be scraped"""
for key in self.failure_types.keys():
for key in self.failure_types:
self.progress.update(self.failure_types[key], total=total)

async def add_failure(self, failure_type: [str, int]) -> None:
Expand Down
2 changes: 1 addition & 1 deletion cyberdrop_dl/ui/prompts/general_prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def import_cyberdrop_v4_items_prompt(manager: Manager) -> None:
"""Import Cyberdrop_V4 Items"""
while True:
console.clear()
console.print(f"Editing Config Values")
console.print("Editing Config Values")
action = inquirer.select(
message="What would you like to do?",
choices=[
Expand Down
10 changes: 5 additions & 5 deletions cyberdrop_dl/ui/prompts/settings_user_prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
def create_new_config_prompt(manager: Manager) -> None:
"""Create a new config file"""
console.clear()
console.print(f"Create a new config file")
console.print("Create a new config file")
config_name = inquirer.text(
message="Enter the name of the config:",
validate=EmptyInputValidator("Input should not be empty")
Expand All @@ -40,7 +40,7 @@ def edit_config_values_prompt(manager: Manager) -> None:

while True:
console.clear()
console.print(f"Editing Config Values")
console.print("Editing Config Values")
action = inquirer.select(
message="What would you like to do?",
choices=[
Expand Down Expand Up @@ -129,7 +129,7 @@ def edit_download_options_prompt(config: Dict) -> None:
], long_instruction="ARROW KEYS: Navigate | TAB: Select | ENTER: Confirm"
).execute()

for key in config["Download_Options"].keys():
for key in config["Download_Options"]:
config["Download_Options"][key] = False

for key in action:
Expand Down Expand Up @@ -272,7 +272,7 @@ def edit_ignore_options_prompt(config: Dict) -> None:
], long_instruction="ARROW KEYS: Move | TAB: Select | ENTER: Confirm",
).execute()

for key in config["Ignore_Options"].keys():
for key in config["Ignore_Options"]:
config["Ignore_Options"][key] = False

for key in action:
Expand Down Expand Up @@ -335,7 +335,7 @@ def edit_runtime_options_prompt(config: Dict) -> None:
long_instruction="10 is the default (uses pythons logging numerical levels)",
).execute()

for key in config["Runtime_Options"].keys():
for key in config["Runtime_Options"]:
config["Runtime_Options"][key] = False

for key in action:
Expand Down
2 changes: 1 addition & 1 deletion cyberdrop_dl/utils/args/browser_cookie_extraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def wrapper(self, *args, **kwargs):
except PermissionError:
console = Console()
console.clear()
console.print(f"We've encountered a Permissions Error. Please close all browsers and try again.", style="bold red")
console.print("We've encountered a Permissions Error. Please close all browsers and try again.", style="bold red")
console.print("If you are still having issues, make sure all browsers processes are closed in a Task Manager.", style="bold red")
console.print("Nothing has been saved.", style="bold red")
inquirer.confirm(message="Press enter to return menu.").execute()
Expand Down
8 changes: 3 additions & 5 deletions cyberdrop_dl/utils/transfer/first_time_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,10 @@ def startup(self) -> None:
OLD_APP_STORAGE: Path = Path(platformdirs.user_config_dir("Cyberdrop-DL"))
OLD_DOWNLOAD_STORAGE = Path(platformdirs.user_downloads_path()) / "Cyberdrop-DL Downloads"

check = False
if APP_STORAGE.exists():
if (APP_STORAGE / "Cache" / "cache.yaml").is_file():
check = self.check_cache_for_moved(APP_STORAGE / "Cache" / "cache.yaml")
if check:
return
cache_file = APP_STORAGE / "Cache" / "cache.yaml"
if cache_file.is_file() and self.check_cache_for_moved(cache_file):
return

OLD_FILES = Path("./Old Files")
OLD_FILES.mkdir(parents=True, exist_ok=True)
Expand Down
5 changes: 1 addition & 4 deletions cyberdrop_dl/utils/utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,7 @@ def error_handling_wrapper(func):
"""Wrapper handles errors for url scraping"""
@wraps(func)
async def wrapper(self, *args, **kwargs):
if isinstance(args[0], URL):
link = args[0]
else:
link = args[0].url
link = args[0] if isinstance(args[0], URL) else args[0].url

try:
return await func(self, *args, **kwargs)
Expand Down

0 comments on commit 8e103d5

Please sign in to comment.