refactor: utilities.py

NTFSvolume · Nov 7, 2024 · bd1f501 · bd1f501
1 parent 139f0e2
commit bd1f501
Show file tree

Hide file tree

Showing 51 changed files with 556 additions and 532 deletions.
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
@@ -74,12 +74,12 @@ def __init__(self, manager: Manager, client_manager: ClientManager) -> None:
     def add_request_log_hooks(self) -> None:
         async def on_request_start(*args):
             params: aiohttp.TraceRequestStartParams = args[2]
-            await log(f"Starting download {params.method} request to {params.url}", 40)
+            log(f"Starting download {params.method} request to {params.url}", 40)
 
         async def on_request_end(*args):
             params: aiohttp.TraceRequestEndParams = args[2]
-            await log(f"Finishing download {params.method} request to {params.url}", 40)
-            await log(f"Response status for {params.url}: {params.response.status}", 40)
+            log(f"Finishing download {params.method} request to {params.url}", 40)
+            log(f"Response status for {params.url}: {params.response.status}", 40)
 
         trace_config = aiohttp.TraceConfig()
         trace_config.on_request_start.append(on_request_start)
@@ -142,7 +142,7 @@ async def _download(
                     await self.manager.progress_manager.download_progress.add_skipped()
                     return False
                 if not proceed:
-                    await log(f"Skipping {media_item.url} as it has already been downloaded", 10)
+                    log(f"Skipping {media_item.url} as it has already been downloaded", 10)
                     await self.manager.progress_manager.download_progress.add_previously_completed(False)
                     await self.process_completed(media_item, domain)
                     await self.handle_media_item_completion(media_item, downloaded=False)
@@ -197,7 +197,7 @@ async def _append_content(
     async def download_file(self, manager: Manager, domain: str, media_item: MediaItem) -> bool:
         """Starts a file."""
         if self.manager.config_manager.settings_data["Download_Options"]["skip_download_mark_completed"]:
-            await log(f"Download Skip {media_item.url} due to mark completed option", 10)
+            log(f"Download Skip {media_item.url} due to mark completed option", 10)
             await self.manager.progress_manager.download_progress.add_skipped()
             # set completed path
             await self.mark_incomplete(media_item, domain)
@@ -250,7 +250,7 @@ async def handle_media_item_completion(self, media_item: MediaItem, downloaded:
             if not downloaded:
                 self.manager.path_manager.add_prev(media_item)
         except Exception:
-            await log(f"Error handling media item completion of: {media_item.complete_file}", 10, exc_info=True)
+            log(f"Error handling media item completion of: {media_item.complete_file}", 10, exc_info=True)
 
     """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
 
@@ -282,13 +282,13 @@ async def get_final_file_info(self, media_item: MediaItem, domain: str) -> tuple
         if not skip and self.manager.config_manager.settings_data["Ignore_Options"]["skip_hosts"]:
             skip_hosts = self.manager.config_manager.settings_data["Ignore_Options"]["skip_hosts"]
             if any(host in media_item.url.host for host in skip_hosts):
-                await log(f"Download Skip {media_item.url} due to skip_hosts config", 10)
+                log(f"Download Skip {media_item.url} due to skip_hosts config", 10)
                 skip = True
 
         if not skip and self.manager.config_manager.settings_data["Ignore_Options"]["only_hosts"]:
             only_hosts = self.manager.config_manager.settings_data["Ignore_Options"]["only_hosts"]
             if not any(host in media_item.url.host for host in only_hosts):
-                await log(f"Download Skip {media_item.url} due to only_hosts config", 10)
+                log(f"Download Skip {media_item.url} due to only_hosts config", 10)
                 skip = True
 
         if skip:
@@ -298,7 +298,7 @@ async def get_final_file_info(self, media_item: MediaItem, domain: str) -> tuple
             if expected_size:
                 file_size_check = await self.check_filesize_limits(media_item)
                 if not file_size_check:
-                    await log(f"Download Skip {media_item.url} due to filesize restrictions", 10)
+                    log(f"Download Skip {media_item.url} due to filesize restrictions", 10)
                     proceed = False
                     skip = True
                     return proceed, skip

diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
@@ -90,7 +90,7 @@ async def hash_item(self, file: Path | str, original_filename: str, referer: URL
                     referer,
                 )
         except Exception as e:
-            await log(f"Error hashing {file} : {e}", 40, exc_info=True)
+            log(f"Error hashing {file} : {e}", 40, exc_info=True)
         self.hashes[key] = hash
         return hash
 
@@ -99,7 +99,7 @@ async def hash_item_during_download(self, media_item: MediaItem) -> None:
             if self.manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["hash_while_downloading"]:
                 await self.hash_item(media_item.complete_file, media_item.original_filename, media_item.referer)
         except Exception as e:
-            await log(f"After hash processing failed: {media_item.complete_file} with error {e}", 40, exc_info=True)
+            log(f"After hash processing failed: {media_item.complete_file} with error {e}", 40, exc_info=True)
 
     async def cleanup_dupes(self) -> None:
         async with self.manager.live_manager.get_hash_live():
@@ -134,7 +134,7 @@ async def final_dupe_cleanup(self, final_dict: dict[str, dict]) -> None:
                             continue
                         try:
                             self.send2trash(ele)
-                            await log(f"Sent prev download: {ele!s} to trash with hash {hash}", 10)
+                            log(f"Sent prev download: {ele!s} to trash with hash {hash}", 10)
                             await self.manager.progress_manager.hash_progress.add_removed_prev_file()
                         except OSError:
                             continue
@@ -145,7 +145,7 @@ async def final_dupe_cleanup(self, final_dict: dict[str, dict]) -> None:
                             continue
                         try:
                             self.send2trash(ele)
-                            await log(f"Sent prev download: {ele!s} to trash with hash {hash}", 10)
+                            log(f"Sent prev download: {ele!s} to trash with hash {hash}", 10)
                             await self.manager.progress_manager.hash_progress.add_removed_prev_file()
                         except OSError:
                             continue
@@ -154,7 +154,7 @@ async def final_dupe_cleanup(self, final_dict: dict[str, dict]) -> None:
                     try:
                         if selected_file.exists():
                             self.send2trash(selected_file)
-                            await log(f"Sent new download:{selected_file} to trash with hash {hash}", 10)
+                            log(f"Sent new download:{selected_file} to trash with hash {hash}", 10)
                             await self.manager.progress_manager.hash_progress.add_removed_file()
 
                     except OSError:
@@ -171,7 +171,7 @@ async def get_file_hashes_dict(self) -> dict:
                 if hash:
                     hashes_dict[hash][size].append(item)
             except Exception as e:
-                await log(f"After hash processing failed: {item} with error {e}", 40, exc_info=True)
+                log(f"After hash processing failed: {item} with error {e}", 40, exc_info=True)
         return hashes_dict
 
     async def get_candiate_per_group(self, hashes_dict: dict[str, dict[int, list[Path]]]) -> dict:
@@ -188,7 +188,7 @@ async def get_candiate_per_group(self, hashes_dict: dict[str, dict[int, list[Pat
                 for file in filter(lambda x: x != selected_file, files):
                     try:
                         self.send2trash(file)
-                        await log(f"Sent new download : {file} to trash with hash {hash}", 10)
+                        log(f"Sent new download : {file} to trash with hash {hash}", 10)
                         await self.manager.progress_manager.hash_progress.add_removed_file()
                     except OSError:
                         pass

diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
@@ -60,12 +60,12 @@ def __init__(self, client_manager: ClientManager) -> None:
     def add_request_log_hooks(self) -> None:
         async def on_request_start(*args):
             params: aiohttp.TraceRequestStartParams = args[2]
-            await log(f"Starting download {params.method} request to {params.url}", 40)
+            log(f"Starting download {params.method} request to {params.url}", 40)
 
         async def on_request_end(*args):
             params: aiohttp.TraceRequestEndParams = args[2]
-            await log(f"Finishing download {params.method} request to {params.url}", 40)
-            await log(f"Response status for {params.url}: {params.response.status}", 40)
+            log(f"Finishing download {params.method} request to {params.url}", 40)
+            log(f"Response status for {params.url}: {params.response.status}", 40)
 
         trace_config = aiohttp.TraceConfig()
         trace_config.on_request_start.append(on_request_start)

diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
@@ -46,7 +46,7 @@ async def wrapper(self: Downloader, *args, **kwargs) -> None:
 
                 log_message = f"with status {e.status} and message: {e.message}"
 
-                await log(f"{self.log_prefix} failed: {media_item.url} {log_message}", 40)
+                log(f"{self.log_prefix} failed: {media_item.url} {log_message}", 40)
 
                 if media_item.current_attempt >= max_attempts:
                     await self.manager.progress_manager.download_stats_progress.add_failure(e.ui_message)
@@ -59,7 +59,7 @@ async def wrapper(self: Downloader, *args, **kwargs) -> None:
                     break
 
                 retrying_message = f"Retrying {self.log_prefix.lower()}: {media_item.url} ,retry attempt: {media_item.current_attempt + 1}"
-                await log(retrying_message, 20)
+                log(retrying_message, 20)
                 continue
 
             except CDLBaseError as e:
@@ -83,7 +83,7 @@ async def wrapper(self: Downloader, *args, **kwargs) -> None:
                     log_message = log_message_short = ui_message = f"{status} - {message}"
 
             failed_message = f"{self.log_prefix} failed: {media_item.url} with error: {log_message}"
-            await log(failed_message, 40, exc_info=exc_info)
+            log(failed_message, 40, exc_info=exc_info)
             await self.attempt_task_removal(media_item)
             await self.manager.log_manager.write_download_error_log(media_item.url, log_message_short, origin)
             await self.manager.progress_manager.download_stats_progress.add_failure(ui_message)
@@ -130,19 +130,19 @@ async def run(self, media_item: MediaItem) -> None:
             self.processed_items.append(media_item.url.path)
             await self.manager.progress_manager.download_progress.update_total()
 
-            await log(f"{self.log_prefix} starting: {media_item.url}", 20)
+            log(f"{self.log_prefix} starting: {media_item.url}", 20)
             async with self.manager.client_manager.download_session_limit:
                 try:
                     if isinstance(media_item.file_lock_reference_name, Field):
                         media_item.file_lock_reference_name = media_item.filename
                     await self._file_lock.check_lock(media_item.file_lock_reference_name)
                     await self.download(media_item)
                 except Exception as e:
-                    await log(f"{self.log_prefix} failed: {media_item.url} with error {e}", 40, exc_info=True)
+                    log(f"{self.log_prefix} failed: {media_item.url} with error {e}", 40, exc_info=True)
                     await self.manager.progress_manager.download_stats_progress.add_failure("Unknown")
                     await self.manager.progress_manager.download_progress.add_failed()
                 else:
-                    await log(f"{self.log_prefix} finished: {media_item.url}", 20)
+                    log(f"{self.log_prefix} finished: {media_item.url}", 20)
                 finally:
                     await self._file_lock.release_lock(media_item.file_lock_reference_name)
         self._semaphore.release()
@@ -200,7 +200,7 @@ async def download(self, media_item: MediaItem) -> None:
         except (DownloadError, aiohttp.ClientResponseError) as e:
             ui_message = getattr(e, "ui_message", e.status)
             log_message_short = log_message = f"{e.status} - {e.message}"
-            await log(f"{self.log_prefix} failed: {media_item.url} with error: {log_message}", 40)
+            log(f"{self.log_prefix} failed: {media_item.url} with error: {log_message}", 40)
             await self.manager.log_manager.write_download_error_log(media_item.url, log_message_short, origin)
             await self.manager.progress_manager.download_stats_progress.add_failure(ui_message)
             await self.manager.progress_manager.download_progress.add_failed()

diff --git a/cyberdrop_dl/main.py b/cyberdrop_dl/main.py
@@ -77,8 +77,8 @@ async def runtime(manager: Manager) -> None:
 
 async def post_runtime(manager: Manager) -> None:
     """Actions to complete after main runtime, and before ui shutdown."""
-    await log_spacer(20)
-    await log_with_color(
+    log_spacer(20)
+    log_with_color(
         f"Running Post-Download Processes For Config: {manager.config_manager.loaded_config}...\n",
         "green",
         20,
@@ -140,10 +140,10 @@ async def switch_config_and_reset_logger(manager: Manager, config_name: str) ->
     logger = logging.getLogger("cyberdrop_dl")
     if manager.args_manager.all_configs:
         if len(logger.handlers) > 0:
-            await log("Picking new config...", 20)
+            log("Picking new config...", 20)
         manager.config_manager.change_config(config_name)
         if len(logger.handlers) > 0:
-            await log(f"Changing config to {config_name}...", 20)
+            log(f"Changing config to {config_name}...", 20)
             old_file_handler = logger.handlers[0]
             logger.removeHandler(logger.handlers[0])
             old_file_handler.close()
@@ -185,32 +185,32 @@ async def director(manager: Manager) -> None:
             await switch_config_and_reset_logger(manager, current_config)
         configs_to_run.pop(0)
 
-        await log(f"Using Debug Log: {debug_log_file_path if debug_log_file_path else None}", 10)
-        await log("Starting Async Processes...", 20)
+        log(f"Using Debug Log: {debug_log_file_path if debug_log_file_path else None}", 10)
+        log("Starting Async Processes...", 20)
         await manager.async_startup()
 
-        await log_spacer(20)
-        await log("Starting CDL...\n", 20)
+        log_spacer(20)
+        log("Starting CDL...\n", 20)
         if not manager.args_manager.sort_all_configs:
             try:
                 async with manager.live_manager.get_main_live(stop=True):
                     await runtime(manager)
                     await post_runtime(manager)
             except Exception:
-                await log("\nAn error occurred, please report this to the developer:", 50, exc_info=True)
+                log("\nAn error occurred, please report this to the developer:", 50, exc_info=True)
                 sys.exit(1)
 
-        await log_spacer(20)
+        log_spacer(20)
         await manager.progress_manager.print_stats(start_time)
 
         if not configs_to_run:
-            await log_spacer(20)
-            await log("Checking for Updates...", 20)
-            await check_latest_pypi()
-            await log_spacer(20)
-            await log("Closing Program...", 20)
+            log_spacer(20)
+            log("Checking for Updates...", 20)
+            check_latest_pypi()
+            log_spacer(20)
+            log("Closing Program...", 20)
             await manager.close()
-            await log_with_color("Finished downloading. Enjoy :)", "green", 20, show_in_stats=False)
+            log_with_color("Finished downloading. Enjoy :)", "green", 20, show_in_stats=False)
 
         await send_webhook_message(manager)
         await sent_apprise_notifications(manager)

diff --git a/cyberdrop_dl/managers/download_manager.py b/cyberdrop_dl/managers/download_manager.py
@@ -24,21 +24,21 @@ def __init__(self) -> None:
     async def check_lock(self, filename: str) -> None:
         """Checks if the file is locked."""
         try:
-            await log_debug(f"Checking lock for {filename}", 40)
+            log_debug(f"Checking lock for {filename}", 40)
             await self._locked_files[filename].acquire()
-            await log_debug(f"Lock for {filename} acquired", 40)
+            log_debug(f"Lock for {filename} acquired", 40)
         except KeyError:
-            await log_debug(f"Lock for {filename} does not exist", 40)
+            log_debug(f"Lock for {filename} does not exist", 40)
             self._locked_files[filename] = asyncio.Lock()
             await self._locked_files[filename].acquire()
-            await log_debug(f"Lock for {filename} acquired", 40)
+            log_debug(f"Lock for {filename} acquired", 40)
 
     async def release_lock(self, filename: str) -> None:
         """Releases the file lock."""
         with contextlib.suppress(KeyError, RuntimeError):
-            await log_debug(f"Releasing lock for {filename}", 40)
+            log_debug(f"Releasing lock for {filename}", 40)
             self._locked_files[filename].release()
-            await log_debug(f"Lock for {filename} released", 40)
+            log_debug(f"Lock for {filename} released", 40)
 
 
 class DownloadManager:

diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.py
@@ -38,7 +38,7 @@ async def get_live(self, layout: Layout, stop: bool = False) -> AsyncGenerator[L
                 if not self.manager.args_manager.no_ui:
                     Console().clear()
         except Exception as e:
-            await log(f"Issue with rich live {e}", level=10, exc_info=True)
+            log(f"Issue with rich live {e}", level=10, exc_info=True)
 
     @asynccontextmanager
     async def get_main_live(self, stop: bool = False) -> AsyncGenerator[Live]:

diff --git a/cyberdrop_dl/managers/log_manager.py b/cyberdrop_dl/managers/log_manager.py
@@ -65,8 +65,8 @@ async def update_last_forum_post(self) -> None:
         if not input_file.is_file() or not self.last_post_log.is_file():
             return
 
-        await log_spacer(20)
-        await log("Updating Last Forum Posts...\n", 20)
+        log_spacer(20)
+        log("Updating Last Forum Posts...\n", 20)
 
         current_urls, current_base_urls, new_urls, new_base_urls = [], [], [], []
         async with aiofiles.open(input_file, encoding="utf8") as f:
@@ -101,11 +101,11 @@ async def update_last_forum_post(self) -> None:
                 old_url = current_urls[index]
                 if old_url == new_url:
                     continue
-                await log(f"Updating {base}\n  {old_url = }\n  {new_url = }", 20)
+                log(f"Updating {base}\n  {old_url = }\n  {new_url = }", 20)
                 updated_urls[index] = new_url
 
         if updated_urls == current_urls:
-            await log("No URLs updated", 20)
+            log("No URLs updated", 20)
             return
 
         async with aiofiles.open(input_file, "w", encoding="utf8") as f: