fix: async references

cyberdrop_dl/clients/hash_client.py

@@ -3,9 +3,10 @@
 import asyncio
 import time
 from collections import defaultdict
+from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from pathlib import Path
-from typing import TYPE_CHECKING, AsyncGenerator
+from typing import TYPE_CHECKING
 
 from send2trash import send2trash
 
@@ -35,7 +36,7 @@ async def _hash_directory_scanner_helper(manager: Manager, path: Path):
     start_time = time.perf_counter()
     async with hash_scan_directory_context(manager):
         await manager.hash_manager.hash_client.hash_directory(path)
-        await manager.progress_manager.print_stats(start_time)
+        manager.progress_manager.print_stats(start_time)
 
 
 class HashClient:

cyberdrop_dl/main.py

@@ -43,8 +43,6 @@ def startup() -> Manager:
         if not manager.args_manager.immediate_download:
             program_ui(manager)
 
-        return manager
-
     except InvalidYamlError as e:
         print_to_console(e.message_rich)
         sys.exit(1)
@@ -53,6 +51,9 @@ def startup() -> Manager:
         print_to_console("Exiting...")
         sys.exit(0)
 
+    else:
+        return manager
+
 
 async def runtime(manager: Manager) -> None:
     """Main runtime loop for the program, this will run until all scraping and downloading is complete."""
@@ -185,7 +186,7 @@ async def director(manager: Manager) -> None:
             with manager.live_manager.get_main_live(stop=True):
                 await runtime(manager)
                 await post_runtime(manager)
-        except Exception as e:
+        except* Exception as e:
             log_with_color(
                 f"An error occurred, please report this to the developer: {e}",
                 "bold red",

cyberdrop_dl/managers/live_manager.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Generator
+from typing import TYPE_CHECKING
 
 from rich.live import Live
 from rich.progress import Progress, SpinnerColumn, TextColumn
 
-from cyberdrop_dl.utils.logger import console
+from cyberdrop_dl.utils.logger import console, log
 
 if TYPE_CHECKING:
     from rich.layout import Layout
@@ -38,10 +39,14 @@ def get_live(self, layout: Layout, stop: bool = False) -> Generator[Live]:
             self.live.update(show, refresh=True)
             yield self.live
 
-        except Exception as e:
-            msg = f"Issue with rich live {e}"
-            raise Exception(msg) from e
-
+        except* Exception as e:
+            msg = f"Issue with rich live: {e}"
+            log(msg, 50, exc_info=True)
+            if isinstance(e, ExceptionGroup):
+                for sub_exception in e.exceptions:
+                    msg = f"Multiple exception caught: {type(sub_exception).__name__} - {sub_exception}"
+                    log(msg, 50, exc_info=sub_exception)
+            raise e
         finally:
             if stop:
                 self.live.stop()

cyberdrop_dl/managers/real_debrid/api.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 import time
+from collections.abc import Generator
 from contextlib import contextmanager
 from datetime import date, datetime, timedelta
-from typing import TYPE_CHECKING, Generator
+from typing import TYPE_CHECKING
 
 from requests import Session
 from requests.exceptions import RequestException
@@ -82,11 +83,12 @@ def handle_response(response: Response) -> dict | str | None:
         try:
             response.raise_for_status()
             JSONResp: dict = response.json()
-            return JSONResp
         except RequestException:
             raise RealDebridError(response) from None
         except AttributeError:
             return response.text
+        else:
+            return JSONResp
 
     @contextmanager
     def rate_limiter(self, buffer: float = 0.2) -> Generator:

cyberdrop_dl/scraper/crawlers/celebforum_crawler.py

@@ -117,7 +117,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
 
                 if scrape_post:
                     date = int(post.select_one(self.post_date_selector).get(self.post_date_attribute))
-                    new_scrape_item = await self.create_scrape_item(
+                    new_scrape_item = self.create_scrape_item(
                         scrape_item,
                         thread_url,
                         title,
@@ -161,7 +161,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
     async def post(self, scrape_item: ScrapeItem, post_content: Tag, post_number: int) -> None:
         """Scrapes a post."""
         if self.manager.config_manager.settings_data["Download_Options"]["separate_posts"]:
-            scrape_item = await self.create_scrape_item(scrape_item, scrape_item.url, "")
+            scrape_item = self.create_scrape_item(scrape_item, scrape_item.url, "")
             scrape_item.add_to_parent_title("post-" + str(post_number))
 
         scrape_item.type = FORUM_POST
@@ -202,7 +202,7 @@ async def links(self, scrape_item: ScrapeItem, post_content: Tag) -> int:
 
             try:
                 if self.domain not in link.host:
-                    new_scrape_item = await self.create_scrape_item(scrape_item, link, "")
+                    new_scrape_item = self.create_scrape_item(scrape_item, link, "")
                     await self.handle_external_links(new_scrape_item)
                 elif self.attachment_url_part in link.parts:
                     await self.handle_internal_links(link, scrape_item)
@@ -241,7 +241,7 @@ async def images(self, scrape_item: ScrapeItem, post_content: Tag) -> int:
             link = URL(link)
 
             if self.domain not in link.host:
-                new_scrape_item = await self.create_scrape_item(scrape_item, link, "")
+                new_scrape_item = self.create_scrape_item(scrape_item, link, "")
                 await self.handle_external_links(new_scrape_item)
             elif self.attachment_url_part in link.parts:
                 await self.handle_internal_links(link, scrape_item)
@@ -270,7 +270,7 @@ async def videos(self, scrape_item: ScrapeItem, post_content: Tag) -> int:
                 link = "https:" + link
 
             link = URL(link)
-            new_scrape_item = await self.create_scrape_item(scrape_item, link, "")
+            new_scrape_item = self.create_scrape_item(scrape_item, link, "")
             await self.handle_external_links(new_scrape_item)
             new_children += 1
             if scrape_item.children_limit and (new_children + scrape_item.children) >= scrape_item.children_limit:
@@ -290,7 +290,7 @@ async def embeds(self, scrape_item: ScrapeItem, post_content: Tag) -> int:
             link = link.replace("ifr", "watch")
 
             link = URL(link)
-            new_scrape_item = await self.create_scrape_item(scrape_item, link, "")
+            new_scrape_item = self.create_scrape_item(scrape_item, link, "")
             await self.handle_external_links(new_scrape_item)
             new_children += 1
             if scrape_item.children_limit and (new_children + scrape_item.children) >= scrape_item.children_limit:
@@ -321,7 +321,7 @@ async def attachments(self, scrape_item: ScrapeItem, post_content: Tag) -> int:
             link = URL(link)
 
             if self.domain not in link.host:
-                new_scrape_item = await self.create_scrape_item(scrape_item, link, "")
+                new_scrape_item = self.create_scrape_item(scrape_item, link, "")
                 await self.handle_external_links(new_scrape_item)
             elif self.attachment_url_part in link.parts:
                 await self.handle_internal_links(link, scrape_item)
@@ -338,5 +338,5 @@ async def attachments(self, scrape_item: ScrapeItem, post_content: Tag) -> int:
     async def handle_internal_links(self, link: URL, scrape_item: ScrapeItem) -> None:
         """Handles internal links."""
         filename, ext = get_filename_and_ext(link.name, True)
-        new_scrape_item = await self.create_scrape_item(scrape_item, link, "Attachments", True)
+        new_scrape_item = self.create_scrape_item(scrape_item, link, "Attachments", True)
         await self.handle_file(link, new_scrape_item, filename, ext)

cyberdrop_dl/scraper/crawlers/chevereto_crawler.py

@@ -3,7 +3,8 @@
 import calendar
 import datetime
 import re
-from typing import TYPE_CHECKING, AsyncGenerator, ClassVar
+from collections.abc import AsyncGenerator
+from typing import TYPE_CHECKING, ClassVar
 
 from aiolimiter import AsyncLimiter
 from bs4 import BeautifulSoup
@@ -27,7 +28,7 @@
 
 
 class CheveretoCrawler(Crawler):
-    JPG_CHURCH_DOMAINS = {
+    JPG_CHURCH_DOMAINS: ClassVar[tuple[str, ...]] = {
         "jpg.homes",
         "jpg.church",
         "jpg.fish",
@@ -42,7 +43,7 @@ class CheveretoCrawler(Crawler):
         "host.church",
     }
 
-    PRIMARY_BASE_DOMAINS = {
+    PRIMARY_BASE_DOMAINS: ClassVar[dict[str, URL]] = {
         "imagepond.net": URL("https://imagepond.net"),
         "jpg.church": URL("https://jpg5.su"),
         "img.kiwi": URL("https://img.kiwi"),
@@ -104,7 +105,7 @@ async def profile(self, scrape_item: ScrapeItem) -> None:
                 if link.startswith("/"):
                     link = self.primary_base_domain / link[1:]
                 link = URL(link)
-                new_scrape_item = await self.create_scrape_item(
+                new_scrape_item = self.create_scrape_item(
                     scrape_item,
                     link,
                     title,
@@ -159,7 +160,7 @@ async def album(self, scrape_item: ScrapeItem) -> None:
                 sub_album_link = self.primary_base_domain / sub_album_link[1:]
 
             sub_album_link = URL(sub_album_link)
-            new_scrape_item = await self.create_scrape_item(scrape_item, sub_album_link, "", True)
+            new_scrape_item = self.create_scrape_item(scrape_item, sub_album_link, "", True)
             self.manager.task_group.create_task(self.run(new_scrape_item))
 
         async for soup in self.web_pager(scrape_item.url):
@@ -169,7 +170,7 @@ async def album(self, scrape_item: ScrapeItem) -> None:
                 if link.startswith("/"):
                     link = self.primary_base_domain / link[1:]
                 link = URL(link)
-                new_scrape_item = await self.create_scrape_item(
+                new_scrape_item = self.create_scrape_item(
                     scrape_item,
                     link,
                     title,
@@ -203,7 +204,7 @@ async def image(self, scrape_item: ScrapeItem) -> None:
                 break
 
         if date:
-            date = await self.parse_datetime(date)
+            date = self.parse_datetime(date)
             scrape_item.possible_datetime = date
 
         filename, ext = get_filename_and_ext(link.name)
@@ -242,7 +243,7 @@ async def get_sort_by_new_url(url: URL) -> URL:
         return url.with_query({"sort": "date_desc", "page": 1})
 
     @staticmethod
-    async def parse_datetime(date: str) -> int:
+    def parse_datetime(date: str) -> int:
         """Parses a datetime string into a unix timestamp."""
         date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
         return calendar.timegm(date.timetuple())

cyberdrop_dl/scraper/crawlers/coomer_crawler.py

@@ -71,14 +71,14 @@ async def favorites(self, scrape_item: ScrapeItem) -> None:
                 id = user["id"]
                 service = user["service"]
                 url = self.primary_base_domain / service / "user" / id
-                new_scrape_item = await self.create_scrape_item(scrape_item, url, None, True, None, None)
+                new_scrape_item = self.create_scrape_item(scrape_item, url, None, True, None, None)
                 self.manager.task_group.create_task(self.run(new_scrape_item))
 
     @error_handling_wrapper
     async def profile(self, scrape_item: ScrapeItem) -> None:
         """Scrapes a profile."""
         offset = 0
-        service, user = await self.get_service_and_user(scrape_item)
+        service, user = self.get_service_and_user(scrape_item)
         user_str = await self.get_user_str_from_profile(scrape_item)
         api_call = self.api_url / service / "user" / user
         scrape_item.type = FILE_HOST_PROFILE
@@ -198,13 +198,13 @@ async def create_new_scrape_item(
                 post_title = post_id + " - " + post_title
 
         new_title = self.create_title(user, None, None)
-        new_scrape_item = await self.create_scrape_item(
+        new_scrape_item = self.create_scrape_item(
             old_scrape_item,
             link,
             new_title,
             True,
             None,
-            await self.parse_datetime(date),
+            self.parse_datetime(date),
             add_parent=add_parent,
         )
         new_scrape_item.add_to_parent_title(post_title)
@@ -223,7 +223,7 @@ async def get_user_str_from_profile(self, scrape_item: ScrapeItem) -> str:
         return soup.select_one("span[itemprop=name]").text
 
     @staticmethod
-    async def get_service_and_user(scrape_item: ScrapeItem) -> tuple[str, str]:
+    def get_service_and_user(scrape_item: ScrapeItem) -> tuple[str, str]:
         """Gets the service and user from a scrape item."""
         user = scrape_item.url.parts[3]
         service = scrape_item.url.parts[1]
@@ -238,7 +238,7 @@ async def get_service_user_and_post(scrape_item: ScrapeItem) -> tuple[str, str,
         return service, user, post
 
     @staticmethod
-    async def parse_datetime(date: str) -> int:
+    def parse_datetime(date: str) -> int:
         """Parses a datetime string."""
         date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
         return calendar.timegm(date.timetuple())

cyberdrop_dl/scraper/crawlers/cyberdrop_crawler.py

@@ -72,7 +72,7 @@ async def album(self, scrape_item: ScrapeItem) -> None:
 
         date = soup.select("p[class=title]")
         if date:
-            date = await self.parse_datetime(date[-1].text)
+            date = self.parse_datetime(date[-1].text)
 
         links = soup.select("div[class*=image-container] a[class=image]")
         for link in links:
@@ -81,7 +81,7 @@ async def album(self, scrape_item: ScrapeItem) -> None:
                 link = self.primary_base_domain.with_path(link)
             link = URL(link)
 
-            new_scrape_item = await self.create_scrape_item(
+            new_scrape_item = self.create_scrape_item(
                 scrape_item,
                 link,
                 title,
@@ -146,7 +146,7 @@ def is_cdn(url: URL) -> bool:
         return bool(re.match(CDN_POSSIBILITIES, url.host))
 
     @staticmethod
-    async def parse_datetime(date: str) -> int:
+    def parse_datetime(date: str) -> int:
         """Parses a datetime string into a unix timestamp."""
         date = datetime.datetime.strptime(date, "%d.%m.%Y")
         return calendar.timegm(date.timetuple())

cyberdrop_dl/scraper/crawlers/cyberfile_crawler.py

@@ -113,7 +113,7 @@ async def folder(self, scrape_item: ScrapeItem) -> None:
                 elif file_id:
                     link = URL(tile.get("dtfullurl"))
                 if link:
-                    new_scrape_item = await self.create_scrape_item(
+                    new_scrape_item = self.create_scrape_item(
                         scrape_item,
                         link,
                         title,
@@ -125,7 +125,7 @@ async def folder(self, scrape_item: ScrapeItem) -> None:
                     log(f"Couldn't find folder or file id for {scrape_item.url} element", 30)
                     continue
 
-                new_scrape_item = await self.create_scrape_item(
+                new_scrape_item = self.create_scrape_item(
                     scrape_item,
                     link,
                     title,
@@ -188,7 +188,7 @@ async def shared(self, scrape_item: ScrapeItem) -> None:
                     link = URL(tile.get("dtfullurl"))
 
                 if link:
-                    new_scrape_item = await self.create_scrape_item(
+                    new_scrape_item = self.create_scrape_item(
                         scrape_item,
                         link,
                         title,
@@ -201,7 +201,7 @@ async def shared(self, scrape_item: ScrapeItem) -> None:
                     log(f"Couldn't find folder or file id for {scrape_item.url} element", 30)
                     continue
 
-                new_scrape_item = await self.create_scrape_item(
+                new_scrape_item = self.create_scrape_item(
                     scrape_item,
                     link,
                     title,
@@ -277,7 +277,7 @@ async def handle_content_id(self, scrape_item: ScrapeItem, content_id: int) -> N
         file_detail_table = ajax_soup.select('table[class="table table-bordered table-striped"]')[-1]
         uploaded_row = file_detail_table.select("tr")[-2]
         uploaded_date = uploaded_row.select_one("td[class=responsiveTable]").text.strip()
-        uploaded_date = await self.parse_datetime(uploaded_date)
+        uploaded_date = self.parse_datetime(uploaded_date)
         scrape_item.possible_datetime = uploaded_date
 
         filename, ext = get_filename_and_ext(link.name)
@@ -286,7 +286,7 @@ async def handle_content_id(self, scrape_item: ScrapeItem, content_id: int) -> N
     """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
 
     @staticmethod
-    async def parse_datetime(date: str) -> int:
+    def parse_datetime(date: str) -> int:
         """Parses a datetime string into a unix timestamp."""
         date = datetime.datetime.strptime(date, "%d/%m/%Y %H:%M:%S")
         return calendar.timegm(date.timetuple())