Merge pull request #485 from NTFSvolume/deprecate_get_domain_db

fix: deprecate `get_db_domain` (chevereto)

cyberdrop_dl/scraper/crawlers/chevereto_crawler.py

@@ -72,7 +72,7 @@ class CheveretoCrawler(Crawler):
     def __init__(self, manager: Manager, site: str) -> None:
         super().__init__(manager, site, self.FOLDER_DOMAINS.get(site, "Chevereto"))
         self.primary_base_domain = self.PRIMARY_BASE_DOMAINS.get(site, URL(f"https://{site}"))
-        self.request_limiter = AsyncLimiter(2, 1)
+        self.request_limiter = AsyncLimiter(1, 1)
         self.next_page_selector = "a[data-pagination=next]"
         self.album_title_selector = "a[data-text=album-name]"
         self.album_img_selector = "a[class='image-container --media'] img"

cyberdrop_dl/utils/database/tables/history_table.py

@@ -30,24 +30,6 @@ def get_db_path(url: URL, referer: str = "") -> str:
     return url_path
 
 
-def get_db_domain(domain: str) -> str:
-    """Gets the domain to be put into the DB and checked from the DB."""
-    if domain in (
-        "img.kiwi",
-        "jpg.church",
-        "jpg.homes",
-        "jpg.fish",
-        "jpg.fishing",
-        "jpg.pet",
-        "jpeg.pet",
-        "jpg1.su",
-        "jpg2.su",
-        "jpg3.su",
-    ):
-        domain = "sharex"
-    return domain
-
-
 class HistoryTable:
     def __init__(self, db_conn: aiosqlite.Connection) -> None:
         self.db_conn: aiosqlite.Connection = db_conn
@@ -60,29 +42,32 @@ async def startup(self) -> None:
         await self.fix_primary_keys()
         await self.add_columns_media()
         await self.fix_bunkr_v4_entries()
+        await self.fix_chevereto_domains()
+
+    async def fix_chevereto_domains(self) -> None:
+        query = """UPDATE media SET domain = 'jpg5.su' WHERE domain = 'sharex'"""
+        cursor = await self.db_conn.cursor()
+        await cursor.execute(query)
+        await self.db_conn.commit()
 
     async def check_complete(self, domain: str, url: URL, referer: URL) -> bool:
         """Checks whether an individual file has completed given its domain and url path."""
         if self.ignore_history:
             return False
 
-        domain = get_db_domain(domain)
-
         url_path = get_db_path(url, domain)
         cursor = await self.db_conn.cursor()
-        result = await cursor.execute(
-            """SELECT referer, completed FROM media WHERE domain = ? and url_path = ?""",
-            (domain, url_path),
-        )
+        query = """SELECT referer, completed FROM media WHERE domain = ? and url_path = ?"""
+        result = await cursor.execute(query, (domain, url_path))
         sql_file_check = await result.fetchone()
         if sql_file_check and sql_file_check[1] != 0:
             # Update the referer if it has changed so that check_complete_by_referer can work
-            if str(referer) != sql_file_check[0]:
-                await cursor.execute(
-                    """UPDATE media SET referer = ? WHERE domain = ? and url_path = ?""",
-                    (str(referer), domain, url_path),
-                )
+            if str(referer) != sql_file_check[0] and url != referer:
+                log(f"Updating referer of {url} from {sql_file_check[0]} to {referer}")
+                query = """UPDATE media SET referer = ? WHERE domain = ? and url_path = ?"""
+                await cursor.execute(query, (str(referer), domain, url_path))
                 await self.db_conn.commit()
+
             return True
         return False
 
@@ -91,7 +76,6 @@ async def check_album(self, domain: str, album_id: str) -> dict[Any, Any]:
         if self.ignore_history:
             return {}
 
-        domain = get_db_domain(domain)
         cursor = await self.db_conn.cursor()
         result = await cursor.execute(
             """SELECT url_path, completed FROM media WHERE domain = ? and album_id = ?""",
@@ -102,7 +86,7 @@ async def check_album(self, domain: str, album_id: str) -> dict[Any, Any]:
 
     async def set_album_id(self, domain: str, media_item: MediaItem) -> None:
         """Sets an album_id in the database."""
-        domain = get_db_domain(domain)
+
         url_path = get_db_path(media_item.url, str(media_item.referer))
         await self.db_conn.execute(
             """UPDATE media SET album_id = ? WHERE domain = ? and url_path = ?""",
@@ -115,7 +99,6 @@ async def check_complete_by_referer(self, domain: str, referer: URL) -> bool:
         if self.ignore_history:
             return False
 
-        domain = get_db_domain(domain)
         cursor = await self.db_conn.cursor()
         result = await cursor.execute(
             """SELECT completed FROM media WHERE domain = ? and referer = ?""",
@@ -126,7 +109,7 @@ async def check_complete_by_referer(self, domain: str, referer: URL) -> bool:
 
     async def insert_incompleted(self, domain: str, media_item: MediaItem) -> None:
         """Inserts an uncompleted file into the database."""
-        domain = get_db_domain(domain)
+
         url_path = get_db_path(media_item.url, str(media_item.referer))
         download_filename = media_item.download_filename or ""
         try:
@@ -160,7 +143,7 @@ async def insert_incompleted(self, domain: str, media_item: MediaItem) -> None:
 
     async def mark_complete(self, domain: str, media_item: MediaItem) -> None:
         """Mark a download as completed in the database."""
-        domain = get_db_domain(domain)
+
         url_path = get_db_path(media_item.url, str(media_item.referer))
         await self.db_conn.execute(
             """UPDATE media SET completed = 1, completed_at = CURRENT_TIMESTAMP WHERE domain = ? and url_path = ?""",
@@ -170,7 +153,7 @@ async def mark_complete(self, domain: str, media_item: MediaItem) -> None:
 
     async def add_filesize(self, domain: str, media_item: MediaItem) -> None:
         """Add the file size to the db."""
-        domain = get_db_domain(domain)
+
         url_path = get_db_path(media_item.url, str(media_item.referer))
         file_size = pathlib.Path(media_item.complete_file).stat().st_size
         await self.db_conn.execute(
@@ -188,7 +171,7 @@ async def check_filename_exists(self, filename: str) -> bool:
 
     async def get_downloaded_filename(self, domain: str, media_item: MediaItem) -> str:
         """Returns the downloaded filename from the database."""
-        domain = get_db_domain(domain)
+
         url_path = get_db_path(media_item.url, str(media_item.referer))
         cursor = await self.db_conn.cursor()
         result = await cursor.execute(