Finally fix a lot of those unknown errors.

Jules-WinnfieldX · Dec 15, 2023 · 35245a7 · 35245a7
1 parent b21f2c1
commit 35245a7
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 18 deletions.
diff --git a/cyberdrop_dl/__init__.py b/cyberdrop_dl/__init__.py
@@ -1 +1 @@
-__version__ = "5.0.112"
+__version__ = "5.0.113"
diff --git a/cyberdrop_dl/scraper/crawlers/imgkiwi_crawler.py b/cyberdrop_dl/scraper/crawlers/imgkiwi_crawler.py
@@ -18,7 +18,7 @@
 
 class ImgKiwiCrawler(Crawler):
     def __init__(self, manager: Manager):
-        super().__init__(manager, "sharex", "ImgKiwi")
+        super().__init__(manager, "img.kiwi", "ImgKiwi")
         self.request_limiter = AsyncLimiter(10, 1)
 
     """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""

diff --git a/cyberdrop_dl/scraper/crawlers/jpgchurch_crawler.py b/cyberdrop_dl/scraper/crawlers/jpgchurch_crawler.py
@@ -18,7 +18,7 @@
 
 class JPGChurchCrawler(Crawler):
     def __init__(self, manager: Manager):
-        super().__init__(manager, "sharex", "JPGChurch")
+        super().__init__(manager, "jpg.church", "JPGChurch")
         self.primary_base_domain = URL("https://jpg3.su")
         self.request_limiter = AsyncLimiter(10, 1)
 
@@ -28,7 +28,7 @@ async def fetch(self, scrape_item: ScrapeItem) -> None:
         """Determines where to send the scrape item based on the url"""
         task_id = await self.scraping_progress.add_task(scrape_item.url)
 
-        if await self.check_direct_link(scrape_item.url) or scrape_item.url.suffix in FILE_FORMATS['Images']:
+        if await self.check_direct_link(scrape_item.url):
             await self.handle_direct_link(scrape_item)
         else:
             scrape_item.url = self.primary_base_domain / scrape_item.url.path[1:]

diff --git a/cyberdrop_dl/scraper/scraper.py b/cyberdrop_dl/scraper/scraper.py
@@ -42,13 +42,14 @@ def __init__(self, manager: Manager):
                                  "cyberdrop": "cyberdrop", "cyberfile": "cyberfile", "e-hentai": "e-hentai",
                                  "erome": "erome", "fapello": "fapello", "f95zone": "f95zone", "gofile": "gofile",
                                  "hotpic": "hotpic", "ibb.co": "imgbb", "imageban": "imageban", "imgbox": "imgbox",
-                                 "imgur": "imgur", "img.kiwi": "sharex", "jpg.church": "sharex", "jpg.homes": "sharex",
-                                 "jpg.fish": "sharex", "jpg.fishing": "sharex", "jpg.pet": "sharex",
-                                 "jpeg.pet": "sharex", "jpg1.su": "sharex", "jpg2.su": "sharex", "jpg3.su": "sharex",
-                                 "kemono": "kemono", "leakedmodels": "leakedmodels", "mediafire": "mediafire",
-                                 "nudostar.com": "nudostar.com", "nudostar.tv": "nudostartv", "omegascans": "omegascans",
-                                 "pimpandhost": "pimpandhost", "pixeldrain": "pixeldrain", "postimg": "postimg",
-                                 "reddit": "reddit", "redd.it": "reddit", "redgifs": "redgifs",
+                                 "imgur": "imgur", "img.kiwi": "img.kiwi", "jpg.church": "jpg.church",
+                                 "jpg.homes": "jpg.church", "jpg.fish": "jpg.church", "jpg.fishing": "jpg.church",
+                                 "jpg.pet": "jpg.church", "jpeg.pet": "jpg.church", "jpg1.su": "jpg.church",
+                                 "jpg2.su": "jpg.church", "jpg3.su": "jpg.church", "kemono": "kemono",
+                                 "leakedmodels": "leakedmodels", "mediafire": "mediafire",
+                                 "nudostar.com": "nudostar.com", "nudostar.tv": "nudostartv",
+                                 "omegascans": "omegascans", "pimpandhost": "pimpandhost", "pixeldrain": "pixeldrain",
+                                 "postimg": "postimg", "reddit": "reddit", "redd.it": "reddit", "redgifs": "redgifs",
                                  "rule34.xxx": "rule34.xxx", "rule34.xyz": "rule34.xyz", "saint": "saint",
                                  "scrolller": "scrolller", "simpcity": "simpcity",
                                  "socialmediagirls": "socialmediagirls",  "toonily": "toonily", "xbunker": "xbunker",

diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.py
@@ -25,6 +25,14 @@ async def get_db_path(url: URL, referer: str = "") -> str:
     return url_path
 
 
+async def get_db_domain(domain: str) -> str:
+    """Gets the domain to be put into the DB and checked from the DB"""
+    if domain in ("img.kiwi", "jpg.church", "jpg.homes", "jpg.fish", "jpg.fishing", "jpg.pet", "jpeg.pet", "jpg1.su",
+                  "jpg2.su", "jpg3.su"):
+        domain = "sharex"
+    return domain
+
+
 class HistoryTable:
     def __init__(self, db_conn: aiosqlite.Connection):
         self.db_conn: aiosqlite.Connection = db_conn
@@ -40,25 +48,38 @@ async def check_complete(self, domain: str, url: URL) -> bool:
         """Checks whether an individual file has completed given its domain and url path"""
         if self.ignore_history:
             return False
+
+        domain = await get_db_domain(domain)
+
         url_path = await get_db_path(url, domain)
         cursor = await self.db_conn.cursor()
-        result = await cursor.execute("""SELECT completed FROM media WHERE domain = ? and url_path = ?""", (domain, url_path))
+        result = await cursor.execute("""SELECT completed FROM media WHERE domain = ? and url_path = ?""",
+                                      (domain, url_path))
         sql_file_check = await result.fetchone()
         return sql_file_check and sql_file_check[0] != 0
 
     async def insert_incompleted(self, domain: str, media_item: MediaItem) -> None:
         """Inserts an uncompleted file into the database"""
+        domain = await get_db_domain(domain)
         url_path = await get_db_path(media_item.url, str(media_item.referer))
         download_filename = media_item.download_filename if isinstance(media_item.download_filename, str) else ""
-        await self.db_conn.execute("""UPDATE media SET domain = ? WHERE domain = 'no_crawler' and url_path = ? and referer = ?""", (domain, url_path, str(media_item.referer)))
-        await self.db_conn.execute("""INSERT OR IGNORE INTO media (domain, url_path, referer, download_path, download_filename, original_filename, completed) VALUES (?, ?, ?, ?, ?, ?, ?)""", (domain, url_path, str(media_item.referer), str(media_item.download_folder), download_filename, media_item.original_filename, 0))
-        await self.db_conn.execute("""UPDATE media SET download_filename = ? WHERE domain = ? and url_path = ?""", (download_filename, domain, url_path))
+        await self.db_conn.execute(
+            """UPDATE media SET domain = ? WHERE domain = 'no_crawler' and url_path = ? and referer = ?""",
+            (domain, url_path, str(media_item.referer)))
+        await self.db_conn.execute(
+            """INSERT OR IGNORE INTO media (domain, url_path, referer, download_path, download_filename, original_filename, completed) VALUES (?, ?, ?, ?, ?, ?, ?)""",
+            (domain, url_path, str(media_item.referer), str(media_item.download_folder), download_filename,
+             media_item.original_filename, 0))
+        await self.db_conn.execute("""UPDATE media SET download_filename = ? WHERE domain = ? and url_path = ?""",
+                                   (download_filename, domain, url_path))
         await self.db_conn.commit()
 
     async def mark_complete(self, domain: str, media_item: MediaItem) -> None:
         """Mark a download as completed in the database"""
+        domain = await get_db_domain(domain)
         url_path = await get_db_path(media_item.url, str(media_item.referer))
-        await self.db_conn.execute("""UPDATE media SET completed = 1 WHERE domain = ? and url_path = ?""", (domain, url_path))
+        await self.db_conn.execute("""UPDATE media SET completed = 1 WHERE domain = ? and url_path = ?""",
+                                   (domain, url_path))
         await self.db_conn.commit()
 
     async def check_filename_exists(self, filename: str) -> bool:
@@ -70,9 +91,11 @@ async def check_filename_exists(self, filename: str) -> bool:
 
     async def get_downloaded_filename(self, domain: str, media_item: MediaItem) -> str:
         """Returns the downloaded filename from the database"""
+        domain = await get_db_domain(domain)
         url_path = await get_db_path(media_item.url, str(media_item.referer))
         cursor = await self.db_conn.cursor()
-        result = await cursor.execute("""SELECT download_filename FROM media WHERE domain = ? and url_path = ?""", (domain, url_path))
+        result = await cursor.execute("""SELECT download_filename FROM media WHERE domain = ? and url_path = ?""",
+                                      (domain, url_path))
         sql_file_check = await result.fetchone()
         return sql_file_check[0] if sql_file_check else None
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cyberdrop-dl"
-version = "5.0.112"
+version = "5.0.113"
 description = "Bulk downloader for multiple file hosts"
 authors = ["Jules Winnfield <[email protected]>"]
 readme = "README.md"