check for post number in forum fragment

Jules-WinnfieldX · Jan 3, 2024 · ae0d168 · ae0d168
1 parent 663e4bf
commit ae0d168
Show file tree

Hide file tree

Showing 10 changed files with 120 additions and 101 deletions.
diff --git a/cyberdrop_dl/__init__.py b/cyberdrop_dl/__init__.py
@@ -1 +1 @@
-__version__ = "5.1.32"
+__version__ = "5.1.33"
diff --git a/cyberdrop_dl/scraper/crawlers/celebforum_crawler.py b/cyberdrop_dl/scraper/crawlers/celebforum_crawler.py
@@ -76,7 +76,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
         thread_url = scrape_item.url
         post_number = 0
         if len(scrape_item.url.parts) > 3:
-            if "post-" in str(scrape_item.url.parts[3]):
+            if "post-" in str(scrape_item.url.parts[3]) or "post-" in scrape_item.url.fragment:
                 url_parts = str(scrape_item.url).rsplit("post-", 1)
                 thread_url = URL(url_parts[0].rstrip("#"))
                 post_number = int(url_parts[-1].strip("/")) if len(url_parts) == 2 else 0

diff --git a/cyberdrop_dl/scraper/crawlers/f95zone_crawler.py b/cyberdrop_dl/scraper/crawlers/f95zone_crawler.py
@@ -76,7 +76,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
         thread_url = scrape_item.url
         post_number = 0
         if len(scrape_item.url.parts) > 3:
-            if "post-" in str(scrape_item.url.parts[3]):
+            if "post-" in str(scrape_item.url.parts[3]) or "post-" in scrape_item.url.fragment:
                 url_parts = str(scrape_item.url).rsplit("post-", 1)
                 thread_url = URL(url_parts[0].rstrip("#"))
                 post_number = int(url_parts[-1].strip("/")) if len(url_parts) == 2 else 0

diff --git a/cyberdrop_dl/scraper/crawlers/leakedmodels_crawler.py b/cyberdrop_dl/scraper/crawlers/leakedmodels_crawler.py
@@ -80,7 +80,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
         thread_url = scrape_item.url
         post_number = 0
         if len(scrape_item.url.parts) > 4:
-            if "post-" in str(scrape_item.url.parts[4]):
+            if "post-" in str(scrape_item.url.parts[4]) or "post-" in scrape_item.url.fragment:
                 url_parts = str(scrape_item.url).rsplit("post-", 1)
                 thread_url = URL(url_parts[0].rstrip("#"))
                 post_number = int(url_parts[-1].strip("/")) if len(url_parts) == 2 else 0

diff --git a/cyberdrop_dl/scraper/crawlers/nudostar_crawler.py b/cyberdrop_dl/scraper/crawlers/nudostar_crawler.py
@@ -77,7 +77,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
         thread_url = scrape_item.url
         post_number = 0
         if len(scrape_item.url.parts) > 4:
-            if "post-" in str(scrape_item.url.parts[4]):
+            if "post-" in str(scrape_item.url.parts[4]) or "post-" in scrape_item.url.fragment:
                 url_parts = str(scrape_item.url).rsplit("post-", 1)
                 thread_url = URL(url_parts[0].rstrip("#"))
                 post_number = int(url_parts[-1].strip("/")) if len(url_parts) == 2 else 0

diff --git a/cyberdrop_dl/scraper/crawlers/simpcity_crawler.py b/cyberdrop_dl/scraper/crawlers/simpcity_crawler.py
@@ -77,7 +77,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
         thread_url = scrape_item.url
         post_number = 0
         if len(scrape_item.url.parts) > 3:
-            if "post-" in str(scrape_item.url.parts[3]):
+            if "post-" in str(scrape_item.url.parts[3]) or "post-" in scrape_item.url.fragment:
                 url_parts = str(scrape_item.url).rsplit("post-", 1)
                 thread_url = URL(url_parts[0].rstrip("#"))
                 post_number = int(url_parts[-1].strip("/")) if len(url_parts) == 2 else 0

diff --git a/cyberdrop_dl/scraper/crawlers/socialmediagirls_crawler.py b/cyberdrop_dl/scraper/crawlers/socialmediagirls_crawler.py
@@ -77,7 +77,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
         thread_url = scrape_item.url
         post_number = 0
         if len(scrape_item.url.parts) > 3:
-            if "post-" in str(scrape_item.url.parts[3]):
+            if "post-" in str(scrape_item.url.parts[3]) or "post-" in scrape_item.url.fragment:
                 url_parts = str(scrape_item.url).rsplit("post-", 1)
                 thread_url = URL(url_parts[0].rstrip("#"))
                 post_number = int(url_parts[-1].strip("/")) if len(url_parts) == 2 else 0

diff --git a/cyberdrop_dl/scraper/crawlers/xbunker_crawler.py b/cyberdrop_dl/scraper/crawlers/xbunker_crawler.py
@@ -81,7 +81,7 @@ async def forum(self, scrape_item: ScrapeItem) -> None:
         thread_url = scrape_item.url
         post_number = 0
         if len(scrape_item.url.parts) > 3:
-            if "post-" in str(scrape_item.url.parts[3]):
+            if "post-" in str(scrape_item.url.parts[3]) or "post-" in scrape_item.url.fragment:
                 url_parts = str(scrape_item.url).rsplit("post-", 1)
                 thread_url = URL(url_parts[0].rstrip("#"))
                 post_number = int(url_parts[-1].strip("/")) if len(url_parts) == 2 else 0

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cyberdrop-dl"
-version = "5.1.32"
+version = "5.1.33"
 description = "Bulk downloader for multiple file hosts"
 authors = ["Jules Winnfield <[email protected]>"]
 readme = "README.md"