From 07f2ad63294b1fe8e50057235ab687fc723f6bd0 Mon Sep 17 00:00:00 2001 From: Jules-WinnfieldX Date: Fri, 8 Dec 2023 22:33:48 -0700 Subject: [PATCH] Change to path, move lock release --- cyberdrop_dl/__init__.py | 2 +- cyberdrop_dl/scraper/crawler.py | 6 +++--- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cyberdrop_dl/__init__.py b/cyberdrop_dl/__init__.py index 0c179ae86..53c2b933b 100644 --- a/cyberdrop_dl/__init__.py +++ b/cyberdrop_dl/__init__.py @@ -1 +1 @@ -__version__ = "5.0.96" +__version__ = "5.0.97" diff --git a/cyberdrop_dl/scraper/crawler.py b/cyberdrop_dl/scraper/crawler.py index 182e6e7fe..609e02c8d 100644 --- a/cyberdrop_dl/scraper/crawler.py +++ b/cyberdrop_dl/scraper/crawler.py @@ -57,10 +57,10 @@ async def run_loop(self) -> None: self.complete = False await self._lock.acquire() - if item.url not in self.scraped_items: - self._lock.release() + if item.url.path not in self.scraped_items: await log(f"Scrape Starting: {item.url}") - self.scraped_items.append(item.url) + self.scraped_items.append(item.url.path) + self._lock.release() await self.fetch(item) await log(f"Scrape Finished: {item.url}") else: diff --git a/pyproject.toml b/pyproject.toml index 245e16576..b4beeb097 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cyberdrop-dl" -version = "5.0.96" +version = "5.0.97" description = "Bulk downloader for multiple file hosts" authors = ["Jules Winnfield "] readme = "README.md"