diff --git a/cyberdrop_dl/scraper/crawler.py b/cyberdrop_dl/scraper/crawler.py index e92f3337f..f7ff5746a 100644 --- a/cyberdrop_dl/scraper/crawler.py +++ b/cyberdrop_dl/scraper/crawler.py @@ -44,7 +44,7 @@ def __init__(self, manager: Manager, domain: str, folder_domain: str | None = No self.downloader = field(init=False) self.scraping_progress = manager.progress_manager.scraping_progress self.client: ScraperClient = field(init=False) - self._lock = asyncio.Lock() + self._semaphore = asyncio.Semaphore(20) self.domain = domain self.folder_domain = folder_domain or domain.capitalize() @@ -65,7 +65,7 @@ async def run(self, item: ScrapeItem) -> None: if not item.url.host: return self.waiting_items += 1 - async with self._lock: + async with self._semaphore: self.waiting_items -= 1 if item.url.path_qs not in self.scraped_items: log(f"Scraping: {item.url}", 20)