From 01677d40811603007e1f94a87ff1bbb1e9e1a345 Mon Sep 17 00:00:00 2001 From: NTFSvolume <172021377+NTFSvolume@users.noreply.github.com> Date: Thu, 7 Nov 2024 05:41:02 -0500 Subject: [PATCH] fix: prepare files before looping through them --- cyberdrop_dl/scraper/crawlers/coomer_crawler.py | 7 +++++-- cyberdrop_dl/scraper/crawlers/kemono_crawler.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cyberdrop_dl/scraper/crawlers/coomer_crawler.py b/cyberdrop_dl/scraper/crawlers/coomer_crawler.py index 0ad9773b7..b06d4812f 100644 --- a/cyberdrop_dl/scraper/crawlers/coomer_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/coomer_crawler.py @@ -139,10 +139,13 @@ async def handle_file(file_obj): add_parent=scrape_item.url.joinpath("post", post_id)) files = [] - if post['file']: + if post.get('file'): files.append(post['file']) - for file in files.extend(post['attachments']): + if post.get('attachments'): + files.extend(post['attachments']) + + for file in files: await handle_file(file) scrape_item.children += 1 if scrape_item.children_limit: diff --git a/cyberdrop_dl/scraper/crawlers/kemono_crawler.py b/cyberdrop_dl/scraper/crawlers/kemono_crawler.py index 55dfe3a52..500c9b943 100644 --- a/cyberdrop_dl/scraper/crawlers/kemono_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/kemono_crawler.py @@ -146,10 +146,13 @@ async def handle_file(file_obj): await self.create_new_scrape_item(link, scrape_item, user_str, post_title, post_id, date) files = [] - if post['file']: + if post.get('file'): files.append(post['file']) - for file in files.extend(post['attachments']): + if post.get('attachments'): + files.extend(post['attachments']) + + for file in files: if scrape_item.children_limit: if scrape_item.children >= scrape_item.children_limit: raise ScrapeItemMaxChildrenReached(origin = scrape_item)