From 442c405e5839b8cd3856f6f3259cd196b37bb4df Mon Sep 17 00:00:00 2001 From: Jules-WinnfieldX Date: Wed, 6 Mar 2024 19:53:51 -0700 Subject: [PATCH] Remove ddos-guard cookies --- .../scraper/crawlers/bunkrr_crawler.py | 21 --- .../scraper/crawlers/coomer_crawler.py | 18 --- .../scraper/crawlers/kemono_crawler.py | 19 --- .../settings_authentication_prompts.py | 120 +++--------------- .../utils/args/browser_cookie_extraction.py | 17 --- cyberdrop_dl/utils/args/config_definitions.py | 7 - 6 files changed, 16 insertions(+), 186 deletions(-) diff --git a/cyberdrop_dl/scraper/crawlers/bunkrr_crawler.py b/cyberdrop_dl/scraper/crawlers/bunkrr_crawler.py index 2ae5595fc..36c4655d2 100644 --- a/cyberdrop_dl/scraper/crawlers/bunkrr_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/bunkrr_crawler.py @@ -21,11 +21,8 @@ class BunkrrCrawler(Crawler): def __init__(self, manager: Manager): super().__init__(manager, "bunkrr", "Bunkrr") self.primary_base_domain = URL("https://bunkr.sk") - self.ddos_guard_domain = URL("https://bunkr.sk") self.request_limiter = AsyncLimiter(10, 1) - self.cookies_set = False - """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" async def fetch(self, scrape_item: ScrapeItem) -> None: @@ -33,8 +30,6 @@ async def fetch(self, scrape_item: ScrapeItem) -> None: task_id = await self.scraping_progress.add_task(scrape_item.url) scrape_item.url = await self.get_stream_link(scrape_item.url) - await self.set_cookies() - if scrape_item.url.host.startswith("get"): scrape_item.url = await self.reinforced_link(scrape_item.url) scrape_item.url = await self.get_stream_link(scrape_item.url) @@ -194,19 +189,3 @@ async def parse_datetime(self, date: str) -> int: """Parses a datetime string into a unix timestamp""" date = datetime.datetime.strptime(date, "%H:%M:%S %d/%m/%Y") return calendar.timegm(date.timetuple()) - - """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" - - async def set_cookies(self): - """Sets the cookies for the client""" - if self.cookies_set: - return - - if self.manager.config_manager.authentication_data['DDOS-Guard']['bunkrr_ddg1']: - self.client.client_manager.cookies.update_cookies({"__ddg1_": self.manager.config_manager.authentication_data['DDOS-Guard']['bunkrr_ddg1']}, response_url=self.ddos_guard_domain) - if self.manager.config_manager.authentication_data['DDOS-Guard']['bunkrr_ddg2']: - self.client.client_manager.cookies.update_cookies({"__ddg2_": self.manager.config_manager.authentication_data['DDOS-Guard']['bunkrr_ddg2']}, response_url=self.ddos_guard_domain) - if self.manager.config_manager.authentication_data['DDOS-Guard']['bunkrr_ddgid']: - self.client.client_manager.cookies.update_cookies({"__ddgid_": self.manager.config_manager.authentication_data['DDOS-Guard']['bunkrr_ddgid']}, response_url=self.ddos_guard_domain) - - self.cookies_set = True diff --git a/cyberdrop_dl/scraper/crawlers/coomer_crawler.py b/cyberdrop_dl/scraper/crawlers/coomer_crawler.py index 1fe16f61f..cdc37ad1e 100644 --- a/cyberdrop_dl/scraper/crawlers/coomer_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/coomer_crawler.py @@ -23,16 +23,12 @@ def __init__(self, manager: Manager): self.api_url = URL("https://coomer.su/api/v1") self.request_limiter = AsyncLimiter(10, 1) - self.cookies_set = False - """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" async def fetch(self, scrape_item: ScrapeItem) -> None: """Determines where to send the scrape item based on the url""" task_id = await self.scraping_progress.add_task(scrape_item.url) - await self.set_cookies() - if "thumbnails" in scrape_item.url.parts: parts = [x for x in scrape_item.url.parts if x not in ("thumbnail", "/")] link = URL(f"https://{scrape_item.url.host}/{'/'.join(parts)}") @@ -152,17 +148,3 @@ async def create_new_scrape_item(self, link: URL, old_scrape_item: ScrapeItem, u new_scrape_item = await self.create_scrape_item(old_scrape_item, link, new_title, True, None, await self.parse_datetime(date)) await new_scrape_item.add_to_parent_title(post_title) self.manager.task_group.create_task(self.run(new_scrape_item)) - - """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" - - async def set_cookies(self): - """Sets the cookies for the client""" - if self.cookies_set: - return - - if self.manager.config_manager.authentication_data['DDOS-Guard']['coomer_ddg1']: - self.client.client_manager.cookies.update_cookies( - {"__ddg1_": self.manager.config_manager.authentication_data['DDOS-Guard']['coomer_ddg1']}, - response_url=self.ddos_guard_domain) - - self.cookies_set = True diff --git a/cyberdrop_dl/scraper/crawlers/kemono_crawler.py b/cyberdrop_dl/scraper/crawlers/kemono_crawler.py index bcc3aec1c..7eea4ed46 100644 --- a/cyberdrop_dl/scraper/crawlers/kemono_crawler.py +++ b/cyberdrop_dl/scraper/crawlers/kemono_crawler.py @@ -19,21 +19,16 @@ class KemonoCrawler(Crawler): def __init__(self, manager: Manager): super().__init__(manager, "kemono", "Kemono") self.primary_base_domain = URL("https://kemono.su") - self.ddos_guard_domain = URL("https://*.kemono.su") self.api_url = URL("https://kemono.su/api/v1") self.services = ['patreon', 'fanbox', 'fantia', 'afdian', 'boosty', 'dlsite', 'gumroad', 'subscribestar'] self.request_limiter = AsyncLimiter(10, 1) - self.cookies_set = False - """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" async def fetch(self, scrape_item: ScrapeItem) -> None: """Determines where to send the scrape item based on the url""" task_id = await self.scraping_progress.add_task(scrape_item.url) - await self.set_cookies() - if "thumbnails" in scrape_item.url.parts: parts = [x for x in scrape_item.url.parts if x not in ("thumbnail", "/")] link = URL(f"https://{scrape_item.url.host}/{'/'.join(parts)}") @@ -166,17 +161,3 @@ async def create_new_scrape_item(self, link: URL, old_scrape_item: ScrapeItem, u new_scrape_item = await self.create_scrape_item(old_scrape_item, link, new_title, True, None, await self.parse_datetime(date)) await new_scrape_item.add_to_parent_title(post_title) self.manager.task_group.create_task(self.run(new_scrape_item)) - - """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" - - async def set_cookies(self): - """Sets the cookies for the client""" - if self.cookies_set: - return - - if self.manager.config_manager.authentication_data['DDOS-Guard']['kemono_ddg1']: - self.client.client_manager.cookies.update_cookies( - {"__ddg1_": self.manager.config_manager.authentication_data['DDOS-Guard']['kemono_ddg1']}, - response_url=self.ddos_guard_domain) - - self.cookies_set = True diff --git a/cyberdrop_dl/ui/prompts/settings_authentication_prompts.py b/cyberdrop_dl/ui/prompts/settings_authentication_prompts.py index 4ecab233c..de5402316 100644 --- a/cyberdrop_dl/ui/prompts/settings_authentication_prompts.py +++ b/cyberdrop_dl/ui/prompts/settings_authentication_prompts.py @@ -6,7 +6,7 @@ from InquirerPy.base.control import Choice from rich.console import Console -from cyberdrop_dl.utils.args.browser_cookie_extraction import get_forum_cookies, get_ddos_guard_cookies +from cyberdrop_dl.utils.args.browser_cookie_extraction import get_forum_cookies from cyberdrop_dl.managers.manager import Manager if TYPE_CHECKING: @@ -27,35 +27,30 @@ def edit_authentication_values_prompt(manager: Manager) -> None: action = inquirer.select( message="What would you like to do?", choices=[ - Choice(1, "Edit DDOS-Guard Cookies"), - Choice(2, "Edit Forum Authentication Values"), - Choice(3, "Edit JDownloader Authentication Values"), - Choice(4, "Edit Reddit Authentication Values"), - Choice(5, "Edit GoFile API Key"), - Choice(6, "Edit Imgur Client ID"), - Choice(7, "Edit PixelDrain API Key"), - Choice(8, "Done"), + Choice(1, "Edit Forum Authentication Values"), + Choice(2, "Edit JDownloader Authentication Values"), + Choice(3, "Edit Reddit Authentication Values"), + Choice(4, "Edit GoFile API Key"), + Choice(5, "Edit Imgur Client ID"), + Choice(6, "Edit PixelDrain API Key"), + Choice(7, "Done"), ], long_instruction="ARROW KEYS: Navigate | ENTER: Select" ).execute() - # Edit DDOS-Guard Cookies - if action == 1: - edit_ddos_guard_cookies_prompt(manager) - # Edit Forums - if action == 2: + if action == 1: edit_forum_authentication_values_prompt(manager) # Edit JDownloader - elif action == 3: + elif action == 2: edit_jdownloader_authentication_values_prompt(auth) # Edit Reddit - elif action == 4: + elif action == 3: edit_reddit_authentication_values_prompt(auth) # Edit GoFile API Key - elif action == 5: + elif action == 4: console.clear() gofile_api_key = inquirer.text( message="Enter the GoFile API Key:", @@ -65,7 +60,7 @@ def edit_authentication_values_prompt(manager: Manager) -> None: auth["GoFile"]["gofile_api_key"] = gofile_api_key # Edit Imgur Client ID - elif action == 6: + elif action == 5: console.clear() imgur_client_id = inquirer.text( message="Enter the Imgur Client ID:", @@ -76,7 +71,7 @@ def edit_authentication_values_prompt(manager: Manager) -> None: auth["Imgur"]["imgur_client_id"] = imgur_client_id # Edit PixelDrain API Key - elif action == 7: + elif action == 6: console.clear() pixeldrain_api_key = inquirer.text( message="Enter the PixelDrain API Key:", @@ -86,94 +81,11 @@ def edit_authentication_values_prompt(manager: Manager) -> None: auth["PixelDrain"]["pixeldrain_api_key"] = pixeldrain_api_key # Done - elif action == 8: + elif action == 7: manager.config_manager.write_updated_authentication_config() return -def edit_ddos_guard_cookies_prompt(manager: Manager) -> None: - """Edit the forum authentication values""" - while True: - console.clear() - console.print("Editing DDOS-Guard Cookie Values") - action = inquirer.select( - message="What would you like to do?", - choices=[ - Choice(1, "Browser Cookie Extraction"), - Choice(2, "Enter Cookie Values Manually"), - Choice(3, "Done"), - ], long_instruction="ARROW KEYS: Navigate | ENTER: Select" - ).execute() - - # Browser Cookie Extraction - if action == 1: - action = inquirer.select( - message="Which browser should we load cookies from?", - choices=[ - Choice("chrome", "Chrome"), - Choice("firefox", "FireFox"), - Choice("edge", "Edge"), - Choice("safari", "Safari"), - Choice("opera", "Opera"), - Choice("brave", "Brave"), - Choice(1, "Done"), - ], long_instruction="ARROW KEYS: Navigate | ENTER: Select" - ).execute() - - # Done - if action == 1: - continue - - # Browser Selection - if action == "chrome": - get_ddos_guard_cookies(manager, "chrome") - elif action == "firefox": - get_ddos_guard_cookies(manager, "firefox") - elif action == "edge": - get_ddos_guard_cookies(manager, "edge") - elif action == "safari": - get_ddos_guard_cookies(manager, "safari") - elif action == "opera": - get_ddos_guard_cookies(manager, "opera") - elif action == "brave": - get_ddos_guard_cookies(manager, "brave") - return - - # Enter Cookie Values Manually - elif action == 2: - bunkr_ddg1 = inquirer.text( - message="Enter your DDG1 value for Bunkr:", - default=manager.config_manager.authentication_data["DDOS-Guard"]["bunkrr_ddg1"], - ).execute() - bunkr_ddg2 = inquirer.text( - message="Enter your DDG2 value for Bunkr:", - default=manager.config_manager.authentication_data["DDOS-Guard"]["bunkrr_ddg2"], - ).execute() - bunkr_ddgid = inquirer.text( - message="Enter your DDGID value for Bunkr:", - default=manager.config_manager.authentication_data["DDOS-Guard"]["bunkrr_ddgid"], - ).execute() - - coomer_ddg1 = inquirer.text( - message="Enter your DDG1 value for Coomer:", - default=manager.config_manager.authentication_data["DDOS-Guard"]["coomer_ddg1"], - ).execute() - kemono_ddg1 = inquirer.text( - message="Enter your DDG1 value for Kemono:", - default=manager.config_manager.authentication_data["DDOS-Guard"]["kemono_ddg1"], - ).execute() - - manager.config_manager.authentication_data["DDOS-Guard"]["bunkr_ddg1"] = bunkr_ddg1 - manager.config_manager.authentication_data["DDOS-Guard"]["bunkr_ddg2"] = bunkr_ddg2 - manager.config_manager.authentication_data["DDOS-Guard"]["bunkr_ddgid"] = bunkr_ddgid - manager.config_manager.authentication_data["DDOS-Guard"]["coomer_ddg1"] = coomer_ddg1 - manager.config_manager.authentication_data["DDOS-Guard"]["kemono_ddg1"] = kemono_ddg1 - - return - elif action == 3: - return - - def edit_forum_authentication_values_prompt(manager: Manager) -> None: """Edit the forum authentication values""" while True: @@ -222,7 +134,7 @@ def edit_forum_authentication_values_prompt(manager: Manager) -> None: get_forum_cookies(manager, "brave") return - # Enter Cookie Values Manually + # Enter Cred Values Manually elif action == 2: celebforum_username = inquirer.text( message="Enter your CelebForum Username:", diff --git a/cyberdrop_dl/utils/args/browser_cookie_extraction.py b/cyberdrop_dl/utils/args/browser_cookie_extraction.py index 9b1714313..cf64c616c 100644 --- a/cyberdrop_dl/utils/args/browser_cookie_extraction.py +++ b/cyberdrop_dl/utils/args/browser_cookie_extraction.py @@ -51,23 +51,6 @@ def get_forum_cookies(manager: Manager, browser: str) -> None: manager.cache_manager.save("browser", browser) -# noinspection PyProtectedMember -@cookie_wrapper -def get_ddos_guard_cookies(manager: Manager, browser: str) -> None: - """Get the cookies for DDOS-Guard""" - auth_args: Dict = manager.config_manager.authentication_data - for ddos_guard in SupportedDomains.supported_ddos_guard: - try: - cookie = get_cookie(browser, ddos_guard) - auth_args['DDOS-Guard'][f'{SupportedDomains.supported_ddos_guard_map[ddos_guard]}_ddg1'] = cookie._cookies[ddos_guard]['/']['__ddg1_'].value - auth_args['DDOS-Guard'][f'{SupportedDomains.supported_ddos_guard_map[ddos_guard]}_ddg2'] = cookie._cookies[ddos_guard]['/']['__ddg2_'].value - auth_args['DDOS-Guard'][f'{SupportedDomains.supported_ddos_guard_map[ddos_guard]}_ddgid'] = cookie._cookies[ddos_guard]['/']['__ddgid_'].value - except KeyError: - pass - - manager.cache_manager.save("browser", browser) - - def get_cookie(browser: str, domain: str): """Get the cookies for a specific domain""" if browser == 'chrome': diff --git a/cyberdrop_dl/utils/args/config_definitions.py b/cyberdrop_dl/utils/args/config_definitions.py index a9e00ecc3..379e79b79 100644 --- a/cyberdrop_dl/utils/args/config_definitions.py +++ b/cyberdrop_dl/utils/args/config_definitions.py @@ -6,13 +6,6 @@ authentication_settings: Dict = { - "DDOS-Guard": { - "bunkrr_ddg1": "", - "bunkrr_ddg2": "", - "bunkrr_ddgid": "", - "coomer_ddg1": "", - "kemono_ddg1": "", - }, "Forums": { "celebforum_xf_user_cookie": "", "celebforum_username": "",