From e79134b2321c9ffd73fa9b1dd5c4c6f3ceb5fa03 Mon Sep 17 00:00:00 2001 From: jabbey1 Date: Wed, 13 Dec 2023 15:11:59 -0800 Subject: [PATCH] - Improved login method - Decreased hardcoding of CSS selectors - Improved artist folder creation - Possible better exiting --- UGDownloader/CTKGUI.py | 11 ++----- UGDownloader/DLoader.py | 23 ++++++-------- UGDownloader/DriverSetup.py | 26 ++++++++++------ UGDownloader/UGDownloader.pyw | 6 ++++ UGDownloader/Utils.py | 57 ++++++++++++++++++++++------------- 5 files changed, 71 insertions(+), 52 deletions(-) diff --git a/UGDownloader/CTKGUI.py b/UGDownloader/CTKGUI.py index 01339d3..eae5ebb 100644 --- a/UGDownloader/CTKGUI.py +++ b/UGDownloader/CTKGUI.py @@ -34,7 +34,6 @@ class App(customtkinter.CTk): def __init__(self, ): super().__init__() - # todo self.driver = None Utils.folder_check() self.resizable(False, False) @@ -294,6 +293,7 @@ def download_button_event(self): self.DOWNLOADING = True driver = DriverSetup.start_browser(artist, headless, browser, cookies) + try: thread = threading.Thread(target=lambda: start_download(driver, artist, user, password, self, filetype)) @@ -336,7 +336,6 @@ def get_todl_data(self): self.todl_table.insert('', 'end', values=(f'{item}',)) def exit_program(self): - try: print('Closing browser...') driver.quit() @@ -344,12 +343,8 @@ def exit_program(self): except: pass self.destroy() - # subprocess.call("taskkill /F /IM chromedriver.exe", shell=True) - # subprocess.call("taskkill /F /IM geckodriver.exe", shell=True) - try: + if os.path.exists('geckodriver.log'): os.remove('geckodriver.log') - except Exception: - pass def validate(artist: str, user: str, password: str) -> bool: @@ -457,7 +452,7 @@ def write(self, string): self.console_output.configure(state='disabled') def flush(self): - pass + self.console_output.update_idletasks() if __name__ == "__main__": diff --git a/UGDownloader/DLoader.py b/UGDownloader/DLoader.py index 3db10f2..c785400 100644 --- a/UGDownloader/DLoader.py +++ b/UGDownloader/DLoader.py @@ -13,7 +13,8 @@ DOWNLOAD_BUTTON_SELECTOR = "button[class='rPQkl yDkT4 IxFbd exTWY lTEpj qOnLe']" TAB_BLOCKED_SELECTOR = '.XqAW0.ViYGM.g2AHx' -def download_tab(driver: webdriver, url: str) -> List[int, int]: + +def download_tab(driver: webdriver, url: str) -> List[int]: """Download the file. Navigates to page, scrolls to the bottom where the download button is, and then clicks. If the click fails, or the button isn't there, the fallback method is called. Returns values to keep track of total number of downloads and failures""" @@ -83,13 +84,13 @@ def link_handler(driver: webdriver, tab_links: list, file_type_wanted: str) -> l if file_type_wanted in ('Guitar Pro', 'Both'): try: driver.find_element(By.LINK_TEXT, 'Guitar Pro').click() - tab_links += collect_links_guitar_pro(driver) + tab_links.extend(collect_links_guitar_pro(driver)) except (TypeError, selenium.common.exceptions.NoSuchElementException): print('There are no available Guitar Pro tabs for this artist.') if file_type_wanted in ('Powertab', 'Both'): try: driver.find_element(By.LINK_TEXT, 'Power').click() - tab_links += collect_links_powertab(driver) + tab_links.extend(collect_links_powertab(driver)) except (TypeError, selenium.common.exceptions.NoSuchElementException): print('There are no available Powertabs for this artist.') elif file_type_wanted == 'Text': @@ -136,19 +137,15 @@ def collect_links_powertab(driver: webdriver) -> list: return tab_links -def create_artist_folder(artist: str) -> str: +def create_artist_folder(artist: str) -> Path: """Build a path to the artist's folder, inside of Tabs where the files will be downloaded. First, builds path, and then determines if there's a folder there already. If not, creates folder. Returns the path to the folder.""" - dl_path = str(Path.cwd()) + '\\Tabs\\' + artist - if path.isdir(dl_path): - print("Using folder at " + dl_path) + dl_path = Path.cwd() / 'Tabs' / artist + if dl_path.is_dir(): + print("Using folder at " + str(dl_path)) return dl_path - try: - mkdir(dl_path) - except OSError as error: - print(error) - else: - print("Folder created at " + dl_path) + dl_path.mkdir(parents=True, exist_ok=True) + print("Folder created at " + str(dl_path)) return dl_path # return path so GUI can set download directory in browser diff --git a/UGDownloader/DriverSetup.py b/UGDownloader/DriverSetup.py index 8ba1dc0..97517a2 100644 --- a/UGDownloader/DriverSetup.py +++ b/UGDownloader/DriverSetup.py @@ -13,7 +13,7 @@ def start_browser(artist: str, headless: bool, which_browser: str, no_cookies: b and options tailored to each browser. Sets the path of and installs the relevant driver.""" dl_path = DLoader.create_artist_folder(artist) if which_browser == 'Firefox': - firefox_options = set_firefox_options(dl_path, headless, no_cookies) + firefox_options = set_firefox_options(str(dl_path), headless, no_cookies) print(f'Starting Firefox, downloading latest Gecko driver.\n') firefox_service = Service(path='_UGDownloaderFiles') firefox_service.creation_flags = CREATE_NO_WINDOW @@ -22,14 +22,14 @@ def start_browser(artist: str, headless: bool, which_browser: str, no_cookies: b # driver = webdriver.Firefox(options=options, executable_path='geckodriver.exe') # get local copy of driver else: - chrome_options = set_chrome_options(dl_path, headless, no_cookies) + chrome_options = set_chrome_options(str(dl_path), headless, no_cookies) print(f'Starting Chrome, downloading latest chromedriver.\n') chrome_service = Service(path='_UGDownloaderFiles') chrome_service.creation_flags = CREATE_NO_WINDOW driver = webdriver.Chrome(options=chrome_options, service=chrome_service) # next three lines allow chrome to download files while in headless mode driver.command_executor._commands["send_command"] = ("POST", '/session/$sessionId/chromium/send_command') - params = {'cmd': 'Page.setDownloadBehavior', 'params': {'behavior': 'allow', 'downloadPath': dl_path}} + params = {'cmd': 'Page.setDownloadBehavior', 'params': {'behavior': 'allow', 'downloadPath': str(dl_path)}} driver.execute("send_command", params) driver.which_browser = which_browser return driver @@ -39,13 +39,19 @@ def set_firefox_options(dl_path: str, headless: bool, no_cookies: bool) -> FFOpt """Configure the firefox driver. Sets the download directory, and browser options including headless mode. No cookies pop-up workaround for firefox at this point""" firefox_options = FFOptions() - firefox_options.set_preference("browser.download.folderList", 2) - firefox_options.set_preference("browser.download.manager.showWhenStarting", False) - firefox_options.set_preference("browser.download.dir", dl_path) - firefox_options.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/x-gzip") - firefox_options.set_preference('permissions.default.stylesheet', 2) - firefox_options.set_preference('permissions.default.image', 2) - firefox_options.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false') + + preferences = { + "browser.download.folderList": 2, + "browser.download.manager.showWhenStarting": False, + "browser.download.dir": dl_path, + "browser.helperApps.neverAsk.saveToDisk": "application/x-gzip", + "permissions.default.stylesheet": 2, + "permissions.default.image": 2, + "dom.ipc.plugins.enabled.libflashplayer.so": 'false' + } + + for key, value in preferences.items(): + firefox_options.set_preference(key, value) if no_cookies: print('Currently, no cookies pop-up removing add-on is included for Firefox, please try Chrome instead if you ' diff --git a/UGDownloader/UGDownloader.pyw b/UGDownloader/UGDownloader.pyw index 5bf4349..38fe99e 100644 --- a/UGDownloader/UGDownloader.pyw +++ b/UGDownloader/UGDownloader.pyw @@ -1,4 +1,10 @@ +import logging import CTKGUI +logging.basicConfig(filename='myapp.log', level=logging.DEBUG, format='%(levelname)s:%(message)s') +logging.info('Started') + app = CTKGUI.App() app.mainloop() + +logging.info('Finished') \ No newline at end of file diff --git a/UGDownloader/Utils.py b/UGDownloader/Utils.py index 0311087..78363f0 100644 --- a/UGDownloader/Utils.py +++ b/UGDownloader/Utils.py @@ -1,3 +1,5 @@ +from selenium.common.exceptions import NoSuchElementException +import logging import sys from datetime import datetime from os import path, mkdir @@ -66,27 +68,40 @@ def folder_check(): def login(driver: webdriver, user: str, password: str): """logs in, but will be defeated if a captcha is present. Must be used when the driver is on a page where a login button exists. If you aren't already logged in, this will be most pages""" - driver.find_element(By.CSS_SELECTOR, '.exTWY').click() # login button - sleep(1) - form = driver.find_element(By.CSS_SELECTOR, "form > div.PictU") - # todo test below - # form = WebDriverWait(driver, 20).until(ec.presence_of_element_located((By.CSS_SELECTOR, "form > div.PictU"))) - username_textbox = form.find_element(By.CSS_SELECTOR, 'input[name=username]') - password_textbox = form.find_element(By.CSS_SELECTOR, 'input[name=password]') - submit_button = form.find_element(By.CSS_SELECTOR, 'button[type=submit]') - - username_textbox.send_keys(user) - password_textbox.send_keys(password) - sleep(1) - submit_button.click() - # call method from captcha class, if figure out how to bypass captcha - # this popup sometimes takes some time to appear, wait until it's clickable - element = WebDriverWait(driver, 20).until( - ec.element_to_be_clickable((By.CSS_SELECTOR, - 'button.RwBUh:nth-child(1) > svg:nth-child(1) > path:nth-child(1)'))) - element.click() - sleep(.5) - print('Logged in') + + # CSS selectors + login_button_selector = '.exTWY' + form_selector = "form > div.PictU" + username_selector = 'input[name=username]' + password_selector = 'input[name=password]' + submit_selector = 'button[type=submit]' + popup_selector = 'button.RwBUh:nth-child(1) > svg:nth-child(1) > path:nth-child(1)' + + try: + # Click on login button + driver.find_element(By.CSS_SELECTOR, login_button_selector).click() + + # Wait for form to be present + form = WebDriverWait(driver, 20).until(ec.presence_of_element_located((By.CSS_SELECTOR, form_selector))) + # Find login elements + username_textbox = form.find_element(By.CSS_SELECTOR, username_selector) + password_textbox = form.find_element(By.CSS_SELECTOR, password_selector) + submit_button = form.find_element(By.CSS_SELECTOR, submit_selector) + + # Enter username and password + username_textbox.send_keys(user) + password_textbox.send_keys(password) + sleep(1) + submit_button.click() + + # Wait for popup to be clickable + popup_element = WebDriverWait(driver, 20).until(ec.element_to_be_clickable((By.CSS_SELECTOR, popup_selector))) + popup_element.click() + sleep(.5) + print('Logged in') + + except NoSuchElementException: + print('Error: Could not find one of the login elements.') def failure_log_new_attempt():