From 6b8cc8bf5019861d4320e0da59f45faa3d8e8117 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 11:32:54 -0500 Subject: [PATCH 01/17] not having the sorted dir not exists should be okay add option to set the scan directory independent of the download_dir skip scanning for database if cdl_only is false --- cyberdrop_dl/managers/args_manager.py | 11 +++- cyberdrop_dl/managers/config_manager.py | 7 +++ cyberdrop_dl/managers/path_manager.py | 4 ++ .../ui/prompts/settings_user_prompts.py | 7 +++ cyberdrop_dl/utils/args/args.py | 1 + cyberdrop_dl/utils/args/config_definitions.py | 1 + cyberdrop_dl/utils/sorting.py | 50 ++++++++++--------- 7 files changed, 57 insertions(+), 24 deletions(-) diff --git a/cyberdrop_dl/managers/args_manager.py b/cyberdrop_dl/managers/args_manager.py index 845932f14..9d9a3084d 100644 --- a/cyberdrop_dl/managers/args_manager.py +++ b/cyberdrop_dl/managers/args_manager.py @@ -41,6 +41,9 @@ def __init__(self): self.sort_downloads = field(init=False) self.sort_cdl_only = field(init=True) self.sort_folder = None + self.scan_folder=None + + # Logs self.main_log_filename = None @@ -108,7 +111,8 @@ def startup(self) -> None: self.sort_cdl_only = False if self.parsed_args['sort_folder']: self.sort_folder = Path(self.parsed_args['sort_folder']) - + if self.parsed_args['scan_folder']: + self.scan_folder = Path(self.parsed_args['scan_folder']) if self.parsed_args['main_log_filename']: self.main_log_filename = self.parsed_args['main_log_filename'] if self.parsed_args['last_forum_post_filename']: @@ -155,3 +159,8 @@ def startup(self) -> None: del self.parsed_args['sort_downloads'] del self.parsed_args['sort_cdl_only'] del self.parsed_args['sort_folder'] + del self.parsed_args['scan_folder'] + del self.parsed_args['completed_after'] + del self.parsed_args['completed_before'] + + diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py index a6df82b50..f3c7e3231 100644 --- a/cyberdrop_dl/managers/config_manager.py +++ b/cyberdrop_dl/managers/config_manager.py @@ -47,6 +47,7 @@ def get_keys(dl, keys=None) -> set: return set(keys) + class ConfigManager: def __init__(self, manager: 'Manager'): self.manager = manager @@ -104,6 +105,7 @@ def load_configs(self) -> None: self.settings_data['Logs']['log_folder'] = APP_STORAGE / "Configs" / self.loaded_config / "Logs" self.settings_data['Logs']['webhook_url'] = "" self.settings_data['Sorting']['sort_folder'] = DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads" + self.settings_data['Sorting']['scan_folder'] = None self.write_updated_settings_config() def _verify_authentication_config(self) -> None: @@ -128,6 +130,8 @@ def _verify_settings_config(self) -> None: self.settings_data['Logs']['log_folder'] = Path(self.settings_data['Logs']['log_folder']) self.settings_data['Logs']['webhook_url'] = str(self.settings_data['Logs']['webhook_url']) self.settings_data['Sorting']['sort_folder'] = Path(self.settings_data['Sorting']['sort_folder']) + self.settings_data['Sorting']['scan_folder'] = Path(self.settings_data['Sorting']['scan_folder']) if self.settings_data['Sorting']['scan_folder'] else None + # change to ints self.settings_data['File_Size_Limits']['maximum_image_size'] = int( @@ -222,6 +226,7 @@ def create_new_config(self, new_settings: Path, settings_data: Dict) -> None: settings_data['Logs']['log_folder'] = str(settings_data['Logs']['log_folder']) settings_data['Logs']['webhook_url'] = str(settings_data['Logs']['webhook_url']) settings_data['Sorting']['sort_folder'] = str(settings_data['Sorting']['sort_folder']) + settings_data['Sorting']['scan_folder'] = str(settings_data['Sorting']['scan_folder']) _save_yaml(new_settings, settings_data) def write_updated_authentication_config(self) -> None: @@ -236,6 +241,8 @@ def write_updated_settings_config(self) -> None: settings_data['Logs']['log_folder'] = str(settings_data['Logs']['log_folder']) settings_data['Logs']['webhook_url'] = str(settings_data['Logs']['webhook_url']) settings_data['Sorting']['sort_folder'] = str(settings_data['Sorting']['sort_folder']) + settings_data['Sorting']['scan_folder'] = str(settings_data['Sorting']['scan_folder']) + _save_yaml(self.settings, settings_data) def write_updated_global_settings_config(self) -> None: diff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.py index ea68744f5..3b2bbf692 100644 --- a/cyberdrop_dl/managers/path_manager.py +++ b/cyberdrop_dl/managers/path_manager.py @@ -26,6 +26,8 @@ def __init__(self, manager: 'Manager'): self.download_dir: Path = field(init=False) self.sorted_dir: Path = field(init=False) + self.scan_dir: Path = field(init=False) + self.log_dir: Path = field(init=False) self.cache_dir: Path = field(init=False) @@ -61,6 +63,8 @@ def startup(self) -> None: """Startup process for the Directory Manager""" self.download_dir = self.manager.config_manager.settings_data['Files']['download_folder'] if not self.manager.args_manager.download_dir else self.manager.args_manager.download_dir self.sorted_dir = self.manager.config_manager.settings_data['Sorting']['sort_folder'] if not self.manager.args_manager.sort_folder else self.manager.args_manager.sort_folder + + self.scan_dir = self.manager.config_manager.settings_data['Sorting']['scan_folder'] if not self.manager.args_manager.scan_folder else self.manager.args_manager.scan_folder self.log_dir = self.manager.config_manager.settings_data['Logs']['log_folder'] if not self.manager.args_manager.log_dir else self.manager.args_manager.log_dir self.input_file = self.manager.config_manager.settings_data['Files']['input_file'] if not self.manager.args_manager.input_file else self.manager.args_manager.input_file self.history_db = self.cache_dir / "cyberdrop.db" diff --git a/cyberdrop_dl/ui/prompts/settings_user_prompts.py b/cyberdrop_dl/ui/prompts/settings_user_prompts.py index 75462e1d6..47d1e7514 100644 --- a/cyberdrop_dl/ui/prompts/settings_user_prompts.py +++ b/cyberdrop_dl/ui/prompts/settings_user_prompts.py @@ -397,6 +397,12 @@ def edit_sort_options_prompt(manager: Manager, config: Dict) -> None: sort_folder = inquirer.filepath( message="Enter the folder you want to sort files into:", default=str(config['Sorting']['sort_folder']), + vi_mode=manager.vi_mode, + ).execute() + + scan_folder = inquirer.filepath( + message="Enter the folder you want to scan for files", + default=str(config['Sorting']['scan_folder'] or config['Files']['download_folder']), validate=PathValidator(is_dir=True, message="Input is not a directory"), vi_mode=manager.vi_mode, ).execute() @@ -432,6 +438,7 @@ def edit_sort_options_prompt(manager: Manager, config: Dict) -> None: ).execute() config['Sorting']['sort_folder'] = Path(sort_folder) + config['Sorting']['scan_folder'] = Path(scan_folder) if bool(scan_folder) else None config['Sorting']['sort_incremementer_format'] = sort_incremementer_format config['Sorting']['sorted_audio'] = sorted_audio config['Sorting']['sorted_video'] = sorted_video diff --git a/cyberdrop_dl/utils/args/args.py b/cyberdrop_dl/utils/args/args.py index 23be96398..347a20e87 100644 --- a/cyberdrop_dl/utils/args/args.py +++ b/cyberdrop_dl/utils/args/args.py @@ -82,6 +82,7 @@ def parse_args() -> argparse.Namespace: sorting_options.add_argument("--sort-downloads", action="store_true", help="sort downloads into folders", default=False) sorting_options.add_argument("--sort-cdl-only", action="store_true", help="only sort CDL files.", default=True) sorting_options.add_argument("--sort_folder", type=str, help="path to where you want CDL to store it's log files", default="") + sorting_options.add_argument("--scan_folder", type=str, help="path to scan for files, if not set then the download_dir is used", default="") ui_options = parser.add_argument_group("UI_Options") ui_options.add_argument("--vi-mode", action="store_true", help="enable VIM keybindings for UI", default=None) diff --git a/cyberdrop_dl/utils/args/config_definitions.py b/cyberdrop_dl/utils/args/config_definitions.py index 102c47164..6ba984885 100644 --- a/cyberdrop_dl/utils/args/config_definitions.py +++ b/cyberdrop_dl/utils/args/config_definitions.py @@ -106,6 +106,7 @@ "Sorting": { "sort_downloads": False, "sort_folder": str(DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"), + "scan_folder":None, "sort_cdl_only": True, "sort_incremementer_format": " ({i})", "sorted_audio": "{sort_dir}/{base_dir}/Audio/{filename}{ext}", diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index 7f671e862..44f50eb81 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -27,7 +27,7 @@ def get_file_date_in_us_ca_formats(file: Path) -> tuple[str, str]: class Sorter: def __init__(self, manager: 'Manager'): - self.download_dir = manager.path_manager.download_dir + self.download_dir = manager.path_manager.scan_dir or manager.path_manager.download_dir self.sorted_downloads = manager.path_manager.sorted_dir self.incrementer_format = manager.config_manager.settings_data['Sorting']['sort_incremementer_format'] self.sort_cdl_only = manager.config_manager.settings_data['Sorting']['sort_cdl_only'] @@ -80,6 +80,8 @@ async def check_dir_parents(self) -> bool: async def sort(self) -> None: """Sorts the files in the download directory into their respective folders""" await log_with_color("\nSorting Downloads: Please Wait", "cyan", 20) + #make sort dir + self.sorted_downloads.mkdir(parents=True, exist_ok=True) if await self.check_dir_parents(): return @@ -88,33 +90,13 @@ async def sort(self) -> None: await log_with_color("Download Directory does not exist", "red", 40) return - unique_download_paths = await self.db_manager.history_table.get_unique_download_paths() - download_folders = [Path(path[0]) for path in unique_download_paths] - existing_folders = [] - - for folder in download_folders: - try: - relative_folder = folder.relative_to(self.download_dir) - base_folder = self.download_dir / relative_folder.parts[0] - print(relative_folder.i4rejohodf) - except Exception as e: - if e.__class__ == ValueError: - continue - logger.log(40, f"Error: {e}\n\nfolder: {folder}\ndownload_dir: {self.download_dir}\nrelative_folder: {relative_folder}") - raise e - - if base_folder.exists(): - existing_folders.append(base_folder) - - download_folders.extend(existing_folders) - download_folders = list(set(download_folders)) + download_folders=await self.get_download_folder() for folder in self.download_dir.iterdir(): if not folder.is_dir(): continue - if folder not in download_folders and self.sort_cdl_only: + if self.sort_cdl_only and folder not in download_folders: continue - files = await self.find_files_in_dir(folder) for file in files: ext = file.suffix.lower() @@ -138,6 +120,28 @@ async def sort(self) -> None: await log_with_color(f"Organized: {self.video_count} Video Files", "green", 20) await log_with_color(f"Organized: {self.other_count} Other Files", "green", 20) + async def get_download_folder(self): + """Gets the download folder""" + if not self.sort_cdl_only: + return [] + unique_download_paths = await self.db_manager.history_table.get_unique_download_paths() + download_folders = download_folders = [Path(download_path[0]) for download_path in unique_download_paths if Path(download_path[0]).is_dir() and Path(download_path[0]) != self.download_dir] + existing_folders = [] + for folder in download_folders: + try: + relative_folder = folder.relative_to(self.download_dir) + base_folder = self.download_dir / relative_folder.parts[0] + print(relative_folder) + except Exception as e: + if e.__class__ == ValueError: + continue + logger.log(40, f"Error: {e}\n\nfolder: {folder}\ndownload_dir: {self.download_dir}\nrelative_folder: {relative_folder}") + raise e + if base_folder.exists(): + existing_folders.append(base_folder) + download_folders.extend(existing_folders) + download_folders = list(set(download_folders)) + return download_folders async def sort_audio(self, file: Path, base_name: str) -> None: """Sorts an audio file into the sorted audio folder""" self.audio_count += 1 From c3e8aba661895d03c2f631dd997f2db080a97878 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 12:06:44 -0500 Subject: [PATCH 02/17] purge folder as we go fix posix path issue for scan_folder --- cyberdrop_dl/managers/manager.py | 2 ++ cyberdrop_dl/utils/sorting.py | 2 ++ cyberdrop_dl/utils/utilities.py | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py index e3c8a983e..9250b5ddd 100644 --- a/cyberdrop_dl/managers/manager.py +++ b/cyberdrop_dl/managers/manager.py @@ -187,6 +187,8 @@ async def args_logging(self) -> None: print_settings['Files']['download_folder'] = str(print_settings['Files']['download_folder']) print_settings["Logs"]["log_folder"] = str(print_settings["Logs"]["log_folder"]) print_settings['Sorting']['sort_folder'] = str(print_settings['Sorting']['sort_folder']) + print_settings['Sorting']['scan_folder'] = str(print_settings['Sorting']['scan_folder']) if str(print_settings['Sorting']['scan_folder']) else "" + input_file = str(self.path_manager.input_file) download_dir = str(self.path_manager.download_dir) diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index 44f50eb81..ea64afefe 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -111,10 +111,12 @@ async def sort(self) -> None: await self.sort_video(file, folder.name) else: await self.sort_other(file, folder.name) + await purge_self(folder) await asyncio.sleep(5) await purge_dir(self.download_dir) + await log_with_color(f"Organized: {self.audio_count} Audio Files", "green", 20) await log_with_color(f"Organized: {self.image_count} Image Files", "green", 20) await log_with_color(f"Organized: {self.video_count} Video Files", "green", 20) diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py index dda6b7802..84de60d84 100644 --- a/cyberdrop_dl/utils/utilities.py +++ b/cyberdrop_dl/utils/utilities.py @@ -234,6 +234,16 @@ async def purge_dir(dirname: Path) -> None: list(map(os.rmdir, deleted)) +async def purge_self(dirname: Path) -> None: + """Purges directory if empyty""" + dir_tree = list(os.walk(dirname, topdown=False)) + if len(dir_tree) == 0: + os.rmdir(dirname) + + + + + async def check_partials_and_empty_folders(manager: Manager): """Checks for partial downloads and empty folders""" if manager.config_manager.settings_data['Runtime_Options']['delete_partial_files']: From 308e89084e4ee84953d7a0ac518dfce99160755a Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 12:10:17 -0500 Subject: [PATCH 03/17] rename purge_dir as purge_dir_tree and have purge_dir be remove the dir itself --- cyberdrop_dl/utils/sorting.py | 6 +++--- cyberdrop_dl/utils/utilities.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index ea64afefe..04a8a7bb0 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -10,7 +10,7 @@ from PIL import Image from videoprops import get_audio_properties, get_video_properties -from cyberdrop_dl.utils.utilities import FILE_FORMATS, log_with_color, purge_dir +from cyberdrop_dl.utils.utilities import FILE_FORMATS, log_with_color, purge_dir_tree,purge_dir logger = logging.getLogger('cyberdrop_dl') @@ -111,10 +111,10 @@ async def sort(self) -> None: await self.sort_video(file, folder.name) else: await self.sort_other(file, folder.name) - await purge_self(folder) + await purge_dir(folder) await asyncio.sleep(5) - await purge_dir(self.download_dir) + await purge_dir_tree(self.download_dir) await log_with_color(f"Organized: {self.audio_count} Audio Files", "green", 20) diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py index 84de60d84..690d97c37 100644 --- a/cyberdrop_dl/utils/utilities.py +++ b/cyberdrop_dl/utils/utilities.py @@ -221,7 +221,7 @@ async def remove_id(manager: Manager, filename: str, ext: str) -> Tuple[str, str """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~""" -async def purge_dir(dirname: Path) -> None: +async def purge_dir_tree(dirname: Path) -> None: """Purges empty directories""" deleted = [] dir_tree = list(os.walk(dirname, topdown=False)) @@ -234,7 +234,7 @@ async def purge_dir(dirname: Path) -> None: list(map(os.rmdir, deleted)) -async def purge_self(dirname: Path) -> None: +async def purge_dir(dirname: Path) -> None: """Purges directory if empyty""" dir_tree = list(os.walk(dirname, topdown=False)) if len(dir_tree) == 0: @@ -262,9 +262,9 @@ async def check_partials_and_empty_folders(manager: Manager): if not manager.config_manager.settings_data['Runtime_Options']['skip_check_for_empty_folders']: await log_with_color("Checking for empty folders...", "yellow", 20) - await purge_dir(manager.path_manager.download_dir) + await purge_dir_tree(manager.path_manager.download_dir) if isinstance(manager.path_manager.sorted_dir, Path): - await purge_dir(manager.path_manager.sorted_dir) + await purge_dir_tree(manager.path_manager.sorted_dir) async def check_latest_pypi(): From 3027c59134204aa4297d63b529d513fefaa377ca Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 12:21:26 -0500 Subject: [PATCH 04/17] remove purge dir because it is not needed --- cyberdrop_dl/utils/sorting.py | 4 ++-- cyberdrop_dl/utils/utilities.py | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index 04a8a7bb0..bb1d23dcd 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -10,7 +10,7 @@ from PIL import Image from videoprops import get_audio_properties, get_video_properties -from cyberdrop_dl.utils.utilities import FILE_FORMATS, log_with_color, purge_dir_tree,purge_dir +from cyberdrop_dl.utils.utilities import FILE_FORMATS, log_with_color, purge_dir_tree logger = logging.getLogger('cyberdrop_dl') @@ -111,7 +111,7 @@ async def sort(self) -> None: await self.sort_video(file, folder.name) else: await self.sort_other(file, folder.name) - await purge_dir(folder) + await purge_dir_tree(folder) await asyncio.sleep(5) await purge_dir_tree(self.download_dir) diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py index 690d97c37..066313ead 100644 --- a/cyberdrop_dl/utils/utilities.py +++ b/cyberdrop_dl/utils/utilities.py @@ -234,14 +234,6 @@ async def purge_dir_tree(dirname: Path) -> None: list(map(os.rmdir, deleted)) -async def purge_dir(dirname: Path) -> None: - """Purges directory if empyty""" - dir_tree = list(os.walk(dirname, topdown=False)) - if len(dir_tree) == 0: - os.rmdir(dirname) - - - async def check_partials_and_empty_folders(manager: Manager): From 7694f8543776d75abd097c44cc43dcf40a8fe777 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 13:22:20 -0500 Subject: [PATCH 05/17] add sorted progress bar --- cyberdrop_dl/managers/live_manager.py | 14 ++++++++ cyberdrop_dl/managers/progress_manager.py | 8 +++++ cyberdrop_dl/ui/progress/sort_progress.py | 44 +++++++++++++++++++++++ cyberdrop_dl/utils/sorting.py | 44 +++++++++++++---------- 4 files changed, 91 insertions(+), 19 deletions(-) create mode 100644 cyberdrop_dl/ui/progress/sort_progress.py diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.py index 9eab2cb08..504201d59 100644 --- a/cyberdrop_dl/managers/live_manager.py +++ b/cyberdrop_dl/managers/live_manager.py @@ -64,3 +64,17 @@ async def get_hash_live(self,stop=False): + @asynccontextmanager + async def get_sort_live(self,stop=False): + try: + if self.manager.args_manager.no_ui: + yield + else: + self.live.start() + self.live.update(self.manager.progress_manager.sort_layout,refresh=True) + yield + if stop: + self.live.stop() + except Exception as e: + await log(f"Issue with rich live {e}",level=10) + await log(f"Issue with rich live {traceback.format_exc()}",level=10) \ No newline at end of file diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py index e2472dbdb..fdd3f0316 100644 --- a/cyberdrop_dl/managers/progress_manager.py +++ b/cyberdrop_dl/managers/progress_manager.py @@ -5,6 +5,8 @@ from cyberdrop_dl.ui.progress.downloads_progress import DownloadsProgress from cyberdrop_dl.ui.progress.hash_progress import HashProgress +from cyberdrop_dl.ui.progress.sort_progress import SortProgress + from cyberdrop_dl.ui.progress.file_progress import FileProgress from cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgress @@ -30,12 +32,16 @@ def __init__(self, manager: 'Manager'): self.download_stats_progress: DownloadStatsProgress = DownloadStatsProgress() self.scrape_stats_progress: ScrapeStatsProgress = ScrapeStatsProgress() self.hash_progress: HashProgress = HashProgress(manager) + self.sort_progress: SortProgress= SortProgress(manager) + self.ui_refresh_rate = manager.config_manager.global_settings_data['UI_Options']['refresh_rate'] self.layout: Layout = field(init=False) self.hash_remove_layout: Layout = field(init=False) self.hash_layout: Layout = field(init=False) + self.sort_layout: Layout = field(init=False) + async def startup(self) -> None: """Startup process for the progress manager""" @@ -58,6 +64,8 @@ async def startup(self) -> None: self.layout = progress_layout self.hash_remove_layout = hash_remove_layout self.hash_layout=await self.hash_progress.get_hash_progress() + self.sort_layout=await self.sort_progress.get_sort_progress() + async def print_stats(self) -> None: """Prints the stats of the program""" diff --git a/cyberdrop_dl/ui/progress/sort_progress.py b/cyberdrop_dl/ui/progress/sort_progress.py new file mode 100644 index 000000000..7ea4d9a13 --- /dev/null +++ b/cyberdrop_dl/ui/progress/sort_progress.py @@ -0,0 +1,44 @@ +from typing import TYPE_CHECKING + +from rich.console import Group +from rich.panel import Panel +from rich.progress import Progress, BarColumn +from humanfriendly import format_size + +if TYPE_CHECKING: + from cyberdrop_dl.managers.manager import Manager + +class SortProgress: + """Class that keeps track of sorted files""" + + def __init__(self, manager: 'Manager'): + self.manager = manager + self.sort_progress = Progress("[progress.description]{task.description}", + BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.2f}%", + "{task.completed} of {task.total} Folders") + + self.sorted_dirs = 0 + + self.sort_progress_group = Group( self.sort_progress) + self.sorted_dir_task_id = self.sort_progress.add_task("[green]Completed", total=0) + + + async def set_total(self,total) -> None: + """sets the total number of directories to be be sorted""" + self.sort_progress.update(self.sorted_dir_task_id,total=total) + + + + async def get_sort_progress(self) -> Panel: + """Returns the progress bar""" + return Panel(self.sort_progress_group, title=f"Config: {self.manager.config_manager.loaded_config}", border_style="green", padding=(1, 1)) + + + + async def add_sorted_dir(self) -> None: + """Adds a completed dir to the progress bar""" + self.sort_progress.advance(self.sorted_dir_task_id , 1) + self.sorted_dirs += 1 + + diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index bb1d23dcd..f77e124e5 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -27,6 +27,7 @@ def get_file_date_in_us_ca_formats(file: Path) -> tuple[str, str]: class Sorter: def __init__(self, manager: 'Manager'): + self.manager = manager self.download_dir = manager.path_manager.scan_dir or manager.path_manager.download_dir self.sorted_downloads = manager.path_manager.sorted_dir self.incrementer_format = manager.config_manager.settings_data['Sorting']['sort_incremementer_format'] @@ -91,27 +92,32 @@ async def sort(self) -> None: return download_folders=await self.get_download_folder() - - for folder in self.download_dir.iterdir(): - if not folder.is_dir(): - continue - if self.sort_cdl_only and folder not in download_folders: - continue - files = await self.find_files_in_dir(folder) - for file in files: - ext = file.suffix.lower() - if '.part' in ext: - continue + async with self.manager.live_manager.get_sort_live(stop=True): + all_scan_folders=list(self.download_dir.iterdir()) + await self.manager.progress_manager.sort_progress.set_total(len(all_scan_folders)) - if ext in FILE_FORMATS['Audio']: - await self.sort_audio(file, folder.name) - elif ext in FILE_FORMATS['Images']: - await self.sort_image(file, folder.name) - elif ext in FILE_FORMATS['Videos']: - await self.sort_video(file, folder.name) + for folder in all_scan_folders: + if not folder.is_dir(): + continue + if self.sort_cdl_only and folder not in download_folders: + continue else: - await self.sort_other(file, folder.name) - await purge_dir_tree(folder) + files = await self.find_files_in_dir(folder) + for file in files: + ext = file.suffix.lower() + if '.part' in ext: + continue + + if ext in FILE_FORMATS['Audio']: + await self.sort_audio(file, folder.name) + elif ext in FILE_FORMATS['Images']: + await self.sort_image(file, folder.name) + elif ext in FILE_FORMATS['Videos']: + await self.sort_video(file, folder.name) + else: + await self.sort_other(file, folder.name) + await purge_dir_tree(folder) + await self.manager.progress_manager.sort_progress.add_sorted_dir() await asyncio.sleep(5) await purge_dir_tree(self.download_dir) From 8f34fd80db02a85a1c64fdbd5d4043f77b0507a8 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 13:25:37 -0500 Subject: [PATCH 06/17] folders only --- cyberdrop_dl/utils/sorting.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index f77e124e5..29a732bc7 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -93,14 +93,12 @@ async def sort(self) -> None: download_folders=await self.get_download_folder() async with self.manager.live_manager.get_sort_live(stop=True): - all_scan_folders=list(self.download_dir.iterdir()) + all_scan_folders=list(filter(lambda x:x.is_dir(),self.download_dir.iterdir())) await self.manager.progress_manager.sort_progress.set_total(len(all_scan_folders)) for folder in all_scan_folders: - if not folder.is_dir(): - continue if self.sort_cdl_only and folder not in download_folders: - continue + pass else: files = await self.find_files_in_dir(folder) for file in files: From 571b976d0df42a9a55236b6f68bbac0616b0edc6 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 13:35:35 -0500 Subject: [PATCH 07/17] the sort arg should make clear it a negative or be postive --- cyberdrop_dl/utils/args/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyberdrop_dl/utils/args/args.py b/cyberdrop_dl/utils/args/args.py index d60ee9cb6..54b17296b 100644 --- a/cyberdrop_dl/utils/args/args.py +++ b/cyberdrop_dl/utils/args/args.py @@ -80,7 +80,7 @@ def parse_args() -> argparse.Namespace: sorting_options = parser.add_argument_group("Sorting") sorting_options.add_argument("--sort-downloads", action="store_true", help="sort downloads into folders", default=False) - sorting_options.add_argument("--sort-all-downloads", action="store_false", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=True) + sorting_options.add_argument("--sort-all-downloads", action="store_true", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=False) sorting_options.add_argument("--sort_folder", type=str, help="path to where you want CDL to store it's log files", default="") sorting_options.add_argument("--scan_folder", type=str, help="path to scan for files, if not set then the download_dir is used", default="") From 1f3fc45023e363a6912c044531a7a04007f2f719 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 13:46:27 -0500 Subject: [PATCH 08/17] reverse for now --- cyberdrop_dl/utils/args/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyberdrop_dl/utils/args/args.py b/cyberdrop_dl/utils/args/args.py index 54b17296b..d60ee9cb6 100644 --- a/cyberdrop_dl/utils/args/args.py +++ b/cyberdrop_dl/utils/args/args.py @@ -80,7 +80,7 @@ def parse_args() -> argparse.Namespace: sorting_options = parser.add_argument_group("Sorting") sorting_options.add_argument("--sort-downloads", action="store_true", help="sort downloads into folders", default=False) - sorting_options.add_argument("--sort-all-downloads", action="store_true", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=False) + sorting_options.add_argument("--sort-all-downloads", action="store_false", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=True) sorting_options.add_argument("--sort_folder", type=str, help="path to where you want CDL to store it's log files", default="") sorting_options.add_argument("--scan_folder", type=str, help="path to scan for files, if not set then the download_dir is used", default="") From 9c81a79d70d15ed2994b2a9e90de9cc33eeab942 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 13:47:54 -0500 Subject: [PATCH 09/17] increment number --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c41960d52..8a60c65f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cyberdrop-dl-patched" -version = "5.4.70" +version = "5.5" description = "Bulk downloader for multiple file hosts" authors = ["Jacob B "] readme = "README.md" From 2cc86e704f8b3acbf4e69242825e4ba69be48200 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 13:51:12 -0500 Subject: [PATCH 10/17] change key --- cyberdrop_dl/managers/args_manager.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cyberdrop_dl/managers/args_manager.py b/cyberdrop_dl/managers/args_manager.py index 384033779..a0319ab6f 100644 --- a/cyberdrop_dl/managers/args_manager.py +++ b/cyberdrop_dl/managers/args_manager.py @@ -157,7 +157,7 @@ def startup(self) -> None: del self.parsed_args['proxy'] del self.parsed_args['links'] del self.parsed_args['sort_downloads'] - del self.parsed_args['sort_cdl_only'] + del self.parsed_args['sort_all_downloads'] del self.parsed_args['sort_folder'] del self.parsed_args['scan_folder'] del self.parsed_args['completed_after'] diff --git a/pyproject.toml b/pyproject.toml index 8a60c65f6..67e0853cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cyberdrop-dl-patched" -version = "5.5" +version = "5.5.0" description = "Bulk downloader for multiple file hosts" authors = ["Jacob B "] readme = "README.md" From 8433350e2f5bb721f9e6bb655155c52722d1a02f Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 15:03:23 -0500 Subject: [PATCH 11/17] only define once --- cyberdrop_dl/utils/sorting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index 29a732bc7..7a63e061e 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -131,7 +131,7 @@ async def get_download_folder(self): if not self.sort_cdl_only: return [] unique_download_paths = await self.db_manager.history_table.get_unique_download_paths() - download_folders = download_folders = [Path(download_path[0]) for download_path in unique_download_paths if Path(download_path[0]).is_dir() and Path(download_path[0]) != self.download_dir] + download_folders = [Path(download_path[0]) for download_path in unique_download_paths if Path(download_path[0]).is_dir() and Path(download_path[0]) != self.download_dir] existing_folders = [] for folder in download_folders: try: From 9b4950b2a9b600316aa128555cc76fc17785b7cd Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 15:04:29 -0500 Subject: [PATCH 12/17] remove test print --- cyberdrop_dl/utils/sorting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py index 7a63e061e..820d7fb70 100644 --- a/cyberdrop_dl/utils/sorting.py +++ b/cyberdrop_dl/utils/sorting.py @@ -137,7 +137,6 @@ async def get_download_folder(self): try: relative_folder = folder.relative_to(self.download_dir) base_folder = self.download_dir / relative_folder.parts[0] - print(relative_folder) except Exception as e: if e.__class__ == ValueError: continue From 5ad006aaee6c1e3a3c4255f81046e40f9872407b Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 15:13:09 -0500 Subject: [PATCH 13/17] increment progres --- cyberdrop_dl/utils/changelog.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cyberdrop_dl/utils/changelog.py b/cyberdrop_dl/utils/changelog.py index 1c470bf84..f8406153e 100644 --- a/cyberdrop_dl/utils/changelog.py +++ b/cyberdrop_dl/utils/changelog.py @@ -14,4 +14,26 @@ \t\t- Fix improper path handling of the `sort_cdl_only` config option. \tFor more details, visit the wiki: https://script-ware.gitbook.io + + +C\bCH\bHA\bAN\bNG\bGE\bEL\bLO\bOG\bG +\tVersion 5.5.0 + +D\bDE\bES\bSC\bCR\bRI\bIP\bPT\bTI\bIO\bON\bN +\tThis update introduces the following changes: +\t\t1. Finalizes new sorting feature +\t\t2. add scanning directory for sorting +\t\t3. adds progress bar for sorting + + + +\tDetails: +\t\t- skips need to scan db if sort_cdl_only is false +\t\t- progress bar for current progress of sorting files,incremented for each folder +\t\t- allow for setting a different folder to scan that is independent of the download folder +\tFor more details, visit the wiki: https://script-ware.gitbook.io + + + + """ From 4f58aa1284528732b39d484d69f83889d7e08dff Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 15:17:44 -0500 Subject: [PATCH 14/17] sync docs --- docs/reference/cli-arguments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.md index e7ea1f69d..f3a728f5b 100644 --- a/docs/reference/cli-arguments.md +++ b/docs/reference/cli-arguments.md @@ -78,7 +78,7 @@ For items not explained below, you can find their counterparts in the Configurat // Sorting Options --sort-downloads ---sort-cdl-only : Only sort files that were downloaded by Cyberdrop-DL +--sort-all-downloads : Only sort files that were downloaded by Cyberdrop-DL --sort-folder // UI Options From e268b8b4c8c5767440f0591ede645adb7a9facf2 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 15:19:36 -0500 Subject: [PATCH 15/17] add scan dir to sorting options --- docs/reference/cli-arguments.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.md index f3a728f5b..3422be6bd 100644 --- a/docs/reference/cli-arguments.md +++ b/docs/reference/cli-arguments.md @@ -78,8 +78,10 @@ For items not explained below, you can find their counterparts in the Configurat // Sorting Options --sort-downloads ---sort-all-downloads : Only sort files that were downloaded by Cyberdrop-DL +--sort-all-downloads : Only sort all downloads with in a the scan_dir --sort-folder +--scan-dir : set a directory to scan + // UI Options --vi-mode From 05912e5060064f9b08aa32f8bb10f426bb967cb2 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 15:20:10 -0500 Subject: [PATCH 16/17] fix spelling --- docs/reference/cli-arguments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.md index 3422be6bd..a2c08ca12 100644 --- a/docs/reference/cli-arguments.md +++ b/docs/reference/cli-arguments.md @@ -78,7 +78,7 @@ For items not explained below, you can find their counterparts in the Configurat // Sorting Options --sort-downloads ---sort-all-downloads : Only sort all downloads with in a the scan_dir +--sort-all-downloads : Only sort all downloads within a the scan_dir --sort-folder --scan-dir : set a directory to scan From 8e188db3afdb5771c85c4212cb955a21db7f8139 Mon Sep 17 00:00:00 2001 From: datawhores Date: Thu, 12 Sep 2024 15:22:43 -0500 Subject: [PATCH 17/17] make sort_all_downloads clearer --- cyberdrop_dl/managers/args_manager.py | 2 +- cyberdrop_dl/utils/args/args.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cyberdrop_dl/managers/args_manager.py b/cyberdrop_dl/managers/args_manager.py index a0319ab6f..8f69f15a0 100644 --- a/cyberdrop_dl/managers/args_manager.py +++ b/cyberdrop_dl/managers/args_manager.py @@ -107,7 +107,7 @@ def startup(self) -> None: self.log_dir = Path(self.parsed_args['log_folder']) if self.parsed_args['sort_downloads']: self.sort_downloads = True - if self.parsed_args['sort_all_downloads']: + if not self.parsed_args['sort_all_downloads']: self.sort_cdl_only = False if self.parsed_args['sort_folder']: self.sort_folder = Path(self.parsed_args['sort_folder']) diff --git a/cyberdrop_dl/utils/args/args.py b/cyberdrop_dl/utils/args/args.py index d60ee9cb6..54b17296b 100644 --- a/cyberdrop_dl/utils/args/args.py +++ b/cyberdrop_dl/utils/args/args.py @@ -80,7 +80,7 @@ def parse_args() -> argparse.Namespace: sorting_options = parser.add_argument_group("Sorting") sorting_options.add_argument("--sort-downloads", action="store_true", help="sort downloads into folders", default=False) - sorting_options.add_argument("--sort-all-downloads", action="store_false", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=True) + sorting_options.add_argument("--sort-all-downloads", action="store_true", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=False) sorting_options.add_argument("--sort_folder", type=str, help="path to where you want CDL to store it's log files", default="") sorting_options.add_argument("--scan_folder", type=str, help="path to scan for files, if not set then the download_dir is used", default="")