Merge pull request #83 from datawhores/sorting2

Sorting2

cyberdrop_dl/managers/args_manager.py

@@ -41,6 +41,9 @@ def __init__(self):
         self.sort_downloads = field(init=False)
         self.sort_cdl_only = field(init=True)
         self.sort_folder = None
+        self.scan_folder=None
+
+
 
         # Logs
         self.main_log_filename = None
@@ -104,11 +107,12 @@ def startup(self) -> None:
             self.log_dir = Path(self.parsed_args['log_folder'])
         if self.parsed_args['sort_downloads']:
             self.sort_downloads = True
-        if self.parsed_args['sort_all_downloads']:
+        if not self.parsed_args['sort_all_downloads']:
             self.sort_cdl_only = False
         if self.parsed_args['sort_folder']:
             self.sort_folder = Path(self.parsed_args['sort_folder'])
-
+        if self.parsed_args['scan_folder']:
+            self.scan_folder = Path(self.parsed_args['scan_folder'])   
         if self.parsed_args['main_log_filename']:
             self.main_log_filename = self.parsed_args['main_log_filename']
         if self.parsed_args['last_forum_post_filename']:
@@ -153,5 +157,10 @@ def startup(self) -> None:
         del self.parsed_args['proxy']
         del self.parsed_args['links']
         del self.parsed_args['sort_downloads']
-        del self.parsed_args['sort_cdl_only']
+        del self.parsed_args['sort_all_downloads']
         del self.parsed_args['sort_folder']
+        del self.parsed_args['scan_folder']
+        del self.parsed_args['completed_after']
+        del self.parsed_args['completed_before']
+
+

cyberdrop_dl/managers/config_manager.py

@@ -47,6 +47,7 @@ def get_keys(dl, keys=None) -> set:
     return set(keys)
 
 
+
 class ConfigManager:
     def __init__(self, manager: 'Manager'):
         self.manager = manager
@@ -104,6 +105,7 @@ def load_configs(self) -> None:
             self.settings_data['Logs']['log_folder'] = APP_STORAGE / "Configs" / self.loaded_config / "Logs"
             self.settings_data['Logs']['webhook_url'] = ""
             self.settings_data['Sorting']['sort_folder'] = DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"
+            self.settings_data['Sorting']['scan_folder'] = None
             self.write_updated_settings_config()
 
     def _verify_authentication_config(self) -> None:
@@ -128,6 +130,8 @@ def _verify_settings_config(self) -> None:
         self.settings_data['Logs']['log_folder'] = Path(self.settings_data['Logs']['log_folder'])
         self.settings_data['Logs']['webhook_url'] = str(self.settings_data['Logs']['webhook_url'])
         self.settings_data['Sorting']['sort_folder'] = Path(self.settings_data['Sorting']['sort_folder'])
+        self.settings_data['Sorting']['scan_folder'] = Path(self.settings_data['Sorting']['scan_folder']) if self.settings_data['Sorting']['scan_folder'] else None
+
 
         # change to ints
         self.settings_data['File_Size_Limits']['maximum_image_size'] = int(
@@ -222,6 +226,7 @@ def create_new_config(self, new_settings: Path, settings_data: Dict) -> None:
         settings_data['Logs']['log_folder'] = str(settings_data['Logs']['log_folder'])
         settings_data['Logs']['webhook_url'] = str(settings_data['Logs']['webhook_url'])
         settings_data['Sorting']['sort_folder'] = str(settings_data['Sorting']['sort_folder'])
+        settings_data['Sorting']['scan_folder'] = str(settings_data['Sorting']['scan_folder'])
         _save_yaml(new_settings, settings_data)
 
     def write_updated_authentication_config(self) -> None:
@@ -236,6 +241,8 @@ def write_updated_settings_config(self) -> None:
         settings_data['Logs']['log_folder'] = str(settings_data['Logs']['log_folder'])
         settings_data['Logs']['webhook_url'] = str(settings_data['Logs']['webhook_url'])
         settings_data['Sorting']['sort_folder'] = str(settings_data['Sorting']['sort_folder'])
+        settings_data['Sorting']['scan_folder'] = str(settings_data['Sorting']['scan_folder'])
+
         _save_yaml(self.settings, settings_data)
 
     def write_updated_global_settings_config(self) -> None:

cyberdrop_dl/managers/live_manager.py

@@ -64,3 +64,17 @@ async def get_hash_live(self,stop=False):
 
 
 
+    @asynccontextmanager
+    async def get_sort_live(self,stop=False):
+        try:
+            if self.manager.args_manager.no_ui:
+                yield  
+            else:
+                self.live.start()
+                self.live.update(self.manager.progress_manager.sort_layout,refresh=True)
+                yield
+            if stop:
+                self.live.stop()
+        except Exception as e:
+            await log(f"Issue with rich live {e}",level=10)
+            await log(f"Issue with rich live {traceback.format_exc()}",level=10)

cyberdrop_dl/managers/manager.py

@@ -187,6 +187,8 @@ async def args_logging(self) -> None:
         print_settings['Files']['download_folder'] = str(print_settings['Files']['download_folder'])
         print_settings["Logs"]["log_folder"] = str(print_settings["Logs"]["log_folder"])
         print_settings['Sorting']['sort_folder'] = str(print_settings['Sorting']['sort_folder'])
+        print_settings['Sorting']['scan_folder'] = str(print_settings['Sorting']['scan_folder']) if str(print_settings['Sorting']['scan_folder']) else ""
+
 
         input_file = str(self.path_manager.input_file)
         download_dir = str(self.path_manager.download_dir)

cyberdrop_dl/managers/path_manager.py

@@ -26,6 +26,8 @@ def __init__(self, manager: 'Manager'):
 
         self.download_dir: Path = field(init=False)
         self.sorted_dir: Path = field(init=False)
+        self.scan_dir: Path = field(init=False)
+
         self.log_dir: Path = field(init=False)
 
         self.cache_dir: Path = field(init=False)
@@ -61,6 +63,8 @@ def startup(self) -> None:
         """Startup process for the Directory Manager"""
         self.download_dir = self.manager.config_manager.settings_data['Files']['download_folder'] if not self.manager.args_manager.download_dir else self.manager.args_manager.download_dir
         self.sorted_dir = self.manager.config_manager.settings_data['Sorting']['sort_folder'] if not self.manager.args_manager.sort_folder else self.manager.args_manager.sort_folder
+
+        self.scan_dir = self.manager.config_manager.settings_data['Sorting']['scan_folder'] if not self.manager.args_manager.scan_folder else self.manager.args_manager.scan_folder
         self.log_dir = self.manager.config_manager.settings_data['Logs']['log_folder'] if not self.manager.args_manager.log_dir else self.manager.args_manager.log_dir
         self.input_file = self.manager.config_manager.settings_data['Files']['input_file'] if not self.manager.args_manager.input_file else self.manager.args_manager.input_file
         self.history_db = self.cache_dir / "cyberdrop.db"

cyberdrop_dl/managers/progress_manager.py

@@ -5,6 +5,8 @@
 
 from cyberdrop_dl.ui.progress.downloads_progress import DownloadsProgress
 from cyberdrop_dl.ui.progress.hash_progress import HashProgress
+from cyberdrop_dl.ui.progress.sort_progress import SortProgress
+
 
 from cyberdrop_dl.ui.progress.file_progress import FileProgress
 from cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgress
@@ -30,12 +32,16 @@ def __init__(self, manager: 'Manager'):
         self.download_stats_progress: DownloadStatsProgress = DownloadStatsProgress()
         self.scrape_stats_progress: ScrapeStatsProgress = ScrapeStatsProgress()
         self.hash_progress: HashProgress = HashProgress(manager)
+        self.sort_progress: SortProgress= SortProgress(manager)
+
 
         self.ui_refresh_rate = manager.config_manager.global_settings_data['UI_Options']['refresh_rate']
 
         self.layout: Layout = field(init=False)
         self.hash_remove_layout: Layout = field(init=False)
         self.hash_layout: Layout = field(init=False)
+        self.sort_layout: Layout = field(init=False)
+
 
     async def startup(self) -> None:
         """Startup process for the progress manager"""
@@ -58,6 +64,8 @@ async def startup(self) -> None:
         self.layout = progress_layout
         self.hash_remove_layout = hash_remove_layout
         self.hash_layout=await self.hash_progress.get_hash_progress()
+        self.sort_layout=await self.sort_progress.get_sort_progress()
+
 
     async def print_stats(self) -> None:
         """Prints the stats of the program"""

cyberdrop_dl/ui/progress/sort_progress.py

@@ -0,0 +1,44 @@
+from typing import TYPE_CHECKING
+
+from rich.console import Group
+from rich.panel import Panel
+from rich.progress import Progress, BarColumn
+from humanfriendly import format_size
+
+if TYPE_CHECKING:
+    from cyberdrop_dl.managers.manager import Manager
+
+class SortProgress:
+    """Class that keeps track of sorted files"""
+
+    def __init__(self, manager: 'Manager'):
+        self.manager = manager
+        self.sort_progress =  Progress("[progress.description]{task.description}",
+                                 BarColumn(bar_width=None),
+                                 "[progress.percentage]{task.percentage:>3.2f}%",
+                                 "{task.completed} of {task.total} Folders")
+
+        self.sorted_dirs = 0
+
+        self.sort_progress_group = Group( self.sort_progress)
+        self.sorted_dir_task_id = self.sort_progress.add_task("[green]Completed", total=0)
+
+
+    async def set_total(self,total) -> None:
+        """sets the total number of directories to be be sorted"""
+        self.sort_progress.update(self.sorted_dir_task_id,total=total)
+
+
+
+    async def get_sort_progress(self) -> Panel:
+        """Returns the progress bar"""
+        return Panel(self.sort_progress_group, title=f"Config: {self.manager.config_manager.loaded_config}", border_style="green", padding=(1, 1))
+
+
+
+    async def add_sorted_dir(self) -> None:
+        """Adds a completed dir to the progress bar"""
+        self.sort_progress.advance(self.sorted_dir_task_id , 1)
+        self.sorted_dirs += 1
+
+

cyberdrop_dl/ui/prompts/settings_user_prompts.py

@@ -397,6 +397,12 @@ def edit_sort_options_prompt(manager: Manager, config: Dict) -> None:
         sort_folder = inquirer.filepath(
             message="Enter the folder you want to sort files into:",
             default=str(config['Sorting']['sort_folder']),
+            vi_mode=manager.vi_mode,
+        ).execute()
+
+        scan_folder = inquirer.filepath(
+            message="Enter the folder you want to scan for files",
+            default=str(config['Sorting']['scan_folder'] or config['Files']['download_folder']),
             validate=PathValidator(is_dir=True, message="Input is not a directory"),
             vi_mode=manager.vi_mode,
         ).execute()
@@ -432,6 +438,7 @@ def edit_sort_options_prompt(manager: Manager, config: Dict) -> None:
         ).execute()
 
         config['Sorting']['sort_folder'] = Path(sort_folder)
+        config['Sorting']['scan_folder'] = Path(scan_folder) if bool(scan_folder) else None
         config['Sorting']['sort_incremementer_format'] = sort_incremementer_format
         config['Sorting']['sorted_audio'] = sorted_audio
         config['Sorting']['sorted_video'] = sorted_video

cyberdrop_dl/utils/args/args.py

@@ -80,8 +80,9 @@ def parse_args() -> argparse.Namespace:
 
     sorting_options = parser.add_argument_group("Sorting")
     sorting_options.add_argument("--sort-downloads", action="store_true", help="sort downloads into folders", default=False)
-    sorting_options.add_argument("--sort-all-downloads", action="store_false", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=True)
+    sorting_options.add_argument("--sort-all-downloads", action="store_true", help="sort all downloads, not just those downloader by Cyberdrop-DL", default=False)
     sorting_options.add_argument("--sort_folder", type=str, help="path to where you want CDL to store it's log files", default="")
+    sorting_options.add_argument("--scan_folder", type=str, help="path to scan for files, if not set then the download_dir is used", default="")
 
     ui_options = parser.add_argument_group("UI_Options")
     ui_options.add_argument("--vi-mode", action="store_true", help="enable VIM keybindings for UI", default=None)

cyberdrop_dl/utils/args/config_definitions.py

@@ -106,6 +106,7 @@
     "Sorting": {
         "sort_downloads": False,
         "sort_folder": str(DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"),
+        "scan_folder":None,
         "sort_cdl_only": True,
         "sort_incremementer_format": " ({i})",
         "sorted_audio": "{sort_dir}/{base_dir}/Audio/{filename}{ext}",

cyberdrop_dl/utils/changelog.py

@@ -14,4 +14,26 @@
 \t\t- Fix improper path handling of the `sort_cdl_only` config option.
 
 \tFor more details, visit the wiki: https://script-ware.gitbook.io
+
+
+C\bCH\bHA\bAN\bNG\bGE\bEL\bLO\bOG\bG
+\tVersion 5.5.0
+
+D\bDE\bES\bSC\bCR\bRI\bIP\bPT\bTI\bIO\bON\bN
+\tThis update introduces the following changes:
+\t\t1. Finalizes new sorting feature
+\t\t2. add scanning directory for sorting
+\t\t3. adds progress bar for sorting
+
+
+
+\tDetails:
+\t\t- skips need to scan db if sort_cdl_only is false
+\t\t- progress bar for current progress of sorting files,incremented for each folder
+\t\t- allow for setting a different folder to scan that is independent of the download folder
+\tFor more details, visit the wiki: https://script-ware.gitbook.io
+
+
+
+
 """

cyberdrop_dl/utils/sorting.py

@@ -10,7 +10,7 @@
 from PIL import Image
 from videoprops import get_audio_properties, get_video_properties
 
-from cyberdrop_dl.utils.utilities import FILE_FORMATS, log_with_color, purge_dir
+from cyberdrop_dl.utils.utilities import FILE_FORMATS, log_with_color, purge_dir_tree
 
 logger = logging.getLogger('cyberdrop_dl')
 
@@ -27,7 +27,8 @@ def get_file_date_in_us_ca_formats(file: Path) -> tuple[str, str]:
 
 class Sorter:
     def __init__(self, manager: 'Manager'):
-        self.download_dir = manager.path_manager.download_dir
+        self.manager = manager
+        self.download_dir = manager.path_manager.scan_dir or manager.path_manager.download_dir
         self.sorted_downloads = manager.path_manager.sorted_dir
         self.incrementer_format = manager.config_manager.settings_data['Sorting']['sort_incremementer_format']
         self.sort_cdl_only = manager.config_manager.settings_data['Sorting']['sort_cdl_only']
@@ -80,6 +81,8 @@ async def check_dir_parents(self) -> bool:
     async def sort(self) -> None:
         """Sorts the files in the download directory into their respective folders"""
         await log_with_color("\nSorting Downloads: Please Wait", "cyan", 20)
+        #make sort dir
+        self.sorted_downloads.mkdir(parents=True, exist_ok=True)
 
         if await self.check_dir_parents():
             return
@@ -88,10 +91,48 @@ async def sort(self) -> None:
             await log_with_color("Download Directory does not exist", "red", 40)
             return
 
+        download_folders=await self.get_download_folder()
+        async with self.manager.live_manager.get_sort_live(stop=True):
+            all_scan_folders=list(filter(lambda x:x.is_dir(),self.download_dir.iterdir()))
+            await self.manager.progress_manager.sort_progress.set_total(len(all_scan_folders))
+
+            for folder in all_scan_folders:
+                if self.sort_cdl_only and folder not in download_folders:
+                    pass
+                else:
+                    files = await self.find_files_in_dir(folder)
+                    for file in files:
+                        ext = file.suffix.lower()
+                        if '.part' in ext:
+                            continue
+
+                        if ext in FILE_FORMATS['Audio']:
+                            await self.sort_audio(file, folder.name)
+                        elif ext in FILE_FORMATS['Images']:
+                            await self.sort_image(file, folder.name)
+                        elif ext in FILE_FORMATS['Videos']:
+                            await self.sort_video(file, folder.name)
+                        else:
+                            await self.sort_other(file, folder.name)
+                        await purge_dir_tree(folder)
+                await self.manager.progress_manager.sort_progress.add_sorted_dir()
+
+        await asyncio.sleep(5)
+        await purge_dir_tree(self.download_dir)
+
+
+        await log_with_color(f"Organized: {self.audio_count} Audio Files", "green", 20)
+        await log_with_color(f"Organized: {self.image_count} Image Files", "green", 20)
+        await log_with_color(f"Organized: {self.video_count} Video Files", "green", 20)
+        await log_with_color(f"Organized: {self.other_count} Other Files", "green", 20)
+
+    async def get_download_folder(self):
+        """Gets the download folder"""
+        if not self.sort_cdl_only:
+            return []
         unique_download_paths = await self.db_manager.history_table.get_unique_download_paths()
         download_folders = [Path(download_path[0]) for download_path in unique_download_paths if Path(download_path[0]).is_dir() and Path(download_path[0]) != self.download_dir]
         existing_folders = []
-
         for folder in download_folders:
             try:
                 relative_folder = folder.relative_to(self.download_dir)
@@ -101,42 +142,11 @@ async def sort(self) -> None:
                     continue
                 logger.log(40, f"Error: {e}\n\nfolder: {folder}\ndownload_dir: {self.download_dir}\nrelative_folder: {relative_folder}")
                 raise e
-
-            if base_folder.is_dir():
+            if base_folder.exists():
                 existing_folders.append(base_folder)
-
         download_folders.extend(existing_folders)
         download_folders = list(set(download_folders))
-
-        for folder in self.download_dir.iterdir():
-            if not folder.is_dir():
-                continue
-            if folder not in download_folders and self.sort_cdl_only:
-                continue
-
-            files = await self.find_files_in_dir(folder)
-            for file in files:
-                ext = file.suffix.lower()
-                if '.part' in ext:
-                    continue
-
-                if ext in FILE_FORMATS['Audio']:
-                    await self.sort_audio(file, folder.name)
-                elif ext in FILE_FORMATS['Images']:
-                    await self.sort_image(file, folder.name)
-                elif ext in FILE_FORMATS['Videos']:
-                    await self.sort_video(file, folder.name)
-                else:
-                    await self.sort_other(file, folder.name)
-
-        await asyncio.sleep(5)
-        await purge_dir(self.download_dir)
-
-        await log_with_color(f"Organized: {self.audio_count} Audio Files", "green", 20)
-        await log_with_color(f"Organized: {self.image_count} Image Files", "green", 20)
-        await log_with_color(f"Organized: {self.video_count} Video Files", "green", 20)
-        await log_with_color(f"Organized: {self.other_count} Other Files", "green", 20)
-
+        return download_folders
     async def sort_audio(self, file: Path, base_name: str) -> None:
         """Sorts an audio file into the sorted audio folder"""
         self.audio_count += 1