Add progress options, and extend redraw to redraw both screens

cyberdrop_dl/__init__.py

@@ -1 +1 @@
-__version__ = "5.3.13"
+__version__ = "5.3.14"

cyberdrop_dl/main.py

@@ -106,7 +106,7 @@ async def director(manager: Manager) -> None:
         if not manager.args_manager.sort_all_configs:
             try:
                 if not manager.args_manager.no_ui:
-                    with Live(manager.progress_manager.layout, refresh_per_second=manager.config_manager.settings_data['Runtime_Options']['ui_refresh_rate']):
+                    with Live(manager.progress_manager.layout, refresh_per_second=manager.config_manager.global_settings_data['UI_Options']['refresh_rate']):
                         await runtime(manager)
                 else:
                     await runtime(manager)

cyberdrop_dl/managers/args_manager.py

@@ -21,8 +21,6 @@ def __init__(self):
         self.no_ui = False
         self.load_config_from_args = False
         self.load_config_name = ""
-        self.vi_mode = None
-
         self.other_links: list = []
 
         # Files
@@ -42,6 +40,9 @@ def __init__(self):
         self.unsupported_urls_filename = None
         self.download_error_urls_filename = None
         self.scrape_error_urls_filename = None
+
+        # UI
+        self.vi_mode = None
 
     def startup(self) -> None:
         """Parses arguments and sets variables accordingly"""

cyberdrop_dl/managers/config_manager.py

@@ -141,14 +141,13 @@ def _verify_settings_config(self) -> None:
         self.settings_data['File_Size_Limits']['minimum_other_size'] = int(
             self.settings_data['File_Size_Limits']['minimum_other_size'])
 
-        self.settings_data['Runtime_Options']['ui_refresh_rate'] = int(
-            self.settings_data['Runtime_Options']['ui_refresh_rate'])
         self.settings_data['Runtime_Options']['log_level'] = int(self.settings_data['Runtime_Options']['log_level'])
 
         self.global_settings_data['General']['max_file_name_length'] = int(
             self.global_settings_data['General']['max_file_name_length'])
         self.global_settings_data['General']['max_folder_name_length'] = int(
             self.global_settings_data['General']['max_folder_name_length'])
+
         self.global_settings_data['Rate_Limiting_Options']['connection_timeout'] = int(
             self.global_settings_data['Rate_Limiting_Options']['connection_timeout'])
         self.global_settings_data['Rate_Limiting_Options']['download_attempts'] = int(
@@ -164,6 +163,13 @@ def _verify_settings_config(self) -> None:
         self.global_settings_data['Rate_Limiting_Options']['read_timeout'] = int(
             self.global_settings_data['Rate_Limiting_Options']['read_timeout'])
 
+        self.global_settings_data['UI_Options']['refresh_rate'] = int(
+            self.global_settings_data['UI_Options']['refresh_rate'])
+        self.global_settings_data['UI_Options']['scraping_item_limit'] = int(
+            self.global_settings_data['UI_Options']['scraping_item_limit'])
+        self.global_settings_data['UI_Options']['downloading_item_limit'] = int(
+            self.global_settings_data['UI_Options']['downloading_item_limit'])
+
         if get_keys(default_settings_data) == get_keys(existing_settings_data):
             return
 

cyberdrop_dl/managers/manager.py

@@ -39,6 +39,7 @@ def __init__(self):
         self.task_group: asyncio.TaskGroup = field(init=False)
         self.task_list: list = []
         self.scrape_mapper = field(init=False)
+
         self.vi_mode: bool = None
 
     def startup(self) -> None:
@@ -54,7 +55,7 @@ def startup(self) -> None:
         self.cache_manager.startup(self.path_manager.cache_dir / "cache.yaml")
         self.config_manager = ConfigManager(self)
         self.config_manager.startup()
-        self.vi_mode = self.config_manager.global_settings_data['General']['vi_mode'] if self.args_manager.vi_mode is None else self.args_manager.vi_mode
+        self.vi_mode = self.config_manager.global_settings_data['UI_Options']['vi_mode'] if self.args_manager.vi_mode is None else self.args_manager.vi_mode
 
         self.path_manager.startup()
         self.log_manager = LogManager(self)

cyberdrop_dl/managers/progress_manager.py

@@ -16,16 +16,18 @@
 class ProgressManager:
     def __init__(self, manager: 'Manager'):
         # File Download Bars
-        self.file_progress: FileProgress = FileProgress(5, manager)
+        self.file_progress: FileProgress = FileProgress(manager.config_manager.global_settings_data['UI_Options']['downloading_item_limit'], manager)
 
         # Scraping Printout
-        self.scraping_progress: ScrapingProgress = ScrapingProgress(5, manager)
+        self.scraping_progress: ScrapingProgress = ScrapingProgress(manager.config_manager.global_settings_data['UI_Options']['scraping_item_limit'], manager)
 
         # Overall Progress Bars & Stats
         self.download_progress: DownloadsProgress = DownloadsProgress()
         self.download_stats_progress: DownloadStatsProgress = DownloadStatsProgress()
         self.scrape_stats_progress: ScrapeStatsProgress = ScrapeStatsProgress()
-
+
+        self.ui_refresh_rate = manager.config_manager.global_settings_data['UI_Options']['refresh_rate']
+
         self.layout: Layout = field(init=False)
 
     async def startup(self) -> None:

cyberdrop_dl/ui/progress/file_progress.py

@@ -60,7 +60,7 @@ async def get_queue_length(self) -> int:
 
         return total
 
-    async def redraw(self) -> None:
+    async def redraw(self, passed=False) -> None:
         """Redraws the progress bar"""
         while len(self.visible_tasks) > self.tasks_visibility_limit:
             task_id = self.visible_tasks.pop(0)
@@ -81,6 +81,9 @@ async def redraw(self) -> None:
             self.queue.update(self.queue_task_id, description=self.queue_str.format(color=self.color, number=queue_length, type_str=self.type_str), visible=True)
         else:
             self.queue.update(self.queue_task_id, visible=False)
+
+        if not passed:
+            await self.manager.progress_manager.scraping_progress.redraw(True)
 
     async def add_task(self, file: str, expected_size: Optional[int]) -> TaskID:
         """Adds a new task to the progress bar"""

cyberdrop_dl/ui/progress/scraping_progress.py

@@ -50,7 +50,7 @@ async def get_queue_length(self) -> int:
 
         return total
 
-    async def redraw(self) -> None:
+    async def redraw(self, passed=False) -> None:
         """Redraws the progress bar"""
         while len(self.visible_tasks) > self.tasks_visibility_limit:
             task_id = self.visible_tasks.pop(0)
@@ -71,6 +71,9 @@ async def redraw(self) -> None:
             self.queue.update(self.queue_task_id, description=self.queue_str.format(color=self.color, number=queue_length, type_str=self.type_str), visible=True)
         else:
             self.queue.update(self.queue_task_id, visible=False)
+
+        if not passed:
+            await self.manager.progress_manager.file_progress.redraw(True)
 
     async def add_task(self, url: URL) -> TaskID:
         """Adds a new task to the progress bar"""

cyberdrop_dl/ui/prompts/settings_global_prompts.py

@@ -23,7 +23,8 @@ def edit_global_settings_prompt(manager: Manager) -> None:
             choices=[
                 Choice(1, "Edit General Settings"),
                 Choice(2, "Edit Rate Limiting Settings"),
-                Choice(3, "Done"),
+                Choice(3, "Edit UI Options"),
+                Choice(4, "Done"),
             ],
             vi_mode=manager.vi_mode,
         ).execute()
@@ -35,9 +36,13 @@ def edit_global_settings_prompt(manager: Manager) -> None:
         # Edit Rate Limiting Settings
         elif action == 2:
             edit_rate_limiting_settings_prompt(manager)
+
+        # Edit UI Settings
+        elif action == 3:
+            edit_progress_settings_prompt(manager)
 
         # Done
-        elif action == 3:
+        elif action == 4:
             manager.config_manager.write_updated_global_settings_config()
             break
 
@@ -81,11 +86,6 @@ def edit_general_settings_prompt(manager: Manager) -> None:
         float_allowed=False,
         vi_mode=manager.vi_mode,
     ).execute()
-    manager.vi_mode = inquirer.confirm(
-        message="Enable VI/VIM keybindings?",
-        default=bool(manager.config_manager.global_settings_data['General']['vi_mode']),
-        vi_mode=manager.vi_mode,
-    ).execute()
 
     manager.config_manager.global_settings_data['General']['allow_insecure_connections'] = allow_insecure_connections
     manager.config_manager.global_settings_data['General']['user_agent'] = user_agent
@@ -94,20 +94,42 @@ def edit_general_settings_prompt(manager: Manager) -> None:
     manager.config_manager.global_settings_data['General']['max_filename_length'] = int(max_filename_length)
     manager.config_manager.global_settings_data['General']['max_folder_name_length'] = int(max_folder_name_length)
     manager.config_manager.global_settings_data['General']['required_free_space'] = int(required_free_space)
-    manager.config_manager.global_settings_data['General']['vi_mode'] = manager.vi_mode
+
 
 def edit_progress_settings_prompt(manager: Manager) -> None:
     """Edit the progress settings"""
     console.clear()
     console.print("Editing Progress Settings")
     refresh_rate = inquirer.number(
         message="Refresh Rate:",
-        default=int(manager.config_manager.global_settings_data['Progress_Options']['refresh_rate']),
+        default=int(manager.config_manager.global_settings_data['UI_Options']['refresh_rate']),
+        float_allowed=False,
+        vi_mode=manager.vi_mode,
+    ).execute()
+    scraping_item_limit = inquirer.number(
+        message="Number of lines to allow for scraping items before overflow:",
+        default=int(manager.config_manager.global_settings_data['UI_Options']['scraping_item_limit']),
+        float_allowed=False,
+        vi_mode=manager.vi_mode,
+    ).execute()
+    downloading_item_limit = inquirer.number(
+        message="Refresh Rate:",
+        default=int(manager.config_manager.global_settings_data['UI_Options']['downloading_item_limit']),
         float_allowed=False,
         vi_mode=manager.vi_mode,
     ).execute()
+
+    vi_mode = inquirer.confirm(
+        message="Enable VI/VIM keybindings?",
+        default=bool(manager.config_manager.global_settings_data['UI_Options']['vi_mode']),
+        vi_mode=manager.vi_mode,
+    ).execute()
 
-    manager.config_manager.global_settings_data['Progress_Options']['refresh_rate'] = int(refresh_rate)
+    manager.config_manager.global_settings_data['UI_Options']['refresh_rate'] = int(refresh_rate)
+    manager.config_manager.global_settings_data['UI_Options']['scraping_item_limit'] = int(scraping_item_limit)
+    manager.config_manager.global_settings_data['UI_Options']['downloading_item_limit'] = int(downloading_item_limit)
+    manager.config_manager.global_settings_data['UI_Options']['vi_mode'] = vi_mode
+    manager.vi_mode = vi_mode
 
 
 def edit_rate_limiting_settings_prompt(manager: Manager) -> None:

cyberdrop_dl/ui/prompts/settings_user_prompts.py

@@ -360,23 +360,13 @@ def edit_runtime_options_prompt(manager: Manager, config: Dict) -> None:
         vi_mode=manager.vi_mode,
     ).execute()
 
-    ui_refresh_rate = inquirer.number(
-        message="Enter the desired UI refresh rate:",
-        default=int(config['Runtime_Options']['ui_refresh_rate']),
-        min_allowed=1,
-        validate=NumberValidator(),
-        long_instruction="10 is the default",
-        vi_mode=manager.vi_mode,
-    ).execute()
-
     for key in config["Runtime_Options"]:
         config["Runtime_Options"][key] = False
 
     for key in action:
         config["Runtime_Options"][key] = True
 
     config['Runtime_Options']['log_level'] = int(log_level)
-    config['Runtime_Options']['ui_refresh_rate'] = int(ui_refresh_rate)
 
 
 def edit_sort_options_prompt(manager: Manager, config: Dict) -> None:

cyberdrop_dl/utils/args/args.py

@@ -17,7 +17,6 @@ def parse_args() -> argparse.Namespace:
     general.add_argument("--download-all-configs", action="store_true", help="Skip the UI and go straight to downloading (runs all configs sequentially)", default=False)
     general.add_argument("--sort-all-configs", action="store_true", help="Sort all configs sequentially", default=False)
     general.add_argument("--retry-failed", action="store_true", help="retry failed downloads", default=False)
-    general.add_argument("--vi-mode", action="store_true", help="enable VIM keybindings for UI", default=None)
 
     # File Paths
     file_paths = parser.add_argument_group("File_Paths")
@@ -69,12 +68,19 @@ def parse_args() -> argparse.Namespace:
     runtime_options.add_argument("--skip-check-for-empty-folders", action="store_true", help="skip check (and removal) for empty folders", default=False)
     runtime_options.add_argument("--delete-partial-files", action="store_true", help="delete partial downloads", default=False)
     runtime_options.add_argument("--send-unsupported-to-jdownloader", action="store_true", help="send unsupported urls to jdownloader", default=False)
-    runtime_options.add_argument("--ui-refresh-rate", type=int, help="set the UI refresh rate (default: %(default)s)", default=10)
     runtime_options.add_argument("--update-last-forum-post", action="store_true", help="update the last forum post", default=False)
 
     sorting_options = parser.add_argument_group("Sorting")
     sorting_options.add_argument("--sort-downloads", action="store_true", help="sort downloads into folders", default=False)
     sorting_options.add_argument("--sort_folder", type=str, help="path to where you want CDL to store it's log files", default="")
+
+    ui_options = parser.add_argument_group("UI_Options")
+    ui_options.add_argument("--vi-mode", action="store_true", help="enable VIM keybindings for UI", default=None)
+    ui_options.add_argument("--refresh-rate", type=int, help="refresh rate for the UI (default: %(default)s)", default=10)
+    ui_options.add_argument("--scraping-item-limit", type=int, help="number of lines to allow for scraping items before overflow (default: %(default)s)", default=5)
+    ui_options.add_argument("--downloading-item-limit", type=int, help="number of lines to allow for downloading items before overflow (default: %(default)s)", default=5)
+
+
 
     # Links
     parser.add_argument("links", metavar="link", nargs="*", help="link to content to download (passing multiple links is supported)", default=[])

cyberdrop_dl/utils/args/config_definitions.py

@@ -99,7 +99,6 @@
         "skip_check_for_empty_folders": False,
         "delete_partial_files": False,
         "send_unsupported_to_jdownloader": False,
-        "ui_refresh_rate": 10,
         "update_last_forum_post": False,
     },
     "Sorting": {
@@ -123,7 +122,6 @@
         "max_file_name_length": 95,
         "max_folder_name_length": 60,
         "required_free_space": 5,
-        "vi_mode": False,
     },
     "Rate_Limiting_Options": {
         "connection_timeout": 15,
@@ -134,5 +132,11 @@
         "max_simultaneous_downloads": 15,
         "max_simultaneous_downloads_per_domain": 5,
     },
+    "UI_Options": {
+        "vi_mode": False,
+        "refresh_rate": 10,
+        "scraping_item_limit": 5,
+        "downloading_item_limit": 5,
+    }
 }
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cyberdrop-dl"
-version = "5.3.13"
+version = "5.3.14"
 description = "Bulk downloader for multiple file hosts"
 authors = ["Jules Winnfield <[email protected]>"]
 readme = "README.md"