feat: skip slow downloads

cyberdrop_dl/clients/download_client.py

@@ -3,6 +3,8 @@
 import asyncio
 import copy
 import itertools
+import time
+from datetime import timedelta
 from functools import partial, wraps
 from http import HTTPStatus
 from pathlib import Path
@@ -13,7 +15,12 @@
 from aiohttp import ClientSession
 from yarl import URL
 
-from cyberdrop_dl.clients.errors import DownloadError, InsufficientFreeSpaceError, InvalidContentTypeError
+from cyberdrop_dl.clients.errors import (
+    DownloadError,
+    InsufficientFreeSpaceError,
+    InvalidContentTypeError,
+    SlowDownloadError,
+)
 from cyberdrop_dl.utils.constants import DEBUG_VAR, FILE_FORMATS
 from cyberdrop_dl.utils.logger import log
 
@@ -194,13 +201,31 @@ async def _append_content(
         media_item.partial_file.parent.mkdir(parents=True, exist_ok=True)
         if not media_item.partial_file.is_file():
             media_item.partial_file.touch()
+
+        last_slow_speed_read = None
+        download_speed_threshold = self.manager.config_manager.settings_data.runtime_options.slow_downloads_speed
+
+        def check_download_speed():
+            nonlocal last_slow_speed_read
+            speed = self.manager.progress_manager.file_progress.get_speed(media_item.task_id)
+            if speed > download_speed_threshold:
+                last_slow_speed_read = None
+                return
+            if not last_slow_speed_read:
+                last_slow_speed_read = time.perf_counter()
+            elif time.perf_counter() - last_slow_speed_read > timedelta(seconds=10):
+                raise SlowDownloadError(origin=media_item)
+
         async with aiofiles.open(media_item.partial_file, mode="ab") as f:  # type: ignore
             async for chunk in content.iter_chunked(self.client_manager.speed_limiter.chunk_size):
                 chunk_size = len(chunk)
                 await self.client_manager.speed_limiter.acquire(chunk_size)
                 await asyncio.sleep(0)
                 await f.write(chunk)
                 update_progress(chunk_size)
+                if download_speed_threshold:
+                    check_download_speed()
+
         if not content.total_bytes and not media_item.partial_file.stat().st_size:
             media_item.partial_file.unlink()
             raise DownloadError(status=HTTPStatus.INTERNAL_SERVER_ERROR, message="File is empty")

cyberdrop_dl/clients/errors.py

@@ -90,6 +90,13 @@ def __init__(
         super().__init__(ui_message, message=message, status=status, origin=origin)
 
 
+class SlowDownloadError(DownloadError):
+    def __init__(self, origin: ScrapeItem | MediaItem | URL | None = None) -> None:
+        """This error will be thrown when a download fails."""
+        ui_message = "Slow Download"
+        super().__init__(ui_message, origin=origin)
+
+
 class InsufficientFreeSpaceError(CDLBaseError):
     def __init__(self, origin: ScrapeItem | MediaItem | URL | None = None) -> None:
         """This error will be thrown when no enough storage is available."""

cyberdrop_dl/config_definitions/config_settings.py

@@ -132,6 +132,7 @@ class RuntimeOptions(BaseModel):
     jdownloader_autostart: bool = False
     jdownloader_whitelist: list[NonEmptyStr] = []
     deep_scrape: bool = False
+    slow_downloads_speed: ByteSize = ByteSize(0)
 
     @field_validator("jdownloader_download_dir", mode="before")
     @classmethod

cyberdrop_dl/ui/progress/file_progress.py

@@ -64,3 +64,11 @@ def advance_file(self, task_id: TaskID, amount: int) -> None:
         """Advances the progress of the given task by the given amount."""
         self.downloaded_data += amount
         self._progress.advance(task_id, amount)
+
+    def get_speed(self, task_id: TaskID) -> float:
+        if task_id not in self._tasks:
+            msg = "Task ID not found"
+            raise ValueError(msg)
+
+        task = self._progress._tasks[task_id]
+        return task.finished_speed or task.speed or 0