chore: merge with master

.github/workflows/create_release.yaml

@@ -13,38 +13,53 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Check for tag
+      - name: Check for version tag
         id: check_tag
         run: |
-          if [ -z "${GITHUB_REF##*/}" ]; then
+          TAG_NAME=${GITHUB_REF##*/}
+          if [ -z "$TAG_NAME" ]; then
             echo "Error: No tag provided. Please provide a tag to run this workflow."
             exit 1
           fi
+          echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV
 
       - name: Extract version number
         id: extract_version
         run: |
-          TAG_NAME=${GITHUB_REF##*/}
-          VERSION_NUMBER=$(echo "$TAG_NAME" | grep -o '[0-9]' | head -n 1)
+          TAG_NAME=${{ env.TAG_NAME }}
+          VERSION_NUMBER=$(echo "$TAG_NAME" | grep -oE '^[0-9]+' | head -n 1)
           if [ -z "$VERSION_NUMBER" ]; then
             echo "Error: No numeric version found in tag name."
             exit 1
           fi
           echo "VERSION_NUMBER=$VERSION_NUMBER" >> $GITHUB_ENV
 
-      - name: Create zip from subfolder
+      - name: Extract changelog content
+        id: extract_changelog
+        run: |
+          TAG_NAME=${{ env.TAG_NAME }}
+          CHANGELOG_CONTENT=$(awk -v version="$TAG_NAME" '
+          /^#+ \[/ { if (p) { exit }; if ($2 == "["version"]") { p=1; next} } p && NF
+          ' "CHANGELOG.md")
+          if [ -z "$CHANGELOG_CONTENT" ]; then
+            echo "Error: No changelog content found for tag $TAG_NAME."
+            exit 1
+          fi
+          echo "CHANGELOG_CONTENT<<EOF" >> $GITHUB_ENV
+          echo "$CHANGELOG_CONTENT" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Create start scripts zip
         run: |
-          mkdir -p zip/Cyberdrop-DL.v${VERSION_NUMBER}
-          cp -r scripts/release/* zip/Cyberdrop-DL.v${VERSION_NUMBER}
-          zip -r Cyberdrop-DL.v${VERSION_NUMBER}.zip zip/*
+          mkdir -p zip/Cyberdrop-DL.v${{ env.VERSION_NUMBER }}
+          cp -r scripts/release/* zip/Cyberdrop-DL.v${{ env.VERSION_NUMBER }}
+          cd zip
+          zip -r ../Cyberdrop-DL.v${{ env.VERSION_NUMBER }}.zip *
 
-      - name: Create release
+      - name: Create GH release
         id: create_release
         uses: softprops/action-gh-release@v2
         with:
-          name: Cyberdrop-DL V${VERSION_NUMBER} Start Files
-          body: |
-            If you install and run the program with PIP yourself, you do not need anything here.
-            Download the Cyberdrop-DL zip file below, you do not need to download the source code zip files.
-            The zip file contains start files that will download and install Cyberdrop-DL and keep it up to date.
-          files: Cyberdrop-DL.v${VERSION_NUMBER}.zip
+          name: Cyberdrop-DL V${{ env.VERSION_NUMBER }} Start Files
+          body: ${{ env.CHANGELOG_CONTENT }}
+          files: Cyberdrop-DL.v${{ env.VERSION_NUMBER }}.zip

.github/workflows/ruff.yaml

@@ -8,12 +8,12 @@ jobs:
         uses: actions/checkout@v4
 
       - name: ruff lint
-        uses: astral-sh/ruff-action@v1
+        uses: astral-sh/ruff-action@v3
         with:
-          version: v0.8.0
+          version: "0.8.0"
 
       - name: ruff format check
-        uses: astral-sh/ruff-action@v1
+        uses: astral-sh/ruff-action@v3
         with:
           args: "format --check"
-          version: v0.8.0
+          version: "0.8.0"

.gitignore

@@ -4,7 +4,8 @@
 /AppData/
 /Downloads/
 /Old Files/
-cyberdrop_dl_debug.log
+*.log
+.coverage
 
 # Python cache
 __pycache__
@@ -21,3 +22,5 @@ venv
 
 # MacOS system files
 **.DS_Store
+.aider*
+.env

.pre-commit-config.yaml

@@ -2,15 +2,29 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
-      - id: trailing-whitespace
-        args: [--markdown-linebreak-ext=md]
-      - id: end-of-file-fixer
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-illegal-windows-names
+      - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
       - id: check-toml
       - id: check-yaml
       - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: name-tests-test
+        args: [ --pytest-test-first ]
+      - id: trailing-whitespace
+        args: [ --markdown-linebreak-ext=md ]
+
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.24.2
+    hooks:
+      - id: toml-sort-fix
+        files: ^(.*\.toml)$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.2
+    rev: v0.8.4
     hooks:
       - id: ruff
         args: [ --fix ]

CHANGELOG.md

@@ -5,7 +5,88 @@ All notable changes to this project will be documented here. For more details, v
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [6.0.0] - 2024-12-20
+## [6.2.0] - 2025-01-10
+
+### Added
+
+- PixHost.to support
+- Forum.AllPornComix.com support (forum)
+- Bellazon.com support (forum)
+- TitsInTops.com support (forum)
+- Video support for chevereto sites (`imagepond.net`, `img.kiwi` and `jpg5.su`)
+- Filesystem URLs support for images and videos (pixeldrain)
+- Indirect links support (saint)
+
+### Changed
+
+- Better MediaFire errors handling
+- Better RealDebrid errors handling
+- Do not overload 403 and 404 errors
+- Use canonical URLs for files in database (chevereto)
+
+### Fixed
+
+- Files being hashed twice when using `Hashing.IN_PLACE`
+- Post information parsing (reddit)
+- Sub-path parsing, ex: submitted, comments (reddit)
+- "Loose Files" folder not being created (all crawlers)
+- Delete startup log file if empty
+- Make a single GoFile token acquisition request
+- Make a single login request try per forum
+- DDoS-Guard detection
+
+
+## [6.1.1] - 2025-01-05
+
+### Changed
+
+- Remove crawlers scrape lock
+
+### Fixed
+
+- v5 to v6 transfer [database] (`hash` table missing on second startup)
+- v4 to v6 transfer [logs] ('downloader.log' being used by another process)
+
+## [6.1.0] - 2025-01-04
+
+### Fixed
+
+- Fallback to old file password check (cyberfile)
+- Handle stream redirects (bunkr)
+- Files being saved with the current date instead of the upload date
+- Sanitize filenames before downloading
+- Apprise notifications integration
+- Log startup errors to a file (validation, YAML, database and unknown errors)
+- Miscalculation of download and scrape queue length
+- Sorting videos with invalid or missing fps metadata
+- e-Hentai crawler
+
+### Changed
+
+- Set 512MB as the minimum required free space
+- `--console-log-level` will have no effect unless `--no-ui` was used
+- OS notification are enabled by default on Windows
+
+### Deprecated
+
+- `--output-folder` is deprecated. Use `--download-folder`
+- v4 support (importing v4 config and files) will be removed in a future version
+
+### Added
+
+- `--deep-scrape` option: https://script-ware.gitbook.io/cyberdrop-dl/reference/configuration-options/settings/runtime_options#deep_scrape
+- `--separate-posts-format` option: https://script-ware.gitbook.io/cyberdrop-dl/reference/configuration-options/settings/download_options#separate_posts_format
+- Allow any file name for cookie files
+- Redact username from log tracebacks
+
+
+## [6.0.1] - 2024-12-24
+
+### Fixed
+
+- Database v5 to v6 transfer
+
+## [6.0.0] - 2024-12-23
 
 ⚠️**BREAKING CHANGES**
 
@@ -20,6 +101,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 > 3. Removed `--sort-cdl-only` option
 
 > 4. The following arguments are deprecated and will be removed in a future version:
+>  - `--output-folder`
 >  - `--download-all-configs`
 >  - `--sort-all-configs`
 >  - `--sort-all-downloads`
@@ -31,6 +113,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 This update introduces the following changes:
+- Add Python 3.13 support
 - Add requests caching to speed up future download sessions for forums, Coomer, and Kemono.
 - Update Kemono & Coomer crawlers to use API calls
 - Update GoFile crawler to the new site design

CONTRIBUTING.md

@@ -16,7 +16,7 @@ you can craft it and directly [submit it as a Pull Request](#submitting-a-pull-r
 
 ### Submitting an Issue
 
-Before openning an issue, search through existing issues to ensure you are not opening a duplicate.
+Before opening an issue, search through existing issues to ensure you are not opening a duplicate.
 You must also [read the wiki](https://script-ware.gitbook.io/cyberdrop-dl/frequently-asked-questions) to learn how to solve most common problems.
 
 If your issue appears to be a bug, and hasn't been reported, open a new issue. Please do not report duplicate issues.
@@ -55,8 +55,7 @@ If everything is okay, your changes will be merged into the project.
 
 ## Setting up the development environment
 
-1. Install a [supported version of Python](https://www.python.org/downloads/). Cyberdrop-DL supports python `3.11` through `3.12`
-(python `3.13` is **NOT** supported yet)
+1. Install a [supported version of Python](https://www.python.org/downloads/). Cyberdrop-DL supports python `3.11` through `3.13`
 
 2. Clone the repo
 

README.md

@@ -7,6 +7,7 @@
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/cyberdrop-dl-patched)](https://pypi.org/project/cyberdrop-dl-patched/)
 [![Docs](https://img.shields.io/badge/docs-wiki-blue?link=https%3A%2F%2Fscript-ware.gitbook.io%2Fcyberdrop-dl)](https://script-ware.gitbook.io/cyberdrop-dl)
 [![GitHub License](https://img.shields.io/github/license/jbsparrow/CyberDropDownloader)](https://github.com/jbsparrow/CyberDropDownloader/blob/master/LICENSE)
+[![linting - Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/jbsparrow/CyberDropDownloader/actions/workflows/ruff.yaml)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/cyberdrop-dl-patched)](https://pypistats.org/packages/cyberdrop-dl-patched)
 
 [![Discord](https://discordapp.com/api/guilds/1070206871564197908/widget.png?style=banner2)](https://discord.com/invite/P5nsbKErwy)

cyberdrop_dl/clients/errors.py

@@ -6,16 +6,15 @@
 
 from yarl import URL
 
+from cyberdrop_dl.utils.constants import VALIDATION_ERROR_FOOTER
+
 if TYPE_CHECKING:
+    from requests import Response
     from yaml.constructor import ConstructorError
 
     from cyberdrop_dl.scraper.crawler import ScrapeItem
     from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
 
-VALIDATION_ERROR_FOOTER = """
-Read the documentation for guidance on how to resolve this error: https://script-ware.gitbook.io/cyberdrop-dl/reference/configuration-options
-Please note, this is not a bug. Do not open issues related to this"""
-
 
 class CDLBaseError(Exception):
     """Base exception for cyberdrop-dl errors."""
@@ -105,6 +104,39 @@ def __init__(self, origin: ScrapeItem | MediaItem | URL | None = None) -> None:
         super().__init__(ui_message, origin=origin)
 
 
+class MediaFireError(CDLBaseError):
+    def __init__(
+        self, status: str | int, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None
+    ) -> None:
+        """This error will be thrown when a scrape fails."""
+        ui_message = f"{status} MediaFire Error"
+        super().__init__(ui_message, message=message, status=status, origin=origin)
+
+
+class RealDebridError(CDLBaseError):
+    """Base RealDebrid API error."""
+
+    def __init__(self, response: Response, error_codes: dict[int, str]) -> None:
+        url = URL(response.url)
+        self.path = url.path
+        try:
+            JSONResp: dict = response.json()
+            code = JSONResp.get("error_code")
+            if code == 16:
+                code = 7
+            error = error_codes.get(code, "Unknown error")
+
+        except AttributeError:
+            code = response.status_code
+            error = f"{code} - {HTTPStatus(code).phrase}"
+
+        error = error.capitalize()
+
+        """This error will be thrown when a scrape fails."""
+        ui_message = f"{code} RealDebrid Error"
+        super().__init__(ui_message, message=error, status=code, origin=url)
+
+
 class ScrapeError(CDLBaseError):
     def __init__(
         self, status: str | int, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None