Merge branch 'update' into dev

.github/workflows/codeql-analysis.yml

@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl

.github/workflows/create_release.yaml

@@ -0,0 +1,49 @@
+name: Create Release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  create_release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check for tag
+        id: check_tag
+        run: |
+          if [ -z "${GITHUB_REF##*/}" ]; then
+            echo "Error: No tag provided. Please provide a tag to run this workflow."
+            exit 1
+          fi
+
+      - name: Extract version number
+        id: extract_version
+        run: |
+          TAG_NAME=${GITHUB_REF##*/}
+          VERSION_NUMBER=$(echo "$TAG_NAME" | grep -o '[0-9]' | head -n 1)
+          if [ -z "$VERSION_NUMBER" ]; then
+            echo "Error: No numeric version found in tag name."
+            exit 1
+          fi
+          echo "VERSION_NUMBER=$VERSION_NUMBER" >> $GITHUB_ENV
+
+      - name: Create zip from subfolder
+        run: |
+          cd scripts/release
+          zip -r ../../Cyberdrop-DL.v${VERSION_NUMBER}.zip .
+
+      - name: Create release
+        id: create_release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: Cyberdrop-DL V${VERSION_NUMBER} Start Files
+          body: |
+            If you install and run the program with PIP yourself, you do not need anything here.
+            Download the Cyberdrop-DL zip file below, you do not need to download the source code zip files.
+            The zip file contains start files that will download and install Cyberdrop-DL and keep it up to date. 
+          files: Cyberdrop-DL.v${VERSION_NUMBER}.zip 

.github/workflows/publish-to-pypi.yaml

@@ -6,6 +6,9 @@ on:
       - master
     tags:
       - '*.*.*'
+    paths:
+      - 'pyproject.toml'
+  workflow_dispatch:
 
 jobs:
   release:

.github/workflows/ruff.yaml

@@ -0,0 +1,19 @@
+name: Ruff
+on: [ push, pull_request ]
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+
+      - name: ruff lint
+        uses: astral-sh/ruff-action@v1
+        with:
+          changed-files: "true"
+
+      - name: ruff format check
+        uses: astral-sh/ruff-action@v1
+        with:
+          args: "format --check"
+          changed-files: "true"

.gitignore

@@ -17,3 +17,6 @@ venv
 
 # Compiled package
 /dist/
+
+# MacOS system files
+**.DS_Store

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0  
+    hooks:
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+      - id: end-of-file-fixer
+      - id: check-toml
+      - id: check-yaml
+      - id: detect-private-key
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.2
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format

CHANGELOG.md

@@ -5,7 +5,40 @@ All notable changes to this project will be documented here. For more details, v
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [5.7.0] - 2024-10-23
+## [5.7.1] - 2024-11-05
+
+⚠️**BREAKING CHANGES**
+
+> All output files (except for the main log file) are now CSV files with headers for each column (`Scrape_Errors`, `Download_Errors`, `Unsupported_URLs` and `Last_Forum_Post`). A custom filename for each file can still be set via config, but the extensions will always be `.csv`
+
+This update introduces the following changes:
+
+1. Adds the option to limit how many items are scraped 
+2. Add support for scraping a users' coomer favorites
+3. Add integration to handle downloads supported by https://real-debrid.com
+4. Add support for https://nekohouse.su profiles and posts
+5. Add support for https://imagepond.net URLs
+6. Add support for password protected albums from chevereto sites (`jpg5`, `Img.kiwi` and `Imagepond`)
+7. Show `total runtime` and `total downloaded data` on final report
+8. Add support to send the main log file as an attachment to the `webhook_url` report
+9. Add support to sent CDL report via email, telegram and many other services via Apprise
+10. Add support for `%` encoded URLs in the input file
+11. General logging improvements and bug fixes
+
+#### Details:
+
+- Users can limit the number of items to scrape by type, using the `--maximum-number-of-children` parameter. For more details on how to use this feature, visit the wiki: https://script-ware.gitbook.io/cyberdrop-dl/reference/configuration-options/settings#download-options
+- Add support for scraping a users' coomer favorites by allowing the user to pass the coomer favorites page URL as an input URL (https://coomer.su/favorites). This requires them to have their coomer session token in the `authentication.yaml` file.
+- Add real-debrid integration to download from any site that they support (`mega.nz`,`rapidgator`, `google drive`, `k2s`, etc). User needs to provide their API key in the `authentication.yaml` file in order to allow downloads 
+- Nekohouse URLs can now be scraped and downloaded by CDL
+- Users can now get the stats report of the run via multiple services and include the main log as an attachment. For more information on how to setup notifications, visit: https://script-ware.gitbook.io/cyberdrop-dl/reference/notifications
+- Fix parsing of bunkr file extensions when `--remove-generated-id` is enabled
+- Remove console markdown data from log files
+- Fix `only_hosts` skip logic
+- Better handling of some unknown errors
+
+
+## [5.7.0] - 2024-10-25
 
 This update introduces the following changes:
 1. Rotating log files 
@@ -16,7 +49,7 @@ This update introduces the following changes:
 6. Add support for password protected Cyberfile URLs
 7. Simplify some UI elements
 8. Improve jdownloader intergration
-9. Implement rich logger
+9.  Implement rich logger
 10. Add a "Check for Updates" UI option
 11. General bug fixes
 
@@ -41,6 +74,7 @@ This update introduces the following changes:
 - Fix cyberfile deleted folders not being correctly scraped
 - Update logging to include when a file is being downloaded with no crawler
 
+
 ## [5.6.54] - 2024-10-21
 
 This update introduces the following changes:
@@ -83,6 +117,7 @@ This update introduces the following changes:
 - Using the flag `--skip-referer-seen-before` will skip downloading files from any referer that have been scraped before. The file (s) will always be skipped regardless of whether the referer was successfully scraped or not
 - Fixes album_id property not being saved to database on supported crawlers
 
+
 ## [5.6.50] - 2024-10-07
 
 This update introduces the following changes:

CONTRIBUTING.md

@@ -0,0 +1,133 @@
+# Contributing
+
+## Found an Issue?
+
+If you find a bug in the source code or a mistake in the documentation, you can
+[open an Issue](#submitting-an-issue) to the GitHub Repository or you can
+[submit a Pull Request](#submitting-a-pull-request-pr) with a fix if you know how to code.
+
+## Want a Feature?
+
+You can *request* a new feature by [opening an issue](#submitting-an-issue). If you would like to *implement* a new feature,
+please open an issue or a discussion with a proposal first. If your feature is *small* and easy to implement,
+you can craft it and directly [submit it as a Pull Request](#submitting-a-pull-request-pr).
+
+## Submission Guidelines
+
+### Submitting an Issue
+
+Before openning an issue, search through existing issues to ensure you are not opening a duplicate.
+You must also [read the wiki](https://script-ware.gitbook.io/cyberdrop-dl/frequently-asked-questions) to learn how to solve most common problems.
+
+If your issue appears to be a bug, and hasn't been reported, open a new issue. Please do not report duplicate issues.
+Providing the following information will allow your issue to be dealt with quickly:
+
+- **Overview of the Issue** - always attach you logs to any new issue.
+- **Version** - what version of Cyberdrop-DL are you running. You should always update to the latest release before opening an issue since 
+the issue may have been fixed already
+- **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you
+- **Operating System** - what OS and version are you using
+- **Reproduce the Error** - provide a live example or a unambiguous set of steps
+- **Related Issues** - has a similar issue been reported before?
+- **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be
+  causing the problem (line of code or commit)
+
+You can open a new issue by providing the above information at https://github.com/jbsparrow/CyberDropDownloader/issues/new/choose.
+
+### Submitting a Pull Request (PR)
+
+Before you submit your Pull Request (PR) consider the following guidelines:
+
+- Search the [repository](https://github.com/jbsparrow/CyberDropDownloader/pulls) for an open or closed PR
+  that relates to your submission. You don't want to duplicate effort.
+- Clone the repo and make your changes on a new branch in your fork
+- Follow [code style conventions](#code-style)
+- Commit your changes using a descriptive commit message
+- Push your fork to GitHub
+- In GitHub, create a pull request to the `master` branch of the repository. 
+- Add a description to your PR. If the PR is small (such as a typo fix), you can go brief. 
+If it contains a lot of changes, it's better to write more details.
+If your changes are user-facing (e.g. a new feature in the UI, a change in behavior, or a bugfix) 
+please include a short message to add to the changelog.
+- Wait for a maintainer to review your PR and then address any comments they might have.
+
+If everything is okay, your changes will be merged into the project.
+
+## Setting up the development environment
+
+1. Install a [supported version of Python](https://www.python.org/downloads/). Cyberdrop-DL supports python `3.11` through `3.12`
+(python `3.13` is **NOT** supported yet)
+
+2. Clone the repo
+
+```shell
+git clone "https://github.com/jbsparrow/CyberDropDownloader"
+cd CyberDropDownloader
+```
+
+3. Install `pipx` (optional, but recommended): https://pipx.pypa.io/stable/installation/
+
+4. Install `poetry`, the project management package Cyberdrop-DL uses
+
+> If you installed `pipx`:
+
+```shell
+pipx install poetry
+```
+
+> With regular `pip`:
+
+```shell
+pip install poetry
+```
+
+5. Install the project's dependencies
+
+```shell
+poetry install
+```
+
+6. Install the pre-commit hooks:
+
+```shell
+poetry run pre-commit install
+```
+
+## Code Style
+
+### Standards
+
+`Formatting`: This project uses [ruff](https://docs.astral.sh/ruff) for formatting, linting and import sorting.
+
+`Typechecking`: Typechecking is not enforced but highly recommended.
+
+`Line Width`: We use a line width of 120.
+
+### Code formatting with pre-commit hooks
+
+This project uses git pre-commit hooks to perform formatting and linting before a commit is allowed,
+to ensure consistent style and catch some common issues early on.
+
+Once installed, hooks will run when you commit. If the formatting isn't quite right or a linter catches something,
+the commit will be rejected and `ruff` will try to fix the files. If `ruff` can not fix all the issues,
+you will need to look at the output and fix them manually. When everything is fixed (either by `ruff` itself or manually)
+all you need to do is `git add` those files again and retry your commit.
+
+### Manual code formatting
+
+We recommend [setting up your IDE](https://docs.astral.sh/ruff/editors/) to format and check with `ruff`, but you can always run
+`poetry run ruff check --fix` then `poetry run ruff format` in the root directory before submitting a pull request.
+If you're using VScode, you can set it to [auto format python files with ruff on save](#editor-settings) in your `settings.json`
+
+## Editor settings
+
+If you use VScode and have `ruff` installed as a formatter, you might find the following `settings.json` useful:
+
+```json
+{
+  "[python]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  }
+}
+```

README.md

@@ -49,6 +49,7 @@ Brand new and improved! Cyberdrop-DL now has an updated paint job, fantastic new
   - kemono
   - leakedmodels
   - mediafire
+  - nekohouse
   - nudostar.com
   - nudostar.tv
   - omegascans

cyberdrop_dl/__init__.py

@@ -1,2 +1,3 @@
 import importlib.metadata
+
 __version__ = importlib.metadata.version('cyberdrop-dl-patched')