diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-build-and-push.yml index 9fb21ec..2f3062e 100644 --- a/.github/workflows/docker-build-and-push.yml +++ b/.github/workflows/docker-build-and-push.yml @@ -12,7 +12,7 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push-image: + build-and-publish-alpine: runs-on: ubuntu-latest permissions: contents: read @@ -43,6 +43,29 @@ jobs: tags: ${{ env.REGISTRY }}/cc-youcube/youcube:latest,${{ env.REGISTRY }}/cc-youcube/youcube:alpine labels: ${{ steps.meta.outputs.labels }} + build-and-publish-nvidia: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout 🛎️ + uses: actions/checkout@v3 + + - name: Login to container registry 🔐 + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata 🏷️ + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: 🔨 Build and Publish nvidia 🚀 uses: docker/build-push-action@v3 with: diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index dbd3d32..d7ef0bc 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -11,7 +11,7 @@ on: - "src/**" jobs: - build-image: + build-alpine-image: runs-on: ubuntu-latest steps: @@ -23,6 +23,13 @@ jobs: with: context: src + build-nvidia-image: + runs-on: ubuntu-latest + + steps: + - name: Checkout 🛎️ + uses: actions/checkout@v3 + - name: Build nvidia image 🔨 uses: docker/build-push-action@v4 with: diff --git a/.gitignore b/.gitignore index 8fc3088..f4336f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea *.dfpwm *.32vid dictionary.dic diff --git a/README.md b/README.md index d77aae8..e17a7e3 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ python src/youcube.py Environment variables you can use to configure the server: | Variable | Default | Description | -|-------------------------------|------------|--------------------------------------------------------------------------------------------------------------------| +| ----------------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------ | | `HOST` | `0.0.0.0` | The host where the web server runs on. | | `PORT` | `5000` | The port where the web server should run on | | `FFMPEG_PATH` | `ffmpeg` | Path to the FFmpeg executable | @@ -55,7 +55,7 @@ And [Sanic Builtin values]. ## Docker Compose ```yml -version: "2.0" +--- services: youcube: image: ghcr.io/cc-youcube/youcube:latest @@ -63,6 +63,7 @@ services: hostname: youcube ports: - 5000:5000 +... ``` [spotify application]: https://developer.spotify.com/dashboard/applications @@ -79,4 +80,4 @@ services: [preview]: .README/preview-server.png [Python Version: 3.7+]: https://img.shields.io/badge/Python-3.7+-green?style=for-the-badge&logo=Python&logoColor=white [Python Lint Workflow Status]: https://img.shields.io/github/actions/workflow/status/CC-YouCube/server/pylint.yml?branch=main&label=Python%20Lint&logo=github&style=for-the-badge -[Sanic Builtin values]: https://sanic.dev/en/guide/running/configuration.md#builtin-values \ No newline at end of file +[Sanic Builtin values]: https://sanic.dev/en/guide/running/configuration.md#builtin-values diff --git a/pyproject.toml b/pyproject.toml index 7169556..8afcc1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,41 +1,40 @@ -[build-system] -requires = ["setuptools", "setuptools-scm"] -build-backend = "setuptools.build_meta" - -[project] -name = "youcube-server" -version = "1.0.0" -authors = [ - {name = "Commandcracker"}, -] -description = "A server which provides a WebSocket API for YouCube clients" -readme = "README.md" -requires-python = ">=3.7" -keywords = ["youtube", "youcube", "computercraft", "minecraft"] -license = {text = "GPL-3.0"} -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Natural Language :: English", - "Topic :: Multimedia :: Video", - "Topic :: Multimedia :: Sound/Audio :: Players", -] -# Required for pulling dependencies from requirements.txt -dynamic = ["dependencies"] - -[project.urls] -Homepage = "https://youcube.madefor.cc" -Repository = "https://github.com/CC-YouCube/server" -Documentation = "https://youcube.madefor.cc/guides/server/installation/" - -[tool.setuptools.dynamic] -# Pull dependencies from requirements.txt -dependencies = {file = ["src/requirements.txt"]} - -[tool.setuptools.packages.find] -where = ["src"] # list of folders that contain the packages (["."] by default) -include = ["youcube*"] # package names should match these glob patterns (["*"] by default) - -[tool.autopep8] -ignore = "E701" - +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "youcube-server" +version = "1.0.0" +authors = [{ name = "Commandcracker" }] +description = "A server which provides a WebSocket API for YouCube clients" +readme = "README.md" +requires-python = ">=3.7" +keywords = ["youtube", "youcube", "computercraft", "minecraft"] +license = { text = "GPL-3.0" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Topic :: Multimedia :: Video", + "Topic :: Multimedia :: Sound/Audio :: Players", +] +# Required for pulling dependencies from requirements.txt +dynamic = ["dependencies"] + +[project.urls] +Homepage = "https://youcube.madefor.cc" +Repository = "https://github.com/CC-YouCube/server" +Documentation = "https://youcube.madefor.cc/guides/server/installation/" + +[tool.setuptools.dynamic] +# Pull dependencies from requirements.txt +dependencies = { file = ["src/requirements.txt"] } + +[tool.setuptools.packages.find] +where = ["src"] # list of folders that contain the packages (["."] by default) +include = [ + "youcube*", +] # package names should match these glob patterns (["*"] by default) + +[tool.autopep8] +ignore = "E701" diff --git a/spellcheck_wordlist.txt b/spellcheck_wordlist.txt index bced916..4de8969 100644 --- a/spellcheck_wordlist.txt +++ b/spellcheck_wordlist.txt @@ -9,14 +9,17 @@ Popen ffmpeg subprocess utils +spotipy # Web Development websocket YouCube Spotify html +dev # Programming Keywords +str env untrusted usr @@ -64,9 +67,15 @@ github dlp geeksforgeeks www +metalink # Video Streaming yc yt dl youtube + +# Other +f'Attachment +WIP +xml diff --git a/src/Dockerfile b/src/Dockerfile index ca41a7d..4ca720c 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -1,51 +1,65 @@ FROM ghcr.io/commandcracker/ffmpeg:latest AS ffmpeg -FROM ffmpeg as sanjuuni +FROM ffmpeg AS sanjuuni -ENV SANJUUNI_VERSION=ed34c266cd489e4db796ca803e247b48b43853e0 +ENV SANJUUNI_VERSION=778644b164c8877e56f9f5512480dde857133815 -ARG SANJUUNI_SHA512SUM="9e482e3b8f8885e8f458856f11d5ee4c27a0aa469b8c54abe1aef943f630ca27eb148c8779ba7a053c1abcce298513e98b614747a77ae1c0cbc86a0a7c95a6d8 *sanjuuni.tar.gz" +ARG SANJUUNI_SHA512SUM="353b97e53ec2daba3046b3062c8a389c75066ea410f9abe089467a4648e83afe926cd65ad0904a0d59eca0520e3174e0f3190987cfee3abbdc9141a04a80ef1a *sanjuuni.tar.gz" + +SHELL ["/bin/ash", "-eo", "pipefail", "-c"] RUN set -eux; \ - apk add --no-cache --update g++ zlib-dev poco-dev make; \ - wget --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ - echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ - mkdir --parents sanjuuni; \ - tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz; \ - rm sanjuuni.tar.gz; + apk add --no-cache --update \ + g++=13.2.1_git20240309-r0 \ + zlib-dev=1.3.1-r1 \ + poco-dev=1.12.4-r0 \ + make=4.4.1-r2; \ + wget -q --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ + echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ + mkdir --parents sanjuuni; \ + tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz; \ + rm sanjuuni.tar.gz; WORKDIR /sanjuuni RUN set -eux; \ - ./configure; \ - make + ./configure; \ + make + +FROM ghcr.io/commandcracker/alpine-pypy3.10-pip:3.20.1-pypy-7.3.14-pip-24.1.1 AS builder -FROM ghcr.io/commandcracker/alpine-pypy3.10-pip:3.19.0-pypy-7.3.14-pip-23.3.2 AS builder +WORKDIR / COPY requirements.txt . COPY youcube ./youcube COPY compile.py . RUN set -eux; \ - apk add --no-cache --update build-base; \ - pip install --no-cache-dir -U setuptools -r requirements.txt; \ - python3 compile.py; \ - pip uninstall pip -y + apk add --no-cache --update build-base=0.5-r3; \ + pip install --no-cache-dir -U setuptools -r requirements.txt; \ + python3 compile.py; \ + pip uninstall pip -y -FROM alpine:3.19.0 +FROM alpine:3.20.1 WORKDIR /opt/server RUN set -eux; \ - apk add --no-cache --update \ - # pypy requirements - libffi libbz2 \ - # sanjuuni requirements - poco \ - # ffmpeg requirements - libgcc libstdc++ ca-certificates libgomp expat; \ - apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community libssl1.1 libcrypto1.1; \ - chown 1000 /opt/server/ + apk add --no-cache --update \ + # CVE-2024-5535 TODO: remove when base image is updated + openssl \ + # pypy requirements + libffi=3.4.6-r0 libbz2=1.0.8-r6 \ + # sanjuuni requirements + poco=1.12.4-r0 \ + # ffmpeg requirements + libgcc=13.2.1_git20240309-r0 \ + libstdc++=13.2.1_git20240309-r0 \ + ca-certificates=20240226-r0 \ + libgomp=13.2.1_git20240309-r0 \ + expat=2.6.2-r0; \ + apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community libssl1.1=1.1.1u-r1 libcrypto1.1=1.1.1u-r1; \ + chown 1000 /opt/server/ COPY --from=builder /opt/pypy /opt/pypy # add ffmpeg @@ -54,14 +68,14 @@ COPY --from=ffmpeg /usr/local /usr/local COPY --from=sanjuuni /sanjuuni/sanjuuni /usr/local/bin ENV \ - # Make sure we use the virtualenv: - PATH="/opt/pypy/bin:$PATH" \ - # Use ffmpeg libs - LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \ - # yt-dlp cache dir - XDG_CACHE_HOME="/opt/server/.yt-dlp-cache" \ - # FIXME: Add UVLOOP support for alpine pypy - SANIC_NO_UVLOOP=true + # Make sure we use the virtualenv: + PATH="/opt/pypy/bin:$PATH" \ + # Use ffmpeg libs + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \ + # yt-dlp cache dir + XDG_CACHE_HOME="/opt/server/.yt-dlp-cache" \ + # FIXME: Add UVLOOP support for alpine pypy + SANIC_NO_UVLOOP=true USER 1000 diff --git a/src/Dockerfile.nvidia b/src/Dockerfile.nvidia index 5dfc775..0b1baba 100644 --- a/src/Dockerfile.nvidia +++ b/src/Dockerfile.nvidia @@ -1,53 +1,59 @@ +#!/usr/bin/env dockerfile-shebang + FROM ghcr.io/commandcracker/ubuntu-ffmpeg:latest AS ffmpeg -FROM ffmpeg as sanjuuni +FROM ffmpeg AS sanjuuni + +ENV SANJUUNI_VERSION=778644b164c8877e56f9f5512480dde857133815 -ENV SANJUUNI_VERSION=0.4 +ARG SANJUUNI_SHA512SUM="353b97e53ec2daba3046b3062c8a389c75066ea410f9abe089467a4648e83afe926cd65ad0904a0d59eca0520e3174e0f3190987cfee3abbdc9141a04a80ef1a *sanjuuni.tar.gz" -ARG SANJUUNI_SHA512SUM="952a6c608d167f37faad53ee7f2e0de8090a02bf73b6455fae7c6b6f648dd6a188e7749fe26caeee85126b2a38d7391389c19afb0100e9962dc551188b9de6ae *sanjuuni.tar.gz" +SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN set -eux; \ - apt-get update; \ - apt-get install \ - ocl-icd-opencl-dev \ - wget \ - clang \ - make \ - libpoco-dev -y; \ - wget --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ - echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ - mkdir --parents sanjuuni; \ - tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz + apt-get update; \ + apt-get install -y --no-install-recommends \ + ocl-icd-opencl-dev=2.2.14-3 \ + wget=1.21.2-2ubuntu1.1 \ + clang=1:14.0-55~exp2 \ + make=4.3-4.1build1 \ + libpoco-dev=1.11.0-3; \ + wget --progress=dot:giga --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ + echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ + mkdir --parents sanjuuni; \ + tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz WORKDIR /sanjuuni RUN set -eux; \ - ./configure; \ - make + ./configure; \ + make FROM ffmpeg +WORKDIR /youcube + COPY --from=sanjuuni /sanjuuni/sanjuuni /usr/local/bin COPY requirements.txt . COPY youcube /youcube -RUN set -eux; \ - apt-get update; \ - apt-get install \ - libpoco-dev \ - python3-pip \ - gnupg \ - curl -y; \ - pip install --no-cache-dir -r requirements.txt; \ - curl -s -L https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key add -; \ - distribution=$(. /etc/os-release;echo $ID$VERSION_ID); \ - curl -s -L https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list; \ - apt-get update; \ - apt-get install \ - nvidia-opencl-dev \ - nvidia-container-runtime -y +SHELL ["/bin/bash", "-o", "pipefail", "-c"] -WORKDIR /youcube +# hadolint ignore=SC1091 +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libpoco-dev=1.11.0-3 \ + python3-pip=22.0.2+dfsg-1 \ + ocl-icd-libopencl1; \ + pip install -U pip; \ + pip install --no-cache-dir -r requirements.txt; \ + mkdir -p /etc/OpenCL/vendors; \ + echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +ENV \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility ENTRYPOINT ["python3", "youcube.py"] diff --git a/src/compile.py b/src/compile.py index 40a7c65..0ba1822 100644 --- a/src/compile.py +++ b/src/compile.py @@ -1,34 +1,29 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Compiles YC to pyc files -""" - -from py_compile import compile as py_compile -from pathlib import Path -from os.path import isdir, join -from os import rename - - -def main() -> None: - """Starts the compilation""" - blacklist = [ - "__main__.py" - ] - - for path in Path("youcube").rglob('*.py'): - if not isdir(path) and path.name not in blacklist: - compile_path = py_compile(path, optimize=2) - new_name = Path( - join( - Path(compile_path).parent, - path.name.replace(".py", ".pyc") - ) - ) - rename(compile_path, new_name) - print(path, "->", new_name) - - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Compiles YC to pyc files +""" + +from os import rename +from os.path import isdir, join +from pathlib import Path +from py_compile import compile as py_compile + + +def main() -> None: + """Starts the compilation""" + blacklist = ["__main__.py"] + + for path in Path("youcube").rglob("*.py"): + if not isdir(path) and path.name not in blacklist: + compile_path = py_compile(path, optimize=2) + new_name = Path( + join(Path(compile_path).parent, path.name.replace(".py", ".pyc")) + ) + rename(compile_path, new_name) + print(path, "->", new_name) + + +if __name__ == "__main__": + main() diff --git a/src/docker-compose.nvidia.yml b/src/docker-compose.nvidia.yml index f6c5c9a..79577e3 100644 --- a/src/docker-compose.nvidia.yml +++ b/src/docker-compose.nvidia.yml @@ -1,14 +1,14 @@ -version: "2.0" -services: - youcube: - build: . - image: youcube:nvidia - restart: always - hostname: youcube - ports: - - 5000:5000 - #env_file: .env - runtime: nvidia - environment: - - NVIDIA_VISIBLE_DEVICES=all - - NVIDIA_DRIVER_CAPABILITIES=compute +--- +services: + youcube: + build: + context: . + dockerfile: Dockerfile.nvidia + image: youcube:nvidia + restart: always + hostname: youcube + ports: + - 5000:5000 + #env_file: .env + runtime: nvidia +... diff --git a/src/docker-compose.yml b/src/docker-compose.yml index 5ec66e6..9e657ea 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -1,10 +1,11 @@ -version: "2.0" -services: - youcube: - build: . - image: youcube - restart: always - hostname: youcube - ports: - - 5000:5000 - #env_file: .env +--- +services: + youcube: + build: . + image: youcube + restart: always + hostname: youcube + ports: + - 5000:5000 + #env_file: .env +... diff --git a/src/requirements.txt b/src/requirements.txt index 991f68a..28a0ad7 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,5 +1,5 @@ -sanic~=23.12.1 -#uvloop~=0.19.0; platform_system != "Windows" -yt-dlp~=2023.12.30 -ujson~=5.9.0 -spotipy~=2.23.0 +sanic~=24.6.0 +#uvloop~=0.19.0; platform_system != "Windows" +yt-dlp~=2024.7.2 +#orjson~=3.10.6 +spotipy~=2.24.0 diff --git a/src/youcube/__main__.py b/src/youcube/__main__.py index 08dd398..ae5c730 100644 --- a/src/youcube/__main__.py +++ b/src/youcube/__main__.py @@ -1,12 +1,12 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Runs the main function -""" - -# Built-in modules -from youcube import main - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Runs the main function +""" + +# Built-in modules +from youcube import main + +if __name__ == "__main__": + main() diff --git a/src/youcube/yc_colours.py b/src/youcube/yc_colours.py index 359a990..1fb2b1f 100644 --- a/src/youcube/yc_colours.py +++ b/src/youcube/yc_colours.py @@ -1,37 +1,38 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Colors using ANSI escape codes -https://en.wikipedia.org/wiki/ANSI_escape_code -""" - -# pylint: disable=too-few-public-methods - - -class Foreground: - """ - [3-bit and 4-bit](https://en.wikipedia.org/wiki/ANSI_escape_code#3-bit_and_4-bit) - """ - BLACK = "\033[30m" - RED = "\033[31m" - GREEN = "\033[32m" - YELLOW = "\033[33m" - BLUE = "\033[34m" - MAGENTA = "\033[35m" - CYAN = "\033[36m" - WHITE = "\033[37m" - - BRIGHT_BLACK = "\033[90m" - BRIGHT_RED = "\033[91m" - BRIGHT_GREEN = "\033[92m" - BRIGHT_YELLOW = "\033[93m" - BRIGHT_BLUE = "\033[94m" - BRIGHT_MAGENTA = "\033[95m" - BRIGHT_CYAN = "\033[96m" - BRIGHT_WHITE = "\033[97m" - - DEFAULT = "\033[39m" - - -RESET = "\033[m" +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Colors using ANSI escape codes +https://en.wikipedia.org/wiki/ANSI_escape_code +""" + +# pylint: disable=too-few-public-methods + + +class Foreground: + """ + [3-bit and 4-bit](https://en.wikipedia.org/wiki/ANSI_escape_code#3-bit_and_4-bit) + """ + + BLACK = "\033[30m" + RED = "\033[31m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + BLUE = "\033[34m" + MAGENTA = "\033[35m" + CYAN = "\033[36m" + WHITE = "\033[37m" + + BRIGHT_BLACK = "\033[90m" + BRIGHT_RED = "\033[91m" + BRIGHT_GREEN = "\033[92m" + BRIGHT_YELLOW = "\033[93m" + BRIGHT_BLUE = "\033[94m" + BRIGHT_MAGENTA = "\033[95m" + BRIGHT_CYAN = "\033[96m" + BRIGHT_WHITE = "\033[97m" + + DEFAULT = "\033[39m" + + +RESET = "\033[m" diff --git a/src/youcube/yc_download.py b/src/youcube/yc_download.py index d9a83fd..a2e0206 100644 --- a/src/youcube/yc_download.py +++ b/src/youcube/yc_download.py @@ -6,36 +6,36 @@ """ # Built-in modules -from tempfile import TemporaryDirectory from asyncio import run_coroutine_threadsafe -from os import listdir, getenv -from os.path import join, dirname, abspath +from os import getenv, listdir +from os.path import abspath, dirname, join +from tempfile import TemporaryDirectory # Local modules -#TODO: change sanic logging format -from yc_logging import YTDLPLogger, logger, NO_COLOR +from yc_colours import RESET, Foreground +from yc_logging import NO_COLOR, YTDLPLogger, logger from yc_magic import run_with_live_output -from yc_colours import Foreground, RESET from yc_spotify import SpotifyURLProcessor from yc_utils import ( - remove_ansi_escape_codes, - remove_whitespace, cap_width_and_height, create_data_folder_if_not_present, + get_audio_name, + get_video_name, is_audio_already_downloaded, is_video_already_downloaded, - get_audio_name, - get_video_name + remove_ansi_escape_codes, + remove_whitespace, ) +# optional pip modules try: - from ujson import dumps + from orjson import dumps except ModuleNotFoundError: from json import dumps # pip modules -from yt_dlp import YoutubeDL from sanic import Websocket +from yt_dlp import YoutubeDL # pylint settings # pylint: disable=pointless-string-statement @@ -51,20 +51,17 @@ def download_video( - temp_dir: str, - media_id: str, - resp: Websocket, - loop, - width: int, - height: int + temp_dir: str, media_id: str, resp: Websocket, loop, width: int, height: int ): """ Converts the downloaded video to 32vid """ - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Converting video to 32vid ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps({"action": "status", "message": "Converting video to 32vid ..."}) + ), + loop, + ) if NO_COLOR: prefix = "[Sanjuuni]" @@ -73,43 +70,43 @@ def download_video( def handler(line): logger.debug("%s%s", prefix, line) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": line - })), loop) + run_coroutine_threadsafe( + resp.send(dumps({"action": "status", "message": line})), loop + ) returncode = run_with_live_output( [ SANJUUNI_PATH, "--width=" + str(width), "--height=" + str(height), - "-i", join(temp_dir, listdir(temp_dir)[0]), + "-i", + join(temp_dir, listdir(temp_dir)[0]), "--raw", - "-o", join( - DATA_FOLDER, - get_video_name(media_id, width, height) - ), - "--disable-opencl" if DISABLE_OPENCL else "" + "-o", + join(DATA_FOLDER, get_video_name(media_id, width, height)), + "--disable-opencl" if DISABLE_OPENCL else "", ], - handler + handler, ) if returncode != 0: logger.warning("Sanjuuni exited with %s", returncode) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "error", - "message": "Faild to convert video!" - })), loop) + run_coroutine_threadsafe( + resp.send(dumps({"action": "error", "message": "Faild to convert video!"})), + loop, + ) def download_audio(temp_dir: str, media_id: str, resp: Websocket, loop): """ Converts the downloaded audio to dfpwm """ - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Converting audio to dfpwm ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps({"action": "status", "message": "Converting audio to dfpwm ..."}) + ), + loop, + ) if NO_COLOR: prefix = "[FFmpeg]" @@ -123,21 +120,25 @@ def handler(line): returncode = run_with_live_output( [ FFMPEG_PATH, - "-i", join(temp_dir, listdir(temp_dir)[0]), - "-f", "dfpwm", - "-ar", "48000", - "-ac", "1", - join(DATA_FOLDER, get_audio_name(media_id)) + "-i", + join(temp_dir, listdir(temp_dir)[0]), + "-f", + "dfpwm", + "-ar", + "48000", + "-ac", + "1", + join(DATA_FOLDER, get_audio_name(media_id)), ], - handler + handler, ) if returncode != 0: logger.warning("FFmpeg exited with %s", returncode) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "error", - "message": "Faild to convert audio!" - })), loop) + run_coroutine_threadsafe( + resp.send(dumps({"action": "error", "message": "Faild to convert audio!"})), + loop, + ) def download( @@ -146,7 +147,7 @@ def download( loop, width: int, height: int, - spotify_url_processor: SpotifyURLProcessor + spotify_url_processor: SpotifyURLProcessor, ) -> (dict[str, any], list): """ Downloads and converts the media from the give URL @@ -160,35 +161,44 @@ def download( def my_hook(info): """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" - if info.get('status') == "downloading": - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": remove_ansi_escape_codes( - f"download {remove_whitespace(info.get('_percent_str'))} " - f"ETA {info.get('_eta_str')}" - ) - })), loop) + if info.get("status") == "downloading": + run_coroutine_threadsafe( + resp.send( + dumps( + { + "action": "status", + "message": remove_ansi_escape_codes( + f"download {remove_whitespace(info.get('_percent_str'))} " + f"ETA {info.get('_eta_str')}" + ), + } + ) + ), + loop, + ) # FIXME: Cleanup on Exception with TemporaryDirectory(prefix="youcube-") as temp_dir: yt_dl_options = { - "format": - "worst[ext=mp4]/worst" if is_video else - "worstaudio/worst", + "format": "worst[ext=mp4]/worst" if is_video else "worstaudio/worst", "outtmpl": join(temp_dir, "%(id)s.%(ext)s"), "default_search": "auto", "restrictfilenames": True, "extract_flat": "in_playlist", "progress_hooks": [my_hook], - "logger": YTDLPLogger() + "logger": YTDLPLogger(), } yt_dl = YoutubeDL(yt_dl_options) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Getting resource information ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps( + {"action": "status", "message": "Getting resource information ..."} + ) + ), + loop, + ) playlist_videos = [] @@ -206,7 +216,7 @@ def my_hook(info): data = yt_dl.extract_info(url, download=False) if data.get("extractor") == "generic": - data["id"] = 'g' + data.get("webpage_url_domain") + data.get("id") + data["id"] = "g" + data.get("webpage_url_domain") + data.get("id") """ If the data is a playlist, we need to get the first video and return it, @@ -226,18 +236,14 @@ def my_hook(info): so we need to get missing information by running the extractor again. """ if data.get("extractor") == "youtube" and ( - data.get("view_count") is None or - data.get("like_count") is None + data.get("view_count") is None or data.get("like_count") is None ): data = yt_dl.extract_info(data.get("id"), download=False) media_id = data.get("id") if data.get("is_live"): - return { - "action": "error", - "message": "Livestreams are not supported" - } + return {"action": "error", "message": "Livestreams are not supported"} create_data_folder_if_not_present() @@ -245,10 +251,12 @@ def my_hook(info): video_downloaded = is_video_already_downloaded(media_id, width, height) if not audio_downloaded or (not video_downloaded and is_video): - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Downloading resource ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps({"action": "status", "message": "Downloading resource ..."}) + ), + loop, + ) yt_dl.process_ie_result(data, download=True) diff --git a/src/youcube/yc_logging.py b/src/youcube/yc_logging.py index e457ead..961d880 100644 --- a/src/youcube/yc_logging.py +++ b/src/youcube/yc_logging.py @@ -1,103 +1,103 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Everything logging related -""" - -# Built-in modules -from logging import ( - Formatter, - Logger, - StreamHandler, - LogRecord, - getLogger, - DEBUG, - INFO, - WARNING, - ERROR, - CRITICAL -) -from os import getenv -from yc_colours import Foreground, RESET - -LOGLEVEL = getenv("LOGLEVEL") or DEBUG -NO_COLOR = getenv("NO_COLOR") or False -# Don't call "getLogger" every time we need the logger -logger = getLogger("__main__") - - -class ColordFormatter(Formatter): - """Logging colored formatter, adapted from https://stackoverflow.com/a/56944256/3638629""" - - # noinspection SpellCheckingInspection - def __init__(self, fmt=None, datefmt="%H:%M:%S") -> None: - super().__init__() - self.fmt = fmt - self.datefmt = datefmt - self.formats = { - DEBUG: f"{Foreground.BRIGHT_BLACK}{self.fmt}{RESET}", - INFO: f"{Foreground.BRIGHT_WHITE}{self.fmt}{RESET}", - WARNING: f"{Foreground.BRIGHT_YELLOW}{self.fmt}{RESET}", - ERROR: f"{Foreground.BRIGHT_RED}{self.fmt}{RESET}", - CRITICAL: f"{Foreground.RED}{self.fmt}{RESET}" - } - - def format(self, record: LogRecord) -> str: - log_fmt = self.formats.get(record.levelno) - formatter = Formatter(log_fmt, datefmt=self.datefmt) - return formatter.format(record) - - -class YTDLPLogger: - """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" - - def __init__(self) -> None: - if NO_COLOR: - self.prefix = "[yt-dlp] " - else: - self.prefix = f"{Foreground.BRIGHT_MAGENTA}[yt-dlp]{RESET} " - - def debug(self, msg: str) -> None: - """Pass msg to the main logger""" - - # For compatibility with youtube-dl, both debug and info are passed into debug - # You can distinguish them by the prefix '[debug] ' - if msg.startswith('[debug] '): - pass - else: - self.info(msg) - - def info(self, msg: str) -> None: - """Pass msg to the main logger""" - logger.debug("%s%s", self.prefix, msg) - - def warning(self, msg: str) -> None: - """Pass msg to the main logger""" - logger.warning("%s%s", self.prefix, msg) - - def error(self, msg: str) -> None: - """Pass msg to the main logger""" - logger.error("%s%s", self.prefix, msg) - - -def setup_logging() -> Logger: - """Sets the main logger up""" - logger.setLevel(LOGLEVEL) - - # noinspection SpellCheckingInspection - if NO_COLOR: - formatter = Formatter( - fmt="[%(asctime)s %(levelname)s] [YouCube] %(message)s" - ) - else: - formatter = ColordFormatter( - # pylint: disable-next=line-too-long - fmt=f"[%(asctime)s %(levelname)s] {Foreground.BRIGHT_WHITE}[You{Foreground.RED}Cube]{RESET} %(message)s" - ) - - logging_handler = StreamHandler() - logging_handler.setFormatter(formatter) - logger.addHandler(logging_handler) - - return logger +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Everything logging related +""" + +# Built-in modules +from logging import ( + CRITICAL, + DEBUG, + ERROR, + INFO, + WARNING, + Formatter, + Logger, + LogRecord, + StreamHandler, + getLogger, +) +from os import getenv + +# local modules +from yc_colours import RESET, Foreground + +LOGLEVEL = getenv("LOGLEVEL") or DEBUG +NO_COLOR = getenv("NO_COLOR") or False +# Don't call "getLogger" every time we need the logger +logger = getLogger("__main__") + + +class ColordFormatter(Formatter): + """Logging colored formatter, adapted from https://stackoverflow.com/a/56944256/3638629""" + + # noinspection SpellCheckingInspection + def __init__(self, fmt=None, datefmt="%H:%M:%S") -> None: + super().__init__() + self.fmt = fmt + self.datefmt = datefmt + self.formats = { + DEBUG: f"{Foreground.BRIGHT_BLACK}{self.fmt}{RESET}", + INFO: f"{Foreground.BRIGHT_WHITE}{self.fmt}{RESET}", + WARNING: f"{Foreground.BRIGHT_YELLOW}{self.fmt}{RESET}", + ERROR: f"{Foreground.BRIGHT_RED}{self.fmt}{RESET}", + CRITICAL: f"{Foreground.RED}{self.fmt}{RESET}", + } + + def format(self, record: LogRecord) -> str: + log_fmt = self.formats.get(record.levelno) + formatter = Formatter(log_fmt, datefmt=self.datefmt) + return formatter.format(record) + + +class YTDLPLogger: + """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" + + def __init__(self) -> None: + if NO_COLOR: + self.prefix = "[yt-dlp] " + else: + self.prefix = f"{Foreground.BRIGHT_MAGENTA}[yt-dlp]{RESET} " + + def debug(self, msg: str) -> None: + """Pass msg to the main logger""" + + # For compatibility with youtube-dl, both debug and info are passed into debug + # You can distinguish them by the prefix '[debug] ' + if msg.startswith("[debug] "): + pass + else: + self.info(msg) + + def info(self, msg: str) -> None: + """Pass msg to the main logger""" + logger.debug("%s%s", self.prefix, msg) + + def warning(self, msg: str) -> None: + """Pass msg to the main logger""" + logger.warning("%s%s", self.prefix, msg) + + def error(self, msg: str) -> None: + """Pass msg to the main logger""" + logger.error("%s%s", self.prefix, msg) + + +def setup_logging() -> Logger: + """Sets the main logger up""" + logger.setLevel(LOGLEVEL) + + # noinspection SpellCheckingInspection + if NO_COLOR: + formatter = Formatter(fmt="[%(asctime)s %(levelname)s] [YouCube] %(message)s") + else: + formatter = ColordFormatter( + # pylint: disable-next=line-too-long + fmt=f"[%(asctime)s %(levelname)s] {Foreground.BRIGHT_WHITE}[You{Foreground.RED}Cube]{RESET} %(message)s" + ) + + logging_handler = StreamHandler() + logging_handler.setFormatter(formatter) + logger.addHandler(logging_handler) + + return logger diff --git a/src/youcube/yc_magic.py b/src/youcube/yc_magic.py index 99970c8..6d1678f 100644 --- a/src/youcube/yc_magic.py +++ b/src/youcube/yc_magic.py @@ -1,138 +1,132 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Black Magic with threads, asyncio and subprocesses -""" - -# Built-in modules -from typing import Any, Callable -from types import FrameType -from threading import Thread -from subprocess import Popen, PIPE -from asyncio import Event -from sys import settrace - - -class ThreadSaveAsyncioEventWithReturnValue(Event): - """ - Thread-save version of asyncio.Event with result / Return value - """ - - def __init__(self) -> None: - super().__init__() - self.result = None - - # pylint: disable-next=fixme - # TODO: clear() method - - def set(self): - # pylint: disable-next=fixme - # FIXME: The _loop attribute is not documented as public api! - self._loop.call_soon_threadsafe(super().set) - - -def run_with_thread_save_asyncio_event_with_return_value( - event: ThreadSaveAsyncioEventWithReturnValue, - func: Callable[[], Any], - *args -) -> None: - """ - Runs a function and calls a ThreadSaveAsyncioEventWithReturnValue - This function is meant to run in a thread - """ - result = func(*args) - event.result = result - event.set() - - -async def run_function_in_thread_from_async_function( - func: Callable[[], Any], - *args -) -> object: - """ - Runs a function in a thread from an async function - """ - event = ThreadSaveAsyncioEventWithReturnValue() - Thread( - target=run_with_thread_save_asyncio_event_with_return_value, - args=(event, func, *args) - ).start() - await event.wait() - return event.result - - -class KillableThread(Thread): - """ - A Thread that can be canceled by running kill on it - https://www.geeksforgeeks.org/python-different-ways-to-kill-a-thread/ - """ - - def __init__(self, *args, **keywords) -> None: - Thread.__init__(self, *args, **keywords) - self.killed = False - - def start(self) -> None: - # pylint: disable-next=attribute-defined-outside-init - self.__run_backup = self.run - self.run = self.__run - Thread.start(self) - - def __run(self) -> None: - settrace(self.globaltrace) - self.__run_backup() - self.run = self.__run_backup - - # pylint: disable-next=unused-argument - def globaltrace(self, frame: FrameType, event: str, arg: Any) -> None: - """ - Allows calling "localtrace" from global - """ - if event == 'call': - return self.localtrace - return None - - # pylint: disable-next=unused-argument - def localtrace(self, frame: FrameType, event: str, arg: Any) -> None: - """ - Uses trace to check if the Thread needs to be killed - """ - if self.killed and event == 'line': - raise SystemExit() - return self.localtrace - - def kill(self) -> None: - """Kills the Thread""" - self.killed = True - - -def run_with_live_output(cmd: list, handler: Callable[[str], None]) -> int: - """ - Runs a subprocess and allows handling output live - """ - with Popen( - cmd, - stdout=PIPE, - stderr=PIPE - ) as process: - - def live_output(): - line = [] - while True: - read = process.stderr.read(1) - if read in (b"\r", b"\n"): # handle \n and \r as new line characters - if len(line) != 0: # ignore empty line - handler("".join(line)) - line.clear() - else: - line.append(read.decode("utf-8")) - - thread = KillableThread(target=live_output) - thread.start() - - process.wait() - thread.kill() - - return process.returncode - -# pylint: disable=unused-argument +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Black Magic with threads, asyncio and subprocesses +""" + +# Built-in modules +from asyncio import Event +from subprocess import PIPE, Popen +from sys import settrace +from threading import Thread +from types import FrameType +from typing import Any, Callable + + +class ThreadSaveAsyncioEventWithReturnValue(Event): + """ + Thread-save version of asyncio.Event with result / Return value + """ + + def __init__(self) -> None: + super().__init__() + self.result = None + + # pylint: disable-next=fixme + # TODO: clear() method + + def set(self): + # pylint: disable-next=fixme + # FIXME: The _loop attribute is not documented as public api! + self._loop.call_soon_threadsafe(super().set) + + +def run_with_thread_save_asyncio_event_with_return_value( + event: ThreadSaveAsyncioEventWithReturnValue, func: Callable[[], Any], *args +) -> None: + """ + Runs a function and calls a ThreadSaveAsyncioEventWithReturnValue + This function is meant to run in a thread + """ + result = func(*args) + event.result = result + event.set() + + +async def run_function_in_thread_from_async_function( + func: Callable[[], Any], *args +) -> object: + """ + Runs a function in a thread from an async function + """ + event = ThreadSaveAsyncioEventWithReturnValue() + Thread( + target=run_with_thread_save_asyncio_event_with_return_value, + args=(event, func, *args), + ).start() + await event.wait() + return event.result + + +class KillableThread(Thread): + """ + A Thread that can be canceled by running kill on it + https://www.geeksforgeeks.org/python-different-ways-to-kill-a-thread/ + """ + + def __init__(self, *args, **keywords) -> None: + Thread.__init__(self, *args, **keywords) + self.killed = False + + def start(self) -> None: + # pylint: disable-next=attribute-defined-outside-init + self.__run_backup = self.run + self.run = self.__run + Thread.start(self) + + def __run(self) -> None: + settrace(self.globaltrace) + self.__run_backup() + self.run = self.__run_backup + + # pylint: disable-next=unused-argument + def globaltrace(self, frame: FrameType, event: str, arg: Any) -> None: + """ + Allows calling "localtrace" from global + """ + if event == "call": + return self.localtrace + return None + + # pylint: disable-next=unused-argument + def localtrace(self, frame: FrameType, event: str, arg: Any) -> None: + """ + Uses trace to check if the Thread needs to be killed + """ + if self.killed and event == "line": + raise SystemExit() + return self.localtrace + + def kill(self) -> None: + """Kills the Thread""" + self.killed = True + + +def run_with_live_output(cmd: list, handler: Callable[[str], None]) -> int: + """ + Runs a subprocess and allows handling output live + """ + with Popen(cmd, stdout=PIPE, stderr=PIPE) as process: + + def live_output(): + line = [] + while True: + read = process.stderr.read(1) + if read in (b"\r", b"\n"): # handle \n and \r as new line characters + if len(line) != 0: # ignore empty line + handler("".join(line)) + line.clear() + else: + line.append(read.decode("utf-8")) + + thread = KillableThread(target=live_output) + thread.start() + + process.wait() + thread.kill() + + return process.returncode + + +# pylint: disable=unused-argument diff --git a/src/youcube/yc_spotify.py b/src/youcube/yc_spotify.py index 402fc50..581df03 100644 --- a/src/youcube/yc_spotify.py +++ b/src/youcube/yc_spotify.py @@ -5,23 +5,23 @@ Spotify support module """ + # Built-in modules +from enum import Enum from logging import getLogger from os import getenv -from enum import Enum from re import match as re_match from typing import Union # pip modules -from spotipy import SpotifyClientCredentials, MemoryCacheHandler +from spotipy import MemoryCacheHandler, SpotifyClientCredentials from spotipy.client import Spotify -# https://github.com/spotipy-dev/spotipy/issues/1071 -_regex_spotify_url = r'^(http[s]?:\/\/)?open.spotify.com\/.*(?Ptrack|artist|album|playlist|show|episode|user)\/(?P[0-9A-Za-z]+)(\?.*)?$' # pylint: disable=missing-function-docstring # pylint: disable=missing-class-docstring + class SpotifyTypes(Enum): TRACK = "track" ARTIST = "artist" @@ -74,8 +74,7 @@ def spotify_artist(self, spotify_id: str) -> list: return playlist def spotify_show(self, spotify_id: str) -> list: - episodes = self.spotify.show_episodes( - spotify_id, market=self.spotify_market) + episodes = self.spotify.show_episodes(spotify_id, market=self.spotify_market) playlist = [] for track in episodes["items"]: @@ -108,13 +107,13 @@ def auto(self, url: str) -> Union[str, list]: SpotifyTypes.ARTIST: self.spotify_artist, SpotifyTypes.SHOW: self.spotify_show, SpotifyTypes.EPISODE: self.spotify_episode, - SpotifyTypes.USER: self.spotify_user + SpotifyTypes.USER: self.spotify_user, } # pylint: disable=protected-access for match in [ re_match(Spotify._regex_spotify_uri, url), - re_match(_regex_spotify_url, url) + re_match(Spotify._regex_spotify_url, url), ]: # pylint: enable=protected-access if match: @@ -142,7 +141,7 @@ def main() -> None: auth_manager=SpotifyClientCredentials( client_id=spotify_client_id, client_secret=spotify_client_secret, - cache_handler=MemoryCacheHandler() + cache_handler=MemoryCacheHandler(), ) ) else: @@ -157,15 +156,14 @@ def main() -> None: "https://open.spotify.com/episode/0UCTRy5frRHxD6SktX9dbV", "https://open.spotify.com/show/5fA3Ze7Ni75iXAEZaEkJIu", "https://open.spotify.com/user/besdkg6w64xf0rt713643tgvt", - "https://open.spotify.com/playlist/5UrcnHexRYVEprv5DJBPER" + "https://open.spotify.com/playlist/5UrcnHexRYVEprv5DJBPER", ] # pylint: disable-next=import-outside-toplevel from yc_colours import Foreground for url in test_urls: - print(Foreground.BLUE + url + Foreground.WHITE, - spotify_url_processor.auto(url)) + print(Foreground.BLUE + url + Foreground.WHITE, spotify_url_processor.auto(url)) if __name__ == "__main__": diff --git a/src/youcube/yc_utils.py b/src/youcube/yc_utils.py index 0034c04..fa9cd8f 100644 --- a/src/youcube/yc_utils.py +++ b/src/youcube/yc_utils.py @@ -1,106 +1,110 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Utils for string manipulation, data management etc. -""" - -# Built-in modules -from re import RegexFlag, compile as re_compile -from typing import Tuple -from os import mkdir -from os.path import join, dirname, abspath, exists - - -def remove_whitespace(string: str) -> str: - """ - Removes all Spaces / Whitespace from a string - """ - return string.replace(" ", "") - - -# Only compile "ansi_escape_codes" once -ansi_escape_codes = re_compile(r''' - \x1B # ESC - (?: # 7-bit C1 Fe (except CSI) - [@-Z\\-_] - | # or [ for CSI, followed by a control sequence - \[ - [0-?]* # Parameter bytes - [ -/]* # Intermediate bytes - [@-~] # Final byte - ) -''', RegexFlag.VERBOSE) - - -def remove_ansi_escape_codes(text: str) -> str: - """ - Remove all Ansi Escape codes - (7-bit C1 ANSI sequences) - """ - return ansi_escape_codes.sub('', text) - - -def cap_width(width: int) -> int: - """Caps the width""" - return min(width, 328) - - -def cap_height(height: int) -> int: - """Caps the height""" - return min(height, 243) - - -def cap_width_and_height(width: int, height: int) -> Tuple[int, int]: - """Caps the width and height""" - return cap_width(width), cap_height(height) - - -VIDEO_FORMAT = "32vid" -AUDIO_FORMAT = "dfpwm" -DATA_FOLDER = join(dirname(abspath(__file__)), "data") - - -def get_video_name(media_id: str, width: int, height: int) -> str: - """Returns the file name of the requested video""" - return f"{media_id}({width}x{height}).{VIDEO_FORMAT}" - - -def get_audio_name(media_id: str) -> str: - """Returns the file name of the requested audio""" - return f"{media_id}.{AUDIO_FORMAT}" - - -def get_video_path(media_id: str, width: int, height: int) -> str: - """Returns the relative path to the requested video""" - return join(DATA_FOLDER, get_video_name(media_id, width, height)) - - -def get_audio_path(media_id: str) -> str: - """Returns the relative path to the requested audio""" - return join(DATA_FOLDER, get_audio_name(media_id)) - - -def create_data_folder_if_not_present(): - """Creates the data folder if it does not exist""" - if not exists(DATA_FOLDER): - mkdir(DATA_FOLDER) - - -def is_audio_already_downloaded(media_id: str) -> bool: - """Returns True if the given audio is already downloaded""" - return exists(get_audio_path(media_id)) - - -def is_video_already_downloaded(media_id: str, width: int, height: int) -> bool: - """Returns True if the given video is already downloaded""" - return exists(get_video_path(media_id, width, height)) - - -# Only compile "allowed_characters" once -allowed_characters = re_compile('^[a-zA-Z0-9-._]*$') - - -def is_save(string: str) -> bool: - """Returns True if the given string does not contain special characters""" - return bool(allowed_characters.match(string)) +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Utils for string manipulation, data management etc. +""" + +# Built-in modules +from os import mkdir +from os.path import abspath, dirname, exists, join +from re import RegexFlag +from re import compile as re_compile +from typing import Tuple + + +def remove_whitespace(string: str) -> str: + """ + Removes all Spaces / Whitespace from a string + """ + return string.replace(" ", "") + + +# Only compile "ansi_escape_codes" once +ansi_escape_codes = re_compile( + r""" + \x1B # ESC + (?: # 7-bit C1 Fe (except CSI) + [@-Z\\-_] + | # or [ for CSI, followed by a control sequence + \[ + [0-?]* # Parameter bytes + [ -/]* # Intermediate bytes + [@-~] # Final byte + ) +""", + RegexFlag.VERBOSE, +) + + +def remove_ansi_escape_codes(text: str) -> str: + """ + Remove all Ansi Escape codes + (7-bit C1 ANSI sequences) + """ + return ansi_escape_codes.sub("", text) + + +def cap_width(width: int) -> int: + """Caps the width""" + return min(width, 328) + + +def cap_height(height: int) -> int: + """Caps the height""" + return min(height, 243) + + +def cap_width_and_height(width: int, height: int) -> Tuple[int, int]: + """Caps the width and height""" + return cap_width(width), cap_height(height) + + +VIDEO_FORMAT = "32vid" +AUDIO_FORMAT = "dfpwm" +DATA_FOLDER = join(dirname(abspath(__file__)), "data") + + +def get_video_name(media_id: str, width: int, height: int) -> str: + """Returns the file name of the requested video""" + return f"{media_id}({width}x{height}).{VIDEO_FORMAT}" + + +def get_audio_name(media_id: str) -> str: + """Returns the file name of the requested audio""" + return f"{media_id}.{AUDIO_FORMAT}" + + +def get_video_path(media_id: str, width: int, height: int) -> str: + """Returns the relative path to the requested video""" + return join(DATA_FOLDER, get_video_name(media_id, width, height)) + + +def get_audio_path(media_id: str) -> str: + """Returns the relative path to the requested audio""" + return join(DATA_FOLDER, get_audio_name(media_id)) + + +def create_data_folder_if_not_present(): + """Creates the data folder if it does not exist""" + if not exists(DATA_FOLDER): + mkdir(DATA_FOLDER) + + +def is_audio_already_downloaded(media_id: str) -> bool: + """Returns True if the given audio is already downloaded""" + return exists(get_audio_path(media_id)) + + +def is_video_already_downloaded(media_id: str, width: int, height: int) -> bool: + """Returns True if the given video is already downloaded""" + return exists(get_video_path(media_id, width, height)) + + +# Only compile "allowed_characters" once +allowed_characters = re_compile("^[a-zA-Z0-9-._]*$") + + +def is_save(string: str) -> bool: + """Returns True if the given string does not contain special characters""" + return bool(allowed_characters.match(string)) diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index 6aa3fd3..047d430 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -6,31 +6,24 @@ """ # built-in modules -from datetime import datetime -from time import sleep -from os.path import join, exists -from os import getenv, remove from asyncio import get_event_loop -from typing import ( - Union, - Tuple, - Type, - List, - Any -) from base64 import b64encode -from shutil import which +from datetime import datetime from multiprocessing import Manager +from os import getenv, remove +from os.path import exists, join +from shutil import which +from time import sleep +from typing import Any, List, Tuple, Type, Union +# optional pip module try: - from ujson import ( - JSONDecodeError, - dumps, - loads as load_json - ) + from orjson import JSONDecodeError, dumps + from orjson import loads as load_json except ModuleNotFoundError: + from json import dumps + from json import loads as load_json from json.decoder import JSONDecodeError - from json import dumps, loads as load_json try: from types import UnionType @@ -39,29 +32,21 @@ # pip modules -from sanic import ( - Sanic, - Request, - Websocket -) -from sanic.response import text -from sanic.handlers import ErrorHandler +from sanic import Request, Sanic, Websocket +from sanic.compat import open_async from sanic.exceptions import SanicException -from spotipy import SpotifyClientCredentials, MemoryCacheHandler +from sanic.handlers import ErrorHandler +from sanic.response import raw, text +from spotipy import MemoryCacheHandler, SpotifyClientCredentials from spotipy.client import Spotify # local modules -from yc_utils import ( - is_save, - cap_width_and_height, - get_video_name, - get_audio_name -) -from yc_colours import Foreground, RESET -from yc_download import download, DATA_FOLDER, FFMPEG_PATH, SANJUUNI_PATH +from yc_colours import RESET, Foreground +from yc_download import DATA_FOLDER, FFMPEG_PATH, SANJUUNI_PATH, download +from yc_logging import NO_COLOR, setup_logging from yc_magic import run_function_in_thread_from_async_function -from yc_logging import setup_logging, NO_COLOR from yc_spotify import SpotifyURLProcessor +from yc_utils import cap_width_and_height, get_audio_name, get_video_name, is_save VERSION = "0.0.0-poc.1.0.2" API_VERSION = "0.0.0-poc.1.0.0" # https://commandcracker.github.io/YouCube/ @@ -86,33 +71,84 @@ # pylint: disable=fixme # pylint: disable=multiple-statements +""" +Ubuntu nvida support fix and maby alpine support ? +us async base64 ? +use HTTP (and Streaming) +Add uvloop support https://github.com/CC-YouCube/server/issues/6 +""" + +""" +1 dfpwm chunk = 16 +MAX_DOWNLOAD = 16 * 1024 * 1024 = 16777216 +WEBSOCKET_MESSAGE = 128 * 1024 = 131072 +(MAX_DOWNLOAD = 128 * WEBSOCKET_MESSAGE) + +the speaker can accept a maximum of 128 x 1024 samples 16KiB + +playAudio +This accepts a list of audio samples as amplitudes between -128 and 127. +These are stored in an internal buffer and played back at 48kHz. +If this buffer is full, this function will return false. +""" + +"""Related CC-Tweaked issues +Streaming HTTP response https://github.com/cc-tweaked/CC-Tweaked/issues/1181 +Speaker Networks https://github.com/cc-tweaked/CC-Tweaked/issues/1488 +Pocket computers do not have many usecases without network access +https://github.com/cc-tweaked/CC-Tweaked/issues/1406 +Speaker limit to 8 https://github.com/cc-tweaked/CC-Tweaked/issues/1313 +Some way to notify player through pocket computer with modem +https://github.com/cc-tweaked/CC-Tweaked/issues/1148 +Memory limits for computers https://github.com/cc-tweaked/CC-Tweaked/issues/1580 +""" + +"""TODO: Add those: +AudioDevices: + - Speaker Note (Sound) https://tweaked.cc/peripheral/speaker.html + - Notblock https://www.youtube.com/watch?v=XY5UvTxD9dA + - Create Steam whistles https://www.youtube.com/watch?v=dgZ4F7U19do + https://github.com/danielathome19/MIDIToComputerCraft/tree/master + +Video Formats: + - 32vid binary https://github.com/MCJack123/sanjuuni + - qtv https://github.com/Axisok/qtccv + +Audio Formats: + - DFPWM ffmpeg fallback ? https://github.com/asiekierka/pixmess/blob/master/scraps/aucmp.py + - PCM + - NBS https://github.com/Xella37/NBS-Tunes-CC + - MIDI https://github.com/OpenPrograms/Sangar-Programs/blob/master/midi.lua + - XM https://github.com/MCJack123/tracc + +Audio u. Video preview / thumbnail: + - NFP https://tweaked.cc/library/cc.image.nft.html + - bimg https://github.com/SkyTheCodeMaster/bimg + - as 1 qtv frame + - as 1 32vid frame +""" + logger = setup_logging() +# TODO: change sanic logging format -def get_vid(vid_file: str, tracker: int) -> List[str]: - """ - Returns given line of 32vid file - """ - with open(vid_file, "r", encoding="utf-8") as file: - file.seek(tracker) +async def get_vid(vid_file: str, tracker: int) -> List[str]: + """Returns given line of 32vid file""" + async with await open_async(file=vid_file, mode="r", encoding="utf-8") as file: + await file.seek(tracker) lines = [] for _unused in range(FRAMES_AT_ONCE): - lines.append(file.readline()[:-1]) # remove \n - file.close() + lines.append((await file.readline())[:-1]) # remove \n return lines -def get_chunk(media_file: str, chunkindex: int) -> bytes: - """ - Returns a chunk of the given media file - """ - with open(media_file, "rb") as file: - file.seek(chunkindex * CHUNKS_AT_ONCE) - chunk = file.read(CHUNKS_AT_ONCE) - file.close() +async def getchunk(media_file: str, chunkindex: int) -> bytes: + """Returns a chunk of the given media file""" + async with await open_async(file=media_file, mode="rb") as file: + await file.seek(chunkindex * CHUNKS_AT_ONCE) + return await file.read(CHUNKS_AT_ONCE) - return chunk # pylint: enable=redefined-outer-name @@ -121,16 +157,8 @@ def assert_resp( __obj_name: str, __obj: Any, __class_or_tuple: Union[ - Type, UnionType, - Tuple[ - Union[ - Type, - UnionType, - Tuple[Any, ...] - ], - ... - ] - ] + Type, UnionType, Tuple[Union[Type, UnionType, Tuple[Any, ...]], ...] + ], ) -> Union[dict, None]: """ "assert" / isinstance that returns a dict that can be send as a ws response @@ -138,7 +166,7 @@ def assert_resp( if not isinstance(__obj, __class_or_tuple): return { "action": "error", - "message": f"{__obj_name} must be a {__class_or_tuple.__name__}" + "message": f"{__obj_name} must be a {__class_or_tuple.__name__}", } return None @@ -154,7 +182,7 @@ def assert_resp( auth_manager=SpotifyClientCredentials( client_id=spotify_client_id, client_secret=spotify_client_secret, - cache_handler=MemoryCacheHandler() + cache_handler=MemoryCacheHandler(), ) ) @@ -179,7 +207,8 @@ async def request_media(message: dict, resp: Websocket, request: Request): loop = get_event_loop() # get "url" url = message.get("url") - if error := assert_resp("url", url, str): return error + if error := assert_resp("url", url, str): + return error # TODO: assert_resp width and height out, files = await run_function_in_thread_from_async_function( download, @@ -188,7 +217,7 @@ async def request_media(message: dict, resp: Websocket, request: Request): loop, message.get("width"), message.get("height"), - spotify_url_processor + spotify_url_processor, ) for file in files: request.app.shared_ctx.data[file] = datetime.now() @@ -198,102 +227,77 @@ async def request_media(message: dict, resp: Websocket, request: Request): async def get_chunk(message: dict, _unused, request: Request): # get "chunkindex" chunkindex = message.get("chunkindex") - if error := assert_resp("chunkindex", chunkindex, int): return error + if error := assert_resp("chunkindex", chunkindex, int): + return error # get "id" media_id = message.get("id") - if error := assert_resp("media_id", media_id, str): return error + if error := assert_resp("media_id", media_id, str): + return error if is_save(media_id): file_name = get_audio_name(message.get("id")) - file = join( - DATA_FOLDER, - file_name - ) - + file = join(DATA_FOLDER, file_name) + request.app.shared_ctx.data[file_name] = datetime.now() - chunk = get_chunk(file, chunkindex) + chunk = await getchunk(file, chunkindex) - return { - "action": "chunk", - "chunk": b64encode(chunk).decode("ascii") - } + return {"action": "chunk", "chunk": b64encode(chunk).decode("ascii")} logger.warning("User tried to use special Characters") - return { - "action": "error", - "message": "You dare not use special Characters" - } + return {"action": "error", "message": "You dare not use special Characters"} @staticmethod async def get_vid(message: dict, _unused, request: Request): # get "line" tracker = message.get("tracker") - if error := assert_resp("tracker", tracker, int): return error + if error := assert_resp("tracker", tracker, int): + return error # get "id" media_id = message.get("id") - if error := assert_resp("id", media_id, str): return error + if error := assert_resp("id", media_id, str): + return error # get "width" - width = message.get('width') - if error := assert_resp("width", width, int): return error + width = message.get("width") + if error := assert_resp("width", width, int): + return error # get "height" - height = message.get('height') - if error := assert_resp("height", height, int): return error + height = message.get("height") + if error := assert_resp("height", height, int): + return error # cap height and width width, height = cap_width_and_height(width, height) if is_save(media_id): - file_name = get_video_name(message.get('id'), width, height) - file = join( - DATA_FOLDER, - file_name - ) - + file_name = get_video_name(message.get("id"), width, height) + file = join(DATA_FOLDER, file_name) + request.app.shared_ctx.data[file_name] = datetime.now() - return { - "action": "vid", - "lines": get_vid(file, tracker) - } + return {"action": "vid", "lines": await get_vid(file, tracker)} - return { - "action": "error", - "message": "You dare not use special Characters" - } + return {"action": "error", "message": "You dare not use special Characters"} @staticmethod async def handshake(*_unused): return { "action": "handshake", - "server": { - "version": VERSION - }, - "api": { - "version": API_VERSION - }, - "capabilities": { - "video": [ - "32vid" - ], - "audio": [ - "dfpwm" - ] - } + "server": {"version": VERSION}, + "api": {"version": API_VERSION}, + "capabilities": {"video": ["32vid"], "audio": ["dfpwm"]}, } # pylint: enable=missing-function-docstring class CustomErrorHandler(ErrorHandler): - """ - Error handler for sanic - """ + """Error handler for sanic""" def default(self, request: Request, exception: Union[SanicException, Exception]): - ''' handles errors that have no error handlers assigned ''' + """handles errors that have no error handlers assigned""" if isinstance(exception, SanicException) and exception.status_code == 426: # TODO: Respond with nice html that tells the user how to install YC @@ -306,7 +310,7 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) return super().default(request, exception) -app = Sanic(__name__) +app = Sanic("youcube") app.error_handler = CustomErrorHandler() # FIXME: The Client is not Responsing to Websocket pings app.config.WEBSOCKET_PING_INTERVAL = 0 @@ -318,7 +322,7 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) # add all actions from default action set for method in dir(Actions): - if not method.startswith('__'): + if not method.startswith("__"): actions[method] = getattr(Actions, method) @@ -327,43 +331,108 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) def data_cache_cleaner(data: dict): + """ + Checks for outdated cache entries every DATA_CACHE_CLEANUP_INTERVAL (default 300) Seconds and + deletes them if they have not been used for DATA_CACHE_CLEANUP_AFTER (default 3600) Seconds. + """ try: while True: sleep(DATA_CACHE_CLEANUP_INTERVAL) for file_name, last_used in data.items(): - if (datetime.now() - last_used).total_seconds() > DATA_CACHE_CLEANUP_AFTER: + if ( + datetime.now() - last_used + ).total_seconds() > DATA_CACHE_CLEANUP_AFTER: file_path = join(DATA_FOLDER, file_name) if exists(file_path): remove(file_path) - logger.debug(f'Deleted "{file_name}"') + logger.debug('Deleted "%s"', file_name) data.pop(file_name) except KeyboardInterrupt: pass +# pylint: disable=redefined-outer-name @app.main_process_ready async def ready(app: Sanic, _): + """See https://sanic.dev/en/guide/basics/listeners.html""" if DATA_CACHE_CLEANUP_INTERVAL > 0 and DATA_CACHE_CLEANUP_AFTER > 0: - app.manager.manage("Data-Cache-Cleaner", data_cache_cleaner, {"data": app.shared_ctx.data}) + app.manager.manage( + "Data-Cache-Cleaner", data_cache_cleaner, {"data": app.shared_ctx.data} + ) + @app.main_process_start async def main_start(app: Sanic): - + """See https://sanic.dev/en/guide/basics/listeners.html""" app.shared_ctx.data = Manager().dict() - + if which(FFMPEG_PATH) is None: logger.warning("FFmpeg not found.") if which(SANJUUNI_PATH) is None: logger.warning("Sanjuuni not found.") - + if spotipy: logger.info("Spotipy Enabled") else: logger.info("Spotipy Disabled") +@app.route("/dfpwm//") +async def stream_dfpwm(_request: Request, media_id: str, chunkindex: int): + """WIP HTTP mode""" + return raw(await getchunk(join(DATA_FOLDER, get_audio_name(media_id)), chunkindex)) + + +@app.route("/32vid////") # , stream=True +async def stream_32vid( + _request: Request, media_id: str, width: int, height: int, tracker: int +): + """WIP HTTP mode""" + return raw( + "\n".join( + await get_vid(join(DATA_FOLDER, get_video_name(media_id, width, height)), tracker) + ) + ) + + +"""" +from sanic import response +@app.route("/dfpwm/") +async def stream_dfpwm(request: Request, id: str): + file_name = get_audio_name(id) + file = join(DATA_FOLDER, get_audio_name(id)) + return await response.file_stream( + file, + chunk_size=CHUNKS_AT_ONCE, + mime_type="application/metalink4+xml", + headers={ + "Content-Disposition": f'Attachment; filename="{file_name}"', + "Content-Type": "application/metalink4+xml", + }, + ) + +@app.route("/32vid///", stream=True) +async def stream_32vid(request: Request, id: str, width: int, height: int): + file_name = get_video_name(id, width, height) + file = join( + DATA_FOLDER, + file_name + ) + return await response.file_stream( + file, + chunk_size=10, + mime_type="application/metalink4+xml", + headers={ + "Content-Disposition": f'Attachment; filename="{file_name}"', + "Content-Type": "application/metalink4+xml", + }, + ) +""" +# pylint: enable=redefined-outer-name + + @app.websocket("/") # pylint: disable-next=invalid-name async def wshandler(request: Request, ws: Websocket): @@ -375,11 +444,7 @@ async def wshandler(request: Request, ws: Websocket): logger.info("%sConnected!", prefix) - logger.debug( - "%sMy headers are: %s", - prefix, - request.headers - ) + logger.debug("%sMy headers are: %s", prefix, request.headers) while True: message = await ws.recv() @@ -389,10 +454,7 @@ async def wshandler(request: Request, ws: Websocket): message: dict = load_json(message) except JSONDecodeError: logger.debug("%sFaild to parse Json", prefix) - await ws.send(dumps({ - "action": "error", - "message": "Faild to parse Json" - })) + await ws.send(dumps({"action": "error", "message": "Faild to parse Json"})) if message.get("action") in actions: response = await actions[message.get("action")](message, ws, request) @@ -404,10 +466,10 @@ def main() -> None: Run all needed services """ port = int(getenv("PORT", "5000")) - host = getenv("HOST", "0.0.0.0") + host = getenv("HOST", "127.0.0.1") fast = not getenv("NO_FAST") - app.run(host=host, port=port, fast=fast) + app.run(host=host, port=port, fast=fast, access_log=True) if __name__ == "__main__":