From e66f448bbd790d09f9fbe639495160602d844bbf Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Mon, 15 Jan 2024 14:56:20 +0100 Subject: [PATCH 1/8] Updated wordlist Added spotipy,dev --- spellcheck_wordlist.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spellcheck_wordlist.txt b/spellcheck_wordlist.txt index bced916..4499fed 100644 --- a/spellcheck_wordlist.txt +++ b/spellcheck_wordlist.txt @@ -9,12 +9,14 @@ Popen ffmpeg subprocess utils +spotipy # Web Development websocket YouCube Spotify html +dev # Programming Keywords env From c50f5847c0987aac64f8d8f83d9fe162b6a47fd3 Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Mon, 15 Jan 2024 15:12:45 +0100 Subject: [PATCH 2/8] make pylint happy --- src/youcube/yc_download.py | 1 - src/youcube/yc_spotify.py | 5 +++-- src/youcube/youcube.py | 23 +++++++++++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/youcube/yc_download.py b/src/youcube/yc_download.py index d9a83fd..0a5cfdd 100644 --- a/src/youcube/yc_download.py +++ b/src/youcube/yc_download.py @@ -12,7 +12,6 @@ from os.path import join, dirname, abspath # Local modules -#TODO: change sanic logging format from yc_logging import YTDLPLogger, logger, NO_COLOR from yc_magic import run_with_live_output from yc_colours import Foreground, RESET diff --git a/src/youcube/yc_spotify.py b/src/youcube/yc_spotify.py index 402fc50..5cb3884 100644 --- a/src/youcube/yc_spotify.py +++ b/src/youcube/yc_spotify.py @@ -17,7 +17,8 @@ from spotipy.client import Spotify # https://github.com/spotipy-dev/spotipy/issues/1071 -_regex_spotify_url = r'^(http[s]?:\/\/)?open.spotify.com\/.*(?Ptrack|artist|album|playlist|show|episode|user)\/(?P[0-9A-Za-z]+)(\?.*)?$' +# pylint: disable-next=line-too-long +REGEX_SPOTIFY_URL = r'^(http[s]?:\/\/)?open.spotify.com\/.*(?Ptrack|artist|album|playlist|show|episode|user)\/(?P[0-9A-Za-z]+)(\?.*)?$' # pylint: disable=missing-function-docstring # pylint: disable=missing-class-docstring @@ -114,7 +115,7 @@ def auto(self, url: str) -> Union[str, list]: # pylint: disable=protected-access for match in [ re_match(Spotify._regex_spotify_uri, url), - re_match(_regex_spotify_url, url) + re_match(REGEX_SPOTIFY_URL, url) ]: # pylint: enable=protected-access if match: diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index 6aa3fd3..d924217 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -87,6 +87,7 @@ # pylint: disable=multiple-statements logger = setup_logging() +#TODO: change sanic logging format def get_vid(vid_file: str, tracker: int) -> List[str]: @@ -210,7 +211,7 @@ async def get_chunk(message: dict, _unused, request: Request): DATA_FOLDER, file_name ) - + request.app.shared_ctx.data[file_name] = datetime.now() chunk = get_chunk(file, chunkindex) @@ -251,7 +252,7 @@ async def get_vid(message: dict, _unused, request: Request): DATA_FOLDER, file_name ) - + request.app.shared_ctx.data[file_name] = datetime.now() return { @@ -327,6 +328,10 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) def data_cache_cleaner(data: dict): + """ + Checks for outdated cache entries every DATA_CACHE_CLEANUP_INTERVAL (default 300) Seconds and + deletes them if they have not been used for DATA_CACHE_CLEANUP_AFTER (default 3600) Seconds. + """ try: while True: sleep(DATA_CACHE_CLEANUP_INTERVAL) @@ -335,35 +340,41 @@ def data_cache_cleaner(data: dict): file_path = join(DATA_FOLDER, file_name) if exists(file_path): remove(file_path) - logger.debug(f'Deleted "{file_name}"') + logger.debug('Deleted "%s"', file_name) data.pop(file_name) except KeyboardInterrupt: pass +# pylint: disable=redefined-outer-name @app.main_process_ready async def ready(app: Sanic, _): + """See https://sanic.dev/en/guide/basics/listeners.html""" if DATA_CACHE_CLEANUP_INTERVAL > 0 and DATA_CACHE_CLEANUP_AFTER > 0: app.manager.manage("Data-Cache-Cleaner", data_cache_cleaner, {"data": app.shared_ctx.data}) + @app.main_process_start async def main_start(app: Sanic): - + """See https://sanic.dev/en/guide/basics/listeners.html""" app.shared_ctx.data = Manager().dict() - + if which(FFMPEG_PATH) is None: logger.warning("FFmpeg not found.") if which(SANJUUNI_PATH) is None: logger.warning("Sanjuuni not found.") - + if spotipy: logger.info("Spotipy Enabled") else: logger.info("Spotipy Disabled") +# pylint: enable=redefined-outer-name + + @app.websocket("/") # pylint: disable-next=invalid-name async def wshandler(request: Request, ws: Websocket): From 848fa63d38f014874dd06536590bf69bbe24fb71 Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Wed, 24 Jan 2024 17:20:06 +0100 Subject: [PATCH 3/8] Things will change :D --- README.md | 4 +- pyproject.toml | 81 +++++---- src/Dockerfile | 70 ++++---- src/Dockerfile.nvidia | 66 +++---- src/compile.py | 64 ++++--- src/docker-compose.nvidia.yml | 28 +-- src/docker-compose.yml | 20 +-- src/youcube/__main__.py | 25 +-- src/youcube/yc_colours.py | 76 ++++---- src/youcube/yc_download.py | 164 +++++++++--------- src/youcube/yc_logging.py | 207 +++++++++++----------- src/youcube/yc_magic.py | 271 ++++++++++++++--------------- src/youcube/yc_spotify.py | 23 +-- src/youcube/yc_utils.py | 217 +++++++++++------------ src/youcube/youcube.py | 315 +++++++++++++++++++--------------- 15 files changed, 854 insertions(+), 777 deletions(-) diff --git a/README.md b/README.md index d77aae8..222ad8d 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ python src/youcube.py Environment variables you can use to configure the server: | Variable | Default | Description | -|-------------------------------|------------|--------------------------------------------------------------------------------------------------------------------| +| ----------------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------ | | `HOST` | `0.0.0.0` | The host where the web server runs on. | | `PORT` | `5000` | The port where the web server should run on | | `FFMPEG_PATH` | `ffmpeg` | Path to the FFmpeg executable | @@ -79,4 +79,4 @@ services: [preview]: .README/preview-server.png [Python Version: 3.7+]: https://img.shields.io/badge/Python-3.7+-green?style=for-the-badge&logo=Python&logoColor=white [Python Lint Workflow Status]: https://img.shields.io/github/actions/workflow/status/CC-YouCube/server/pylint.yml?branch=main&label=Python%20Lint&logo=github&style=for-the-badge -[Sanic Builtin values]: https://sanic.dev/en/guide/running/configuration.md#builtin-values \ No newline at end of file +[Sanic Builtin values]: https://sanic.dev/en/guide/running/configuration.md#builtin-values diff --git a/pyproject.toml b/pyproject.toml index 7169556..8afcc1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,41 +1,40 @@ -[build-system] -requires = ["setuptools", "setuptools-scm"] -build-backend = "setuptools.build_meta" - -[project] -name = "youcube-server" -version = "1.0.0" -authors = [ - {name = "Commandcracker"}, -] -description = "A server which provides a WebSocket API for YouCube clients" -readme = "README.md" -requires-python = ">=3.7" -keywords = ["youtube", "youcube", "computercraft", "minecraft"] -license = {text = "GPL-3.0"} -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Natural Language :: English", - "Topic :: Multimedia :: Video", - "Topic :: Multimedia :: Sound/Audio :: Players", -] -# Required for pulling dependencies from requirements.txt -dynamic = ["dependencies"] - -[project.urls] -Homepage = "https://youcube.madefor.cc" -Repository = "https://github.com/CC-YouCube/server" -Documentation = "https://youcube.madefor.cc/guides/server/installation/" - -[tool.setuptools.dynamic] -# Pull dependencies from requirements.txt -dependencies = {file = ["src/requirements.txt"]} - -[tool.setuptools.packages.find] -where = ["src"] # list of folders that contain the packages (["."] by default) -include = ["youcube*"] # package names should match these glob patterns (["*"] by default) - -[tool.autopep8] -ignore = "E701" - +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "youcube-server" +version = "1.0.0" +authors = [{ name = "Commandcracker" }] +description = "A server which provides a WebSocket API for YouCube clients" +readme = "README.md" +requires-python = ">=3.7" +keywords = ["youtube", "youcube", "computercraft", "minecraft"] +license = { text = "GPL-3.0" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Topic :: Multimedia :: Video", + "Topic :: Multimedia :: Sound/Audio :: Players", +] +# Required for pulling dependencies from requirements.txt +dynamic = ["dependencies"] + +[project.urls] +Homepage = "https://youcube.madefor.cc" +Repository = "https://github.com/CC-YouCube/server" +Documentation = "https://youcube.madefor.cc/guides/server/installation/" + +[tool.setuptools.dynamic] +# Pull dependencies from requirements.txt +dependencies = { file = ["src/requirements.txt"] } + +[tool.setuptools.packages.find] +where = ["src"] # list of folders that contain the packages (["."] by default) +include = [ + "youcube*", +] # package names should match these glob patterns (["*"] by default) + +[tool.autopep8] +ignore = "E701" diff --git a/src/Dockerfile b/src/Dockerfile index ca41a7d..a33b722 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -6,46 +6,58 @@ ENV SANJUUNI_VERSION=ed34c266cd489e4db796ca803e247b48b43853e0 ARG SANJUUNI_SHA512SUM="9e482e3b8f8885e8f458856f11d5ee4c27a0aa469b8c54abe1aef943f630ca27eb148c8779ba7a053c1abcce298513e98b614747a77ae1c0cbc86a0a7c95a6d8 *sanjuuni.tar.gz" +SHELL ["/bin/ash", "-eo", "pipefail", "-c"] + RUN set -eux; \ - apk add --no-cache --update g++ zlib-dev poco-dev make; \ - wget --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ - echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ - mkdir --parents sanjuuni; \ - tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz; \ - rm sanjuuni.tar.gz; + apk add --no-cache --update \ + g++=13.2.1_git20231014-r0 \ + zlib-dev=1.3.1-r0 \ + poco-dev=1.12.4-r0 \ + make=4.4.1-r2; \ + wget -q --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ + echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ + mkdir --parents sanjuuni; \ + tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz; \ + rm sanjuuni.tar.gz; WORKDIR /sanjuuni RUN set -eux; \ - ./configure; \ - make + ./configure; \ + make FROM ghcr.io/commandcracker/alpine-pypy3.10-pip:3.19.0-pypy-7.3.14-pip-23.3.2 AS builder +WORKDIR / + COPY requirements.txt . COPY youcube ./youcube COPY compile.py . RUN set -eux; \ - apk add --no-cache --update build-base; \ - pip install --no-cache-dir -U setuptools -r requirements.txt; \ - python3 compile.py; \ - pip uninstall pip -y + apk add --no-cache --update build-base=0.5-r3; \ + pip install --no-cache-dir -U setuptools -r requirements.txt; \ + python3 compile.py; \ + pip uninstall pip -y FROM alpine:3.19.0 WORKDIR /opt/server RUN set -eux; \ - apk add --no-cache --update \ - # pypy requirements - libffi libbz2 \ - # sanjuuni requirements - poco \ - # ffmpeg requirements - libgcc libstdc++ ca-certificates libgomp expat; \ - apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community libssl1.1 libcrypto1.1; \ - chown 1000 /opt/server/ + apk add --no-cache --update \ + # pypy requirements + libffi=3.4.4-r3 libbz2=1.0.8-r6 \ + # sanjuuni requirements + poco=1.12.4-r0 \ + # ffmpeg requirements + libgcc=13.2.1_git20231014-r \ + libstdc++=13.2.1_git20231014-r0 \ + ca-certificates=20230506-r0 \ + libgomp=13.2.1_git20231014-r0 \ + expat=2.5.0-r2; \ + apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community libssl1.1=1.1.1u-r1 libcrypto1.1=1.1.1u-r1; \ + chown 1000 /opt/server/ COPY --from=builder /opt/pypy /opt/pypy # add ffmpeg @@ -54,14 +66,14 @@ COPY --from=ffmpeg /usr/local /usr/local COPY --from=sanjuuni /sanjuuni/sanjuuni /usr/local/bin ENV \ - # Make sure we use the virtualenv: - PATH="/opt/pypy/bin:$PATH" \ - # Use ffmpeg libs - LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \ - # yt-dlp cache dir - XDG_CACHE_HOME="/opt/server/.yt-dlp-cache" \ - # FIXME: Add UVLOOP support for alpine pypy - SANIC_NO_UVLOOP=true + # Make sure we use the virtualenv: + PATH="/opt/pypy/bin:$PATH" \ + # Use ffmpeg libs + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \ + # yt-dlp cache dir + XDG_CACHE_HOME="/opt/server/.yt-dlp-cache" \ + # FIXME: Add UVLOOP support for alpine pypy + SANIC_NO_UVLOOP=true USER 1000 diff --git a/src/Dockerfile.nvidia b/src/Dockerfile.nvidia index 5dfc775..978ce49 100644 --- a/src/Dockerfile.nvidia +++ b/src/Dockerfile.nvidia @@ -6,48 +6,56 @@ ENV SANJUUNI_VERSION=0.4 ARG SANJUUNI_SHA512SUM="952a6c608d167f37faad53ee7f2e0de8090a02bf73b6455fae7c6b6f648dd6a188e7749fe26caeee85126b2a38d7391389c19afb0100e9962dc551188b9de6ae *sanjuuni.tar.gz" +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + RUN set -eux; \ - apt-get update; \ - apt-get install \ - ocl-icd-opencl-dev \ - wget \ - clang \ - make \ - libpoco-dev -y; \ - wget --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ - echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ - mkdir --parents sanjuuni; \ - tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz + apt-get update; \ + apt-get install \ + ocl-icd-opencl-dev=2.2.14-3 \ + wget=1.21.2-2ubuntu1 \ + clang=1:14.0-55~exp2 \ + make=4.3-4.1build1 \ + libpoco-dev=1.11.0-3 -y --no-install-recommends; \ + wget --progress=dot:giga --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ + echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ + mkdir --parents sanjuuni; \ + tar --extract --directory sanjuuni --strip-components=1 --file=sanjuuni.tar.gz WORKDIR /sanjuuni RUN set -eux; \ - ./configure; \ - make + ./configure; \ + make FROM ffmpeg +WORKDIR /youcube + COPY --from=sanjuuni /sanjuuni/sanjuuni /usr/local/bin COPY requirements.txt . COPY youcube /youcube -RUN set -eux; \ - apt-get update; \ - apt-get install \ - libpoco-dev \ - python3-pip \ - gnupg \ - curl -y; \ - pip install --no-cache-dir -r requirements.txt; \ - curl -s -L https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key add -; \ - distribution=$(. /etc/os-release;echo $ID$VERSION_ID); \ - curl -s -L https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list; \ - apt-get update; \ - apt-get install \ - nvidia-opencl-dev \ - nvidia-container-runtime -y +SHELL ["/bin/bash", "-o", "pipefail", "-c"] -WORKDIR /youcube +# hadolint ignore=SC1091 +RUN set -eux; \ + apt-get update; \ + apt-get install \ + libpoco-dev=1.11.0-3 \ + python3-pip=22.0.2+dfsg-1 \ + gnupg=2.2.27-3ubuntu2.1 \ + libcurl4=7.81.0-1 \ + curl=7.81.0-1 -y --no-install-recommends; \ + pip install --no-cache-dir -r requirements.txt; \ + curl -s -L "https://nvidia.github.io/nvidia-container-runtime/gpgkey" | apt-key add -; \ + distribution=$(. /etc/os-release;echo "$ID$VERSION_ID"); \ + curl -s -L "https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.list" | tee /etc/apt/sources.list.d/nvidia-container-runtime.list; \ + apt-get update; \ + apt-get install \ + nvidia-opencl-dev=11.5.1-1ubuntu1 \ + nvidia-container-runtime=3.13.0-1 -y --no-install-recommends; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/* ENTRYPOINT ["python3", "youcube.py"] diff --git a/src/compile.py b/src/compile.py index 40a7c65..0ef0a7b 100644 --- a/src/compile.py +++ b/src/compile.py @@ -1,34 +1,30 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Compiles YC to pyc files -""" - -from py_compile import compile as py_compile -from pathlib import Path -from os.path import isdir, join -from os import rename - - -def main() -> None: - """Starts the compilation""" - blacklist = [ - "__main__.py" - ] - - for path in Path("youcube").rglob('*.py'): - if not isdir(path) and path.name not in blacklist: - compile_path = py_compile(path, optimize=2) - new_name = Path( - join( - Path(compile_path).parent, - path.name.replace(".py", ".pyc") - ) - ) - rename(compile_path, new_name) - print(path, "->", new_name) - - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Compiles YC to pyc files +""" + +from os import rename +from os.path import isdir, join +from pathlib import Path +from py_compile import compile as py_compile + + +def main() -> None: + """Starts the compilation""" + blacklist = ["__main__.py"] + + for path in Path("youcube").rglob("*.py"): + if not isdir(path) and path.name not in blacklist: + compile_path = py_compile(path, optimize=2) + new_name = Path( + join(Path(compile_path).parent, path.name.replace(".py", ".pyc")) + ) + rename(compile_path, new_name) + print(path, "->", new_name) + + +if __name__ == "__main__": + main() + diff --git a/src/docker-compose.nvidia.yml b/src/docker-compose.nvidia.yml index f6c5c9a..f2d2d38 100644 --- a/src/docker-compose.nvidia.yml +++ b/src/docker-compose.nvidia.yml @@ -1,14 +1,14 @@ -version: "2.0" -services: - youcube: - build: . - image: youcube:nvidia - restart: always - hostname: youcube - ports: - - 5000:5000 - #env_file: .env - runtime: nvidia - environment: - - NVIDIA_VISIBLE_DEVICES=all - - NVIDIA_DRIVER_CAPABILITIES=compute +version: "2.0" +services: + youcube: + build: . + image: youcube:nvidia + restart: always + hostname: youcube + ports: + - 5000:5000 + #env_file: .env + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute diff --git a/src/docker-compose.yml b/src/docker-compose.yml index 5ec66e6..10fc378 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -1,10 +1,10 @@ -version: "2.0" -services: - youcube: - build: . - image: youcube - restart: always - hostname: youcube - ports: - - 5000:5000 - #env_file: .env +version: "2.0" +services: + youcube: + build: . + image: youcube + restart: always + hostname: youcube + ports: + - 5000:5000 + #env_file: .env diff --git a/src/youcube/__main__.py b/src/youcube/__main__.py index 08dd398..384758c 100644 --- a/src/youcube/__main__.py +++ b/src/youcube/__main__.py @@ -1,12 +1,13 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Runs the main function -""" - -# Built-in modules -from youcube import main - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Runs the main function +""" + +# Built-in modules +from youcube import main + +if __name__ == "__main__": + main() + diff --git a/src/youcube/yc_colours.py b/src/youcube/yc_colours.py index 359a990..c375ed5 100644 --- a/src/youcube/yc_colours.py +++ b/src/youcube/yc_colours.py @@ -1,37 +1,39 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Colors using ANSI escape codes -https://en.wikipedia.org/wiki/ANSI_escape_code -""" - -# pylint: disable=too-few-public-methods - - -class Foreground: - """ - [3-bit and 4-bit](https://en.wikipedia.org/wiki/ANSI_escape_code#3-bit_and_4-bit) - """ - BLACK = "\033[30m" - RED = "\033[31m" - GREEN = "\033[32m" - YELLOW = "\033[33m" - BLUE = "\033[34m" - MAGENTA = "\033[35m" - CYAN = "\033[36m" - WHITE = "\033[37m" - - BRIGHT_BLACK = "\033[90m" - BRIGHT_RED = "\033[91m" - BRIGHT_GREEN = "\033[92m" - BRIGHT_YELLOW = "\033[93m" - BRIGHT_BLUE = "\033[94m" - BRIGHT_MAGENTA = "\033[95m" - BRIGHT_CYAN = "\033[96m" - BRIGHT_WHITE = "\033[97m" - - DEFAULT = "\033[39m" - - -RESET = "\033[m" +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Colors using ANSI escape codes +https://en.wikipedia.org/wiki/ANSI_escape_code +""" + +# pylint: disable=too-few-public-methods + + +class Foreground: + """ + [3-bit and 4-bit](https://en.wikipedia.org/wiki/ANSI_escape_code#3-bit_and_4-bit) + """ + + BLACK = "\033[30m" + RED = "\033[31m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + BLUE = "\033[34m" + MAGENTA = "\033[35m" + CYAN = "\033[36m" + WHITE = "\033[37m" + + BRIGHT_BLACK = "\033[90m" + BRIGHT_RED = "\033[91m" + BRIGHT_GREEN = "\033[92m" + BRIGHT_YELLOW = "\033[93m" + BRIGHT_BLUE = "\033[94m" + BRIGHT_MAGENTA = "\033[95m" + BRIGHT_CYAN = "\033[96m" + BRIGHT_WHITE = "\033[97m" + + DEFAULT = "\033[39m" + + +RESET = "\033[m" + diff --git a/src/youcube/yc_download.py b/src/youcube/yc_download.py index 0a5cfdd..5ec535d 100644 --- a/src/youcube/yc_download.py +++ b/src/youcube/yc_download.py @@ -6,35 +6,36 @@ """ # Built-in modules -from tempfile import TemporaryDirectory from asyncio import run_coroutine_threadsafe -from os import listdir, getenv -from os.path import join, dirname, abspath +from os import getenv, listdir +from os.path import abspath, dirname, join +from tempfile import TemporaryDirectory # Local modules -from yc_logging import YTDLPLogger, logger, NO_COLOR +from yc_colours import RESET, Foreground +from yc_logging import NO_COLOR, YTDLPLogger, logger from yc_magic import run_with_live_output -from yc_colours import Foreground, RESET from yc_spotify import SpotifyURLProcessor from yc_utils import ( - remove_ansi_escape_codes, - remove_whitespace, cap_width_and_height, create_data_folder_if_not_present, + get_audio_name, + get_video_name, is_audio_already_downloaded, is_video_already_downloaded, - get_audio_name, - get_video_name + remove_ansi_escape_codes, + remove_whitespace, ) +# optional pip modules try: from ujson import dumps except ModuleNotFoundError: from json import dumps # pip modules -from yt_dlp import YoutubeDL from sanic import Websocket +from yt_dlp import YoutubeDL # pylint settings # pylint: disable=pointless-string-statement @@ -50,20 +51,17 @@ def download_video( - temp_dir: str, - media_id: str, - resp: Websocket, - loop, - width: int, - height: int + temp_dir: str, media_id: str, resp: Websocket, loop, width: int, height: int ): """ Converts the downloaded video to 32vid """ - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Converting video to 32vid ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps({"action": "status", "message": "Converting video to 32vid ..."}) + ), + loop, + ) if NO_COLOR: prefix = "[Sanjuuni]" @@ -72,43 +70,43 @@ def download_video( def handler(line): logger.debug("%s%s", prefix, line) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": line - })), loop) + run_coroutine_threadsafe( + resp.send(dumps({"action": "status", "message": line})), loop + ) returncode = run_with_live_output( [ SANJUUNI_PATH, "--width=" + str(width), "--height=" + str(height), - "-i", join(temp_dir, listdir(temp_dir)[0]), + "-i", + join(temp_dir, listdir(temp_dir)[0]), "--raw", - "-o", join( - DATA_FOLDER, - get_video_name(media_id, width, height) - ), - "--disable-opencl" if DISABLE_OPENCL else "" + "-o", + join(DATA_FOLDER, get_video_name(media_id, width, height)), + "--disable-opencl" if DISABLE_OPENCL else "", ], - handler + handler, ) if returncode != 0: logger.warning("Sanjuuni exited with %s", returncode) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "error", - "message": "Faild to convert video!" - })), loop) + run_coroutine_threadsafe( + resp.send(dumps({"action": "error", "message": "Faild to convert video!"})), + loop, + ) def download_audio(temp_dir: str, media_id: str, resp: Websocket, loop): """ Converts the downloaded audio to dfpwm """ - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Converting audio to dfpwm ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps({"action": "status", "message": "Converting audio to dfpwm ..."}) + ), + loop, + ) if NO_COLOR: prefix = "[FFmpeg]" @@ -122,21 +120,25 @@ def handler(line): returncode = run_with_live_output( [ FFMPEG_PATH, - "-i", join(temp_dir, listdir(temp_dir)[0]), - "-f", "dfpwm", - "-ar", "48000", - "-ac", "1", - join(DATA_FOLDER, get_audio_name(media_id)) + "-i", + join(temp_dir, listdir(temp_dir)[0]), + "-f", + "dfpwm", + "-ar", + "48000", + "-ac", + "1", + join(DATA_FOLDER, get_audio_name(media_id)), ], - handler + handler, ) if returncode != 0: logger.warning("FFmpeg exited with %s", returncode) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "error", - "message": "Faild to convert audio!" - })), loop) + run_coroutine_threadsafe( + resp.send(dumps({"action": "error", "message": "Faild to convert audio!"})), + loop, + ) def download( @@ -145,7 +147,7 @@ def download( loop, width: int, height: int, - spotify_url_processor: SpotifyURLProcessor + spotify_url_processor: SpotifyURLProcessor, ) -> (dict[str, any], list): """ Downloads and converts the media from the give URL @@ -159,35 +161,44 @@ def download( def my_hook(info): """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" - if info.get('status') == "downloading": - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": remove_ansi_escape_codes( - f"download {remove_whitespace(info.get('_percent_str'))} " - f"ETA {info.get('_eta_str')}" - ) - })), loop) + if info.get("status") == "downloading": + run_coroutine_threadsafe( + resp.send( + dumps( + { + "action": "status", + "message": remove_ansi_escape_codes( + f"download {remove_whitespace(info.get('_percent_str'))} " + f"ETA {info.get('_eta_str')}" + ), + } + ) + ), + loop, + ) # FIXME: Cleanup on Exception with TemporaryDirectory(prefix="youcube-") as temp_dir: yt_dl_options = { - "format": - "worst[ext=mp4]/worst" if is_video else - "worstaudio/worst", + "format": "worst[ext=mp4]/worst" if is_video else "worstaudio/worst", "outtmpl": join(temp_dir, "%(id)s.%(ext)s"), "default_search": "auto", "restrictfilenames": True, "extract_flat": "in_playlist", "progress_hooks": [my_hook], - "logger": YTDLPLogger() + "logger": YTDLPLogger(), } yt_dl = YoutubeDL(yt_dl_options) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Getting resource information ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps( + {"action": "status", "message": "Getting resource information ..."} + ) + ), + loop, + ) playlist_videos = [] @@ -205,7 +216,7 @@ def my_hook(info): data = yt_dl.extract_info(url, download=False) if data.get("extractor") == "generic": - data["id"] = 'g' + data.get("webpage_url_domain") + data.get("id") + data["id"] = "g" + data.get("webpage_url_domain") + data.get("id") """ If the data is a playlist, we need to get the first video and return it, @@ -225,18 +236,14 @@ def my_hook(info): so we need to get missing information by running the extractor again. """ if data.get("extractor") == "youtube" and ( - data.get("view_count") is None or - data.get("like_count") is None + data.get("view_count") is None or data.get("like_count") is None ): data = yt_dl.extract_info(data.get("id"), download=False) media_id = data.get("id") if data.get("is_live"): - return { - "action": "error", - "message": "Livestreams are not supported" - } + return {"action": "error", "message": "Livestreams are not supported"} create_data_folder_if_not_present() @@ -244,10 +251,12 @@ def my_hook(info): video_downloaded = is_video_already_downloaded(media_id, width, height) if not audio_downloaded or (not video_downloaded and is_video): - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Downloading resource ..." - })), loop) + run_coroutine_threadsafe( + resp.send( + dumps({"action": "status", "message": "Downloading resource ..."}) + ), + loop, + ) yt_dl.process_ie_result(data, download=True) @@ -284,3 +293,4 @@ def my_hook(info): files.append(get_video_name(media_id, width, height)) return out, files + diff --git a/src/youcube/yc_logging.py b/src/youcube/yc_logging.py index e457ead..770fcaa 100644 --- a/src/youcube/yc_logging.py +++ b/src/youcube/yc_logging.py @@ -1,103 +1,104 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Everything logging related -""" - -# Built-in modules -from logging import ( - Formatter, - Logger, - StreamHandler, - LogRecord, - getLogger, - DEBUG, - INFO, - WARNING, - ERROR, - CRITICAL -) -from os import getenv -from yc_colours import Foreground, RESET - -LOGLEVEL = getenv("LOGLEVEL") or DEBUG -NO_COLOR = getenv("NO_COLOR") or False -# Don't call "getLogger" every time we need the logger -logger = getLogger("__main__") - - -class ColordFormatter(Formatter): - """Logging colored formatter, adapted from https://stackoverflow.com/a/56944256/3638629""" - - # noinspection SpellCheckingInspection - def __init__(self, fmt=None, datefmt="%H:%M:%S") -> None: - super().__init__() - self.fmt = fmt - self.datefmt = datefmt - self.formats = { - DEBUG: f"{Foreground.BRIGHT_BLACK}{self.fmt}{RESET}", - INFO: f"{Foreground.BRIGHT_WHITE}{self.fmt}{RESET}", - WARNING: f"{Foreground.BRIGHT_YELLOW}{self.fmt}{RESET}", - ERROR: f"{Foreground.BRIGHT_RED}{self.fmt}{RESET}", - CRITICAL: f"{Foreground.RED}{self.fmt}{RESET}" - } - - def format(self, record: LogRecord) -> str: - log_fmt = self.formats.get(record.levelno) - formatter = Formatter(log_fmt, datefmt=self.datefmt) - return formatter.format(record) - - -class YTDLPLogger: - """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" - - def __init__(self) -> None: - if NO_COLOR: - self.prefix = "[yt-dlp] " - else: - self.prefix = f"{Foreground.BRIGHT_MAGENTA}[yt-dlp]{RESET} " - - def debug(self, msg: str) -> None: - """Pass msg to the main logger""" - - # For compatibility with youtube-dl, both debug and info are passed into debug - # You can distinguish them by the prefix '[debug] ' - if msg.startswith('[debug] '): - pass - else: - self.info(msg) - - def info(self, msg: str) -> None: - """Pass msg to the main logger""" - logger.debug("%s%s", self.prefix, msg) - - def warning(self, msg: str) -> None: - """Pass msg to the main logger""" - logger.warning("%s%s", self.prefix, msg) - - def error(self, msg: str) -> None: - """Pass msg to the main logger""" - logger.error("%s%s", self.prefix, msg) - - -def setup_logging() -> Logger: - """Sets the main logger up""" - logger.setLevel(LOGLEVEL) - - # noinspection SpellCheckingInspection - if NO_COLOR: - formatter = Formatter( - fmt="[%(asctime)s %(levelname)s] [YouCube] %(message)s" - ) - else: - formatter = ColordFormatter( - # pylint: disable-next=line-too-long - fmt=f"[%(asctime)s %(levelname)s] {Foreground.BRIGHT_WHITE}[You{Foreground.RED}Cube]{RESET} %(message)s" - ) - - logging_handler = StreamHandler() - logging_handler.setFormatter(formatter) - logger.addHandler(logging_handler) - - return logger +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Everything logging related +""" + +# Built-in modules +from logging import ( + CRITICAL, + DEBUG, + ERROR, + INFO, + WARNING, + Formatter, + Logger, + LogRecord, + StreamHandler, + getLogger, +) +from os import getenv + +# local modules +from yc_colours import RESET, Foreground + +LOGLEVEL = getenv("LOGLEVEL") or DEBUG +NO_COLOR = getenv("NO_COLOR") or False +# Don't call "getLogger" every time we need the logger +logger = getLogger("__main__") + + +class ColordFormatter(Formatter): + """Logging colored formatter, adapted from https://stackoverflow.com/a/56944256/3638629""" + + # noinspection SpellCheckingInspection + def __init__(self, fmt=None, datefmt="%H:%M:%S") -> None: + super().__init__() + self.fmt = fmt + self.datefmt = datefmt + self.formats = { + DEBUG: f"{Foreground.BRIGHT_BLACK}{self.fmt}{RESET}", + INFO: f"{Foreground.BRIGHT_WHITE}{self.fmt}{RESET}", + WARNING: f"{Foreground.BRIGHT_YELLOW}{self.fmt}{RESET}", + ERROR: f"{Foreground.BRIGHT_RED}{self.fmt}{RESET}", + CRITICAL: f"{Foreground.RED}{self.fmt}{RESET}", + } + + def format(self, record: LogRecord) -> str: + log_fmt = self.formats.get(record.levelno) + formatter = Formatter(log_fmt, datefmt=self.datefmt) + return formatter.format(record) + + +class YTDLPLogger: + """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" + + def __init__(self) -> None: + if NO_COLOR: + self.prefix = "[yt-dlp] " + else: + self.prefix = f"{Foreground.BRIGHT_MAGENTA}[yt-dlp]{RESET} " + + def debug(self, msg: str) -> None: + """Pass msg to the main logger""" + + # For compatibility with youtube-dl, both debug and info are passed into debug + # You can distinguish them by the prefix '[debug] ' + if msg.startswith("[debug] "): + pass + else: + self.info(msg) + + def info(self, msg: str) -> None: + """Pass msg to the main logger""" + logger.debug("%s%s", self.prefix, msg) + + def warning(self, msg: str) -> None: + """Pass msg to the main logger""" + logger.warning("%s%s", self.prefix, msg) + + def error(self, msg: str) -> None: + """Pass msg to the main logger""" + logger.error("%s%s", self.prefix, msg) + + +def setup_logging() -> Logger: + """Sets the main logger up""" + logger.setLevel(LOGLEVEL) + + # noinspection SpellCheckingInspection + if NO_COLOR: + formatter = Formatter(fmt="[%(asctime)s %(levelname)s] [YouCube] %(message)s") + else: + formatter = ColordFormatter( + # pylint: disable-next=line-too-long + fmt=f"[%(asctime)s %(levelname)s] {Foreground.BRIGHT_WHITE}[You{Foreground.RED}Cube]{RESET} %(message)s" + ) + + logging_handler = StreamHandler() + logging_handler.setFormatter(formatter) + logger.addHandler(logging_handler) + + return logger + diff --git a/src/youcube/yc_magic.py b/src/youcube/yc_magic.py index 99970c8..32a51ee 100644 --- a/src/youcube/yc_magic.py +++ b/src/youcube/yc_magic.py @@ -1,138 +1,133 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Black Magic with threads, asyncio and subprocesses -""" - -# Built-in modules -from typing import Any, Callable -from types import FrameType -from threading import Thread -from subprocess import Popen, PIPE -from asyncio import Event -from sys import settrace - - -class ThreadSaveAsyncioEventWithReturnValue(Event): - """ - Thread-save version of asyncio.Event with result / Return value - """ - - def __init__(self) -> None: - super().__init__() - self.result = None - - # pylint: disable-next=fixme - # TODO: clear() method - - def set(self): - # pylint: disable-next=fixme - # FIXME: The _loop attribute is not documented as public api! - self._loop.call_soon_threadsafe(super().set) - - -def run_with_thread_save_asyncio_event_with_return_value( - event: ThreadSaveAsyncioEventWithReturnValue, - func: Callable[[], Any], - *args -) -> None: - """ - Runs a function and calls a ThreadSaveAsyncioEventWithReturnValue - This function is meant to run in a thread - """ - result = func(*args) - event.result = result - event.set() - - -async def run_function_in_thread_from_async_function( - func: Callable[[], Any], - *args -) -> object: - """ - Runs a function in a thread from an async function - """ - event = ThreadSaveAsyncioEventWithReturnValue() - Thread( - target=run_with_thread_save_asyncio_event_with_return_value, - args=(event, func, *args) - ).start() - await event.wait() - return event.result - - -class KillableThread(Thread): - """ - A Thread that can be canceled by running kill on it - https://www.geeksforgeeks.org/python-different-ways-to-kill-a-thread/ - """ - - def __init__(self, *args, **keywords) -> None: - Thread.__init__(self, *args, **keywords) - self.killed = False - - def start(self) -> None: - # pylint: disable-next=attribute-defined-outside-init - self.__run_backup = self.run - self.run = self.__run - Thread.start(self) - - def __run(self) -> None: - settrace(self.globaltrace) - self.__run_backup() - self.run = self.__run_backup - - # pylint: disable-next=unused-argument - def globaltrace(self, frame: FrameType, event: str, arg: Any) -> None: - """ - Allows calling "localtrace" from global - """ - if event == 'call': - return self.localtrace - return None - - # pylint: disable-next=unused-argument - def localtrace(self, frame: FrameType, event: str, arg: Any) -> None: - """ - Uses trace to check if the Thread needs to be killed - """ - if self.killed and event == 'line': - raise SystemExit() - return self.localtrace - - def kill(self) -> None: - """Kills the Thread""" - self.killed = True - - -def run_with_live_output(cmd: list, handler: Callable[[str], None]) -> int: - """ - Runs a subprocess and allows handling output live - """ - with Popen( - cmd, - stdout=PIPE, - stderr=PIPE - ) as process: - - def live_output(): - line = [] - while True: - read = process.stderr.read(1) - if read in (b"\r", b"\n"): # handle \n and \r as new line characters - if len(line) != 0: # ignore empty line - handler("".join(line)) - line.clear() - else: - line.append(read.decode("utf-8")) - - thread = KillableThread(target=live_output) - thread.start() - - process.wait() - thread.kill() - - return process.returncode - -# pylint: disable=unused-argument +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Black Magic with threads, asyncio and subprocesses +""" + +# Built-in modules +from asyncio import Event +from subprocess import PIPE, Popen +from sys import settrace +from threading import Thread +from types import FrameType +from typing import Any, Callable + + +class ThreadSaveAsyncioEventWithReturnValue(Event): + """ + Thread-save version of asyncio.Event with result / Return value + """ + + def __init__(self) -> None: + super().__init__() + self.result = None + + # pylint: disable-next=fixme + # TODO: clear() method + + def set(self): + # pylint: disable-next=fixme + # FIXME: The _loop attribute is not documented as public api! + self._loop.call_soon_threadsafe(super().set) + + +def run_with_thread_save_asyncio_event_with_return_value( + event: ThreadSaveAsyncioEventWithReturnValue, func: Callable[[], Any], *args +) -> None: + """ + Runs a function and calls a ThreadSaveAsyncioEventWithReturnValue + This function is meant to run in a thread + """ + result = func(*args) + event.result = result + event.set() + + +async def run_function_in_thread_from_async_function( + func: Callable[[], Any], *args +) -> object: + """ + Runs a function in a thread from an async function + """ + event = ThreadSaveAsyncioEventWithReturnValue() + Thread( + target=run_with_thread_save_asyncio_event_with_return_value, + args=(event, func, *args), + ).start() + await event.wait() + return event.result + + +class KillableThread(Thread): + """ + A Thread that can be canceled by running kill on it + https://www.geeksforgeeks.org/python-different-ways-to-kill-a-thread/ + """ + + def __init__(self, *args, **keywords) -> None: + Thread.__init__(self, *args, **keywords) + self.killed = False + + def start(self) -> None: + # pylint: disable-next=attribute-defined-outside-init + self.__run_backup = self.run + self.run = self.__run + Thread.start(self) + + def __run(self) -> None: + settrace(self.globaltrace) + self.__run_backup() + self.run = self.__run_backup + + # pylint: disable-next=unused-argument + def globaltrace(self, frame: FrameType, event: str, arg: Any) -> None: + """ + Allows calling "localtrace" from global + """ + if event == "call": + return self.localtrace + return None + + # pylint: disable-next=unused-argument + def localtrace(self, frame: FrameType, event: str, arg: Any) -> None: + """ + Uses trace to check if the Thread needs to be killed + """ + if self.killed and event == "line": + raise SystemExit() + return self.localtrace + + def kill(self) -> None: + """Kills the Thread""" + self.killed = True + + +def run_with_live_output(cmd: list, handler: Callable[[str], None]) -> int: + """ + Runs a subprocess and allows handling output live + """ + with Popen(cmd, stdout=PIPE, stderr=PIPE) as process: + + def live_output(): + line = [] + while True: + read = process.stderr.read(1) + if read in (b"\r", b"\n"): # handle \n and \r as new line characters + if len(line) != 0: # ignore empty line + handler("".join(line)) + line.clear() + else: + line.append(read.decode("utf-8")) + + thread = KillableThread(target=live_output) + thread.start() + + process.wait() + thread.kill() + + return process.returncode + + +# pylint: disable=unused-argument + diff --git a/src/youcube/yc_spotify.py b/src/youcube/yc_spotify.py index 5cb3884..7588b24 100644 --- a/src/youcube/yc_spotify.py +++ b/src/youcube/yc_spotify.py @@ -5,24 +5,26 @@ Spotify support module """ + # Built-in modules +from enum import Enum from logging import getLogger from os import getenv -from enum import Enum from re import match as re_match from typing import Union # pip modules -from spotipy import SpotifyClientCredentials, MemoryCacheHandler +from spotipy import MemoryCacheHandler, SpotifyClientCredentials from spotipy.client import Spotify # https://github.com/spotipy-dev/spotipy/issues/1071 # pylint: disable-next=line-too-long -REGEX_SPOTIFY_URL = r'^(http[s]?:\/\/)?open.spotify.com\/.*(?Ptrack|artist|album|playlist|show|episode|user)\/(?P[0-9A-Za-z]+)(\?.*)?$' +REGEX_SPOTIFY_URL = r"^(http[s]?:\/\/)?open.spotify.com\/.*(?Ptrack|artist|album|playlist|show|episode|user)\/(?P[0-9A-Za-z]+)(\?.*)?$" # pylint: disable=missing-function-docstring # pylint: disable=missing-class-docstring + class SpotifyTypes(Enum): TRACK = "track" ARTIST = "artist" @@ -75,8 +77,7 @@ def spotify_artist(self, spotify_id: str) -> list: return playlist def spotify_show(self, spotify_id: str) -> list: - episodes = self.spotify.show_episodes( - spotify_id, market=self.spotify_market) + episodes = self.spotify.show_episodes(spotify_id, market=self.spotify_market) playlist = [] for track in episodes["items"]: @@ -109,13 +110,13 @@ def auto(self, url: str) -> Union[str, list]: SpotifyTypes.ARTIST: self.spotify_artist, SpotifyTypes.SHOW: self.spotify_show, SpotifyTypes.EPISODE: self.spotify_episode, - SpotifyTypes.USER: self.spotify_user + SpotifyTypes.USER: self.spotify_user, } # pylint: disable=protected-access for match in [ re_match(Spotify._regex_spotify_uri, url), - re_match(REGEX_SPOTIFY_URL, url) + re_match(REGEX_SPOTIFY_URL, url), ]: # pylint: enable=protected-access if match: @@ -143,7 +144,7 @@ def main() -> None: auth_manager=SpotifyClientCredentials( client_id=spotify_client_id, client_secret=spotify_client_secret, - cache_handler=MemoryCacheHandler() + cache_handler=MemoryCacheHandler(), ) ) else: @@ -158,16 +159,16 @@ def main() -> None: "https://open.spotify.com/episode/0UCTRy5frRHxD6SktX9dbV", "https://open.spotify.com/show/5fA3Ze7Ni75iXAEZaEkJIu", "https://open.spotify.com/user/besdkg6w64xf0rt713643tgvt", - "https://open.spotify.com/playlist/5UrcnHexRYVEprv5DJBPER" + "https://open.spotify.com/playlist/5UrcnHexRYVEprv5DJBPER", ] # pylint: disable-next=import-outside-toplevel from yc_colours import Foreground for url in test_urls: - print(Foreground.BLUE + url + Foreground.WHITE, - spotify_url_processor.auto(url)) + print(Foreground.BLUE + url + Foreground.WHITE, spotify_url_processor.auto(url)) if __name__ == "__main__": main() + diff --git a/src/youcube/yc_utils.py b/src/youcube/yc_utils.py index 0034c04..4689341 100644 --- a/src/youcube/yc_utils.py +++ b/src/youcube/yc_utils.py @@ -1,106 +1,111 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Utils for string manipulation, data management etc. -""" - -# Built-in modules -from re import RegexFlag, compile as re_compile -from typing import Tuple -from os import mkdir -from os.path import join, dirname, abspath, exists - - -def remove_whitespace(string: str) -> str: - """ - Removes all Spaces / Whitespace from a string - """ - return string.replace(" ", "") - - -# Only compile "ansi_escape_codes" once -ansi_escape_codes = re_compile(r''' - \x1B # ESC - (?: # 7-bit C1 Fe (except CSI) - [@-Z\\-_] - | # or [ for CSI, followed by a control sequence - \[ - [0-?]* # Parameter bytes - [ -/]* # Intermediate bytes - [@-~] # Final byte - ) -''', RegexFlag.VERBOSE) - - -def remove_ansi_escape_codes(text: str) -> str: - """ - Remove all Ansi Escape codes - (7-bit C1 ANSI sequences) - """ - return ansi_escape_codes.sub('', text) - - -def cap_width(width: int) -> int: - """Caps the width""" - return min(width, 328) - - -def cap_height(height: int) -> int: - """Caps the height""" - return min(height, 243) - - -def cap_width_and_height(width: int, height: int) -> Tuple[int, int]: - """Caps the width and height""" - return cap_width(width), cap_height(height) - - -VIDEO_FORMAT = "32vid" -AUDIO_FORMAT = "dfpwm" -DATA_FOLDER = join(dirname(abspath(__file__)), "data") - - -def get_video_name(media_id: str, width: int, height: int) -> str: - """Returns the file name of the requested video""" - return f"{media_id}({width}x{height}).{VIDEO_FORMAT}" - - -def get_audio_name(media_id: str) -> str: - """Returns the file name of the requested audio""" - return f"{media_id}.{AUDIO_FORMAT}" - - -def get_video_path(media_id: str, width: int, height: int) -> str: - """Returns the relative path to the requested video""" - return join(DATA_FOLDER, get_video_name(media_id, width, height)) - - -def get_audio_path(media_id: str) -> str: - """Returns the relative path to the requested audio""" - return join(DATA_FOLDER, get_audio_name(media_id)) - - -def create_data_folder_if_not_present(): - """Creates the data folder if it does not exist""" - if not exists(DATA_FOLDER): - mkdir(DATA_FOLDER) - - -def is_audio_already_downloaded(media_id: str) -> bool: - """Returns True if the given audio is already downloaded""" - return exists(get_audio_path(media_id)) - - -def is_video_already_downloaded(media_id: str, width: int, height: int) -> bool: - """Returns True if the given video is already downloaded""" - return exists(get_video_path(media_id, width, height)) - - -# Only compile "allowed_characters" once -allowed_characters = re_compile('^[a-zA-Z0-9-._]*$') - - -def is_save(string: str) -> bool: - """Returns True if the given string does not contain special characters""" - return bool(allowed_characters.match(string)) +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Utils for string manipulation, data management etc. +""" + +# Built-in modules +from os import mkdir +from os.path import abspath, dirname, exists, join +from re import RegexFlag +from re import compile as re_compile +from typing import Tuple + + +def remove_whitespace(string: str) -> str: + """ + Removes all Spaces / Whitespace from a string + """ + return string.replace(" ", "") + + +# Only compile "ansi_escape_codes" once +ansi_escape_codes = re_compile( + r""" + \x1B # ESC + (?: # 7-bit C1 Fe (except CSI) + [@-Z\\-_] + | # or [ for CSI, followed by a control sequence + \[ + [0-?]* # Parameter bytes + [ -/]* # Intermediate bytes + [@-~] # Final byte + ) +""", + RegexFlag.VERBOSE, +) + + +def remove_ansi_escape_codes(text: str) -> str: + """ + Remove all Ansi Escape codes + (7-bit C1 ANSI sequences) + """ + return ansi_escape_codes.sub("", text) + + +def cap_width(width: int) -> int: + """Caps the width""" + return min(width, 328) + + +def cap_height(height: int) -> int: + """Caps the height""" + return min(height, 243) + + +def cap_width_and_height(width: int, height: int) -> Tuple[int, int]: + """Caps the width and height""" + return cap_width(width), cap_height(height) + + +VIDEO_FORMAT = "32vid" +AUDIO_FORMAT = "dfpwm" +DATA_FOLDER = join(dirname(abspath(__file__)), "data") + + +def get_video_name(media_id: str, width: int, height: int) -> str: + """Returns the file name of the requested video""" + return f"{media_id}({width}x{height}).{VIDEO_FORMAT}" + + +def get_audio_name(media_id: str) -> str: + """Returns the file name of the requested audio""" + return f"{media_id}.{AUDIO_FORMAT}" + + +def get_video_path(media_id: str, width: int, height: int) -> str: + """Returns the relative path to the requested video""" + return join(DATA_FOLDER, get_video_name(media_id, width, height)) + + +def get_audio_path(media_id: str) -> str: + """Returns the relative path to the requested audio""" + return join(DATA_FOLDER, get_audio_name(media_id)) + + +def create_data_folder_if_not_present(): + """Creates the data folder if it does not exist""" + if not exists(DATA_FOLDER): + mkdir(DATA_FOLDER) + + +def is_audio_already_downloaded(media_id: str) -> bool: + """Returns True if the given audio is already downloaded""" + return exists(get_audio_path(media_id)) + + +def is_video_already_downloaded(media_id: str, width: int, height: int) -> bool: + """Returns True if the given video is already downloaded""" + return exists(get_video_path(media_id, width, height)) + + +# Only compile "allowed_characters" once +allowed_characters = re_compile("^[a-zA-Z0-9-._]*$") + + +def is_save(string: str) -> bool: + """Returns True if the given string does not contain special characters""" + return bool(allowed_characters.match(string)) + diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index d924217..af621be 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -6,31 +6,24 @@ """ # built-in modules -from datetime import datetime -from time import sleep -from os.path import join, exists -from os import getenv, remove from asyncio import get_event_loop -from typing import ( - Union, - Tuple, - Type, - List, - Any -) from base64 import b64encode -from shutil import which +from datetime import datetime from multiprocessing import Manager +from os import getenv, remove +from os.path import exists, join +from shutil import which +from time import sleep +from typing import Any, List, Tuple, Type, Union +# optional pip module try: - from ujson import ( - JSONDecodeError, - dumps, - loads as load_json - ) + from ujson import JSONDecodeError, dumps + from ujson import loads as load_json except ModuleNotFoundError: + from json import dumps + from json import loads as load_json from json.decoder import JSONDecodeError - from json import dumps, loads as load_json try: from types import UnionType @@ -39,29 +32,21 @@ # pip modules -from sanic import ( - Sanic, - Request, - Websocket -) -from sanic.response import text -from sanic.handlers import ErrorHandler +from sanic import Request, Sanic, Websocket +from sanic.compat import open_async from sanic.exceptions import SanicException -from spotipy import SpotifyClientCredentials, MemoryCacheHandler +from sanic.handlers import ErrorHandler +from sanic.response import raw, text +from spotipy import MemoryCacheHandler, SpotifyClientCredentials from spotipy.client import Spotify # local modules -from yc_utils import ( - is_save, - cap_width_and_height, - get_video_name, - get_audio_name -) -from yc_colours import Foreground, RESET -from yc_download import download, DATA_FOLDER, FFMPEG_PATH, SANJUUNI_PATH +from yc_colours import RESET, Foreground +from yc_download import DATA_FOLDER, FFMPEG_PATH, SANJUUNI_PATH, download +from yc_logging import NO_COLOR, setup_logging from yc_magic import run_function_in_thread_from_async_function -from yc_logging import setup_logging, NO_COLOR from yc_spotify import SpotifyURLProcessor +from yc_utils import cap_width_and_height, get_audio_name, get_video_name, is_save VERSION = "0.0.0-poc.1.0.2" API_VERSION = "0.0.0-poc.1.0.0" # https://commandcracker.github.io/YouCube/ @@ -81,39 +66,86 @@ FRAMES_AT_ONCE = 10 +""" +Ubuntu nvida support fix and maby alpine support ? +us async base64 ? +use HTTP (and Streaming) +Add uvloop support https://github.com/CC-YouCube/server/issues/6 +""" + +""" +1 dfpwm chunk = 16 +MAX_DOWNLOAD = 16 * 1024 * 1024 = 16777216 +WEBSOCKET_MESSAGE = 128 * 1024 = 131072 +(MAX_DOWNLOAD = 128 * WEBSOCKET_MESSAGE) + +the speaker can accept a maximum of 128 x 1024 samples 16KiB + +playAudio +This accepts a list of audio samples as amplitudes between -128 and 127. +These are stored in an internal buffer and played back at 48kHz. If this buffer is full, this function will return false. +""" + +"""Related CC-Tweaked issues +Streaming HTTP response https://github.com/cc-tweaked/CC-Tweaked/issues/1181 +Speaker Networks https://github.com/cc-tweaked/CC-Tweaked/issues/1488 +Pocket computers do not have many usecases without network access https://github.com/cc-tweaked/CC-Tweaked/issues/1406 +Speaker limit to 8 https://github.com/cc-tweaked/CC-Tweaked/issues/1313 +Some way to notify player through pocket computer with modem https://github.com/cc-tweaked/CC-Tweaked/issues/1148 +Memory limits for computers https://github.com/cc-tweaked/CC-Tweaked/issues/1580 +""" + +"""TODO: Add those: +AudioDevices: + - Speaker Note (Sound) https://tweaked.cc/peripheral/speaker.html + - Notblock https://www.youtube.com/watch?v=XY5UvTxD9dA + - Create Steam whistles https://www.youtube.com/watch?v=dgZ4F7U19do + https://github.com/danielathome19/MIDIToComputerCraft/tree/master + +Video Formats: + - 32vid binary https://github.com/MCJack123/sanjuuni + - qtv https://github.com/Axisok/qtccv + +Audio Formats: + - DFPWM ffmpeg fallback ? https://github.com/asiekierka/pixmess/blob/master/scraps/aucmp.py + - PCM + - NBS https://github.com/Xella37/NBS-Tunes-CC + - MIDI https://github.com/OpenPrograms/Sangar-Programs/blob/master/midi.lua + - XM https://github.com/MCJack123/tracc + +Audio u. Video preview / thumbnail: + - NFP https://tweaked.cc/library/cc.image.nft.html + - bimg https://github.com/SkyTheCodeMaster/bimg + - as 1 qtv frame + - as 1 32vid frame +""" + # pylint settings # pylint: disable=pointless-string-statement # pylint: disable=fixme # pylint: disable=multiple-statements logger = setup_logging() -#TODO: change sanic logging format +# TODO: change sanic logging format -def get_vid(vid_file: str, tracker: int) -> List[str]: - """ - Returns given line of 32vid file - """ - with open(vid_file, "r", encoding="utf-8") as file: - file.seek(tracker) +async def get_vid(vid_file: str, tracker: int) -> List[str]: + """Returns given line of 32vid file""" + async with await open_async(file=vid_file, mode="r", encoding="utf-8") as file: + await file.seek(tracker) lines = [] for _unused in range(FRAMES_AT_ONCE): - lines.append(file.readline()[:-1]) # remove \n - file.close() + lines.append((await file.readline())[:-1]) # remove \n return lines -def get_chunk(media_file: str, chunkindex: int) -> bytes: - """ - Returns a chunk of the given media file - """ - with open(media_file, "rb") as file: - file.seek(chunkindex * CHUNKS_AT_ONCE) - chunk = file.read(CHUNKS_AT_ONCE) - file.close() +async def getchunk(media_file: str, chunkindex: int) -> bytes: + """Returns a chunk of the given media file""" + async with await open_async(file=media_file, mode="rb") as file: + await file.seek(chunkindex * CHUNKS_AT_ONCE) + return await file.read(CHUNKS_AT_ONCE) - return chunk # pylint: enable=redefined-outer-name @@ -122,16 +154,8 @@ def assert_resp( __obj_name: str, __obj: Any, __class_or_tuple: Union[ - Type, UnionType, - Tuple[ - Union[ - Type, - UnionType, - Tuple[Any, ...] - ], - ... - ] - ] + Type, UnionType, Tuple[Union[Type, UnionType, Tuple[Any, ...]], ...] + ], ) -> Union[dict, None]: """ "assert" / isinstance that returns a dict that can be send as a ws response @@ -139,7 +163,7 @@ def assert_resp( if not isinstance(__obj, __class_or_tuple): return { "action": "error", - "message": f"{__obj_name} must be a {__class_or_tuple.__name__}" + "message": f"{__obj_name} must be a {__class_or_tuple.__name__}", } return None @@ -155,7 +179,7 @@ def assert_resp( auth_manager=SpotifyClientCredentials( client_id=spotify_client_id, client_secret=spotify_client_secret, - cache_handler=MemoryCacheHandler() + cache_handler=MemoryCacheHandler(), ) ) @@ -180,7 +204,8 @@ async def request_media(message: dict, resp: Websocket, request: Request): loop = get_event_loop() # get "url" url = message.get("url") - if error := assert_resp("url", url, str): return error + if error := assert_resp("url", url, str): + return error # TODO: assert_resp width and height out, files = await run_function_in_thread_from_async_function( download, @@ -189,7 +214,7 @@ async def request_media(message: dict, resp: Websocket, request: Request): loop, message.get("width"), message.get("height"), - spotify_url_processor + spotify_url_processor, ) for file in files: request.app.shared_ctx.data[file] = datetime.now() @@ -199,102 +224,77 @@ async def request_media(message: dict, resp: Websocket, request: Request): async def get_chunk(message: dict, _unused, request: Request): # get "chunkindex" chunkindex = message.get("chunkindex") - if error := assert_resp("chunkindex", chunkindex, int): return error + if error := assert_resp("chunkindex", chunkindex, int): + return error # get "id" media_id = message.get("id") - if error := assert_resp("media_id", media_id, str): return error + if error := assert_resp("media_id", media_id, str): + return error if is_save(media_id): file_name = get_audio_name(message.get("id")) - file = join( - DATA_FOLDER, - file_name - ) + file = join(DATA_FOLDER, file_name) request.app.shared_ctx.data[file_name] = datetime.now() - chunk = get_chunk(file, chunkindex) + chunk = await get_chunk(file, chunkindex) - return { - "action": "chunk", - "chunk": b64encode(chunk).decode("ascii") - } + return {"action": "chunk", "chunk": b64encode(chunk).decode("ascii")} logger.warning("User tried to use special Characters") - return { - "action": "error", - "message": "You dare not use special Characters" - } + return {"action": "error", "message": "You dare not use special Characters"} @staticmethod async def get_vid(message: dict, _unused, request: Request): # get "line" tracker = message.get("tracker") - if error := assert_resp("tracker", tracker, int): return error + if error := assert_resp("tracker", tracker, int): + return error # get "id" media_id = message.get("id") - if error := assert_resp("id", media_id, str): return error + if error := assert_resp("id", media_id, str): + return error # get "width" - width = message.get('width') - if error := assert_resp("width", width, int): return error + width = message.get("width") + if error := assert_resp("width", width, int): + return error # get "height" - height = message.get('height') - if error := assert_resp("height", height, int): return error + height = message.get("height") + if error := assert_resp("height", height, int): + return error # cap height and width width, height = cap_width_and_height(width, height) if is_save(media_id): - file_name = get_video_name(message.get('id'), width, height) - file = join( - DATA_FOLDER, - file_name - ) + file_name = get_video_name(message.get("id"), width, height) + file = join(DATA_FOLDER, file_name) request.app.shared_ctx.data[file_name] = datetime.now() - return { - "action": "vid", - "lines": get_vid(file, tracker) - } + return {"action": "vid", "lines": await get_vid(file, tracker)} - return { - "action": "error", - "message": "You dare not use special Characters" - } + return {"action": "error", "message": "You dare not use special Characters"} @staticmethod async def handshake(*_unused): return { "action": "handshake", - "server": { - "version": VERSION - }, - "api": { - "version": API_VERSION - }, - "capabilities": { - "video": [ - "32vid" - ], - "audio": [ - "dfpwm" - ] - } + "server": {"version": VERSION}, + "api": {"version": API_VERSION}, + "capabilities": {"video": ["32vid"], "audio": ["dfpwm"]}, } # pylint: enable=missing-function-docstring class CustomErrorHandler(ErrorHandler): - """ - Error handler for sanic - """ + """Error handler for sanic""" def default(self, request: Request, exception: Union[SanicException, Exception]): - ''' handles errors that have no error handlers assigned ''' + """handles errors that have no error handlers assigned""" if isinstance(exception, SanicException) and exception.status_code == 426: # TODO: Respond with nice html that tells the user how to install YC @@ -319,7 +319,7 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) # add all actions from default action set for method in dir(Actions): - if not method.startswith('__'): + if not method.startswith("__"): actions[method] = getattr(Actions, method) @@ -329,14 +329,16 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) def data_cache_cleaner(data: dict): """ - Checks for outdated cache entries every DATA_CACHE_CLEANUP_INTERVAL (default 300) Seconds and + Checks for outdated cache entries every DATA_CACHE_CLEANUP_INTERVAL (default 300) Seconds and deletes them if they have not been used for DATA_CACHE_CLEANUP_AFTER (default 3600) Seconds. """ try: while True: sleep(DATA_CACHE_CLEANUP_INTERVAL) for file_name, last_used in data.items(): - if (datetime.now() - last_used).total_seconds() > DATA_CACHE_CLEANUP_AFTER: + if ( + datetime.now() - last_used + ).total_seconds() > DATA_CACHE_CLEANUP_AFTER: file_path = join(DATA_FOLDER, file_name) if exists(file_path): remove(file_path) @@ -352,7 +354,9 @@ def data_cache_cleaner(data: dict): async def ready(app: Sanic, _): """See https://sanic.dev/en/guide/basics/listeners.html""" if DATA_CACHE_CLEANUP_INTERVAL > 0 and DATA_CACHE_CLEANUP_AFTER > 0: - app.manager.manage("Data-Cache-Cleaner", data_cache_cleaner, {"data": app.shared_ctx.data}) + app.manager.manage( + "Data-Cache-Cleaner", data_cache_cleaner, {"data": app.shared_ctx.data} + ) @app.main_process_start @@ -372,6 +376,55 @@ async def main_start(app: Sanic): logger.info("Spotipy Disabled") +@app.route("/dfpwm//") +async def stream_dfpwm(request: Request, id: str, chunkindex: int): + return raw(await get_chunk(join(DATA_FOLDER, get_audio_name(id)), chunkindex)) + + +@app.route("/32vid////", stream=True) +async def stream_32vid( + request: Request, id: str, width: int, height: int, tracker: int +): + return raw( + "\n".join( + await get_vid(join(DATA_FOLDER, get_video_name(id, width, height)), tracker) + ) + ) + + +"""" +from sanic import response +@app.route("/dfpwm/") +async def stream_dfpwm(request: Request, id: str): + file_name = get_audio_name(id) + file = join(DATA_FOLDER, get_audio_name(id)) + return await response.file_stream( + file, + chunk_size=CHUNKS_AT_ONCE, + mime_type="application/metalink4+xml", + headers={ + "Content-Disposition": f'Attachment; filename="{file_name}"', + "Content-Type": "application/metalink4+xml", + }, + ) + +@app.route("/32vid///", stream=True) +async def stream_32vid(request: Request, id: str, width: int, height: int): + file_name = get_video_name(id, width, height) + file = join( + DATA_FOLDER, + file_name + ) + return await response.file_stream( + file, + chunk_size=10, + mime_type="application/metalink4+xml", + headers={ + "Content-Disposition": f'Attachment; filename="{file_name}"', + "Content-Type": "application/metalink4+xml", + }, + ) +""" # pylint: enable=redefined-outer-name @@ -386,11 +439,7 @@ async def wshandler(request: Request, ws: Websocket): logger.info("%sConnected!", prefix) - logger.debug( - "%sMy headers are: %s", - prefix, - request.headers - ) + logger.debug("%sMy headers are: %s", prefix, request.headers) while True: message = await ws.recv() @@ -400,10 +449,7 @@ async def wshandler(request: Request, ws: Websocket): message: dict = load_json(message) except JSONDecodeError: logger.debug("%sFaild to parse Json", prefix) - await ws.send(dumps({ - "action": "error", - "message": "Faild to parse Json" - })) + await ws.send(dumps({"action": "error", "message": "Faild to parse Json"})) if message.get("action") in actions: response = await actions[message.get("action")](message, ws, request) @@ -415,11 +461,12 @@ def main() -> None: Run all needed services """ port = int(getenv("PORT", "5000")) - host = getenv("HOST", "0.0.0.0") + host = getenv("HOST", "127.0.0.1") fast = not getenv("NO_FAST") - app.run(host=host, port=port, fast=fast) + app.run(host=host, port=port, fast=fast, access_log=True) if __name__ == "__main__": main() + From a6ee163015114f8c08e649cf44bd9a13d4328827 Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Fri, 5 Jul 2024 20:09:05 +0200 Subject: [PATCH 4/8] Update docker images, fixed docker nvidia opencl support, updated all requirements --- src/Dockerfile | 28 ++++++++++++++------------- src/Dockerfile.nvidia | 36 +++++++++++++++++------------------ src/docker-compose.nvidia.yml | 10 +++++----- src/docker-compose.yml | 3 ++- src/requirements.txt | 10 +++++----- src/youcube/yc_download.py | 2 +- src/youcube/yc_spotify.py | 5 +---- src/youcube/youcube.py | 12 ++++++------ 8 files changed, 52 insertions(+), 54 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index a33b722..4ca720c 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -1,17 +1,17 @@ FROM ghcr.io/commandcracker/ffmpeg:latest AS ffmpeg -FROM ffmpeg as sanjuuni +FROM ffmpeg AS sanjuuni -ENV SANJUUNI_VERSION=ed34c266cd489e4db796ca803e247b48b43853e0 +ENV SANJUUNI_VERSION=778644b164c8877e56f9f5512480dde857133815 -ARG SANJUUNI_SHA512SUM="9e482e3b8f8885e8f458856f11d5ee4c27a0aa469b8c54abe1aef943f630ca27eb148c8779ba7a053c1abcce298513e98b614747a77ae1c0cbc86a0a7c95a6d8 *sanjuuni.tar.gz" +ARG SANJUUNI_SHA512SUM="353b97e53ec2daba3046b3062c8a389c75066ea410f9abe089467a4648e83afe926cd65ad0904a0d59eca0520e3174e0f3190987cfee3abbdc9141a04a80ef1a *sanjuuni.tar.gz" SHELL ["/bin/ash", "-eo", "pipefail", "-c"] RUN set -eux; \ apk add --no-cache --update \ - g++=13.2.1_git20231014-r0 \ - zlib-dev=1.3.1-r0 \ + g++=13.2.1_git20240309-r0 \ + zlib-dev=1.3.1-r1 \ poco-dev=1.12.4-r0 \ make=4.4.1-r2; \ wget -q --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ @@ -26,7 +26,7 @@ RUN set -eux; \ ./configure; \ make -FROM ghcr.io/commandcracker/alpine-pypy3.10-pip:3.19.0-pypy-7.3.14-pip-23.3.2 AS builder +FROM ghcr.io/commandcracker/alpine-pypy3.10-pip:3.20.1-pypy-7.3.14-pip-24.1.1 AS builder WORKDIR / @@ -40,22 +40,24 @@ RUN set -eux; \ python3 compile.py; \ pip uninstall pip -y -FROM alpine:3.19.0 +FROM alpine:3.20.1 WORKDIR /opt/server RUN set -eux; \ apk add --no-cache --update \ + # CVE-2024-5535 TODO: remove when base image is updated + openssl \ # pypy requirements - libffi=3.4.4-r3 libbz2=1.0.8-r6 \ + libffi=3.4.6-r0 libbz2=1.0.8-r6 \ # sanjuuni requirements poco=1.12.4-r0 \ # ffmpeg requirements - libgcc=13.2.1_git20231014-r \ - libstdc++=13.2.1_git20231014-r0 \ - ca-certificates=20230506-r0 \ - libgomp=13.2.1_git20231014-r0 \ - expat=2.5.0-r2; \ + libgcc=13.2.1_git20240309-r0 \ + libstdc++=13.2.1_git20240309-r0 \ + ca-certificates=20240226-r0 \ + libgomp=13.2.1_git20240309-r0 \ + expat=2.6.2-r0; \ apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/community libssl1.1=1.1.1u-r1 libcrypto1.1=1.1.1u-r1; \ chown 1000 /opt/server/ diff --git a/src/Dockerfile.nvidia b/src/Dockerfile.nvidia index 978ce49..0b1baba 100644 --- a/src/Dockerfile.nvidia +++ b/src/Dockerfile.nvidia @@ -1,21 +1,23 @@ +#!/usr/bin/env dockerfile-shebang + FROM ghcr.io/commandcracker/ubuntu-ffmpeg:latest AS ffmpeg -FROM ffmpeg as sanjuuni +FROM ffmpeg AS sanjuuni -ENV SANJUUNI_VERSION=0.4 +ENV SANJUUNI_VERSION=778644b164c8877e56f9f5512480dde857133815 -ARG SANJUUNI_SHA512SUM="952a6c608d167f37faad53ee7f2e0de8090a02bf73b6455fae7c6b6f648dd6a188e7749fe26caeee85126b2a38d7391389c19afb0100e9962dc551188b9de6ae *sanjuuni.tar.gz" +ARG SANJUUNI_SHA512SUM="353b97e53ec2daba3046b3062c8a389c75066ea410f9abe089467a4648e83afe926cd65ad0904a0d59eca0520e3174e0f3190987cfee3abbdc9141a04a80ef1a *sanjuuni.tar.gz" SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN set -eux; \ apt-get update; \ - apt-get install \ + apt-get install -y --no-install-recommends \ ocl-icd-opencl-dev=2.2.14-3 \ - wget=1.21.2-2ubuntu1 \ + wget=1.21.2-2ubuntu1.1 \ clang=1:14.0-55~exp2 \ make=4.3-4.1build1 \ - libpoco-dev=1.11.0-3 -y --no-install-recommends; \ + libpoco-dev=1.11.0-3; \ wget --progress=dot:giga --output-document=sanjuuni.tar.gz https://github.com/MCJack123/sanjuuni/archive/${SANJUUNI_VERSION}.tar.gz; \ echo "${SANJUUNI_SHA512SUM}" | sha512sum -c -; \ mkdir --parents sanjuuni; \ @@ -41,21 +43,17 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # hadolint ignore=SC1091 RUN set -eux; \ apt-get update; \ - apt-get install \ + apt-get install -y --no-install-recommends \ libpoco-dev=1.11.0-3 \ python3-pip=22.0.2+dfsg-1 \ - gnupg=2.2.27-3ubuntu2.1 \ - libcurl4=7.81.0-1 \ - curl=7.81.0-1 -y --no-install-recommends; \ + ocl-icd-libopencl1; \ + pip install -U pip; \ pip install --no-cache-dir -r requirements.txt; \ - curl -s -L "https://nvidia.github.io/nvidia-container-runtime/gpgkey" | apt-key add -; \ - distribution=$(. /etc/os-release;echo "$ID$VERSION_ID"); \ - curl -s -L "https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.list" | tee /etc/apt/sources.list.d/nvidia-container-runtime.list; \ - apt-get update; \ - apt-get install \ - nvidia-opencl-dev=11.5.1-1ubuntu1 \ - nvidia-container-runtime=3.13.0-1 -y --no-install-recommends; \ - apt-get clean; \ - rm -rf /var/lib/apt/lists/* + mkdir -p /etc/OpenCL/vendors; \ + echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +ENV \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility ENTRYPOINT ["python3", "youcube.py"] diff --git a/src/docker-compose.nvidia.yml b/src/docker-compose.nvidia.yml index f2d2d38..79577e3 100644 --- a/src/docker-compose.nvidia.yml +++ b/src/docker-compose.nvidia.yml @@ -1,7 +1,9 @@ -version: "2.0" +--- services: youcube: - build: . + build: + context: . + dockerfile: Dockerfile.nvidia image: youcube:nvidia restart: always hostname: youcube @@ -9,6 +11,4 @@ services: - 5000:5000 #env_file: .env runtime: nvidia - environment: - - NVIDIA_VISIBLE_DEVICES=all - - NVIDIA_DRIVER_CAPABILITIES=compute +... diff --git a/src/docker-compose.yml b/src/docker-compose.yml index 10fc378..9e657ea 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -1,4 +1,4 @@ -version: "2.0" +--- services: youcube: build: . @@ -8,3 +8,4 @@ services: ports: - 5000:5000 #env_file: .env +... diff --git a/src/requirements.txt b/src/requirements.txt index 991f68a..28a0ad7 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,5 +1,5 @@ -sanic~=23.12.1 -#uvloop~=0.19.0; platform_system != "Windows" -yt-dlp~=2023.12.30 -ujson~=5.9.0 -spotipy~=2.23.0 +sanic~=24.6.0 +#uvloop~=0.19.0; platform_system != "Windows" +yt-dlp~=2024.7.2 +#orjson~=3.10.6 +spotipy~=2.24.0 diff --git a/src/youcube/yc_download.py b/src/youcube/yc_download.py index 5ec535d..147d633 100644 --- a/src/youcube/yc_download.py +++ b/src/youcube/yc_download.py @@ -29,7 +29,7 @@ # optional pip modules try: - from ujson import dumps + from orjson import dumps except ModuleNotFoundError: from json import dumps diff --git a/src/youcube/yc_spotify.py b/src/youcube/yc_spotify.py index 7588b24..2a76328 100644 --- a/src/youcube/yc_spotify.py +++ b/src/youcube/yc_spotify.py @@ -17,9 +17,6 @@ from spotipy import MemoryCacheHandler, SpotifyClientCredentials from spotipy.client import Spotify -# https://github.com/spotipy-dev/spotipy/issues/1071 -# pylint: disable-next=line-too-long -REGEX_SPOTIFY_URL = r"^(http[s]?:\/\/)?open.spotify.com\/.*(?Ptrack|artist|album|playlist|show|episode|user)\/(?P[0-9A-Za-z]+)(\?.*)?$" # pylint: disable=missing-function-docstring # pylint: disable=missing-class-docstring @@ -116,7 +113,7 @@ def auto(self, url: str) -> Union[str, list]: # pylint: disable=protected-access for match in [ re_match(Spotify._regex_spotify_uri, url), - re_match(REGEX_SPOTIFY_URL, url), + re_match(Spotify._regex_spotify_url, url), ]: # pylint: enable=protected-access if match: diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index af621be..1c1e031 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -18,8 +18,8 @@ # optional pip module try: - from ujson import JSONDecodeError, dumps - from ujson import loads as load_json + from orjson import JSONDecodeError, dumps + from orjson import loads as load_json except ModuleNotFoundError: from json import dumps from json import loads as load_json @@ -237,7 +237,7 @@ async def get_chunk(message: dict, _unused, request: Request): file = join(DATA_FOLDER, file_name) request.app.shared_ctx.data[file_name] = datetime.now() - chunk = await get_chunk(file, chunkindex) + chunk = await getchunk(file, chunkindex) return {"action": "chunk", "chunk": b64encode(chunk).decode("ascii")} logger.warning("User tried to use special Characters") @@ -307,7 +307,7 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) return super().default(request, exception) -app = Sanic(__name__) +app = Sanic("youcube") app.error_handler = CustomErrorHandler() # FIXME: The Client is not Responsing to Websocket pings app.config.WEBSOCKET_PING_INTERVAL = 0 @@ -378,10 +378,10 @@ async def main_start(app: Sanic): @app.route("/dfpwm//") async def stream_dfpwm(request: Request, id: str, chunkindex: int): - return raw(await get_chunk(join(DATA_FOLDER, get_audio_name(id)), chunkindex)) + return raw(await getchunk(join(DATA_FOLDER, get_audio_name(id)), chunkindex)) -@app.route("/32vid////", stream=True) +@app.route("/32vid////") # , stream=True async def stream_32vid( request: Request, id: str, width: int, height: int, tracker: int ): From 3f9aa7e74bf6c63b052ed01e2dd079b0603082e9 Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Fri, 5 Jul 2024 20:20:37 +0200 Subject: [PATCH 5/8] Update Readme docker compose --- .gitignore | 1 + README.md | 3 ++- src/compile.py | 1 - src/youcube/__main__.py | 1 - src/youcube/yc_colours.py | 1 - src/youcube/yc_download.py | 1 - src/youcube/yc_logging.py | 1 - src/youcube/yc_magic.py | 1 - src/youcube/yc_spotify.py | 1 - src/youcube/yc_utils.py | 1 - src/youcube/youcube.py | 4 ++-- 11 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 8fc3088..f4336f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea *.dfpwm *.32vid dictionary.dic diff --git a/README.md b/README.md index 222ad8d..e17a7e3 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ And [Sanic Builtin values]. ## Docker Compose ```yml -version: "2.0" +--- services: youcube: image: ghcr.io/cc-youcube/youcube:latest @@ -63,6 +63,7 @@ services: hostname: youcube ports: - 5000:5000 +... ``` [spotify application]: https://developer.spotify.com/dashboard/applications diff --git a/src/compile.py b/src/compile.py index 0ef0a7b..0ba1822 100644 --- a/src/compile.py +++ b/src/compile.py @@ -27,4 +27,3 @@ def main() -> None: if __name__ == "__main__": main() - diff --git a/src/youcube/__main__.py b/src/youcube/__main__.py index 384758c..ae5c730 100644 --- a/src/youcube/__main__.py +++ b/src/youcube/__main__.py @@ -10,4 +10,3 @@ if __name__ == "__main__": main() - diff --git a/src/youcube/yc_colours.py b/src/youcube/yc_colours.py index c375ed5..1fb2b1f 100644 --- a/src/youcube/yc_colours.py +++ b/src/youcube/yc_colours.py @@ -36,4 +36,3 @@ class Foreground: RESET = "\033[m" - diff --git a/src/youcube/yc_download.py b/src/youcube/yc_download.py index 147d633..a2e0206 100644 --- a/src/youcube/yc_download.py +++ b/src/youcube/yc_download.py @@ -293,4 +293,3 @@ def my_hook(info): files.append(get_video_name(media_id, width, height)) return out, files - diff --git a/src/youcube/yc_logging.py b/src/youcube/yc_logging.py index 770fcaa..961d880 100644 --- a/src/youcube/yc_logging.py +++ b/src/youcube/yc_logging.py @@ -101,4 +101,3 @@ def setup_logging() -> Logger: logger.addHandler(logging_handler) return logger - diff --git a/src/youcube/yc_magic.py b/src/youcube/yc_magic.py index 32a51ee..6d1678f 100644 --- a/src/youcube/yc_magic.py +++ b/src/youcube/yc_magic.py @@ -130,4 +130,3 @@ def live_output(): # pylint: disable=unused-argument - diff --git a/src/youcube/yc_spotify.py b/src/youcube/yc_spotify.py index 2a76328..581df03 100644 --- a/src/youcube/yc_spotify.py +++ b/src/youcube/yc_spotify.py @@ -168,4 +168,3 @@ def main() -> None: if __name__ == "__main__": main() - diff --git a/src/youcube/yc_utils.py b/src/youcube/yc_utils.py index 4689341..fa9cd8f 100644 --- a/src/youcube/yc_utils.py +++ b/src/youcube/yc_utils.py @@ -108,4 +108,3 @@ def is_video_already_downloaded(media_id: str, width: int, height: int) -> bool: def is_save(string: str) -> bool: """Returns True if the given string does not contain special characters""" return bool(allowed_characters.match(string)) - diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index 1c1e031..60198d3 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -83,7 +83,8 @@ playAudio This accepts a list of audio samples as amplitudes between -128 and 127. -These are stored in an internal buffer and played back at 48kHz. If this buffer is full, this function will return false. +These are stored in an internal buffer and played back at 48kHz. +If this buffer is full, this function will return false. """ """Related CC-Tweaked issues @@ -469,4 +470,3 @@ def main() -> None: if __name__ == "__main__": main() - From 748ee87b2137e50bcd41850ba3dd4bfcbca1ea18 Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Fri, 5 Jul 2024 20:32:05 +0200 Subject: [PATCH 6/8] Update Wordlist --- spellcheck_wordlist.txt | 5 +++++ src/youcube/youcube.py | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/spellcheck_wordlist.txt b/spellcheck_wordlist.txt index 4499fed..a9bbd31 100644 --- a/spellcheck_wordlist.txt +++ b/spellcheck_wordlist.txt @@ -19,6 +19,7 @@ html dev # Programming Keywords +str env untrusted usr @@ -66,9 +67,13 @@ github dlp geeksforgeeks www +metalink # Video Streaming yc yt dl youtube + +# Other +f'Attachment diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index 60198d3..3668b5b 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -90,9 +90,11 @@ """Related CC-Tweaked issues Streaming HTTP response https://github.com/cc-tweaked/CC-Tweaked/issues/1181 Speaker Networks https://github.com/cc-tweaked/CC-Tweaked/issues/1488 -Pocket computers do not have many usecases without network access https://github.com/cc-tweaked/CC-Tweaked/issues/1406 +Pocket computers do not have many usecases without network access +https://github.com/cc-tweaked/CC-Tweaked/issues/1406 Speaker limit to 8 https://github.com/cc-tweaked/CC-Tweaked/issues/1313 -Some way to notify player through pocket computer with modem https://github.com/cc-tweaked/CC-Tweaked/issues/1148 +Some way to notify player through pocket computer with modem +https://github.com/cc-tweaked/CC-Tweaked/issues/1148 Memory limits for computers https://github.com/cc-tweaked/CC-Tweaked/issues/1580 """ @@ -377,18 +379,20 @@ async def main_start(app: Sanic): logger.info("Spotipy Disabled") -@app.route("/dfpwm//") -async def stream_dfpwm(request: Request, id: str, chunkindex: int): - return raw(await getchunk(join(DATA_FOLDER, get_audio_name(id)), chunkindex)) +@app.route("/dfpwm//") +async def stream_dfpwm(_request: Request, media_id: str, chunkindex: int): + """WIP HTTP mode""" + return raw(await getchunk(join(DATA_FOLDER, get_audio_name(media_id)), chunkindex)) -@app.route("/32vid////") # , stream=True +@app.route("/32vid////") # , stream=True async def stream_32vid( - request: Request, id: str, width: int, height: int, tracker: int + _request: Request, media_id: str, width: int, height: int, tracker: int ): + """WIP HTTP mode""" return raw( "\n".join( - await get_vid(join(DATA_FOLDER, get_video_name(id, width, height)), tracker) + await get_vid(join(DATA_FOLDER, get_video_name(media_id, width, height)), tracker) ) ) From 74ada8578504880ca8e9834bdad84c4387198ac5 Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Fri, 5 Jul 2024 20:34:44 +0200 Subject: [PATCH 7/8] Make all linters happy --- spellcheck_wordlist.txt | 2 ++ src/youcube/youcube.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/spellcheck_wordlist.txt b/spellcheck_wordlist.txt index a9bbd31..4de8969 100644 --- a/spellcheck_wordlist.txt +++ b/spellcheck_wordlist.txt @@ -77,3 +77,5 @@ youtube # Other f'Attachment +WIP +xml diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index 3668b5b..047d430 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -66,6 +66,11 @@ FRAMES_AT_ONCE = 10 +# pylint settings +# pylint: disable=pointless-string-statement +# pylint: disable=fixme +# pylint: disable=multiple-statements + """ Ubuntu nvida support fix and maby alpine support ? us async base64 ? @@ -123,11 +128,6 @@ - as 1 32vid frame """ -# pylint settings -# pylint: disable=pointless-string-statement -# pylint: disable=fixme -# pylint: disable=multiple-statements - logger = setup_logging() # TODO: change sanic logging format From 8bf18bbe3f525e1818f116bb8bf606708cacba5f Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Fri, 5 Jul 2024 20:44:17 +0200 Subject: [PATCH 8/8] Make docker build and publish faster --- .github/workflows/docker-build-and-push.yml | 25 ++++++++++++++++++++- .github/workflows/docker-build.yml | 9 +++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-build-and-push.yml index 9fb21ec..2f3062e 100644 --- a/.github/workflows/docker-build-and-push.yml +++ b/.github/workflows/docker-build-and-push.yml @@ -12,7 +12,7 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push-image: + build-and-publish-alpine: runs-on: ubuntu-latest permissions: contents: read @@ -43,6 +43,29 @@ jobs: tags: ${{ env.REGISTRY }}/cc-youcube/youcube:latest,${{ env.REGISTRY }}/cc-youcube/youcube:alpine labels: ${{ steps.meta.outputs.labels }} + build-and-publish-nvidia: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout 🛎️ + uses: actions/checkout@v3 + + - name: Login to container registry 🔐 + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata 🏷️ + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: 🔨 Build and Publish nvidia 🚀 uses: docker/build-push-action@v3 with: diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index dbd3d32..d7ef0bc 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -11,7 +11,7 @@ on: - "src/**" jobs: - build-image: + build-alpine-image: runs-on: ubuntu-latest steps: @@ -23,6 +23,13 @@ jobs: with: context: src + build-nvidia-image: + runs-on: ubuntu-latest + + steps: + - name: Checkout 🛎️ + uses: actions/checkout@v3 + - name: Build nvidia image 🔨 uses: docker/build-push-action@v4 with: