From a3b863baa2937a9e927dc9cf453b24df50ff4c3c Mon Sep 17 00:00:00 2001 From: Commandcracker Date: Mon, 15 Jan 2024 14:52:45 +0100 Subject: [PATCH] Clear cache Implemented cleanup process, that checks for outdated cache entries every DATA_CACHE_CLEANUP_INTERVAL(default 300) Seconds and deletes them if they have not been used for DATA_CACHE_CLEANUP_AFTER (default 3600) Seconds. resolves #1 --- README.md | 160 ++++++----- src/youcube/yc_download.py | 567 +++++++++++++++++++------------------ src/youcube/youcube.py | 133 +++++---- 3 files changed, 434 insertions(+), 426 deletions(-) diff --git a/README.md b/README.md index 26812e5..d77aae8 100644 --- a/README.md +++ b/README.md @@ -1,78 +1,82 @@ -# YouCube Server - -[![Python Version: 3.7+]](https://www.python.org/downloads/) -[![Python Lint Workflow Status]](https://github.com/CC-YouCube/server/actions/workflows/pylint.yml) - -![preview] - -YouCube has a some public servers, which you can use if you don't want to host your own server. \ -The client has the public servers set by default, so you can just run the client, and you're good to go. \ -Moor Information about the servers can be seen on the [doc]. - -## Requirements - -- [yt-dlp/FFmpeg] / [FFmpeg 5.1+] -- [sanjuuni] -- [Python 3.7+] - - [sanic] - - [yt-dlp] - - [ujson] (Optional) - - [spotipy] - -You can install the required packages with [pip] by running: - -```shell -pip install -r src/requirements.txt -``` - -## Starting the Server - -```bash -python src/youcube.py -``` - -## Environment variables - -Environment variables you can use to configure the server. - -| Variable | Default | Description | -|-------------------------|------------|---------------------------------------------------| -| `HOST` | `0.0.0.0` | The host where the web server runs on. | -| `PORT` | `5000` | The port where the web server should run on | -| `TRUSTED_PROXIES` | | Trusted proxies (separated by comma`,`) | -| `FFMPEG_PATH` | `ffmpeg` | Path to the FFmpeg executable | -| `SANJUUNI_PATH` | `sanjuuni` | Path to the Sanjuuni executable | -| `NO_COLOR` | `False` | Disable colored output | -| `LOGLEVEL` | `DEBUG` | Python Log level of the main logger | -| `DISABLE_OPENCL` | `False` | Disables sanjuuni GPU acceleration | -| `NO_FAST` | `False` | Disable Sanic worker processes maximization | -| `SPOTIPY_CLIENT_ID` | | The Client ID from your [spotify application] | -| `SPOTIPY_CLIENT_SECRET` | | The Client Secret from your [spotify application] | - -## Docker Compose - -```yml -version: "2.0" -services: - youcube: - image: ghcr.io/cc-youcube/youcube:latest - restart: always - hostname: youcube - ports: - - 5000:5000 -``` - -[spotify application]: https://developer.spotify.com/dashboard/applications -[pip]: https://pip.pypa.io/en/stable/installation -[yt-dlp/FFmpeg]: https://github.com/yt-dlp/FFmpeg-Builds -[FFmpeg 5.1+]: https://ffmpeg.org -[sanjuuni]: https://github.com/MCJack123/sanjuuni -[Python 3.7+]: https://www.python.org/downloads -[sanic]: https://sanic.dev -[yt-dlp]: https://pypi.org/project/yt-dlp -[ujson]: https://pypi.org/project/ujson -[spotipy]: https://pypi.org/project/spotipy -[doc]: https://youcube.madefor.cc/api -[preview]: .README/preview-server.png -[Python Version: 3.7+]: https://img.shields.io/badge/Python-3.7+-green?style=for-the-badge&logo=Python&logoColor=white -[Python Lint Workflow Status]: https://img.shields.io/github/actions/workflow/status/CC-YouCube/server/pylint.yml?branch=main&label=Python%20Lint&logo=github&style=for-the-badge +# YouCube Server + +[![Python Version: 3.7+]](https://www.python.org/downloads/) +[![Python Lint Workflow Status]](https://github.com/CC-YouCube/server/actions/workflows/pylint.yml) + +![preview] + +YouCube has a some public servers, which you can use if you don't want to host your own server. \ +The client has the public servers set by default, so you can just run the client, and you're good to go. \ +Moor Information about the servers can be seen on the [doc]. + +## Requirements + +- [yt-dlp/FFmpeg] / [FFmpeg 5.1+] +- [sanjuuni] +- [Python 3.7+] + - [sanic] + - [yt-dlp] + - [ujson] (Optional) + - [spotipy] + +You can install the required packages with [pip] by running: + +```shell +pip install -r src/requirements.txt +``` + +## Starting the Server + +```bash +python src/youcube.py +``` + +## Environment variables + +Environment variables you can use to configure the server: + +| Variable | Default | Description | +|-------------------------------|------------|--------------------------------------------------------------------------------------------------------------------| +| `HOST` | `0.0.0.0` | The host where the web server runs on. | +| `PORT` | `5000` | The port where the web server should run on | +| `FFMPEG_PATH` | `ffmpeg` | Path to the FFmpeg executable | +| `SANJUUNI_PATH` | `sanjuuni` | Path to the Sanjuuni executable | +| `NO_COLOR` | `False` | Disable colored output | +| `LOGLEVEL` | `DEBUG` | Python Log level of the main logger | +| `DISABLE_OPENCL` | `False` | Disables sanjuuni GPU acceleration | +| `NO_FAST` | `False` | Disable Sanic worker processes maximization | +| `SPOTIPY_CLIENT_ID` | | The Client ID from your [spotify application] | +| `SPOTIPY_CLIENT_SECRET` | | The Client Secret from your [spotify application] | +| `DATA_CACHE_CLEANUP_INTERVAL` | `300` | Time interval (in seconds) for the data cache cleaner to wait before checking for outdated cache entries. | +| `DATA_CACHE_CLEANUP_AFTER` | `3600` | Time threshold (in seconds) for considering a cache entry outdated. Cache entries older than this will be removed. | + +And [Sanic Builtin values]. + +## Docker Compose + +```yml +version: "2.0" +services: + youcube: + image: ghcr.io/cc-youcube/youcube:latest + restart: always + hostname: youcube + ports: + - 5000:5000 +``` + +[spotify application]: https://developer.spotify.com/dashboard/applications +[pip]: https://pip.pypa.io/en/stable/installation +[yt-dlp/FFmpeg]: https://github.com/yt-dlp/FFmpeg-Builds +[FFmpeg 5.1+]: https://ffmpeg.org +[sanjuuni]: https://github.com/MCJack123/sanjuuni +[Python 3.7+]: https://www.python.org/downloads +[sanic]: https://sanic.dev +[yt-dlp]: https://pypi.org/project/yt-dlp +[ujson]: https://pypi.org/project/ujson +[spotipy]: https://pypi.org/project/spotipy +[doc]: https://youcube.madefor.cc/api +[preview]: .README/preview-server.png +[Python Version: 3.7+]: https://img.shields.io/badge/Python-3.7+-green?style=for-the-badge&logo=Python&logoColor=white +[Python Lint Workflow Status]: https://img.shields.io/github/actions/workflow/status/CC-YouCube/server/pylint.yml?branch=main&label=Python%20Lint&logo=github&style=for-the-badge +[Sanic Builtin values]: https://sanic.dev/en/guide/running/configuration.md#builtin-values \ No newline at end of file diff --git a/src/youcube/yc_download.py b/src/youcube/yc_download.py index 61a4667..d9a83fd 100644 --- a/src/youcube/yc_download.py +++ b/src/youcube/yc_download.py @@ -1,280 +1,287 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" -Download Functionality of YC -""" - -# Built-in modules -from tempfile import TemporaryDirectory -from asyncio import run_coroutine_threadsafe -from os import listdir, getenv -from os.path import join, dirname, abspath - -# Local modules -from yc_logging import YTDLPLogger, logger, NO_COLOR -from yc_magic import run_with_live_output -from yc_colours import Foreground, RESET -from yc_spotify import SpotifyURLProcessor -from yc_utils import ( - remove_ansi_escape_codes, - remove_whitespace, - cap_width_and_height, - create_data_folder_if_not_present, - is_audio_already_downloaded, - is_video_already_downloaded, - get_audio_name, - get_video_name -) - -try: - from ujson import dumps -except ModuleNotFoundError: - from json import dumps - -# pip modules -from yt_dlp import YoutubeDL -from sanic import Websocket - -# pylint settings -# pylint: disable=pointless-string-statement -# pylint: disable=fixme -# pylint: disable=too-many-locals -# pylint: disable=too-many-arguments -# pylint: disable=too-many-branches - -DATA_FOLDER = join(dirname(abspath(__file__)), "data") -FFMPEG_PATH = getenv("FFMPEG_PATH", "ffmpeg") -SANJUUNI_PATH = getenv("SANJUUNI_PATH", "sanjuuni") -DISABLE_OPENCL = bool(getenv("DISABLE_OPENCL")) - - -def download_video( - temp_dir: str, - media_id: str, - resp: Websocket, - loop, - width: int, - height: int -): - """ - Converts the downloaded video to 32vid - """ - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Converting video to 32vid ..." - })), loop) - - if NO_COLOR: - prefix = "[Sanjuuni]" - else: - prefix = f"{Foreground.BRIGHT_YELLOW}[Sanjuuni]{RESET} " - - def handler(line): - logger.debug("%s%s", prefix, line) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": line - })), loop) - - returncode = run_with_live_output( - [ - SANJUUNI_PATH, - "--width=" + str(width), - "--height=" + str(height), - "-i", join(temp_dir, listdir(temp_dir)[0]), - "--raw", - "-o", join( - DATA_FOLDER, - get_video_name(media_id, width, height) - ), - "--disable-opencl" if DISABLE_OPENCL else "" - ], - handler - ) - - if returncode != 0: - logger.warning("Sanjuuni exited with %s", returncode) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "error", - "message": "Faild to convert video!" - })), loop) - - -def download_audio(temp_dir: str, media_id: str, resp: Websocket, loop): - """ - Converts the downloaded audio to dfpwm - """ - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Converting audio to dfpwm ..." - })), loop) - - if NO_COLOR: - prefix = "[FFmpeg]" - else: - prefix = f"{Foreground.BRIGHT_GREEN}[FFmpeg]{RESET} " - - def handler(line): - logger.debug("%s%s", prefix, line) - # TODO: send message to resp - - returncode = run_with_live_output( - [ - FFMPEG_PATH, - "-i", join(temp_dir, listdir(temp_dir)[0]), - "-f", "dfpwm", - "-ar", "48000", - "-ac", "1", - join(DATA_FOLDER, get_audio_name(media_id)) - ], - handler - ) - - if returncode != 0: - logger.warning("FFmpeg exited with %s", returncode) - run_coroutine_threadsafe(resp.send(dumps({ - "action": "error", - "message": "Faild to convert audio!" - })), loop) - - -def download( - url: str, - resp: Websocket, - loop, - width: int, - height: int, - spotify_url_processor: SpotifyURLProcessor -) -> str: - """ - Downloads and converts the media from the give URL - """ - - is_video = width is not None and height is not None - - # cap height and width - if width and height: - width, height = cap_width_and_height(width, height) - - def my_hook(info): - """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" - if info.get('status') == "downloading": - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": remove_ansi_escape_codes( - f"download {remove_whitespace(info.get('_percent_str'))} " - f"ETA {info.get('_eta_str')}" - ) - })), loop) - - with TemporaryDirectory(prefix="youcube-") as temp_dir: - yt_dl_options = { - "format": - "worst[ext=mp4]/worst" if is_video else - "worstaudio/worst", - "outtmpl": join(temp_dir, "%(id)s.%(ext)s"), - "default_search": "auto", - "restrictfilenames": True, - "extract_flat": "in_playlist", - "progress_hooks": [my_hook], - "logger": YTDLPLogger() - } - - yt_dl = YoutubeDL(yt_dl_options) - - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Getting resource information ..." - })), loop) - - playlist_videos = [] - - if spotify_url_processor: - # Spotify FIXME: The first media key is sometimes duplicated - processed_url = spotify_url_processor.auto(url) - if processed_url: - if isinstance(processed_url, list): - url = spotify_url_processor.auto(processed_url[0]) - processed_url.pop(0) - playlist_videos = processed_url - else: - url = processed_url - - data = yt_dl.extract_info(url, download=False) - - if data.get("extractor") == "generic": - data["id"] = 'g' + data.get("webpage_url_domain") + data.get("id") - - """ - If the data is a playlist, we need to get the first video and return it, - also, we need to grep all video in the playlist to provide support. - """ - if data.get("_type") == "playlist": - for video in data.get("entries"): - playlist_videos.append(video.get("id")) - - playlist_videos.pop(0) - - data = data["entries"][0] - - """ - If the video is extract from a playlist, - the video is extracted flat, - so we need to get missing information by running the extractor again. - """ - if data.get("extractor") == "youtube" and ( - data.get("view_count") is None or - data.get("like_count") is None - ): - data = yt_dl.extract_info(data.get("id"), download=False) - - media_id = data.get("id") - - if data.get("is_live"): - return { - "action": "error", - "message": "Livestreams are not supported" - } - - create_data_folder_if_not_present() - - audio_downloaded = is_audio_already_downloaded(media_id) - video_downloaded = is_video_already_downloaded(media_id, width, height) - - if not audio_downloaded or (not video_downloaded and is_video): - run_coroutine_threadsafe(resp.send(dumps({ - "action": "status", - "message": "Downloading resource ..." - })), loop) - - yt_dl.process_ie_result(data, download=True) - - # TODO: Thread audio & video download - - if not audio_downloaded: - download_audio(temp_dir, media_id, resp, loop) - - if not video_downloaded and is_video: - download_video(temp_dir, media_id, resp, loop, width, height) - - out = { - "action": "media", - "id": media_id, - # "fulltitle": data.get("fulltitle"), - "title": data.get("title"), - "like_count": data.get("like_count"), - "view_count": data.get("view_count"), - # "upload_date": data.get("upload_date"), - # "tags": data.get("tags"), - # "description": data.get("description"), - # "categories": data.get("categories"), - # "channel_name": data.get("channel"), - # "channel_id": data.get("channel_id") - } - - # Only return playlist_videos if there are videos in playlist_videos - if len(playlist_videos) > 0: - out["playlist_videos"] = playlist_videos - - return out +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Download Functionality of YC +""" + +# Built-in modules +from tempfile import TemporaryDirectory +from asyncio import run_coroutine_threadsafe +from os import listdir, getenv +from os.path import join, dirname, abspath + +# Local modules +#TODO: change sanic logging format +from yc_logging import YTDLPLogger, logger, NO_COLOR +from yc_magic import run_with_live_output +from yc_colours import Foreground, RESET +from yc_spotify import SpotifyURLProcessor +from yc_utils import ( + remove_ansi_escape_codes, + remove_whitespace, + cap_width_and_height, + create_data_folder_if_not_present, + is_audio_already_downloaded, + is_video_already_downloaded, + get_audio_name, + get_video_name +) + +try: + from ujson import dumps +except ModuleNotFoundError: + from json import dumps + +# pip modules +from yt_dlp import YoutubeDL +from sanic import Websocket + +# pylint settings +# pylint: disable=pointless-string-statement +# pylint: disable=fixme +# pylint: disable=too-many-locals +# pylint: disable=too-many-arguments +# pylint: disable=too-many-branches + +DATA_FOLDER = join(dirname(abspath(__file__)), "data") +FFMPEG_PATH = getenv("FFMPEG_PATH", "ffmpeg") +SANJUUNI_PATH = getenv("SANJUUNI_PATH", "sanjuuni") +DISABLE_OPENCL = bool(getenv("DISABLE_OPENCL")) + + +def download_video( + temp_dir: str, + media_id: str, + resp: Websocket, + loop, + width: int, + height: int +): + """ + Converts the downloaded video to 32vid + """ + run_coroutine_threadsafe(resp.send(dumps({ + "action": "status", + "message": "Converting video to 32vid ..." + })), loop) + + if NO_COLOR: + prefix = "[Sanjuuni]" + else: + prefix = f"{Foreground.BRIGHT_YELLOW}[Sanjuuni]{RESET} " + + def handler(line): + logger.debug("%s%s", prefix, line) + run_coroutine_threadsafe(resp.send(dumps({ + "action": "status", + "message": line + })), loop) + + returncode = run_with_live_output( + [ + SANJUUNI_PATH, + "--width=" + str(width), + "--height=" + str(height), + "-i", join(temp_dir, listdir(temp_dir)[0]), + "--raw", + "-o", join( + DATA_FOLDER, + get_video_name(media_id, width, height) + ), + "--disable-opencl" if DISABLE_OPENCL else "" + ], + handler + ) + + if returncode != 0: + logger.warning("Sanjuuni exited with %s", returncode) + run_coroutine_threadsafe(resp.send(dumps({ + "action": "error", + "message": "Faild to convert video!" + })), loop) + + +def download_audio(temp_dir: str, media_id: str, resp: Websocket, loop): + """ + Converts the downloaded audio to dfpwm + """ + run_coroutine_threadsafe(resp.send(dumps({ + "action": "status", + "message": "Converting audio to dfpwm ..." + })), loop) + + if NO_COLOR: + prefix = "[FFmpeg]" + else: + prefix = f"{Foreground.BRIGHT_GREEN}[FFmpeg]{RESET} " + + def handler(line): + logger.debug("%s%s", prefix, line) + # TODO: send message to resp + + returncode = run_with_live_output( + [ + FFMPEG_PATH, + "-i", join(temp_dir, listdir(temp_dir)[0]), + "-f", "dfpwm", + "-ar", "48000", + "-ac", "1", + join(DATA_FOLDER, get_audio_name(media_id)) + ], + handler + ) + + if returncode != 0: + logger.warning("FFmpeg exited with %s", returncode) + run_coroutine_threadsafe(resp.send(dumps({ + "action": "error", + "message": "Faild to convert audio!" + })), loop) + + +def download( + url: str, + resp: Websocket, + loop, + width: int, + height: int, + spotify_url_processor: SpotifyURLProcessor +) -> (dict[str, any], list): + """ + Downloads and converts the media from the give URL + """ + + is_video = width is not None and height is not None + + # cap height and width + if width and height: + width, height = cap_width_and_height(width, height) + + def my_hook(info): + """https://github.com/yt-dlp/yt-dlp#adding-logger-and-progress-hook""" + if info.get('status') == "downloading": + run_coroutine_threadsafe(resp.send(dumps({ + "action": "status", + "message": remove_ansi_escape_codes( + f"download {remove_whitespace(info.get('_percent_str'))} " + f"ETA {info.get('_eta_str')}" + ) + })), loop) + + # FIXME: Cleanup on Exception + with TemporaryDirectory(prefix="youcube-") as temp_dir: + yt_dl_options = { + "format": + "worst[ext=mp4]/worst" if is_video else + "worstaudio/worst", + "outtmpl": join(temp_dir, "%(id)s.%(ext)s"), + "default_search": "auto", + "restrictfilenames": True, + "extract_flat": "in_playlist", + "progress_hooks": [my_hook], + "logger": YTDLPLogger() + } + + yt_dl = YoutubeDL(yt_dl_options) + + run_coroutine_threadsafe(resp.send(dumps({ + "action": "status", + "message": "Getting resource information ..." + })), loop) + + playlist_videos = [] + + if spotify_url_processor: + # Spotify FIXME: The first media key is sometimes duplicated + processed_url = spotify_url_processor.auto(url) + if processed_url: + if isinstance(processed_url, list): + url = spotify_url_processor.auto(processed_url[0]) + processed_url.pop(0) + playlist_videos = processed_url + else: + url = processed_url + + data = yt_dl.extract_info(url, download=False) + + if data.get("extractor") == "generic": + data["id"] = 'g' + data.get("webpage_url_domain") + data.get("id") + + """ + If the data is a playlist, we need to get the first video and return it, + also, we need to grep all video in the playlist to provide support. + """ + if data.get("_type") == "playlist": + for video in data.get("entries"): + playlist_videos.append(video.get("id")) + + playlist_videos.pop(0) + + data = data["entries"][0] + + """ + If the video is extract from a playlist, + the video is extracted flat, + so we need to get missing information by running the extractor again. + """ + if data.get("extractor") == "youtube" and ( + data.get("view_count") is None or + data.get("like_count") is None + ): + data = yt_dl.extract_info(data.get("id"), download=False) + + media_id = data.get("id") + + if data.get("is_live"): + return { + "action": "error", + "message": "Livestreams are not supported" + } + + create_data_folder_if_not_present() + + audio_downloaded = is_audio_already_downloaded(media_id) + video_downloaded = is_video_already_downloaded(media_id, width, height) + + if not audio_downloaded or (not video_downloaded and is_video): + run_coroutine_threadsafe(resp.send(dumps({ + "action": "status", + "message": "Downloading resource ..." + })), loop) + + yt_dl.process_ie_result(data, download=True) + + # TODO: Thread audio & video download + + if not audio_downloaded: + download_audio(temp_dir, media_id, resp, loop) + + if not video_downloaded and is_video: + download_video(temp_dir, media_id, resp, loop, width, height) + + out = { + "action": "media", + "id": media_id, + # "fulltitle": data.get("fulltitle"), + "title": data.get("title"), + "like_count": data.get("like_count"), + "view_count": data.get("view_count"), + # "upload_date": data.get("upload_date"), + # "tags": data.get("tags"), + # "description": data.get("description"), + # "categories": data.get("categories"), + # "channel_name": data.get("channel"), + # "channel_id": data.get("channel_id") + } + + # Only return playlist_videos if there are videos in playlist_videos + if len(playlist_videos) > 0: + out["playlist_videos"] = playlist_videos + + files = [] + files.append(get_audio_name(media_id)) + if is_video: + files.append(get_video_name(media_id, width, height)) + + return out, files diff --git a/src/youcube/youcube.py b/src/youcube/youcube.py index 13f9158..6aa3fd3 100644 --- a/src/youcube/youcube.py +++ b/src/youcube/youcube.py @@ -6,8 +6,10 @@ """ # built-in modules -from os.path import join -from os import getenv +from datetime import datetime +from time import sleep +from os.path import join, exists +from os import getenv, remove from asyncio import get_event_loop from typing import ( Union, @@ -18,6 +20,7 @@ ) from base64 import b64encode from shutil import which +from multiprocessing import Manager try: from ujson import ( @@ -83,6 +86,8 @@ # pylint: disable=fixme # pylint: disable=multiple-statements +logger = setup_logging() + def get_vid(vid_file: str, tracker: int) -> List[str]: """ @@ -109,37 +114,6 @@ def get_chunk(media_file: str, chunkindex: int) -> bytes: return chunk - -class UntrustedProxy(Exception): - """ - Occurs when someone connects through an untrusted proxy - """ - - def __str__(self) -> str: - return "A client is not using a trusted proxy!" - -# pylint: disable=redefined-outer-name - - -def get_client_ip(request: Request, trusted_proxies: list) -> str: - """ - Returns the real client IP - """ - peername_host = request.ip - - if trusted_proxies is None: - return peername_host - - if peername_host in trusted_proxies: - x_forwarded_for = request.headers.get('X-Forwarded-For') - - if x_forwarded_for is not None: - x_forwarded_for = x_forwarded_for.split(",")[0] - - return x_forwarded_for or request.headers.get('True-Client-Ip') - - raise UntrustedProxy - # pylint: enable=redefined-outer-name @@ -169,18 +143,13 @@ def assert_resp( return None -logger = setup_logging() - # pylint: disable=duplicate-code spotify_client_id = getenv("SPOTIPY_CLIENT_ID") spotify_client_secret = getenv("SPOTIPY_CLIENT_SECRET") # pylint: disable-next=invalid-name spotipy = None -# TODO: only print once - if spotify_client_id and spotify_client_secret: - logger.info("Spotipy Enabled") spotipy = Spotify( auth_manager=SpotifyClientCredentials( client_id=spotify_client_id, @@ -188,8 +157,6 @@ def assert_resp( cache_handler=MemoryCacheHandler() ) ) -else: - logger.info("Spotipy Disabled") # pylint: disable-next=invalid-name spotify_url_processor = None @@ -208,13 +175,13 @@ class Actions: # pylint: disable=missing-function-docstring @staticmethod - async def request_media(message: dict, resp: Websocket): + async def request_media(message: dict, resp: Websocket, request: Request): loop = get_event_loop() # get "url" url = message.get("url") if error := assert_resp("url", url, str): return error # TODO: assert_resp width and height - return await run_function_in_thread_from_async_function( + out, files = await run_function_in_thread_from_async_function( download, url, resp, @@ -223,10 +190,12 @@ async def request_media(message: dict, resp: Websocket): message.get("height"), spotify_url_processor ) + for file in files: + request.app.shared_ctx.data[file] = datetime.now() + return out @staticmethod - async def get_chunk(message: dict, _unused): - # TODO: clear cache + async def get_chunk(message: dict, _unused, request: Request): # get "chunkindex" chunkindex = message.get("chunkindex") if error := assert_resp("chunkindex", chunkindex, int): return error @@ -236,11 +205,13 @@ async def get_chunk(message: dict, _unused): if error := assert_resp("media_id", media_id, str): return error if is_save(media_id): + file_name = get_audio_name(message.get("id")) file = join( DATA_FOLDER, - get_audio_name(message.get("id")) + file_name ) - + + request.app.shared_ctx.data[file_name] = datetime.now() chunk = get_chunk(file, chunkindex) return { @@ -254,7 +225,7 @@ async def get_chunk(message: dict, _unused): } @staticmethod - async def get_vid(message: dict, _unused): + async def get_vid(message: dict, _unused, request: Request): # get "line" tracker = message.get("tracker") if error := assert_resp("tracker", tracker, int): return error @@ -275,10 +246,13 @@ async def get_vid(message: dict, _unused): width, height = cap_width_and_height(width, height) if is_save(media_id): + file_name = get_video_name(message.get('id'), width, height) file = join( DATA_FOLDER, - get_video_name(message.get('id'), width, height) + file_name ) + + request.app.shared_ctx.data[file_name] = datetime.now() return { "action": "vid", @@ -347,34 +321,57 @@ def default(self, request: Request, exception: Union[SanicException, Exception]) if not method.startswith('__'): actions[method] = getattr(Actions, method) -trusted_proxies = getenv("TRUSTED_PROXIES") -# pylint: disable-next=invalid-name -proxies = None - -if trusted_proxies is not None: - proxies = [] - for proxy in trusted_proxies.split(","): - proxies.append(proxy) - -# TODO: only print once - -if which(FFMPEG_PATH) is None: - logger.warning("FFmpeg not found.") - -if which(SANJUUNI_PATH) is None: - logger.warning("Sanjuuni not found.") +DATA_CACHE_CLEANUP_INTERVAL = int(getenv("DATA_CACHE_CLEANUP_INTERVAL", "300")) +DATA_CACHE_CLEANUP_AFTER = int(getenv("DATA_CACHE_CLEANUP_AFTER", "3600")) + + +def data_cache_cleaner(data: dict): + try: + while True: + sleep(DATA_CACHE_CLEANUP_INTERVAL) + for file_name, last_used in data.items(): + if (datetime.now() - last_used).total_seconds() > DATA_CACHE_CLEANUP_AFTER: + file_path = join(DATA_FOLDER, file_name) + if exists(file_path): + remove(file_path) + logger.debug(f'Deleted "{file_name}"') + data.pop(file_name) + + except KeyboardInterrupt: + pass + + +@app.main_process_ready +async def ready(app: Sanic, _): + if DATA_CACHE_CLEANUP_INTERVAL > 0 and DATA_CACHE_CLEANUP_AFTER > 0: + app.manager.manage("Data-Cache-Cleaner", data_cache_cleaner, {"data": app.shared_ctx.data}) + +@app.main_process_start +async def main_start(app: Sanic): + + app.shared_ctx.data = Manager().dict() + + if which(FFMPEG_PATH) is None: + logger.warning("FFmpeg not found.") + + if which(SANJUUNI_PATH) is None: + logger.warning("Sanjuuni not found.") + + if spotipy: + logger.info("Spotipy Enabled") + else: + logger.info("Spotipy Disabled") @app.websocket("/") # pylint: disable-next=invalid-name async def wshandler(request: Request, ws: Websocket): """Handels web-socket requests""" - client_ip = get_client_ip(request, proxies) if NO_COLOR: - prefix = f"[{client_ip}] " + prefix = f"[{request.client_ip}] " else: - prefix = f"{Foreground.BLUE}[{client_ip}]{RESET} " + prefix = f"{Foreground.BLUE}[{request.client_ip}]{RESET} " logger.info("%sConnected!", prefix) @@ -398,7 +395,7 @@ async def wshandler(request: Request, ws: Websocket): })) if message.get("action") in actions: - response = await actions[message.get("action")](message, ws) + response = await actions[message.get("action")](message, ws, request) await ws.send(dumps(response))