diff --git a/Cargo.lock b/Cargo.lock index c3a18cb..d0042d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -727,6 +727,7 @@ dependencies = [ "humantime-serde", "lazy_static", "log", + "openssl-sys", "quick-xml", "regex", "reqwest", @@ -1071,6 +1072,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "111.22.0+1.1.1q" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.75" @@ -1080,6 +1090,7 @@ dependencies = [ "autocfg", "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/Cargo.toml b/Cargo.toml index 463a18f..40fb5de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ tokio = { version = "1.20.0", features = ["full"] } # Web actix-web = "4" reqwest = { version = "0.11", features = ["gzip", "json"] } +openssl-sys = { version = "0.9", features = ["vendored"] } # Utilities anyhow = "1.0" @@ -24,7 +25,7 @@ humantime = "2.1.0" humantime-serde = "1.1.1" serde = { version = "1.0", features = ["derive"] } toml = "0.5" -quick-xml = { version = "0.23", features = [ "serialize" ] } +quick-xml = { version = "0.23", features = ["serialize"] } chrono = { version = "0.4.0", features = ["serde"] } regex = "1" serde_regex = "1.1.0" @@ -36,3 +37,4 @@ log = "0.4" [profile.release] lto = true +strip = true diff --git a/README.md b/README.md new file mode 100644 index 0000000..fa13cbc --- /dev/null +++ b/README.md @@ -0,0 +1,139 @@ +# hoshinova + +> Monitor YouTube channels and automatically run +> [ytarchive](https://github.com/Kethsar/ytarchive) when the channel goes live. + +**⚠️ Unstable Software**: This program is under heavy development. It works, but +will still undergo a lot of breaking changes. Upgrade with caution. + +## Install + +Make sure you have [ytarchive](https://github.com/Kethsar/ytarchive) and +[ffmpeg](https://ffmpeg.org/) installed and executable in your PATH +([guide](https://github.com/HoloArchivists/hollow_memories)). + +You can +[download the latest release](https://github.com/HoloArchivists/hoshinova/releases), +or build it yourself. You'll need to have [Rust](https://www.rust-lang.org/) +installed. + +```bash +# Clone the repository +git clone https://github.com/HoloArchivists/hoshinova + +# Build and run +cd hoshinova && cargo run --release +``` + +## Configure + +Copy the `config.example.toml` file to `config.toml` and edit the file as +needed. + +### ytarchive configuration + +```toml +[ytarchive] +executable_path = "ytarchive" +working_directory = "temp" +args = [ + "--vp9", "--thumbnail", "--add-metadata", "--threads", "4", + "--output", "%(upload_date)s %(title)s [%(channel)s] (%(id)s)" +] +quality = "best" +``` + +The default configuration should work for most cases. If you don't have +`ytarchive` in your PATH, you can specify absolute path in the `executable_path` +section (for example, `/home/user/bin/ytarchive`). + +You can also set a different `working_directory`. This is the place where +ytarchive will download videos to while it's live. After it's done, the files +will be moved to the `output_directory` configured in each channel (see below). + +By default, the `--wait` flag is added automatically. You can add more flags +too, if you need to use cookies, change the number of threads, etc. Just note +that each argument needs to be a separate item in the list (for example, +`["--threads", "4"]` instead of `["--threads 4"]`). + +### scrapers and notifiers + +```toml +[scraper.rss] +poll_interval = "30s" +``` + +Right now there's only an RSS scraper. More may be added in the future. You can +change the `poll_interval`, which specifies how long to wait between checking +the RSS feeds of each channel. + +```toml +[notifier.discord] +webhook_url = "https://discordapp.com/api/webhooks/123456789012345678/abcdefghijklmnopqrstuvwxyz" +notify_on = ["waiting", "recording", "done", "failed"] +``` + +This part is optional. You can remove this section if you don't want any +notifications. + +Right now you can only send notifications to Discord. You can get the +`webhook_url` by following +[these instructions](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks). +The `notify_on` setting lets you specify which events you want to be notified +about. Right now there are only 4 events: + +| Event | Description | +| ----------- | ---------------------------------------------------------- | +| `waiting` | The stream waiting room is available but it's not live yet | +| `recording` | The stream has just started and is being recorded | +| `done` | The stream is over | +| `failed` | Something went wrong while recording the stream | + +### channel configuration + +```toml +[[channel]] +id = "UCP0BspO_AMEe3aQqqpo89Dg" +name = "Moona Hoshinova" +filters = ["(?i)MoonUtau|Karaoke|Archive"] +outpath = "./videos/moona" +``` + +This part can be copy-pasted multiple times to monitor and record multiple +channels. The `id` field is the channel ID. It's the ending part of e.g. +`https://www.youtube.com/channel/UCP0BspO_AMEe3aQqqpo89Dg`. + +> If you have a `https://www.youtube.com/c/SomeName` URL you can use this +> bookmarklet to convert it to a `/channel/` URL: +> +> ``` +> javascript:window.location=ytInitialData.metadata.channelMetadataRenderer.channelUrl +> ``` + +The `name` can be anything, it's just to help you identify the channel in the +config file. + +`filters` is a list of regular expressions to match on video titles. You can +[check the syntax here](https://docs.rs/regex/latest/regex/#syntax). + +`outpath` is the output folder where you want the resulting videos to be moved +to. + +## Creating release builds + +Use the helper script `build.sh` to generate optimized release binaries for +multiple targets. It uses `cross-rs`, which uses Docker, to automatically set up +the build environment for cross-compilation. + +If you run into any linking issues, run `cargo clean` and try again. + +## Support + +This is very early in development. New features will be added, and existing +features may be changed or removed without notice. We do not make any guarantees +on the software's stability. + +That being said, we're open to accepting input, bug reports, and contributions. +If you run into any issues, feel free to +[hop on our Discord](https://discord.gg/y53h4pHB3n), or +[file an issue](https://github.com/HoloArchivists/hoshinova/issues/new/choose). diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..68bf1f8 --- /dev/null +++ b/build.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +# +# This script is used to generate release binaries for hoshinova. It uses +# cross-rs to cross-compile the project to multiple architectures. +# +# If you run into any issues with linking with glibc, run `cargo clean`. +# See: https://github.com/cross-rs/cross/issues/724 +# + +# Install cross +cargo install cross --git https://github.com/cross-rs/cross + +# Set up latest version of upx from git. The current latest release +# (v3.96-git-d7ba31cab8ce+) does not work with the binary generated by rust. +# See: https://github.com/upx/upx/issues/476 +mkdir -p target +upxdir=target/.upx +upx=./$upxdir/src/upx.out + +if pushd $upxdir; then git pull; popd; else git clone https://github.com/upx/upx.git $upxdir; fi +pushd $upxdir + echo '*' > .gitignore + git submodule update --init --recursive + make +popd + +targets=(x86_64-unknown-linux-musl aarch64-unknown-linux-musl x86_64-pc-windows-gnu) + +for target in "${targets[@]}"; do + echo "Building for $target" + cross build --target $target --release + $upx target/$target/release/hoshinova \ + || $upx target/$target/release/hoshinova.exe +done + +echo "Done!" diff --git a/config.example.toml b/config.example.toml index 6a58d58..914f391 100644 --- a/config.example.toml +++ b/config.example.toml @@ -5,8 +5,7 @@ executable_path = "ytarchive" working_directory = "temp" args = [ - "--vp9", "--thumbnail", "--add-metadata", - "--threads", "4", + "--vp9", "--thumbnail", "--add-metadata", "--threads", "4", "--output", "%(upload_date)s %(title)s [%(channel)s] (%(id)s)" ] quality = "best" @@ -18,6 +17,11 @@ poll_interval = "30s" webhook_url = "https://discordapp.com/api/webhooks/123456789012345678/abcdefghijklmnopqrstuvwxyz" notify_on = ["waiting", "recording", "done", "failed"] +# Coming soon, a web interface to view and manage tasks. +# Optional, remove this section to disable. +[webserver] +bind_address = "127.0.0.1:1104" + [[channel]] id = "UCP0BspO_AMEe3aQqqpo89Dg" name = "Moona Hoshinova" diff --git a/src/config.rs b/src/config.rs index 73c609b..0854b64 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,6 +7,7 @@ pub struct Config { pub ytarchive: YtarchiveConfig, pub scraper: ScraperConfig, pub notifier: NotifierConfig, + pub webserver: Option, pub channel: Vec, } @@ -40,6 +41,11 @@ pub struct NotifierDiscordConfig { pub notify_on: Vec, } +#[derive(Clone, Deserialize, Debug)] +pub struct WebserverConfig { + pub bind_address: String, +} + #[derive(Clone, Deserialize, Debug)] pub struct ChannelConfig { pub id: String, diff --git a/src/main.rs b/src/main.rs index 07db94c..63b1bd2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ extern crate log; use crate::module::Module; use crate::msgbus::MessageBus; +use actix_web::{App, HttpServer}; use anyhow::{anyhow, Result}; use clap::Parser; use std::{process::Command, sync::Arc}; @@ -10,6 +11,7 @@ use tokio::sync::RwLock; mod config; mod module; mod msgbus; +mod web; pub static APP_NAME: &str = concat!(env!("CARGO_PKG_NAME"), " v", env!("CARGO_PKG_VERSION")); pub static APP_USER_AGENT: &str = concat!( @@ -67,7 +69,7 @@ fn test_ytarchive(path: &str) -> Result { #[tokio::main] async fn main() -> Result<()> { // Initialize logging - env_logger::init(); + env_logger::init_from_env(env_logger::Env::default().default_filter_or("info")); info!("{}", APP_NAME); // Parse command line arguments @@ -108,6 +110,24 @@ async fn main() -> Result<()> { let h_recorder = run_module!(bus, module::recorder::YTArchive::new(config.clone())); let h_notifier = run_module!(bus, module::notifier::Discord::new(config.clone())); + // Start webserver + let h_server = tokio::spawn(async move { + let config = config.clone(); + let config = &*config.read().await; + if let Some(webserver) = &config.webserver { + let ws = HttpServer::new(|| App::new().configure(web::configure)) + .bind(webserver.bind_address.clone()) + .map_err(|e| anyhow!("Failed to bind to address: {}", e))? + .run(); + info!("Starting webserver on {}", webserver.bind_address); + return ws + .await + .map_err(|e| anyhow!("Failed to start webserver: {}", e)); + }; + debug!("No webserver configured"); + Ok::<(), anyhow::Error>(()) + }); + // Listen for signals let closer = bus.add_tx(); let h_signal = tokio::spawn(async move { @@ -123,7 +143,7 @@ async fn main() -> Result<()> { let h_bus = tokio::task::spawn(async move { bus.start().await }); // Wait for all tasks to finish - futures::try_join!(h_scraper, h_recorder, h_notifier, h_signal, h_bus) + futures::try_join!(h_scraper, h_recorder, h_notifier, h_signal, h_bus, h_server) .map(|_| ()) .map_err(|e| anyhow!("Task errored: {}", e)) } diff --git a/src/module/notifier.rs b/src/module/notifier.rs index bd3b442..27ebee0 100644 --- a/src/module/notifier.rs +++ b/src/module/notifier.rs @@ -1,7 +1,7 @@ -use super::{Message, Module, Notification, Task, TaskStatus}; +use super::{Message, Module, Notification, TaskStatus}; use crate::msgbus::BusTx; use crate::{config::Config, APP_NAME, APP_USER_AGENT}; -use anyhow::{anyhow, Result}; +use anyhow::Result; use async_trait::async_trait; use reqwest::Client; use serde::Serialize; diff --git a/src/module/scraper.rs b/src/module/scraper.rs index 497ddde..f0cbf93 100644 --- a/src/module/scraper.rs +++ b/src/module/scraper.rs @@ -17,13 +17,13 @@ pub struct RSS { client: Client, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize)] struct RSSFeed { #[serde(rename = "entry", default)] entries: Vec, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize)] struct FeedEntry { #[serde(rename = "videoId")] video_id: String, @@ -31,23 +31,20 @@ struct FeedEntry { channel_id: String, title: String, author: Author, - published: chrono::DateTime, - updated: chrono::DateTime, group: MediaGroup, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize)] struct Author { name: String, - uri: String, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize)] struct MediaGroup { thumbnail: Thumbnail, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize)] struct Thumbnail { url: String, } diff --git a/src/web/mod.rs b/src/web/mod.rs new file mode 100644 index 0000000..4a39238 --- /dev/null +++ b/src/web/mod.rs @@ -0,0 +1,10 @@ +use actix_web::{get, web::ServiceConfig, HttpResponse, Responder}; + +pub fn configure(app: &mut ServiceConfig) { + app.service(hello); +} + +#[get("/")] +async fn hello() -> impl Responder { + HttpResponse::Ok().body("Hello world!") +}