Clean up for first release

Cargo.toml

@@ -14,6 +14,7 @@ tokio = { version = "1.20.0", features = ["full"] }
 # Web
 actix-web = "4"
 reqwest = { version = "0.11", features = ["gzip", "json"] }
+openssl-sys = { version = "0.9", features = ["vendored"] }
 
 # Utilities
 anyhow = "1.0"
@@ -24,7 +25,7 @@ humantime = "2.1.0"
 humantime-serde = "1.1.1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "0.5"
-quick-xml = { version = "0.23", features = [ "serialize" ] }
+quick-xml = { version = "0.23", features = ["serialize"] }
 chrono = { version = "0.4.0", features = ["serde"] }
 regex = "1"
 serde_regex = "1.1.0"
@@ -36,3 +37,4 @@ log = "0.4"
 
 [profile.release]
 lto = true
+strip = true

README.md

@@ -0,0 +1,139 @@
+# hoshinova
+
+> Monitor YouTube channels and automatically run
+> [ytarchive](https://github.com/Kethsar/ytarchive) when the channel goes live.
+
+**⚠️ Unstable Software**: This program is under heavy development. It works, but
+will still undergo a lot of breaking changes. Upgrade with caution.
+
+## Install
+
+Make sure you have [ytarchive](https://github.com/Kethsar/ytarchive) and
+[ffmpeg](https://ffmpeg.org/) installed and executable in your PATH
+([guide](https://github.com/HoloArchivists/hollow_memories)).
+
+You can
+[download the latest release](https://github.com/HoloArchivists/hoshinova/releases),
+or build it yourself. You'll need to have [Rust](https://www.rust-lang.org/)
+installed.
+
+```bash
+# Clone the repository
+git clone https://github.com/HoloArchivists/hoshinova
+
+# Build and run
+cd hoshinova && cargo run --release
+```
+
+## Configure
+
+Copy the `config.example.toml` file to `config.toml` and edit the file as
+needed.
+
+### ytarchive configuration
+
+```toml
+[ytarchive]
+executable_path = "ytarchive"
+working_directory = "temp"
+args = [
+  "--vp9", "--thumbnail", "--add-metadata", "--threads", "4",
+  "--output", "%(upload_date)s %(title)s [%(channel)s] (%(id)s)"
+]
+quality = "best"
+```
+
+The default configuration should work for most cases. If you don't have
+`ytarchive` in your PATH, you can specify absolute path in the `executable_path`
+section (for example, `/home/user/bin/ytarchive`).
+
+You can also set a different `working_directory`. This is the place where
+ytarchive will download videos to while it's live. After it's done, the files
+will be moved to the `output_directory` configured in each channel (see below).
+
+By default, the `--wait` flag is added automatically. You can add more flags
+too, if you need to use cookies, change the number of threads, etc. Just note
+that each argument needs to be a separate item in the list (for example,
+`["--threads", "4"]` instead of `["--threads 4"]`).
+
+### scrapers and notifiers
+
+```toml
+[scraper.rss]
+poll_interval = "30s"
+```
+
+Right now there's only an RSS scraper. More may be added in the future. You can
+change the `poll_interval`, which specifies how long to wait between checking
+the RSS feeds of each channel.
+
+```toml
+[notifier.discord]
+webhook_url = "https://discordapp.com/api/webhooks/123456789012345678/abcdefghijklmnopqrstuvwxyz"
+notify_on = ["waiting", "recording", "done", "failed"]
+```
+
+This part is optional. You can remove this section if you don't want any
+notifications.
+
+Right now you can only send notifications to Discord. You can get the
+`webhook_url` by following
+[these instructions](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
+The `notify_on` setting lets you specify which events you want to be notified
+about. Right now there are only 4 events:
+
+| Event       | Description                                                |
+| ----------- | ---------------------------------------------------------- |
+| `waiting`   | The stream waiting room is available but it's not live yet |
+| `recording` | The stream has just started and is being recorded          |
+| `done`      | The stream is over                                         |
+| `failed`    | Something went wrong while recording the stream            |
+
+### channel configuration
+
+```toml
+[[channel]]
+id = "UCP0BspO_AMEe3aQqqpo89Dg"
+name = "Moona Hoshinova"
+filters = ["(?i)MoonUtau|Karaoke|Archive"]
+outpath = "./videos/moona"
+```
+
+This part can be copy-pasted multiple times to monitor and record multiple
+channels. The `id` field is the channel ID. It's the ending part of e.g.
+`https://www.youtube.com/channel/UCP0BspO_AMEe3aQqqpo89Dg`.
+
+> If you have a `https://www.youtube.com/c/SomeName` URL you can use this
+> bookmarklet to convert it to a `/channel/` URL:
+>
+> ```
+> javascript:window.location=ytInitialData.metadata.channelMetadataRenderer.channelUrl
+> ```
+
+The `name` can be anything, it's just to help you identify the channel in the
+config file.
+
+`filters` is a list of regular expressions to match on video titles. You can
+[check the syntax here](https://docs.rs/regex/latest/regex/#syntax).
+
+`outpath` is the output folder where you want the resulting videos to be moved
+to.
+
+## Creating release builds
+
+Use the helper script `build.sh` to generate optimized release binaries for
+multiple targets. It uses `cross-rs`, which uses Docker, to automatically set up
+the build environment for cross-compilation.
+
+If you run into any linking issues, run `cargo clean` and try again.
+
+## Support
+
+This is very early in development. New features will be added, and existing
+features may be changed or removed without notice. We do not make any guarantees
+on the software's stability.
+
+That being said, we're open to accepting input, bug reports, and contributions.
+If you run into any issues, feel free to
+[hop on our Discord](https://discord.gg/y53h4pHB3n), or
+[file an issue](https://github.com/HoloArchivists/hoshinova/issues/new/choose).

build.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+
+#
+# This script is used to generate release binaries for hoshinova. It uses
+# cross-rs to cross-compile the project to multiple architectures.
+#
+# If you run into any issues with linking with glibc, run `cargo clean`.
+# See: https://github.com/cross-rs/cross/issues/724
+#
+
+# Install cross
+cargo install cross --git https://github.com/cross-rs/cross
+
+# Set up latest version of upx from git. The current latest release
+# (v3.96-git-d7ba31cab8ce+) does not work with the binary generated by rust.
+# See: https://github.com/upx/upx/issues/476
+mkdir -p target
+upxdir=target/.upx
+upx=./$upxdir/src/upx.out
+
+if pushd $upxdir; then git pull; popd; else git clone https://github.com/upx/upx.git $upxdir; fi
+pushd $upxdir
+  echo '*' > .gitignore
+  git submodule update --init --recursive
+  make
+popd
+
+targets=(x86_64-unknown-linux-musl aarch64-unknown-linux-musl x86_64-pc-windows-gnu)
+
+for target in "${targets[@]}"; do
+  echo "Building for $target"
+  cross build --target $target --release
+  $upx target/$target/release/hoshinova \
+  || $upx target/$target/release/hoshinova.exe
+done
+
+echo "Done!"

config.example.toml

@@ -5,8 +5,7 @@
 executable_path = "ytarchive"
 working_directory = "temp"
 args = [
-  "--vp9", "--thumbnail", "--add-metadata",
-  "--threads", "4",
+  "--vp9", "--thumbnail", "--add-metadata", "--threads", "4",
   "--output", "%(upload_date)s %(title)s [%(channel)s] (%(id)s)"
 ]
 quality = "best"
@@ -18,6 +17,11 @@ poll_interval = "30s"
 webhook_url = "https://discordapp.com/api/webhooks/123456789012345678/abcdefghijklmnopqrstuvwxyz"
 notify_on = ["waiting", "recording", "done", "failed"]
 
+# Coming soon, a web interface to view and manage tasks.
+# Optional, remove this section to disable.
+[webserver]
+bind_address = "127.0.0.1:1104"
+
 [[channel]]
 id = "UCP0BspO_AMEe3aQqqpo89Dg"
 name = "Moona Hoshinova"

src/config.rs

@@ -7,6 +7,7 @@ pub struct Config {
     pub ytarchive: YtarchiveConfig,
     pub scraper: ScraperConfig,
     pub notifier: NotifierConfig,
+    pub webserver: Option<WebserverConfig>,
     pub channel: Vec<ChannelConfig>,
 }
 
@@ -40,6 +41,11 @@ pub struct NotifierDiscordConfig {
     pub notify_on: Vec<TaskStatus>,
 }
 
+#[derive(Clone, Deserialize, Debug)]
+pub struct WebserverConfig {
+    pub bind_address: String,
+}
+
 #[derive(Clone, Deserialize, Debug)]
 pub struct ChannelConfig {
     pub id: String,

src/main.rs

@@ -2,6 +2,7 @@
 extern crate log;
 use crate::module::Module;
 use crate::msgbus::MessageBus;
+use actix_web::{App, HttpServer};
 use anyhow::{anyhow, Result};
 use clap::Parser;
 use std::{process::Command, sync::Arc};
@@ -10,6 +11,7 @@ use tokio::sync::RwLock;
 mod config;
 mod module;
 mod msgbus;
+mod web;
 
 pub static APP_NAME: &str = concat!(env!("CARGO_PKG_NAME"), " v", env!("CARGO_PKG_VERSION"));
 pub static APP_USER_AGENT: &str = concat!(
@@ -67,7 +69,7 @@ fn test_ytarchive(path: &str) -> Result<String> {
 #[tokio::main]
 async fn main() -> Result<()> {
     // Initialize logging
-    env_logger::init();
+    env_logger::init_from_env(env_logger::Env::default().default_filter_or("info"));
     info!("{}", APP_NAME);
 
     // Parse command line arguments
@@ -108,6 +110,24 @@ async fn main() -> Result<()> {
     let h_recorder = run_module!(bus, module::recorder::YTArchive::new(config.clone()));
     let h_notifier = run_module!(bus, module::notifier::Discord::new(config.clone()));
 
+    // Start webserver
+    let h_server = tokio::spawn(async move {
+        let config = config.clone();
+        let config = &*config.read().await;
+        if let Some(webserver) = &config.webserver {
+            let ws = HttpServer::new(|| App::new().configure(web::configure))
+                .bind(webserver.bind_address.clone())
+                .map_err(|e| anyhow!("Failed to bind to address: {}", e))?
+                .run();
+            info!("Starting webserver on {}", webserver.bind_address);
+            return ws
+                .await
+                .map_err(|e| anyhow!("Failed to start webserver: {}", e));
+        };
+        debug!("No webserver configured");
+        Ok::<(), anyhow::Error>(())
+    });
+
     // Listen for signals
     let closer = bus.add_tx();
     let h_signal = tokio::spawn(async move {
@@ -123,7 +143,7 @@ async fn main() -> Result<()> {
     let h_bus = tokio::task::spawn(async move { bus.start().await });
 
     // Wait for all tasks to finish
-    futures::try_join!(h_scraper, h_recorder, h_notifier, h_signal, h_bus)
+    futures::try_join!(h_scraper, h_recorder, h_notifier, h_signal, h_bus, h_server)
         .map(|_| ())
         .map_err(|e| anyhow!("Task errored: {}", e))
 }

src/module/notifier.rs

@@ -1,7 +1,7 @@
-use super::{Message, Module, Notification, Task, TaskStatus};
+use super::{Message, Module, Notification, TaskStatus};
 use crate::msgbus::BusTx;
 use crate::{config::Config, APP_NAME, APP_USER_AGENT};
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use async_trait::async_trait;
 use reqwest::Client;
 use serde::Serialize;

src/module/scraper.rs

@@ -17,37 +17,34 @@ pub struct RSS {
     client: Client,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 struct RSSFeed {
     #[serde(rename = "entry", default)]
     entries: Vec<FeedEntry>,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 struct FeedEntry {
     #[serde(rename = "videoId")]
     video_id: String,
     #[serde(rename = "channelId")]
     channel_id: String,
     title: String,
     author: Author,
-    published: chrono::DateTime<chrono::Utc>,
-    updated: chrono::DateTime<chrono::Utc>,
     group: MediaGroup,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 struct Author {
     name: String,
-    uri: String,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 struct MediaGroup {
     thumbnail: Thumbnail,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 struct Thumbnail {
     url: String,
 }

src/web/mod.rs

@@ -0,0 +1,10 @@
+use actix_web::{get, web::ServiceConfig, HttpResponse, Responder};
+
+pub fn configure(app: &mut ServiceConfig) {
+    app.service(hello);
+}
+
+#[get("/")]
+async fn hello() -> impl Responder {
+    HttpResponse::Ok().body("Hello world!")
+}