diff --git a/Cargo.lock b/Cargo.lock index cde6bf2..83d6cc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,7 +171,7 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2", - "time 0.3.11", + "time 0.3.12", "url", ] @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" +checksum = "c91f1f46651137be86f3a2b9a8359f9ab421d04d941c62b5982e1ca21113adf9" [[package]] name = "async-compression" @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" dependencies = [ "proc-macro2", "quote", @@ -334,9 +334,9 @@ checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] name = "bytestring" @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.8" +version = "3.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190814073e85d238f31ff738fcb0bf6910cedeb73376c87cd69291028966fd83" +checksum = "a3dbbb6653e7c55cc8595ad3e1f7be8f32aba4eb7ff7f0fd1163d4f3d137c0a9" dependencies = [ "atty", "bitflags", @@ -395,9 +395,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.2.7" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759bf187376e1afa7b85b959e6a664a3e7a95203415dba952ad19139e798f902" +checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" dependencies = [ "heck", "proc-macro-error", @@ -428,7 +428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" dependencies = [ "percent-encoding", - "time 0.3.11", + "time 0.3.12", "version_check", ] @@ -460,6 +460,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "derive_more" version = "0.99.17" @@ -637,9 +643,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", @@ -677,9 +683,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "607c8a29735385251a339424dd462993c0fed8fa09d378f259377df08c126022" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" @@ -705,6 +711,7 @@ dependencies = [ "async-trait", "chrono", "clap", + "data-encoding", "env_logger", "futures", "humantime", @@ -841,9 +848,9 @@ checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itoa" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] name = "jobserver" @@ -856,9 +863,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.58" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" dependencies = [ "wasm-bindgen", ] @@ -877,9 +884,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b" [[package]] name = "local-channel" @@ -1019,9 +1026,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "os_str_bytes" -version = "6.1.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" [[package]] name = "parking_lot" @@ -1048,9 +1055,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" +checksum = "9423e2b32f7a043629287a536f21951e8c6a82482d0acb1eeebfc90bc2225b22" [[package]] name = "percent-encoding" @@ -1102,9 +1109,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" +checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" dependencies = [ "unicode-ident", ] @@ -1121,9 +1128,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ "proc-macro2", ] @@ -1160,9 +1167,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] @@ -1297,18 +1304,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" dependencies = [ "base64", ] [[package]] name = "ryu" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" [[package]] name = "same-file" @@ -1337,24 +1344,24 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" +checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711" [[package]] name = "serde" -version = "1.0.138" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1578c6245786b9d168c5447eeacfb96856573ca56c9d68fdcf394be134882a47" +checksum = "e590c437916fb6b221e1d00df6e3294f3fccd70ca7e92541c475d6ed6ef5fee2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.138" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023e9b1467aef8a10fb88f25611870ada9800ef7e22afce356bb0d2387b6f27c" +checksum = "34b5b8d809babe02f538c2cfec6f2c1ed10804c0e5a6a041a049a4f5588ccc2e" dependencies = [ "proc-macro2", "quote", @@ -1363,9 +1370,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7" dependencies = [ "itoa", "ryu", @@ -1429,9 +1436,12 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" @@ -1463,9 +1473,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" dependencies = [ "proc-macro2", "quote", @@ -1520,11 +1530,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" +checksum = "74b7cc93fc23ba97fde84f7eea56c55d1ba183f495c6715defdfc7b9cb8c870f" dependencies = [ "itoa", + "js-sys", "libc", "num_threads", "time-macros", @@ -1553,9 +1564,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57aec3cfa4c296db7255446efb4928a6be304b431a806216105542a67b6ca82e" +checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" dependencies = [ "autocfg", "bytes", @@ -1625,9 +1636,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ "cfg-if", "log", @@ -1637,9 +1648,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" dependencies = [ "once_cell", ] @@ -1697,9 +1708,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" +checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" [[package]] name = "unicode-normalization" @@ -1769,9 +1780,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1779,13 +1790,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -1794,9 +1805,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f" +checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" dependencies = [ "cfg-if", "js-sys", @@ -1806,9 +1817,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1816,9 +1827,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" dependencies = [ "proc-macro2", "quote", @@ -1829,15 +1840,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" +checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" [[package]] name = "web-sys" -version = "0.3.58" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" +checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 2539025..2ae00c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ serde_json = "1.0" clap = { version = "3.2.8", features = ["derive"] } env_logger = "0.8.4" log = "0.4" +data-encoding = "2.3.2" [profile.release] lto = true diff --git a/src/config.rs b/src/config.rs index ee2577f..1292813 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,5 @@ -use crate::module::TaskStatus; use anyhow::{anyhow, Result}; +use crate::module::Status; use serde::{Deserialize, Serialize}; use ts_rs::TS; @@ -50,7 +50,7 @@ pub struct NotifierConfig { #[ts(export, export_to = "web/src/bindings/")] pub struct NotifierDiscordConfig { pub webhook_url: String, - pub notify_on: Vec, + pub notify_on: Vec, } #[derive(Clone, TS, Serialize, Deserialize, Debug)] diff --git a/src/main.rs b/src/main.rs index b1d7cbf..508a7c4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -108,6 +108,7 @@ async fn main() -> Result<()> { let config = Arc::new(RwLock::new(config)); let h_scraper = run_module!(bus, module::scraper::RSS::new(config.clone())); let h_recorder = run_module!(bus, module::recorder::YTArchive::new(config.clone())); + let h_json = run_module!(bus, module::recorder::Json::new(config.clone())); let h_notifier = run_module!(bus, module::notifier::Discord::new(config.clone())); let h_webserver = run_module!(bus, module::web::WebServer::new(config.clone())); @@ -129,6 +130,7 @@ async fn main() -> Result<()> { futures::try_join!( h_scraper, h_recorder, + h_json, h_notifier, h_signal, h_bus, diff --git a/src/module/mod.rs b/src/module/mod.rs index 2941b9f..8659a1f 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -1,4 +1,4 @@ -use self::recorder::YTAStatus; +use self::recorder::{JsonStatus, YTAStatus}; use crate::{config::Config, msgbus::BusTx}; use anyhow::Result; use async_trait::async_trait; @@ -18,6 +18,7 @@ pub enum Message { ToRecord(Task), ToNotify(Notification), RecordingStatus(RecordingStatus), + MetadataStatus(MetadataStatus), } #[derive(Debug, Clone, TS, Serialize, Deserialize)] @@ -36,7 +37,14 @@ pub struct Task { #[ts(export, export_to = "web/src/bindings/")] pub struct Notification { pub task: Task, - pub status: TaskStatus, + pub status: Status, +} + +#[derive(Debug, Clone, PartialEq, Serialize, TS)] +#[ts(export, export_to = "web/src/bindings/")] +pub enum Status { + Task(TaskStatus), + Playability(PlayabilityStatus), } #[derive(Debug, Clone, TS)] @@ -46,6 +54,83 @@ pub struct RecordingStatus { pub status: YTAStatus, } +#[derive(Debug, Clone, TS)] +#[ts(export, export_to = "web/src/bindings/")] +pub struct MetadataStatus { + pub task: Task, + pub status: JsonStatus, +} + +#[derive(Debug, Clone, PartialEq, TS)] +#[ts(export, export_to = "web/src/bindings/")] +pub enum PlayabilityStatus { + MembersOnly, + Privated, + Copyrighted, + Removed, + Unlisted, + OnLive, + Ok, + Offline, + LoginRequired, + Unknown, +} +impl Serialize for PlayabilityStatus { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(match self { + PlayabilityStatus::Ok => "ok", + PlayabilityStatus::OnLive => "live", + PlayabilityStatus::Removed => "removed", + PlayabilityStatus::Offline => "offline", + PlayabilityStatus::MembersOnly => "members_only", + PlayabilityStatus::Unknown => "unknown", + PlayabilityStatus::Privated => "privated", + PlayabilityStatus::Unlisted => "unlisted", + PlayabilityStatus::Copyrighted => "copyrighted", + PlayabilityStatus::LoginRequired => "login_required", + }) + } +} + +impl<'de> Deserialize<'de> for PlayabilityStatus { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + match &*s { + "ok" => Ok(PlayabilityStatus::Ok), + "live" => Ok(PlayabilityStatus::OnLive), + "removed" => Ok(PlayabilityStatus::Removed), + "offline" => Ok(PlayabilityStatus::Offline), + "members_only" => Ok(PlayabilityStatus::MembersOnly), + "unknown" => Ok(PlayabilityStatus::Unknown), + "privated" => Ok(PlayabilityStatus::Privated), + "unlisted" => Ok(PlayabilityStatus::Unlisted), + "copyrighted" => Ok(PlayabilityStatus::Copyrighted), + "login_required" => Ok(PlayabilityStatus::LoginRequired), + _ => Err(serde::de::Error::unknown_variant( + &s, + &[ + "ok", + "live", + "removed", + "offline", + "members_only", + "unknown", + "privated", + "unlisted", + "copyrighted", + "login_required", + ], + )), + } + } +} + #[derive(Debug, Clone, PartialEq, TS)] #[ts(export, export_to = "web/src/bindings/")] pub enum TaskStatus { @@ -69,7 +154,7 @@ impl Serialize for TaskStatus { } } -impl<'de> serde::Deserialize<'de> for TaskStatus { +impl<'de> Deserialize<'de> for TaskStatus { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -88,6 +173,50 @@ impl<'de> serde::Deserialize<'de> for TaskStatus { } } +impl<'de> Deserialize<'de> for Status { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + match &*s { + "ok" => Ok(Status::Playability(PlayabilityStatus::Ok)), + "live" => Ok(Status::Playability(PlayabilityStatus::OnLive)), + "removed" => Ok(Status::Playability(PlayabilityStatus::Removed)), + "offline" => Ok(Status::Playability(PlayabilityStatus::Offline)), + "members_only" => Ok(Status::Playability(PlayabilityStatus::MembersOnly)), + "unknown" => Ok(Status::Playability(PlayabilityStatus::Unknown)), + "privated" => Ok(Status::Playability(PlayabilityStatus::Privated)), + "unlisted" => Ok(Status::Playability(PlayabilityStatus::Unlisted)), + "copyrighted" => Ok(Status::Playability(PlayabilityStatus::Copyrighted)), + "login_required" => Ok(Status::Playability(PlayabilityStatus::LoginRequired)), + "waiting" => Ok(Status::Task(TaskStatus::Waiting)), + "recording" => Ok(Status::Task(TaskStatus::Recording)), + "done" => Ok(Status::Task(TaskStatus::Done)), + "failed" => Ok(Status::Task(TaskStatus::Failed)), + _ => Err(serde::de::Error::unknown_variant( + &s, + &[ + "ok", + "live", + "removed", + "offline", + "members_only", + "unknown", + "privated", + "unlisted", + "copyrighted", + "login_required", + "waiting", + "recording", + "done", + "failed", + ], + )), + } + } +} + #[async_trait] pub trait Module { fn new(config: Arc>) -> Self; diff --git a/src/module/notifier.rs b/src/module/notifier.rs index ad4c526..3cf06d3 100644 --- a/src/module/notifier.rs +++ b/src/module/notifier.rs @@ -1,4 +1,4 @@ -use super::{Message, Module, Notification, TaskStatus}; +use super::{Message, Module, Notification, PlayabilityStatus, Status, TaskStatus}; use crate::msgbus::BusTx; use crate::{config::Config, APP_NAME, APP_USER_AGENT}; use anyhow::Result; @@ -83,10 +83,22 @@ impl Module for Discord { } let (title, color) = match status { - TaskStatus::Waiting => ("Waiting for Live", 0xebd045), - TaskStatus::Recording => ("Recording", 0x58b9ff), - TaskStatus::Done => ("Done", 0x45eb45), - TaskStatus::Failed => ("Failed", 0xeb4545), + Status::Task(TaskStatus::Waiting) => ("Waiting for Live", 0xebd045), + Status::Task(TaskStatus::Recording) => ("Recording", 0x58b9ff), + Status::Task(TaskStatus::Done) => ("Done", 0x45eb45), + Status::Task(TaskStatus::Failed) => ("Failed", 0xeb4545), + Status::Playability(PlayabilityStatus::Ok) => ("Available", 0x45eb45), + Status::Playability(PlayabilityStatus::MembersOnly) => ("Members Only", 0xeb4545), + Status::Playability(PlayabilityStatus::Copyrighted) => ("Copyrighted", 0xeb4545), + Status::Playability(PlayabilityStatus::Removed) => ("Removed", 0xeb4545), + Status::Playability(PlayabilityStatus::LoginRequired) => { + ("Login Required", 0xeb4545) + } + Status::Playability(PlayabilityStatus::Unlisted) => ("Unlisted", 0xeb4545), + Status::Playability(PlayabilityStatus::Unknown) => ("Unknown", 0xeb4545), + Status::Playability(PlayabilityStatus::OnLive) => ("Live", 0xeb4545), + Status::Playability(PlayabilityStatus::Privated) => ("Privated", 0xeb4545), + Status::Playability(PlayabilityStatus::Offline) => ("Offline", 0xeb4545), }; let timestamp = chrono::Utc::now().to_rfc3339(); @@ -105,7 +117,7 @@ impl Module for Discord { footer: DiscordEmbedFooter { text: APP_NAME.into(), }, - timestamp: timestamp, + timestamp, thumbnail: DiscordEmbedThumbnail { url: task.video_picture, }, diff --git a/src/module/recorder.rs b/src/module/recorder.rs index cc0df71..d277978 100644 --- a/src/module/recorder.rs +++ b/src/module/recorder.rs @@ -1,14 +1,16 @@ -use super::{Message, Module, Notification, Task, TaskStatus}; -use crate::msgbus::BusTx; -use crate::{config::Config, module::RecordingStatus}; +use super::{Message, Module, Notification, PlayabilityStatus, Status, Task, TaskStatus}; +use crate::{config::Config, module::MetadataStatus, module::RecordingStatus, APP_NAME}; +use crate::{msgbus::BusTx, APP_USER_AGENT}; use anyhow::{anyhow, Result}; use async_trait::async_trait; use chrono::{DateTime, Utc}; +use data_encoding::BASE64URL; use lazy_static::lazy_static; use regex::Regex; -use serde::Serialize; -use std::collections::HashSet; +use reqwest::Client; +use serde::{Deserialize, Serialize}; use std::{ + collections::{HashMap, HashSet}, fs, path::Path, process::Stdio, @@ -23,6 +25,353 @@ use tokio::{ }; use ts_rs::TS; +struct Priority { + video: [u16; 20], + audio: [u16; 7], +} + +const PRIORITY: Priority = Priority { + video: [ + 337, 315, 266, 138, // 2160p60 + 313, 336, // 2160p + 308, // 1440p60 + 271, 264, // 1440p + 335, 303, 299, // 1080p60 + 248, 169, 137, // 1080p + 334, 302, 298, // 720p60 + 247, 136, // 720p + ], + audio: [251, 141, 171, 140, 250, 249, 139], +}; + +#[derive(Clone, Serialize, Deserialize, Debug)] +struct VideoInfo { + title: String, + id: String, + channel_name: String, + channel_url: String, + description: String, + thumbnail: String, + thumbnail_url: String, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct JsonSchema { + video: String, + audio: String, + metadata: VideoInfo, + version: String, + create_time: String, +} + +impl JsonSchema { + fn new(video: String, audio: String, metadata: VideoInfo) -> JsonSchema { + JsonSchema { + video, + audio, + metadata, + version: APP_NAME.to_string(), + create_time: chrono::prelude::Utc::now().to_rfc3339(), + } + } +} + +pub struct Json {} + +impl Json { + /// This function generates a serialiazable struct loosely + /// following the schema of auto-ytarchive-raw + async fn get(client: Client, task: Task, bus: &mut BusTx) -> Result<()> { + let task_name = format!("[{}][{}][{}]", task.video_id, task.channel_name, task.title); + + // Ensure the output directory exists + tokio::fs::create_dir_all(&task.output_directory) + .await + .map_err(|e| anyhow!("Failed to create output directory: {:?}", e))?; + + // Fetch the video page + let mut status = JsonStatus::new(); + bus.send(Message::MetadataStatus(MetadataStatus { + task: task.clone(), + status: status.clone(), + })) + .await?; + let url = format!("https://www.youtube.com/watch?v={}", task.video_id); + let res = client + .get(&url) + .send() + .await + .map_err(|e| anyhow!("Error fetching video page: {}", e))? + .text() + .await + .map_err(|e| anyhow!("Error fetching video page: {}", e))?; + + // Find streams (above playability because we borrow res) + let mut map_itag_url = HashMap::new(); + let itag_re = + Regex::new(r#"itag":(\d+?),"url":"([^"]+)"#).expect("Failed to compile itag regex"); + for capture in itag_re.captures_iter(&res) { + let itag = capture[1].to_string(); + let url = capture[2].to_string(); + if url.contains("noclen") { + map_itag_url.insert(itag, url); + } + } + + // Description + let description; + if res.contains(r#"description":{"simpleText":"#) { + // Should probably refactor the re to avoid this if + let description_re = Regex::new(r#""description":\{"simpleText":"(.+?)"},"#) + .expect("Failed to compile description regex"); + description = match description_re + .captures(&res) + .expect("Description not found") + .get(1) + { + Some(text) => text.as_str().to_string(), + None => "".to_string(), + }; + } else { + description = String::new(); + } + + // Check playability status + let (message, playability_status) = match res { + html if html.contains(r#"offerId":"sponsors_only_video"#) => { + info!("{} Stream is members only", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::MembersOnly), + })), + PlayabilityStatus::MembersOnly, + ) + } + html if html.contains(r#"status":"UNPLAYABLE"#) => { + info!("{} Stream is copyrighted", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::Copyrighted), + })), + PlayabilityStatus::Copyrighted, + ) + } + html if html.contains(r#"status":"LOGIN_REQUIRED"#) => { + info!("{} Stream is private", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::Privated), + })), + PlayabilityStatus::Privated, + ) + } + html if html.contains(r#"status":"ERROR"#) => { + info!("{} Stream is removed", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::Removed), + })), + PlayabilityStatus::Removed, + ) + } + html if html.contains(r#"status":"OK"#) => match html { + html if html.contains("\"isUnlisted\":true") => { + info!("{} Stream is unlisted", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::Unlisted), + })), + PlayabilityStatus::Unlisted, + ) + } + html if html.contains("hlsManifestUrl") => { + info!("{} Stream is live", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::OnLive), + })), + PlayabilityStatus::OnLive, + ) + } + _ => { + info!("{} Stream is available", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::Ok), + })), + PlayabilityStatus::Ok, + ) + } + }, + html if html.contains(r#"status":"LIVE_STREAM_OFFLINE"#) => { + info!("{} Stream is offline", task_name); + ( + (Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::Offline), + })), + PlayabilityStatus::Offline, + ) + } + html if html.contains(r#"status":"LOGIN_REQUIRED"#) => { + info!("{} Stream requires login", task_name); + ( + Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::LoginRequired), + }), + PlayabilityStatus::LoginRequired, + ) + } + _ => { + info!("{} Unknown status", task_name); + ( + Message::ToNotify(Notification { + task: task.clone(), + status: Status::Playability(PlayabilityStatus::Unknown), + }), + PlayabilityStatus::Unknown, + ) + } + }; + bus.send(message).await?; + + let mut video = String::new(); + let mut video_quality = String::new(); + let mut audio = String::new(); + let mut audio_quality = String::new(); + + for itag in PRIORITY.video { + match map_itag_url.get(&itag.to_string()) { + Some(url) => { + video = url.to_string(); + video_quality = itag.to_string() + } + _ => (), + } + } + if video == String::new() { + warn!("{} got empty video sources.", task.video_id) + } + for itag in PRIORITY.audio { + match map_itag_url.get(&itag.to_string()) { + Some(url) => { + audio = url.to_string(); + audio_quality = itag.to_string() + } + _ => (), + } + } + if audio == String::new() { + warn!("{} got empty audio sources.", task.video_id) + } + + status.video_quality = Some(video_quality); + status.audio_quality = Some(audio_quality); + + status.playability = Some(playability_status); + // Getting thumbnail + let image_data = client.get(&url).send().await?.bytes().await?; + let thumbnail = format!("data:image/jpeg;base64,{}", BASE64URL.encode(&image_data)); + + let metadata = VideoInfo { + title: task.title.to_owned(), + id: task.video_id.to_owned(), + thumbnail, + description, + thumbnail_url: task.video_picture.to_owned(), + channel_name: task.channel_name.to_owned(), + channel_url: format!( + "https://www.youtube.com/channel/{}", + task.channel_id.to_owned() + ), + }; + let json = JsonSchema::new(video, audio, metadata); + let json_string = serde_json::to_string_pretty(&json).expect("Failed to serialize JSON"); + tokio::fs::write( + format!("{}/{}.json", &task.output_directory, &task.video_id), + json_string.as_bytes(), + ) + .await?; + status.state = JsonState::Finished; + bus.send(Message::MetadataStatus(MetadataStatus { + task: task.clone(), + status: status.clone(), + })) + .await?; + + Ok(()) + } +} + +#[async_trait] +impl Module for Json { + fn new(_config: Arc>) -> Self { + Self {} + } + + async fn run(&self, tx: &BusTx, rx: &mut mpsc::Receiver) -> Result<()> { + // Listen for new messages + while let Some(message) = rx.recv().await { + match message { + Message::ToRecord(task) => { + debug!("Spawning thread for task: {:?}", task); + let mut tx = tx.clone(); + let client = Client::builder() + .user_agent(APP_USER_AGENT) + .build() + .expect("Failed to create client"); + tokio::spawn(async move { + if let Err(e) = Json::get(client, task, &mut tx).await { + error!("Failed to get json for task: {:?}", e); + }; + }); + } + _ => (), + } + } + + debug!("JSON module finished"); + Ok(()) + } +} + +/// The current state of ytarchive. +#[derive(Debug, Clone, Serialize, TS)] +pub struct JsonStatus { + state: JsonState, + playability: Option, + video_quality: Option, + audio_quality: Option, + output_file: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, TS)] +pub enum JsonState { + Idle, + Finished, +} + +impl JsonStatus { + pub fn new() -> Self { + Self { + state: JsonState::Idle, + playability: None, + video_quality: None, + audio_quality: None, + output_file: None, + } + } +} + pub struct YTArchive { config: Arc>, active_ids: Arc>>, @@ -185,12 +534,13 @@ impl YTArchive { status.parse_line(&line); // Push the current status to the bus - if let Err(_) = bus + if (bus .send(Message::RecordingStatus(RecordingStatus { task: task.clone(), status: status.clone(), })) - .await + .await) + .is_err() { break; } @@ -205,21 +555,21 @@ impl YTArchive { info!("{} Waiting for stream to go live", task_name); Some(Message::ToNotify(Notification { task: task.clone(), - status: TaskStatus::Waiting, + status: Status::Task(TaskStatus::Waiting), })) } YTAState::Recording => { info!("{} Recording started", task_name); Some(Message::ToNotify(Notification { task: task.clone(), - status: TaskStatus::Recording, + status: Status::Task(TaskStatus::Recording), })) } YTAState::Finished => { info!("{} Recording finished", task_name); Some(Message::ToNotify(Notification { task: task.clone(), - status: TaskStatus::Done, + status: Status::Task(TaskStatus::Done), })) } YTAState::AlreadyProcessed => { @@ -230,7 +580,7 @@ impl YTArchive { info!("{} Recording failed: interrupted", task_name); Some(Message::ToNotify(Notification { task: task.clone(), - status: TaskStatus::Failed, + status: Status::Task(TaskStatus::Failed), })) } _ => None, @@ -238,7 +588,7 @@ impl YTArchive { if let Some(message) = message { // Exit the loop if message failed to send - if let Err(_) = bus.send(message).await { + if (bus.send(message).await).is_err() { break; } } diff --git a/src/module/scraper.rs b/src/module/scraper.rs index c3d775a..411f349 100644 --- a/src/module/scraper.rs +++ b/src/module/scraper.rs @@ -1,9 +1,8 @@ use super::{Message, Module, Task}; use crate::{config, msgbus::BusTx, youtube, APP_USER_AGENT}; -use anyhow::{anyhow, Result}; +use anyhow::Result; use async_trait::async_trait; use futures::stream::{self, Stream, StreamExt}; -use lazy_static::lazy_static; use reqwest::Client; use serde::Deserialize; use std::{ @@ -145,7 +144,7 @@ impl Module for RSS { let err = self .run_loop(scraped.clone()) .await - .map(|task| tx.send(Message::ToRecord(task.clone()))) + .map(|task| tx.send(Message::ToRecord(task))) .buffer_unordered(4) .collect::>>() .await diff --git a/web/src/pages/TasksPage.tsx b/web/src/pages/TasksPage.tsx index 6377a9e..b850e91 100644 --- a/web/src/pages/TasksPage.tsx +++ b/web/src/pages/TasksPage.tsx @@ -27,9 +27,30 @@ import { useQueryConfig } from '../api/config'; import { YTAState } from '../bindings/YTAState'; import { Task } from '../bindings/Task'; -const SleepingPanda = React.lazy(() => import('../lotties/SleepingPanda')); +const PlayabilityBadge = ({ state }: { state: State }) => ( + + {stateString(state)} + +); -const TaskStateBadge = ({ state }: { state: YTAState }) => ( +const TaskStateBadge = ({ state }: { state: State }) => (