From a614335193fc9f9e6b0bf4b29c7a8dabed33d4fc Mon Sep 17 00:00:00 2001 From: Hizkia Felix Date: Tue, 10 Jan 2023 14:50:57 +0800 Subject: [PATCH] Add support for ytarchive-0.3.2 --- Cargo.lock | 2 +- Cargo.toml | 2 +- Dockerfile | 4 ++-- src/module/recorder.rs | 36 +++++++++++++++++++++++++++++++++-- src/module/scraper.rs | 37 ++++++++++++++++++++++++++---------- src/msgbus.rs | 4 ++-- web/src/pages/ConfigPage.tsx | 2 +- web/src/pages/TasksPage.tsx | 3 +-- 8 files changed, 69 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cde6bf2..ac94694 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -698,7 +698,7 @@ dependencies = [ [[package]] name = "hoshinova" -version = "0.2.2" +version = "0.2.3" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 4882e0f..105dae0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hoshinova" -version = "0.2.2" +version = "0.2.3" edition = "2021" repository = "https://github.com/HoloArchivists/hoshinova" homepage = "https://github.com/HoloArchivists/hoshinova" diff --git a/Dockerfile b/Dockerfile index 66b8302..491769a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:16 as web-deps WORKDIR /src/web COPY web/package.json web/yarn.lock ./ -RUN yarn install +RUN yarn install --frozen-lockfile # Create base image for building Rust FROM rust:1.62-alpine AS rust-build-image @@ -49,7 +49,7 @@ FROM alpine AS runner WORKDIR /app RUN set -ex; \ apk add --no-cache ffmpeg wget unzip; \ - wget -O /app/ytarchive.zip https://github.com/Kethsar/ytarchive/releases/download/latest/ytarchive_linux_amd64.zip; \ + wget -O /app/ytarchive.zip https://github.com/Kethsar/ytarchive/releases/download/v0.3.2/ytarchive_linux_amd64.zip; \ unzip /app/ytarchive.zip -d /usr/local/bin/; \ rm /app/ytarchive.zip; \ apk del wget unzip; diff --git a/src/module/recorder.rs b/src/module/recorder.rs index f5b7ec7..20eb455 100644 --- a/src/module/recorder.rs +++ b/src/module/recorder.rs @@ -398,6 +398,7 @@ pub enum YTAState { AlreadyProcessed, Ended, Interrupted, + Errored, } fn strip_ansi(s: &str) -> String { @@ -459,7 +460,27 @@ impl YTAStatus { if let Some(x) = parts.next() { self.total_size = Some(strip_ansi(x.trim())); }; - } else if self.version == None && line.starts_with("ytarchive ") { + return; + } else if line.starts_with("Audio Fragments: ") { + self.state = YTAState::Recording; + let mut parts = line.split(';').map(|s| s.split(':').nth(1).unwrap_or("")); + if let Some(x) = parts.next() { + self.audio_fragments = x.trim().parse().ok(); + }; + if let Some(x) = parts.next() { + self.total_size = Some(strip_ansi(x.trim())); + }; + return; + } + + // New versions of ytarchive prepend a timestamp to the output + let line = if self.version == Some("0.3.2".into()) { + line[20..].trim() + } else { + line + }; + + if self.version == None && line.starts_with("ytarchive ") { self.version = Some(strip_ansi(&line[10..])); } else if self.video_quality == None && line.starts_with("Selected quality: ") { self.video_quality = Some(strip_ansi(&line[18..])); @@ -483,7 +504,18 @@ impl YTAStatus { self.output_file = Some(strip_ansi(&line[12..])); } else if line.contains("User Interrupt") { self.state = YTAState::Interrupted; - } else if line.trim().is_empty() || line.contains("Loaded cookie file") { + } else if line.contains("Error retrieving player response") + || line.contains("unable to retrieve") + || line.contains("error writing the muxcmd file") + || line.contains("Something must have gone wrong with ffmpeg") + || line.contains("At least one error occurred") + { + self.state = YTAState::Errored; + } else if line.trim().is_empty() + || line.contains("Loaded cookie file") + || line.starts_with("Video Title: ") + || line.starts_with("Channel: ") + { // Ignore } else { warn!("Unknown ytarchive output: {}", line); diff --git a/src/module/scraper.rs b/src/module/scraper.rs index c13cacb..f1c4f28 100644 --- a/src/module/scraper.rs +++ b/src/module/scraper.rs @@ -63,6 +63,14 @@ impl RSS { let max_age = chrono::Duration::from_std(self.config.read().await.scraper.rss.ignore_older_than) .context("Failed to convert ignore_older_than to chrono::Duration")?; + debug!( + "Ignoring videos older than {}", + max_age + .to_std() + .map(humantime::format_duration) + .map(|s| s.to_string()) + .unwrap_or_else(|_| "???".into()) + ); // Fetch the RSS feed let url = format!( @@ -86,16 +94,25 @@ impl RSS { .filter_map(move |entry| { let mut scraped = scraped.lock().unwrap(); - // Skip if video has already been scraped, or if it's too old - if scraped.contains(&entry.video_id) - || entry.updated < chrono::Utc::now() - max_age - || !channel.filters.iter().any(|filter| { - // Or if the video doesn't match the filters - filter.is_match(&entry.title) - || (channel.match_description - && filter.is_match(&entry.group.description)) - }) - { + if scraped.contains(&entry.video_id) { + // Skip if video has already been scraped + debug!("Skipping {}: already scraped", entry.video_id); + return None; + } else if entry.updated < chrono::Utc::now() - max_age { + // Or if the video is too old + debug!( + "Skipping {}: too old ({} < {})", + entry.video_id, + entry.updated, + chrono::Utc::now() - max_age + ); + return None; + } else if !channel.filters.iter().any(|filter| { + filter.is_match(&entry.title) + || (channel.match_description && filter.is_match(&entry.group.description)) + }) { + // Or if the video doesn't match the filters + debug!("Skipping {}: doesn't match filters", entry.video_id); return None; } diff --git a/src/msgbus.rs b/src/msgbus.rs index c169131..fcd3d68 100644 --- a/src/msgbus.rs +++ b/src/msgbus.rs @@ -45,10 +45,10 @@ impl MessageBus { /// closed. pub async fn start(&mut self) { 'out: while let Some(BusMessage::Message(msg)) = self.mix_rx.recv().await { - for tx in &mut self.mix_tx { + for (n, tx) in &mut self.mix_tx.iter().enumerate() { match tx.try_send(msg.clone()) { Err(e) => { - error!("Failed to send message: {}", e); + error!("Failed to send message to queue {}: {}", n, e); break 'out; } _ => (), diff --git a/web/src/pages/ConfigPage.tsx b/web/src/pages/ConfigPage.tsx index 651e08c..105c8e8 100644 --- a/web/src/pages/ConfigPage.tsx +++ b/web/src/pages/ConfigPage.tsx @@ -77,7 +77,7 @@ const ConfigPage = () => { ? 'Reloading...' : isEditable ? 'Save and apply' - : 'Reload configuration'} + : 'Reload from disk'} {isEditable ? null : ( diff --git a/web/src/pages/TasksPage.tsx b/web/src/pages/TasksPage.tsx index ad82434..96ca92b 100644 --- a/web/src/pages/TasksPage.tsx +++ b/web/src/pages/TasksPage.tsx @@ -25,7 +25,6 @@ import { closeAllModals, openModal } from '@mantine/modals'; import { showNotification } from '@mantine/notifications'; import { useQueryConfig } from '../api/config'; import { YTAState } from '../bindings/YTAState'; -import { Task } from '../bindings/Task'; const SleepingPanda = React.lazy(() => import('../lotties/SleepingPanda')); @@ -40,7 +39,7 @@ const TaskStateBadge = ({ state }: { state: YTAState }) => ( ? 'yellow' : state === 'Idle' || state === 'AlreadyProcessed' || state === 'Ended' ? 'gray' - : state === 'Interrupted' + : state === 'Interrupted' || state === 'Errored' ? 'red' : 'violet' }