fix ts, remove js rendering, edit during playback

apps/desktop/src-tauri/src/editor_instance.rs

@@ -9,8 +9,9 @@ use cap_rendering::{ProjectUniforms, RecordingDecoders, RenderOptions, RenderVid
 use std::ops::Deref;
 use std::{path::PathBuf, sync::Arc};
 use tokio::sync::{mpsc, watch, Mutex};
+use tracing::instrument::WithSubscriber;
 
-type PreviewFrameInstruction = (u32, ProjectConfiguration);
+type PreviewFrameInstruction = u32;
 
 pub struct EditorState {
     pub playhead_position: u32,
@@ -30,6 +31,10 @@ pub struct EditorInstance {
     pub state: Mutex<EditorState>,
     on_state_change: Box<dyn Fn(&EditorState) + Send + Sync + 'static>,
     pub preview_tx: watch::Sender<Option<PreviewFrameInstruction>>,
+    pub project_config: (
+        watch::Sender<ProjectConfiguration>,
+        watch::Receiver<ProjectConfiguration>,
+    ),
 }
 
 impl EditorInstance {
@@ -112,6 +117,11 @@ impl EditorInstance {
 
         let (preview_tx, preview_rx) = watch::channel(None);
 
+        let project_config = std::fs::read_to_string(project_path.join("project-config.json"))
+            .ok()
+            .and_then(|s| serde_json::from_str(&s).ok())
+            .unwrap_or_default();
+
         let this = Arc::new(Self {
             id: video_id,
             project_path,
@@ -128,6 +138,7 @@ impl EditorInstance {
             }),
             on_state_change: Box::new(on_state_change),
             preview_tx,
+            project_config: watch::channel(project_config),
         });
 
         this.state.lock().await.preview_task =
@@ -159,7 +170,7 @@ impl EditorInstance {
         (self.on_state_change)(&state);
     }
 
-    pub async fn start_playback(self: Arc<Self>, project: ProjectConfiguration) {
+    pub async fn start_playback(self: Arc<Self>) {
         let (mut handle, prev) = {
             let Ok(mut state) = self.state.try_lock() else {
                 return;
@@ -174,7 +185,7 @@ impl EditorInstance {
                 decoders: self.decoders.clone(),
                 recordings: self.recordings,
                 start_frame_number,
-                project,
+                project: self.project_config.0.subscribe(),
             }
             .start()
             .await;
@@ -210,18 +221,23 @@ impl EditorInstance {
 
     fn spawn_preview_renderer(
         self: Arc<Self>,
-        mut preview_rx: watch::Receiver<Option<PreviewFrameInstruction>>,
+        mut preview_rx: watch::Receiver<Option<u32>>,
     ) -> tokio::task::JoinHandle<()> {
         tokio::spawn(async move {
             loop {
                 preview_rx.changed().await.unwrap();
-                let Some((frame_number, project)) = preview_rx.borrow().deref().clone() else {
+                let Some(frame_number) = preview_rx.borrow().deref().clone() else {
                     continue;
                 };
 
-                let Some(time) = project.timeline.as_ref().map(|timeline| {
-                    timeline.get_recording_time(frame_number as f64 / FPS as f64)
-                }).unwrap_or(Some(frame_number as f64 / FPS as f64)) else {
+                let project = self.project_config.1.borrow().clone();
+
+                let Some(time) = project
+                    .timeline
+                    .as_ref()
+                    .map(|timeline| timeline.get_recording_time(frame_number as f64 / FPS as f64))
+                    .unwrap_or(Some(frame_number as f64 / FPS as f64))
+                else {
                     continue;
                 };
 

apps/desktop/src-tauri/src/lib.rs

@@ -882,7 +882,6 @@ async fn render_to_file_impl(
 #[derive(Deserialize, specta::Type, tauri_specta::Event, Debug, Clone)]
 struct RenderFrameEvent {
     frame_number: u32,
-    project: ProjectConfiguration,
 }
 
 #[derive(Serialize, specta::Type, tauri_specta::Event, Debug, Clone)]
@@ -907,10 +906,10 @@ pub struct AudioData {
 
 #[tauri::command]
 #[specta::specta]
-async fn start_playback(app: AppHandle, video_id: String, project: ProjectConfiguration) {
+async fn start_playback(app: AppHandle, video_id: String) {
     upsert_editor_instance(&app, video_id)
         .await
-        .start_playback(project)
+        .start_playback()
         .await
 }
 
@@ -931,7 +930,7 @@ async fn stop_playback(app: AppHandle, video_id: String) {
 struct SerializedEditorInstance {
     frames_socket_url: String,
     recording_duration: f64,
-    saved_project_config: Option<ProjectConfiguration>,
+    saved_project_config: ProjectConfiguration,
     recordings: ProjectRecordings,
     path: PathBuf,
 }
@@ -947,11 +946,10 @@ async fn create_editor_instance(
     Ok(SerializedEditorInstance {
         frames_socket_url: format!("ws://localhost:{}{FRAMES_WS_PATH}", editor_instance.ws_port),
         recording_duration: editor_instance.recordings.duration(),
-        saved_project_config: std::fs::read_to_string(
-            editor_instance.project_path.join("project-config.json"),
-        )
-        .ok()
-        .and_then(|s| serde_json::from_str(&s).ok()),
+        saved_project_config: {
+            let project_config = editor_instance.project_config.1.borrow();
+            project_config.clone()
+        },
         recordings: editor_instance.recordings,
         path: editor_instance.project_path.clone(),
     })
@@ -1367,14 +1365,16 @@ async fn set_playhead_position(app: AppHandle, video_id: String, frame_number: u
 
 #[tauri::command]
 #[specta::specta]
-async fn save_project_config(app: AppHandle, video_id: String, config: ProjectConfiguration) {
+async fn set_project_config(app: AppHandle, video_id: String, config: ProjectConfiguration) {
     let editor_instance = upsert_editor_instance(&app, video_id).await;
 
     std::fs::write(
         editor_instance.project_path.join("project-config.json"),
         serde_json::to_string_pretty(&json!(config)).unwrap(),
     )
     .unwrap();
+
+    editor_instance.project_config.0.send(config).ok();
 }
 
 #[tauri::command(async)]
@@ -2048,7 +2048,7 @@ pub fn run() {
             stop_playback,
             set_playhead_position,
             open_in_finder,
-            save_project_config,
+            set_project_config,
             open_editor,
             open_main_window,
             permissions::open_permission_settings,
@@ -2234,10 +2234,7 @@ pub fn run() {
         .expect("error while running tauri application")
         .run(|handle, event| match event {
             #[cfg(target_os = "macos")]
-            tauri::RunEvent::Reopen {
-                has_visible_windows,
-                ..
-            } => open_main_window(handle.clone()),
+            tauri::RunEvent::Reopen { .. } => open_main_window(handle.clone()),
             _ => {}
         });
 }
@@ -2293,10 +2290,7 @@ async fn create_editor_instance_impl(app: &AppHandle, video_id: String) -> Arc<E
     RenderFrameEvent::listen_any(app, {
         let instance = instance.clone();
         move |e| {
-            instance
-                .preview_tx
-                .send(Some((e.payload.frame_number, e.payload.project)))
-                .ok();
+            instance.preview_tx.send(Some(e.payload.frame_number)).ok();
         }
     });
 

apps/desktop/src-tauri/src/playback.rs

@@ -16,7 +16,7 @@ pub struct Playback {
     pub render_constants: Arc<RenderVideoConstants>,
     pub decoders: RecordingDecoders,
     pub start_frame_number: u32,
-    pub project: ProjectConfiguration,
+    pub project: watch::Receiver<ProjectConfiguration>,
     pub recordings: ProjectRecordings,
 }
 
@@ -52,10 +52,10 @@ impl Playback {
             let start = Instant::now();
 
             let mut frame_number = self.start_frame_number + 1;
-            let uniforms = ProjectUniforms::new(&self.render_constants, &self.project);
 
             let duration = self
                 .project
+                .borrow()
                 .timeline()
                 .map(|t| t.duration())
                 .unwrap_or(f64::MAX);
@@ -76,7 +76,9 @@ impl Playback {
                     break;
                 };
 
-                let time = if let Some(timeline) = self.project.timeline() {
+                let project = self.project.borrow().clone();
+
+                let time = if let Some(timeline) = project.timeline() {
                     match timeline.get_recording_time(frame_number as f64 / FPS as f64) {
                         Some(time) => time,
                         None => break,
@@ -85,20 +87,20 @@ impl Playback {
                     frame_number as f64 / FPS as f64
                 };
 
-                let debug = Instant::now();
                 tokio::select! {
                     _ = stop_rx.changed() => {
                        break;
                     },
                     Some((screen_frame, camera_frame)) = self.decoders.get_frames((time * FPS as f64) as u32) => {
                         // println!("decoded frame in {:?}", debug.elapsed());
+                        let uniforms = ProjectUniforms::new(&self.render_constants, &project);
 
                         self
                             .renderer
                             .render_frame(
                                 screen_frame,
                                 camera_frame,
-                                self.project.background.source.clone(),
+                                project.background.source.clone(),
                                 uniforms.clone()
                             )
                             .await;
@@ -141,7 +143,7 @@ struct AudioPlayback {
     stop_rx: watch::Receiver<bool>,
     start_frame_number: u32,
     duration: f64,
-    project: ProjectConfiguration,
+    project: watch::Receiver<ProjectConfiguration>,
 }
 
 impl AudioPlayback {
@@ -172,7 +174,7 @@ impl AudioPlayback {
 
             let next_sample = move || {
                 time += time_inc;
-                let time = self.project.timeline()?.get_recording_time(time)?;
+                let time = self.project.borrow().timeline()?.get_recording_time(time)?;
 
                 let index = time / duration * data.len() as f64;
 

apps/desktop/src/routes/(window-chrome)/index.tsx

@@ -408,7 +408,7 @@ export default function () {
                 e.preventDefault();
 
                 commands.setRecordingOptions({
-                  ...data.options,
+                  ...options.data,
                   audioInputName: null,
                 });
               }}

apps/desktop/src/routes/(window-chrome)/settings/recordings.tsx

@@ -107,22 +107,24 @@ function RecordingItem(props: {
       </div>
       <div class="flex items-center">
         <button
+          type="button"
           onClick={(e) => {
             e.stopPropagation();
             props.onOpenFolder();
           }}
           class="p-2 hover:bg-gray-200 rounded-full mr-2"
         >
-          <IconLucideFolder size={20} />
+          <IconLucideFolder class="size-5" />
         </button>
         <button
+          type="button"
           onClick={(e) => {
             e.stopPropagation();
             props.onClick();
           }}
           class="p-2 hover:bg-gray-200 rounded-full"
         >
-          <IconLucideEye size={20} />
+          <IconLucideEye class="size-5" />
         </button>
       </div>
     </li>

apps/desktop/src/routes/(window-chrome)/settings/screenshots.tsx

@@ -106,22 +106,24 @@ function ScreenshotItem(props: {
       </div>
       <div class="flex items-center">
         <button
+          type="button"
           onClick={(e) => {
             e.stopPropagation();
             props.onOpenFolder();
           }}
           class="p-2 hover:bg-gray-200 rounded-full mr-2"
         >
-          <IconLucideFolder size={20} />
+          <IconLucideFolder class="size-6" />
         </button>
         <button
+          type="button"
           onClick={(e) => {
             e.stopPropagation();
             props.onClick();
           }}
           class="p-2 hover:bg-gray-200 rounded-full"
         >
-          <IconLucideEye size={20} />
+          <IconLucideEye class="size-6" />
         </button>
       </div>
     </li>

apps/desktop/src/routes/editor/Editor.tsx

@@ -89,7 +89,6 @@ function Inner() {
   const renderFrame = throttle((time: number) => {
     events.renderFrameEvent.emit({
       frame_number: Math.max(Math.floor(time * 30), 0),
-      project: project,
     });
   }, 1000 / 60);
 
@@ -123,7 +122,7 @@ function Inner() {
         await commands.stopPlayback(videoId);
         setPlaying(false);
       } else {
-        await commands.startPlayback(videoId, project);
+        await commands.startPlayback(videoId);
         setPlaying(true);
       }
     } catch (error) {

apps/desktop/src/routes/editor/Player.tsx

@@ -181,9 +181,7 @@ export function Player() {
             <button
               type="button"
               onClick={() =>
-                commands
-                  .startPlayback(videoId, project)
-                  .then(() => setPlaying(true))
+                commands.startPlayback(videoId).then(() => setPlaying(true))
               }
             >
               <IconCapPlayCircle class="size-[1.5rem]" />

apps/desktop/src/routes/editor/Timeline.tsx

@@ -53,14 +53,16 @@ export function Timeline() {
         commands.setPlayheadPosition(
           videoId,
           Math.round(
-            30 * editorInstance.recordingDuration * ((e.clientX - left) / width)
+            30 *
+              editorInstance.recordingDuration *
+              ((e.clientX - left!) / width!)
           )
         );
       }}
       onMouseMove={(e) => {
         const { left, width } = timelineBounds;
         setPreviewTime(
-          editorInstance.recordingDuration * ((e.clientX - left) / width)
+          editorInstance.recordingDuration * ((e.clientX - left!) / width!)
         );
       }}
       onMouseLeave={() => {
@@ -224,7 +226,7 @@ export function Timeline() {
 
                       const newEnd =
                         end +
-                        ((event.clientX - downEvent.clientX) / width) *
+                        ((event.clientX - downEvent.clientX) / width!) *
                           duration();
 
                       setProject(

apps/desktop/src/routes/editor/context.ts

@@ -42,7 +42,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider(
           trackStore(project);
         },
         debounce(() => {
-          commands.saveProjectConfig(editorInstanceContext.videoId, project);
+          commands.setProjectConfig(editorInstanceContext.videoId, project);
         }),
         { defer: true }
       )

apps/desktop/src/routes/editor/editorInstanceContext.ts

@@ -31,7 +31,6 @@ export const [EditorInstanceContextProvider, useEditorInstanceContext] =
       ws.onopen = () => {
         events.renderFrameEvent.emit({
           frame_number: Math.floor(0),
-          project: instance.data.savedProjectConfig ?? DEFAULT_PROJECT_CONFIG,
         });
       };
       ws.binaryType = "arraybuffer";