New architecture for vk-video initialization. (#857)

compositor_pipeline/src/pipeline.rs

@@ -133,7 +133,7 @@ pub struct PipelineCtx {
     pub download_dir: Arc<PathBuf>,
     pub event_emitter: Arc<EventEmitter>,
     #[cfg(feature = "vk-video")]
-    pub vulkan_ctx: Option<Arc<vk_video::VulkanCtx>>,
+    pub vulkan_ctx: Option<graphics_context::VulkanCtx>,
 }
 
 impl std::fmt::Debug for PipelineCtx {
@@ -156,6 +156,7 @@ impl Pipeline {
                 opts.force_gpu,
                 opts.wgpu_features,
                 Default::default(),
+                None,
             )?),
             #[cfg(not(feature = "vk-video"))]
             None => None,

compositor_pipeline/src/pipeline/decoder/video/vulkan_video.rs

@@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration};
 use compositor_render::{Frame, FrameData, InputId, Resolution};
 use crossbeam_channel::{Receiver, Sender};
 use tracing::{debug, error, span, trace, warn, Level};
-use vk_video::{VulkanCtx, WgpuTexturesDeocder};
+use vk_video::VulkanDevice;
 
 use crate::{
     error::InputInitError,
@@ -17,7 +17,7 @@ pub fn start_vulkan_video_decoder_thread(
     frame_sender: Sender<PipelineEvent<Frame>>,
     input_id: InputId,
 ) -> Result<(), InputInitError> {
-    let Some(vulkan_ctx) = pipeline_ctx.vulkan_ctx.as_ref().map(|ctx| ctx.clone()) else {
+    let Some(vulkan_ctx) = pipeline_ctx.vulkan_ctx.clone() else {
         return Err(InputInitError::VulkanContextRequiredForVulkanDecoder);
     };
 
@@ -33,7 +33,7 @@ pub fn start_vulkan_video_decoder_thread(
             )
             .entered();
             run_decoder_thread(
-                vulkan_ctx,
+                vulkan_ctx.device,
                 init_result_sender,
                 chunks_receiver,
                 frame_sender,
@@ -47,12 +47,12 @@ pub fn start_vulkan_video_decoder_thread(
 }
 
 fn run_decoder_thread(
-    vulkan_ctx: Arc<VulkanCtx>,
+    vulkan_device: Arc<VulkanDevice>,
     init_result_sender: Sender<Result<(), InputInitError>>,
     chunks_receiver: Receiver<PipelineEvent<EncodedChunk>>,
     frame_sender: Sender<PipelineEvent<Frame>>,
 ) {
-    let mut decoder = match WgpuTexturesDeocder::new(vulkan_ctx) {
+    let mut decoder = match vulkan_device.create_wgpu_textures_decoder() {
         Ok(decoder) => {
             init_result_sender.send(Ok(())).unwrap();
             decoder

compositor_pipeline/src/pipeline/graphics_context.rs

@@ -1,14 +1,23 @@
 use crate::error::InitPipelineError;
-use compositor_render::{create_wgpu_ctx, error::InitRendererEngineError};
+use compositor_render::{create_wgpu_ctx, error::InitRendererEngineError, WgpuComponents};
 use std::sync::Arc;
 
+#[cfg(feature = "vk-video")]
+#[derive(Debug, Clone)]
+pub struct VulkanCtx {
+    pub device: Arc<vk_video::VulkanDevice>,
+    pub instance: Arc<vk_video::VulkanInstance>,
+}
+
 #[derive(Debug)]
 pub struct GraphicsContext {
     pub device: Arc<wgpu::Device>,
     pub queue: Arc<wgpu::Queue>,
+    pub adapter: Arc<wgpu::Adapter>,
+    pub instance: Arc<wgpu::Instance>,
 
     #[cfg(feature = "vk-video")]
-    pub vulkan_ctx: Option<Arc<vk_video::VulkanCtx>>,
+    pub vulkan_ctx: Option<VulkanCtx>,
 }
 
 impl GraphicsContext {
@@ -17,6 +26,7 @@ impl GraphicsContext {
         force_gpu: bool,
         features: wgpu::Features,
         limits: wgpu::Limits,
+        mut compatible_surface: Option<&mut wgpu::Surface<'_>>,
     ) -> Result<Self, InitPipelineError> {
         use compositor_render::{required_wgpu_features, set_required_wgpu_limits};
         use tracing::warn;
@@ -26,22 +36,36 @@ impl GraphicsContext {
 
         let limits = set_required_wgpu_limits(limits);
 
-        match vk_video::VulkanCtx::new(vulkan_features, limits.clone()) {
-            Ok(ctx) => Ok(GraphicsContext {
-                device: ctx.wgpu_ctx.device.clone(),
-                queue: ctx.wgpu_ctx.queue.clone(),
-                vulkan_ctx: Some(ctx.into()),
+        match vk_video::VulkanInstance::new().and_then(|instance| {
+            let device =
+                instance.create_device(vulkan_features, limits.clone(), &mut compatible_surface)?;
+
+            Ok((instance, device))
+        }) {
+            Ok((instance, device)) => Ok(GraphicsContext {
+                device: device.wgpu_device.clone(),
+                queue: device.wgpu_queue.clone(),
+                adapter: device.wgpu_adapter.clone(),
+                instance: instance.wgpu_instance.clone(),
+                vulkan_ctx: Some(VulkanCtx { instance, device }),
             }),
 
             Err(err) => {
                 warn!("Cannot initialize vulkan video decoding context. Reason: {err}. Initializing without vulkan video support.");
 
-                let (device, queue) = create_wgpu_ctx(force_gpu, features, limits)
+                let WgpuComponents {
+                    instance,
+                    adapter,
+                    device,
+                    queue,
+                } = create_wgpu_ctx(force_gpu, features, limits, compatible_surface.as_deref())
                     .map_err(InitRendererEngineError::FailedToInitWgpuCtx)?;
 
                 Ok(GraphicsContext {
                     device,
                     queue,
+                    adapter,
+                    instance,
                     vulkan_ctx: None,
                 })
             }
@@ -53,10 +77,21 @@ impl GraphicsContext {
         force_gpu: bool,
         features: wgpu::Features,
         limits: wgpu::Limits,
+        compatible_surface: Option<&mut wgpu::Surface<'_>>,
     ) -> Result<Self, InitPipelineError> {
-        let (device, queue) = create_wgpu_ctx(force_gpu, features, limits)
+        let WgpuComponents {
+            instance,
+            adapter,
+            device,
+            queue,
+        } = create_wgpu_ctx(force_gpu, features, limits, compatible_surface.as_deref())
             .map_err(InitRendererEngineError::FailedToInitWgpuCtx)?;
 
-        Ok(GraphicsContext { device, queue })
+        Ok(GraphicsContext {
+            device,
+            queue,
+            adapter,
+            instance,
+        })
     }
 }

compositor_render/src/lib.rs

@@ -20,7 +20,7 @@ pub use state::RendererOptions;
 pub use state::RendererSpec;
 
 pub use wgpu::WgpuFeatures;
-pub use wgpu::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits};
+pub use wgpu::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits, WgpuComponents};
 
 pub mod image {
     pub use crate::transformations::image_renderer::{ImageSource, ImageSpec, ImageType};

compositor_render/src/wgpu.rs

@@ -7,7 +7,7 @@ pub(crate) mod texture;
 pub(crate) mod utils;
 
 pub(crate) use ctx::WgpuCtx;
-pub use ctx::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits};
+pub use ctx::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits, WgpuComponents};
 pub use wgpu::Features as WgpuFeatures;
 
 #[must_use]

compositor_render/src/wgpu/ctx.rs

@@ -34,7 +34,8 @@ impl WgpuCtx {
                 Self::new_from_device_queue(device, queue)?
             }
             None => {
-                let (device, queue) = create_wgpu_ctx(force_gpu, features, Default::default())?;
+                let WgpuComponents { device, queue, .. } =
+                    create_wgpu_ctx(force_gpu, features, Default::default(), None)?;
                 Self::new_from_device_queue(device, queue)?
             }
         };
@@ -101,11 +102,20 @@ pub fn set_required_wgpu_limits(limits: wgpu::Limits) -> wgpu::Limits {
     }
 }
 
+#[derive(Clone)]
+pub struct WgpuComponents {
+    pub device: Arc<wgpu::Device>,
+    pub queue: Arc<wgpu::Queue>,
+    pub adapter: Arc<wgpu::Adapter>,
+    pub instance: Arc<wgpu::Instance>,
+}
+
 pub fn create_wgpu_ctx(
     force_gpu: bool,
     features: wgpu::Features,
     limits: wgpu::Limits,
-) -> Result<(Arc<wgpu::Device>, Arc<wgpu::Queue>), CreateWgpuCtxError> {
+    compatible_surface: Option<&wgpu::Surface<'_>>,
+) -> Result<WgpuComponents, CreateWgpuCtxError> {
     let instance = wgpu::Instance::new(wgpu::InstanceDescriptor {
         backends: wgpu::Backends::all(),
         ..Default::default()
@@ -117,7 +127,7 @@ pub fn create_wgpu_ctx(
     let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptionsBase {
         power_preference: wgpu::PowerPreference::HighPerformance,
         force_fallback_adapter: false,
-        compatible_surface: None,
+        compatible_surface,
     }))
     .ok_or(CreateWgpuCtxError::NoAdapter)?;
 
@@ -148,7 +158,12 @@ pub fn create_wgpu_ctx(
         },
         None,
     ))?;
-    Ok((device.into(), queue.into()))
+    Ok(WgpuComponents {
+        instance: instance.into(),
+        adapter: adapter.into(),
+        device: device.into(),
+        queue: queue.into(),
+    })
 }
 
 fn uniform_bind_group_layout(device: &wgpu::Device) -> wgpu::BindGroupLayout {

integration_tests/examples/manual_graphics_initialization.rs

@@ -9,27 +9,18 @@ fn main() {
     };
     use live_compositor::config::read_config;
 
-    let graphics_context =
-        GraphicsContext::new(false, wgpu::Features::default(), wgpu::Limits::default()).unwrap();
+    let graphics_context = GraphicsContext::new(
+        false,
+        wgpu::Features::default(),
+        wgpu::Limits::default(),
+        None,
+    )
+    .unwrap();
 
     let _device = graphics_context.device.clone();
     let _queue = graphics_context.queue.clone();
-
-    let _adapter = graphics_context
-        .vulkan_ctx
-        .as_ref()
-        .unwrap()
-        .wgpu_ctx
-        .adapter
-        .clone();
-
-    let _instance = graphics_context
-        .vulkan_ctx
-        .as_ref()
-        .unwrap()
-        .wgpu_ctx
-        .instance
-        .clone();
+    let _adapter = graphics_context.adapter.clone();
+    let _instance = graphics_context.instance.clone();
 
     let config = read_config();
 

integration_tests/examples/raw_channel_input.rs

@@ -40,7 +40,7 @@ fn main() {
     ffmpeg_next::format::network::init();
     let config = read_config();
     logger::init_logger(config.logger);
-    let ctx = GraphicsContext::new(false, Default::default(), Default::default()).unwrap();
+    let ctx = GraphicsContext::new(false, Default::default(), Default::default(), None).unwrap();
     let (wgpu_device, wgpu_queue) = (ctx.device.clone(), ctx.queue.clone());
     // no chromium support, so we can ignore _event_loop
     let (pipeline, _event_loop) = Pipeline::new(Options {

integration_tests/examples/raw_channel_output.rs

@@ -53,7 +53,7 @@ fn main() {
     logger::init_logger(read_config().logger);
     let mut config = read_config();
     config.queue_options.ahead_of_time_processing = true;
-    let ctx = GraphicsContext::new(false, Default::default(), Default::default()).unwrap();
+    let ctx = GraphicsContext::new(false, Default::default(), Default::default(), None).unwrap();
     let (wgpu_device, wgpu_queue) = (ctx.device.clone(), ctx.queue.clone());
     // no chromium support, so we can ignore _event_loop
     let (pipeline, _event_loop) = Pipeline::new(Options {

integration_tests/examples/vulkan.rs

@@ -73,7 +73,7 @@ fn client_code() -> Result<()> {
     const INPUT_PORT: u16 = 8006;
     const OUTPUT_PORT: u16 = 8004;
 
-    const VIDEOS: u16 = 1;
+    const VIDEOS: u16 = 6;
     start_ffmpeg_receive(Some(OUTPUT_PORT), None)?;
 
     let config = read_config();
@@ -176,7 +176,7 @@ fn client_code() -> Result<()> {
     Pipeline::start(&pipeline);
 
     for i in 0..VIDEOS {
-        start_ffmpeg_send(IP, Some(INPUT_PORT + 2 * i), None, TestSample::Sample)?;
+        start_ffmpeg_send(IP, Some(INPUT_PORT + 2 * i), None, TestSample::BigBuckBunny)?;
     }
 
     let event_loop_fallback = || {

src/snapshot_tests/utils.rs

@@ -1,14 +1,10 @@
 use core::panic;
-use std::{
-    io::Write,
-    sync::{Arc, OnceLock},
-    time::Duration,
-};
+use std::{io::Write, sync::OnceLock, time::Duration};
 
 use bytes::BufMut;
 use compositor_render::{
     create_wgpu_ctx, web_renderer, Frame, FrameData, Framerate, Renderer, RendererOptions,
-    WgpuFeatures, YuvPlanes,
+    WgpuComponents, WgpuFeatures, YuvPlanes,
 };
 use crossbeam_channel::bounded;
 use tracing::error;
@@ -57,13 +53,16 @@ pub(super) fn yuv_frame_to_rgba(frame: &Frame, planes: &YuvPlanes) -> Vec<u8> {
     rgba_data
 }
 
-fn get_wgpu_ctx() -> (Arc<wgpu::Device>, Arc<wgpu::Queue>) {
-    static CTX: OnceLock<(Arc<wgpu::Device>, Arc<wgpu::Queue>)> = OnceLock::new();
-    CTX.get_or_init(|| create_wgpu_ctx(false, Default::default(), Default::default()).unwrap())
-        .clone()
+fn get_wgpu_ctx() -> WgpuComponents {
+    static CTX: OnceLock<WgpuComponents> = OnceLock::new();
+    CTX.get_or_init(|| {
+        create_wgpu_ctx(false, Default::default(), Default::default(), None).unwrap()
+    })
+    .clone()
 }
 
 pub(super) fn create_renderer() -> Renderer {
+    let wgpu_ctx = get_wgpu_ctx();
     let (renderer, _event_loop) = Renderer::new(RendererOptions {
         web_renderer: web_renderer::WebRendererInitOptions {
             enable: false,
@@ -73,15 +72,15 @@ pub(super) fn create_renderer() -> Renderer {
         framerate: Framerate { num: 30, den: 1 },
         stream_fallback_timeout: Duration::from_secs(3),
         wgpu_features: WgpuFeatures::default(),
-        wgpu_ctx: Some(get_wgpu_ctx()),
+        wgpu_ctx: Some((wgpu_ctx.device.clone(), wgpu_ctx.queue.clone())),
         load_system_fonts: false,
     })
     .unwrap();
     renderer
 }
 
 fn read_rgba_texture(texture: &wgpu::Texture) -> bytes::Bytes {
-    let (device, queue) = get_wgpu_ctx();
+    let WgpuComponents { device, queue, .. } = get_wgpu_ctx();
     let buffer = new_download_buffer(&device, texture);
 
     let mut encoder = device.create_command_encoder(&Default::default());

vk-video/Cargo.toml

@@ -11,6 +11,7 @@ repository = "https://github.com/software-mansion/live-compositor"
 
 [dependencies]
 ash = "0.38.0"
+bytemuck = { version = "1.19.0", features = ["derive"] }
 bytes = "1"
 derivative = "2.2.0"
 h264-reader = { git = "https://github.com/membraneframework-labs/h264-reader.git", branch = "live-compositor" }
@@ -21,6 +22,8 @@ wgpu =  "23.0.0"
 
 [dev-dependencies]
 tracing-subscriber = "0.3.18"
+winit = "0.29"
+clap = { version = "4.5.20", features = ["derive"] }
 
 [build-dependencies]
 cfg_aliases = "0.2.1"