From 5eebc9d4d80aa4b7c37bdbc556d65e6417814164 Mon Sep 17 00:00:00 2001 From: Jerzy Wilczek Date: Tue, 12 Nov 2024 16:23:46 +0100 Subject: [PATCH] New architecture for vk-video initialization. This change allows users to initialize `vk-video` with wgpu surfaces for rendering to an on-screen window. This was previously overlooked and there was no possibility to guarantee a `VulkanCtx` was able to render on screen. This patch also adds an example h264 player to vk-video. --- Cargo.lock | 924 +++++++++++++++++- compositor_pipeline/src/pipeline.rs | 14 +- .../pipeline/decoder/video/vulkan_video.rs | 12 +- .../src/pipeline/graphics_context.rs | 55 +- compositor_render/src/lib.rs | 2 +- compositor_render/src/wgpu.rs | 2 +- compositor_render/src/wgpu/ctx.rs | 23 +- .../manual_graphics_initialization.rs | 27 +- .../examples/raw_channel_input.rs | 2 +- .../examples/raw_channel_output.rs | 2 +- integration_tests/examples/vulkan.rs | 4 +- src/snapshot_tests/utils.rs | 21 +- vk-video/Cargo.toml | 3 + vk-video/examples/basic.rs | 14 +- vk-video/examples/player/main.rs | 14 + vk-video/examples/player/player.rs | 64 ++ vk-video/examples/player/player/decoder.rs | 43 + vk-video/examples/player/player/renderer.rs | 353 +++++++ vk-video/examples/player/player/shader.wgsl | 37 + vk-video/examples/wgpu.rs | 18 +- vk-video/src/lib.rs | 28 +- vk-video/src/vulkan_decoder.rs | 58 +- .../src/vulkan_decoder/session_resources.rs | 8 +- .../session_resources/images.rs | 6 +- .../session_resources/parameters.rs | 4 +- vk-video/src/vulkan_decoder/vulkan_ctx.rs | 319 +++--- vk-video/src/vulkan_decoder/wrappers/video.rs | 4 +- 27 files changed, 1789 insertions(+), 272 deletions(-) create mode 100644 vk-video/examples/player/main.rs create mode 100644 vk-video/examples/player/player.rs create mode 100644 vk-video/examples/player/player/decoder.rs create mode 100644 vk-video/examples/player/player/renderer.rs create mode 100644 vk-video/examples/player/player/shader.wgsl diff --git a/Cargo.lock b/Cargo.lock index 6838ec633..07a98612b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,22 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ab_glyph" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3672c180e71eeaaac3a541fbbc5f5ad4def8b747c595ad30d674e43049f7b0" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71b1793ee61086797f5c80b6efa2b8ffa6d5dd703f118545808a7f2e27f7046" + [[package]] name = "addr2line" version = "0.20.0" @@ -24,6 +40,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -44,6 +61,33 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "android-activity" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee91c0c2905bae44f84bfa4e044536541df26b7703fd0888deeb9060fcc44289" +dependencies = [ + "android-properties", + "bitflags 2.6.0", + "cc", + "cesu8", + "jni", + "jni-sys", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-sys", + "num_enum", + "thiserror", +] + +[[package]] +name = "android-properties" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -68,6 +112,55 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.72" @@ -95,6 +188,12 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "as-raw-xcb-connection" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" + [[package]] name = "ascii" version = "1.1.0" @@ -324,6 +423,25 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae85a0696e7ea3b835a453750bf002770776609115e6d25c6d2ff28a8200f7e7" +dependencies = [ + "objc-sys", +] + +[[package]] +name = "block2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b55663a85f33501257357e6421bb33e769d5c9ffb5ba0921c975a123e35e68" +dependencies = [ + "block-sys", + "objc2", +] + [[package]] name = "bumpalo" version = "3.13.0" @@ -362,6 +480,32 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "calloop" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba7adb4dd5aa98e5553510223000e7148f621165ec5f9acd7113f6ca4995298" +dependencies = [ + "bitflags 2.6.0", + "log", + "polling", + "rustix", + "slab", + "thiserror", +] + +[[package]] +name = "calloop-wayland-source" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0ea9b9476c7fad82841a8dbb380e2eae480c21910feba80725b46931ed8f02" +dependencies = [ + "calloop", + "rustix", + "wayland-backend", + "wayland-client", +] + [[package]] name = "cc" version = "1.1.10" @@ -372,6 +516,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -439,6 +589,46 @@ dependencies = [ "libloading 0.7.4", ] +[[package]] +name = "clap" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.75", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + [[package]] name = "cmake" version = "0.1.50" @@ -464,6 +654,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compositor_api" version = "0.1.0" @@ -566,6 +772,15 @@ dependencies = [ "wgpu", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -592,6 +807,19 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types 0.5.0", + "libc", +] + [[package]] name = "core-graphics-types" version = "0.1.3" @@ -703,6 +931,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "cursor-icon" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" + [[package]] name = "cxx" version = "1.0.126" @@ -791,6 +1025,21 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + +[[package]] +name = "dlib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" +dependencies = [ + "libloading 0.8.0", +] + [[package]] name = "document-features" version = "0.2.10" @@ -800,6 +1049,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dyn-clone" version = "1.0.14" @@ -1163,6 +1418,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" +dependencies = [ + "libc", + "windows-targets 0.48.1", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -1305,7 +1570,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.0.1", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -1349,12 +1614,30 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex-slice" version = "0.1.4" @@ -1510,6 +1793,17 @@ dependencies = [ "cc", ] +[[package]] +name = "icrate" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d3aaff8a54577104bafdf686ff18565c3b6903ca5782a2026ef06e2c7aa319" +dependencies = [ + "block2", + "dispatch", + "objc2", +] + [[package]] name = "idna" version = "0.4.0" @@ -1558,12 +1852,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.15.0", ] [[package]] @@ -1606,6 +1900,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -1621,6 +1921,22 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni-sys" version = "0.3.0" @@ -1730,6 +2046,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall 0.5.7", +] + [[package]] name = "link-cplusplus" version = "1.0.9" @@ -1992,7 +2319,7 @@ dependencies = [ "cfg_aliases 0.1.1", "codespan-reporting", "hexf-parse", - "indexmap 2.0.1", + "indexmap 2.6.0", "log", "rustc-hash 1.1.0", "spirv", @@ -2056,6 +2383,27 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.6.0", + "jni-sys", + "log", + "ndk-sys", + "num_enum", + "raw-window-handle", + "thiserror", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "ndk-sys" version = "0.5.0+25.2.9519653" @@ -2169,10 +2517,31 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.2", "libc", ] +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.75", +] + [[package]] name = "objc" version = "0.2.7" @@ -2182,6 +2551,28 @@ dependencies = [ "malloc_buf", ] +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "559c5a40fdd30eb5e344fbceacf7595a81e242529fb4e21cf5f43fb4f11ff98d" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d079845b37af429bfe5dfa76e6d087d788031045b25cfc6fd898486fd9847666" + [[package]] name = "object" version = "0.31.1" @@ -2251,6 +2642,15 @@ dependencies = [ "libc", ] +[[package]] +name = "orbclient" +version = "0.3.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba0b26cec2e24f08ed8bb31519a9333140a6599b867dac464bb150bdb796fd43" +dependencies = [ + "libredox", +] + [[package]] name = "overload" version = "0.1.1" @@ -2258,8 +2658,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] -name = "parking_lot" -version = "0.12.1" +name = "owned_ttf_parser" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec719bbf3b2a81c109a4e20b1f129b5566b7dce654bc3872f6a05abf82b2c4" +dependencies = [ + "ttf-parser 0.25.0", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ @@ -2275,7 +2684,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "smallvec", "windows-targets 0.48.1", ] @@ -2364,6 +2773,21 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "polling" +version = "3.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2790cd301dec6cd3b7a025e4815cf825724a51c98dccfe6a3e55f05ffb6511" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.4.0", + "pin-project-lite", + "rustix", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "pollster" version = "0.3.0" @@ -2401,6 +2825,15 @@ dependencies = [ "num-integer", ] +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -2425,6 +2858,15 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.36" @@ -2544,6 +2986,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "regex" version = "1.10.6" @@ -2651,7 +3102,7 @@ dependencies = [ "png", "rgb", "svgtypes", - "tiny-skia", + "tiny-skia 0.10.0", "usvg", ] @@ -2874,6 +3325,15 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.22" @@ -2906,6 +3366,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2918,6 +3384,19 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" +[[package]] +name = "sctk-adwaita" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70b31447ca297092c5a9916fc3b955203157b37c19ca8edde4f52e9843e602c7" +dependencies = [ + "ab_glyph", + "log", + "memmap2 0.9.4", + "smithay-client-toolkit", + "tiny-skia 0.11.4", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -2995,7 +3474,7 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -3149,6 +3628,40 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "smithay-client-toolkit" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "922fd3eeab3bd820d76537ce8f582b1cf951eceb5475c28500c7457d9d17f53a" +dependencies = [ + "bitflags 2.6.0", + "calloop", + "calloop-wayland-source", + "cursor-icon", + "libc", + "log", + "memmap2 0.9.4", + "rustix", + "thiserror", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", +] + +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.5.7" @@ -3198,6 +3711,12 @@ dependencies = [ "float-cmp", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -3306,7 +3825,7 @@ checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", "windows-sys 0.48.0", ] @@ -3373,7 +3892,21 @@ dependencies = [ "cfg-if", "log", "png", - "tiny-skia-path", + "tiny-skia-path 0.10.0", +] + +[[package]] +name = "tiny-skia" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d13394d44dae3207b52a326c0c85a8bf87f1541f23b0d143811088497b09ab" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "log", + "tiny-skia-path 0.11.4", ] [[package]] @@ -3387,6 +3920,17 @@ dependencies = [ "strict-num", ] +[[package]] +name = "tiny-skia-path" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e7fc0c2e86a30b117d0462aa261b72b7a99b7ebd7deb3a14ceda95c5bdc93" +dependencies = [ + "arrayref", + "bytemuck", + "strict-num", +] + [[package]] name = "tiny_http" version = "0.12.0" @@ -3491,6 +4035,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap 2.6.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -3678,6 +4239,12 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c591d83f69777866b9126b24c6dd9a18351f177e49d625920d19f989fd31cf8" +[[package]] +name = "ttf-parser" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5902c5d130972a0000f60860bfbf46f7ca3db5391eddfedd1b8728bd9dc96c0e" + [[package]] name = "tungstenite" version = "0.21.0" @@ -3871,7 +4438,7 @@ dependencies = [ "rctree", "strict-num", "svgtypes", - "tiny-skia-path", + "tiny-skia-path 0.10.0", ] [[package]] @@ -3880,6 +4447,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.0" @@ -3914,8 +4487,10 @@ name = "vk-video" version = "0.1.0" dependencies = [ "ash", + "bytemuck", "bytes", "cfg_aliases 0.2.1", + "clap", "derivative", "h264-reader", "thiserror", @@ -3923,6 +4498,17 @@ dependencies = [ "tracing-subscriber 0.3.18", "vk-mem", "wgpu", + "winit", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", ] [[package]] @@ -4018,6 +4604,115 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wayland-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056535ced7a150d45159d3a8dc30f91a2e2d588ca0b23f70e56033622b8016f6" +dependencies = [ + "cc", + "downcast-rs", + "rustix", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66249d3fc69f76fd74c82cc319300faa554e9d865dab1f7cd66cc20db10b280" +dependencies = [ + "bitflags 2.6.0", + "rustix", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-csd-frame" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625c5029dbd43d25e6aa9615e88b829a5cad13b2819c4ae129fdbb7c31ab4c7e" +dependencies = [ + "bitflags 2.6.0", + "cursor-icon", + "wayland-backend", +] + +[[package]] +name = "wayland-cursor" +version = "0.31.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b08bc3aafdb0035e7fe0fdf17ba0c09c268732707dca4ae098f60cb28c9e4c" +dependencies = [ + "rustix", + "wayland-client", + "xcursor", +] + +[[package]] +name = "wayland-protocols" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f81f365b8b4a97f422ac0e8737c438024b5951734506b0e1d775c73030561f4" +dependencies = [ + "bitflags 2.6.0", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-plasma" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23803551115ff9ea9bce586860c5c5a971e360825a0309264102a9495a5ff479" +dependencies = [ + "bitflags 2.6.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1f61b76b6c2d8742e10f9ba5c3737f6530b4c243132c2a2ccc8aa96fe25cd6" +dependencies = [ + "bitflags 2.6.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597f2001b2e5fc1121e3d5b9791d3e78f05ba6bfa4641053846248e3a13661c3" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efa8ac0d8e8ed3e3b5c9fc92c7881406a268e11555abe36493efabe649a29e09" +dependencies = [ + "dlib", + "log", + "once_cell", + "pkg-config", +] + [[package]] name = "web-sys" version = "0.3.72" @@ -4028,6 +4723,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webrtc-util" version = "0.8.1" @@ -4090,7 +4795,7 @@ dependencies = [ "bitflags 2.6.0", "cfg_aliases 0.1.1", "document-features", - "indexmap 2.0.1", + "indexmap 2.6.0", "log", "naga", "once_cell", @@ -4324,6 +5029,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -4342,6 +5056,30 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.1" @@ -4373,6 +5111,12 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" @@ -4391,6 +5135,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.0" @@ -4409,6 +5159,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.0" @@ -4433,6 +5189,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.0" @@ -4451,6 +5213,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.0" @@ -4463,6 +5231,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" @@ -4481,6 +5255,12 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.0" @@ -4493,6 +5273,120 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winit" +version = "0.29.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d59ad965a635657faf09c8f062badd885748428933dad8e8bdd64064d92e5ca" +dependencies = [ + "ahash", + "android-activity", + "atomic-waker", + "bitflags 2.6.0", + "bytemuck", + "calloop", + "cfg_aliases 0.1.1", + "core-foundation", + "core-graphics", + "cursor-icon", + "icrate", + "js-sys", + "libc", + "log", + "memmap2 0.9.4", + "ndk", + "ndk-sys", + "objc2", + "once_cell", + "orbclient", + "percent-encoding", + "raw-window-handle", + "redox_syscall 0.3.5", + "rustix", + "sctk-adwaita", + "smithay-client-toolkit", + "smol_str", + "unicode-segmentation", + "wasm-bindgen", + "wasm-bindgen-futures", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-plasma", + "web-sys", + "web-time", + "windows-sys 0.48.0", + "x11-dl", + "x11rb", + "xkbcommon-dl", +] + +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "x11rb" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d91ffca73ee7f68ce055750bf9f6eca0780b8c85eff9bc046a3b0da41755e12" +dependencies = [ + "as-raw-xcb-connection", + "gethostname", + "libc", + "libloading 0.8.0", + "once_cell", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d" + +[[package]] +name = "xcursor" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef33da6b1660b4ddbfb3aef0ade110c8b8a781a3b6382fa5f2b5b040fd55f61" + +[[package]] +name = "xkbcommon-dl" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039de8032a9a8856a6be89cea3e5d12fdd82306ab7c94d74e6deab2460651c5" +dependencies = [ + "bitflags 2.6.0", + "dlib", + "log", + "once_cell", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "xml-rs" version = "0.8.21" diff --git a/compositor_pipeline/src/pipeline.rs b/compositor_pipeline/src/pipeline.rs index 465f4b641..b47b484eb 100644 --- a/compositor_pipeline/src/pipeline.rs +++ b/compositor_pipeline/src/pipeline.rs @@ -133,7 +133,9 @@ pub struct PipelineCtx { pub download_dir: Arc, pub event_emitter: Arc, #[cfg(feature = "vk-video")] - pub vulkan_ctx: Option>, + pub vulkan_instance: Option>, + #[cfg(feature = "vk-video")] + pub vulkan_device: Option>, } impl std::fmt::Debug for PipelineCtx { @@ -156,6 +158,7 @@ impl Pipeline { opts.force_gpu, opts.wgpu_features, Default::default(), + None, )?), #[cfg(not(feature = "vk-video"))] None => None, @@ -194,7 +197,14 @@ impl Pipeline { download_dir: download_dir.into(), event_emitter, #[cfg(feature = "vk-video")] - vulkan_ctx: preinitialized_ctx.and_then(|ctx| ctx.vulkan_ctx), + vulkan_device: preinitialized_ctx + .as_ref() + .and_then(|ctx| ctx.vulkan_ctx.as_ref()) + .map(|ctx| ctx.device.clone()), + #[cfg(feature = "vk-video")] + vulkan_instance: preinitialized_ctx + .and_then(|ctx| ctx.vulkan_ctx) + .map(|ctx| ctx.instance.clone()), }, }; diff --git a/compositor_pipeline/src/pipeline/decoder/video/vulkan_video.rs b/compositor_pipeline/src/pipeline/decoder/video/vulkan_video.rs index 149723124..120f28f44 100644 --- a/compositor_pipeline/src/pipeline/decoder/video/vulkan_video.rs +++ b/compositor_pipeline/src/pipeline/decoder/video/vulkan_video.rs @@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; use compositor_render::{Frame, FrameData, InputId, Resolution}; use crossbeam_channel::{Receiver, Sender}; use tracing::{debug, error, span, trace, warn, Level}; -use vk_video::{VulkanCtx, WgpuTexturesDeocder}; +use vk_video::VulkanDevice; use crate::{ error::InputInitError, @@ -17,7 +17,11 @@ pub fn start_vulkan_video_decoder_thread( frame_sender: Sender>, input_id: InputId, ) -> Result<(), InputInitError> { - let Some(vulkan_ctx) = pipeline_ctx.vulkan_ctx.as_ref().map(|ctx| ctx.clone()) else { + let Some(vulkan_ctx) = pipeline_ctx + .vulkan_device + .as_ref() + .map(|device| device.clone()) + else { return Err(InputInitError::VulkanContextRequiredForVulkanDecoder); }; @@ -47,12 +51,12 @@ pub fn start_vulkan_video_decoder_thread( } fn run_decoder_thread( - vulkan_ctx: Arc, + vulkan_device: Arc, init_result_sender: Sender>, chunks_receiver: Receiver>, frame_sender: Sender>, ) { - let mut decoder = match WgpuTexturesDeocder::new(vulkan_ctx) { + let mut decoder = match vulkan_device.create_wgpu_textures_decoder() { Ok(decoder) => { init_result_sender.send(Ok(())).unwrap(); decoder diff --git a/compositor_pipeline/src/pipeline/graphics_context.rs b/compositor_pipeline/src/pipeline/graphics_context.rs index d0bc15d0c..2ec3bb793 100644 --- a/compositor_pipeline/src/pipeline/graphics_context.rs +++ b/compositor_pipeline/src/pipeline/graphics_context.rs @@ -1,14 +1,23 @@ use crate::error::InitPipelineError; -use compositor_render::{create_wgpu_ctx, error::InitRendererEngineError}; +use compositor_render::{create_wgpu_ctx, error::InitRendererEngineError, WgpuComponents}; use std::sync::Arc; +#[cfg(feature = "vk-video")] +#[derive(Debug)] +pub struct VulkanCtx { + pub device: Arc, + pub instance: Arc, +} + #[derive(Debug)] pub struct GraphicsContext { pub device: Arc, pub queue: Arc, + pub adapter: Arc, + pub instance: Arc, #[cfg(feature = "vk-video")] - pub vulkan_ctx: Option>, + pub vulkan_ctx: Option, } impl GraphicsContext { @@ -17,6 +26,7 @@ impl GraphicsContext { force_gpu: bool, features: wgpu::Features, limits: wgpu::Limits, + mut compatible_surface: Option<&mut wgpu::Surface<'_>>, ) -> Result { use compositor_render::{required_wgpu_features, set_required_wgpu_limits}; use tracing::warn; @@ -26,22 +36,36 @@ impl GraphicsContext { let limits = set_required_wgpu_limits(limits); - match vk_video::VulkanCtx::new(vulkan_features, limits.clone()) { - Ok(ctx) => Ok(GraphicsContext { - device: ctx.wgpu_ctx.device.clone(), - queue: ctx.wgpu_ctx.queue.clone(), - vulkan_ctx: Some(ctx.into()), + match vk_video::VulkanInstance::new().and_then(|instance| { + let device = + instance.create_device(vulkan_features, limits.clone(), &mut compatible_surface)?; + + Ok((instance, device)) + }) { + Ok((instance, device)) => Ok(GraphicsContext { + device: device.wgpu_device.clone(), + queue: device.wgpu_queue.clone(), + adapter: device.wgpu_adapter.clone(), + instance: instance.wgpu_instance.clone(), + vulkan_ctx: Some(VulkanCtx { instance, device }), }), Err(err) => { warn!("Cannot initialize vulkan video decoding context. Reason: {err}. Initializing without vulkan video support."); - let (device, queue) = create_wgpu_ctx(force_gpu, features, limits) + let WgpuComponents { + instance, + adapter, + device, + queue, + } = create_wgpu_ctx(force_gpu, features, limits, compatible_surface.as_deref()) .map_err(InitRendererEngineError::FailedToInitWgpuCtx)?; Ok(GraphicsContext { device, queue, + adapter, + instance, vulkan_ctx: None, }) } @@ -53,10 +77,21 @@ impl GraphicsContext { force_gpu: bool, features: wgpu::Features, limits: wgpu::Limits, + compatible_surface: Option<&mut wgpu::Surface<'_>>, ) -> Result { - let (device, queue) = create_wgpu_ctx(force_gpu, features, limits) + let WgpuComponents { + instance, + adapter, + device, + queue, + } = create_wgpu_ctx(force_gpu, features, limits, compatible_surface.as_deref()) .map_err(InitRendererEngineError::FailedToInitWgpuCtx)?; - Ok(GraphicsContext { device, queue }) + Ok(GraphicsContext { + device, + queue, + adapter, + instance, + }) } } diff --git a/compositor_render/src/lib.rs b/compositor_render/src/lib.rs index 5d598473f..3e3a4e3ea 100644 --- a/compositor_render/src/lib.rs +++ b/compositor_render/src/lib.rs @@ -20,7 +20,7 @@ pub use state::RendererOptions; pub use state::RendererSpec; pub use wgpu::WgpuFeatures; -pub use wgpu::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits}; +pub use wgpu::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits, WgpuComponents}; pub mod image { pub use crate::transformations::image_renderer::{ImageSource, ImageSpec, ImageType}; diff --git a/compositor_render/src/wgpu.rs b/compositor_render/src/wgpu.rs index ff9921a65..e3e7196da 100644 --- a/compositor_render/src/wgpu.rs +++ b/compositor_render/src/wgpu.rs @@ -7,7 +7,7 @@ pub(crate) mod texture; pub(crate) mod utils; pub(crate) use ctx::WgpuCtx; -pub use ctx::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits}; +pub use ctx::{create_wgpu_ctx, required_wgpu_features, set_required_wgpu_limits, WgpuComponents}; pub use wgpu::Features as WgpuFeatures; #[must_use] diff --git a/compositor_render/src/wgpu/ctx.rs b/compositor_render/src/wgpu/ctx.rs index 4bf23706f..96aab0f06 100644 --- a/compositor_render/src/wgpu/ctx.rs +++ b/compositor_render/src/wgpu/ctx.rs @@ -34,7 +34,8 @@ impl WgpuCtx { Self::new_from_device_queue(device, queue)? } None => { - let (device, queue) = create_wgpu_ctx(force_gpu, features, Default::default())?; + let WgpuComponents { device, queue, .. } = + create_wgpu_ctx(force_gpu, features, Default::default(), None)?; Self::new_from_device_queue(device, queue)? } }; @@ -101,11 +102,20 @@ pub fn set_required_wgpu_limits(limits: wgpu::Limits) -> wgpu::Limits { } } +#[derive(Clone)] +pub struct WgpuComponents { + pub device: Arc, + pub queue: Arc, + pub adapter: Arc, + pub instance: Arc, +} + pub fn create_wgpu_ctx( force_gpu: bool, features: wgpu::Features, limits: wgpu::Limits, -) -> Result<(Arc, Arc), CreateWgpuCtxError> { + compatible_surface: Option<&wgpu::Surface<'_>>, +) -> Result { let instance = wgpu::Instance::new(wgpu::InstanceDescriptor { backends: wgpu::Backends::all(), ..Default::default() @@ -117,7 +127,7 @@ pub fn create_wgpu_ctx( let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptionsBase { power_preference: wgpu::PowerPreference::HighPerformance, force_fallback_adapter: false, - compatible_surface: None, + compatible_surface, })) .ok_or(CreateWgpuCtxError::NoAdapter)?; @@ -148,7 +158,12 @@ pub fn create_wgpu_ctx( }, None, ))?; - Ok((device.into(), queue.into())) + Ok(WgpuComponents { + instance: instance.into(), + adapter: adapter.into(), + device: device.into(), + queue: queue.into(), + }) } fn uniform_bind_group_layout(device: &wgpu::Device) -> wgpu::BindGroupLayout { diff --git a/integration_tests/examples/manual_graphics_initialization.rs b/integration_tests/examples/manual_graphics_initialization.rs index 236edd7f9..2ae7e096f 100644 --- a/integration_tests/examples/manual_graphics_initialization.rs +++ b/integration_tests/examples/manual_graphics_initialization.rs @@ -9,27 +9,18 @@ fn main() { }; use live_compositor::config::read_config; - let graphics_context = - GraphicsContext::new(false, wgpu::Features::default(), wgpu::Limits::default()).unwrap(); + let graphics_context = GraphicsContext::new( + false, + wgpu::Features::default(), + wgpu::Limits::default(), + None, + ) + .unwrap(); let _device = graphics_context.device.clone(); let _queue = graphics_context.queue.clone(); - - let _adapter = graphics_context - .vulkan_ctx - .as_ref() - .unwrap() - .wgpu_ctx - .adapter - .clone(); - - let _instance = graphics_context - .vulkan_ctx - .as_ref() - .unwrap() - .wgpu_ctx - .instance - .clone(); + let _adapter = graphics_context.adapter.clone(); + let _instance = graphics_context.instance.clone(); let config = read_config(); diff --git a/integration_tests/examples/raw_channel_input.rs b/integration_tests/examples/raw_channel_input.rs index 06fb7d50e..90b0ec577 100644 --- a/integration_tests/examples/raw_channel_input.rs +++ b/integration_tests/examples/raw_channel_input.rs @@ -40,7 +40,7 @@ fn main() { ffmpeg_next::format::network::init(); let config = read_config(); logger::init_logger(config.logger); - let ctx = GraphicsContext::new(false, Default::default(), Default::default()).unwrap(); + let ctx = GraphicsContext::new(false, Default::default(), Default::default(), None).unwrap(); let (wgpu_device, wgpu_queue) = (ctx.device.clone(), ctx.queue.clone()); // no chromium support, so we can ignore _event_loop let (pipeline, _event_loop) = Pipeline::new(Options { diff --git a/integration_tests/examples/raw_channel_output.rs b/integration_tests/examples/raw_channel_output.rs index 4f7143a47..3b21da594 100644 --- a/integration_tests/examples/raw_channel_output.rs +++ b/integration_tests/examples/raw_channel_output.rs @@ -53,7 +53,7 @@ fn main() { logger::init_logger(read_config().logger); let mut config = read_config(); config.queue_options.ahead_of_time_processing = true; - let ctx = GraphicsContext::new(false, Default::default(), Default::default()).unwrap(); + let ctx = GraphicsContext::new(false, Default::default(), Default::default(), None).unwrap(); let (wgpu_device, wgpu_queue) = (ctx.device.clone(), ctx.queue.clone()); // no chromium support, so we can ignore _event_loop let (pipeline, _event_loop) = Pipeline::new(Options { diff --git a/integration_tests/examples/vulkan.rs b/integration_tests/examples/vulkan.rs index dc7fd4e7f..46e8e14e7 100644 --- a/integration_tests/examples/vulkan.rs +++ b/integration_tests/examples/vulkan.rs @@ -73,7 +73,7 @@ fn client_code() -> Result<()> { const INPUT_PORT: u16 = 8006; const OUTPUT_PORT: u16 = 8004; - const VIDEOS: u16 = 1; + const VIDEOS: u16 = 6; start_ffmpeg_receive(Some(OUTPUT_PORT), None)?; let config = read_config(); @@ -176,7 +176,7 @@ fn client_code() -> Result<()> { Pipeline::start(&pipeline); for i in 0..VIDEOS { - start_ffmpeg_send(IP, Some(INPUT_PORT + 2 * i), None, TestSample::Sample)?; + start_ffmpeg_send(IP, Some(INPUT_PORT + 2 * i), None, TestSample::BigBuckBunny)?; } let event_loop_fallback = || { diff --git a/src/snapshot_tests/utils.rs b/src/snapshot_tests/utils.rs index 6de77f988..d3250eecf 100644 --- a/src/snapshot_tests/utils.rs +++ b/src/snapshot_tests/utils.rs @@ -1,14 +1,10 @@ use core::panic; -use std::{ - io::Write, - sync::{Arc, OnceLock}, - time::Duration, -}; +use std::{io::Write, sync::OnceLock, time::Duration}; use bytes::BufMut; use compositor_render::{ create_wgpu_ctx, web_renderer, Frame, FrameData, Framerate, Renderer, RendererOptions, - WgpuFeatures, YuvPlanes, + WgpuComponents, WgpuFeatures, YuvPlanes, }; use crossbeam_channel::bounded; use tracing::error; @@ -57,13 +53,16 @@ pub(super) fn yuv_frame_to_rgba(frame: &Frame, planes: &YuvPlanes) -> Vec { rgba_data } -fn get_wgpu_ctx() -> (Arc, Arc) { - static CTX: OnceLock<(Arc, Arc)> = OnceLock::new(); - CTX.get_or_init(|| create_wgpu_ctx(false, Default::default(), Default::default()).unwrap()) +fn get_wgpu_ctx() -> WgpuComponents { + static CTX: OnceLock = OnceLock::new(); + CTX.get_or_init(|| { + create_wgpu_ctx(false, Default::default(), Default::default(), None).unwrap() + }) .clone() } pub(super) fn create_renderer() -> Renderer { + let wgpu_ctx = get_wgpu_ctx(); let (renderer, _event_loop) = Renderer::new(RendererOptions { web_renderer: web_renderer::WebRendererInitOptions { enable: false, @@ -73,7 +72,7 @@ pub(super) fn create_renderer() -> Renderer { framerate: Framerate { num: 30, den: 1 }, stream_fallback_timeout: Duration::from_secs(3), wgpu_features: WgpuFeatures::default(), - wgpu_ctx: Some(get_wgpu_ctx()), + wgpu_ctx: Some((wgpu_ctx.device.clone(), wgpu_ctx.queue.clone())), load_system_fonts: false, }) .unwrap(); @@ -81,7 +80,7 @@ pub(super) fn create_renderer() -> Renderer { } fn read_rgba_texture(texture: &wgpu::Texture) -> bytes::Bytes { - let (device, queue) = get_wgpu_ctx(); + let WgpuComponents { device, queue, .. } = get_wgpu_ctx(); let buffer = new_download_buffer(&device, texture); let mut encoder = device.create_command_encoder(&Default::default()); diff --git a/vk-video/Cargo.toml b/vk-video/Cargo.toml index ea60ae5fc..a31f2ff0d 100644 --- a/vk-video/Cargo.toml +++ b/vk-video/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/software-mansion/live-compositor" [dependencies] ash = "0.38.0" +bytemuck = { version = "1.19.0", features = ["derive"] } bytes = "1" derivative = "2.2.0" h264-reader = { git = "https://github.com/membraneframework-labs/h264-reader.git", branch = "live-compositor" } @@ -21,6 +22,8 @@ wgpu = "23.0.0" [dev-dependencies] tracing-subscriber = "0.3.18" +winit = "0.29" +clap = { version = "4.5.20", features = ["derive"] } [build-dependencies] cfg_aliases = "0.2.1" diff --git a/vk-video/examples/basic.rs b/vk-video/examples/basic.rs index 17eaaf4ad..a39fc93d5 100644 --- a/vk-video/examples/basic.rs +++ b/vk-video/examples/basic.rs @@ -2,7 +2,7 @@ fn main() { use std::io::Write; - use vk_video::Frame; + use vk_video::{Frame, VulkanInstance}; let subscriber = tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::INFO) @@ -18,17 +18,19 @@ fn main() { let h264_bytestream = std::fs::read(&args[1]).unwrap_or_else(|_| panic!("read {}", args[1])); - let vulkan_ctx = std::sync::Arc::new( - vk_video::VulkanCtx::new( + let vulkan_instance = VulkanInstance::new().unwrap(); + let vulkan_device = vulkan_instance + .create_device( wgpu::Features::empty(), wgpu::Limits { max_push_constant_size: 128, ..Default::default() }, + &mut None, ) - .unwrap(), - ); - let mut decoder = vk_video::BytesDecoder::new(vulkan_ctx).unwrap(); + .unwrap(); + + let mut decoder = vulkan_device.create_bytes_decoder().unwrap(); let mut output_file = std::fs::File::create("output.nv12").unwrap(); diff --git a/vk-video/examples/player/main.rs b/vk-video/examples/player/main.rs new file mode 100644 index 000000000..c10caef60 --- /dev/null +++ b/vk-video/examples/player/main.rs @@ -0,0 +1,14 @@ +#[cfg(vulkan)] +mod player; + +#[cfg(vulkan)] +fn main() { + player::run() +} + +#[cfg(not(vulkan))] +fn main() { + println!( + "This crate doesn't work on your operating system, because it does not support vulkan" + ); +} diff --git a/vk-video/examples/player/player.rs b/vk-video/examples/player/player.rs new file mode 100644 index 000000000..4fd14ddda --- /dev/null +++ b/vk-video/examples/player/player.rs @@ -0,0 +1,64 @@ +use std::{path::PathBuf, sync::mpsc, time::Duration}; + +use clap::Parser; +use vk_video::VulkanInstance; +use winit::{event_loop::EventLoop, window::WindowBuilder}; + +mod decoder; +mod renderer; + +const FRAMES_BUFFER_LEN: usize = 3; + +#[derive(Parser)] +#[command(version, about, long_about=None)] +struct Args { + /// an .h264 file to play + filename: PathBuf, + + /// framerate to play the video at + framerate: u64, +} + +struct FrameWithPts { + frame: wgpu::Texture, + /// Presentation timestamp + pts: Duration, +} + +pub fn run() { + let args = Args::parse(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .finish(); + + tracing::subscriber::set_global_default(subscriber).unwrap(); + + let file = std::fs::File::open(&args.filename).expect("open file"); + + let event_loop = EventLoop::new().unwrap(); + let window = WindowBuilder::new().build(&event_loop).unwrap(); + + let vulkan_instance = VulkanInstance::new().unwrap(); + + let mut surface = vulkan_instance + .wgpu_instance + .create_surface(&window) + .unwrap(); + + let vulkan_device = vulkan_instance + .create_device( + wgpu::Features::empty(), + wgpu::Limits::default(), + &mut Some(&mut surface), + ) + .unwrap(); + + let (tx, rx) = mpsc::sync_channel(FRAMES_BUFFER_LEN); + let vulkan_device_clone = vulkan_device.clone(); + + std::thread::spawn(move || { + decoder::run_decoder(tx, args.framerate, vulkan_device_clone, file); + }); + + renderer::run_renderer(event_loop, &window, surface, &vulkan_device, rx); +} diff --git a/vk-video/examples/player/player/decoder.rs b/vk-video/examples/player/player/decoder.rs new file mode 100644 index 000000000..09ab0018e --- /dev/null +++ b/vk-video/examples/player/player/decoder.rs @@ -0,0 +1,43 @@ +use std::{ + io::Read, + sync::{mpsc::SyncSender, Arc}, + time::Duration, +}; + +use bytes::BytesMut; +use vk_video::VulkanDevice; + +use super::FrameWithPts; + +pub fn run_decoder( + tx: SyncSender, + framerate: u64, + vulkan_device: Arc, + mut bytestream_reader: impl Read, +) { + let mut decoder = vulkan_device.create_wgpu_textures_decoder().unwrap(); + let frame_interval = 1.0 / (framerate as f64); + let mut frame_number = 0u64; + let mut buffer = BytesMut::zeroed(4096); + + while let Ok(n) = bytestream_reader.read(&mut buffer) { + if n == 0 { + return; + } + + let decoded = decoder.decode(&buffer[..n], None).unwrap(); + + for f in decoded { + let result = FrameWithPts { + frame: f.frame, + pts: Duration::from_secs_f64(frame_number as f64 * frame_interval), + }; + + frame_number += 1; + + if tx.send(result).is_err() { + return; + } + } + } +} diff --git a/vk-video/examples/player/player/renderer.rs b/vk-video/examples/player/player/renderer.rs new file mode 100644 index 000000000..e68d625c2 --- /dev/null +++ b/vk-video/examples/player/player/renderer.rs @@ -0,0 +1,353 @@ +use std::sync::{mpsc::Receiver, Arc}; + +use vk_video::VulkanDevice; +use wgpu::util::DeviceExt; +use winit::{ + dpi::PhysicalSize, + event::{ElementState, Event, KeyEvent, WindowEvent}, + event_loop::EventLoop, + keyboard::{KeyCode, PhysicalKey}, + window::Window, +}; + +use super::FrameWithPts; + +pub fn run_renderer<'a>( + event_loop: EventLoop<()>, + window: &'a Window, + surface: wgpu::Surface<'a>, + vulkan_device: &VulkanDevice, + rx: Receiver, +) { + let mut current_frame = rx.recv().unwrap(); + let mut next_frame = None; + + window.set_title("vk-video example player"); + window.set_resizable(false); + let _ = window.request_inner_size(PhysicalSize::new( + current_frame.frame.size().width, + current_frame.frame.size().height, + )); + + let mut renderer = Renderer::new(surface, vulkan_device, window); + + let start_timestamp = std::time::Instant::now(); + event_loop + .run(move |event, cf| match event { + Event::WindowEvent { window_id, event } if window_id == window.id() => match event { + WindowEvent::KeyboardInput { + event: + KeyEvent { + state: ElementState::Pressed, + physical_key: PhysicalKey::Code(KeyCode::Escape), + .. + }, + .. + } + | WindowEvent::CloseRequested => cf.exit(), + + WindowEvent::RedrawRequested => { + window.request_redraw(); + if next_frame.is_none() { + if let Ok(f) = rx.try_recv() { + next_frame = Some(f); + } + } + + let current_pts = std::time::Instant::now() - start_timestamp; + if let Some(next_frame_pts) = next_frame.as_ref().map(|f| f.pts) { + if next_frame_pts < current_pts { + current_frame = next_frame.take().unwrap(); + } + } + + let _ = window.request_inner_size(PhysicalSize::new( + current_frame.frame.size().width, + current_frame.frame.size().height, + )); + + renderer.render(¤t_frame.frame, window).unwrap(); + } + + WindowEvent::Resized(new_size) => renderer.resize(new_size), + _ => {} + }, + _ => {} + }) + .unwrap(); +} + +#[derive(Debug, Clone, Copy, bytemuck::Zeroable, bytemuck::Pod)] +#[repr(C)] +struct Vertex { + position: [f32; 3], + texture_coords: [f32; 2], +} + +impl Vertex { + const ATTRIBUTES: &[wgpu::VertexAttribute] = + &wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x2]; + const LAYOUT: wgpu::VertexBufferLayout<'static> = wgpu::VertexBufferLayout { + step_mode: wgpu::VertexStepMode::Vertex, + attributes: Self::ATTRIBUTES, + array_stride: std::mem::size_of::() as wgpu::BufferAddress, + }; +} + +const VERTICES: &[Vertex] = &[ + Vertex { + position: [-1.0, 1.0, 0.0], + texture_coords: [0.0, 0.0], + }, + Vertex { + position: [-1.0, -1.0, 0.0], + texture_coords: [0.0, 1.0], + }, + Vertex { + position: [1.0, -1.0, 0.0], + texture_coords: [1.0, 1.0], + }, + Vertex { + position: [1.0, 1.0, 0.0], + texture_coords: [1.0, 0.0], + }, +]; + +const INDICES: &[u16] = &[0, 1, 3, 1, 2, 3]; + +struct Renderer<'a> { + surface: wgpu::Surface<'a>, + device: Arc, + queue: Arc, + surface_configuration: wgpu::SurfaceConfiguration, + sampler: wgpu::Sampler, + vertex_buffer: wgpu::Buffer, + index_buffer: wgpu::Buffer, + pipeline: wgpu::RenderPipeline, +} + +impl<'a> Renderer<'a> { + fn new(surface: wgpu::Surface<'a>, vulkan_device: &VulkanDevice, window: &Window) -> Self { + let device = vulkan_device.wgpu_device.clone(); + let queue = vulkan_device.wgpu_queue.clone(); + let size = window.inner_size(); + let surface_capabilities = surface.get_capabilities(&vulkan_device.wgpu_adapter); + let surface_texture_format = surface_capabilities + .formats + .iter() + .find(|f| f.is_srgb()) + .copied() + .unwrap_or(surface_capabilities.formats[0]); + + let surface_configuration = wgpu::SurfaceConfiguration { + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + width: size.width, + height: size.height, + format: surface_texture_format, + view_formats: vec![ + surface_texture_format, + surface_texture_format.remove_srgb_suffix(), + ], + alpha_mode: surface_capabilities.alpha_modes[0], + present_mode: surface_capabilities.present_modes[0], + desired_maximum_frame_latency: 2, + }; + + surface.configure(&device, &surface_configuration); + + let vertex_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("vertex buffer"), + usage: wgpu::BufferUsages::VERTEX, + contents: bytemuck::cast_slice(VERTICES), + }); + + let index_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("index buffer"), + usage: wgpu::BufferUsages::INDEX, + contents: bytemuck::cast_slice(INDICES), + }); + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("bgl"), + entries: &[ + wgpu::BindGroupLayoutEntry { + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + }, + wgpu::BindGroupLayoutEntry { + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + }, + wgpu::BindGroupLayoutEntry { + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + }, + ], + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Nearest, + min_filter: wgpu::FilterMode::Nearest, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("pipeline layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let shader_module_descriptor = wgpu::include_wgsl!("shader.wgsl"); + let shader_module = device.create_shader_module(shader_module_descriptor); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("render pipeline"), + layout: Some(&pipeline_layout), + cache: None, + vertex: wgpu::VertexState { + module: &shader_module, + buffers: &[Vertex::LAYOUT], + compilation_options: Default::default(), + entry_point: None, + }, + fragment: Some(wgpu::FragmentState { + module: &shader_module, + targets: &[Some(wgpu::ColorTargetState { + format: surface_configuration.format.remove_srgb_suffix(), + blend: Some(wgpu::BlendState::REPLACE), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + entry_point: None, + }), + + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + strip_index_format: None, + cull_mode: Some(wgpu::Face::Back), + front_face: wgpu::FrontFace::Ccw, + polygon_mode: wgpu::PolygonMode::Fill, + conservative: false, + unclipped_depth: false, + }, + multisample: wgpu::MultisampleState { + count: 1, + mask: !0, + alpha_to_coverage_enabled: false, + }, + multiview: None, + depth_stencil: None, + }); + + Self { + surface, + device, + queue, + surface_configuration, + sampler, + index_buffer, + vertex_buffer, + pipeline, + } + } + + fn resize(&mut self, size: PhysicalSize) { + if size.width > 0 && size.height > 0 { + self.surface_configuration.width = size.width; + self.surface_configuration.height = size.height; + self.surface + .configure(&self.device, &self.surface_configuration); + } + } + + fn render(&mut self, frame: &wgpu::Texture, window: &Window) -> Result<(), wgpu::SurfaceError> { + let device = &self.device; + let surface = self.surface.get_current_texture()?; + let surface_view = surface.texture.create_view(&wgpu::TextureViewDescriptor { + format: Some(surface.texture.format().remove_srgb_suffix()), + ..Default::default() + }); + let texture_view_y = frame.create_view(&wgpu::TextureViewDescriptor { + label: Some("y texture"), + format: Some(wgpu::TextureFormat::R8Unorm), + aspect: wgpu::TextureAspect::Plane0, + dimension: Some(wgpu::TextureViewDimension::D2), + ..Default::default() + }); + + let texture_view_uv = frame.create_view(&wgpu::TextureViewDescriptor { + label: Some("uv texture"), + format: Some(wgpu::TextureFormat::Rg8Unorm), + aspect: wgpu::TextureAspect::Plane1, + dimension: Some(wgpu::TextureViewDimension::D2), + ..Default::default() + }); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("bind group"), + layout: &self.pipeline.get_bind_group_layout(0), + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&texture_view_y), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&texture_view_uv), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + + let mut command_encoder = device.create_command_encoder(&Default::default()); + + { + let mut render_pass = command_encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("render pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &surface_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::WHITE), + store: wgpu::StoreOp::Store, + }, + })], + ..Default::default() + }); + + render_pass.set_pipeline(&self.pipeline); + render_pass.set_bind_group(0, &bind_group, &[]); + render_pass.set_vertex_buffer(0, self.vertex_buffer.slice(..)); + render_pass.set_index_buffer(self.index_buffer.slice(..), wgpu::IndexFormat::Uint16); + render_pass.draw_indexed(0..INDICES.len() as u32, 0, 0..1); + } + + self.queue.submit(Some(command_encoder.finish())); + window.pre_present_notify(); + surface.present(); + + Ok(()) + } +} diff --git a/vk-video/examples/player/player/shader.wgsl b/vk-video/examples/player/player/shader.wgsl new file mode 100644 index 000000000..1aa05c41c --- /dev/null +++ b/vk-video/examples/player/player/shader.wgsl @@ -0,0 +1,37 @@ +struct VertexInput { + @location(0) position: vec3, + @location(1) tex_coords: vec2, +} + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) tex_coords: vec2, +} + +@vertex +fn vs_main(input: VertexInput) -> VertexOutput { + var output: VertexOutput; + + output.position = vec4(input.position, 1.0); + output.tex_coords = input.tex_coords; + + return output; +} + +@group(0) @binding(0) var y_texture: texture_2d; +@group(0) @binding(1) var uv_texture: texture_2d; +@group(0) @binding(2) var sampler_: sampler; + +@fragment +fn fs_main(input: VertexOutput) -> @location(0) vec4 { + var y = textureSample(y_texture, sampler_, input.tex_coords).x; + var uv = textureSample(uv_texture, sampler_, input.tex_coords); + var u = uv.x; + var v = uv.y; + + let r = y + 1.40200 * (v - 128.0 / 255.0); + let g = y - 0.34414 * (u - 128.0 / 255.0) - 0.71414 * (v - 128.0 / 255.0); + let b = y + 1.77200 * (u - 128.0 / 255.0); + + return vec4(clamp(r, 0.0, 1.0), clamp(g, 0.0, 1.0), clamp(b, 0.0, 1.0), 1.0); +} diff --git a/vk-video/examples/wgpu.rs b/vk-video/examples/wgpu.rs index 95d56096a..6ef6346ad 100644 --- a/vk-video/examples/wgpu.rs +++ b/vk-video/examples/wgpu.rs @@ -2,7 +2,7 @@ fn main() { use std::io::Write; - use vk_video::Frame; + use vk_video::{Frame, VulkanInstance}; let subscriber = tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::INFO) @@ -17,25 +17,27 @@ fn main() { } let h264_bytestream = std::fs::read(&args[1]).unwrap_or_else(|_| panic!("read {}", args[1])); - let vulkan_ctx = std::sync::Arc::new( - vk_video::VulkanCtx::new( + let vulkan_instance = VulkanInstance::new().unwrap(); + let vulkan_device = vulkan_instance + .create_device( wgpu::Features::empty(), wgpu::Limits { max_push_constant_size: 128, ..Default::default() }, + &mut None, ) - .unwrap(), - ); - let mut decoder = vk_video::WgpuTexturesDeocder::new(vulkan_ctx.clone()).unwrap(); + .unwrap(); + + let mut decoder = vulkan_device.create_wgpu_textures_decoder().unwrap(); let mut output_file = std::fs::File::create("output.nv12").unwrap(); for chunk in h264_bytestream.chunks(256) { let frames = decoder.decode(chunk, None).unwrap(); - let device = &vulkan_ctx.wgpu_ctx.device; - let queue = &vulkan_ctx.wgpu_ctx.queue; + let device = &vulkan_device.wgpu_device; + let queue = &vulkan_device.wgpu_queue; for Frame { frame, .. } in frames { let decoded_frame = download_wgpu_texture(device, queue, frame); output_file.write_all(&decoded_frame).unwrap(); diff --git a/vk-video/src/lib.rs b/vk-video/src/lib.rs index 92fcbb26f..5948eed65 100644 --- a/vk-video/src/lib.rs +++ b/vk-video/src/lib.rs @@ -6,9 +6,7 @@ use parser::Parser; use vulkan_decoder::{FrameSorter, VulkanDecoder}; pub use parser::ParserError; -pub use vulkan_decoder::{VulkanCtx, VulkanCtxError, VulkanDecoderError}; - -pub use vulkan_decoder::WgpuCtx; +pub use vulkan_decoder::{VulkanCtxError, VulkanDecoderError, VulkanDevice, VulkanInstance}; #[derive(Debug, thiserror::Error)] pub enum DecoderError { @@ -31,18 +29,6 @@ pub struct WgpuTexturesDeocder<'a> { } impl WgpuTexturesDeocder<'_> { - pub fn new(vulkan_ctx: std::sync::Arc) -> Result { - let parser = Parser::default(); - let vulkan_decoder = VulkanDecoder::new(vulkan_ctx)?; - let frame_sorter = FrameSorter::::new(); - - Ok(Self { - parser, - vulkan_decoder, - frame_sorter, - }) - } - // TODO: the below hasn't been verified. /// The produced textures have the [`wgpu::TextureFormat::NV12`] format and can be used as a copy source or a texture binding. pub fn decode( @@ -72,18 +58,6 @@ pub struct BytesDecoder<'a> { } impl BytesDecoder<'_> { - pub fn new(vulkan_ctx: std::sync::Arc) -> Result { - let parser = Parser::default(); - let vulkan_decoder = VulkanDecoder::new(vulkan_ctx)?; - let frame_sorter = FrameSorter::>::new(); - - Ok(Self { - parser, - vulkan_decoder, - frame_sorter, - }) - } - /// The result is a sequence of frames. Te payload of each [`Frame`] struct is a [`Vec`]. Each [`Vec`] contains a single /// decoded frame in the [NV12 format](https://en.wikipedia.org/wiki/YCbCr#4:2:0). pub fn decode( diff --git a/vk-video/src/vulkan_decoder.rs b/vk-video/src/vulkan_decoder.rs index d6486d478..1777a69d6 100644 --- a/vk-video/src/vulkan_decoder.rs +++ b/vk-video/src/vulkan_decoder.rs @@ -18,7 +18,7 @@ pub(crate) use frame_sorter::FrameSorter; pub use vulkan_ctx::*; pub struct VulkanDecoder<'a> { - vulkan_ctx: Arc, + vulkan_device: Arc, video_session_resources: Option>, command_buffers: CommandBuffers, _command_pools: CommandPools, @@ -99,7 +99,7 @@ pub enum VulkanDecoderError { } impl<'a> VulkanDecoder<'a> { - pub fn new(vulkan_ctx: Arc) -> Result { + pub fn new(vulkan_ctx: Arc) -> Result { let decode_pool = Arc::new(CommandPool::new( vulkan_ctx.device.clone(), vulkan_ctx.queues.h264_decode.idx, @@ -128,7 +128,7 @@ impl<'a> VulkanDecoder<'a> { }; Ok(Self { - vulkan_ctx, + vulkan_device: vulkan_ctx, video_session_resources: None, _command_pools: command_pools, command_buffers: CommandBuffers { @@ -237,14 +237,14 @@ impl VulkanDecoder<'_> { fn process_sps(&mut self, sps: &SeqParameterSet) -> Result<(), VulkanDecoderError> { match self.video_session_resources.as_mut() { Some(session) => session.process_sps( - &self.vulkan_ctx, + &self.vulkan_device, &self.command_buffers.decode_buffer, sps.clone(), &self.sync_structures.fence_memory_barrier_completed, )?, None => { self.video_session_resources = Some(VideoSessionResources::new_from_sps( - &self.vulkan_ctx, + &self.vulkan_device, &self.command_buffers.decode_buffer, sps.clone(), &self.sync_structures.fence_memory_barrier_completed, @@ -298,7 +298,7 @@ impl VulkanDecoder<'_> { // upload data to a buffer let size = Self::pad_size_to_alignment( decode_information.rbsp_bytes.len() as u64, - self.vulkan_ctx + self.vulkan_device .video_capabilities .min_bitstream_buffer_offset_alignment, ); @@ -310,7 +310,7 @@ impl VulkanDecoder<'_> { .ok_or(VulkanDecoderError::NoSession)?; let decode_buffer = Buffer::new_with_decode_data( - self.vulkan_ctx.allocator.clone(), + self.vulkan_device.allocator.clone(), &decode_information.rbsp_bytes, size, &video_session_resources.profile_info, @@ -337,7 +337,7 @@ impl VulkanDecoder<'_> { ); unsafe { - self.vulkan_ctx.device.cmd_pipeline_barrier2( + self.vulkan_device.device.cmd_pipeline_barrier2( *self.command_buffers.decode_buffer, &vk::DependencyInfo::default().memory_barriers(&[memory_barrier]), ) @@ -357,7 +357,7 @@ impl VulkanDecoder<'_> { .reference_slots(&reference_slots); unsafe { - self.vulkan_ctx + self.vulkan_device .device .video_queue_ext .cmd_begin_video_coding_khr(*self.command_buffers.decode_buffer, &begin_info) @@ -369,7 +369,7 @@ impl VulkanDecoder<'_> { .flags(vk::VideoCodingControlFlagsKHR::RESET); unsafe { - self.vulkan_ctx + self.vulkan_device .device .video_queue_ext .cmd_control_video_coding_khr( @@ -481,7 +481,7 @@ impl VulkanDecoder<'_> { } unsafe { - self.vulkan_ctx + self.vulkan_device .device .video_decode_queue_ext .cmd_decode_video_khr(*self.command_buffers.decode_buffer, &decode_info) @@ -492,7 +492,7 @@ impl VulkanDecoder<'_> { } unsafe { - self.vulkan_ctx + self.vulkan_device .device .video_queue_ext .cmd_end_video_coding_khr( @@ -503,7 +503,7 @@ impl VulkanDecoder<'_> { self.command_buffers.decode_buffer.end()?; - self.vulkan_ctx.queues.h264_decode.submit( + self.vulkan_device.queues.h264_decode.submit( &self.command_buffers.decode_buffer, &[], &[( @@ -552,8 +552,8 @@ impl VulkanDecoder<'_> { }; let queue_indices = [ - self.vulkan_ctx.queues.transfer.idx as u32, - self.vulkan_ctx.queues.wgpu.idx as u32, + self.vulkan_device.queues.transfer.idx as u32, + self.vulkan_device.queues.wgpu.idx as u32, ]; let create_info = vk::ImageCreateInfo::default() @@ -574,7 +574,10 @@ impl VulkanDecoder<'_> { .queue_family_indices(&queue_indices) .initial_layout(vk::ImageLayout::UNDEFINED); - let image = Arc::new(Image::new(self.vulkan_ctx.allocator.clone(), &create_info)?); + let image = Arc::new(Image::new( + self.vulkan_device.allocator.clone(), + &create_info, + )?); self.command_buffers .vulkan_to_wgpu_transfer_buffer @@ -617,7 +620,7 @@ impl VulkanDecoder<'_> { }); unsafe { - self.vulkan_ctx.device.cmd_pipeline_barrier2( + self.vulkan_device.device.cmd_pipeline_barrier2( *self.command_buffers.vulkan_to_wgpu_transfer_buffer, &vk::DependencyInfo::default() .image_memory_barriers(&[memory_barrier_src, memory_barrier_dst]), @@ -664,7 +667,7 @@ impl VulkanDecoder<'_> { ]; unsafe { - self.vulkan_ctx.device.cmd_copy_image( + self.vulkan_device.device.cmd_copy_image( *self.command_buffers.vulkan_to_wgpu_transfer_buffer, decode_output.image, vk::ImageLayout::TRANSFER_SRC_OPTIMAL, @@ -691,7 +694,7 @@ impl VulkanDecoder<'_> { .new_layout(vk::ImageLayout::GENERAL); unsafe { - self.vulkan_ctx.device.cmd_pipeline_barrier2( + self.vulkan_device.device.cmd_pipeline_barrier2( *self.command_buffers.vulkan_to_wgpu_transfer_buffer, &vk::DependencyInfo::default() .image_memory_barriers(&[memory_barrier_src, memory_barrier_dst]), @@ -700,7 +703,7 @@ impl VulkanDecoder<'_> { self.command_buffers.vulkan_to_wgpu_transfer_buffer.end()?; - self.vulkan_ctx.queues.transfer.submit( + self.vulkan_device.queues.transfer.submit( &self.command_buffers.vulkan_to_wgpu_transfer_buffer, &[( decode_output.wait_semaphore, @@ -759,9 +762,8 @@ impl VulkanDecoder<'_> { }; let wgpu_texture = unsafe { - self.vulkan_ctx - .wgpu_ctx - .device + self.vulkan_device + .wgpu_device .create_texture_from_hal::( hal_texture, &wgpu::TextureDescriptor { @@ -906,7 +908,7 @@ impl VulkanDecoder<'_> { }); unsafe { - self.vulkan_ctx.device.cmd_pipeline_barrier2( + self.vulkan_device.device.cmd_pipeline_barrier2( *self.command_buffers.gpu_to_mem_transfer_buffer, &vk::DependencyInfo::default().image_memory_barriers(&[memory_barrier]), ) @@ -915,7 +917,7 @@ impl VulkanDecoder<'_> { let y_plane_size = dimensions.width as u64 * dimensions.height as u64; let dst_buffer = Buffer::new_transfer( - self.vulkan_ctx.allocator.clone(), + self.vulkan_device.allocator.clone(), y_plane_size * 3 / 2, TransferDirection::GpuToMem, )?; @@ -956,7 +958,7 @@ impl VulkanDecoder<'_> { ]; unsafe { - self.vulkan_ctx.device.cmd_copy_image_to_buffer( + self.vulkan_device.device.cmd_copy_image_to_buffer( *self.command_buffers.gpu_to_mem_transfer_buffer, image, vk::ImageLayout::TRANSFER_SRC_OPTIMAL, @@ -974,7 +976,7 @@ impl VulkanDecoder<'_> { .new_layout(current_image_layout); unsafe { - self.vulkan_ctx.device.cmd_pipeline_barrier2( + self.vulkan_device.device.cmd_pipeline_barrier2( *self.command_buffers.gpu_to_mem_transfer_buffer, &vk::DependencyInfo::default().image_memory_barriers(&[memory_barrier]), ) @@ -982,7 +984,7 @@ impl VulkanDecoder<'_> { self.command_buffers.gpu_to_mem_transfer_buffer.end()?; - self.vulkan_ctx.queues.transfer.submit( + self.vulkan_device.queues.transfer.submit( &self.command_buffers.gpu_to_mem_transfer_buffer, wait_semaphores, signal_semaphores, diff --git a/vk-video/src/vulkan_decoder/session_resources.rs b/vk-video/src/vulkan_decoder/session_resources.rs index b57892e45..9c41a1fb1 100644 --- a/vk-video/src/vulkan_decoder/session_resources.rs +++ b/vk-video/src/vulkan_decoder/session_resources.rs @@ -10,7 +10,7 @@ use parameters::VideoSessionParametersManager; use super::{ h264_level_idc_to_max_dpb_mbs, vk_to_h264_level_idc, CommandBuffer, DecodeQueryPool, Fence, - H264ProfileInfo, SeqParameterSetExt, VideoSession, VulkanCtx, VulkanDecoderError, + H264ProfileInfo, SeqParameterSetExt, VideoSession, VulkanDecoderError, VulkanDevice, }; mod images; @@ -55,7 +55,7 @@ fn calculate_max_num_reorder_frames(sps: &SeqParameterSet) -> Result { pub(crate) fn new_from_sps( - vulkan_ctx: &VulkanCtx, + vulkan_ctx: &VulkanDevice, decode_buffer: &CommandBuffer, sps: SeqParameterSet, fence_memory_barrier_completed: &Fence, @@ -131,7 +131,7 @@ impl VideoSessionResources<'_> { pub(crate) fn process_sps( &mut self, - vulkan_ctx: &VulkanCtx, + vulkan_ctx: &VulkanDevice, decode_buffer: &CommandBuffer, sps: SeqParameterSet, fence_memory_barrier_completed: &Fence, @@ -200,7 +200,7 @@ impl VideoSessionResources<'_> { } fn new_decoding_images<'a>( - vulkan_ctx: &VulkanCtx, + vulkan_ctx: &VulkanDevice, profile: &H264ProfileInfo, max_coded_extent: vk::Extent2D, max_dpb_slots: u32, diff --git a/vk-video/src/vulkan_decoder/session_resources/images.rs b/vk-video/src/vulkan_decoder/session_resources/images.rs index 79b0e60f1..ab88132fc 100644 --- a/vk-video/src/vulkan_decoder/session_resources/images.rs +++ b/vk-video/src/vulkan_decoder/session_resources/images.rs @@ -4,7 +4,7 @@ use ash::vk; use crate::{ vulkan_decoder::{H264ProfileInfo, Image, ImageView}, - VulkanCtx, VulkanDecoderError, + VulkanDecoderError, VulkanDevice, }; pub(crate) struct DecodingImages<'a> { @@ -22,7 +22,7 @@ pub(crate) struct DecodingImageBundle<'a> { impl<'a> DecodingImageBundle<'a> { #[allow(clippy::too_many_arguments)] pub(crate) fn new( - vulkan_ctx: &VulkanCtx, + vulkan_ctx: &VulkanDevice, format: &vk::VideoFormatPropertiesKHR<'a>, dimensions: vk::Extent2D, image_usage: vk::ImageUsageFlags, @@ -156,7 +156,7 @@ impl<'a> DecodingImages<'a> { } pub(crate) fn new( - vulkan_ctx: &VulkanCtx, + vulkan_ctx: &VulkanDevice, profile: &H264ProfileInfo, dpb_format: &vk::VideoFormatPropertiesKHR<'a>, dst_format: &Option>, diff --git a/vk-video/src/vulkan_decoder/session_resources/parameters.rs b/vk-video/src/vulkan_decoder/session_resources/parameters.rs index e5f366f67..97af83f06 100644 --- a/vk-video/src/vulkan_decoder/session_resources/parameters.rs +++ b/vk-video/src/vulkan_decoder/session_resources/parameters.rs @@ -7,7 +7,7 @@ use crate::{ vulkan_decoder::{ Device, VideoSessionParameters, VkPictureParameterSet, VkSequenceParameterSet, }, - VulkanCtx, VulkanDecoderError, + VulkanDecoderError, VulkanDevice, }; /// Since `VideoSessionParameters` can only add sps and pps values (inserting sps or pps with an @@ -23,7 +23,7 @@ pub(crate) struct VideoSessionParametersManager { impl VideoSessionParametersManager { pub(crate) fn new( - vulkan_ctx: &VulkanCtx, + vulkan_ctx: &VulkanDevice, session: vk::VideoSessionKHR, ) -> Result { Ok(Self { diff --git a/vk-video/src/vulkan_decoder/vulkan_ctx.rs b/vk-video/src/vulkan_decoder/vulkan_ctx.rs index 24d94c37b..132ab4a3f 100644 --- a/vk-video/src/vulkan_decoder/vulkan_ctx.rs +++ b/vk-video/src/vulkan_decoder/vulkan_ctx.rs @@ -5,9 +5,13 @@ use std::{ use ash::{vk, Entry}; use tracing::{debug, error, warn}; +use wgpu::hal::Adapter; + +use crate::{parser::Parser, BytesDecoder, DecoderError, WgpuTexturesDeocder}; use super::{ - Allocator, CommandBuffer, CommandPool, DebugMessenger, Device, Instance, VulkanDecoderError, + Allocator, CommandBuffer, CommandPool, DebugMessenger, Device, FrameSorter, Instance, + VulkanDecoder, VulkanDecoderError, }; const REQUIRED_EXTENSIONS: &[&CStr] = &[ @@ -43,103 +47,15 @@ pub enum VulkanCtxError { StringConversionError(#[from] std::ffi::FromBytesUntilNulError), } -pub struct VulkanCtx { +pub struct VulkanInstance { _entry: Arc, - _instance: Arc, - _physical_device: vk::PhysicalDevice, - pub(crate) device: Arc, - pub(crate) allocator: Arc, - pub(crate) queues: Queues, + instance: Arc, _debug_messenger: Option, - pub(crate) video_capabilities: vk::VideoCapabilitiesKHR<'static>, - pub(crate) h264_dpb_format_properties: vk::VideoFormatPropertiesKHR<'static>, - pub(crate) h264_dst_format_properties: Option>, - pub(crate) h264_caps: vk::VideoDecodeH264CapabilitiesKHR<'static>, - pub wgpu_ctx: WgpuCtx, -} - -pub struct WgpuCtx { - pub instance: Arc, - pub adapter: Arc, - pub device: Arc, - pub queue: Arc, + pub wgpu_instance: Arc, } -pub(crate) struct CommandPools { - pub(crate) _decode_pool: Arc, - pub(crate) _transfer_pool: Arc, -} - -pub(crate) struct Queue { - pub(crate) queue: std::sync::Mutex, - pub(crate) idx: usize, - _video_properties: vk::QueueFamilyVideoPropertiesKHR<'static>, - pub(crate) query_result_status_properties: - vk::QueueFamilyQueryResultStatusPropertiesKHR<'static>, - device: Arc, -} - -impl Queue { - pub(crate) fn supports_result_status_queries(&self) -> bool { - self.query_result_status_properties - .query_result_status_support - == vk::TRUE - } - - pub(crate) fn submit( - &self, - buffer: &CommandBuffer, - wait_semaphores: &[(vk::Semaphore, vk::PipelineStageFlags2)], - signal_semaphores: &[(vk::Semaphore, vk::PipelineStageFlags2)], - fence: Option, - ) -> Result<(), VulkanDecoderError> { - fn to_sem_submit_info( - submits: &[(vk::Semaphore, vk::PipelineStageFlags2)], - ) -> Vec { - submits - .iter() - .map(|&(sem, stage)| { - vk::SemaphoreSubmitInfo::default() - .semaphore(sem) - .stage_mask(stage) - }) - .collect::>() - } - - let wait_semaphores = to_sem_submit_info(wait_semaphores); - let signal_semaphores = to_sem_submit_info(signal_semaphores); - - let buffer_submit_info = - [vk::CommandBufferSubmitInfo::default().command_buffer(buffer.buffer)]; - - let submit_info = [vk::SubmitInfo2::default() - .wait_semaphore_infos(&wait_semaphores) - .signal_semaphore_infos(&signal_semaphores) - .command_buffer_infos(&buffer_submit_info)]; - - unsafe { - self.device.queue_submit2( - *self.queue.lock().unwrap(), - &submit_info, - fence.unwrap_or(vk::Fence::null()), - )? - }; - - Ok(()) - } -} - -pub(crate) struct Queues { - pub(crate) transfer: Queue, - pub(crate) h264_decode: Queue, - pub(crate) wgpu: Queue, -} - -impl VulkanCtx { - pub fn new( - wgpu_features: wgpu::Features, - wgpu_limits: wgpu::Limits, - ) -> Result { +impl VulkanInstance { + pub fn new() -> Result, VulkanCtxError> { let entry = Arc::new(unsafe { Entry::load()? }); let api_version = vk::make_api_version(0, 1, 3, 0); @@ -232,20 +148,44 @@ impl VulkanCtx { )? }; - let physical_devices = unsafe { instance.enumerate_physical_devices()? }; + let wgpu_instance = + unsafe { wgpu::Instance::from_hal::(wgpu_instance) }; + + Ok(Self { + _entry: entry, + instance, + _debug_messenger: debug_messenger, + wgpu_instance: wgpu_instance.into(), + } + .into()) + } + + /// The `compatible surface` being a `&mut Option<&mut wgpu::Surface<'_>>` is a result of + /// weirdness in wgpu API, which we fixed upstream. When wgpu releases, this will be converted to + /// `Option<&wgpu::Surface<'_>>` + pub fn create_device( + &self, + wgpu_features: wgpu::Features, + wgpu_limits: wgpu::Limits, + compatible_surface: &mut Option<&mut wgpu::Surface<'_>>, + ) -> Result, VulkanCtxError> { + let physical_devices = unsafe { self.instance.enumerate_physical_devices()? }; let ChosenDevice { physical_device, + wgpu_adapter, queue_indices, h264_dpb_format_properties, h264_dst_format_properties, video_capabilities, h264_caps, - } = find_device(&physical_devices, &instance, REQUIRED_EXTENSIONS)?; - - let wgpu_adapter = wgpu_instance - .expose_adapter(physical_device) - .ok_or(VulkanCtxError::WgpuAdapterNotCreated)?; + } = find_device( + &physical_devices, + &self.instance, + &self.wgpu_instance, + REQUIRED_EXTENSIONS, + compatible_surface, + )?; let wgpu_features = wgpu_features | wgpu::Features::TEXTURE_FORMAT_NV12; @@ -281,15 +221,19 @@ impl VulkanCtx { .add_to_device_create(device_create_info) .push_next(&mut vk_synch_2_feature); - let device = unsafe { instance.create_device(physical_device, &device_create_info, None)? }; - let video_queue_ext = ash::khr::video_queue::Device::new(&instance, &device); - let video_decode_queue_ext = ash::khr::video_decode_queue::Device::new(&instance, &device); + let device = unsafe { + self.instance + .create_device(physical_device, &device_create_info, None)? + }; + let video_queue_ext = ash::khr::video_queue::Device::new(&self.instance, &device); + let video_decode_queue_ext = + ash::khr::video_decode_queue::Device::new(&self.instance, &device); let device = Arc::new(Device { device, video_queue_ext, video_decode_queue_ext, - _instance: instance.clone(), + _instance: self.instance.clone(), }); let h264_decode_queue = @@ -350,14 +294,12 @@ impl VulkanCtx { }; let allocator = Arc::new(Allocator::new( - instance.clone(), + self.instance.clone(), physical_device, device.clone(), )?); - let wgpu_instance = - unsafe { wgpu::Instance::from_hal::(wgpu_instance) }; - let wgpu_adapter = unsafe { wgpu_instance.create_adapter_from_hal(wgpu_adapter) }; + let wgpu_adapter = unsafe { self.wgpu_instance.create_adapter_from_hal(wgpu_adapter) }; let (wgpu_device, wgpu_queue) = unsafe { wgpu_adapter.create_device_from_hal( wgpu_device, @@ -371,38 +313,80 @@ impl VulkanCtx { )? }; - let wgpu_ctx = WgpuCtx { - instance: Arc::new(wgpu_instance), - adapter: Arc::new(wgpu_adapter), - device: Arc::new(wgpu_device), - queue: Arc::new(wgpu_queue), - }; - - Ok(Self { - _entry: entry, - _instance: instance, + Ok(VulkanDevice { _physical_device: physical_device, device, allocator, queues, - _debug_messenger: debug_messenger, video_capabilities, h264_dpb_format_properties, h264_dst_format_properties, h264_caps, - wgpu_ctx, + wgpu_device: wgpu_device.into(), + wgpu_queue: wgpu_queue.into(), + wgpu_adapter: wgpu_adapter.into(), + } + .into()) + } +} + +impl std::fmt::Debug for VulkanInstance { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VulkanInstance").finish() + } +} + +pub struct VulkanDevice { + _physical_device: vk::PhysicalDevice, + pub(crate) device: Arc, + pub(crate) allocator: Arc, + pub(crate) queues: Queues, + pub(crate) video_capabilities: vk::VideoCapabilitiesKHR<'static>, + pub(crate) h264_dpb_format_properties: vk::VideoFormatPropertiesKHR<'static>, + pub(crate) h264_dst_format_properties: Option>, + pub(crate) h264_caps: vk::VideoDecodeH264CapabilitiesKHR<'static>, + pub wgpu_device: Arc, + pub wgpu_queue: Arc, + pub wgpu_adapter: Arc, +} + +impl VulkanDevice { + pub fn create_wgpu_textures_decoder( + self: &Arc, + ) -> Result { + let parser = Parser::default(); + let vulkan_decoder = VulkanDecoder::new(self.clone())?; + let frame_sorter = FrameSorter::::new(); + + Ok(WgpuTexturesDeocder { + parser, + vulkan_decoder, + frame_sorter, + }) + } + + pub fn create_bytes_decoder(self: &Arc) -> Result { + let parser = Parser::default(); + let vulkan_decoder = VulkanDecoder::new(self.clone())?; + let frame_sorter = FrameSorter::>::new(); + + Ok(BytesDecoder { + parser, + vulkan_decoder, + frame_sorter, }) } } -impl std::fmt::Debug for VulkanCtx { +impl std::fmt::Debug for VulkanDevice { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("VulkanCtx").finish() + f.debug_struct("VulkanDevice").finish() } } struct ChosenDevice<'a> { physical_device: vk::PhysicalDevice, + wgpu_adapter: wgpu::hal::ExposedAdapter, queue_indices: QueueIndices<'a>, h264_dpb_format_properties: vk::VideoFormatPropertiesKHR<'a>, h264_dst_format_properties: Option>, @@ -413,11 +397,31 @@ struct ChosenDevice<'a> { fn find_device<'a>( devices: &[vk::PhysicalDevice], instance: &Instance, + wgpu_instance: &wgpu::Instance, required_extension_names: &[&CStr], + compatible_surface: &mut Option<&mut wgpu::Surface<'_>>, ) -> Result, VulkanCtxError> { for &device in devices { let properties = unsafe { instance.get_physical_device_properties(device) }; + let wgpu_instance = unsafe { wgpu_instance.as_hal::() }.unwrap(); + + let wgpu_adapter = wgpu_instance + .expose_adapter(device) + .ok_or(VulkanCtxError::WgpuAdapterNotCreated)?; + + if let Some(surface) = compatible_surface { + let surface_capabilities = unsafe { + (*surface).as_hal::(|surface| { + surface.and_then(|surface| wgpu_adapter.adapter.surface_capabilities(surface)) + }) + }; + + if surface_capabilities.is_none() { + continue; + } + } + let mut vk_13_features = vk::PhysicalDeviceVulkan13Features::default(); let mut features = vk::PhysicalDeviceFeatures2::default().push_next(&mut vk_13_features); @@ -628,6 +632,7 @@ fn find_device<'a>( return Ok(ChosenDevice { physical_device: device, + wgpu_adapter, queue_indices: QueueIndices { transfer: QueueIndex { idx: transfer_queue_idx, @@ -703,6 +708,76 @@ fn query_video_format_properties<'a>( Ok(format_properties) } +pub(crate) struct CommandPools { + pub(crate) _decode_pool: Arc, + pub(crate) _transfer_pool: Arc, +} + +pub(crate) struct Queue { + pub(crate) queue: std::sync::Mutex, + pub(crate) idx: usize, + _video_properties: vk::QueueFamilyVideoPropertiesKHR<'static>, + pub(crate) query_result_status_properties: + vk::QueueFamilyQueryResultStatusPropertiesKHR<'static>, + device: Arc, +} + +impl Queue { + pub(crate) fn supports_result_status_queries(&self) -> bool { + self.query_result_status_properties + .query_result_status_support + == vk::TRUE + } + + pub(crate) fn submit( + &self, + buffer: &CommandBuffer, + wait_semaphores: &[(vk::Semaphore, vk::PipelineStageFlags2)], + signal_semaphores: &[(vk::Semaphore, vk::PipelineStageFlags2)], + fence: Option, + ) -> Result<(), VulkanDecoderError> { + fn to_sem_submit_info( + submits: &[(vk::Semaphore, vk::PipelineStageFlags2)], + ) -> Vec { + submits + .iter() + .map(|&(sem, stage)| { + vk::SemaphoreSubmitInfo::default() + .semaphore(sem) + .stage_mask(stage) + }) + .collect::>() + } + + let wait_semaphores = to_sem_submit_info(wait_semaphores); + let signal_semaphores = to_sem_submit_info(signal_semaphores); + + let buffer_submit_info = + [vk::CommandBufferSubmitInfo::default().command_buffer(buffer.buffer)]; + + let submit_info = [vk::SubmitInfo2::default() + .wait_semaphore_infos(&wait_semaphores) + .signal_semaphore_infos(&signal_semaphores) + .command_buffer_infos(&buffer_submit_info)]; + + unsafe { + self.device.queue_submit2( + *self.queue.lock().unwrap(), + &submit_info, + fence.unwrap_or(vk::Fence::null()), + )? + }; + + Ok(()) + } +} + +pub(crate) struct Queues { + pub(crate) transfer: Queue, + pub(crate) h264_decode: Queue, + pub(crate) wgpu: Queue, +} + struct QueueIndex<'a> { idx: usize, video_properties: vk::QueueFamilyVideoPropertiesKHR<'a>, diff --git a/vk-video/src/vulkan_decoder/wrappers/video.rs b/vk-video/src/vulkan_decoder/wrappers/video.rs index 315441719..7c3757820 100644 --- a/vk-video/src/vulkan_decoder/wrappers/video.rs +++ b/vk-video/src/vulkan_decoder/wrappers/video.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use ash::vk; -use crate::{vulkan_decoder::VulkanDecoderError, VulkanCtx}; +use crate::{vulkan_decoder::VulkanDecoderError, VulkanDevice}; use super::{Device, MemoryAllocation, VideoQueueExt}; @@ -96,7 +96,7 @@ pub(crate) struct VideoSession { impl VideoSession { pub(crate) fn new( - vulkan_ctx: &VulkanCtx, + vulkan_ctx: &VulkanDevice, profile_info: &vk::VideoProfileInfoKHR, max_coded_extent: vk::Extent2D, max_dpb_slots: u32,