diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 079373db..f3180790 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -74,6 +74,10 @@ jobs: # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} + # make sure certain cargo features compile + cargo check -p hyperlight-host --features crashdump + cargo check -p hyperlight-host --features print_debug + # without any driver (shouldn't compile) just test-rust-feature-compilation-fail ${{ matrix.config }} diff --git a/Cargo.lock b/Cargo.lock index 942f4ec6..7d5ab033 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arbitrary" @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -1060,7 +1060,6 @@ dependencies = [ "hyperlight-testing", "log", "strum", - "strum_macros 0.26.4", "tracing", ] @@ -1107,6 +1106,7 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry-otlp", + "opentelemetry-semantic-conventions", "opentelemetry_sdk", "page_size", "paste", @@ -1125,7 +1125,7 @@ dependencies = [ "strum", "tempfile", "termcolor", - "thiserror 2.0.3", + "thiserror 2.0.6", "tokio", "tracing", "tracing-chrome", @@ -1446,9 +1446,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.164" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libfuzzer-sys" @@ -1709,9 +1709,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "opentelemetry" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +checksum = "0f3cebff57f7dbd1255b44d8bddc2cebeb0ea677dbaa2e25a3070a91b318f660" dependencies = [ "futures-core", "futures-sink", @@ -1723,9 +1723,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e1f9c8b032d4f635c730c0efcf731d5e2530ea13fa8bef7939ddc8420696bd" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", @@ -1737,13 +1737,14 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tonic", + "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d3968ce3aefdcca5c27e3c4ea4391b37547726a70893aab52d3de95d5f8b34" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1751,11 +1752,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + [[package]] name = "opentelemetry_sdk" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3" +checksum = "27b742c1cae4693792cc564e58d75a2a0ba29421a34a85b50da92efa89ecb2bc" dependencies = [ "async-trait", "futures-channel", @@ -1770,6 +1777,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", + "tracing", ] [[package]] @@ -2488,24 +2496,11 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.25.0" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn", + "strum_macros", ] [[package]] @@ -2588,11 +2583,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.6", ] [[package]] @@ -2608,9 +2603,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", @@ -2812,9 +2807,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -2824,9 +2819,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -2846,9 +2841,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -2882,9 +2877,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" dependencies = [ "js-sys", "once_cell", @@ -2900,9 +2895,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" dependencies = [ "serde", "tracing-core", @@ -2910,9 +2905,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", diff --git a/docs/debugging-hyperlight.md b/docs/debugging-hyperlight.md index b933c775..9ff3b822 100644 --- a/docs/debugging-hyperlight.md +++ b/docs/debugging-hyperlight.md @@ -39,6 +39,6 @@ cargo test --package hyperlight-host --test integration_test --features print_de ## Dumping the memory configuration, virtual processor register state and memory contents on a crash or unexpected VM Exit -To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `dump_on_crash` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message. +To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message. There are no tools at this time to analyze the dump file, but it can be useful for debugging. diff --git a/docs/hyperlight-metrics-logs-and-traces.md b/docs/hyperlight-metrics-logs-and-traces.md index accb3370..3a22e5a8 100644 --- a/docs/hyperlight-metrics-logs-and-traces.md +++ b/docs/hyperlight-metrics-logs-and-traces.md @@ -82,13 +82,13 @@ In the [examples/otlp_tracing](../src/hyperlight_host/examples/otlp_tracing) dir #### Linux ```bash -RUST_LOG='none,hyperlight-host=info,tracing=info' cargo run --example otlp_tracing +RUST_LOG='none,hyperlight_host=info,tracing=info' cargo run --example otlp_tracing ``` #### Windows ```powershell -$env:RUST_LOG='none,hyperlight-host=info,tracing=info';cargo run --example otlp_tracing +$env:RUST_LOG='none,hyperlight_host=info,tracing=info';cargo run --example otlp_tracing ``` The sample will run and generate trace data until any key is pressed. @@ -96,7 +96,7 @@ The sample will run and generate trace data until any key is pressed. To view the trace data, leave the example running and use the jaegertracing/all-in-one container image with the following command: ```console - docker run -d --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 4317:4317 -p 16686:16686 jaegertracing/all-in-one:1.51 + docker run -d --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 4317:4317 -p 16686:16686 jaegertracing/all-in-one:1.60 ``` NOTE: when running this on windows that this is a linux container, so you will need to ensure that docker is configured to run linux containers using WSL2. Alternatively, you can download the Jaeger binaries from [here](https://www.jaegertracing.io/download/). Extract the archive and run the `jaeger-all-in-one` executable as follows: diff --git a/proposals/NNNN-hip-template/README.md b/proposals/NNNN-hip-template/README.md index b6ca1622..fb8d91b2 100644 --- a/proposals/NNNN-hip-template/README.md +++ b/proposals/NNNN-hip-template/README.md @@ -1,4 +1,4 @@ -# HIP NNNN - SKIP NAME +# HIP NNNN - HIP NAME - [Summary](#summary) diff --git a/src/hyperlight_common/Cargo.toml b/src/hyperlight_common/Cargo.toml index 1080f5be..b0ff2c14 100644 --- a/src/hyperlight_common/Cargo.toml +++ b/src/hyperlight_common/Cargo.toml @@ -16,11 +16,10 @@ workspace = true [dependencies] flatbuffers = { version = "24.3.25", default-features = false } -anyhow = { version = "1.0.72", default-features = false } +anyhow = { version = "1.0.94", default-features = false } log = "0.4.20" -tracing = { version = "0.1.27", optional = true } -strum = {version = "0.25", default-features = false, features = ["derive"]} -strum_macros = {version = "0.26", features =[]} +tracing = { version = "0.1.41", optional = true } +strum = {version = "0.26", default-features = false, features = ["derive"]} [features] default = ["tracing"] diff --git a/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs b/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs index 0add85f6..14fa02b9 100644 --- a/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs +++ b/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs @@ -16,7 +16,7 @@ limitations under the License. use anyhow::{bail, Error, Result}; use log::Level; -use strum_macros::EnumIter; +use strum::EnumIter; #[cfg(feature = "tracing")] use tracing::{instrument, Span}; diff --git a/src/hyperlight_guest/Cargo.toml b/src/hyperlight_guest/Cargo.toml index 98220695..d70559c9 100644 --- a/src/hyperlight_guest/Cargo.toml +++ b/src/hyperlight_guest/Cargo.toml @@ -19,7 +19,7 @@ printf = [] # compile printf alloca = [] # compile alloca wrapper [dependencies] -anyhow = { version = "1.0.45", default-features = false } +anyhow = { version = "1.0.94", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["alloc"] } buddy_system_allocator = "0.11.0" hyperlight-common = { workspace = true } @@ -27,5 +27,5 @@ spin = "0.9.8" log = { version = "0.4", default-features = false } [build-dependencies] -cc = "1.0" +cc = "1.2" cfg-if = "1.0" diff --git a/src/hyperlight_guest/build.rs b/src/hyperlight_guest/build.rs index 533e6422..34ecbe89 100644 --- a/src/hyperlight_guest/build.rs +++ b/src/hyperlight_guest/build.rs @@ -133,11 +133,6 @@ fn cargo_main() { cfg.define("__x86_64__", None); cfg.define("__LITTLE_ENDIAN__", None); - cfg.define("malloc", "hlmalloc"); - cfg.define("calloc", "hlcalloc"); - cfg.define("free", "hlfree"); - cfg.define("realloc", "hlrealloc"); - // silence compiler warnings cfg.flag("-Wno-sign-compare"); cfg.flag("-Wno-bitwise-op-parentheses"); diff --git a/src/hyperlight_guest/src/memory.rs b/src/hyperlight_guest/src/memory.rs index f777cf4a..70390cbd 100644 --- a/src/hyperlight_guest/src/memory.rs +++ b/src/hyperlight_guest/src/memory.rs @@ -25,19 +25,42 @@ use crate::entrypoint::abort_with_code; extern crate alloc; -#[no_mangle] -pub extern "C" fn hlmalloc(size: usize) -> *mut c_void { - alloc_helper(size, false) -} +/* + C-wrappers for Rust's registered global allocator. -pub fn alloc_helper(size: usize, zero: bool) -> *mut c_void { - // Allocate a block that includes space for both layout information and data + Each memory allocation via `malloc/calloc/realloc` is stored together with a `alloc::Layout` describing + the size and alignment of the allocation. This layout is stored just before the actual raw memory returned to the caller. + + Example: A call to malloc(64) will allocate space for both an `alloc::Layout` and 64 bytes of memory: + + ---------------------------------------------------------------------------------------- + | Layout { size: 64 + size_of::(), ... } | 64 bytes of memory | ... + ---------------------------------------------------------------------------------------- + ^ + | + | + ptr returned to caller +*/ + +// We assume the maximum alignment for any value is the alignment of u128. +const MAX_ALIGN: usize = align_of::(); + +/// Allocates a block of memory with the given size. The memory is only guaranteed to be initialized to 0s if `zero` is true, otherwise +/// it may or may not be initialized. +/// +/// # Safety +/// The returned pointer must be freed with `memory::free` when it is no longer needed, otherwise memory will leak. +unsafe fn alloc_helper(size: usize, zero: bool) -> *mut c_void { if size == 0 { return ptr::null_mut(); } - let total_size = size + size_of::(); - let layout = Layout::from_size_align(total_size, align_of::()).unwrap(); + // Allocate a block that includes space for both layout information and data + let total_size = size + .checked_add(size_of::()) + .expect("data and layout size should not overflow in alloc"); + let layout = Layout::from_size_align(total_size, MAX_ALIGN).expect("Invalid layout"); + unsafe { let raw_ptr = match zero { true => alloc::alloc::alloc_zeroed(layout), @@ -53,14 +76,36 @@ pub fn alloc_helper(size: usize, zero: bool) -> *mut c_void { } } +/// Allocates a block of memory with the given size. +/// The memory is not guaranteed to be initialized to 0s. +/// +/// # Safety +/// The returned pointer must be freed with `memory::free` when it is no longer needed, otherwise memory will leak. #[no_mangle] -pub extern "C" fn hlcalloc(n: usize, size: usize) -> *mut c_void { - let total_size = n * size; +pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + alloc_helper(size, false) +} + +/// Allocates a block of memory for an array of `nmemb` elements, each of `size` bytes. +/// The memory is initialized to 0s. +/// +/// # Safety +/// The returned pointer must be freed with `memory::free` when it is no longer needed, otherwise memory will leak. +#[no_mangle] +pub unsafe extern "C" fn calloc(nmemb: usize, size: usize) -> *mut c_void { + let total_size = nmemb + .checked_mul(size) + .expect("nmemb * size should not overflow in calloc"); + alloc_helper(total_size, true) } +/// Frees the memory block pointed to by `ptr`. +/// +/// # Safety +/// `ptr` must be a pointer to a memory block previously allocated by `memory::malloc`, `memory::calloc`, or `memory::realloc`. #[no_mangle] -pub extern "C" fn hlfree(ptr: *mut c_void) { +pub unsafe extern "C" fn free(ptr: *mut c_void) { if !ptr.is_null() { unsafe { let block_start = (ptr as *const Layout).sub(1); @@ -70,26 +115,43 @@ pub extern "C" fn hlfree(ptr: *mut c_void) { } } +/// Changes the size of the memory block pointed to by `ptr` to `size` bytes. If the returned ptr is non-null, +/// any usage of the old memory block is immediately undefined behavior. +/// +/// # Safety +/// `ptr` must be a pointer to a memory block previously allocated by `memory::malloc`, `memory::calloc`, or `memory::realloc`. #[no_mangle] -pub extern "C" fn hlrealloc(ptr: *mut c_void, size: usize) -> *mut c_void { +pub unsafe extern "C" fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void { if ptr.is_null() { // If the pointer is null, treat as a malloc - return hlmalloc(size); + return malloc(size); + } + + if size == 0 { + // If the size is 0, treat as a free and return null + free(ptr); + return ptr::null_mut(); } unsafe { + let total_new_size = size + .checked_add(size_of::()) + .expect("data and layout size should not overflow in realloc"); + let block_start = (ptr as *const Layout).sub(1); - let layout = block_start.read(); - let total_new_size = size + size_of::(); + let old_layout = block_start.read(); + let new_layout = Layout::from_size_align(total_new_size, MAX_ALIGN).unwrap(); + let new_block_start = - alloc::alloc::realloc(block_start as *mut u8, layout, total_new_size) as *mut Layout; + alloc::alloc::realloc(block_start as *mut u8, old_layout, total_new_size) + as *mut Layout; if new_block_start.is_null() { // Realloc failed abort_with_code(ErrorCode::MallocFailed as i32); } else { - // Return the pointer just after the layout information - // since old layout should still as it would have been copied + // Update the stored Layout, then return ptr to memory right after the Layout. + new_block_start.write(new_layout); new_block_start.add(1) as *mut c_void } } diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 7c0cc122..53c8ce48 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -24,7 +24,7 @@ workspace = true goblin = { version = "0.9" } rand = { version = "0.8.5" } cfg-if = { version = "1.0.0" } -libc = { version = "0.2.155" } +libc = { version = "0.2.167" } paste = "1.0" flatbuffers = "24.3.25" page_size = "0.6.0" @@ -35,16 +35,16 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" log = "0.4.20" once_cell = { version = "1.18.0" } -tracing = { version = "0.1.37", features = ["log"] } +tracing = { version = "0.1.41", features = ["log"] } tracing-log = "0.2.0" -tracing-core = "0.1.31" +tracing-core = "0.1.33" hyperlight-common = { workspace = true, default-features = true } vmm-sys-util = "0.12.1" crossbeam = "0.8.0" crossbeam-channel = "0.5.8" -thiserror = "2.0.0" +thiserror = "2.0.6" prometheus = "0.13.3" -strum = { version = "0.25", features = ["derive"] } +strum = { version = "0.26", features = ["derive"] } tempfile = { version = "3.10", optional = true } serde_yaml = "0.9" anyhow = "1.0" @@ -85,17 +85,18 @@ serde = "1.0" proptest = "1.2.0" tempfile = "3.8.0" crossbeam-queue = "0.3.8" -tracing-serde = "0.1.3" +tracing-serde = "0.2.0" serial_test = "3.1.1" hyperlight-testing = { workspace = true } env_logger = "0.11.5" tracing-forest = { version = "0.1.6", features = ["uuid", "chrono", "smallvec", "serde", "env-filter"] } -tracing = "0.1.37" -tracing-subscriber = { version = "0.3.18", features = ["std", "env-filter"] } -tracing-opentelemetry = "0.27.0" -opentelemetry = "0.26.0" -opentelemetry-otlp = { version = "0.26.0", features = ["default"] } -opentelemetry_sdk = { version = "0.26.0", features = ["rt-tokio"] } +tracing = "0.1.41" +tracing-subscriber = {version = "0.3.19", features = ["std", "env-filter"]} +tracing-opentelemetry = "0.28.0" +opentelemetry = "0.27.0" +opentelemetry-otlp = { version = "0.27.0", features = ["default"] } +opentelemetry-semantic-conventions = "0.27.0" +opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] } tokio = { version = "1.34.0", features = ["full"] } criterion = "0.5.1" tracing-chrome = "0.7.2" @@ -109,7 +110,7 @@ windows = { version = "0.58", features = [ proc-maps = "0.4.0" [build-dependencies] -anyhow = { version = "1.0.75" } +anyhow = { version = "1.0.94" } cfg_aliases = "0.2.1" built = { version = "0.7.0", features = ["chrono", "git2"] } @@ -120,9 +121,7 @@ function_call_metrics = [] executable_heap = [] # This feature enables printing of debug information to stdout in debug builds print_debug = [] -# This feature enables dunping of the VMs details to a file when an unexpected or error exit occurs in a VM in debug mode -# the name of the file is output to stdout and logged. -dump_on_crash = ["dep:tempfile"] +crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds. kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] mshv2 = ["mshv-bindings2", "mshv-ioctls2"] mshv3 = ["mshv-bindings3", "mshv-ioctls3"] diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index ab333d83..7eded862 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -99,6 +99,10 @@ fn main() -> Result<()> { // the other features they want. mshv2: { all(feature = "mshv2", not(feature="mshv3"), target_os = "linux") }, mshv3: { all(feature = "mshv3", target_os = "linux") }, + // crashdump feature is aliased with debug_assertions to make it only available in debug-builds. + crashdump: { all(feature = "crashdump", debug_assertions) }, + // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. + print_debug: { all(feature = "print_debug", debug_assertions) }, } write_built_file()?; diff --git a/src/hyperlight_host/examples/otlp_tracing/main.rs b/src/hyperlight_host/examples/otlp_tracing/main.rs index d80e0a80..5548c484 100644 --- a/src/hyperlight_host/examples/otlp_tracing/main.rs +++ b/src/hyperlight_host/examples/otlp_tracing/main.rs @@ -17,6 +17,8 @@ limitations under the License. use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType}; use rand::Rng; use tracing::{span, Level}; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; extern crate hyperlight_host; use std::error::Error; @@ -29,16 +31,19 @@ use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox; use hyperlight_host::sandbox_state::transition::Noop; use hyperlight_host::{GuestBinary, MultiUseSandbox, Result as HyperlightResult}; use hyperlight_testing::simple_guest_as_string; -use opentelemetry::global::shutdown_tracer_provider; +use opentelemetry::global::{self, shutdown_tracer_provider}; use opentelemetry::trace::TracerProvider; use opentelemetry::KeyValue; -use opentelemetry_otlp::{new_exporter, new_pipeline, WithExportConfig}; +use opentelemetry_otlp::{SpanExporter, WithExportConfig}; use opentelemetry_sdk::runtime::Tokio; use opentelemetry_sdk::{trace, Resource}; -use tracing_subscriber::layer::SubscriberExt; +use opentelemetry_semantic_conventions::attribute::{SERVICE_NAME, SERVICE_VERSION}; +use opentelemetry_semantic_conventions::SCHEMA_URL; use tracing_subscriber::EnvFilter; use uuid::Uuid; +const ENDPOINT_ADDR: &str = "http://localhost:4317"; + fn fn_writer(_msg: String) -> HyperlightResult { Ok(0) } @@ -47,30 +52,51 @@ fn fn_writer(_msg: String) -> HyperlightResult { #[tokio::main] async fn main() -> Result<(), Box> { - let tracer = new_pipeline() - .tracing() - .with_exporter( - new_exporter() - .tonic() - .with_endpoint("http://localhost:4317/v1/traces"), + init_tracing_subscriber(ENDPOINT_ADDR)?; + + Ok(run_example(true)?) +} + +fn init_tracing_subscriber(addr: &str) -> Result<(), Box> { + let exporter = SpanExporter::builder() + .with_tonic() + .with_endpoint(addr) + .build()?; + + let provider = trace::TracerProvider::builder() + .with_config( + trace::Config::default().with_resource(Resource::from_schema_url( + vec![ + KeyValue::new(SERVICE_NAME, "hyperlight_otel_example"), + KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")), + ], + SCHEMA_URL, + )), ) - .with_trace_config(trace::Config::default().with_resource(Resource::new(vec![ - KeyValue::new("service.name", "hyperlight_otel_example"), - ]))) - .install_batch(Tokio) - .unwrap() - .tracer("trace-demo"); + .with_batch_exporter(exporter, Tokio) + .build(); + + global::set_tracer_provider(provider.clone()); + let tracer = provider.tracer("trace-demo"); + + let otel_layer = OpenTelemetryLayer::new(tracer); - let otel_layer = tracing_opentelemetry::OpenTelemetryLayer::new(tracer); + // Try using the environment otherwise set default filters + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { + EnvFilter::from_default_env() + .add_directive("hyperlight_host=info".parse().unwrap()) + .add_directive("tracing=info".parse().unwrap()) + }); - tracing_subscriber::Registry::default() - .with(EnvFilter::from_default_env()) + tracing_subscriber::registry() + .with(filter) .with(otel_layer) .try_init()?; - Ok(run_example()?) + Ok(()) } -fn run_example() -> HyperlightResult<()> { + +fn run_example(wait_input: bool) -> HyperlightResult<()> { // Get the path to a simple guest binary. let hyperlight_guest_path = simple_guest_as_string().expect("Cannot find the guest binary at the expected location."); @@ -168,9 +194,12 @@ fn run_example() -> HyperlightResult<()> { join_handles.push(handle); } - println!("Press enter to exit..."); - let mut input = String::new(); - stdin().read_line(&mut input)?; + if wait_input { + println!("Press enter to exit..."); + let mut input = String::new(); + stdin().read_line(&mut input)?; + } + *should_exit.try_lock().unwrap() = true; for join_handle in join_handles { let result = join_handle.join(); @@ -180,3 +209,52 @@ fn run_example() -> HyperlightResult<()> { Ok(()) } + +#[cfg(test)] +mod test { + use hyperlight_host::{HyperlightError, Result}; + use tokio::io::AsyncReadExt; + use tokio::net::{TcpListener, TcpStream}; + + use super::*; + + const TESTER_ADDR: &str = "127.0.0.1:4317"; + + async fn handle(mut stream: TcpStream) -> Result<()> { + let mut buf = Vec::with_capacity(128); + let size = stream.read_buf(&mut buf).await?; + + if size > 0 { + Ok(()) + } else { + Err(HyperlightError::Error("Cannot read req body".to_string())) + } + } + + async fn check_otl_connection(addr: &str) -> Result<()> { + let listener = TcpListener::bind(addr).await?; + + let (stream, _) = listener.accept().await?; + + handle(stream).await + } + + #[tokio::test] + async fn test_subscriber() { + // Create task that generates spans + let task = tokio::spawn(async move { + let _ = init_tracing_subscriber(ENDPOINT_ADDR); + + // No need to wait for input, just generate some spans and exit + let _ = run_example(false); + }); + + // Create server that listens and checks to see if traces are received + let result = check_otl_connection(TESTER_ADDR).await; + + // Abort task in case it doesn't finish + task.abort(); + + assert!(result.is_ok()); + } +} diff --git a/src/hyperlight_host/src/hypervisor/crashdump.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs new file mode 100644 index 00000000..a70dc34c --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -0,0 +1,44 @@ +use std::io::Write; + +use tempfile::NamedTempFile; + +use super::Hypervisor; +use crate::{new_error, Result}; + +/// Dump registers + memory regions + raw memory to a tempfile +#[cfg(crashdump)] +pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { + let mut temp_file = NamedTempFile::with_prefix("mem")?; + let hv_details = format!("{:#x?}", hv); + + // write hypervisor details such as registers, info about mapped memory regions, etc. + temp_file.write_all(hv_details.as_bytes())?; + temp_file.write_all(b"================ MEMORY DUMP =================\n")?; + + // write the raw memory dump for each memory region + for region in hv.get_memory_regions() { + if region.host_region.start == 0 || region.host_region.is_empty() { + continue; + } + // SAFETY: we got this memory region from the hypervisor so should never be invalid + let region_slice = unsafe { + std::slice::from_raw_parts( + region.host_region.start as *const u8, + region.host_region.len(), + ) + }; + temp_file.write_all(region_slice)?; + } + temp_file.flush()?; + + // persist the tempfile to disk + let persist_path = temp_file.path().with_extension("dmp"); + temp_file + .persist(&persist_path) + .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?; + + println!("Memory dumped to file: {:?}", persist_path); + log::error!("Memory dumped to file: {:?}", persist_path); + + Ok(()) +} diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 841c8557..5f0a7935 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -53,7 +53,7 @@ use crate::hypervisor::hypervisor_handler::HypervisorHandler; use crate::hypervisor::HyperlightExit; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; -use crate::{debug, log_then_return, new_error, Result}; +use crate::{log_then_return, new_error, Result}; /// Determine whether the HyperV for Linux hypervisor API is present /// and functional. @@ -326,7 +326,7 @@ impl Hypervisor for HypervLinuxDriver { let result = match run_result { Ok(m) => match m.header.message_type { HALT_MESSAGE => { - debug!("mshv - Halt Details : {:#?}", &self); + crate::debug!("mshv - Halt Details : {:#?}", &self); HyperlightExit::Halt() } IO_PORT_INTERCEPT_MESSAGE => { @@ -335,7 +335,7 @@ impl Hypervisor for HypervLinuxDriver { let rip = io_message.header.rip; let rax = io_message.rax; let instruction_length = io_message.header.instruction_length() as u64; - debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self); + crate::debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self); HyperlightExit::IoOut( port_number, rax.to_le_bytes().to_vec(), @@ -346,24 +346,22 @@ impl Hypervisor for HypervLinuxDriver { UNMAPPED_GPA_MESSAGE => { let mimo_message = m.to_memory_info()?; let addr = mimo_message.guest_physical_address; - debug!( + crate::debug!( "mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}", - addr, &self + addr, + &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); HyperlightExit::Mmio(addr) } INVALID_GPA_ACCESS_MESSAGE => { let mimo_message = m.to_memory_info()?; let gpa = mimo_message.guest_physical_address; let access_info = MemoryRegionFlags::try_from(mimo_message)?; - debug!( + crate::debug!( "mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}", - gpa, &self + gpa, + &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); match self.get_memory_access_violation( gpa as usize, &self.mem_regions, @@ -374,9 +372,7 @@ impl Hypervisor for HypervLinuxDriver { } } other => { - debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); + crate::debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self); log_then_return!("unknown Hyper-V run message type {:?}", other); } }, @@ -385,9 +381,7 @@ impl Hypervisor for HypervLinuxDriver { libc::EINTR => HyperlightExit::Cancelled(), libc::EAGAIN => HyperlightExit::Retry(), _ => { - debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); + crate::debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self); log_then_return!("Error running VCPU {:?}", e); } }, @@ -399,6 +393,11 @@ impl Hypervisor for HypervLinuxDriver { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor { self as &mut dyn Hypervisor } + + #[cfg(crashdump)] + fn get_memory_regions(&self) -> &[MemoryRegion] { + &self.mem_regions + } } impl Drop for HypervLinuxDriver { diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 80d16736..5c5bbc60 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -19,7 +19,6 @@ use std::fmt; use std::fmt::{Debug, Formatter}; use std::string::String; -use cfg_if::cfg_if; use hyperlight_common::mem::PAGE_SIZE_USIZE; use tracing::{instrument, Span}; use windows::Win32::Foundation::HANDLE; @@ -463,19 +462,10 @@ impl Hypervisor for HypervWindowsDriver { // see https://learn.microsoft.com/en-us/virtualization/api/hypervisor-platform/funcs/whvexitcontextdatatypes) let instruction_length = exit_context.VpContext._bitfield & 0xF; unsafe { - cfg_if! { - if #[cfg(all(feature = "print_debug", debug_assertions))] { - println!( - "HyperV IO Details :\n Port: {:#x} \n {:#?}", - exit_context.Anonymous.IoPortAccess.PortNumber, &self - ); - } else { - debug!( - "HyperV IO Details :\n Port: {:#x} \n {:#?}", - exit_context.Anonymous.IoPortAccess.PortNumber, &self - ); - } - } + debug!( + "HyperV IO Details :\n Port: {:#x} \n {:#?}", + exit_context.Anonymous.IoPortAccess.PortNumber, &self + ); HyperlightExit::IoOut( exit_context.Anonymous.IoPortAccess.PortNumber, exit_context @@ -508,18 +498,6 @@ impl Hypervisor for HypervWindowsDriver { "HyperV Memory Access Details :\n GPA: {:#?}\n Access Info :{:#?}\n {:#?} ", gpa, access_info, &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - { - if let Err(e) = unsafe { - self.write_dump_file( - self.mem_regions.clone(), - self.source_address.add(PAGE_SIZE_USIZE) as *const u8, - self.size, - ) - } { - println!("Error dumping memory: {}", e); - } - } match self.get_memory_access_violation(gpa as usize, &self.mem_regions, access_info) { @@ -539,18 +517,6 @@ impl Hypervisor for HypervWindowsDriver { "HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}", exit_context.ExitReason, &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - { - if let Err(e) = unsafe { - self.write_dump_file( - self.mem_regions.clone(), - self.source_address.add(PAGE_SIZE_USIZE) as *const u8, - self.size, - ) - } { - println!("Error dumping memory: {}", e); - } - } match self.get_exit_details(exit_context.ExitReason) { Ok(error) => HyperlightExit::Unknown(error), Err(e) => HyperlightExit::Unknown(format!("Error getting exit details: {}", e)), @@ -569,6 +535,11 @@ impl Hypervisor for HypervWindowsDriver { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor { self as &mut dyn Hypervisor } + + #[cfg(crashdump)] + fn get_memory_regions(&self) -> &[MemoryRegion] { + &self.mem_regions + } } #[cfg(test)] diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs index bfb55fa2..8e1645cb 100644 --- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs +++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs @@ -16,6 +16,7 @@ limitations under the License. #[cfg(target_os = "windows")] use core::ffi::c_void; +use std::ops::DerefMut; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; @@ -228,13 +229,9 @@ impl HypervisorHandler { #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] pub(crate) fn start_hypervisor_handler( &mut self, - mut sandbox_memory_manager: SandboxMemoryManager, + sandbox_memory_manager: SandboxMemoryManager, ) -> Result<()> { let configuration = self.configuration.clone(); - let mut hv = set_up_hypervisor_partition( - &mut sandbox_memory_manager, - configuration.outb_handler.clone(), - )?; #[cfg(target_os = "windows")] let in_process = sandbox_memory_manager.is_in_process(); @@ -267,18 +264,8 @@ impl HypervisorHandler { #[cfg(target_os = "linux")] self.execution_variables.run_cancelled.store(false); - #[cfg(target_os = "windows")] - if !in_process { - self.execution_variables - .set_partition_handle(hv.get_partition_handle())?; - } - let to_handler_rx = self.communication_channels.to_handler_rx.clone(); - #[cfg(target_os = "windows")] - let execution_variables = self.execution_variables.clone(); - #[cfg(target_os = "linux")] let mut execution_variables = self.execution_variables.clone(); - // ^^^ this needs to be mut on linux to set_thread_id let from_handler_tx = self.communication_channels.from_handler_tx.clone(); let hv_handler_clone = self.clone(); @@ -295,9 +282,24 @@ impl HypervisorHandler { thread::Builder::new() .name("Hypervisor Handler".to_string()) .spawn(move || -> Result<()> { + let mut hv: Option> = None; for action in to_handler_rx { match action { HypervisorHandlerAction::Initialise => { + { + hv = Some(set_up_hypervisor_partition( + execution_variables.shm.try_lock().unwrap().deref_mut().as_mut().unwrap(), + configuration.outb_handler.clone(), + )?); + } + let hv = hv.as_mut().unwrap(); + + #[cfg(target_os = "windows")] + if !in_process { + execution_variables + .set_partition_handle(hv.get_partition_handle())?; + } + #[cfg(target_os = "linux")] { // We cannot use the Killable trait, so we get the `pthread_t` via a libc @@ -328,6 +330,7 @@ impl HypervisorHandler { .shared_mem .lock .try_read(); + let res = hv.initialise( configuration.peb_addr.clone(), configuration.seed, @@ -362,6 +365,8 @@ impl HypervisorHandler { } } HypervisorHandlerAction::DispatchCallFromHost(function_name) => { + let hv = hv.as_mut().unwrap(); + // Lock to indicate an action is being performed in the hypervisor execution_variables.running.store(true, Ordering::SeqCst); @@ -944,6 +949,13 @@ mod tests { } } + #[test] + fn create_10_sandboxes() { + for _ in 0..10 { + create_multi_use_sandbox(); + } + } + #[test] fn hello_world() -> Result<()> { let mut sandbox = create_multi_use_sandbox(); diff --git a/src/hyperlight_host/src/hypervisor/inprocess.rs b/src/hyperlight_host/src/hypervisor/inprocess.rs index a8ff862e..f7076a5b 100644 --- a/src/hyperlight_host/src/hypervisor/inprocess.rs +++ b/src/hyperlight_host/src/hypervisor/inprocess.rs @@ -18,6 +18,8 @@ use std::fmt::Debug; use std::os::raw::c_void; use super::{HyperlightExit, Hypervisor}; +#[cfg(crashdump)] +use crate::mem::memory_region::MemoryRegion; use crate::sandbox::leaked_outb::LeakedOutBWrapper; use crate::Result; @@ -123,4 +125,9 @@ impl<'a> Hypervisor for InprocessDriver<'a> { fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE { unimplemented!("get_partition_handle should not be needed since we are in in-process mode") } + + #[cfg(crashdump)] + fn get_memory_regions(&self) -> &[MemoryRegion] { + unimplemented!("get_memory_regions is not supported since we are in in-process mode") + } } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 5caf8241..991e1e24 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -17,7 +17,6 @@ limitations under the License. use std::convert::TryFrom; use std::fmt::Debug; -use cfg_if::cfg_if; use kvm_bindings::{kvm_fpu, kvm_regs, kvm_userspace_memory_region, KVM_MEM_READONLY}; use kvm_ioctls::Cap::UserMemory; use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; @@ -32,7 +31,7 @@ use super::{ use crate::hypervisor::hypervisor_handler::HypervisorHandler; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; -use crate::{debug, log_then_return, new_error, Result}; +use crate::{log_then_return, new_error, Result}; /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise #[instrument(skip_all, parent = Span::current(), level = "Trace")] @@ -272,22 +271,20 @@ impl Hypervisor for KVMDriver { let exit_reason = self.vcpu_fd.run(); let result = match exit_reason { Ok(VcpuExit::Hlt) => { - debug!("KVM - Halt Details : {:#?}", &self); + crate::debug!("KVM - Halt Details : {:#?}", &self); HyperlightExit::Halt() } Ok(VcpuExit::IoOut(port, data)) => { - // because vcpufd.run() mutably borrows self we cannot pass self to debug! macro here - debug!("KVM IO Details : \nPort : {}\nData : {:?}", port, data); + // because vcpufd.run() mutably borrows self we cannot pass self to crate::debug! macro here + crate::debug!("KVM IO Details : \nPort : {}\nData : {:?}", port, data); // KVM does not need to set RIP or instruction length so these are set to 0 HyperlightExit::IoOut(port, data.to_vec(), 0, 0) } Ok(VcpuExit::MmioRead(addr, _)) => { - debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); - let gpa = addr as usize; + crate::debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self); + match self.get_memory_access_violation( - gpa, + addr as usize, &self.mem_regions, MemoryRegionFlags::READ, ) { @@ -296,12 +293,10 @@ impl Hypervisor for KVMDriver { } } Ok(VcpuExit::MmioWrite(addr, _)) => { - debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); - let gpa = addr as usize; + crate::debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self); + match self.get_memory_access_violation( - gpa, + addr as usize, &self.mem_regions, MemoryRegionFlags::WRITE, ) { @@ -314,26 +309,13 @@ impl Hypervisor for KVMDriver { libc::EINTR => HyperlightExit::Cancelled(), libc::EAGAIN => HyperlightExit::Retry(), _ => { - debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); + crate::debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self); log_then_return!("Error running VCPU {:?}", e); } }, Ok(other) => { - cfg_if! { - if #[cfg(all(feature = "print_debug", debug_assertions))] { - let _ = other; - debug!("KVM Other Exit: \n {:#?}", &self); - HyperlightExit::Unknown("Unexpected KVM Exit".to_string()) - } else if #[cfg(all(feature = "dump_on_crash", debug_assertions))] { - self.dump_on_crash(self.mem_regions.clone()); - HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other)) - } else{ - debug!("KVM Other Exit {:?}", other); - HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other)) - } - } + crate::debug!("KVM Other Exit {:?}", other); + HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other)) } }; Ok(result) @@ -343,6 +325,11 @@ impl Hypervisor for KVMDriver { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor { self as &mut dyn Hypervisor } + + #[cfg(crashdump)] + fn get_memory_regions(&self) -> &[MemoryRegion] { + &self.mem_regions + } } #[cfg(test)] diff --git a/src/hyperlight_host/src/hypervisor/metrics.rs b/src/hyperlight_host/src/hypervisor/metrics.rs index 499f3516..a02d19ed 100644 --- a/src/hyperlight_host/src/hypervisor/metrics.rs +++ b/src/hyperlight_host/src/hypervisor/metrics.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; use std::sync::Once; use once_cell::sync::OnceCell; -use strum::{EnumIter, EnumVariantNames, IntoStaticStr}; +use strum::{EnumIter, IntoStaticStr, VariantNames}; use tracing::{instrument, Span}; use crate::metrics::{ @@ -55,7 +55,7 @@ static HYPERVISOR_METRIC_DEFINITIONS: &[HyperlightMetricDefinition] = /// The enum is required to derive from EnumIter, EnumVariantNames, IntoStaticStr /// and strum(serialize_all = "snake_case") performs the name conversion from CamelCase to snake_case /// when the enum variant is serialized to a string -#[derive(Debug, EnumIter, EnumVariantNames, IntoStaticStr)] +#[derive(Debug, EnumIter, VariantNames, IntoStaticStr)] #[strum(serialize_all = "snake_case")] pub(super) enum HypervisorMetric { NumberOfCancelledGuestExecutions, diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 4f13ff2b..c34d7eb1 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -55,6 +55,9 @@ pub(crate) mod windows_hypervisor_platform; #[cfg(target_os = "windows")] pub(crate) mod wrappers; +#[cfg(crashdump)] +pub(crate) mod crashdump; + use std::fmt::Debug; use std::sync::{Arc, Mutex}; @@ -184,70 +187,8 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { #[cfg(target_os = "windows")] fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE; - /// Dump memory to a file on crash - #[cfg(all(debug_assertions, feature = "dump_on_crash", target_os = "linux"))] - fn dump_on_crash(&self, mem_regions: Vec) { - let memory_size = mem_regions - .iter() - .map(|region| region.guest_region.end - region.guest_region.start) - .sum(); - if let Err(e) = unsafe { - self.write_dump_file( - mem_regions.clone(), - mem_regions[0].host_region.start as *const u8, - memory_size, - ) - } { - println!("Error dumping memory: {}", e); - } - } - - /// A function that takes an address and a size and writes the memory at that address to a file in the temp/tmp directory - /// # Safety - /// This function is unsafe because it is writing memory to a file, make sure that the address is valid and that the size is correct - /// This function is only available when the `dump_on_crash` feature is enabled and running in debug mode - #[cfg(all(feature = "dump_on_crash", debug_assertions))] - unsafe fn write_dump_file( - &self, - regions: Vec, - address: *const u8, - size: usize, - ) -> Result<()> { - use std::io::Write; - - use tempfile::NamedTempFile; - - if address.is_null() || size == 0 { - return Err(new_error!("Invalid address or size")); - } - - let hv_details = format!("{:#?}", self); - let regions_details = format!("{:#?}", regions); - - // Create a temporary file - let mut file = NamedTempFile::with_prefix("mem")?; - let temp_path = file.path().to_path_buf(); - - file.write_all(hv_details.as_bytes())?; - file.write_all(b"\n")?; - file.write_all(regions_details.as_bytes())?; - file.write_all(b"\n")?; - - // SAFETY: Ensure the address and size are valid and accessible - unsafe { - let slice = std::slice::from_raw_parts(address, size); - file.write_all(slice)?; - file.flush()?; - } - let persist_path = temp_path.with_extension("dmp"); - file.persist(&persist_path) - .map_err(|e| new_error!("Failed to persist file: {:?}", e))?; - - print!("Memory dumped to file: {:?}", persist_path); - log::error!("Memory dumped to file: {:?}", persist_path); - - Ok(()) - } + #[cfg(crashdump)] + fn get_memory_regions(&self) -> &[MemoryRegion]; } /// A virtual CPU that can be run until an exit occurs @@ -263,22 +204,29 @@ impl VirtualCPU { mem_access_fn: Arc>, ) -> Result<()> { loop { - match hv.run()? { - HyperlightExit::Halt() => { + match hv.run() { + Ok(HyperlightExit::Halt()) => { break; } - HyperlightExit::IoOut(port, data, rip, instruction_length) => { + Ok(HyperlightExit::IoOut(port, data, rip, instruction_length)) => { hv.handle_io(port, data, rip, instruction_length, outb_handle_fn.clone())? } - HyperlightExit::Mmio(addr) => { + Ok(HyperlightExit::Mmio(addr)) => { + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; + mem_access_fn .clone() .try_lock() .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? .call()?; + log_then_return!("MMIO access address {:#x}", addr); } - HyperlightExit::AccessViolation(addr, tried, region_permission) => { + Ok(HyperlightExit::AccessViolation(addr, tried, region_permission)) => { + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; + if region_permission.intersects(MemoryRegionFlags::STACK_GUARD) { return Err(HyperlightError::StackOverflow()); } @@ -288,7 +236,7 @@ impl VirtualCPU { region_permission )); } - HyperlightExit::Cancelled() => { + Ok(HyperlightExit::Cancelled()) => { // Shutdown is returned when the host has cancelled execution // After termination, the main thread will re-initialize the VM if let Some(hvh) = hv_handler { @@ -301,10 +249,19 @@ impl VirtualCPU { int_counter_inc!(&NumberOfCancelledGuestExecutions); log_then_return!(ExecutionCanceledByHost()); } - HyperlightExit::Unknown(reason) => { + Ok(HyperlightExit::Unknown(reason)) => { + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; + log_then_return!("Unexpected VM Exit {:?}", reason); } - HyperlightExit::Retry() => continue, + Ok(HyperlightExit::Retry()) => continue, + Err(e) => { + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; + + return Err(e); + } } } diff --git a/src/hyperlight_host/src/lib.rs b/src/hyperlight_host/src/lib.rs index 7170552d..2dac703f 100644 --- a/src/hyperlight_host/src/lib.rs +++ b/src/hyperlight_host/src/lib.rs @@ -143,19 +143,14 @@ macro_rules! log_then_return { }; } +// same as log::debug!, but will additionally print to stdout if the print_debug feature is enabled #[macro_export] macro_rules! debug { - ($($arg:tt)+) => { - // If the print_debug feature is enabled, print the debug message to the console - #[cfg(all(feature = "print_debug", debug_assertions))] - { - (println!($($arg)+)) - } - - // Then log/trace the debug message - (log::debug!($($arg)+)) + #[cfg(print_debug)] + println!($($arg)+); + log::debug!($($arg)+); } } diff --git a/src/hyperlight_host/src/mem/pe/pe_info.rs b/src/hyperlight_host/src/mem/pe/pe_info.rs index eb4b792c..97a99bfa 100644 --- a/src/hyperlight_host/src/mem/pe/pe_info.rs +++ b/src/hyperlight_host/src/mem/pe/pe_info.rs @@ -23,7 +23,7 @@ use goblin::pe::PE; use tracing::{instrument, Span}; use crate::mem::pe::base_relocations; -use crate::{debug, log_then_return, Result}; +use crate::{log_then_return, Result}; const IMAGE_REL_BASED_DIR64: u8 = 10; const IMAGE_REL_BASED_ABSOLUTE: u8 = 0; @@ -106,9 +106,11 @@ impl PEInfo { let name = section.name().unwrap_or("Unknown"); let virtual_size = section.virtual_size; let raw_size = section.size_of_raw_data; - debug!( + crate::debug!( "Section: {}, Virtual Size: {}, On-Disk Size: {}", - name, virtual_size, raw_size + name, + virtual_size, + raw_size ); if virtual_size > raw_size { @@ -122,16 +124,16 @@ impl PEInfo { data_section_raw_pointer = section.pointer_to_raw_data; data_section_additional_bytes = virtual_size - raw_size; - debug!( + crate::debug!( "Resizing the data section - Additional bytes required: {}", data_section_additional_bytes ); - debug!( + crate::debug!( "Resizing the data section - Existing PE File Size: {} New PE File Size: {}", pe_bytes.len(), pe_bytes.len() + data_section_additional_bytes as usize, ); - debug!( + crate::debug!( "Resizing the data section - Data Section Raw Pointer: {}", data_section_raw_pointer ); @@ -140,19 +142,19 @@ impl PEInfo { end_of_data_index = (section.pointer_to_raw_data + section.size_of_raw_data) as usize; - debug!("End of data index: {}", end_of_data_index); + crate::debug!("End of data index: {}", end_of_data_index); // the remainder of the data is the rest of the file after the .data section if any let next_section = pe.sections.get(i + 1); if let Some(next_section) = next_section { - debug!( + crate::debug!( "Start of section after data index: {}", next_section.pointer_to_raw_data ); } else { - debug!("No more sections after the .data section"); + crate::debug!("No more sections after the .data section"); } } else { log_then_return!( diff --git a/src/hyperlight_host/src/sandbox/metrics.rs b/src/hyperlight_host/src/sandbox/metrics.rs index 840b003a..05ef9bae 100644 --- a/src/hyperlight_host/src/sandbox/metrics.rs +++ b/src/hyperlight_host/src/sandbox/metrics.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; use std::sync::Once; use once_cell::sync::OnceCell; -use strum::{EnumIter, EnumVariantNames, IntoStaticStr}; +use strum::{EnumIter, IntoStaticStr, VariantNames}; use tracing::{instrument, Span}; use crate::metrics::{ @@ -87,7 +87,7 @@ static SANDBOX_METRIC_DEFINITIONS: &[HyperlightMetricDefinition] = &[ /// The enum is required to derive from EnumIter, EnumVariantNames, IntoStaticStr /// and strum(serialize_all = "snake_case") performs the name conversion from CamelCase to snake_case /// when the enum variant is serialized to a string -#[derive(Debug, EnumIter, EnumVariantNames, IntoStaticStr)] +#[derive(Debug, EnumIter, VariantNames, IntoStaticStr)] #[strum(serialize_all = "snake_case")] pub(crate) enum SandboxMetric { GuestErrorCount, diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index eaa573de..6ae2e1a0 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -35,7 +35,7 @@ use crate::sandbox::SandboxConfiguration; use crate::sandbox_state::sandbox::EvolvableSandbox; use crate::sandbox_state::transition::Noop; use crate::{ - debug, log_build_details, log_then_return, new_error, MultiUseSandbox, Result, SingleUseSandbox, + log_build_details, log_then_return, new_error, MultiUseSandbox, Result, SingleUseSandbox, }; /// A preliminary `Sandbox`, not yet ready to execute guest code. @@ -258,7 +258,7 @@ impl UninitializedSandbox { } } - debug!("Sandbox created: {:#?}", sandbox); + crate::debug!("Sandbox created: {:#?}", sandbox); Ok(sandbox) } diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 85e01bbd..a140ef90 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -186,14 +186,14 @@ fn guest_panic() { } #[test] -fn guest_hlmalloc() { +fn guest_malloc() { // this test is rust-only let sbox1: SingleUseSandbox = new_uninit_rust().unwrap().evolve(Noop::default()).unwrap(); let size_to_allocate = 2000; let res = sbox1 .call_guest_function_by_name( - "TestHlMalloc", // uses hlmalloc + "TestMalloc", ReturnType::Int, Some(vec![ParameterValue::Int(size_to_allocate)]), ) @@ -202,7 +202,7 @@ fn guest_hlmalloc() { } #[test] -fn guest_malloc() { +fn guest_allocate_vec() { let sbox1: SingleUseSandbox = new_uninit().unwrap().evolve(Noop::default()).unwrap(); let size_to_allocate = 2000; @@ -220,14 +220,14 @@ fn guest_malloc() { // checks that malloc failures are captured correctly #[test] -fn guest_hlmalloc_abort() { +fn guest_malloc_abort() { let sbox1: SingleUseSandbox = new_uninit_rust().unwrap().evolve(Noop::default()).unwrap(); let size = 20000000; // some big number that should fail when allocated let res = sbox1 .call_guest_function_by_name( - "TestHlMalloc", + "TestMalloc", ReturnType::Int, Some(vec![ParameterValue::Int(size)]), ) diff --git a/src/hyperlight_testing/Cargo.toml b/src/hyperlight_testing/Cargo.toml index 78b883f5..37fcaf69 100644 --- a/src/hyperlight_testing/Cargo.toml +++ b/src/hyperlight_testing/Cargo.toml @@ -3,13 +3,13 @@ name = "hyperlight-testing" edition = "2021" [dependencies] -anyhow = "1.0.72" +anyhow = "1.0.94" log = "0.4" once_cell = "1.19" -tracing = { version = "0.1.37", features = ["log"] } +tracing = { version = "0.1.41", features = ["log"] } tracing-log = "0.2.0" -tracing-core = "0.1.17" -tracing-serde = "0.1" +tracing-core = "0.1.33" +tracing-serde = "0.2" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/tests/c_guests/c_simpleguest/main.c b/src/tests/c_guests/c_simpleguest/main.c index 91a5eb5c..d1e688ee 100644 --- a/src/tests/c_guests/c_simpleguest/main.c +++ b/src/tests/c_guests/c_simpleguest/main.c @@ -19,7 +19,7 @@ double echo_double(double d) { return d; } hl_Vec *set_byte_array_to_zero(const hl_FunctionCall* params) { hl_Vec input = params->parameters[0].value.VecBytes; - uint8_t *x = hlmalloc(input.len); + uint8_t *x = malloc(input.len); for (uintptr_t i = 0; i < input.len; i++) { x[i] = 0; } @@ -90,7 +90,7 @@ int small_var(void) { } int call_malloc(int32_t size) { - void *heap_memory = hlmalloc(size); + void *heap_memory = malloc(size); if (NULL == heap_memory) { hl_set_error(hl_ErrorCode_GuestError, "Malloc Failed"); } @@ -99,12 +99,12 @@ int call_malloc(int32_t size) { } int malloc_and_free(int32_t size) { - void *heap_memory = hlmalloc(size); + void *heap_memory = malloc(size); if (NULL == heap_memory) { hl_set_error(hl_ErrorCode_GuestError, "Malloc Failed"); } - hlfree(heap_memory); + free(heap_memory); return size; } diff --git a/src/tests/rust_guests/callbackguest/Cargo.lock b/src/tests/rust_guests/callbackguest/Cargo.lock index 97b27b27..ff2522cc 100644 --- a/src/tests/rust_guests/callbackguest/Cargo.lock +++ b/src/tests/rust_guests/callbackguest/Cargo.lock @@ -62,12 +62,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -82,7 +76,6 @@ dependencies = [ "flatbuffers", "log", "strum", - "strum_macros 0.26.4", ] [[package]] @@ -227,24 +220,11 @@ dependencies = [ [[package]] name = "strum" -version = "0.25.0" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn", + "strum_macros", ] [[package]] @@ -253,7 +233,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", diff --git a/src/tests/rust_guests/simpleguest/Cargo.lock b/src/tests/rust_guests/simpleguest/Cargo.lock index 6ce9f3e8..f8df7b2a 100644 --- a/src/tests/rust_guests/simpleguest/Cargo.lock +++ b/src/tests/rust_guests/simpleguest/Cargo.lock @@ -54,12 +54,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -74,7 +68,6 @@ dependencies = [ "flatbuffers", "log", "strum", - "strum_macros 0.26.4", ] [[package]] @@ -228,24 +221,11 @@ dependencies = [ [[package]] name = "strum" -version = "0.25.0" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn", + "strum_macros", ] [[package]] @@ -254,7 +234,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index c993bc80..685587f7 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -54,7 +54,7 @@ use hyperlight_guest::guest_function_register::register_function; use hyperlight_guest::host_function_call::{ call_host_function, get_host_value_return_as_int, get_host_value_return_as_ulong, }; -use hyperlight_guest::memory::hlmalloc; +use hyperlight_guest::memory::malloc; use hyperlight_guest::{logging, MIN_STACK_ADDRESS}; use log::{error, LevelFilter}; @@ -623,11 +623,9 @@ fn execute_on_heap(_function_call: &FunctionCall) -> Result> { Ok(get_flatbuffer_result_from_string("fail")) } -#[no_mangle] -#[allow(improper_ctypes_definitions)] -pub extern "C" fn test_rust_malloc(function_call: &FunctionCall) -> Result> { +fn test_rust_malloc(function_call: &FunctionCall) -> Result> { if let ParameterValue::Int(code) = function_call.parameters.clone().unwrap()[0].clone() { - let ptr = hlmalloc(code as usize); + let ptr = unsafe { malloc(code as usize) }; Ok(get_flatbuffer_result_from_int(ptr as i32)) } else { Err(HyperlightGuestError::new( @@ -1012,7 +1010,7 @@ pub extern "C" fn hyperlight_main() { register_function(guest_panic_def); let rust_malloc_def = GuestFunctionDefinition::new( - "TestHlMalloc".to_string(), + "TestMalloc".to_string(), Vec::from(&[ParameterType::Int]), ReturnType::Int, test_rust_malloc as i64,