From 81935d2ce82af5724e38522aa3e18954d471ec88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 01:31:58 +0000 Subject: [PATCH 01/21] Bump libc from 0.2.164 to 0.2.165 Bumps [libc](https://github.com/rust-lang/libc) from 0.2.164 to 0.2.165. - [Release notes](https://github.com/rust-lang/libc/releases) - [Changelog](https://github.com/rust-lang/libc/blob/0.2.165/CHANGELOG.md) - [Commits](https://github.com/rust-lang/libc/compare/0.2.164...0.2.165) --- updated-dependencies: - dependency-name: libc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- src/hyperlight_host/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 942f4ec6..6f712f35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1446,9 +1446,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.164" +version = "0.2.165" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "fcb4d3d38eab6c5239a362fa8bae48c03baf980a6e7079f063942d563ef3533e" [[package]] name = "libfuzzer-sys" diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index b0218786..be78ea7b 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -24,7 +24,7 @@ workspace = true goblin = { version = "0.9" } rand = { version = "0.8.5" } cfg-if = { version = "1.0.0" } -libc = { version = "0.2.155" } +libc = { version = "0.2.165" } paste = "1.0" flatbuffers = "24.3.25" page_size = "0.6.0" From 9e366137de6fca44b6c5c97a2d2868db29d4ac67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:01:11 +0000 Subject: [PATCH 02/21] Bump tracing-core from 0.1.32 to 0.1.33 Bumps [tracing-core](https://github.com/tokio-rs/tracing) from 0.1.32 to 0.1.33. - [Release notes](https://github.com/tokio-rs/tracing/releases) - [Commits](https://github.com/tokio-rs/tracing/compare/tracing-core-0.1.32...tracing-core-0.1.33) --- updated-dependencies: - dependency-name: tracing-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- src/hyperlight_host/Cargo.toml | 2 +- src/hyperlight_testing/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f712f35..5b9da8f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2846,9 +2846,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index be78ea7b..f876fd29 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -37,7 +37,7 @@ log = "0.4.20" once_cell = { version = "1.18.0" } tracing = { version = "0.1.37", features = ["log"] } tracing-log = "0.2.0" -tracing-core = "0.1.31" +tracing-core = "0.1.33" hyperlight-common = { workspace = true, default-features = true } vmm-sys-util = "0.12.1" crossbeam = "0.8.0" diff --git a/src/hyperlight_testing/Cargo.toml b/src/hyperlight_testing/Cargo.toml index 78b883f5..46b46822 100644 --- a/src/hyperlight_testing/Cargo.toml +++ b/src/hyperlight_testing/Cargo.toml @@ -8,7 +8,7 @@ log = "0.4" once_cell = "1.19" tracing = { version = "0.1.37", features = ["log"] } tracing-log = "0.2.0" -tracing-core = "0.1.17" +tracing-core = "0.1.33" tracing-serde = "0.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" From e39936a4154893422432d3c892d8548af8f3e235 Mon Sep 17 00:00:00 2001 From: danbugs Date: Wed, 6 Nov 2024 23:33:57 +0000 Subject: [PATCH 03/21] [performance] Move set_up_hypervisor_partition to inside the hv handler thread As per KVM docs, vCPU ioctls should be issued from the same thread that was used to create the vCPU. In accordance to that, this PR moves set_up_hypervisor_partition to inside the hv handler thread, which manages vCPU interactions like init and dispatching guest funciton calls. Signed-off-by: danbugs --- .../src/hypervisor/hypervisor_handler.rs | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs index bfb55fa2..8e1645cb 100644 --- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs +++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs @@ -16,6 +16,7 @@ limitations under the License. #[cfg(target_os = "windows")] use core::ffi::c_void; +use std::ops::DerefMut; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; @@ -228,13 +229,9 @@ impl HypervisorHandler { #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] pub(crate) fn start_hypervisor_handler( &mut self, - mut sandbox_memory_manager: SandboxMemoryManager, + sandbox_memory_manager: SandboxMemoryManager, ) -> Result<()> { let configuration = self.configuration.clone(); - let mut hv = set_up_hypervisor_partition( - &mut sandbox_memory_manager, - configuration.outb_handler.clone(), - )?; #[cfg(target_os = "windows")] let in_process = sandbox_memory_manager.is_in_process(); @@ -267,18 +264,8 @@ impl HypervisorHandler { #[cfg(target_os = "linux")] self.execution_variables.run_cancelled.store(false); - #[cfg(target_os = "windows")] - if !in_process { - self.execution_variables - .set_partition_handle(hv.get_partition_handle())?; - } - let to_handler_rx = self.communication_channels.to_handler_rx.clone(); - #[cfg(target_os = "windows")] - let execution_variables = self.execution_variables.clone(); - #[cfg(target_os = "linux")] let mut execution_variables = self.execution_variables.clone(); - // ^^^ this needs to be mut on linux to set_thread_id let from_handler_tx = self.communication_channels.from_handler_tx.clone(); let hv_handler_clone = self.clone(); @@ -295,9 +282,24 @@ impl HypervisorHandler { thread::Builder::new() .name("Hypervisor Handler".to_string()) .spawn(move || -> Result<()> { + let mut hv: Option> = None; for action in to_handler_rx { match action { HypervisorHandlerAction::Initialise => { + { + hv = Some(set_up_hypervisor_partition( + execution_variables.shm.try_lock().unwrap().deref_mut().as_mut().unwrap(), + configuration.outb_handler.clone(), + )?); + } + let hv = hv.as_mut().unwrap(); + + #[cfg(target_os = "windows")] + if !in_process { + execution_variables + .set_partition_handle(hv.get_partition_handle())?; + } + #[cfg(target_os = "linux")] { // We cannot use the Killable trait, so we get the `pthread_t` via a libc @@ -328,6 +330,7 @@ impl HypervisorHandler { .shared_mem .lock .try_read(); + let res = hv.initialise( configuration.peb_addr.clone(), configuration.seed, @@ -362,6 +365,8 @@ impl HypervisorHandler { } } HypervisorHandlerAction::DispatchCallFromHost(function_name) => { + let hv = hv.as_mut().unwrap(); + // Lock to indicate an action is being performed in the hypervisor execution_variables.running.store(true, Ordering::SeqCst); @@ -944,6 +949,13 @@ mod tests { } } + #[test] + fn create_10_sandboxes() { + for _ in 0..10 { + create_multi_use_sandbox(); + } + } + #[test] fn hello_world() -> Result<()> { let mut sandbox = create_multi_use_sandbox(); From aa20af9ae690119d966f404a0d238ce210295f10 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg Date: Mon, 25 Nov 2024 20:01:07 +0000 Subject: [PATCH 04/21] Move calling 'dump_on_crash' from each driver, to common VCpu, and move logic to its own module Signed-off-by: Ludvig Liljenberg --- .../src/hypervisor/dump_on_crash.rs | 44 ++++++++ .../src/hypervisor/hyperv_linux.rs | 13 +-- .../src/hypervisor/hyperv_windows.rs | 47 ++------ .../src/hypervisor/inprocess.rs | 7 ++ src/hyperlight_host/src/hypervisor/kvm.rs | 35 ++---- src/hyperlight_host/src/hypervisor/mod.rs | 101 +++++------------- 6 files changed, 105 insertions(+), 142 deletions(-) create mode 100644 src/hyperlight_host/src/hypervisor/dump_on_crash.rs diff --git a/src/hyperlight_host/src/hypervisor/dump_on_crash.rs b/src/hyperlight_host/src/hypervisor/dump_on_crash.rs new file mode 100644 index 00000000..bdc2ab04 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/dump_on_crash.rs @@ -0,0 +1,44 @@ +use std::io::Write; + +use tempfile::NamedTempFile; + +use super::Hypervisor; +use crate::{new_error, Result}; + +/// Dump registers + memory regions + raw memory to a tempfile +#[cfg(feature = "dump_on_crash")] +pub(crate) fn dump_on_crash_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { + let mut temp_file = NamedTempFile::with_prefix("mem")?; + let hv_details = format!("{:#x?}", hv); + + // write hypervisor details such as registers, info about mapped memory regions, etc. + temp_file.write_all(hv_details.as_bytes())?; + temp_file.write_all(b"================ MEMORY DUMP =================\n")?; + + // write the raw memory dump for each memory region + for region in hv.get_memory_regions() { + if region.host_region.start == 0 || region.host_region.is_empty() { + continue; + } + // SAFETY: we got this memory region from the hypervisor so should never be invalid + let region_slice = unsafe { + std::slice::from_raw_parts( + region.host_region.start as *const u8, + region.host_region.len(), + ) + }; + temp_file.write_all(region_slice)?; + } + temp_file.flush()?; + + // persist the tempfile to disk + let persist_path = temp_file.path().with_extension("dmp"); + temp_file + .persist(&persist_path) + .map_err(|e| new_error!("Failed to persist dump_on_crash file: {:?}", e))?; + + println!("Memory dumped to file: {:?}", persist_path); + log::error!("Memory dumped to file: {:?}", persist_path); + + Ok(()) +} diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 4bbaef7b..1a583fe0 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -311,8 +311,6 @@ impl Hypervisor for HypervLinuxDriver { "mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}", addr, &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); HyperlightExit::Mmio(addr) } INVALID_GPA_ACCESS_MESSAGE => { @@ -323,8 +321,6 @@ impl Hypervisor for HypervLinuxDriver { "mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}", gpa, &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); match self.get_memory_access_violation( gpa as usize, &self.mem_regions, @@ -336,8 +332,6 @@ impl Hypervisor for HypervLinuxDriver { } other => { debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); log_then_return!("unknown Hyper-V run message type {:?}", other); } }, @@ -347,8 +341,6 @@ impl Hypervisor for HypervLinuxDriver { libc::EAGAIN => HyperlightExit::Retry(), _ => { debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); log_then_return!("Error running VCPU {:?}", e); } }, @@ -360,6 +352,11 @@ impl Hypervisor for HypervLinuxDriver { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor { self as &mut dyn Hypervisor } + + #[cfg(feature = "dump_on_crash")] + fn get_memory_regions(&self) -> &[MemoryRegion] { + &self.mem_regions + } } impl Drop for HypervLinuxDriver { diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 80d16736..4984f46e 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -19,7 +19,6 @@ use std::fmt; use std::fmt::{Debug, Formatter}; use std::string::String; -use cfg_if::cfg_if; use hyperlight_common::mem::PAGE_SIZE_USIZE; use tracing::{instrument, Span}; use windows::Win32::Foundation::HANDLE; @@ -463,19 +462,10 @@ impl Hypervisor for HypervWindowsDriver { // see https://learn.microsoft.com/en-us/virtualization/api/hypervisor-platform/funcs/whvexitcontextdatatypes) let instruction_length = exit_context.VpContext._bitfield & 0xF; unsafe { - cfg_if! { - if #[cfg(all(feature = "print_debug", debug_assertions))] { - println!( - "HyperV IO Details :\n Port: {:#x} \n {:#?}", - exit_context.Anonymous.IoPortAccess.PortNumber, &self - ); - } else { - debug!( - "HyperV IO Details :\n Port: {:#x} \n {:#?}", - exit_context.Anonymous.IoPortAccess.PortNumber, &self - ); - } - } + debug!( + "HyperV IO Details :\n Port: {:#x} \n {:#?}", + exit_context.Anonymous.IoPortAccess.PortNumber, &self + ); HyperlightExit::IoOut( exit_context.Anonymous.IoPortAccess.PortNumber, exit_context @@ -508,18 +498,6 @@ impl Hypervisor for HypervWindowsDriver { "HyperV Memory Access Details :\n GPA: {:#?}\n Access Info :{:#?}\n {:#?} ", gpa, access_info, &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - { - if let Err(e) = unsafe { - self.write_dump_file( - self.mem_regions.clone(), - self.source_address.add(PAGE_SIZE_USIZE) as *const u8, - self.size, - ) - } { - println!("Error dumping memory: {}", e); - } - } match self.get_memory_access_violation(gpa as usize, &self.mem_regions, access_info) { @@ -539,18 +517,6 @@ impl Hypervisor for HypervWindowsDriver { "HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}", exit_context.ExitReason, &self ); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - { - if let Err(e) = unsafe { - self.write_dump_file( - self.mem_regions.clone(), - self.source_address.add(PAGE_SIZE_USIZE) as *const u8, - self.size, - ) - } { - println!("Error dumping memory: {}", e); - } - } match self.get_exit_details(exit_context.ExitReason) { Ok(error) => HyperlightExit::Unknown(error), Err(e) => HyperlightExit::Unknown(format!("Error getting exit details: {}", e)), @@ -569,6 +535,11 @@ impl Hypervisor for HypervWindowsDriver { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor { self as &mut dyn Hypervisor } + + #[cfg(feature = "dump_on_crash")] + fn get_memory_regions(&self) -> &[MemoryRegion] { + &self.mem_regions + } } #[cfg(test)] diff --git a/src/hyperlight_host/src/hypervisor/inprocess.rs b/src/hyperlight_host/src/hypervisor/inprocess.rs index a8ff862e..bd173747 100644 --- a/src/hyperlight_host/src/hypervisor/inprocess.rs +++ b/src/hyperlight_host/src/hypervisor/inprocess.rs @@ -18,6 +18,8 @@ use std::fmt::Debug; use std::os::raw::c_void; use super::{HyperlightExit, Hypervisor}; +#[cfg(feature = "dump_on_crash")] +use crate::mem::memory_region::MemoryRegion; use crate::sandbox::leaked_outb::LeakedOutBWrapper; use crate::Result; @@ -123,4 +125,9 @@ impl<'a> Hypervisor for InprocessDriver<'a> { fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE { unimplemented!("get_partition_handle should not be needed since we are in in-process mode") } + + #[cfg(feature = "dump_on_crash")] + fn get_memory_regions(&self) -> &[MemoryRegion] { + unimplemented!("get_memory_regions is not supported since we are in in-process mode") + } } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 5caf8241..7f7ec033 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -17,7 +17,6 @@ limitations under the License. use std::convert::TryFrom; use std::fmt::Debug; -use cfg_if::cfg_if; use kvm_bindings::{kvm_fpu, kvm_regs, kvm_userspace_memory_region, KVM_MEM_READONLY}; use kvm_ioctls::Cap::UserMemory; use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; @@ -283,11 +282,9 @@ impl Hypervisor for KVMDriver { } Ok(VcpuExit::MmioRead(addr, _)) => { debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); - let gpa = addr as usize; + match self.get_memory_access_violation( - gpa, + addr as usize, &self.mem_regions, MemoryRegionFlags::READ, ) { @@ -297,11 +294,9 @@ impl Hypervisor for KVMDriver { } Ok(VcpuExit::MmioWrite(addr, _)) => { debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); - let gpa = addr as usize; + match self.get_memory_access_violation( - gpa, + addr as usize, &self.mem_regions, MemoryRegionFlags::WRITE, ) { @@ -315,25 +310,12 @@ impl Hypervisor for KVMDriver { libc::EAGAIN => HyperlightExit::Retry(), _ => { debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self); - #[cfg(all(debug_assertions, feature = "dump_on_crash"))] - self.dump_on_crash(self.mem_regions.clone()); log_then_return!("Error running VCPU {:?}", e); } }, Ok(other) => { - cfg_if! { - if #[cfg(all(feature = "print_debug", debug_assertions))] { - let _ = other; - debug!("KVM Other Exit: \n {:#?}", &self); - HyperlightExit::Unknown("Unexpected KVM Exit".to_string()) - } else if #[cfg(all(feature = "dump_on_crash", debug_assertions))] { - self.dump_on_crash(self.mem_regions.clone()); - HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other)) - } else{ - debug!("KVM Other Exit {:?}", other); - HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other)) - } - } + debug!("KVM Other Exit {:?}", other); + HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other)) } }; Ok(result) @@ -343,6 +325,11 @@ impl Hypervisor for KVMDriver { fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor { self as &mut dyn Hypervisor } + + #[cfg(feature = "dump_on_crash")] + fn get_memory_regions(&self) -> &[MemoryRegion] { + &self.mem_regions + } } #[cfg(test)] diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 4f13ff2b..166f973e 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -55,6 +55,9 @@ pub(crate) mod windows_hypervisor_platform; #[cfg(target_os = "windows")] pub(crate) mod wrappers; +#[cfg(feature = "dump_on_crash")] +pub(crate) mod dump_on_crash; + use std::fmt::Debug; use std::sync::{Arc, Mutex}; @@ -184,70 +187,8 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { #[cfg(target_os = "windows")] fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE; - /// Dump memory to a file on crash - #[cfg(all(debug_assertions, feature = "dump_on_crash", target_os = "linux"))] - fn dump_on_crash(&self, mem_regions: Vec) { - let memory_size = mem_regions - .iter() - .map(|region| region.guest_region.end - region.guest_region.start) - .sum(); - if let Err(e) = unsafe { - self.write_dump_file( - mem_regions.clone(), - mem_regions[0].host_region.start as *const u8, - memory_size, - ) - } { - println!("Error dumping memory: {}", e); - } - } - - /// A function that takes an address and a size and writes the memory at that address to a file in the temp/tmp directory - /// # Safety - /// This function is unsafe because it is writing memory to a file, make sure that the address is valid and that the size is correct - /// This function is only available when the `dump_on_crash` feature is enabled and running in debug mode - #[cfg(all(feature = "dump_on_crash", debug_assertions))] - unsafe fn write_dump_file( - &self, - regions: Vec, - address: *const u8, - size: usize, - ) -> Result<()> { - use std::io::Write; - - use tempfile::NamedTempFile; - - if address.is_null() || size == 0 { - return Err(new_error!("Invalid address or size")); - } - - let hv_details = format!("{:#?}", self); - let regions_details = format!("{:#?}", regions); - - // Create a temporary file - let mut file = NamedTempFile::with_prefix("mem")?; - let temp_path = file.path().to_path_buf(); - - file.write_all(hv_details.as_bytes())?; - file.write_all(b"\n")?; - file.write_all(regions_details.as_bytes())?; - file.write_all(b"\n")?; - - // SAFETY: Ensure the address and size are valid and accessible - unsafe { - let slice = std::slice::from_raw_parts(address, size); - file.write_all(slice)?; - file.flush()?; - } - let persist_path = temp_path.with_extension("dmp"); - file.persist(&persist_path) - .map_err(|e| new_error!("Failed to persist file: {:?}", e))?; - - print!("Memory dumped to file: {:?}", persist_path); - log::error!("Memory dumped to file: {:?}", persist_path); - - Ok(()) - } + #[cfg(feature = "dump_on_crash")] + fn get_memory_regions(&self) -> &[MemoryRegion]; } /// A virtual CPU that can be run until an exit occurs @@ -263,22 +204,29 @@ impl VirtualCPU { mem_access_fn: Arc>, ) -> Result<()> { loop { - match hv.run()? { - HyperlightExit::Halt() => { + match hv.run() { + Ok(HyperlightExit::Halt()) => { break; } - HyperlightExit::IoOut(port, data, rip, instruction_length) => { + Ok(HyperlightExit::IoOut(port, data, rip, instruction_length)) => { hv.handle_io(port, data, rip, instruction_length, outb_handle_fn.clone())? } - HyperlightExit::Mmio(addr) => { + Ok(HyperlightExit::Mmio(addr)) => { + #[cfg(feature = "dump_on_crash")] + dump_on_crash::dump_on_crash_to_tempfile(hv)?; + mem_access_fn .clone() .try_lock() .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? .call()?; + log_then_return!("MMIO access address {:#x}", addr); } - HyperlightExit::AccessViolation(addr, tried, region_permission) => { + Ok(HyperlightExit::AccessViolation(addr, tried, region_permission)) => { + #[cfg(feature = "dump_on_crash")] + dump_on_crash::dump_on_crash_to_tempfile(hv)?; + if region_permission.intersects(MemoryRegionFlags::STACK_GUARD) { return Err(HyperlightError::StackOverflow()); } @@ -288,7 +236,7 @@ impl VirtualCPU { region_permission )); } - HyperlightExit::Cancelled() => { + Ok(HyperlightExit::Cancelled()) => { // Shutdown is returned when the host has cancelled execution // After termination, the main thread will re-initialize the VM if let Some(hvh) = hv_handler { @@ -301,10 +249,19 @@ impl VirtualCPU { int_counter_inc!(&NumberOfCancelledGuestExecutions); log_then_return!(ExecutionCanceledByHost()); } - HyperlightExit::Unknown(reason) => { + Ok(HyperlightExit::Unknown(reason)) => { + #[cfg(feature = "dump_on_crash")] + dump_on_crash::dump_on_crash_to_tempfile(hv)?; + log_then_return!("Unexpected VM Exit {:?}", reason); } - HyperlightExit::Retry() => continue, + Ok(HyperlightExit::Retry()) => continue, + Err(e) => { + #[cfg(feature = "dump_on_crash")] + dump_on_crash::dump_on_crash_to_tempfile(hv)?; + + return Err(e); + } } } From c743308662eacea3578832f473a8ba3956ac6118 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg Date: Mon, 25 Nov 2024 20:15:38 +0000 Subject: [PATCH 05/21] Rename cargo feature 'dump_on_crash' to 'crashdump' and alias with debug_assertions Signed-off-by: Ludvig Liljenberg --- docs/debugging-hyperlight.md | 2 +- src/hyperlight_host/Cargo.toml | 4 +--- src/hyperlight_host/build.rs | 2 ++ .../{dump_on_crash.rs => crashdump.rs} | 6 ++--- .../src/hypervisor/hyperv_linux.rs | 2 +- .../src/hypervisor/hyperv_windows.rs | 2 +- .../src/hypervisor/inprocess.rs | 4 ++-- src/hyperlight_host/src/hypervisor/kvm.rs | 2 +- src/hyperlight_host/src/hypervisor/mod.rs | 22 +++++++++---------- 9 files changed, 23 insertions(+), 23 deletions(-) rename src/hyperlight_host/src/hypervisor/{dump_on_crash.rs => crashdump.rs} (87%) diff --git a/docs/debugging-hyperlight.md b/docs/debugging-hyperlight.md index b933c775..9ff3b822 100644 --- a/docs/debugging-hyperlight.md +++ b/docs/debugging-hyperlight.md @@ -39,6 +39,6 @@ cargo test --package hyperlight-host --test integration_test --features print_de ## Dumping the memory configuration, virtual processor register state and memory contents on a crash or unexpected VM Exit -To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `dump_on_crash` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message. +To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message. There are no tools at this time to analyze the dump file, but it can be useful for debugging. diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index f876fd29..f4c36f41 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -118,9 +118,7 @@ function_call_metrics = [] executable_heap = [] # This feature enables printing of debug information to stdout in debug builds print_debug = [] -# This feature enables dunping of the VMs details to a file when an unexpected or error exit occurs in a VM in debug mode -# the name of the file is output to stdout and logged. -dump_on_crash = ["dep:tempfile"] +crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds. kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"] inprocess = [] diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index f115c6e3..9cc0db12 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -94,6 +94,8 @@ fn main() -> Result<()> { // inprocess feature is aliased with debug_assertions to make it only available in debug-builds. // You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead. inprocess: { all(feature = "inprocess", debug_assertions) }, + // crashdump feature is aliased with debug_assertions to make it only available in debug-builds. + crashdump: { all(feature = "crashdump", debug_assertions) }, } write_built_file()?; diff --git a/src/hyperlight_host/src/hypervisor/dump_on_crash.rs b/src/hyperlight_host/src/hypervisor/crashdump.rs similarity index 87% rename from src/hyperlight_host/src/hypervisor/dump_on_crash.rs rename to src/hyperlight_host/src/hypervisor/crashdump.rs index bdc2ab04..a70dc34c 100644 --- a/src/hyperlight_host/src/hypervisor/dump_on_crash.rs +++ b/src/hyperlight_host/src/hypervisor/crashdump.rs @@ -6,8 +6,8 @@ use super::Hypervisor; use crate::{new_error, Result}; /// Dump registers + memory regions + raw memory to a tempfile -#[cfg(feature = "dump_on_crash")] -pub(crate) fn dump_on_crash_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { +#[cfg(crashdump)] +pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { let mut temp_file = NamedTempFile::with_prefix("mem")?; let hv_details = format!("{:#x?}", hv); @@ -35,7 +35,7 @@ pub(crate) fn dump_on_crash_to_tempfile(hv: &dyn Hypervisor) -> Result<()> { let persist_path = temp_file.path().with_extension("dmp"); temp_file .persist(&persist_path) - .map_err(|e| new_error!("Failed to persist dump_on_crash file: {:?}", e))?; + .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?; println!("Memory dumped to file: {:?}", persist_path); log::error!("Memory dumped to file: {:?}", persist_path); diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 1a583fe0..7633272d 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -353,7 +353,7 @@ impl Hypervisor for HypervLinuxDriver { self as &mut dyn Hypervisor } - #[cfg(feature = "dump_on_crash")] + #[cfg(crashdump)] fn get_memory_regions(&self) -> &[MemoryRegion] { &self.mem_regions } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 4984f46e..5c5bbc60 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -536,7 +536,7 @@ impl Hypervisor for HypervWindowsDriver { self as &mut dyn Hypervisor } - #[cfg(feature = "dump_on_crash")] + #[cfg(crashdump)] fn get_memory_regions(&self) -> &[MemoryRegion] { &self.mem_regions } diff --git a/src/hyperlight_host/src/hypervisor/inprocess.rs b/src/hyperlight_host/src/hypervisor/inprocess.rs index bd173747..f7076a5b 100644 --- a/src/hyperlight_host/src/hypervisor/inprocess.rs +++ b/src/hyperlight_host/src/hypervisor/inprocess.rs @@ -18,7 +18,7 @@ use std::fmt::Debug; use std::os::raw::c_void; use super::{HyperlightExit, Hypervisor}; -#[cfg(feature = "dump_on_crash")] +#[cfg(crashdump)] use crate::mem::memory_region::MemoryRegion; use crate::sandbox::leaked_outb::LeakedOutBWrapper; use crate::Result; @@ -126,7 +126,7 @@ impl<'a> Hypervisor for InprocessDriver<'a> { unimplemented!("get_partition_handle should not be needed since we are in in-process mode") } - #[cfg(feature = "dump_on_crash")] + #[cfg(crashdump)] fn get_memory_regions(&self) -> &[MemoryRegion] { unimplemented!("get_memory_regions is not supported since we are in in-process mode") } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 7f7ec033..74801320 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -326,7 +326,7 @@ impl Hypervisor for KVMDriver { self as &mut dyn Hypervisor } - #[cfg(feature = "dump_on_crash")] + #[cfg(crashdump)] fn get_memory_regions(&self) -> &[MemoryRegion] { &self.mem_regions } diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 166f973e..c34d7eb1 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -55,8 +55,8 @@ pub(crate) mod windows_hypervisor_platform; #[cfg(target_os = "windows")] pub(crate) mod wrappers; -#[cfg(feature = "dump_on_crash")] -pub(crate) mod dump_on_crash; +#[cfg(crashdump)] +pub(crate) mod crashdump; use std::fmt::Debug; use std::sync::{Arc, Mutex}; @@ -187,7 +187,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { #[cfg(target_os = "windows")] fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE; - #[cfg(feature = "dump_on_crash")] + #[cfg(crashdump)] fn get_memory_regions(&self) -> &[MemoryRegion]; } @@ -212,8 +212,8 @@ impl VirtualCPU { hv.handle_io(port, data, rip, instruction_length, outb_handle_fn.clone())? } Ok(HyperlightExit::Mmio(addr)) => { - #[cfg(feature = "dump_on_crash")] - dump_on_crash::dump_on_crash_to_tempfile(hv)?; + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; mem_access_fn .clone() @@ -224,8 +224,8 @@ impl VirtualCPU { log_then_return!("MMIO access address {:#x}", addr); } Ok(HyperlightExit::AccessViolation(addr, tried, region_permission)) => { - #[cfg(feature = "dump_on_crash")] - dump_on_crash::dump_on_crash_to_tempfile(hv)?; + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; if region_permission.intersects(MemoryRegionFlags::STACK_GUARD) { return Err(HyperlightError::StackOverflow()); @@ -250,15 +250,15 @@ impl VirtualCPU { log_then_return!(ExecutionCanceledByHost()); } Ok(HyperlightExit::Unknown(reason)) => { - #[cfg(feature = "dump_on_crash")] - dump_on_crash::dump_on_crash_to_tempfile(hv)?; + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; log_then_return!("Unexpected VM Exit {:?}", reason); } Ok(HyperlightExit::Retry()) => continue, Err(e) => { - #[cfg(feature = "dump_on_crash")] - dump_on_crash::dump_on_crash_to_tempfile(hv)?; + #[cfg(crashdump)] + crashdump::crashdump_to_tempfile(hv)?; return Err(e); } From 79142ee8686eef0e80ec32b6295e947e566efa16 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg Date: Mon, 25 Nov 2024 20:08:34 +0000 Subject: [PATCH 06/21] Add check in CI to make sure cargo feature 'crashdump' and 'print_debug" compiles Signed-off-by: Ludvig Liljenberg --- .github/workflows/dep_rust.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 079373db..f3180790 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -74,6 +74,10 @@ jobs: # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} + # make sure certain cargo features compile + cargo check -p hyperlight-host --features crashdump + cargo check -p hyperlight-host --features print_debug + # without any driver (shouldn't compile) just test-rust-feature-compilation-fail ${{ matrix.config }} From 517652932ad63a68271e7f7a54aa9815be8fbe99 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg Date: Mon, 25 Nov 2024 20:21:30 +0000 Subject: [PATCH 07/21] Expliclity call crate::debug! instead of debug! to differentiate from log::debug, and alias feature with debug_assertions Signed-off-by: Ludvig Liljenberg --- src/hyperlight_host/build.rs | 2 ++ .../src/hypervisor/hyperv_linux.rs | 20 ++++++++++--------- src/hyperlight_host/src/hypervisor/kvm.rs | 16 +++++++-------- src/hyperlight_host/src/lib.rs | 13 ++++-------- src/hyperlight_host/src/mem/pe/pe_info.rs | 20 ++++++++++--------- .../src/sandbox/uninitialized.rs | 4 ++-- 6 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 9cc0db12..2aa321e7 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -96,6 +96,8 @@ fn main() -> Result<()> { inprocess: { all(feature = "inprocess", debug_assertions) }, // crashdump feature is aliased with debug_assertions to make it only available in debug-builds. crashdump: { all(feature = "crashdump", debug_assertions) }, + // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. + print_debug: { all(feature = "print_debug", debug_assertions) }, } write_built_file()?; diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 7633272d..b8a18303 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -37,7 +37,7 @@ use crate::hypervisor::hypervisor_handler::HypervisorHandler; use crate::hypervisor::HyperlightExit; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; -use crate::{debug, log_then_return, new_error, Result}; +use crate::{log_then_return, new_error, Result}; /// Determine whether the HyperV for Linux hypervisor API is present /// and functional. @@ -287,7 +287,7 @@ impl Hypervisor for HypervLinuxDriver { let result = match &self.vcpu_fd.run(hv_message) { Ok(m) => match m.header.message_type { HALT_MESSAGE => { - debug!("mshv - Halt Details : {:#?}", &self); + crate::debug!("mshv - Halt Details : {:#?}", &self); HyperlightExit::Halt() } IO_PORT_INTERCEPT_MESSAGE => { @@ -296,7 +296,7 @@ impl Hypervisor for HypervLinuxDriver { let rip = io_message.header.rip; let rax = io_message.rax; let instruction_length = io_message.header.instruction_length() as u64; - debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self); + crate::debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self); HyperlightExit::IoOut( port_number, rax.to_le_bytes().to_vec(), @@ -307,9 +307,10 @@ impl Hypervisor for HypervLinuxDriver { UNMAPPED_GPA_MESSAGE => { let mimo_message = m.to_memory_info()?; let addr = mimo_message.guest_physical_address; - debug!( + crate::debug!( "mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}", - addr, &self + addr, + &self ); HyperlightExit::Mmio(addr) } @@ -317,9 +318,10 @@ impl Hypervisor for HypervLinuxDriver { let mimo_message = m.to_memory_info()?; let gpa = mimo_message.guest_physical_address; let access_info = MemoryRegionFlags::try_from(mimo_message)?; - debug!( + crate::debug!( "mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}", - gpa, &self + gpa, + &self ); match self.get_memory_access_violation( gpa as usize, @@ -331,7 +333,7 @@ impl Hypervisor for HypervLinuxDriver { } } other => { - debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self); + crate::debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self); log_then_return!("unknown Hyper-V run message type {:?}", other); } }, @@ -340,7 +342,7 @@ impl Hypervisor for HypervLinuxDriver { libc::EINTR => HyperlightExit::Cancelled(), libc::EAGAIN => HyperlightExit::Retry(), _ => { - debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self); + crate::debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self); log_then_return!("Error running VCPU {:?}", e); } }, diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 74801320..991e1e24 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -31,7 +31,7 @@ use super::{ use crate::hypervisor::hypervisor_handler::HypervisorHandler; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; -use crate::{debug, log_then_return, new_error, Result}; +use crate::{log_then_return, new_error, Result}; /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise #[instrument(skip_all, parent = Span::current(), level = "Trace")] @@ -271,17 +271,17 @@ impl Hypervisor for KVMDriver { let exit_reason = self.vcpu_fd.run(); let result = match exit_reason { Ok(VcpuExit::Hlt) => { - debug!("KVM - Halt Details : {:#?}", &self); + crate::debug!("KVM - Halt Details : {:#?}", &self); HyperlightExit::Halt() } Ok(VcpuExit::IoOut(port, data)) => { - // because vcpufd.run() mutably borrows self we cannot pass self to debug! macro here - debug!("KVM IO Details : \nPort : {}\nData : {:?}", port, data); + // because vcpufd.run() mutably borrows self we cannot pass self to crate::debug! macro here + crate::debug!("KVM IO Details : \nPort : {}\nData : {:?}", port, data); // KVM does not need to set RIP or instruction length so these are set to 0 HyperlightExit::IoOut(port, data.to_vec(), 0, 0) } Ok(VcpuExit::MmioRead(addr, _)) => { - debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self); + crate::debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self); match self.get_memory_access_violation( addr as usize, @@ -293,7 +293,7 @@ impl Hypervisor for KVMDriver { } } Ok(VcpuExit::MmioWrite(addr, _)) => { - debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self); + crate::debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self); match self.get_memory_access_violation( addr as usize, @@ -309,12 +309,12 @@ impl Hypervisor for KVMDriver { libc::EINTR => HyperlightExit::Cancelled(), libc::EAGAIN => HyperlightExit::Retry(), _ => { - debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self); + crate::debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self); log_then_return!("Error running VCPU {:?}", e); } }, Ok(other) => { - debug!("KVM Other Exit {:?}", other); + crate::debug!("KVM Other Exit {:?}", other); HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other)) } }; diff --git a/src/hyperlight_host/src/lib.rs b/src/hyperlight_host/src/lib.rs index 7170552d..2dac703f 100644 --- a/src/hyperlight_host/src/lib.rs +++ b/src/hyperlight_host/src/lib.rs @@ -143,19 +143,14 @@ macro_rules! log_then_return { }; } +// same as log::debug!, but will additionally print to stdout if the print_debug feature is enabled #[macro_export] macro_rules! debug { - ($($arg:tt)+) => { - // If the print_debug feature is enabled, print the debug message to the console - #[cfg(all(feature = "print_debug", debug_assertions))] - { - (println!($($arg)+)) - } - - // Then log/trace the debug message - (log::debug!($($arg)+)) + #[cfg(print_debug)] + println!($($arg)+); + log::debug!($($arg)+); } } diff --git a/src/hyperlight_host/src/mem/pe/pe_info.rs b/src/hyperlight_host/src/mem/pe/pe_info.rs index eb4b792c..97a99bfa 100644 --- a/src/hyperlight_host/src/mem/pe/pe_info.rs +++ b/src/hyperlight_host/src/mem/pe/pe_info.rs @@ -23,7 +23,7 @@ use goblin::pe::PE; use tracing::{instrument, Span}; use crate::mem::pe::base_relocations; -use crate::{debug, log_then_return, Result}; +use crate::{log_then_return, Result}; const IMAGE_REL_BASED_DIR64: u8 = 10; const IMAGE_REL_BASED_ABSOLUTE: u8 = 0; @@ -106,9 +106,11 @@ impl PEInfo { let name = section.name().unwrap_or("Unknown"); let virtual_size = section.virtual_size; let raw_size = section.size_of_raw_data; - debug!( + crate::debug!( "Section: {}, Virtual Size: {}, On-Disk Size: {}", - name, virtual_size, raw_size + name, + virtual_size, + raw_size ); if virtual_size > raw_size { @@ -122,16 +124,16 @@ impl PEInfo { data_section_raw_pointer = section.pointer_to_raw_data; data_section_additional_bytes = virtual_size - raw_size; - debug!( + crate::debug!( "Resizing the data section - Additional bytes required: {}", data_section_additional_bytes ); - debug!( + crate::debug!( "Resizing the data section - Existing PE File Size: {} New PE File Size: {}", pe_bytes.len(), pe_bytes.len() + data_section_additional_bytes as usize, ); - debug!( + crate::debug!( "Resizing the data section - Data Section Raw Pointer: {}", data_section_raw_pointer ); @@ -140,19 +142,19 @@ impl PEInfo { end_of_data_index = (section.pointer_to_raw_data + section.size_of_raw_data) as usize; - debug!("End of data index: {}", end_of_data_index); + crate::debug!("End of data index: {}", end_of_data_index); // the remainder of the data is the rest of the file after the .data section if any let next_section = pe.sections.get(i + 1); if let Some(next_section) = next_section { - debug!( + crate::debug!( "Start of section after data index: {}", next_section.pointer_to_raw_data ); } else { - debug!("No more sections after the .data section"); + crate::debug!("No more sections after the .data section"); } } else { log_then_return!( diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index eaa573de..6ae2e1a0 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -35,7 +35,7 @@ use crate::sandbox::SandboxConfiguration; use crate::sandbox_state::sandbox::EvolvableSandbox; use crate::sandbox_state::transition::Noop; use crate::{ - debug, log_build_details, log_then_return, new_error, MultiUseSandbox, Result, SingleUseSandbox, + log_build_details, log_then_return, new_error, MultiUseSandbox, Result, SingleUseSandbox, }; /// A preliminary `Sandbox`, not yet ready to execute guest code. @@ -258,7 +258,7 @@ impl UninitializedSandbox { } } - debug!("Sandbox created: {:#?}", sandbox); + crate::debug!("Sandbox created: {:#?}", sandbox); Ok(sandbox) } From 82d10f89848f4efd924025f0da1c963bac8e5fed Mon Sep 17 00:00:00 2001 From: danbugs Date: Tue, 26 Nov 2024 18:46:20 +0000 Subject: [PATCH 08/21] [docs] small patch to HIP template Signed-off-by: danbugs --- proposals/NNNN-hip-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proposals/NNNN-hip-template/README.md b/proposals/NNNN-hip-template/README.md index b6ca1622..fb8d91b2 100644 --- a/proposals/NNNN-hip-template/README.md +++ b/proposals/NNNN-hip-template/README.md @@ -1,4 +1,4 @@ -# HIP NNNN - SKIP NAME +# HIP NNNN - HIP NAME - [Summary](#summary) From 23192b706d068a0e4eda96ca9b4ec9cc2e00122a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Wed, 27 Nov 2024 19:31:17 +0200 Subject: [PATCH 09/21] Fix otlp tracing (#61) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix otlp_tracing compilation issue Signed-off-by: Doru Blânzeanu * Fix logging filter so that jaeger can get the traces Signed-off-by: Doru Blânzeanu * Fix fmt check error Signed-off-by: Doru Blânzeanu * Add otlp_tracing test to ensure the traces are accessible for collectors Signed-off-by: Doru Blânzeanu --------- Signed-off-by: Doru Blânzeanu --- Cargo.lock | 29 ++-- docs/hyperlight-metrics-logs-and-traces.md | 6 +- src/hyperlight_host/Cargo.toml | 11 +- .../examples/otlp_tracing/main.rs | 124 ++++++++++++++---- 4 files changed, 129 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b9da8f0..071cfbd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1107,6 +1107,7 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry-otlp", + "opentelemetry-semantic-conventions", "opentelemetry_sdk", "page_size", "paste", @@ -1709,9 +1710,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "opentelemetry" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +checksum = "0f3cebff57f7dbd1255b44d8bddc2cebeb0ea677dbaa2e25a3070a91b318f660" dependencies = [ "futures-core", "futures-sink", @@ -1723,9 +1724,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e1f9c8b032d4f635c730c0efcf731d5e2530ea13fa8bef7939ddc8420696bd" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", @@ -1737,13 +1738,14 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tonic", + "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d3968ce3aefdcca5c27e3c4ea4391b37547726a70893aab52d3de95d5f8b34" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1751,11 +1753,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + [[package]] name = "opentelemetry_sdk" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3" +checksum = "27b742c1cae4693792cc564e58d75a2a0ba29421a34a85b50da92efa89ecb2bc" dependencies = [ "async-trait", "futures-channel", @@ -1770,6 +1778,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", + "tracing", ] [[package]] @@ -2882,9 +2891,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" dependencies = [ "js-sys", "once_cell", diff --git a/docs/hyperlight-metrics-logs-and-traces.md b/docs/hyperlight-metrics-logs-and-traces.md index accb3370..3a22e5a8 100644 --- a/docs/hyperlight-metrics-logs-and-traces.md +++ b/docs/hyperlight-metrics-logs-and-traces.md @@ -82,13 +82,13 @@ In the [examples/otlp_tracing](../src/hyperlight_host/examples/otlp_tracing) dir #### Linux ```bash -RUST_LOG='none,hyperlight-host=info,tracing=info' cargo run --example otlp_tracing +RUST_LOG='none,hyperlight_host=info,tracing=info' cargo run --example otlp_tracing ``` #### Windows ```powershell -$env:RUST_LOG='none,hyperlight-host=info,tracing=info';cargo run --example otlp_tracing +$env:RUST_LOG='none,hyperlight_host=info,tracing=info';cargo run --example otlp_tracing ``` The sample will run and generate trace data until any key is pressed. @@ -96,7 +96,7 @@ The sample will run and generate trace data until any key is pressed. To view the trace data, leave the example running and use the jaegertracing/all-in-one container image with the following command: ```console - docker run -d --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 4317:4317 -p 16686:16686 jaegertracing/all-in-one:1.51 + docker run -d --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 4317:4317 -p 16686:16686 jaegertracing/all-in-one:1.60 ``` NOTE: when running this on windows that this is a linux container, so you will need to ensure that docker is configured to run linux containers using WSL2. Alternatively, you can download the Jaeger binaries from [here](https://www.jaegertracing.io/download/). Extract the archive and run the `jaeger-all-in-one` executable as follows: diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index f4c36f41..0966177d 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -89,11 +89,12 @@ hyperlight-testing = { workspace = true } env_logger = "0.11.5" tracing-forest = { version = "0.1.6", features = ["uuid", "chrono", "smallvec", "serde", "env-filter"] } tracing = "0.1.37" -tracing-subscriber = { version = "0.3.18", features = ["std", "env-filter"] } -tracing-opentelemetry = "0.27.0" -opentelemetry = "0.26.0" -opentelemetry-otlp = { version = "0.26.0", features = ["default"] } -opentelemetry_sdk = { version = "0.26.0", features = ["rt-tokio"] } +tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter"]} +tracing-opentelemetry = "0.28.0" +opentelemetry = "0.27.0" +opentelemetry-otlp = { version = "0.27.0", features = ["default"] } +opentelemetry-semantic-conventions = "0.27.0" +opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] } tokio = { version = "1.34.0", features = ["full"] } criterion = "0.5.1" tracing-chrome = "0.7.2" diff --git a/src/hyperlight_host/examples/otlp_tracing/main.rs b/src/hyperlight_host/examples/otlp_tracing/main.rs index d80e0a80..5548c484 100644 --- a/src/hyperlight_host/examples/otlp_tracing/main.rs +++ b/src/hyperlight_host/examples/otlp_tracing/main.rs @@ -17,6 +17,8 @@ limitations under the License. use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType}; use rand::Rng; use tracing::{span, Level}; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; extern crate hyperlight_host; use std::error::Error; @@ -29,16 +31,19 @@ use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox; use hyperlight_host::sandbox_state::transition::Noop; use hyperlight_host::{GuestBinary, MultiUseSandbox, Result as HyperlightResult}; use hyperlight_testing::simple_guest_as_string; -use opentelemetry::global::shutdown_tracer_provider; +use opentelemetry::global::{self, shutdown_tracer_provider}; use opentelemetry::trace::TracerProvider; use opentelemetry::KeyValue; -use opentelemetry_otlp::{new_exporter, new_pipeline, WithExportConfig}; +use opentelemetry_otlp::{SpanExporter, WithExportConfig}; use opentelemetry_sdk::runtime::Tokio; use opentelemetry_sdk::{trace, Resource}; -use tracing_subscriber::layer::SubscriberExt; +use opentelemetry_semantic_conventions::attribute::{SERVICE_NAME, SERVICE_VERSION}; +use opentelemetry_semantic_conventions::SCHEMA_URL; use tracing_subscriber::EnvFilter; use uuid::Uuid; +const ENDPOINT_ADDR: &str = "http://localhost:4317"; + fn fn_writer(_msg: String) -> HyperlightResult { Ok(0) } @@ -47,30 +52,51 @@ fn fn_writer(_msg: String) -> HyperlightResult { #[tokio::main] async fn main() -> Result<(), Box> { - let tracer = new_pipeline() - .tracing() - .with_exporter( - new_exporter() - .tonic() - .with_endpoint("http://localhost:4317/v1/traces"), + init_tracing_subscriber(ENDPOINT_ADDR)?; + + Ok(run_example(true)?) +} + +fn init_tracing_subscriber(addr: &str) -> Result<(), Box> { + let exporter = SpanExporter::builder() + .with_tonic() + .with_endpoint(addr) + .build()?; + + let provider = trace::TracerProvider::builder() + .with_config( + trace::Config::default().with_resource(Resource::from_schema_url( + vec![ + KeyValue::new(SERVICE_NAME, "hyperlight_otel_example"), + KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")), + ], + SCHEMA_URL, + )), ) - .with_trace_config(trace::Config::default().with_resource(Resource::new(vec![ - KeyValue::new("service.name", "hyperlight_otel_example"), - ]))) - .install_batch(Tokio) - .unwrap() - .tracer("trace-demo"); + .with_batch_exporter(exporter, Tokio) + .build(); + + global::set_tracer_provider(provider.clone()); + let tracer = provider.tracer("trace-demo"); + + let otel_layer = OpenTelemetryLayer::new(tracer); - let otel_layer = tracing_opentelemetry::OpenTelemetryLayer::new(tracer); + // Try using the environment otherwise set default filters + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { + EnvFilter::from_default_env() + .add_directive("hyperlight_host=info".parse().unwrap()) + .add_directive("tracing=info".parse().unwrap()) + }); - tracing_subscriber::Registry::default() - .with(EnvFilter::from_default_env()) + tracing_subscriber::registry() + .with(filter) .with(otel_layer) .try_init()?; - Ok(run_example()?) + Ok(()) } -fn run_example() -> HyperlightResult<()> { + +fn run_example(wait_input: bool) -> HyperlightResult<()> { // Get the path to a simple guest binary. let hyperlight_guest_path = simple_guest_as_string().expect("Cannot find the guest binary at the expected location."); @@ -168,9 +194,12 @@ fn run_example() -> HyperlightResult<()> { join_handles.push(handle); } - println!("Press enter to exit..."); - let mut input = String::new(); - stdin().read_line(&mut input)?; + if wait_input { + println!("Press enter to exit..."); + let mut input = String::new(); + stdin().read_line(&mut input)?; + } + *should_exit.try_lock().unwrap() = true; for join_handle in join_handles { let result = join_handle.join(); @@ -180,3 +209,52 @@ fn run_example() -> HyperlightResult<()> { Ok(()) } + +#[cfg(test)] +mod test { + use hyperlight_host::{HyperlightError, Result}; + use tokio::io::AsyncReadExt; + use tokio::net::{TcpListener, TcpStream}; + + use super::*; + + const TESTER_ADDR: &str = "127.0.0.1:4317"; + + async fn handle(mut stream: TcpStream) -> Result<()> { + let mut buf = Vec::with_capacity(128); + let size = stream.read_buf(&mut buf).await?; + + if size > 0 { + Ok(()) + } else { + Err(HyperlightError::Error("Cannot read req body".to_string())) + } + } + + async fn check_otl_connection(addr: &str) -> Result<()> { + let listener = TcpListener::bind(addr).await?; + + let (stream, _) = listener.accept().await?; + + handle(stream).await + } + + #[tokio::test] + async fn test_subscriber() { + // Create task that generates spans + let task = tokio::spawn(async move { + let _ = init_tracing_subscriber(ENDPOINT_ADDR); + + // No need to wait for input, just generate some spans and exit + let _ = run_example(false); + }); + + // Create server that listens and checks to see if traces are received + let result = check_otl_connection(TESTER_ADDR).await; + + // Abort task in case it doesn't finish + task.abort(); + + assert!(result.is_ok()); + } +} From 3f981d237fa39dbcf68c015b16ee185634b34c8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 02:05:49 +0000 Subject: [PATCH 10/21] Bump libc from 0.2.165 to 0.2.167 Bumps [libc](https://github.com/rust-lang/libc) from 0.2.165 to 0.2.167. - [Release notes](https://github.com/rust-lang/libc/releases) - [Changelog](https://github.com/rust-lang/libc/blob/0.2.167/CHANGELOG.md) - [Commits](https://github.com/rust-lang/libc/compare/0.2.165...0.2.167) --- updated-dependencies: - dependency-name: libc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- src/hyperlight_host/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 071cfbd7..5742b01a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1447,9 +1447,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.165" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb4d3d38eab6c5239a362fa8bae48c03baf980a6e7079f063942d563ef3533e" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libfuzzer-sys" diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 0966177d..e7aface1 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -24,7 +24,7 @@ workspace = true goblin = { version = "0.9" } rand = { version = "0.8.5" } cfg-if = { version = "1.0.0" } -libc = { version = "0.2.165" } +libc = { version = "0.2.167" } paste = "1.0" flatbuffers = "24.3.25" page_size = "0.6.0" From 1181e50933b9acd154f9b2c17dd45f5d8a4e1a46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:27:49 +0000 Subject: [PATCH 11/21] Bump cc from 1.2.1 to 1.2.2 Bumps [cc](https://github.com/rust-lang/cc-rs) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/rust-lang/cc-rs/releases) - [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.1...cc-v1.2.2) --- updated-dependencies: - dependency-name: cc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- src/hyperlight_guest/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5742b01a..4929fdce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ "jobserver", "libc", diff --git a/src/hyperlight_guest/Cargo.toml b/src/hyperlight_guest/Cargo.toml index 98220695..8f218b1f 100644 --- a/src/hyperlight_guest/Cargo.toml +++ b/src/hyperlight_guest/Cargo.toml @@ -27,5 +27,5 @@ spin = "0.9.8" log = { version = "0.4", default-features = false } [build-dependencies] -cc = "1.0" +cc = "1.2" cfg-if = "1.0" From 4937f1ae94c2fe6f98e4edbefa88a211aff03045 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 18:00:49 +0000 Subject: [PATCH 12/21] Bump tracing from 0.1.40 to 0.1.41 Bumps [tracing](https://github.com/tokio-rs/tracing) from 0.1.40 to 0.1.41. - [Release notes](https://github.com/tokio-rs/tracing/releases) - [Commits](https://github.com/tokio-rs/tracing/compare/tracing-0.1.40...tracing-0.1.41) --- updated-dependencies: - dependency-name: tracing dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- src/hyperlight_common/Cargo.toml | 2 +- src/hyperlight_host/Cargo.toml | 4 ++-- src/hyperlight_testing/Cargo.toml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4929fdce..30beb429 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2821,9 +2821,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -2833,9 +2833,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", diff --git a/src/hyperlight_common/Cargo.toml b/src/hyperlight_common/Cargo.toml index 1080f5be..063372c4 100644 --- a/src/hyperlight_common/Cargo.toml +++ b/src/hyperlight_common/Cargo.toml @@ -18,7 +18,7 @@ workspace = true flatbuffers = { version = "24.3.25", default-features = false } anyhow = { version = "1.0.72", default-features = false } log = "0.4.20" -tracing = { version = "0.1.27", optional = true } +tracing = { version = "0.1.41", optional = true } strum = {version = "0.25", default-features = false, features = ["derive"]} strum_macros = {version = "0.26", features =[]} diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index e7aface1..d3c7d88e 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -35,7 +35,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" log = "0.4.20" once_cell = { version = "1.18.0" } -tracing = { version = "0.1.37", features = ["log"] } +tracing = { version = "0.1.41", features = ["log"] } tracing-log = "0.2.0" tracing-core = "0.1.33" hyperlight-common = { workspace = true, default-features = true } @@ -88,7 +88,7 @@ serial_test = "3.1.1" hyperlight-testing = { workspace = true } env_logger = "0.11.5" tracing-forest = { version = "0.1.6", features = ["uuid", "chrono", "smallvec", "serde", "env-filter"] } -tracing = "0.1.37" +tracing = "0.1.41" tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter"]} tracing-opentelemetry = "0.28.0" opentelemetry = "0.27.0" diff --git a/src/hyperlight_testing/Cargo.toml b/src/hyperlight_testing/Cargo.toml index 46b46822..7fc871b1 100644 --- a/src/hyperlight_testing/Cargo.toml +++ b/src/hyperlight_testing/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" anyhow = "1.0.72" log = "0.4" once_cell = "1.19" -tracing = { version = "0.1.37", features = ["log"] } +tracing = { version = "0.1.41", features = ["log"] } tracing-log = "0.2.0" tracing-core = "0.1.33" tracing-serde = "0.1" From f8f8ed187681c54e7c3fe449864a172822f02df7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 18:49:37 +0000 Subject: [PATCH 13/21] Bump tracing-serde from 0.1.3 to 0.2.0 (#88) --- Cargo.lock | 4 ++-- src/hyperlight_host/Cargo.toml | 2 +- src/hyperlight_testing/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30beb429..9c33ea2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2909,9 +2909,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" dependencies = [ "serde", "tracing-core", diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index d3c7d88e..4c675489 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -83,7 +83,7 @@ serde = "1.0" proptest = "1.2.0" tempfile = "3.8.0" crossbeam-queue = "0.3.8" -tracing-serde = "0.1.3" +tracing-serde = "0.2.0" serial_test = "3.1.1" hyperlight-testing = { workspace = true } env_logger = "0.11.5" diff --git a/src/hyperlight_testing/Cargo.toml b/src/hyperlight_testing/Cargo.toml index 7fc871b1..bc12fb00 100644 --- a/src/hyperlight_testing/Cargo.toml +++ b/src/hyperlight_testing/Cargo.toml @@ -9,7 +9,7 @@ once_cell = "1.19" tracing = { version = "0.1.41", features = ["log"] } tracing-log = "0.2.0" tracing-core = "0.1.33" -tracing-serde = "0.1" +tracing-serde = "0.2" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" From b115c42e7214dd797da6a9430785803fcdfb5a7e Mon Sep 17 00:00:00 2001 From: Jorge Prendes Date: Tue, 3 Dec 2024 19:21:06 +0000 Subject: [PATCH 14/21] bump strup (#41) Signed-off-by: Jorge Prendes --- Cargo.lock | 20 ++----------- src/hyperlight_common/Cargo.toml | 3 +- .../flatbuffer_wrappers/guest_log_level.rs | 2 +- src/hyperlight_host/Cargo.toml | 2 +- src/hyperlight_host/src/hypervisor/metrics.rs | 4 +-- src/hyperlight_host/src/sandbox/metrics.rs | 4 +-- .../rust_guests/callbackguest/Cargo.lock | 28 +++---------------- src/tests/rust_guests/simpleguest/Cargo.lock | 28 +++---------------- 8 files changed, 18 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c33ea2a..f793fdc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1060,7 +1060,6 @@ dependencies = [ "hyperlight-testing", "log", "strum", - "strum_macros 0.26.4", "tracing", ] @@ -2497,24 +2496,11 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.25.0" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn", + "strum_macros", ] [[package]] diff --git a/src/hyperlight_common/Cargo.toml b/src/hyperlight_common/Cargo.toml index 063372c4..bc5161f8 100644 --- a/src/hyperlight_common/Cargo.toml +++ b/src/hyperlight_common/Cargo.toml @@ -19,8 +19,7 @@ flatbuffers = { version = "24.3.25", default-features = false } anyhow = { version = "1.0.72", default-features = false } log = "0.4.20" tracing = { version = "0.1.41", optional = true } -strum = {version = "0.25", default-features = false, features = ["derive"]} -strum_macros = {version = "0.26", features =[]} +strum = {version = "0.26", default-features = false, features = ["derive"]} [features] default = ["tracing"] diff --git a/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs b/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs index 0add85f6..14fa02b9 100644 --- a/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs +++ b/src/hyperlight_common/src/flatbuffer_wrappers/guest_log_level.rs @@ -16,7 +16,7 @@ limitations under the License. use anyhow::{bail, Error, Result}; use log::Level; -use strum_macros::EnumIter; +use strum::EnumIter; #[cfg(feature = "tracing")] use tracing::{instrument, Span}; diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 4c675489..465b1162 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -44,7 +44,7 @@ crossbeam = "0.8.0" crossbeam-channel = "0.5.8" thiserror = "2.0.0" prometheus = "0.13.3" -strum = { version = "0.25", features = ["derive"] } +strum = { version = "0.26", features = ["derive"] } tempfile = { version = "3.10", optional = true } serde_yaml = "0.9" anyhow = "1.0" diff --git a/src/hyperlight_host/src/hypervisor/metrics.rs b/src/hyperlight_host/src/hypervisor/metrics.rs index 499f3516..a02d19ed 100644 --- a/src/hyperlight_host/src/hypervisor/metrics.rs +++ b/src/hyperlight_host/src/hypervisor/metrics.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; use std::sync::Once; use once_cell::sync::OnceCell; -use strum::{EnumIter, EnumVariantNames, IntoStaticStr}; +use strum::{EnumIter, IntoStaticStr, VariantNames}; use tracing::{instrument, Span}; use crate::metrics::{ @@ -55,7 +55,7 @@ static HYPERVISOR_METRIC_DEFINITIONS: &[HyperlightMetricDefinition] = /// The enum is required to derive from EnumIter, EnumVariantNames, IntoStaticStr /// and strum(serialize_all = "snake_case") performs the name conversion from CamelCase to snake_case /// when the enum variant is serialized to a string -#[derive(Debug, EnumIter, EnumVariantNames, IntoStaticStr)] +#[derive(Debug, EnumIter, VariantNames, IntoStaticStr)] #[strum(serialize_all = "snake_case")] pub(super) enum HypervisorMetric { NumberOfCancelledGuestExecutions, diff --git a/src/hyperlight_host/src/sandbox/metrics.rs b/src/hyperlight_host/src/sandbox/metrics.rs index 840b003a..05ef9bae 100644 --- a/src/hyperlight_host/src/sandbox/metrics.rs +++ b/src/hyperlight_host/src/sandbox/metrics.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; use std::sync::Once; use once_cell::sync::OnceCell; -use strum::{EnumIter, EnumVariantNames, IntoStaticStr}; +use strum::{EnumIter, IntoStaticStr, VariantNames}; use tracing::{instrument, Span}; use crate::metrics::{ @@ -87,7 +87,7 @@ static SANDBOX_METRIC_DEFINITIONS: &[HyperlightMetricDefinition] = &[ /// The enum is required to derive from EnumIter, EnumVariantNames, IntoStaticStr /// and strum(serialize_all = "snake_case") performs the name conversion from CamelCase to snake_case /// when the enum variant is serialized to a string -#[derive(Debug, EnumIter, EnumVariantNames, IntoStaticStr)] +#[derive(Debug, EnumIter, VariantNames, IntoStaticStr)] #[strum(serialize_all = "snake_case")] pub(crate) enum SandboxMetric { GuestErrorCount, diff --git a/src/tests/rust_guests/callbackguest/Cargo.lock b/src/tests/rust_guests/callbackguest/Cargo.lock index 97b27b27..ff2522cc 100644 --- a/src/tests/rust_guests/callbackguest/Cargo.lock +++ b/src/tests/rust_guests/callbackguest/Cargo.lock @@ -62,12 +62,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -82,7 +76,6 @@ dependencies = [ "flatbuffers", "log", "strum", - "strum_macros 0.26.4", ] [[package]] @@ -227,24 +220,11 @@ dependencies = [ [[package]] name = "strum" -version = "0.25.0" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn", + "strum_macros", ] [[package]] @@ -253,7 +233,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", diff --git a/src/tests/rust_guests/simpleguest/Cargo.lock b/src/tests/rust_guests/simpleguest/Cargo.lock index 6ce9f3e8..f8df7b2a 100644 --- a/src/tests/rust_guests/simpleguest/Cargo.lock +++ b/src/tests/rust_guests/simpleguest/Cargo.lock @@ -54,12 +54,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -74,7 +68,6 @@ dependencies = [ "flatbuffers", "log", "strum", - "strum_macros 0.26.4", ] [[package]] @@ -228,24 +221,11 @@ dependencies = [ [[package]] name = "strum" -version = "0.25.0" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn", + "strum_macros", ] [[package]] @@ -254,7 +234,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", From 2a9315dbfa421df53ffb8b385de7ee2ae67ec306 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg Date: Tue, 3 Dec 2024 00:08:51 +0000 Subject: [PATCH 15/21] Fix realloc by updating the stored Layout. Previously the old Layout was copied over but not updated to account for the new size. Signed-off-by: Ludvig Liljenberg --- src/hyperlight_guest/src/memory.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/hyperlight_guest/src/memory.rs b/src/hyperlight_guest/src/memory.rs index f777cf4a..7e57b54c 100644 --- a/src/hyperlight_guest/src/memory.rs +++ b/src/hyperlight_guest/src/memory.rs @@ -78,9 +78,14 @@ pub extern "C" fn hlrealloc(ptr: *mut c_void, size: usize) -> *mut c_void { } unsafe { + let total_new_size = size + .checked_add(size_of::()) + .expect("data and layout size should not overflow in realloc"); + let block_start = (ptr as *const Layout).sub(1); - let layout = block_start.read(); - let total_new_size = size + size_of::(); + let old_layout = block_start.read(); + let new_layout = Layout::from_size_align(total_new_size, align_of::()).unwrap(); + let new_block_start = alloc::alloc::realloc(block_start as *mut u8, layout, total_new_size) as *mut Layout; @@ -88,8 +93,8 @@ pub extern "C" fn hlrealloc(ptr: *mut c_void, size: usize) -> *mut c_void { // Realloc failed abort_with_code(ErrorCode::MallocFailed as i32); } else { - // Return the pointer just after the layout information - // since old layout should still as it would have been copied + // Update the stored Layout, then return ptr to memory right after the Layout. + new_block_start.write(new_layout); new_block_start.add(1) as *mut c_void } } From 65d06e679c7808c663e8693dd5251a0e05092a55 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg Date: Tue, 3 Dec 2024 00:16:30 +0000 Subject: [PATCH 16/21] Remove 'hl' prefix from hlmalloc/hlfree/hlcalling/hlfree and update documentations, and mark the functions unsafe Signed-off-by: Ludvig Liljenberg --- src/hyperlight_guest/build.rs | 5 -- src/hyperlight_guest/src/memory.rs | 87 +++++++++++++++---- src/hyperlight_host/tests/integration_test.rs | 10 +-- src/tests/c_guests/c_simpleguest/main.c | 8 +- src/tests/rust_guests/simpleguest/src/main.rs | 10 +-- 5 files changed, 85 insertions(+), 35 deletions(-) diff --git a/src/hyperlight_guest/build.rs b/src/hyperlight_guest/build.rs index 533e6422..34ecbe89 100644 --- a/src/hyperlight_guest/build.rs +++ b/src/hyperlight_guest/build.rs @@ -133,11 +133,6 @@ fn cargo_main() { cfg.define("__x86_64__", None); cfg.define("__LITTLE_ENDIAN__", None); - cfg.define("malloc", "hlmalloc"); - cfg.define("calloc", "hlcalloc"); - cfg.define("free", "hlfree"); - cfg.define("realloc", "hlrealloc"); - // silence compiler warnings cfg.flag("-Wno-sign-compare"); cfg.flag("-Wno-bitwise-op-parentheses"); diff --git a/src/hyperlight_guest/src/memory.rs b/src/hyperlight_guest/src/memory.rs index 7e57b54c..70390cbd 100644 --- a/src/hyperlight_guest/src/memory.rs +++ b/src/hyperlight_guest/src/memory.rs @@ -25,19 +25,42 @@ use crate::entrypoint::abort_with_code; extern crate alloc; -#[no_mangle] -pub extern "C" fn hlmalloc(size: usize) -> *mut c_void { - alloc_helper(size, false) -} +/* + C-wrappers for Rust's registered global allocator. -pub fn alloc_helper(size: usize, zero: bool) -> *mut c_void { - // Allocate a block that includes space for both layout information and data + Each memory allocation via `malloc/calloc/realloc` is stored together with a `alloc::Layout` describing + the size and alignment of the allocation. This layout is stored just before the actual raw memory returned to the caller. + + Example: A call to malloc(64) will allocate space for both an `alloc::Layout` and 64 bytes of memory: + + ---------------------------------------------------------------------------------------- + | Layout { size: 64 + size_of::(), ... } | 64 bytes of memory | ... + ---------------------------------------------------------------------------------------- + ^ + | + | + ptr returned to caller +*/ + +// We assume the maximum alignment for any value is the alignment of u128. +const MAX_ALIGN: usize = align_of::(); + +/// Allocates a block of memory with the given size. The memory is only guaranteed to be initialized to 0s if `zero` is true, otherwise +/// it may or may not be initialized. +/// +/// # Safety +/// The returned pointer must be freed with `memory::free` when it is no longer needed, otherwise memory will leak. +unsafe fn alloc_helper(size: usize, zero: bool) -> *mut c_void { if size == 0 { return ptr::null_mut(); } - let total_size = size + size_of::(); - let layout = Layout::from_size_align(total_size, align_of::()).unwrap(); + // Allocate a block that includes space for both layout information and data + let total_size = size + .checked_add(size_of::()) + .expect("data and layout size should not overflow in alloc"); + let layout = Layout::from_size_align(total_size, MAX_ALIGN).expect("Invalid layout"); + unsafe { let raw_ptr = match zero { true => alloc::alloc::alloc_zeroed(layout), @@ -53,14 +76,36 @@ pub fn alloc_helper(size: usize, zero: bool) -> *mut c_void { } } +/// Allocates a block of memory with the given size. +/// The memory is not guaranteed to be initialized to 0s. +/// +/// # Safety +/// The returned pointer must be freed with `memory::free` when it is no longer needed, otherwise memory will leak. +#[no_mangle] +pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + alloc_helper(size, false) +} + +/// Allocates a block of memory for an array of `nmemb` elements, each of `size` bytes. +/// The memory is initialized to 0s. +/// +/// # Safety +/// The returned pointer must be freed with `memory::free` when it is no longer needed, otherwise memory will leak. #[no_mangle] -pub extern "C" fn hlcalloc(n: usize, size: usize) -> *mut c_void { - let total_size = n * size; +pub unsafe extern "C" fn calloc(nmemb: usize, size: usize) -> *mut c_void { + let total_size = nmemb + .checked_mul(size) + .expect("nmemb * size should not overflow in calloc"); + alloc_helper(total_size, true) } +/// Frees the memory block pointed to by `ptr`. +/// +/// # Safety +/// `ptr` must be a pointer to a memory block previously allocated by `memory::malloc`, `memory::calloc`, or `memory::realloc`. #[no_mangle] -pub extern "C" fn hlfree(ptr: *mut c_void) { +pub unsafe extern "C" fn free(ptr: *mut c_void) { if !ptr.is_null() { unsafe { let block_start = (ptr as *const Layout).sub(1); @@ -70,11 +115,22 @@ pub extern "C" fn hlfree(ptr: *mut c_void) { } } +/// Changes the size of the memory block pointed to by `ptr` to `size` bytes. If the returned ptr is non-null, +/// any usage of the old memory block is immediately undefined behavior. +/// +/// # Safety +/// `ptr` must be a pointer to a memory block previously allocated by `memory::malloc`, `memory::calloc`, or `memory::realloc`. #[no_mangle] -pub extern "C" fn hlrealloc(ptr: *mut c_void, size: usize) -> *mut c_void { +pub unsafe extern "C" fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void { if ptr.is_null() { // If the pointer is null, treat as a malloc - return hlmalloc(size); + return malloc(size); + } + + if size == 0 { + // If the size is 0, treat as a free and return null + free(ptr); + return ptr::null_mut(); } unsafe { @@ -84,10 +140,11 @@ pub extern "C" fn hlrealloc(ptr: *mut c_void, size: usize) -> *mut c_void { let block_start = (ptr as *const Layout).sub(1); let old_layout = block_start.read(); - let new_layout = Layout::from_size_align(total_new_size, align_of::()).unwrap(); + let new_layout = Layout::from_size_align(total_new_size, MAX_ALIGN).unwrap(); let new_block_start = - alloc::alloc::realloc(block_start as *mut u8, layout, total_new_size) as *mut Layout; + alloc::alloc::realloc(block_start as *mut u8, old_layout, total_new_size) + as *mut Layout; if new_block_start.is_null() { // Realloc failed diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 85e01bbd..a140ef90 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -186,14 +186,14 @@ fn guest_panic() { } #[test] -fn guest_hlmalloc() { +fn guest_malloc() { // this test is rust-only let sbox1: SingleUseSandbox = new_uninit_rust().unwrap().evolve(Noop::default()).unwrap(); let size_to_allocate = 2000; let res = sbox1 .call_guest_function_by_name( - "TestHlMalloc", // uses hlmalloc + "TestMalloc", ReturnType::Int, Some(vec![ParameterValue::Int(size_to_allocate)]), ) @@ -202,7 +202,7 @@ fn guest_hlmalloc() { } #[test] -fn guest_malloc() { +fn guest_allocate_vec() { let sbox1: SingleUseSandbox = new_uninit().unwrap().evolve(Noop::default()).unwrap(); let size_to_allocate = 2000; @@ -220,14 +220,14 @@ fn guest_malloc() { // checks that malloc failures are captured correctly #[test] -fn guest_hlmalloc_abort() { +fn guest_malloc_abort() { let sbox1: SingleUseSandbox = new_uninit_rust().unwrap().evolve(Noop::default()).unwrap(); let size = 20000000; // some big number that should fail when allocated let res = sbox1 .call_guest_function_by_name( - "TestHlMalloc", + "TestMalloc", ReturnType::Int, Some(vec![ParameterValue::Int(size)]), ) diff --git a/src/tests/c_guests/c_simpleguest/main.c b/src/tests/c_guests/c_simpleguest/main.c index 91a5eb5c..d1e688ee 100644 --- a/src/tests/c_guests/c_simpleguest/main.c +++ b/src/tests/c_guests/c_simpleguest/main.c @@ -19,7 +19,7 @@ double echo_double(double d) { return d; } hl_Vec *set_byte_array_to_zero(const hl_FunctionCall* params) { hl_Vec input = params->parameters[0].value.VecBytes; - uint8_t *x = hlmalloc(input.len); + uint8_t *x = malloc(input.len); for (uintptr_t i = 0; i < input.len; i++) { x[i] = 0; } @@ -90,7 +90,7 @@ int small_var(void) { } int call_malloc(int32_t size) { - void *heap_memory = hlmalloc(size); + void *heap_memory = malloc(size); if (NULL == heap_memory) { hl_set_error(hl_ErrorCode_GuestError, "Malloc Failed"); } @@ -99,12 +99,12 @@ int call_malloc(int32_t size) { } int malloc_and_free(int32_t size) { - void *heap_memory = hlmalloc(size); + void *heap_memory = malloc(size); if (NULL == heap_memory) { hl_set_error(hl_ErrorCode_GuestError, "Malloc Failed"); } - hlfree(heap_memory); + free(heap_memory); return size; } diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index c993bc80..685587f7 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -54,7 +54,7 @@ use hyperlight_guest::guest_function_register::register_function; use hyperlight_guest::host_function_call::{ call_host_function, get_host_value_return_as_int, get_host_value_return_as_ulong, }; -use hyperlight_guest::memory::hlmalloc; +use hyperlight_guest::memory::malloc; use hyperlight_guest::{logging, MIN_STACK_ADDRESS}; use log::{error, LevelFilter}; @@ -623,11 +623,9 @@ fn execute_on_heap(_function_call: &FunctionCall) -> Result> { Ok(get_flatbuffer_result_from_string("fail")) } -#[no_mangle] -#[allow(improper_ctypes_definitions)] -pub extern "C" fn test_rust_malloc(function_call: &FunctionCall) -> Result> { +fn test_rust_malloc(function_call: &FunctionCall) -> Result> { if let ParameterValue::Int(code) = function_call.parameters.clone().unwrap()[0].clone() { - let ptr = hlmalloc(code as usize); + let ptr = unsafe { malloc(code as usize) }; Ok(get_flatbuffer_result_from_int(ptr as i32)) } else { Err(HyperlightGuestError::new( @@ -1012,7 +1010,7 @@ pub extern "C" fn hyperlight_main() { register_function(guest_panic_def); let rust_malloc_def = GuestFunctionDefinition::new( - "TestHlMalloc".to_string(), + "TestMalloc".to_string(), Vec::from(&[ParameterType::Int]), ReturnType::Int, test_rust_malloc as i64, From b0ea748900024dfc9095974522b260b937f458ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:33:39 +0000 Subject: [PATCH 17/21] Bump tracing-subscriber from 0.3.18 to 0.3.19 Dependabot couldn't find the original pull request head commit, 62368d31d9e623108b4766dd109aea757fcda4d7. --- Cargo.lock | 4 ++-- src/hyperlight_host/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f793fdc8..2af38c2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2905,9 +2905,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 465b1162..01112b71 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -89,7 +89,7 @@ hyperlight-testing = { workspace = true } env_logger = "0.11.5" tracing-forest = { version = "0.1.6", features = ["uuid", "chrono", "smallvec", "serde", "env-filter"] } tracing = "0.1.41" -tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter"]} +tracing-subscriber = {version = "0.3.19", features = ["std", "env-filter"]} tracing-opentelemetry = "0.28.0" opentelemetry = "0.27.0" opentelemetry-otlp = { version = "0.27.0", features = ["default"] } From f090d9473882e3fe8e4a3efcd9353e79cd8a3b26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 8 Dec 2024 17:28:23 -0500 Subject: [PATCH 18/21] Bump thiserror from 2.0.3 to 2.0.4 (#94) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 2.0.3 to 2.0.4. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/2.0.3...2.0.4) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ src/hyperlight_host/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2af38c2e..f248d1a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1125,7 +1125,7 @@ dependencies = [ "strum", "tempfile", "termcolor", - "thiserror 2.0.3", + "thiserror 2.0.4", "tokio", "tracing", "tracing-chrome", @@ -2583,11 +2583,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.4", ] [[package]] @@ -2603,9 +2603,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" dependencies = [ "proc-macro2", "quote", diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 01112b71..a1aaaada 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -42,7 +42,7 @@ hyperlight-common = { workspace = true, default-features = true } vmm-sys-util = "0.12.1" crossbeam = "0.8.0" crossbeam-channel = "0.5.8" -thiserror = "2.0.0" +thiserror = "2.0.4" prometheus = "0.13.3" strum = { version = "0.26", features = ["derive"] } tempfile = { version = "3.10", optional = true } From 8f0ceb71fb6c7a750ffe4d1e5c44b2a27af1f5d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:15:53 +0000 Subject: [PATCH 19/21] Bump thiserror from 2.0.4 to 2.0.6 (#100) --- Cargo.lock | 12 ++++++------ src/hyperlight_host/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f248d1a8..56833c3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1125,7 +1125,7 @@ dependencies = [ "strum", "tempfile", "termcolor", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", "tracing", "tracing-chrome", @@ -2583,11 +2583,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.6", ] [[package]] @@ -2603,9 +2603,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index a1aaaada..4b2cfca2 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -42,7 +42,7 @@ hyperlight-common = { workspace = true, default-features = true } vmm-sys-util = "0.12.1" crossbeam = "0.8.0" crossbeam-channel = "0.5.8" -thiserror = "2.0.4" +thiserror = "2.0.6" prometheus = "0.13.3" strum = { version = "0.26", features = ["derive"] } tempfile = { version = "3.10", optional = true } From 0bc6ec4792272b30b1ca7676401e97149bc8427b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:16:28 +0000 Subject: [PATCH 20/21] Bump cc from 1.2.2 to 1.2.3 (#101) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56833c3a..d28164c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", From d2d78f6a2e2539270c45041ce3937cf3f296ded5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 23:51:55 +0000 Subject: [PATCH 21/21] Bump anyhow from 1.0.93 to 1.0.94 Bumps [anyhow](https://github.com/dtolnay/anyhow) from 1.0.93 to 1.0.94. - [Release notes](https://github.com/dtolnay/anyhow/releases) - [Commits](https://github.com/dtolnay/anyhow/compare/1.0.93...1.0.94) --- updated-dependencies: - dependency-name: anyhow dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- src/hyperlight_common/Cargo.toml | 2 +- src/hyperlight_guest/Cargo.toml | 2 +- src/hyperlight_host/Cargo.toml | 2 +- src/hyperlight_testing/Cargo.toml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d28164c7..7d5ab033 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arbitrary" diff --git a/src/hyperlight_common/Cargo.toml b/src/hyperlight_common/Cargo.toml index bc5161f8..b0ff2c14 100644 --- a/src/hyperlight_common/Cargo.toml +++ b/src/hyperlight_common/Cargo.toml @@ -16,7 +16,7 @@ workspace = true [dependencies] flatbuffers = { version = "24.3.25", default-features = false } -anyhow = { version = "1.0.72", default-features = false } +anyhow = { version = "1.0.94", default-features = false } log = "0.4.20" tracing = { version = "0.1.41", optional = true } strum = {version = "0.26", default-features = false, features = ["derive"]} diff --git a/src/hyperlight_guest/Cargo.toml b/src/hyperlight_guest/Cargo.toml index 8f218b1f..d70559c9 100644 --- a/src/hyperlight_guest/Cargo.toml +++ b/src/hyperlight_guest/Cargo.toml @@ -19,7 +19,7 @@ printf = [] # compile printf alloca = [] # compile alloca wrapper [dependencies] -anyhow = { version = "1.0.45", default-features = false } +anyhow = { version = "1.0.94", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["alloc"] } buddy_system_allocator = "0.11.0" hyperlight-common = { workspace = true } diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 4b2cfca2..5c0067ae 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -108,7 +108,7 @@ windows = { version = "0.58", features = [ proc-maps = "0.4.0" [build-dependencies] -anyhow = { version = "1.0.75" } +anyhow = { version = "1.0.94" } cfg_aliases = "0.2.1" built = { version = "0.7.0", features = ["chrono", "git2"] } diff --git a/src/hyperlight_testing/Cargo.toml b/src/hyperlight_testing/Cargo.toml index bc12fb00..37fcaf69 100644 --- a/src/hyperlight_testing/Cargo.toml +++ b/src/hyperlight_testing/Cargo.toml @@ -3,7 +3,7 @@ name = "hyperlight-testing" edition = "2021" [dependencies] -anyhow = "1.0.72" +anyhow = "1.0.94" log = "0.4" once_cell = "1.19" tracing = { version = "0.1.41", features = ["log"] }