Initial support for gdb debugging in Hyperlight KVM guest

- The current implementation supports only 4 hardware breakpoints.
- There might be some bugs, testing is still needed

Signed-off-by: Doru Blânzeanu <[email protected]>

src/hyperlight_host/Cargo.toml

@@ -69,6 +69,8 @@ rust-embed = { version = "8.3.0", features = ["debug-embed", "include-exclude",
 sha256 = "1.4.0"
 
 [target.'cfg(unix)'.dependencies]
+gdbstub = "0.7.3"
+gdbstub_arch = "0.3.1"
 seccompiler = { version = "0.4.0", optional = true }
 mshv-bindings = { workspace = true, optional = true }
 mshv-ioctls = { workspace = true, optional = true }
@@ -113,7 +115,7 @@ cfg_aliases = "0.2.1"
 built = { version = "0.7.0", features = ["chrono", "git2"] }
 
 [features]
-default = ["kvm", "mshv", "seccomp"]
+default = ["gdb", "kvm", "mshv", "seccomp"]
 seccomp = ["dep:seccompiler"]
 function_call_metrics = []
 executable_heap = []
@@ -123,6 +125,8 @@ crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
 inprocess = []
+# This enables compilation of gdb stub for easy debug in the guest
+gdb = []
 
 [[bench]]
 name = "benchmarks"

src/hyperlight_host/build.rs

@@ -89,6 +89,7 @@ fn main() -> Result<()> {
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
     // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
     cfg_aliases::cfg_aliases! {
+        gdb: { all(feature = "gdb", debug_assertions, target_os = "linux") },
         kvm: { all(feature = "kvm", target_os = "linux") },
         mshv: { all(feature = "mshv", target_os = "linux") },
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.

src/hyperlight_host/examples/hello-world/main.rs

@@ -16,22 +16,28 @@ limitations under the License.
 
 use std::sync::{Arc, Mutex};
 use std::thread;
+use std::time::Duration;
 
 use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType};
 use hyperlight_host::func::HostFunction0;
+use hyperlight_host::sandbox::SandboxConfiguration;
 use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox;
 use hyperlight_host::sandbox_state::transition::Noop;
 use hyperlight_host::{MultiUseSandbox, UninitializedSandbox};
 
 fn main() -> hyperlight_host::Result<()> {
     // Create an uninitialized sandbox with a guest binary
+    let mut cfg = SandboxConfiguration::default();
+    cfg.set_max_execution_time(Duration::from_secs(0xffff_ffff_ffff));
+    cfg.set_max_initialization_time(Duration::from_secs(0xffff_ffff_ffff));
+
     let mut uninitialized_sandbox = UninitializedSandbox::new(
         hyperlight_host::GuestBinary::FilePath(
             hyperlight_testing::simple_guest_as_string().unwrap(),
         ),
-        None, // default configuration
-        None, // default run options
-        None, // default host print function
+        Some(cfg), // default configuration
+        None,      // default run options
+        None,      // default host print function
     )?;
 
     // Register a host functions

src/hyperlight_host/src/hypervisor/gdb/event_loop.rs

@@ -0,0 +1,103 @@
+use gdbstub::common::Signal;
+use gdbstub::conn::ConnectionExt;
+use gdbstub::stub::run_blocking::{self, WaitForStopReasonError};
+use gdbstub::stub::{DisconnectReason, GdbStub, SingleThreadStopReason};
+
+use super::target::HyperlightKvmSandboxTarget;
+use super::GdbTargetError;
+use crate::hypervisor::gdb::GdbDebug;
+
+pub struct GdbBlockingEventLoop;
+
+impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
+    type Connection = Box<dyn ConnectionExt<Error = std::io::Error>>;
+    type StopReason = SingleThreadStopReason<u64>;
+    type Target = HyperlightKvmSandboxTarget;
+
+    fn wait_for_stop_reason(
+        target: &mut Self::Target,
+        conn: &mut Self::Connection,
+    ) -> Result<
+        run_blocking::Event<Self::StopReason>,
+        run_blocking::WaitForStopReasonError<
+            <Self::Target as gdbstub::target::Target>::Error,
+            <Self::Connection as gdbstub::conn::Connection>::Error,
+        >,
+    > {
+        loop {
+            match target.try_recv() {
+                Ok(_) => {
+                    target.pause_vcpu();
+
+                    // Get the stop reason from the target
+                    let stop_reason = target
+                        .get_stop_reason()
+                        .map_err(WaitForStopReasonError::Target)?;
+
+                    // Resume execution if unknown reason for stop
+                    let Some(stop_response) = stop_reason else {
+                        target
+                            .resume_vcpu()
+                            .map_err(WaitForStopReasonError::Target)?;
+
+                        continue;
+                    };
+
+                    return Ok(run_blocking::Event::TargetStopped(stop_response));
+                }
+                Err(crossbeam_channel::TryRecvError::Empty) => (),
+                Err(_) => {
+                    return Err(run_blocking::WaitForStopReasonError::Target(
+                        GdbTargetError::GdbQueueError,
+                    ));
+                }
+            }
+
+            if conn.peek().map(|b| b.is_some()).unwrap_or(false) {
+                let byte = conn
+                    .read()
+                    .map_err(run_blocking::WaitForStopReasonError::Connection)?;
+
+                return Ok(run_blocking::Event::IncomingData(byte));
+            }
+        }
+    }
+
+    fn on_interrupt(
+        target: &mut Self::Target,
+    ) -> Result<Option<Self::StopReason>, <Self::Target as gdbstub::target::Target>::Error> {
+        target.pause_vcpu();
+
+        Ok(Some(SingleThreadStopReason::SignalWithThread {
+            tid: (),
+            signal: Signal::SIGINT,
+        }))
+    }
+}
+
+pub fn event_loop_thread(
+    debugger: GdbStub<HyperlightKvmSandboxTarget, Box<dyn ConnectionExt<Error = std::io::Error>>>,
+    mut target: HyperlightKvmSandboxTarget,
+) {
+    match debugger.run_blocking::<GdbBlockingEventLoop>(&mut target) {
+        Ok(disconnect_reason) => match disconnect_reason {
+            DisconnectReason::Disconnect => log::info!("Gdb client disconnected"),
+            DisconnectReason::TargetExited(code) => {
+                log::info!("Gdb target exited with code {}", code)
+            }
+            DisconnectReason::TargetTerminated(sig) => {
+                log::info!("Gdb target terminated with signale {}", sig)
+            }
+            DisconnectReason::Kill => log::info!("Gdb sent a kill command"),
+        },
+        Err(e) => {
+            if e.is_target_error() {
+                log::error!("Target encountered a fatal error: {e:?}");
+            } else if e.is_connection_error() {
+                log::error!("connection error: {:?}", e);
+            } else {
+                log::error!("gdbstub got a fatal error {:?}", e);
+            }
+        }
+    }
+}

src/hyperlight_host/src/hypervisor/gdb/mod.rs

@@ -0,0 +1,135 @@
+mod event_loop;
+pub mod target;
+
+use std::net::TcpListener;
+use std::thread;
+
+use crossbeam_channel::{Receiver, Sender, TryRecvError};
+use event_loop::event_loop_thread;
+use gdbstub::conn::ConnectionExt;
+use gdbstub::stub::GdbStub;
+use target::HyperlightKvmSandboxTarget;
+
+#[allow(dead_code)]
+#[derive(Debug)]
+pub enum GdbTargetError {
+    /// Error to set breakpoint
+    GdbBindError,
+    GdbBreakpointError,
+    GdbInstructionPointerError,
+    GdbListenerError,
+    GdbQueueError,
+    GdbReadRegistersError,
+    GdbReceiveMsgError,
+    GdbResumeError,
+    GdbSendMsgError,
+    GdbSetGuestDebugError,
+    GdbSpawnThreadError,
+    GdbStepError,
+    GdbTranslateGvaError,
+    GdbUnexpectedMessageError,
+    GdbWriteRegistersError,
+}
+
+/// Trait that provides common communication methods for targets
+pub trait GdbDebug {
+    /// Sends a message to the Hypervisor
+    fn send(&self, ev: DebugMessage) -> Result<(), GdbTargetError>;
+    /// Waits for a message from the Hypervisor
+    fn recv(&self) -> Result<DebugMessage, GdbTargetError>;
+    /// Checks for a pending message from the Hypervisor
+    fn try_recv(&self) -> Result<DebugMessage, TryRecvError>;
+}
+
+/// Event sent to the VCPU execution loop
+#[derive(Debug)]
+pub enum DebugMessage {
+    /// VCPU stopped in debug
+    VcpuStoppedEv,
+    /// Resume VCPU execution
+    VcpuResumeEv,
+    /// Response ok
+    VcpuOk,
+    /// Response error
+    VcpuErr,
+}
+
+/// Type that takes care of communication between Hypervisor and Gdb
+pub struct GdbConnection {
+    /// Transmit channel
+    tx: Sender<DebugMessage>,
+    /// Receive channel
+    rx: Receiver<DebugMessage>,
+}
+
+impl GdbConnection {
+    pub fn new_pair() -> (Self, Self) {
+        let (hyp_tx, gdb_rx) = crossbeam_channel::unbounded();
+        let (gdb_tx, hyp_rx) = crossbeam_channel::unbounded();
+
+        let gdb_conn = GdbConnection {
+            tx: gdb_tx,
+            rx: gdb_rx,
+        };
+
+        let hyp_conn = GdbConnection {
+            tx: hyp_tx,
+            rx: hyp_rx,
+        };
+
+        (gdb_conn, hyp_conn)
+    }
+
+    /// Sends message over the transmit channel
+    pub fn send(&self, msg: DebugMessage) -> Result<(), GdbTargetError> {
+        self.tx
+            .send(msg)
+            .map_err(|_| GdbTargetError::GdbSendMsgError)
+    }
+
+    /// Waits for a message over the receive channel
+    pub fn recv(&self) -> Result<DebugMessage, GdbTargetError> {
+        self.rx
+            .recv()
+            .map_err(|_| GdbTargetError::GdbReceiveMsgError)
+    }
+
+    /// Checks whether there's a message waiting on the receive channel
+    pub fn try_recv(&self) -> Result<DebugMessage, TryRecvError> {
+        self.rx.try_recv()
+    }
+}
+
+/// Creates a thread that handles gdb protocol
+pub fn create_gdb_thread(mut target: HyperlightKvmSandboxTarget) -> Result<(), GdbTargetError> {
+    // TODO: Address multiple sandboxes scenario
+    let socket = format!("localhost:{}", 8081);
+
+    log::info!("Listening on {:?}", socket);
+    let listener = TcpListener::bind(socket).map_err(|_| GdbTargetError::GdbBindError)?;
+
+    log::info!("Starting GDB thread");
+    let _handle = thread::Builder::new()
+        .name("GDB handler".to_string())
+        .spawn(move || -> Result<(), GdbTargetError> {
+            log::info!("Waiting for GDB connection ... ");
+            let (conn, _) = listener
+                .accept()
+                .map_err(|_| GdbTargetError::GdbListenerError)?;
+            let conn: Box<dyn ConnectionExt<Error = std::io::Error>> = Box::new(conn);
+            let debugger = GdbStub::new(conn);
+
+            if let DebugMessage::VcpuStoppedEv = target.recv()? {
+                target.pause_vcpu();
+
+                event_loop_thread(debugger, target);
+
+                Ok(())
+            } else {
+                Err(GdbTargetError::GdbUnexpectedMessageError)
+            }
+        })
+        .map_err(|_| GdbTargetError::GdbSpawnThreadError)?;
+
+    Ok(())
+}