feat(aarch64): add GDB debugging for Linux KVM

.github/workflows/dep_build_test.yml

@@ -129,6 +129,17 @@ jobs:
       - name: Build
         run: just build ${{ inputs.config }}
 
+      - name: Run ARM64 KVM Gdb tests
+        if: inputs.arch == 'arm64' && inputs.hypervisor == 'kvm' && inputs.config == 'debug'
+        env:
+          RUST_LOG: debug
+        run: |
+          if ! command -v gdb > /dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y --no-install-recommends gdb
+          fi
+          just test-rust-gdb-debugging debug
+
       - name: Run Rust tests
         run: |
           # with default features

CHANGELOG.md

@@ -5,7 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## [Prerelease] - Unreleased
 
 ### Added
+* ARM64 Linux/KVM guest debugging supports software and hardware breakpoints, register and memory access, and hardware single-step.
 * `Sandbox::snapshot_with_options` takes a snapshot with `SnapshotOptions`. `SnapshotOptions::scratch_size` gives the sandbox, and later restores of the snapshot, a different scratch size. A guest can boot with the scratch its initialisation needs and run guest function calls from a snapshot with only the scratch they need, so each restore costs less. It can also give a guest more scratch than it booted with.
+* Support register, memory, continue, hardware single-step, interrupt, and
+  hardware and software breakpoint GDB debugging for Windows ARM64 guests.
 
 ### Changed
 * Core dumps no longer record the guest binary path. The process name and command line are always `<unknown>`, as they already were for sandboxes created from snapshots by @ludfjig in https://github.com/hyperlight-dev/hyperlight/pull/1975
@@ -40,6 +43,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Fixed
 
+* GDB memory reads and writes translate each guest page on hosts with larger pages.
+* ARM64 GDB snapshot recovery handles cancellation and crash detach. Snapshot checkpointing rejects software breakpoints before replacing memory.
+* WHP ARM64 cache synchronization code is read-only to guests.
 * Guest libc accepts null buffers for zero-length writes.
 * Fix calls to guest functions with more than 11 parameters. Registration accepts up to 32, but every call with more than 11 failed.
 

Justfile

@@ -331,7 +331,7 @@ test-no-surrogate target=default-target:
 # runs tests that exercise gdb debugging
 test-rust-gdb-debugging target=default-target features="":
     {{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }} --example guest-debugging {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }}
-    {{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }} -p hyperlight-host --lib {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- hypervisor::gdb::
+    {{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }} -p hyperlight-host --lib {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- hypervisor::gdb:: hypervisor::hyperlight_vm::debug::tests:: gdb_recovery_tests:: hypervisor::virtual_machine::whp::aarch64::debug_tests::
 
 # rust test for crashdump
 test-rust-crashdump target=default-target features="":

docs/how-to-debug-a-hyperlight-guest.md

@@ -7,15 +7,27 @@ to start listening for a gdb connection.
 ## Supported features
 
 The Hyperlight `gdb` feature enables guest debugging to:
-   - stop at an entry point breakpoint which is automatically set by Hyperlight
-   - add and remove HW breakpoints (maximum 4 set breakpoints at a time)
+   - stop before the first vCPU run
    - add and remove SW breakpoints
    - read and write registers
    - read and write addresses
-   - step/continue
+   - continue
+   - single-step
    - get code offset from target
    - stop when a crash occurs and only allow read access to the guest memory and registers
 
+On x86_64, gdb can also add up to four hardware breakpoints.
+ARM64 on Windows/WHP and Linux/KVM supports two hardware breakpoints,
+four-byte `BRK` software breakpoints, and hardware single-step. KVM reports
+debug exceptions through `KVM_EXIT_DEBUG`. Guest exception vectors forward debug
+exceptions to WHP through HVC. Windows ARM64 guest binaries must retain the
+`hyperlight_debug_vectors` symbol so the host can install the vectors before
+the initial debug stop. Watchpoints are not advertised.
+
+ARM64 snapshots reject installed software breakpoints, including those added
+during checkpointing. Restoring a poisoned sandbox discards its software
+breakpoint state and restores clean snapshot memory.
+
 ## Expected behavior
 
 Below is a list describing some cases of expected behavior from a gdb debug 
@@ -170,7 +182,7 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or
       |                        │          |               create_gdb_thread           |                                 |      │
       |                        │          |◄─────────────────────────────────────────┌─┐         vcpu stopped          ┌─┐     │
       |    attach              │         ┌─┐                                         │ │◄──────────────────────────────┴─┘     │
-     ┌─┐───────────────────────┼────────►│ │                                         │ │     entrypoint breakpoint      |      │
+     ┌─┐───────────────────────┼────────►│ │                                         │ │       initial debug stop       |      │
      │ │   attach response     │         │ │                                         │ │                                |      │
      │ │◄──────────────────────┼─────────│ │                                         │ │                                |      │
      │ │                       │         │ │                                         │ │                                |      │

src/hyperlight_common/src/arch/aarch64/exn.rs

@@ -3,6 +3,9 @@
 
 use crate::vmem::bits;
 
+/// HVC immediate for forwarding guest exceptions to the WHP debugger.
+pub const DEBUG_EXCEPTION_HVC: u16 = 0x485a;
+
 const ESR_EC_UNKNOWN: u64 = 0b000000; // udf generates this
 const ESR_EC_DATA_ABORT_LOWER_EL: u64 = 0b100100;
 const ESR_EC_DATA_ABORT_SAME_EL: u64 = 0b100101;

src/hyperlight_guest_bin/src/arch/aarch64/exception/entry.rs

@@ -104,10 +104,24 @@ context_restore:\n
     q_off = const offset_of!(ExceptionContext, q),
 );
 
+// Only synchronous exceptions have a syndrome for debug forwarding.
+// The host resumes debug events at ERET and other synchronous exceptions at the prologue.
 macro_rules! vbar_entry {
     ($et:literal, $ef:literal) => {
         concat!(
             "
+            .if {ExceptionType_",
+            $et,
+            "} == {ExceptionType_Synchronous}\n
+            stp x0, x1, [sp, #-16]!\n
+            mrs x0, mdscr_el1\n
+            tbz x0, #13, 1f\n
+            ldp x0, x1, [sp], #16\n
+            hvc #{debug_hvc}\n
+            eret\n
+        1:\n
+            ldp x0, x1, [sp], #16\n
+            .endif\n
             sub sp, sp, #{ctx_size}\n
             str x30, [sp, #{x30_off}]\n
             bl context_save\n
@@ -129,6 +143,8 @@ macro_rules! vbar_entry {
 global_asm!("
 .balign 0x800\n
 .global vbar\n
+.global hyperlight_debug_vectors\n
+hyperlight_debug_vectors:\n
 vbar:\n",
     vbar_entry!("Synchronous", "CurrentSP0"),
     vbar_entry!("IRQ", "CurrentSP0"),
@@ -148,6 +164,7 @@ vbar:\n",
     vbar_entry!("SError", "LowerAArch32"),
     ctx_size = const size_of::<ExceptionContext>(),
     x30_off = const offset_of!(ExceptionContext, x) + 15 * 0x010,
+    debug_hvc = const hyperlight_common::arch::exn::DEBUG_EXCEPTION_HVC,
     handler = sym super::handle::handle_exception,
     ExceptionType_Synchronous = const ExceptionType::Synchronous as u64,
     ExceptionType_IRQ = const ExceptionType::Irq as u64,

src/hyperlight_host/build.rs

@@ -150,7 +150,8 @@ fn main() -> Result<()> {
     // Essentially the kvm and mshv3 features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
     // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv3")] in the codebase.
     cfg_aliases::cfg_aliases! {
-        gdb: { all(feature = "gdb", debug_assertions, target_arch = "x86_64") },
+        gdb_platform: { any(target_arch = "x86_64", all(target_arch = "aarch64", any(target_os = "windows", all(target_os = "linux", feature = "kvm")))) },
+        gdb: { all(feature = "gdb", debug_assertions, gdb_platform) },
         kvm: { all(feature = "kvm", target_os = "linux") },
         mshv3: { all(feature = "mshv3", target_os = "linux") },
         hvf: { all(feature = "hvf", target_os = "macos") },

src/hyperlight_host/examples/guest-debugging/main.rs

@@ -34,10 +34,12 @@ fn main() -> hyperlight_host::Result<()> {
             .host_function("Sleep5Secs", sleep_5_secs)
             .build()?;
 
-    // Call guest function
+    #[cfg(target_arch = "x86_64")]
     multi_use_sandbox_dbg
         .call::<()>("UseSSE2Registers", ())
         .unwrap();
+    #[cfg(target_arch = "aarch64")]
+    multi_use_sandbox_dbg.call::<()>("NoOp", ()).unwrap();
 
     let message =
         "Hello, World! I am executing inside of a VM with debugger attached :)\n".to_string();
@@ -291,6 +293,161 @@ mod tests {
     }
 
     #[test]
+    #[cfg(target_arch = "aarch64")]
+    #[serial]
+    fn test_gdb_continue_over_sw_breakpoint() {
+        let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-continue");
+        let cmd = format!(
+            "file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
+                target remote :8080
+                set pagination off
+                set logging file {out_file_path}
+                set logging enabled on
+                break simpleguest::no_op
+                commands 1
+                    continue
+                end
+                break simpleguest::print_output
+                commands 2
+                    echo Continued over ARM64 software breakpoint\\n
+                    set logging enabled off
+                    detach
+                    quit
+                end
+                continue
+            "
+        );
+        #[cfg(windows)]
+        let cmd = format!("set osabi none\n{cmd}");
+        let checker =
+            |contents: String| contents.contains("Continued over ARM64 software breakpoint");
+        let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);
+
+        cleanup(&out_file_path, &cmd_file_path);
+        assert!(result.is_ok(), "{}", result.unwrap_err());
+    }
+
+    #[test]
+    #[cfg(target_arch = "aarch64")]
+    #[serial]
+    fn test_gdb_hardware_breakpoint_and_step() {
+        let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-hardware");
+        let cmd = format!(
+            "set osabi none
+                file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
+                target remote :8080
+                set pagination off
+                set logging file {out_file_path}
+                set logging enabled on
+                set $entry_pc = $pc
+                stepi
+                if $pc != $entry_pc + 4
+                    echo Entry step did not execute one instruction\\n
+                    quit 1
+                end
+                hbreak simpleguest::no_op
+                continue
+                set $return_pc = $x30
+                stepi
+                if $pc != $return_pc
+                    echo Hardware RET step missed its destination\\n
+                    quit 1
+                end
+                echo Hardware breakpoint and entry/RET stepping passed\\n
+                set logging enabled off
+                detach
+                quit
+            "
+        );
+        let checker = |contents: String| {
+            contents.contains("Hardware breakpoint and entry/RET stepping passed")
+        };
+        let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);
+        cleanup(&out_file_path, &cmd_file_path);
+        assert!(result.is_ok(), "{}", result.unwrap_err());
+    }
+
+    #[test]
+    #[cfg(target_arch = "aarch64")]
+    #[serial]
+    fn test_gdb_code_writes_and_self_loop() {
+        let (out_file_path, cmd_file_path, manifest_dir) =
+            gdb_test_paths("gdb-aarch64-code-writes");
+        let cmd = format!(
+            "set osabi none
+                file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
+                target remote :8080
+                set pagination off
+                set logging file {out_file_path}
+                set logging enabled on
+                hbreak simpleguest::no_op
+                continue
+                delete 1
+                set language c
+                set $loop_pc = $pc
+                set $instruction = *(unsigned int *)$pc
+                set *(unsigned int *)$pc = 0x14000000
+                stepi
+                if $pc != $loop_pc
+                    echo Self-loop step missed its destination\\n
+                    quit 1
+                end
+                set *(unsigned int *)$pc = $instruction
+                set $return_pc = $x30
+                stepi
+                if $pc != $return_pc
+                    echo Restored RET missed its destination\\n
+                    quit 1
+                end
+                echo Code writes and self-loop stepping passed\\n
+                set logging enabled off
+                detach
+                quit
+            "
+        );
+        let checker =
+            |contents: String| contents.contains("Code writes and self-loop stepping passed");
+        let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);
+        cleanup(&out_file_path, &cmd_file_path);
+        assert!(result.is_ok(), "{}", result.unwrap_err());
+    }
+
+    #[test]
+    #[cfg(target_arch = "aarch64")]
+    #[serial]
+    fn test_gdb_single_step() {
+        let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-step");
+        let cmd = format!(
+            "file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
+                target remote :8080
+                set pagination off
+                set logging file {out_file_path}
+                set logging enabled on
+                break simpleguest::no_op
+                stepi
+                echo Stepped over ARM64 linear instruction\\n
+                continue
+                stepi
+                echo Stepped over ARM64 RET\\n
+                set logging enabled off
+                detach
+                quit
+            "
+        );
+        #[cfg(windows)]
+        let cmd = format!("set osabi none\n{cmd}");
+        let checker = |contents: String| {
+            contents.contains("Stepped over ARM64 linear instruction")
+                && contents.contains("Stepped over ARM64 RET")
+        };
+        let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);
+
+        cleanup(&out_file_path, &cmd_file_path);
+        assert!(result.is_ok(), "{}", result.unwrap_err());
+    }
+
+    #[test]
+    #[cfg(target_arch = "x86_64")]
     #[serial]
     fn test_gdb_sse_check() {
         let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-sse");