Repository: microsoft/openvmm
Pull request #1074: tmk: add aarch64 support
Author: login: jstarks
Created: 2025-03-19T23:31:17Z | Merged: 2025-03-20T17:02:33Z

Add aarch64 support to `tmk_vmm` and `simple_tmk`.

As part of this, I defined a custom target to use when building `simple_tmk` so that it's relatively straightforward to build this on non-Linux platforms. It looks something like:

```
RUSTC_BOOTSTRAP=1 cargo build -p simple_tmk --config openhcl/minimal_rt/aarch64-config.toml
```

On Linux you can still just build with the `aarch64-unknown-linux-musl` target, like we do for `openhcl_boot`, so `RUSTC_BOOTSTRAP=1` remains optional.


Diff:
diff --git a/build_support/macos/sign_and_run.sh b/build_support/macos/sign_and_run.sh
index d955db9c8e..41e81eb942 100755
--- a/build_support/macos/sign_and_run.sh
+++ b/build_support/macos/sign_and_run.sh
@@ -3,7 +3,7 @@
 set -e
 
 case "$CARGO_PKG_NAME" in
-    "openvmm")
+    "openvmm"|"tmk_vmm")
         # Add entitlements for using hypervisor framework.
         entitlements=$(dirname "$0")/entitlements.xml
         codesign --entitlements "$entitlements" -f -s - "$1" > /dev/null
diff --git a/openhcl/minimal_rt/aarch64-config.toml b/openhcl/minimal_rt/aarch64-config.toml
new file mode 100644
index 0000000000..e7af843812
--- /dev/null
+++ b/openhcl/minimal_rt/aarch64-config.toml
@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Config file to use to build minimal_rt binaries for aarch64.
+#
+# The dependency on build-std requires setting RUSTC_BOOTSTRAP=1, or an
+# unstable toolchain. Consider using the aarch64-unknown-linux-musl
+# target if you want to avoid this.
+
+[build]
+# Custom target needed to enable static PIE support, which is not
+# supported by aarch64-minimal.
+target = "minimal_rt/aarch64-minimal_rt-none.json"
+
+[unstable]
+build-std = ["core"]
+
+[env]
+MINIMAL_RT_BUILD = "1"
diff --git a/openhcl/minimal_rt/aarch64-minimal_rt-none.json b/openhcl/minimal_rt/aarch64-minimal_rt-none.json
new file mode 100644
index 0000000000..c311b65732
--- /dev/null
+++ b/openhcl/minimal_rt/aarch64-minimal_rt-none.json
@@ -0,0 +1,18 @@
+{
+  "arch": "aarch64",
+  "crt-objects-fallback": "false",
+  "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32",
+  "disable-redzone": true,
+  "features": "+v8a,+strict-align,+neon,+fp-armv8",
+  "linker": "rust-lld",
+  "linker-flavor": "gnu-lld",
+  "llvm-target": "aarch64-unknown-none",
+  "max-atomic-width": 128,
+  "panic-strategy": "abort",
+  "position-independent-executables": true,
+  "static-position-independent-executables": true,
+  "stack-probes": {
+    "kind": "inline"
+  },
+  "target-pointer-width": "64"
+}
diff --git a/openhcl/minimal_rt/x86_64-config.toml b/openhcl/minimal_rt/x86_64-config.toml
new file mode 100644
index 0000000000..62ab2614ee
--- /dev/null
+++ b/openhcl/minimal_rt/x86_64-config.toml
@@ -0,0 +1,10 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Config file to use to build minimal_rt binaries for x86_64.
+
+[build]
+target = "x86_64-unknown-none"
+
+[env]
+MINIMAL_RT_BUILD = "1"
diff --git a/openhcl/minimal_rt_build/src/lib.rs b/openhcl/minimal_rt_build/src/lib.rs
index 2596ee1914..7630bdd777 100644
--- a/openhcl/minimal_rt_build/src/lib.rs
+++ b/openhcl/minimal_rt_build/src/lib.rs
@@ -29,7 +29,13 @@ pub fn init() {
         return;
     }
 
-    let supported_triple;
+    let triple = std::env::var("TARGET").unwrap();
+    let unsupported = |supported_triple| {
+        panic!(
+            "build is only supported with the {} target, not {}, clear MINIMAL_RT_BUILD",
+            supported_triple, triple
+        );
+    };
     // xtask-fmt allow-target-arch sys-crate
     match std::env::var("CARGO_CFG_TARGET_ARCH").unwrap().as_str() {
         "x86_64" => {
@@ -40,28 +46,41 @@ pub fn init() {
             // various places).
             //
             // No special linker flags are needed.
-            supported_triple = "x86_64-unknown-none";
+            if triple != "x86_64-unknown-none" {
+                unsupported("x86_64-unknown-none");
+            }
         }
         "aarch64" => {
-            // This is the supported triple for aarch64. The -none target does
-            // not enable PIE, which we require so that the boot loader can run
-            // anywhere in PA space.
-            supported_triple = "aarch64-unknown-linux-musl";
-            // Don't include the _start entry point.
-            println!("cargo:rustc-link-arg=-nostartfiles");
-            // Make the executable relocatable.
-            println!("cargo:rustc-link-arg=-static-pie");
+            match triple.as_str() {
+                "aarch64-minimal_rt-none" => {
+                    // This is a custom target, defined via
+                    // aarch64-minimal_rt-none.json. So, it requires
+                    // RUSTC_BOOTSTRAP=1 or an unstable toolchain in order to
+                    // use `-Zbuild-std`.
+                    //
+                    // It is aarch64-unknown-none with support for static PIE
+                    // binaries, which we need to support loading the image
+                    // anywhere in PA space.
+                }
+                "aarch64-unknown-linux-musl" => {
+                    // This target works (it supports static PIE binaries) and
+                    // does not require an unstable toolchain, but it is
+                    // difficult to build from non-Linux host environments.
+                    //
+                    // This does require some tweaks to the linker flags.
+                    //
+                    // Don't include the _start entry point.
+                    println!("cargo:rustc-link-arg=-nostartfiles");
+                    // Make the executable relocatable.
+                    println!("cargo:rustc-link-arg=-static-pie");
+                }
+                _ => {
+                    unsupported("aarch64-unknown-linux-musl");
+                }
+            }
         }
         arch => panic!("unsupported arch {arch}"),
     }
 
-    let triple = std::env::var("TARGET").unwrap();
-    if triple != supported_triple {
-        panic!(
-            "build is only supported with the {} target, clear MINIMAL_RT_BUILD",
-            supported_triple
-        );
-    }
-
     println!("cargo:rustc-cfg=minimal_rt");
 }
diff --git a/tmk/simple_tmk/src/main.rs b/tmk/simple_tmk/src/main.rs
index 9b1ed735c0..f2c7f1cb54 100644
--- a/tmk/simple_tmk/src/main.rs
+++ b/tmk/simple_tmk/src/main.rs
@@ -5,7 +5,7 @@
 
 #![cfg_attr(minimal_rt, no_std, no_main)]
 
-#[cfg(all(minimal_rt, target_arch = "x86_64"))]
+#[cfg(minimal_rt)]
 mod tmk;
 
 #[cfg(not(minimal_rt))]
diff --git a/tmk/simple_tmk/src/tmk.rs b/tmk/simple_tmk/src/tmk.rs
index 7ba9df49ec..1e603a81e4 100644
--- a/tmk/simple_tmk/src/tmk.rs
+++ b/tmk/simple_tmk/src/tmk.rs
@@ -4,7 +4,9 @@
 // UNSAFETY: needed to write low-level TMK code.
 #![expect(unsafe_code)]
 
-use core::arch::global_asm;
+mod aarch64;
+mod x86;
+
 use core::marker::PhantomData;
 
 #[repr(C)]
@@ -21,37 +23,6 @@ impl<'a> Str<'a> {
     }
 }
 
-static HELLO_WORLD: Str<'static> = Str::new("hello world");
-
-global_asm! {
-    ".globl _start",
-    "_start:",
-    "lea rsp, {STACK_SIZE} + {stack}[rip]",
-    "lea rdx, _DYNAMIC[rip]",
-    "lea rdi, __ehdr_start[rip]",
-    "mov rsi, rdi",
-    "call {relocate}",
-    "mov rcx, {TMK_ADDRESS_LOG}",
-    "lea rdx, {hello_world}[rip]",
-    "mov qword ptr [rcx], rdx",
-    "mov rcx, {TMK_ADDRESS_COMPLETE}",
-    "mov byte ptr [rcx], 0",
-    "2: hlt",
-    "jmp 2b",
-    hello_world = sym HELLO_WORLD,
-    relocate = sym minimal_rt::reloc::relocate,
-    stack = sym STACK,
-    STACK_SIZE = const STACK_SIZE - 16,
-    TMK_ADDRESS_LOG = const tmk_protocol::TMK_ADDRESS_LOG,
-    TMK_ADDRESS_COMPLETE = const tmk_protocol::TMK_ADDRESS_COMPLETE,
-}
-
-const STACK_SIZE: usize = 4096;
-#[repr(C, align(16))]
-struct Stack([u8; STACK_SIZE]);
-#[unsafe(no_mangle)]
-static mut STACK: Stack = Stack([0; STACK_SIZE]);
-
 #[panic_handler]
 fn panic(_info: &core::panic::PanicInfo<'_>) -> ! {
     minimal_rt::arch::fault();
diff --git a/tmk/simple_tmk/src/tmk/aarch64.rs b/tmk/simple_tmk/src/tmk/aarch64.rs
new file mode 100644
index 0000000000..46e6c14d67
--- /dev/null
+++ b/tmk/simple_tmk/src/tmk/aarch64.rs
@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Aarch64 entry point and support.
+
+#![cfg(target_arch = "aarch64")]
+
+use super::Str;
+use core::arch::global_asm;
+
+static HELLO_WORLD: Str<'static> = Str::new("hello world");
+
+global_asm! {
+    ".weak _DYNAMIC",
+    ".hidden _DYNAMIC",
+    ".globl _start",
+    "_start:",
+    "adrp x1, {stack}",
+    "add x1, x1, :lo12:{stack}",
+    "add x1, x1, {STACK_SIZE}",
+    "mov sp, x1",
+
+    // Enable the FPU.
+    "mrs     x0, CPACR_EL1",
+    "orr     x0, x0, #(3 << 20)",
+    "orr     x0, x0, #(3 << 16)",
+    "msr     CPACR_EL1, x0",
+    "isb",
+
+    "adrp x0, __ehdr_start",
+    "add x0, x0, :lo12:__ehdr_start",
+    "mov x1, x0",
+    "adrp x2, _DYNAMIC",
+    "add x2, x2, :lo12:_DYNAMIC",
+    "bl {relocate}",
+
+    "ldr x0, ={TMK_ADDRESS_LOG}",
+    "adrp x1, {hello_world}",
+    "add x1, x1, :lo12:{hello_world}",
+    "str x1, [x0]",
+
+    "ldr x0, ={TMK_ADDRESS_COMPLETE}",
+    "mov x1, 0",
+    "str x1, [x0]",
+    "1: wfi",
+    "b 1b",
+    hello_world = sym HELLO_WORLD,
+    relocate = sym minimal_rt::reloc::relocate,
+    stack = sym STACK,
+    STACK_SIZE = const STACK_SIZE,
+    TMK_ADDRESS_LOG = const tmk_protocol::TMK_ADDRESS_LOG,
+    TMK_ADDRESS_COMPLETE = const tmk_protocol::TMK_ADDRESS_COMPLETE,
+}
+
+const STACK_SIZE: usize = 16384;
+#[repr(C, align(16))]
+struct Stack([u8; STACK_SIZE]);
+#[unsafe(no_mangle)]
+static mut STACK: Stack = Stack([0; STACK_SIZE]);
diff --git a/tmk/simple_tmk/src/tmk/x86.rs b/tmk/simple_tmk/src/tmk/x86.rs
new file mode 100644
index 0000000000..6820385750
--- /dev/null
+++ b/tmk/simple_tmk/src/tmk/x86.rs
@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! x86_64 entry point and support.
+
+#![cfg(target_arch = "x86_64")]
+
+use super::Str;
+use core::arch::global_asm;
+
+static HELLO_WORLD: Str<'static> = Str::new("hello world");
+
+global_asm! {
+    ".globl _start",
+    "_start:",
+    "lea rsp, {STACK_SIZE} + {stack}[rip]",
+    "lea rdx, _DYNAMIC[rip]",
+    "lea rdi, __ehdr_start[rip]",
+    "mov rsi, rdi",
+    "call {relocate}",
+    "mov rcx, {TMK_ADDRESS_LOG}",
+    "lea rdx, {hello_world}[rip]",
+    "mov qword ptr [rcx], rdx",
+    "mov rcx, {TMK_ADDRESS_COMPLETE}",
+    "mov byte ptr [rcx], 0",
+    "2: hlt",
+    "jmp 2b",
+    hello_world = sym HELLO_WORLD,
+    relocate = sym minimal_rt::reloc::relocate,
+    stack = sym STACK,
+    STACK_SIZE = const STACK_SIZE,
+    TMK_ADDRESS_LOG = const tmk_protocol::TMK_ADDRESS_LOG,
+    TMK_ADDRESS_COMPLETE = const tmk_protocol::TMK_ADDRESS_COMPLETE,
+}
+
+const STACK_SIZE: usize = 4096;
+#[repr(C, align(16))]
+struct Stack([u8; STACK_SIZE]);
+#[unsafe(no_mangle)]
+static mut STACK: Stack = Stack([0; STACK_SIZE]);
diff --git a/tmk/tmk_vmm/Cargo.toml b/tmk/tmk_vmm/Cargo.toml
index 62149a7499..0a01c0c56b 100644
--- a/tmk/tmk_vmm/Cargo.toml
+++ b/tmk/tmk_vmm/Cargo.toml
@@ -28,7 +28,7 @@ clap = { workspace = true, features = ["derive"] }
 fs-err.workspace = true
 futures.workspace = true
 tracing.workspace = true
-tracing-subscriber.workspace = true
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
 
 [target.'cfg(windows)'.dependencies]
 virt_whp.workspace = true
diff --git a/tmk/tmk_vmm/src/load.rs b/tmk/tmk_vmm/src/load.rs
index 3611d585ef..5ba22999c0 100644
--- a/tmk/tmk_vmm/src/load.rs
+++ b/tmk/tmk_vmm/src/load.rs
@@ -7,12 +7,15 @@ use anyhow::Context as _;
 use fs_err::File;
 use guestmem::GuestMemory;
 use hvdef::Vtl;
+use loader::importer::GuestArch;
 use loader::importer::ImageLoad;
 use loader::importer::X86Register;
+use std::fmt::Debug;
 use std::sync::Arc;
 use virt::VpIndex;
 use vm_topology::memory::MemoryLayout;
 use vm_topology::processor::ProcessorTopology;
+use vm_topology::processor::aarch64::Aarch64Topology;
 use vm_topology::processor::x86::X86Topology;
 
 /// Loads a TMK, returning the initial registers for the BSP.
@@ -25,16 +28,7 @@ pub fn load_x86(
     tmk: &File,
 ) -> anyhow::Result<Arc<virt::x86::X86InitialRegs>> {
     let mut loader = vm_loader::Loader::new(guest_memory.clone(), memory_layout, Vtl::Vtl0);
-    let load_info = loader::elf::load_static_elf(
-        &mut loader,
-        &mut &*tmk,
-        0,
-        0x200000,
-        false,
-        loader::importer::BootPageAcceptance::Exclusive,
-        "tmk",
-    )
-    .context("failed to load tmk")?;
+    let load_info = load_binary(&mut loader, tmk)?;
 
     let page_table_base = load_info.next_available_address;
     let page_tables = page_table::x64::build_page_tables_64(
@@ -80,3 +74,46 @@ pub fn load_x86(
     );
     Ok(regs)
 }
+
+#[cfg_attr(not(guest_arch = "aarch64"), expect(dead_code))]
+pub fn load_aarch64(
+    memory_layout: &MemoryLayout,
+    guest_memory: &GuestMemory,
+    processor_topology: &ProcessorTopology<Aarch64Topology>,
+    caps: &virt::aarch64::Aarch64PartitionCapabilities,
+    tmk: &File,
+) -> anyhow::Result<Arc<virt::aarch64::Aarch64InitialRegs>> {
+    let mut loader = vm_loader::Loader::new(guest_memory.clone(), memory_layout, Vtl::Vtl0);
+    let load_info = load_binary(&mut loader, tmk)?;
+
+    let mut import_reg = |reg| {
+        loader
+            .import_vp_register(reg)
+            .context("failed to set register")
+    };
+
+    import_reg(loader::importer::Aarch64Register::Pc(load_info.entrypoint))?;
+    let regs = vm_loader::initial_regs::aarch64_initial_regs(
+        &loader.initial_regs(),
+        caps,
+        &processor_topology.vp_arch(VpIndex::BSP),
+    );
+
+    Ok(regs)
+}
+
+fn load_binary<R: Debug + GuestArch>(
+    loader: &mut vm_loader::Loader<'_, R>,
+    tmk: &File,
+) -> anyhow::Result<loader::elf::LoadInfo> {
+    loader::elf::load_static_elf(
+        loader,
+        &mut &*tmk,
+        0,
+        0x200000,
+        false,
+        loader::importer::BootPageAcceptance::Exclusive,
+        "tmk",
+    )
+    .context("failed to load tmk")
+}
diff --git a/tmk/tmk_vmm/src/main.rs b/tmk/tmk_vmm/src/main.rs
index 121bd5b45f..03feb1e1d5 100644
--- a/tmk/tmk_vmm/src/main.rs
+++ b/tmk/tmk_vmm/src/main.rs
@@ -17,12 +17,24 @@ use pal_async::DefaultDriver;
 use pal_async::DefaultPool;
 use run::CommonState;
 use std::path::PathBuf;
+use tracing::level_filters::LevelFilter;
 use tracing_subscriber::fmt::format::FmtSpan;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
 
 fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt()
-        .fmt_fields(tracing_helpers::formatter::FieldFormatter)
-        .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
+    tracing_subscriber::registry()
+        .with(
+            tracing_subscriber::fmt::layer()
+                .fmt_fields(tracing_helpers::formatter::FieldFormatter)
+                .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE),
+        )
+        .with(
+            tracing_subscriber::EnvFilter::builder()
+                .with_default_directive(LevelFilter::INFO.into())
+                .with_env_var("TMK_LOG")
+                .from_env_lossy(),
+        )
         .init();
 
     DefaultPool::run_with(do_main)
diff --git a/tmk/tmk_vmm/src/run.rs b/tmk/tmk_vmm/src/run.rs
index 75b5d1c7f8..569b5b4f48 100644
--- a/tmk/tmk_vmm/src/run.rs
+++ b/tmk/tmk_vmm/src/run.rs
@@ -3,15 +3,6 @@
 
 //! Support for running a VM's VPs.
 
-#![cfg_attr(
-    guest_arch = "aarch64",
-    expect(unreachable_code),
-    expect(dead_code),
-    expect(unused_imports),
-    expect(unused_variables),
-    expect(unused_mut)
-)]
-
 use crate::Options;
 use crate::load;
 use anyhow::Context as _;
@@ -35,7 +26,7 @@ use vmcore::vmtime::VmTimeKeeper;
 use vmcore::vmtime::VmTimeSource;
 
 pub struct CommonState {
-    #[allow(dead_code)] // Only used in some configurations.
+    #[cfg_attr(not(target_os = "linux"), expect(dead_code))]
     pub driver: DefaultDriver,
     pub vmtime_keeper: VmTimeKeeper,
     pub vmtime_source: VmTimeSource,
@@ -54,9 +45,11 @@ impl CommonState {
             .context("failed to build processor topology")?;
 
         #[cfg(guest_arch = "aarch64")]
-        anyhow::bail!("aarch64 not supported yet");
-        #[cfg(guest_arch = "aarch64")]
-        let processor_topology = TopologyBuilder::new_aarch64(todo!())
+        let processor_topology =
+            TopologyBuilder::new_aarch64(vm_topology::processor::arch::GicInfo {
+                gic_distributor_base: 0xff000000,
+                gic_redistributors_base: 0xff020000,
+            })
             .build(1)
             .context("failed to build processor topology")?;
 
@@ -82,8 +75,8 @@ impl CommonState {
         let (event_send, mut event_recv) = mesh::channel();
 
         // Load the TMK.
+        let tmk = fs_err::File::open(&self.opts.tmk).context("failed to open tmk")?;
         let regs = {
-            let tmk = fs_err::File::open(&self.opts.tmk).context("failed to open tmk")?;
             #[cfg(guest_arch = "x86_64")]
             {
                 load::load_x86(
@@ -96,7 +89,13 @@ impl CommonState {
             }
             #[cfg(guest_arch = "aarch64")]
             {
-                anyhow::bail!("aarch64 not supported yet");
+                load::load_aarch64(
+                    &self.memory_layout,
+                    guest_memory,
+                    &self.processor_topology,
+                    caps,
+                    &tmk,
+                )?
             }
         };
 
@@ -272,7 +271,12 @@ impl RunnerBuilder {
             }
             #[cfg(guest_arch = "aarch64")]
             {
-                todo!()
+                let virt::aarch64::Aarch64InitialRegs {
+                    registers,
+                    system_registers,
+                } = self.regs.as_ref();
+                state.set_registers(registers)?;
+                state.set_system_registers(system_registers)?;
             }
             state.commit()?;
         }
diff --git a/vmm_core/virt_hvf/src/vp_state.rs b/vmm_core/virt_hvf/src/vp_state.rs
index feaeff9272..fb8f5e5bb8 100644
--- a/vmm_core/virt_hvf/src/vp_state.rs
+++ b/vmm_core/virt_hvf/src/vp_state.rs
@@ -117,7 +117,7 @@ impl AccessVpState for HvfVpStateAccess<'_, '_> {
     }
 
     fn commit(&mut self) -> Result<(), Self::Error> {
-        todo!()
+        Ok(())
     }
 
     fn registers(&mut self) -> Result<virt::aarch64::vp::Registers, Self::Error> {


Reviews:
user: login: smalis-msft
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:05:23Z
user: login: smalis-msft
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:09:31Z
user: login: smalis-msft
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:10:34Z
user: login: daprilik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:55:29Z
user: login: daprilik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:56:22Z
user: login: smalis-msft
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:56:34Z
user: login: smalis-msft
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T16:06:11Z
user: login: jstarks
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T16:16:24Z
user: login: jstarks
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T16:18:42Z
user: login: smalis-msft
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T16:24:35Z
user: login: jstarks
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T16:24:38Z
user: login: smalis-msft
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T16:32:53Z
user: login: jstarks
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T16:51:22Z
user: login: jstarks
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T16:51:40Z

Review comments:
diff_hunk: @@ -272,7 +271,12 @@ impl RunnerBuilder {
             }
             #[cfg(guest_arch = "aarch64")]
             {
-                todo!()
+                let virt::InitialRegs {
path: tmk/tmk_vmm/src/run.rs
user: login: smalis-msft
body: not a virt::aarch64::Aarch64InitialRegs path to match above?
created_at: 2025-03-20T15:05:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -40,28 +46,41 @@ pub fn init() {
             // various places).
             //
             // No special linker flags are needed.
-            supported_triple = "x86_64-unknown-none";
+            if triple != "x86_64-unknown-none" {
+                unsupported("x86_64-unknown-none");
+            }
         }
         "aarch64" => {
-            // This is the supported triple for aarch64. The -none target does
-            // not enable PIE, which we require so that the boot loader can run
-            // anywhere in PA space.
-            supported_triple = "aarch64-unknown-linux-musl";
-            // Don't include the _start entry point.
-            println!("cargo:rustc-link-arg=-nostartfiles");
-            // Make the executable relocatable.
-            println!("cargo:rustc-link-arg=-static-pie");
+            match triple.as_str() {
+                "aarch64-minimal_rt-none" => {
path: openhcl/minimal_rt_build/src/lib.rs
user: login: smalis-msft
body: Have we investigated if there's any interest in adding a target like this upstream? like an official aarch64-unknown-none-staticpie?
created_at: 2025-03-20T15:09:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Config file to use to build minimal_rt binaries for aarch64.
+#
+# This requires setting RUSTC_BOOTSTRAP=1, or an unstable toolchain.
path: openhcl/minimal_rt/aarch64-config.toml
user: login: daprilik
body: ```suggestion
# The dependency on `build-std` necessitates setting RUSTC_BOOTSTRAP=1, or an unstable toolchain.
```

and on that note - why the dependency on build-std?
created_at: 2025-03-20T15:55:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Config file to use to build minimal_rt binaries for aarch64.
+#
+# This requires setting RUSTC_BOOTSTRAP=1, or an unstable toolchain.
path: openhcl/minimal_rt/aarch64-config.toml
user: login: daprilik
body: ah, because the stdlib needs to be built with static PIE, duh.

morning brain is real
created_at: 2025-03-20T15:56:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Config file to use to build minimal_rt binaries for aarch64.
+#
+# This requires setting RUSTC_BOOTSTRAP=1, or an unstable toolchain.
path: openhcl/minimal_rt/aarch64-config.toml
user: login: smalis-msft
body: I believe the dependency is because this is a custom target, so there's no std being shipped via rustup.
created_at: 2025-03-20T15:56:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -54,9 +45,11 @@ impl CommonState {
             .context("failed to build processor topology")?;
path: tmk/tmk_vmm/src/run.rs
user: login: smalis-msft
body: Github won't let me comment on the right line, but is the allow(dead_code) above needed? It looks like we always use the driver due to vmtime.
created_at: 2025-03-20T16:06:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Config file to use to build minimal_rt binaries for aarch64.
+#
+# This requires setting RUSTC_BOOTSTRAP=1, or an unstable toolchain.
path: openhcl/minimal_rt/aarch64-config.toml
user: login: jstarks
body: Right, with the custom target, you must have build-std. And the custom target is for static PIE.
created_at: 2025-03-20T16:16:24Z
author_association: MEMBER
diff_hunk: @@ -40,28 +46,41 @@ pub fn init() {
             // various places).
             //
             // No special linker flags are needed.
-            supported_triple = "x86_64-unknown-none";
+            if triple != "x86_64-unknown-none" {
+                unsupported("x86_64-unknown-none");
+            }
         }
         "aarch64" => {
-            // This is the supported triple for aarch64. The -none target does
-            // not enable PIE, which we require so that the boot loader can run
-            // anywhere in PA space.
-            supported_triple = "aarch64-unknown-linux-musl";
-            // Don't include the _start entry point.
-            println!("cargo:rustc-link-arg=-nostartfiles");
-            // Make the executable relocatable.
-            println!("cargo:rustc-link-arg=-static-pie");
+            match triple.as_str() {
+                "aarch64-minimal_rt-none" => {
path: openhcl/minimal_rt_build/src/lib.rs
user: login: jstarks
body: I think ideally they'd just update the existing target.

But I also kind of think we should just get comfortable with custom targets and build-std, since this probably isn't the last time we'll want to tweak something. E.g., x86_64-unknown-none doesn't use sse2 even though we are fine with it doing so.
created_at: 2025-03-20T16:18:42Z
author_association: MEMBER
diff_hunk: @@ -40,28 +46,41 @@ pub fn init() {
             // various places).
             //
             // No special linker flags are needed.
-            supported_triple = "x86_64-unknown-none";
+            if triple != "x86_64-unknown-none" {
+                unsupported("x86_64-unknown-none");
+            }
         }
         "aarch64" => {
-            // This is the supported triple for aarch64. The -none target does
-            // not enable PIE, which we require so that the boot loader can run
-            // anywhere in PA space.
-            supported_triple = "aarch64-unknown-linux-musl";
-            // Don't include the _start entry point.
-            println!("cargo:rustc-link-arg=-nostartfiles");
-            // Make the executable relocatable.
-            println!("cargo:rustc-link-arg=-static-pie");
+            match triple.as_str() {
+                "aarch64-minimal_rt-none" => {
path: openhcl/minimal_rt_build/src/lib.rs
user: login: smalis-msft
body: Maybe we should keep our naming closer to the traditional target? i.e. name ours aarch64-unknown-none-staticpie, and a future x86_64-unknown-none-sse2, or something.
created_at: 2025-03-20T16:24:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -54,9 +45,11 @@ impl CommonState {
             .context("failed to build processor topology")?;
path: tmk/tmk_vmm/src/run.rs
user: login: jstarks
body: We don't always use it as part of `CommonState`. Only the paravisor path uses it currently. Maybe I'll convert it to an expect for now.
created_at: 2025-03-20T16:24:38Z
author_association: MEMBER
diff_hunk: @@ -40,28 +46,41 @@ pub fn init() {
             // various places).
             //
             // No special linker flags are needed.
-            supported_triple = "x86_64-unknown-none";
+            if triple != "x86_64-unknown-none" {
+                unsupported("x86_64-unknown-none");
+            }
         }
         "aarch64" => {
-            // This is the supported triple for aarch64. The -none target does
-            // not enable PIE, which we require so that the boot loader can run
-            // anywhere in PA space.
-            supported_triple = "aarch64-unknown-linux-musl";
-            // Don't include the _start entry point.
-            println!("cargo:rustc-link-arg=-nostartfiles");
-            // Make the executable relocatable.
-            println!("cargo:rustc-link-arg=-static-pie");
+            match triple.as_str() {
+                "aarch64-minimal_rt-none" => {
path: openhcl/minimal_rt_build/src/lib.rs
user: login: jstarks
body: I think we're less likely to conflict with upstream targets by specifying the vendor. Maybe we could add a suffix, too.
created_at: 2025-03-20T16:51:22Z
author_association: MEMBER
diff_hunk: @@ -40,28 +46,41 @@ pub fn init() {
             // various places).
             //
             // No special linker flags are needed.
-            supported_triple = "x86_64-unknown-none";
+            if triple != "x86_64-unknown-none" {
+                unsupported("x86_64-unknown-none");
+            }
         }
         "aarch64" => {
-            // This is the supported triple for aarch64. The -none target does
-            // not enable PIE, which we require so that the boot loader can run
-            // anywhere in PA space.
-            supported_triple = "aarch64-unknown-linux-musl";
-            // Don't include the _start entry point.
-            println!("cargo:rustc-link-arg=-nostartfiles");
-            // Make the executable relocatable.
-            println!("cargo:rustc-link-arg=-static-pie");
+            match triple.as_str() {
+                "aarch64-minimal_rt-none" => {
path: openhcl/minimal_rt_build/src/lib.rs
user: login: jstarks
body: Can consider later.
created_at: 2025-03-20T16:51:40Z
author_association: MEMBER

PR comments:
user: login: smalis-msft
created_at: 2025-03-20T15:01:35Z
body: What happens if you try to do a normal cross compile from non-linux hosts?
author_association: CONTRIBUTOR
user: login: jstarks
created_at: 2025-03-20T16:22:25Z
body: > What happens if you try to do a normal cross compile from non-linux hosts?

It hits our custom linker thing from config.toml, which relies on having gcc for leveraging the musl sysroot. If you remove that, then on macos at least, it complains about not understanding various linker flags.
author_association: MEMBER