Repository: bytecodealliance/wasmtime
Pull request #9282 (2024-09-19 16:12:46)

Add constant evaluation support for the `struct.new[_default]` Wasm GC instructions


<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->



Patch:
diff --git a/ci/build-build-matrix.js b/ci/build-build-matrix.js
index 321eca46d..4cda1079e 100644
--- a/ci/build-build-matrix.js
+++ b/ci/build-build-matrix.js
@@ -64,6 +64,11 @@ const array = [
     "os": "ubuntu-latest",
     "target": "x86_64-unknown-linux-musl",
   },
+  {
+    "build": "aarch64-windows",
+    "os": "windows-latest",
+    "target": "aarch64-pc-windows-msvc",
+  },
 ];
 
 const builds = [];
diff --git a/ci/build-tarballs.sh b/ci/build-tarballs.sh
index b8e1684c0..229aa2f51 100755
--- a/ci/build-tarballs.sh
+++ b/ci/build-tarballs.sh
@@ -68,7 +68,8 @@ case $build in
     fi
     ;;
 
-  x86_64-mingw*)
+  # Skip the MSI for non-x86_64 builds of Windows
+  x86_64-mingw* | *-windows*)
     cp target/$target/release/wasmtime.exe tmp/$bin_pkgname/wasmtime$min.exe
     fmt=zip
     ;;
diff --git a/cranelift/codegen/src/ir/memflags.rs b/cranelift/codegen/src/ir/memflags.rs
index 9b706bd56..f604a324c 100644
--- a/cranelift/codegen/src/ir/memflags.rs
+++ b/cranelift/codegen/src/ir/memflags.rs
@@ -222,6 +222,20 @@ impl MemFlags {
         }
     }
 
+    /// Return endianness of the memory access, if explicitly specified.
+    ///
+    /// If the endianness is not explicitly specified, this will return `None`,
+    /// which means "native endianness".
+    pub const fn explicit_endianness(self) -> Option<Endianness> {
+        if self.read_bit(BIT_LITTLE_ENDIAN) {
+            Some(Endianness::Little)
+        } else if self.read_bit(BIT_BIG_ENDIAN) {
+            Some(Endianness::Big)
+        } else {
+            None
+        }
+    }
+
     /// Set endianness of the memory access.
     pub fn set_endianness(&mut self, endianness: Endianness) {
         *self = self.with_endianness(endianness);
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
index 8244a7c62..03848fc98 100644
--- a/cranelift/codegen/src/isa/aarch64/abi.rs
+++ b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -6,7 +6,7 @@ use crate::ir::types::*;
 use crate::ir::MemFlags;
 use crate::ir::{dynamic_to_fixed, ExternalName, LibCall, Signature};
 use crate::isa;
-use crate::isa::aarch64::{inst::*, settings as aarch64_settings};
+use crate::isa::aarch64::{inst::*, settings as aarch64_settings, AArch64Backend};
 use crate::isa::unwind::UnwindInst;
 use crate::isa::winch;
 use crate::machinst::*;
@@ -725,6 +725,13 @@ impl ABIMachineSpec for AArch64MachineDeps {
         if incoming_args_diff > 0 {
             // Decrement SP to account for the additional space required by a tail call.
             insts.extend(Self::gen_sp_reg_adjust(-(incoming_args_diff as i32)));
+            if flags.unwind_info() {
+                insts.push(Inst::Unwind {
+                    inst: UnwindInst::StackAlloc {
+                        size: incoming_args_diff,
+                    },
+                });
+            }
 
             // Move fp and lr down.
             if setup_frame {
@@ -921,6 +928,11 @@ impl ABIMachineSpec for AArch64MachineDeps {
         let stack_size = frame_layout.fixed_frame_storage_size + frame_layout.outgoing_args_size;
         if stack_size > 0 {
             insts.extend(Self::gen_sp_reg_adjust(-(stack_size as i32)));
+            if flags.unwind_info() {
+                insts.push(Inst::Unwind {
+                    inst: UnwindInst::StackAlloc { size: stack_size },
+                });
+            }
         }
 
         insts
@@ -1249,7 +1261,7 @@ impl AArch64CallSite {
         mut self,
         ctx: &mut Lower<Inst>,
         args: isle::ValueSlice,
-        isa_flags: &aarch64_settings::Flags,
+        backend: &AArch64Backend,
     ) {
         let new_stack_arg_size =
             u32::try_from(self.sig(ctx.sigs()).sized_stack_arg_space()).unwrap();
@@ -1263,7 +1275,7 @@ impl AArch64CallSite {
 
         let dest = self.dest().clone();
         let uses = self.take_uses();
-        let key = select_api_key(isa_flags, isa::CallConv::Tail, true);
+        let key = select_api_key(&backend.isa_flags, isa::CallConv::Tail, true);
 
         match dest {
             CallDest::ExtName(callee, RelocDistance::Near) => {
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index 44e3b4d0d..0c05db4bc 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -73,66 +73,7 @@ pub struct ExtendedValue {
 
 impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
     isle_lower_prelude_methods!();
-    isle_prelude_caller_methods!(
-        crate::isa::aarch64::abi::AArch64MachineDeps,
-        AArch64CallSite
-    );
-
-    fn gen_return_call(
-        &mut self,
-        callee_sig: SigRef,
-        callee: ExternalName,
-        distance: RelocDistance,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let call_site = AArch64CallSite::from_func(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            &callee,
-            IsTailCall::Yes,
-            distance,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args, &self.backend.isa_flags);
-
-        InstOutput::new()
-    }
-
-    fn gen_return_call_indirect(
-        &mut self,
-        callee_sig: SigRef,
-        callee: Value,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let callee = self.put_in_reg(callee);
-
-        let call_site = AArch64CallSite::from_ptr(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            callee,
-            IsTailCall::Yes,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args, &self.backend.isa_flags);
-
-        InstOutput::new()
-    }
+    isle_prelude_caller_methods!(AArch64CallSite);
 
     fn sign_return_address_disabled(&mut self) -> Option<()> {
         if self.backend.isa_flags.sign_return_address() {
diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs
index c1da4e3f1..fb2f2382f 100644
--- a/cranelift/codegen/src/isa/aarch64/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/mod.rs
@@ -138,10 +138,11 @@ impl TargetIsa for AArch64Backend {
                     )?,
                 ))
             }
-            UnwindInfoKind::Windows => {
-                // TODO: support Windows unwind info on AArch64
-                None
-            }
+            UnwindInfoKind::Windows => Some(UnwindInfo::WindowsArm64(
+                crate::isa::unwind::winarm64::create_unwind_info_from_insts(
+                    &result.buffer.unwind_info[..],
+                )?,
+            )),
             _ => None,
         })
     }
diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs
index 54c4dba16..58a4e4024 100644
--- a/cranelift/codegen/src/isa/pulley_shared/abi.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs
@@ -1,6 +1,7 @@
 //! Implementation of a standard Pulley ABI.
 
 use super::{inst::*, PulleyFlags, PulleyTargetKind};
+use crate::isa::pulley_shared::PulleyBackend;
 use crate::{
     ir::{self, types::*, MemFlags, Signature},
     isa::{self, unwind::UnwindInst},
@@ -657,7 +658,12 @@ impl<P> PulleyABICallSite<P>
 where
     P: PulleyTargetKind,
 {
-    pub fn emit_return_call(self, _ctx: &mut Lower<InstAndKind<P>>, _args: isle::ValueSlice) {
+    pub fn emit_return_call(
+        self,
+        _ctx: &mut Lower<InstAndKind<P>>,
+        _args: isle::ValueSlice,
+        _backend: &PulleyBackend<P>,
+    ) {
         todo!()
     }
 }
diff --git a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs
index 1f68650ef..e1c57bd73 100644
--- a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs
@@ -49,63 +49,7 @@ where
     P: PulleyTargetKind,
 {
     crate::isle_lower_prelude_methods!(InstAndKind<P>);
-    crate::isle_prelude_caller_methods!(PulleyMachineDeps<P>, PulleyABICallSite<P>);
-
-    fn gen_return_call(
-        &mut self,
-        callee_sig: SigRef,
-        callee: ExternalName,
-        distance: RelocDistance,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let call_site = PulleyABICallSite::from_func(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            &callee,
-            IsTailCall::Yes,
-            distance,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args);
-
-        InstOutput::new()
-    }
-
-    fn gen_return_call_indirect(
-        &mut self,
-        callee_sig: SigRef,
-        callee: Value,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let callee = self.put_in_reg(callee);
-
-        let call_site = PulleyABICallSite::from_ptr(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            callee,
-            IsTailCall::Yes,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args);
-
-        InstOutput::new()
-    }
+    crate::isle_prelude_caller_methods!(PulleyABICallSite<P>);
 
     fn lower_br_table(&mut self, _index: Reg, _targets: &[MachLabel]) -> Unit {
         todo!()
diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs
index ce31aecaa..6c9fbe3a3 100644
--- a/cranelift/codegen/src/isa/riscv64/abi.rs
+++ b/cranelift/codegen/src/isa/riscv64/abi.rs
@@ -5,7 +5,7 @@ use crate::ir::types::*;
 
 use crate::isa;
 
-use crate::isa::riscv64::inst::*;
+use crate::isa::riscv64::{inst::*, Riscv64Backend};
 use crate::isa::CallConv;
 use crate::machinst::*;
 
@@ -727,7 +727,12 @@ impl ABIMachineSpec for Riscv64MachineDeps {
 }
 
 impl Riscv64ABICallSite {
-    pub fn emit_return_call(mut self, ctx: &mut Lower<Inst>, args: isle::ValueSlice) {
+    pub fn emit_return_call(
+        mut self,
+        ctx: &mut Lower<Inst>,
+        args: isle::ValueSlice,
+        _backend: &Riscv64Backend,
+    ) {
         let new_stack_arg_size =
             u32::try_from(self.sig(ctx.sigs()).sized_stack_arg_space()).unwrap();
 
diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
index 28a0bd064..c39bfd4e5 100644
--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
@@ -60,63 +60,7 @@ impl<'a, 'b> RV64IsleContext<'a, 'b, MInst, Riscv64Backend> {
 
 impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend> {
     isle_lower_prelude_methods!();
-    isle_prelude_caller_methods!(Riscv64MachineDeps, Riscv64ABICallSite);
-
-    fn gen_return_call(
-        &mut self,
-        callee_sig: SigRef,
-        callee: ExternalName,
-        distance: RelocDistance,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let call_site = Riscv64ABICallSite::from_func(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            &callee,
-            IsTailCall::Yes,
-            distance,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args);
-
-        InstOutput::new()
-    }
-
-    fn gen_return_call_indirect(
-        &mut self,
-        callee_sig: SigRef,
-        callee: Value,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let callee = self.put_in_reg(callee);
-
-        let call_site = Riscv64ABICallSite::from_ptr(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            callee,
-            IsTailCall::Yes,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args);
-
-        InstOutput::new()
-    }
+    isle_prelude_caller_methods!(Riscv64ABICallSite);
 
     fn fpu_op_width_from_ty(&mut self, ty: Type) -> FpuOPWidth {
         match ty {
diff --git a/cranelift/codegen/src/isa/unwind.rs b/cranelift/codegen/src/isa/unwind.rs
index d357b64fa..305e7b470 100644
--- a/cranelift/codegen/src/isa/unwind.rs
+++ b/cranelift/codegen/src/isa/unwind.rs
@@ -11,6 +11,9 @@ pub mod systemv;
 #[cfg(feature = "unwind")]
 pub mod winx64;
 
+#[cfg(feature = "unwind")]
+pub mod winarm64;
+
 /// CFA-based unwind information used on SystemV.
 pub type CfaUnwindInfo = systemv::UnwindInfo;
 
@@ -39,6 +42,9 @@ pub enum UnwindInfo {
     /// System V ABI unwind information.
     #[cfg(feature = "unwind")]
     SystemV(CfaUnwindInfo),
+    /// Windows Arm64 ABI unwind information.
+    #[cfg(feature = "unwind")]
+    WindowsArm64(winarm64::UnwindInfo),
 }
 
 /// Unwind pseudoinstruction used in VCode backends: represents that
@@ -197,3 +203,39 @@ pub enum UnwindInst {
         return_addresses: bool,
     },
 }
+
+struct Writer<'a> {
+    buf: &'a mut [u8],
+    offset: usize,
+}
+
+impl<'a> Writer<'a> {
+    pub fn new(buf: &'a mut [u8]) -> Self {
+        Self { buf, offset: 0 }
+    }
+
+    fn write_u8(&mut self, v: u8) {
+        self.buf[self.offset] = v;
+        self.offset += 1;
+    }
+
+    fn write_u16_le(&mut self, v: u16) {
+        self.buf[self.offset..(self.offset + 2)].copy_from_slice(&v.to_le_bytes());
+        self.offset += 2;
+    }
+
+    fn write_u16_be(&mut self, v: u16) {
+        self.buf[self.offset..(self.offset + 2)].copy_from_slice(&v.to_be_bytes());
+        self.offset += 2;
+    }
+
+    fn write_u32_le(&mut self, v: u32) {
+        self.buf[self.offset..(self.offset + 4)].copy_from_slice(&v.to_le_bytes());
+        self.offset += 4;
+    }
+
+    fn write_u32_be(&mut self, v: u32) {
+        self.buf[self.offset..(self.offset + 4)].copy_from_slice(&v.to_be_bytes());
+        self.offset += 4;
+    }
+}
diff --git a/cranelift/codegen/src/isa/unwind/winarm64.rs b/cranelift/codegen/src/isa/unwind/winarm64.rs
new file mode 100644
index 000000000..a4ab147fb
--- /dev/null
+++ b/cranelift/codegen/src/isa/unwind/winarm64.rs
@@ -0,0 +1,315 @@
+//! Windows Arm64 ABI unwind information.
+
+use alloc::vec::Vec;
+#[cfg(feature = "enable-serde")]
+use serde_derive::{Deserialize, Serialize};
+
+use crate::binemit::CodeOffset;
+use crate::isa::unwind::UnwindInst;
+use crate::result::CodegenResult;
+
+use super::Writer;
+
+/// The supported unwind codes for the Arm64 Windows ABI.
+///
+/// See: <https://learn.microsoft.com/en-us/cpp/build/arm64-exception-handling>
+/// Only what is needed to describe the prologues generated by the Cranelift AArch64 ISA are represented here.
+#[allow(dead_code)]
+#[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub(crate) enum UnwindCode {
+    /// Save int register, or register pair.
+    SaveReg {
+        reg: u8,
+        stack_offset: u16,
+        is_pair: bool,
+    },
+    /// Save floating point register, or register pair.
+    SaveFReg {
+        reg: u8,
+        stack_offset: u16,
+        is_pair: bool,
+    },
+    /// Save frame-pointer register (X29) and LR register pair.
+    SaveFpLrPair {
+        stack_offset: u16,
+    },
+    // Small (<512b) stack allocation.
+    AllocS {
+        size: u16,
+    },
+    // Medium (<32Kb) stack allocation.
+    AllocM {
+        size: u16,
+    },
+    // Large (<256Mb) stack allocation.
+    AllocL {
+        size: u32,
+    },
+    /// PAC sign the LR register.
+    PacSignLr,
+    /// Set the frame-pointer register to the stack-pointer register.
+    SetFp,
+    /// Set the frame-pointer register to the stack-pointer register with an
+    /// offset.
+    AddFp {
+        offset: u16,
+    },
+}
+
+/// Represents Windows Arm64 unwind information.
+///
+/// For information about Windows Arm64 unwind info, see:
+/// <https://learn.microsoft.com/en-us/cpp/build/arm64-exception-handling>
+#[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub struct UnwindInfo {
+    pub(crate) unwind_codes: Vec<UnwindCode>,
+}
+
+impl UnwindInfo {
+    /// Calculate the number of words needed to encode the unwind codes.
+    pub fn code_words(&self) -> u8 {
+        let mut bytes = 0u16;
+        for code in self.unwind_codes.iter() {
+            let next_bytes = match code {
+                UnwindCode::SaveFpLrPair { .. }
+                | UnwindCode::AllocS { .. }
+                | UnwindCode::PacSignLr
+                | UnwindCode::SetFp => 1,
+                UnwindCode::SaveReg { .. }
+                | UnwindCode::SaveFReg { .. }
+                | UnwindCode::AllocM { .. }
+                | UnwindCode::AddFp { .. } => 2,
+                UnwindCode::AllocL { .. } => 4,
+            };
+            bytes = bytes.checked_add(next_bytes).unwrap();
+        }
+
+        bytes.div_ceil(4).try_into().unwrap()
+    }
+
+    /// Emits the unwind information into the given mutable byte slice.
+    ///
+    /// This function will panic if the slice is not at least `emit_size` in length.
+    pub fn emit(&self, buf: &mut [u8]) {
+        fn encode_stack_offset<const BITS: u8>(stack_offset: u16) -> u16 {
+            let encoded = (stack_offset / 8) - 1;
+            assert!(encoded < (1 << BITS), "Stack offset too large");
+            encoded
+        }
+
+        // NOTE: Unwind codes are written in big-endian!
+
+        let mut writer = Writer::new(buf);
+        for code in self.unwind_codes.iter().rev() {
+            match code {
+                &UnwindCode::SaveReg {
+                    reg,
+                    stack_offset,
+                    is_pair,
+                } => {
+                    assert!(reg >= 19, "Can't save registers before X19");
+                    let reg = u16::from(reg - 19);
+                    let encoding = if is_pair {
+                        let mut encoding = 0b11001100_00000000u16;
+                        encoding |= reg << 6;
+                        encoding |= encode_stack_offset::<6>(stack_offset);
+                        encoding
+                    } else {
+                        let mut encoding = 0b11010100_00000000u16;
+                        encoding |= reg << 5;
+                        encoding |= encode_stack_offset::<5>(stack_offset);
+                        encoding
+                    };
+                    writer.write_u16_be(encoding);
+                }
+                &UnwindCode::SaveFReg {
+                    reg,
+                    stack_offset,
+                    is_pair,
+                } => {
+                    assert!(reg >= 8, "Can't save registers before D8");
+                    let reg = u16::from(reg - 8);
+                    let encoding = if is_pair {
+                        let mut encoding = 0b11011010_00000000u16;
+                        encoding |= reg << 6;
+                        encoding |= encode_stack_offset::<6>(stack_offset);
+                        encoding
+                    } else {
+                        let mut encoding = 0b11011110_00000000u16;
+                        encoding |= reg << 5;
+                        encoding |= encode_stack_offset::<5>(stack_offset);
+                        encoding
+                    };
+                    writer.write_u16_be(encoding);
+                }
+                &UnwindCode::SaveFpLrPair { stack_offset } => {
+                    if stack_offset == 0 {
+                        writer.write_u8(0b01000000);
+                    } else {
+                        let encoding = 0b10000000u8
+                            | u8::try_from(encode_stack_offset::<6>(stack_offset)).unwrap();
+                        writer.write_u8(encoding);
+                    }
+                }
+                &UnwindCode::AllocS { size } => {
+                    // Size is measured in double 64-bit words.
+                    let encoding = size / 16;
+                    assert!(encoding < (1 << 5), "Stack alloc size too large");
+                    // Tag is 0b000, so we don't need to encode that.
+                    writer.write_u8(encoding.try_into().unwrap());
+                }
+                &UnwindCode::AllocM { size } => {
+                    // Size is measured in double 64-bit words.
+                    let mut encoding = size / 16;
+                    assert!(encoding < (1 << 11), "Stack alloc size too large");
+                    encoding |= 0b11000 << 11;
+                    writer.write_u16_be(encoding);
+                }
+                &UnwindCode::AllocL { size } => {
+                    // Size is measured in double 64-bit words.
+                    let mut encoding = size / 16;
+                    assert!(encoding < (1 << 24), "Stack alloc size too large");
+                    encoding |= 0b11100000 << 24;
+                    writer.write_u32_be(encoding);
+                }
+                UnwindCode::PacSignLr => {
+                    writer.write_u8(0b11111100);
+                }
+                UnwindCode::SetFp => {
+                    writer.write_u8(0b11100001);
+                }
+                &UnwindCode::AddFp { mut offset } => {
+                    offset /= 8;
+                    assert!(offset & !0xFF == 0, "Offset too large");
+                    let encoding = (0b11100010 << 8) | offset;
+                    writer.write_u16_be(encoding);
+                }
+            }
+        }
+    }
+}
+
+pub(crate) fn create_unwind_info_from_insts(
+    insts: &[(CodeOffset, UnwindInst)],
+) -> CodegenResult<UnwindInfo> {
+    let mut unwind_codes = vec![];
+    let mut last_stackalloc = None;
+    let mut last_clobber_offset = None;
+    for &(_, ref inst) in insts {
+        match inst {
+            &UnwindInst::PushFrameRegs { .. } => {
+                unwind_codes.push(UnwindCode::SaveFpLrPair { stack_offset: 16 });
+                unwind_codes.push(UnwindCode::SetFp);
+            }
+            &UnwindInst::DefineNewFrame {
+                offset_downward_to_clobbers,
+                ..
+            } => {
+                assert!(last_clobber_offset.is_none(), "More than one frame defined");
+                last_clobber_offset = Some(offset_downward_to_clobbers);
+
+                // If we've seen a stackalloc, then we were adjusting the stack
+                // to make space for additional arguments, so encode that now.
+                if let &Some(last_stackalloc) = &last_stackalloc {
+                    assert!(last_stackalloc < (1u32 << 8) * 8);
+                    unwind_codes.push(UnwindCode::AddFp {
+                        offset: u16::try_from(last_stackalloc).unwrap(),
+                    });
+                    unwind_codes.push(UnwindCode::SaveFpLrPair { stack_offset: 0 });
+                    unwind_codes.push(UnwindCode::SetFp);
+                }
+            }
+            &UnwindInst::StackAlloc { size } => {
+                last_stackalloc = Some(size);
+                assert!(size % 16 == 0, "Size must be a multiple of 16");
+                const SMALL_STACK_ALLOC_MAX: u32 = (1 << 5) * 16 - 1;
+                const MEDIUM_STACK_ALLOC_MIN: u32 = SMALL_STACK_ALLOC_MAX + 1;
+                const MEDIUM_STACK_ALLOC_MAX: u32 = (1 << 11) * 16 - 1;
+                const LARGE_STACK_ALLOC_MIN: u32 = MEDIUM_STACK_ALLOC_MAX + 1;
+                const LARGE_STACK_ALLOC_MAX: u32 = (1 << 24) * 16 - 1;
+                match size {
+                    0..=SMALL_STACK_ALLOC_MAX => unwind_codes.push(UnwindCode::AllocS {
+                        size: size.try_into().unwrap(),
+                    }),
+                    MEDIUM_STACK_ALLOC_MIN..=MEDIUM_STACK_ALLOC_MAX => {
+                        unwind_codes.push(UnwindCode::AllocM {
+                            size: size.try_into().unwrap(),
+                        })
+                    }
+                    LARGE_STACK_ALLOC_MIN..=LARGE_STACK_ALLOC_MAX => {
+                        unwind_codes.push(UnwindCode::AllocL { size: size })
+                    }
+                    _ => panic!("Stack allocation size too large"),
+                }
+            }
+            &UnwindInst::SaveReg {
+                clobber_offset,
+                reg,
+            } => {
+                // We're given the clobber offset, but we need to encode how far
+                // the stack was adjusted, so calculate that based on the last
+                // clobber offset we saw.
+                let last_clobber_offset = last_clobber_offset.as_mut().expect("No frame defined");
+                if *last_clobber_offset > clobber_offset {
+                    let stack_offset = *last_clobber_offset - clobber_offset;
+                    *last_clobber_offset = clobber_offset;
+
+                    assert!(stack_offset % 8 == 0, "Offset must be a multiple of 8");
+                    match reg.class() {
+                        regalloc2::RegClass::Int => {
+                            let reg = reg.hw_enc();
+                            if reg < 19 {
+                                panic!("Can't save registers before X19");
+                            }
+                            unwind_codes.push(UnwindCode::SaveReg {
+                                reg,
+                                stack_offset: stack_offset.try_into().unwrap(),
+                                is_pair: false,
+                            });
+                        }
+                        regalloc2::RegClass::Float => {
+                            let reg = reg.hw_enc();
+                            if reg < 8 {
+                                panic!("Can't save registers before D8");
+                            }
+                            unwind_codes.push(UnwindCode::SaveFReg {
+                                reg,
+                                stack_offset: stack_offset.try_into().unwrap(),
+                                is_pair: false,
+                            });
+                        }
+                        regalloc2::RegClass::Vector => unreachable!(),
+                    }
+                } else {
+                    // If we see a clobber offset within the last offset amount,
+                    // then we're actually saving a pair of registers.
+                    let last_unwind_code = unwind_codes.last_mut().unwrap();
+                    match last_unwind_code {
+                        UnwindCode::SaveReg { is_pair, .. } => {
+                            assert_eq!(reg.class(), regalloc2::RegClass::Int);
+                            assert!(!*is_pair);
+                            *is_pair = true;
+                        }
+                        UnwindCode::SaveFReg { is_pair, .. } => {
+                            assert_eq!(reg.class(), regalloc2::RegClass::Float);
+                            assert!(!*is_pair);
+                            *is_pair = true;
+                        }
+                        _ => unreachable!("Previous code should have been a register save"),
+                    }
+                }
+            }
+            &UnwindInst::Aarch64SetPointerAuth { return_addresses } => {
+                assert!(
+                    return_addresses,
+                    "Windows doesn't support explicitly disabling return address signing"
+                );
+                unwind_codes.push(UnwindCode::PacSignLr);
+            }
+        }
+    }
+
+    Ok(UnwindInfo { unwind_codes })
+}
diff --git a/cranelift/codegen/src/isa/unwind/winx64.rs b/cranelift/codegen/src/isa/unwind/winx64.rs
index de87251f7..8a23494b1 100644
--- a/cranelift/codegen/src/isa/unwind/winx64.rs
+++ b/cranelift/codegen/src/isa/unwind/winx64.rs
@@ -1,6 +1,5 @@
 //! Windows x64 ABI unwind information.
 
-use crate::result::{CodegenError, CodegenResult};
 use alloc::vec::Vec;
 use log::warn;
 #[cfg(feature = "enable-serde")]
@@ -8,38 +7,15 @@ use serde_derive::{Deserialize, Serialize};
 
 use crate::binemit::CodeOffset;
 use crate::isa::unwind::UnwindInst;
+use crate::result::{CodegenError, CodegenResult};
+
+use super::Writer;
 
 /// Maximum (inclusive) size of a "small" stack allocation
 const SMALL_ALLOC_MAX_SIZE: u32 = 128;
 /// Maximum (inclusive) size of a "large" stack allocation that can represented in 16-bits
 const LARGE_ALLOC_16BIT_MAX_SIZE: u32 = 524280;
 
-struct Writer<'a> {
-    buf: &'a mut [u8],
-    offset: usize,
-}
-
-impl<'a> Writer<'a> {
-    pub fn new(buf: &'a mut [u8]) -> Self {
-        Self { buf, offset: 0 }
-    }
-
-    fn write_u8(&mut self, v: u8) {
-        self.buf[self.offset] = v;
-        self.offset += 1;
-    }
-
-    fn write_u16_le(&mut self, v: u16) {
-        self.buf[self.offset..(self.offset + 2)].copy_from_slice(&v.to_le_bytes());
-        self.offset += 2;
-    }
-
-    fn write_u32_le(&mut self, v: u32) {
-        self.buf[self.offset..(self.offset + 4)].copy_from_slice(&v.to_le_bytes());
-        self.offset += 4;
-    }
-}
-
 /// The supported unwind codes for the x64 Windows ABI.
 ///
 /// See: <https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64>
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index 8a887a1a3..857e57535 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -4,6 +4,7 @@ use crate::ir::{self, types, LibCall, MemFlags, Signature, TrapCode};
 use crate::ir::{types::*, ExternalName};
 use crate::isa;
 use crate::isa::winch;
+use crate::isa::x64::X64Backend;
 use crate::isa::{unwind::UnwindInst, x64::inst::*, x64::settings as x64_settings, CallConv};
 use crate::machinst::abi::*;
 use crate::machinst::*;
@@ -980,7 +981,12 @@ impl ABIMachineSpec for X64ABIMachineSpec {
 }
 
 impl X64CallSite {
-    pub fn emit_return_call(mut self, ctx: &mut Lower<Inst>, args: isle::ValueSlice) {
+    pub fn emit_return_call(
+        mut self,
+        ctx: &mut Lower<Inst>,
+        args: isle::ValueSlice,
+        _backend: &X64Backend,
+    ) {
         let new_stack_arg_size =
             u32::try_from(self.sig(ctx.sigs()).sized_stack_arg_space()).unwrap();
 
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 3ed4e4cb4..6b6473c07 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -72,63 +72,7 @@ pub(crate) fn lower_branch(
 
 impl Context for IsleContext<'_, '_, MInst, X64Backend> {
     isle_lower_prelude_methods!();
-    isle_prelude_caller_methods!(X64ABIMachineSpec, X64CallSite);
-
-    fn gen_return_call_indirect(
-        &mut self,
-        callee_sig: SigRef,
-        callee: Value,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let callee = self.put_in_reg(callee);
-
-        let call_site = X64CallSite::from_ptr(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            callee,
-            IsTailCall::Yes,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args);
-
-        InstOutput::new()
-    }
-
-    fn gen_return_call(
-        &mut self,
-        callee_sig: SigRef,
-        callee: ExternalName,
-        distance: RelocDistance,
-        args: ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = isa::CallConv::Tail;
-        debug_assert_eq!(
-            self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
-            caller_conv,
-            "Can only do `return_call`s from within a `tail` calling convention function"
-        );
-
-        let call_site = X64CallSite::from_func(
-            self.lower_ctx.sigs(),
-            callee_sig,
-            &callee,
-            IsTailCall::Yes,
-            distance,
-            caller_conv,
-            self.backend.flags().clone(),
-        );
-        call_site.emit_return_call(self.lower_ctx, args);
-
-        InstOutput::new()
-    }
+    isle_prelude_caller_methods!(X64CallSite);
 
     #[inline]
     fn operand_size_of_type_32_64(&mut self, ty: Type) -> OperandSize {
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index 6954dab2a..eb183c4b2 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -753,7 +753,7 @@ pub fn shuffle_imm_as_le_lane_idx(size: u8, bytes: &[u8]) -> Option<u8> {
 #[macro_export]
 #[doc(hidden)]
 macro_rules! isle_prelude_caller_methods {
-    ($abispec:ty, $abicaller:ty) => {
+    ($abicaller:ty) => {
         fn gen_call(
             &mut self,
             sig_ref: SigRef,
@@ -808,6 +808,62 @@ macro_rules! isle_prelude_caller_methods {
 
             crate::machinst::isle::gen_call_common(&mut self.lower_ctx, num_rets, caller, args)
         }
+
+        fn gen_return_call(
+            &mut self,
+            callee_sig: SigRef,
+            callee: ExternalName,
+            distance: RelocDistance,
+            args: ValueSlice,
+        ) -> InstOutput {
+            let caller_conv = isa::CallConv::Tail;
+            debug_assert_eq!(
+                self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
+                caller_conv,
+                "Can only do `return_call`s from within a `tail` calling convention function"
+            );
+
+            let call_site = <$abicaller>::from_func(
+                self.lower_ctx.sigs(),
+                callee_sig,
+                &callee,
+                IsTailCall::Yes,
+                distance,
+                caller_conv,
+                self.backend.flags().clone(),
+            );
+            call_site.emit_return_call(self.lower_ctx, args, self.backend);
+
+            InstOutput::new()
+        }
+
+        fn gen_return_call_indirect(
+            &mut self,
+            callee_sig: SigRef,
+            callee: Value,
+            args: ValueSlice,
+        ) -> InstOutput {
+            let caller_conv = isa::CallConv::Tail;
+            debug_assert_eq!(
+                self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()),
+                caller_conv,
+                "Can only do `return_call`s from within a `tail` calling convention function"
+            );
+
+            let callee = self.put_in_reg(callee);
+
+            let call_site = <$abicaller>::from_ptr(
+                self.lower_ctx.sigs(),
+                callee_sig,
+                callee,
+                IsTailCall::Yes,
+                caller_conv,
+                self.backend.flags().clone(),
+            );
+            call_site.emit_return_call(self.lower_ctx, args, self.backend);
+
+            InstOutput::new()
+        }
     };
 }
 
diff --git a/crates/cranelift/src/gc/enabled.rs b/crates/cranelift/src/gc/enabled.rs
index 0597e2ee9..95fa1881b 100644
--- a/crates/cranelift/src/gc/enabled.rs
+++ b/crates/cranelift/src/gc/enabled.rs
@@ -25,6 +25,10 @@ fn unbarriered_load_gc_ref(
     flags: ir::MemFlags,
 ) -> WasmResult<ir::Value> {
     debug_assert!(ty.is_vmgcref_type());
+    assert!(
+        flags.explicit_endianness().is_none(),
+        "GC references are always native-endian"
+    );
     let gc_ref = builder.ins().load(ir::types::I32, flags, ptr_to_gc_ref, 0);
     if ty != WasmHeapType::I31 {
         builder.declare_value_needs_stack_map(gc_ref);
@@ -40,6 +44,10 @@ fn unbarriered_store_gc_ref(
     flags: ir::MemFlags,
 ) -> WasmResult<()> {
     debug_assert!(ty.is_vmgcref_type());
+    assert!(
+        flags.explicit_endianness().is_none(),
+        "GC references are always native-endian"
+    );
     builder.ins().store(flags, gc_ref, dst, 0);
     Ok(())
 }
diff --git a/crates/cranelift/src/gc/enabled/drc.rs b/crates/cranelift/src/gc/enabled/drc.rs
index 8efaece59..cd351772e 100644
--- a/crates/cranelift/src/gc/enabled/drc.rs
+++ b/crates/cranelift/src/gc/enabled/drc.rs
@@ -315,6 +315,10 @@ impl GcCompiler for DrcCompiler {
         flags: ir::MemFlags,
     ) -> WasmResult<ir::Value> {
         assert!(ty.is_vmgcref_type());
+        assert!(
+            flags.explicit_endianness().is_none(),
+            "GC references are always native-endian"
+        );
 
         let (reference_type, needs_stack_map) = func_env.reference_type(ty.heap_type);
         debug_assert!(needs_stack_map);
@@ -463,6 +467,10 @@ impl GcCompiler for DrcCompiler {
         flags: ir::MemFlags,
     ) -> WasmResult<()> {
         assert!(ty.is_vmgcref_type());
+        assert!(
+            flags.explicit_endianness().is_none(),
+            "GC references are always native-endian"
+        );
 
         let (ref_ty, needs_stack_map) = func_env.reference_type(ty.heap_type);
         debug_assert!(needs_stack_map);
diff --git a/crates/cranelift/src/obj.rs b/crates/cranelift/src/obj.rs
index d73596f68..c6f93bdb0 100644
--- a/crates/cranelift/src/obj.rs
+++ b/crates/cranelift/src/obj.rs
@@ -293,19 +293,19 @@ impl<'a> UnwindInfoBuilder<'a> {
             // actually assembled byte-wise here. Instead that's deferred to
             // happen later during `write_windows_unwind_info` which will apply
             // a further offset to `unwind_address`.
+            //
+            // FIXME: in theory we could "intern" the `unwind_info` value
+            // here within the `.xdata` section. Most of our unwind
+            // information for functions is probably pretty similar in which
+            // case the `.xdata` could be quite small and `.pdata` could
+            // have multiple functions point to the same unwinding
+            // information.
             UnwindInfo::WindowsX64(info) => {
                 let unwind_size = info.emit_size();
                 let mut unwind_info = vec![0; unwind_size];
                 info.emit(&mut unwind_info);
 
                 // `.xdata` entries are always 4-byte aligned
-                //
-                // FIXME: in theory we could "intern" the `unwind_info` value
-                // here within the `.xdata` section. Most of our unwind
-                // information for functions is probably pretty similar in which
-                // case the `.xdata` could be quite small and `.pdata` could
-                // have multiple functions point to the same unwinding
-                // information.
                 while self.windows_xdata.len() % 4 != 0 {
                     self.windows_xdata.push(0x00);
                 }
@@ -320,6 +320,58 @@ impl<'a> UnwindInfoBuilder<'a> {
                 });
             }
 
+            // See https://learn.microsoft.com/en-us/cpp/build/arm64-exception-handling
+            UnwindInfo::WindowsArm64(info) => {
+                let code_words = info.code_words();
+                let mut unwind_codes = vec![0; (code_words * 4) as usize];
+                info.emit(&mut unwind_codes);
+
+                // `.xdata` entries are always 4-byte aligned
+                while self.windows_xdata.len() % 4 != 0 {
+                    self.windows_xdata.push(0x00);
+                }
+
+                // First word:
+                // 0-17:    Function Length
+                // 18-19:   Version (must be 0)
+                // 20:      X bit (is exception data present?)
+                // 21:      E bit (has single packed epilogue?)
+                // 22-26:   Epilogue count
+                // 27-31:   Code words count
+                let requires_extended_counts = code_words > (1 << 5);
+                let encoded_function_len = function_len / 4;
+                assert!(encoded_function_len < (1 << 18), "function too large");
+                let mut word1 = encoded_function_len as u32;
+                if !requires_extended_counts {
+                    word1 |= (code_words as u32) << 27;
+                }
+                let unwind_address = self.windows_xdata.len();
+                self.windows_xdata.extend_from_slice(&word1.to_le_bytes());
+
+                if requires_extended_counts {
+                    // Extended counts word:
+                    // 0-15:    Epilogue count
+                    // 16-23:   Code words count
+                    let extended_counts_word = (code_words as u32) << 16;
+                    self.windows_xdata
+                        .extend_from_slice(&extended_counts_word.to_le_bytes());
+                }
+
+                // Skip epilogue information: Per comment on [`UnwindInst`], we
+                // do not emit information about epilogues.
+
+                // Emit the unwind codes.
+                self.windows_xdata.extend_from_slice(&unwind_codes);
+
+                // Record a `RUNTIME_FUNCTION` which this will point to.
+                // NOTE: `end` is not used, so leave it as 0.
+                self.windows_pdata.push(RUNTIME_FUNCTION {
+                    begin: u32::try_from(function_offset).unwrap(),
+                    end: 0,
+                    unwind_address: u32::try_from(unwind_address).unwrap(),
+                });
+            }
+
             // System-V is different enough that we just record the unwinding
             // information to get processed at a later time.
             UnwindInfo::SystemV(info) => {
@@ -385,11 +437,6 @@ impl<'a> UnwindInfoBuilder<'a> {
         pdata_id: SectionId,
         text_section_size: u64,
     ) {
-        // Currently the binary format supported here only supports
-        // little-endian for x86_64, or at least that's all where it's tested.
-        // This may need updates for other platforms.
-        assert_eq!(obj.architecture(), Architecture::X86_64);
-
         // Append the `.xdata` section, or the actual unwinding information
         // codes and such which were built as we found unwind information for
         // functions.
@@ -409,14 +456,35 @@ impl<'a> UnwindInfoBuilder<'a> {
         // `xdata` section follows the `.text` section so the
         // `text_section_size` is added in to calculate the final
         // `.text`-section-relative address of the unwind information.
-        let mut pdata = Vec::with_capacity(self.windows_pdata.len() * 3 * 4);
-        for info in self.windows_pdata.iter() {
-            pdata.extend_from_slice(&info.begin.to_le_bytes());
-            pdata.extend_from_slice(&info.end.to_le_bytes());
-            let address = text_section_size + u64::from(info.unwind_address);
-            let address = u32::try_from(address).unwrap();
-            pdata.extend_from_slice(&address.to_le_bytes());
-        }
+        let xdata_rva = |address| {
+            let address = u64::from(address);
+            let address = address + text_section_size;
+            u32::try_from(address).unwrap()
+        };
+        let pdata = match obj.architecture() {
+            Architecture::X86_64 => {
+                let mut pdata = Vec::with_capacity(self.windows_pdata.len() * 3 * 4);
+                for info in self.windows_pdata.iter() {
+                    pdata.extend_from_slice(&info.begin.to_le_bytes());
+                    pdata.extend_from_slice(&info.end.to_le_bytes());
+                    pdata.extend_from_slice(&xdata_rva(info.unwind_address).to_le_bytes());
+                }
+                pdata
+            }
+
+            Architecture::Aarch64 => {
+                // Windows Arm64 .pdata also supports packed unwind data, but
+                // we're not currently using that.
+                let mut pdata = Vec::with_capacity(self.windows_pdata.len() * 2 * 4);
+                for info in self.windows_pdata.iter() {
+                    pdata.extend_from_slice(&info.begin.to_le_bytes());
+                    pdata.extend_from_slice(&xdata_rva(info.unwind_address).to_le_bytes());
+                }
+                pdata
+            }
+
+            _ => unimplemented!("unsupported architecture for windows unwind info"),
+        };
         obj.append_section_data(pdata_id, &pdata, 4);
     }
 
diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs
index 7e1c69bbc..8555a561c 100644
--- a/crates/types/src/lib.rs
+++ b/crates/types/src/lib.rs
@@ -1490,6 +1490,8 @@ pub enum ConstOp {
     I64Add,
     I64Sub,
     I64Mul,
+    StructNew { struct_type_index: TypeIndex },
+    StructNewDefault { struct_type_index: TypeIndex },
 }
 
 impl ConstOp {
@@ -1512,6 +1514,12 @@ impl ConstOp {
             O::I64Add => Self::I64Add,
             O::I64Sub => Self::I64Sub,
             O::I64Mul => Self::I64Mul,
+            O::StructNew { struct_type_index } => Self::StructNew {
+                struct_type_index: TypeIndex::from_u32(struct_type_index),
+            },
+            O::StructNewDefault { struct_type_index } => Self::StructNewDefault {
+                struct_type_index: TypeIndex::from_u32(struct_type_index),
+            },
             op => {
                 return Err(wasm_unsupported!(
                     "unsupported opcode in const expression at offset {offset:#x}: {op:?}",
diff --git a/crates/wasmtime/src/runtime/gc/disabled/anyref.rs b/crates/wasmtime/src/runtime/gc/disabled/anyref.rs
index ee62ec4ae..8357516a0 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/anyref.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/anyref.rs
@@ -52,6 +52,11 @@ impl AnyRef {
         None
     }
 
+    pub unsafe fn _from_raw(_store: &mut AutoAssertNoGc<'_>, raw: u32) -> Option<Rooted<Self>> {
+        assert_eq!(raw, 0);
+        None
+    }
+
     pub unsafe fn to_raw(&self, _store: impl AsContextMut) -> Result<u32> {
         match *self {}
     }
diff --git a/crates/wasmtime/src/runtime/gc/disabled/externref.rs b/crates/wasmtime/src/runtime/gc/disabled/externref.rs
index a89babaf5..55ef89e09 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/externref.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/externref.rs
@@ -43,6 +43,11 @@ impl ExternRef {
         None
     }
 
+    pub unsafe fn _from_raw(_store: &mut AutoAssertNoGc<'_>, raw: u32) -> Option<Rooted<Self>> {
+        assert_eq!(raw, 0);
+        None
+    }
+
     pub unsafe fn to_raw(&self, _store: impl AsContextMut) -> Result<u32> {
         match *self {}
     }
diff --git a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
index ffd54b87b..520aff82c 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
@@ -247,7 +247,11 @@ impl AnyRef {
     /// [`ValRaw`]: crate::ValRaw
     pub unsafe fn to_raw(&self, mut store: impl AsContextMut) -> Result<u32> {
         let mut store = AutoAssertNoGc::new(store.as_context_mut().0);
-        let gc_ref = self.inner.try_clone_gc_ref(&mut store)?;
+        self._to_raw(&mut store)
+    }
+
+    pub(crate) unsafe fn _to_raw(&self, store: &mut AutoAssertNoGc<'_>) -> Result<u32> {
+        let gc_ref = self.inner.try_clone_gc_ref(store)?;
         let raw = gc_ref.as_raw_u32();
         store.gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref);
         Ok(raw)
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index bcdee4159..6f6fec630 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -1194,6 +1194,47 @@ where
     }
 }
 
+/// Internal version of `RootScope` that only wraps a `&mut StoreOpaque` rather
+/// than a whole `impl AsContextMut<Data = T>`.
+pub(crate) struct OpaqueRootScope<'a> {
+    store: &'a mut StoreOpaque,
+    scope: usize,
+}
+
+impl Drop for OpaqueRootScope<'_> {
+    fn drop(&mut self) {
+        self.store.exit_gc_lifo_scope(self.scope);
+    }
+}
+
+impl Deref for OpaqueRootScope<'_> {
+    type Target = StoreOpaque;
+
+    fn deref(&self) -> &Self::Target {
+        self.store
+    }
+}
+
+// XXX: Don't use this `DerefMut` implementation to `mem::{swap,replace}` or
+// etc... the underlying `StoreOpaque` in a `OpaqueRootScope`! That will result
+// in truncating the store's GC root set's LIFO roots to the wrong length.
+//
+// We don't implement `DerefMut` for `RootScope` for exactly this reason, but
+// allow it for `OpaqueRootScope` because it is only Wasmtime-internal and not
+// publicly exported.
+impl DerefMut for OpaqueRootScope<'_> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.store
+    }
+}
+
+impl<'a> OpaqueRootScope<'a> {
+    pub(crate) fn new(store: &'a mut StoreOpaque) -> Self {
+        let scope = store.gc_roots().enter_lifo_scope();
+        OpaqueRootScope { store, scope }
+    }
+}
+
 /// A rooted reference to a garbage-collected `T` with arbitrary lifetime.
 ///
 /// A `ManuallyRooted<T>` is a strong handle to a garbage-collected `T`,
diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs
index b537ce1cb..108ec7368 100644
--- a/crates/wasmtime/src/runtime/values.rs
+++ b/crates/wasmtime/src/runtime/values.rs
@@ -235,7 +235,16 @@ impl Val {
     /// This method is unsafe for the reasons that [`ExternRef::from_raw`] and
     /// [`Func::from_raw`] are unsafe. Additionally there's no guarantee
     /// otherwise that `raw` should have the type `ty` specified.
-    pub unsafe fn from_raw(store: impl AsContextMut, raw: ValRaw, ty: ValType) -> Val {
+    pub unsafe fn from_raw(mut store: impl AsContextMut, raw: ValRaw, ty: ValType) -> Val {
+        let mut store = AutoAssertNoGc::new(store.as_context_mut().0);
+        Self::_from_raw(&mut store, raw, &ty)
+    }
+
+    pub(crate) unsafe fn _from_raw(
+        store: &mut AutoAssertNoGc<'_>,
+        raw: ValRaw,
+        ty: &ValType,
+    ) -> Val {
         match ty {
             ValType::I32 => Val::I32(raw.get_i32()),
             ValType::I64 => Val::I64(raw.get_i64()),
@@ -245,12 +254,12 @@ impl Val {
             ValType::Ref(ref_ty) => {
                 let ref_ = match ref_ty.heap_type() {
                     HeapType::Func | HeapType::ConcreteFunc(_) => {
-                        Func::from_raw(store, raw.get_funcref()).into()
+                        Func::_from_raw(store, raw.get_funcref()).into()
                     }
 
                     HeapType::NoFunc => Ref::Func(None),
 
-                    HeapType::Extern => ExternRef::from_raw(store, raw.get_externref()).into(),
+                    HeapType::Extern => ExternRef::_from_raw(store, raw.get_externref()).into(),
 
                     HeapType::NoExtern => Ref::Extern(None),
 
@@ -261,7 +270,7 @@ impl Val {
                     | HeapType::ConcreteArray(_)
                     | HeapType::Struct
                     | HeapType::ConcreteStruct(_) => {
-                        AnyRef::from_raw(store, raw.get_anyref()).into()
+                        AnyRef::_from_raw(store, raw.get_anyref()).into()
                     }
 
                     HeapType::None => Ref::Any(None),
diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs
index f4c178cd0..6eca60a7b 100644
--- a/crates/wasmtime/src/runtime/vm/const_expr.rs
+++ b/crates/wasmtime/src/runtime/vm/const_expr.rs
@@ -1,9 +1,13 @@
 //! Evaluating const expressions.
 
-use crate::prelude::*;
 use crate::runtime::vm::{Instance, VMGcRef, ValRaw, I31};
+use crate::store::AutoAssertNoGc;
+use crate::{prelude::*, StructRef, StructRefPre, StructType, Val};
 use smallvec::SmallVec;
-use wasmtime_environ::{ConstExpr, FuncIndex, GlobalIndex, Module};
+use wasmtime_environ::{
+    ConstExpr, ConstOp, FuncIndex, GlobalIndex, Module, ModuleInternedTypeIndex, WasmCompositeType,
+    WasmSubType,
+};
 
 /// An interpreter for const expressions.
 ///
@@ -26,15 +30,15 @@ impl<'a, 'b> ConstEvalContext<'a, 'b> {
         Self { instance, module }
     }
 
-    fn global_get(&mut self, index: GlobalIndex) -> Result<ValRaw> {
+    fn global_get(&mut self, store: &mut AutoAssertNoGc<'_>, index: GlobalIndex) -> Result<ValRaw> {
         unsafe {
-            let gc_store = (*self.instance.store()).unwrap_gc_store_mut();
             let global = self
                 .instance
                 .defined_or_imported_global_ptr(index)
                 .as_ref()
                 .unwrap();
-            Ok(global.to_val_raw(gc_store, self.module.globals[index].wasm_ty))
+            let mut gc_store = store.unwrap_gc_store_mut();
+            Ok(global.to_val_raw(&mut gc_store, self.module.globals[index].wasm_ty))
         }
     }
 
@@ -43,6 +47,108 @@ impl<'a, 'b> ConstEvalContext<'a, 'b> {
             self.instance.get_func_ref(index).unwrap().cast(),
         ))
     }
+
+    #[cfg(feature = "gc")]
+    fn struct_fields_len(&self, struct_type_index: ModuleInternedTypeIndex) -> usize {
+        let module = self
+            .instance
+            .runtime_module()
+            .expect("should never be allocating a struct type defined in a dummy module");
+
+        let struct_ty = match &module.types()[struct_type_index].composite_type {
+            WasmCompositeType::Struct(s) => s,
+            _ => unreachable!(),
+        };
+
+        struct_ty.fields.len()
+    }
+
+    /// Safety: field values must be of the correct types.
+    #[cfg(feature = "gc")]
+    unsafe fn struct_new(
+        &mut self,
+        store: &mut AutoAssertNoGc<'_>,
+        struct_type_index: ModuleInternedTypeIndex,
+        fields: &[ValRaw],
+    ) -> Result<ValRaw> {
+        let module = self
+            .instance
+            .runtime_module()
+            .expect("should never be allocating a struct type defined in a dummy module");
+        let shared_ty = module
+            .signatures()
+            .shared_type(struct_type_index)
+            .expect("should have an engine type for module type");
+
+        let struct_ty = StructType::from_shared_type_index(store.engine(), shared_ty);
+        let fields = fields
+            .iter()
+            .zip(struct_ty.fields())
+            .map(|(raw, ty)| {
+                let ty = ty.element_type().unpack();
+                let mut store = AutoAssertNoGc::new(store);
+                Val::_from_raw(&mut store, *raw, ty)
+            })
+            .collect::<Vec<_>>();
+
+        let allocator = StructRefPre::_new(store, struct_ty);
+        let struct_ref = StructRef::_new(store, &allocator, &fields)?;
+        let raw = struct_ref.to_anyref()._to_raw(store)?;
+        Ok(ValRaw::anyref(raw))
+    }
+
+    #[cfg(feature = "gc")]
+    fn struct_new_default(
+        &mut self,
+        store: &mut AutoAssertNoGc<'_>,
+        struct_type_index: ModuleInternedTypeIndex,
+    ) -> Result<ValRaw> {
+        let module = self
+            .instance
+            .runtime_module()
+            .expect("should never be allocating a struct type defined in a dummy module");
+
+        let shared_ty = module
+            .signatures()
+            .shared_type(struct_type_index)
+            .expect("should have an engine type for module type");
+
+        let borrowed = module
+            .engine()
+            .signatures()
+            .borrow(shared_ty)
+            .expect("should have a registered type for struct");
+        let WasmSubType {
+            composite_type: WasmCompositeType::Struct(struct_ty),
+            ..
+        } = &*borrowed
+        else {
+            unreachable!("registered type should be a struct");
+        };
+
+        let fields = struct_ty
+            .fields
+            .iter()
+            .map(|ty| match &ty.element_type {
+                wasmtime_environ::WasmStorageType::I8 | wasmtime_environ::WasmStorageType::I16 => {
+                    ValRaw::i32(0)
+                }
+                wasmtime_environ::WasmStorageType::Val(v) => match v {
+                    wasmtime_environ::WasmValType::I32 => ValRaw::i32(0),
+                    wasmtime_environ::WasmValType::I64 => ValRaw::i64(0),
+                    wasmtime_environ::WasmValType::F32 => ValRaw::f32(0.0f32.to_bits()),
+                    wasmtime_environ::WasmValType::F64 => ValRaw::f64(0.0f64.to_bits()),
+                    wasmtime_environ::WasmValType::V128 => ValRaw::v128(0),
+                    wasmtime_environ::WasmValType::Ref(r) => {
+                        assert!(r.nullable);
+                        ValRaw::null()
+                    }
+                },
+            })
+            .collect::<SmallVec<[_; 8]>>();
+
+        unsafe { self.struct_new(store, struct_type_index, &fields) }
+    }
 }
 
 impl ConstExprEvaluator {
@@ -62,52 +168,104 @@ impl ConstExprEvaluator {
     ) -> Result<ValRaw> {
         self.stack.clear();
 
+        let mut store = (*context.instance.store()).store_opaque_mut();
+
+        // Ensure that we don't permanently root any GC references we allocate
+        // during const evaluation, keeping them alive for the duration of the
+        // store's lifetime.
+        #[cfg(feature = "gc")]
+        let mut store = crate::OpaqueRootScope::new(&mut store);
+
+        // We cannot allow GC during const evaluation because the stack of
+        // `ValRaw`s are not rooted. If we had a GC reference on our stack, and
+        // then performed a collection, that on-stack reference's object could
+        // be reclaimed or relocated by the collector, and then when we use the
+        // reference again we would basically get a use-after-free bug.
+        let mut store = AutoAssertNoGc::new(&mut store);
+
         for op in expr.ops() {
             match op {
-                wasmtime_environ::ConstOp::I32Const(i) => self.stack.push(ValRaw::i32(*i)),
-                wasmtime_environ::ConstOp::I64Const(i) => self.stack.push(ValRaw::i64(*i)),
-                wasmtime_environ::ConstOp::F32Const(f) => self.stack.push(ValRaw::f32(*f)),
-                wasmtime_environ::ConstOp::F64Const(f) => self.stack.push(ValRaw::f64(*f)),
-                wasmtime_environ::ConstOp::V128Const(v) => self.stack.push(ValRaw::v128(*v)),
-                wasmtime_environ::ConstOp::GlobalGet(g) => self.stack.push(context.global_get(*g)?),
-                wasmtime_environ::ConstOp::RefNull => self.stack.push(ValRaw::null()),
-                wasmtime_environ::ConstOp::RefFunc(f) => self.stack.push(context.ref_func(*f)?),
-                wasmtime_environ::ConstOp::RefI31 => {
+                ConstOp::I32Const(i) => self.stack.push(ValRaw::i32(*i)),
+                ConstOp::I64Const(i) => self.stack.push(ValRaw::i64(*i)),
+                ConstOp::F32Const(f) => self.stack.push(ValRaw::f32(*f)),
+                ConstOp::F64Const(f) => self.stack.push(ValRaw::f64(*f)),
+                ConstOp::V128Const(v) => self.stack.push(ValRaw::v128(*v)),
+                ConstOp::GlobalGet(g) => self.stack.push(context.global_get(&mut store, *g)?),
+                ConstOp::RefNull => self.stack.push(ValRaw::null()),
+                ConstOp::RefFunc(f) => self.stack.push(context.ref_func(*f)?),
+                ConstOp::RefI31 => {
                     let i = self.pop()?.get_i32();
                     let i31 = I31::wrapping_i32(i);
                     let raw = VMGcRef::from_i31(i31).as_raw_u32();
                     self.stack.push(ValRaw::anyref(raw));
                 }
-                wasmtime_environ::ConstOp::I32Add => {
+                ConstOp::I32Add => {
                     let b = self.pop()?.get_i32();
                     let a = self.pop()?.get_i32();
                     self.stack.push(ValRaw::i32(a.wrapping_add(b)));
                 }
-                wasmtime_environ::ConstOp::I32Sub => {
+                ConstOp::I32Sub => {
                     let b = self.pop()?.get_i32();
                     let a = self.pop()?.get_i32();
                     self.stack.push(ValRaw::i32(a.wrapping_sub(b)));
                 }
-                wasmtime_environ::ConstOp::I32Mul => {
+                ConstOp::I32Mul => {
                     let b = self.pop()?.get_i32();
                     let a = self.pop()?.get_i32();
                     self.stack.push(ValRaw::i32(a.wrapping_mul(b)));
                 }
-                wasmtime_environ::ConstOp::I64Add => {
+                ConstOp::I64Add => {
                     let b = self.pop()?.get_i64();
                     let a = self.pop()?.get_i64();
                     self.stack.push(ValRaw::i64(a.wrapping_add(b)));
                 }
-                wasmtime_environ::ConstOp::I64Sub => {
+                ConstOp::I64Sub => {
                     let b = self.pop()?.get_i64();
                     let a = self.pop()?.get_i64();
                     self.stack.push(ValRaw::i64(a.wrapping_sub(b)));
                 }
-                wasmtime_environ::ConstOp::I64Mul => {
+                ConstOp::I64Mul => {
                     let b = self.pop()?.get_i64();
                     let a = self.pop()?.get_i64();
                     self.stack.push(ValRaw::i64(a.wrapping_mul(b)));
                 }
+
+                #[cfg(not(feature = "gc"))]
+                ConstOp::StructNew { .. } | ConstOp::StructNewDefault { .. } => {
+                    bail!(
+                        "const expr evaluation error: struct operations are not \
+                         supported without the `gc` feature"
+                    )
+                }
+
+                #[cfg(feature = "gc")]
+                ConstOp::StructNew { struct_type_index } => {
+                    let interned_type_index = context.module.types[*struct_type_index];
+                    let len = context.struct_fields_len(interned_type_index);
+
+                    if self.stack.len() < len {
+                        bail!(
+                            "const expr evaluation error: expected at least {len} values on the stack, found {}",
+                            self.stack.len()
+                        )
+                    }
+
+                    let start = self.stack.len() - len;
+                    let s = context.struct_new(
+                        &mut store,
+                        interned_type_index,
+                        &self.stack[start..],
+                    )?;
+                    self.stack.truncate(start);
+                    self.stack.push(s);
+                }
+
+                #[cfg(feature = "gc")]
+                ConstOp::StructNewDefault { struct_type_index } => {
+                    let interned_type_index = context.module.types[*struct_type_index];
+                    self.stack
+                        .push(context.struct_new_default(&mut store, interned_type_index)?);
+                }
             }
         }
 
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
index c806fbb4f..6a7cdc35c 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
@@ -4,25 +4,23 @@ use crate::V128;
 use core::mem;
 
 /// A plain-old-data type that can be stored in a `ValType` or a `StorageType`.
-///
-/// Safety: implementations must be POD and all bit patterns must be valid.
 pub trait PodValType<const SIZE: usize>: Copy {
-    /// Read an instance of `Self` from the given little-endian bytes.
-    fn read_le(le_bytes: &[u8; SIZE]) -> Self;
+    /// Read an instance of `Self` from the given native-endian bytes.
+    fn read_ne(ne_bytes: &[u8; SIZE]) -> Self;
 
-    /// Write `self` into the given memory location, as little-endian bytes.
-    fn write_le(&self, into: &mut [u8; SIZE]);
+    /// Write `self` into the given memory location, as native-endian bytes.
+    fn write_ne(&self, into: &mut [u8; SIZE]);
 }
 
 macro_rules! impl_pod_val_type {
     ( $( $t:ty , )* ) => {
         $(
             impl PodValType<{mem::size_of::<$t>()}> for $t {
-                fn read_le(le_bytes: &[u8; mem::size_of::<$t>()]) -> Self {
-                    <$t>::from_le_bytes(*le_bytes)
+                fn read_ne(ne_bytes: &[u8; mem::size_of::<$t>()]) -> Self {
+                    <$t>::from_ne_bytes(*ne_bytes)
                 }
-                fn write_le(&self, into: &mut [u8; mem::size_of::<$t>()]) {
-                    *into = self.to_le_bytes();
+                fn write_ne(&self, into: &mut [u8; mem::size_of::<$t>()]) {
+                    *into = self.to_ne_bytes();
                 }
             }
         )*
@@ -41,11 +39,11 @@ impl_pod_val_type! {
 }
 
 impl PodValType<{ mem::size_of::<V128>() }> for V128 {
-    fn read_le(le_bytes: &[u8; mem::size_of::<V128>()]) -> Self {
-        u128::from_le_bytes(*le_bytes).into()
+    fn read_ne(ne_bytes: &[u8; mem::size_of::<V128>()]) -> Self {
+        u128::from_ne_bytes(*ne_bytes).into()
     }
-    fn write_le(&self, into: &mut [u8; mem::size_of::<V128>()]) {
-        *into = self.as_u128().to_le_bytes();
+    fn write_ne(&self, into: &mut [u8; mem::size_of::<V128>()]) {
+        *into = self.as_u128().to_ne_bytes();
     }
 }
 
@@ -114,7 +112,7 @@ impl<'a> VMGcObjectDataMut<'a> {
         let offset = usize::try_from(offset).unwrap();
         let end = offset.checked_add(N).unwrap();
         let bytes = self.data.get(offset..end).expect("out of bounds field");
-        T::read_le(bytes.try_into().unwrap())
+        T::read_ne(bytes.try_into().unwrap())
     }
 
     /// Read a POD field out of this object.
@@ -132,7 +130,7 @@ impl<'a> VMGcObjectDataMut<'a> {
         let offset = usize::try_from(offset).unwrap();
         let end = offset.checked_add(N).unwrap();
         let into = self.data.get_mut(offset..end).expect("out of bounds field");
-        val.write_le(into.try_into().unwrap());
+        val.write_ne(into.try_into().unwrap());
     }
 
     impl_pod_methods! {
diff --git a/crates/wasmtime/src/runtime/vm/sys/windows/unwind.rs b/crates/wasmtime/src/runtime/vm/sys/windows/unwind.rs
index eb559302b..86406d6b4 100644
--- a/crates/wasmtime/src/runtime/vm/sys/windows/unwind.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/windows/unwind.rs
@@ -19,11 +19,16 @@ impl UnwindRegistration {
         unwind_info: *const u8,
         unwind_len: usize,
     ) -> Result<UnwindRegistration> {
+        #[cfg(target_arch = "aarch64")]
+        use IMAGE_ARM64_RUNTIME_FUNCTION_ENTRY as Entry;
+        #[cfg(not(target_arch = "aarch64"))]
+        use IMAGE_RUNTIME_FUNCTION_ENTRY as Entry;
+
         assert!(unwind_info as usize % 4 == 0);
-        let unit_len = mem::size_of::<IMAGE_RUNTIME_FUNCTION_ENTRY>();
+        let unit_len = mem::size_of::<Entry>();
         assert!(unwind_len % unit_len == 0);
         if RtlAddFunctionTable(
-            unwind_info as *mut _,
+            unwind_info as *mut Entry,
             (unwind_len / unit_len) as u32,
             base_address as _,
         ) == 0
diff --git a/pulley/fuzz/src/interp.rs b/pulley/fuzz/src/interp.rs
index 2777ef1fc..5fd70b082 100644
--- a/pulley/fuzz/src/interp.rs
+++ b/pulley/fuzz/src/interp.rs
@@ -32,7 +32,7 @@ pub fn interp(ops: Vec<Op>) {
         match vm.call(NonNull::from(&encoded[0]), args, rets.into_iter().copied()) {
             Ok(rets) => assert_eq!(rets.count(), 0),
             Err(pc) => {
-                let pc = pc as usize;
+                let pc = pc.as_ptr() as usize;
 
                 let start = &encoded[0] as *const u8 as usize;
                 let end = encoded.last().unwrap() as *const u8 as usize;
diff --git a/pulley/src/decode.rs b/pulley/src/decode.rs
index 54c84304f..1a0c8b202 100644
--- a/pulley/src/decode.rs
+++ b/pulley/src/decode.rs
@@ -1,5 +1,7 @@
 //! Decoding support for pulley bytecode.
 
+use core::ptr::NonNull;
+
 use alloc::vec::Vec;
 use cranelift_bitset::scalar::ScalarBitSetStorage;
 use cranelift_bitset::ScalarBitSet;
@@ -194,7 +196,7 @@ pub enum Uninhabited {}
 ///
 /// This is a wrapper over a raw pointer to bytecode somewhere in memory.
 #[derive(Clone, Copy, Debug)]
-pub struct UnsafeBytecodeStream(*mut u8);
+pub struct UnsafeBytecodeStream(NonNull<u8>);
 
 impl UnsafeBytecodeStream {
     /// Construct a new `UnsafeBytecodeStream` pointing at the given PC.
@@ -208,8 +210,7 @@ impl UnsafeBytecodeStream {
     /// new PC, this stream must not be used to read just after the
     /// unconditional jump instruction because there is no guarantee that that
     /// memory is part of the bytecode stream or not.
-    pub unsafe fn new(pc: *mut u8) -> Self {
-        assert!(!pc.is_null());
+    pub unsafe fn new(pc: NonNull<u8>) -> Self {
         UnsafeBytecodeStream(pc)
     }
 
@@ -222,20 +223,19 @@ impl UnsafeBytecodeStream {
     /// that the address at `self._as_ptr() + offset` contains valid Pulley
     /// bytecode.
     pub unsafe fn offset(&self, offset: isize) -> Self {
-        UnsafeBytecodeStream(self.0.offset(offset))
+        UnsafeBytecodeStream(NonNull::new_unchecked(self.0.as_ptr().offset(offset)))
     }
 
     /// Get this stream's underlying raw pointer.
-    pub fn as_ptr(&self) -> *mut u8 {
+    pub fn as_ptr(&self) -> NonNull<u8> {
         self.0
     }
 }
 
 impl BytecodeStream for UnsafeBytecodeStream {
     fn read<const N: usize>(&mut self) -> Result<[u8; N], Self::Error> {
-        debug_assert!(!self.0.is_null());
-        let bytes = unsafe { self.0.cast::<[u8; N]>().read() };
-        self.0 = unsafe { self.0.add(N) };
+        let bytes = unsafe { self.0.cast::<[u8; N]>().as_ptr().read() };
+        self.0 = unsafe { NonNull::new_unchecked(self.0.as_ptr().add(N)) };
         Ok(bytes)
     }
 
diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs
index 664b9cf72..24b3e2a59 100644
--- a/pulley/src/interp.rs
+++ b/pulley/src/interp.rs
@@ -72,7 +72,7 @@ impl Vm {
         func: NonNull<u8>,
         args: &[Val],
         rets: impl IntoIterator<Item = RegType> + 'a,
-    ) -> Result<impl Iterator<Item = Val> + 'a, *mut u8> {
+    ) -> Result<impl Iterator<Item = Val> + 'a, NonNull<u8>> {
         // NB: make sure this method stays in sync with
         // `PulleyMachineDeps::compute_arg_locs`!
 
@@ -97,7 +97,7 @@ impl Vm {
             }
         }
 
-        self.run(func.as_ptr())?;
+        self.run(func)?;
 
         let mut x_rets = (0..16).map(|x| XReg::new_unchecked(x));
         let mut f_rets = (0..16).map(|f| FReg::new_unchecked(f));
@@ -119,7 +119,7 @@ impl Vm {
         }))
     }
 
-    unsafe fn run(&mut self, pc: *mut u8) -> Result<(), *mut u8> {
+    unsafe fn run(&mut self, pc: NonNull<u8>) -> Result<(), NonNull<u8>> {
         let mut visitor = InterpreterVisitor {
             state: &mut self.state,
             pc: UnsafeBytecodeStream::new(pc),
@@ -148,23 +148,25 @@ impl Vm {
 
     #[cold]
     #[inline(never)]
-    fn return_to_host(&self) -> Result<(), *mut u8> {
+    fn return_to_host(&self) -> Result<(), NonNull<u8>> {
         Ok(())
     }
 
     #[cold]
     #[inline(never)]
-    fn trap(&self, pc: *mut u8) -> Result<(), *mut u8> {
+    fn trap(&self, pc: NonNull<u8>) -> Result<(), NonNull<u8>> {
         // We are given the VM's PC upon having executed a trap instruction,
         // which is actually pointing to the next instruction after the
         // trap. Back the PC up to point exactly at the trap.
-        let trap_pc = unsafe { pc.byte_sub(ExtendedOpcode::ENCODED_SIZE_OF_TRAP) };
+        let trap_pc = unsafe {
+            NonNull::new_unchecked(pc.as_ptr().byte_sub(ExtendedOpcode::ENCODED_SIZE_OF_TRAP))
+        };
         Err(trap_pc)
     }
 
     #[cold]
     #[inline(never)]
-    fn host_call(&self) -> Result<(), *mut u8> {
+    fn host_call(&self) -> Result<(), NonNull<u8>> {
         todo!()
     }
 }
@@ -624,15 +626,15 @@ impl OpVisitor for InterpreterVisitor<'_> {
             Continuation::ReturnToHost
         } else {
             let return_addr = self.state[XReg::lr].get_ptr();
-            self.pc = unsafe { UnsafeBytecodeStream::new(return_addr) };
+            self.pc = unsafe { UnsafeBytecodeStream::new(NonNull::new_unchecked(return_addr)) };
             // log::trace!("returning to {return_addr:#p}");
             Continuation::Continue
         }
     }
 
     fn call(&mut self, offset: PcRelOffset) -> Self::Return {
-        let return_addr = u64::try_from(self.pc.as_ptr() as usize).unwrap();
-        self.state[XReg::lr].set_u64(return_addr);
+        let return_addr = self.pc.as_ptr();
+        self.state[XReg::lr].set_ptr(return_addr.as_ptr());
         self.pc_rel_jump(offset, 5)
     }
 


Test patch:
diff --git a/pulley/tests/all/interp.rs b/pulley/tests/all/interp.rs
index 085c09cf3..8015cfed1 100644
--- a/pulley/tests/all/interp.rs
+++ b/pulley/tests/all/interp.rs
@@ -13,7 +13,7 @@ fn encoded(ops: &[Op]) -> Vec<u8> {
     encoded
 }
 
-unsafe fn run(vm: &mut Vm, ops: &[Op]) -> Result<(), *mut u8> {
+unsafe fn run(vm: &mut Vm, ops: &[Op]) -> Result<(), NonNull<u8>> {
     let _ = env_logger::try_init();
     let ops = encoded(ops);
     let _ = vm.call(NonNull::from(&ops[..]).cast(), &[], [])?;
diff --git a/tests/wast.rs b/tests/wast.rs
index 7a2f77c9b..00af5cc15 100644
--- a/tests/wast.rs
+++ b/tests/wast.rs
@@ -216,7 +216,6 @@ fn should_fail(test: &Path, strategy: Strategy) -> bool {
         "ref_test.wast",
         "return_call_indirect.wast",
         "return_call.wast",
-        "struct.wast",
         "table_sub.wast",
         "type_canon.wast",
         "type_equivalence.wast",