Repository: bytecodealliance/wasmtime
Pull request #8310 (2024-04-05 21:59:31)

Disable traps by default in `bindgen!` imports
By default previously all return types were wrapped in `wasmtime::Result<T>` to enable any import to return a trap to the wasm guest. This is a fair bit of boilerplate, however, and it's easy to accidentally turn a normal error into a trap. This is additionally somewhat of a power-user method as many imports probably don't end up wanting to trap.

This commit adds a new configuration option to `bindgen!`: `trappable_imports`, and switches the default to `false`. The previous behavior can be recovered with `trappable_imports: true`.

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->



Patch:
diff --git a/RELEASES.md b/RELEASES.md
index 5fbf568bf..9fad07589 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -24,6 +24,15 @@ Unreleased.
 
 --------------------------------------------------------------------------------
 
+## 19.0.1
+
+Released 2024-04-02.
+
+* Fix a panic using tables with the wrong type.
+  [#8284](https://github.com/bytecodealliance/wasmtime/pull/8284)
+
+--------------------------------------------------------------------------------
+
 ## 19.0.0
 
 Released 2024-03-20
@@ -101,8 +110,8 @@ Released 2024-03-20
   specification.
   [#7661](https://github.com/bytecodealliance/wasmtime/pull/7661)
 
-* Support for `externref` has been moved behind a new `gc` Cargo feature. This
-  will also gate support for wasm gc in the future.
+* Support for `externref` and similar features has been moved behind a new `gc`
+  Cargo feature. This will also gate support for Wasm GC in the future.
   [#7975](https://github.com/bytecodealliance/wasmtime/pull/7975)
 
 * Require the `WASMTIME_WASI_CONFIG_PREOPEN_SOCKET_ALLOW` environment variable
diff --git a/cranelift/codegen/src/bitset.rs b/cranelift/codegen/src/bitset.rs
index 7d81b3221..70c86d722 100644
--- a/cranelift/codegen/src/bitset.rs
+++ b/cranelift/codegen/src/bitset.rs
@@ -10,13 +10,35 @@ use core::mem::size_of;
 use core::ops::{Add, BitOr, Shl, Sub};
 
 /// A small bitset built on a single primitive integer type
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Copy, Default, PartialEq, Eq)]
 #[cfg_attr(
     feature = "enable-serde",
     derive(serde_derive::Serialize, serde_derive::Deserialize)
 )]
 pub struct BitSet<T>(pub T);
 
+impl<T> std::fmt::Debug for BitSet<T>
+where
+    T: Into<u32>
+        + From<u8>
+        + BitOr<T, Output = T>
+        + Shl<u8, Output = T>
+        + Sub<T, Output = T>
+        + Add<T, Output = T>
+        + PartialEq
+        + Copy,
+{
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        let mut s = f.debug_struct(std::any::type_name::<Self>());
+        for i in 0..Self::bits() {
+            use std::string::ToString;
+            let i = u32::try_from(i).unwrap();
+            s.field(&i.to_string(), &self.contains(i));
+        }
+        s.finish()
+    }
+}
+
 impl<T> BitSet<T>
 where
     T: Into<u32>
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index c5efafaee..93e955c9a 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -31,6 +31,7 @@ use crate::{timing, CompileError};
 use alloc::string::String;
 use alloc::vec::Vec;
 use cranelift_control::ControlPlane;
+use target_lexicon::Architecture;
 
 #[cfg(feature = "souper-harvest")]
 use crate::souper_harvest::do_souper_harvest;
@@ -282,7 +283,15 @@ impl Context {
 
     /// Perform NaN canonicalizing rewrites on the function.
     pub fn canonicalize_nans(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
-        do_nan_canonicalization(&mut self.func);
+        // Currently only RiscV64 is the only arch that may not have vector support.
+        let has_vector_support = match isa.triple().architecture {
+            Architecture::Riscv64(_) => match isa.isa_flags().iter().find(|f| f.name == "has_v") {
+                Some(value) => value.as_bool().unwrap_or(false),
+                None => false,
+            },
+            _ => true,
+        };
+        do_nan_canonicalization(&mut self.func, has_vector_support);
         self.verify_if(isa)
     }
 
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index 707a01c1a..c0e7ed665 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -44,6 +44,17 @@ pub fn has_side_effect(func: &Function, inst: Inst) -> bool {
     trivially_has_side_effects(opcode) || is_load_with_defined_trapping(opcode, data)
 }
 
+/// Is the given instruction a bitcast to or from a reference type (e.g. `r64`)?
+pub fn is_bitcast_from_ref(func: &Function, inst: Inst) -> bool {
+    let op = func.dfg.insts[inst].opcode();
+    if op != ir::Opcode::Bitcast {
+        return false;
+    }
+
+    let arg = func.dfg.inst_args(inst)[0];
+    func.dfg.value_type(arg).is_ref()
+}
+
 /// Does the given instruction behave as a "pure" node with respect to
 /// aegraph semantics?
 ///
@@ -58,6 +69,7 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
         } => flags.readonly() && flags.notrap(),
         _ => false,
     };
+
     // Multi-value results do not play nicely with much of the egraph
     // infrastructure. They are in practice used only for multi-return
     // calls and some other odd instructions (e.g. uadd_overflow) which,
@@ -70,7 +82,11 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
 
     let op = func.dfg.insts[inst].opcode();
 
-    has_one_result && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
+    has_one_result
+        && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
+        // Cannot optimize ref-y bitcasts, as that can interact badly with
+        // safepoints and stack maps.
+        && !is_bitcast_from_ref(func, inst)
 }
 
 /// Can the given instruction be merged into another copy of itself?
@@ -90,6 +106,9 @@ pub fn is_mergeable_for_egraph(func: &Function, inst: Inst) -> bool {
         && !op.can_store()
         // Can only have idempotent side-effects.
         && (!has_side_effect(func, inst) || op.side_effects_idempotent())
+        // Cannot optimize ref-y bitcasts, as that can interact badly with
+        // safepoints and stack maps.
+        && !is_bitcast_from_ref(func, inst)
 }
 
 /// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
index 7f5903dac..8139ac81f 100644
--- a/cranelift/codegen/src/isa/aarch64/abi.rs
+++ b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -36,9 +36,9 @@ impl Into<AMode> for StackAMode {
     fn into(self) -> AMode {
         match self {
             // Argument area begins after saved frame pointer + return address.
-            StackAMode::ArgOffset(off, ty) => AMode::FPOffset { off: off + 16, ty },
-            StackAMode::NominalSPOffset(off, ty) => AMode::NominalSPOffset { off, ty },
-            StackAMode::SPOffset(off, ty) => AMode::SPOffset { off, ty },
+            StackAMode::IncomingArg(off) => AMode::FPOffset { off: off + 16 },
+            StackAMode::Slot(off) => AMode::NominalSPOffset { off },
+            StackAMode::OutgoingArg(off) => AMode::SPOffset { off },
         }
     }
 }
@@ -457,7 +457,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Inst {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Inst {
         // FIXME: Do something different for dynamic types?
         let mem = mem.into();
         Inst::LoadAddr { rd: into_reg, mem }
@@ -471,7 +471,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
         let mem = AMode::RegOffset {
             rn: base,
             off: offset as i64,
-            ty,
         };
         Inst::gen_load(into_reg, mem, ty, MemFlags::trusted())
     }
@@ -480,7 +479,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
         let mem = AMode::RegOffset {
             rn: base,
             off: offset as i64,
-            ty,
         };
         Inst::gen_store(mem, from_reg, ty, MemFlags::trusted())
     }
@@ -1190,7 +1188,7 @@ impl AArch64MachineDeps {
             insts.extend(Self::gen_sp_reg_adjust(-(guard_size as i32)));
 
             insts.push(Inst::gen_store(
-                AMode::SPOffset { off: 0, ty: I8 },
+                AMode::SPOffset { off: 0 },
                 zero_reg(),
                 I32,
                 MemFlags::trusted(),
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 51107ab36..f706f8048 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -1119,15 +1119,13 @@
         ;; Register plus register offset, scaled by type's size.
         (RegScaled
          (rn Reg)
-         (rm Reg)
-         (ty Type))
+         (rm Reg))
 
         ;; Register plus register offset, scaled by type's size, with index
         ;; sign- or zero-extended first.
         (RegScaledExtended
          (rn Reg)
          (rm Reg)
-         (ty Type)
          (extendop ExtendOp))
 
         ;; Register plus register offset, with index sign- or zero-extended
@@ -1157,18 +1155,15 @@
         ;; offsets with multiple instructions as necessary during code emission.
         (RegOffset
          (rn Reg)
-         (off i64)
-         (ty Type))
+         (off i64))
 
         ;; Offset from the stack pointer.
         (SPOffset
-         (off i64)
-         (ty Type))
+         (off i64))
 
         ;; Offset from the frame pointer.
         (FPOffset
-         (off i64)
-         (ty Type))
+         (off i64))
 
         ;; A reference to a constant which is placed outside of the function's
         ;; body, typically at the end.
@@ -1188,8 +1183,7 @@
         ;; clobber pushes. See the diagram in the documentation for
         ;; [crate::isa::aarch64::abi](the ABI module) for more details.
         (NominalSPOffset
-         (off i64)
-         (ty Type))))
+         (off i64))))
 
 ;; A memory argument to a load/store-pair.
 (type PairAMode (enum
@@ -3169,18 +3163,18 @@
 ;; first and then if that succeeds afterwards try to find an extend.
 (rule 6 (amode_no_more_iconst ty (iadd x (ishl y (iconst (u64_from_imm64 n)))) offset)
         (if-let $true (u64_eq (ty_bytes ty) (u64_shl 1 n)))
-        (amode_reg_scaled (amode_add x offset) y ty))
+        (amode_reg_scaled (amode_add x offset) y))
 (rule 7 (amode_no_more_iconst ty (iadd (ishl y (iconst (u64_from_imm64 n))) x) offset)
         (if-let $true (u64_eq (ty_bytes ty) (u64_shl 1 n)))
-        (amode_reg_scaled (amode_add x offset) y ty))
+        (amode_reg_scaled (amode_add x offset) y))
 
-(decl amode_reg_scaled (Reg Value Type) AMode)
-(rule 0 (amode_reg_scaled base index ty)
-        (AMode.RegScaled base index ty))
-(rule 1 (amode_reg_scaled base (uextend index @ (value_type $I32)) ty)
-        (AMode.RegScaledExtended base index ty (ExtendOp.UXTW)))
-(rule 2 (amode_reg_scaled base (sextend index @ (value_type $I32)) ty)
-        (AMode.RegScaledExtended base index ty (ExtendOp.SXTW)))
+(decl amode_reg_scaled (Reg Value) AMode)
+(rule 0 (amode_reg_scaled base index)
+        (AMode.RegScaled base index))
+(rule 1 (amode_reg_scaled base (uextend index @ (value_type $I32)))
+        (AMode.RegScaledExtended base index (ExtendOp.UXTW)))
+(rule 1 (amode_reg_scaled base (sextend index @ (value_type $I32)))
+        (AMode.RegScaledExtended base index (ExtendOp.SXTW)))
 
 ;; Helper to add a 32-bit signed immediate to the register provided. This will
 ;; select an appropriate `add` instruction to use.
@@ -3519,7 +3513,7 @@
             ;; the frame record that corresponds to the current subroutine on
             ;; the stack; the presence of the record is guaranteed by the
             ;; `preserve_frame_pointers` setting.
-            (addr AMode (AMode.FPOffset 8 $I64))
+            (addr AMode (AMode.FPOffset 8))
             (_ Unit (emit (MInst.ULoad64 dst addr (mem_flags_trusted)))))
            dst))
 
@@ -3529,7 +3523,7 @@
       ;; the frame record. Furthermore, we can use LR as a scratch register
       ;; because the function will set it to the return address immediately
       ;; before returning.
-      (let ((addr AMode (AMode.FPOffset 8 $I64))
+      (let ((addr AMode (AMode.FPOffset 8))
             (lr WritableReg (writable_link_reg))
             (_ Unit (emit (MInst.ULoad64 lr addr (mem_flags_trusted))))
             (_ Unit (emit (MInst.Xpaclri))))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs
index a9f185bf6..3375215d2 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/args.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs
@@ -138,11 +138,10 @@ impl AMode {
 
     /// Memory reference using `reg1 + sizeof(ty) * reg2` as an address, with `reg2` sign- or
     /// zero-extended as per `op`.
-    pub fn reg_plus_reg_scaled_extended(reg1: Reg, reg2: Reg, ty: Type, op: ExtendOp) -> AMode {
+    pub fn reg_plus_reg_scaled_extended(reg1: Reg, reg2: Reg, op: ExtendOp) -> AMode {
         AMode::RegScaledExtended {
             rn: reg1,
             rm: reg2,
-            ty,
             extendop: op,
         }
     }
@@ -162,20 +161,13 @@ impl AMode {
                 rn: allocs.next(rn),
                 rm: allocs.next(rm),
             },
-            &AMode::RegScaled { rn, rm, ty } => AMode::RegScaled {
+            &AMode::RegScaled { rn, rm } => AMode::RegScaled {
                 rn: allocs.next(rn),
                 rm: allocs.next(rm),
-                ty,
             },
-            &AMode::RegScaledExtended {
-                rn,
-                rm,
-                ty,
-                extendop,
-            } => AMode::RegScaledExtended {
+            &AMode::RegScaledExtended { rn, rm, extendop } => AMode::RegScaledExtended {
                 rn: allocs.next(rn),
                 rm: allocs.next(rm),
-                ty,
                 extendop,
             },
             &AMode::RegExtended { rn, rm, extendop } => AMode::RegExtended {
@@ -183,10 +175,9 @@ impl AMode {
                 rm: allocs.next(rm),
                 extendop,
             },
-            &AMode::RegOffset { rn, off, ty } => AMode::RegOffset {
+            &AMode::RegOffset { rn, off } => AMode::RegOffset {
                 rn: allocs.next(rn),
                 off,
-                ty,
             },
             &AMode::SPPreIndexed { .. }
             | &AMode::SPPostIndexed { .. }
@@ -385,19 +376,20 @@ impl PrettyPrint for MemLabel {
     }
 }
 
-fn shift_for_type(ty: Type) -> usize {
-    match ty.bytes() {
+fn shift_for_type(size_bytes: u8) -> usize {
+    match size_bytes {
         1 => 0,
         2 => 1,
         4 => 2,
         8 => 3,
         16 => 4,
-        _ => panic!("unknown type: {}", ty),
+        _ => panic!("unknown type size: {size_bytes}"),
     }
 }
 
 impl PrettyPrint for AMode {
-    fn pretty_print(&self, _: u8, allocs: &mut AllocationConsumer<'_>) -> String {
+    fn pretty_print(&self, size_bytes: u8, allocs: &mut AllocationConsumer<'_>) -> String {
+        debug_assert!(size_bytes != 0);
         match self {
             &AMode::Unscaled { rn, simm9 } => {
                 let reg = pretty_print_reg(rn, allocs);
@@ -410,7 +402,7 @@ impl PrettyPrint for AMode {
             }
             &AMode::UnsignedOffset { rn, uimm12 } => {
                 let reg = pretty_print_reg(rn, allocs);
-                if uimm12.value != 0 {
+                if uimm12.value() != 0 {
                     let uimm12 = uimm12.pretty_print(8, allocs);
                     format!("[{}, {}]", reg, uimm12)
                 } else {
@@ -422,19 +414,14 @@ impl PrettyPrint for AMode {
                 let r2 = pretty_print_reg(rm, allocs);
                 format!("[{}, {}]", r1, r2)
             }
-            &AMode::RegScaled { rn, rm, ty } => {
+            &AMode::RegScaled { rn, rm } => {
                 let r1 = pretty_print_reg(rn, allocs);
                 let r2 = pretty_print_reg(rm, allocs);
-                let shift = shift_for_type(ty);
+                let shift = shift_for_type(size_bytes);
                 format!("[{}, {}, LSL #{}]", r1, r2, shift)
             }
-            &AMode::RegScaledExtended {
-                rn,
-                rm,
-                ty,
-                extendop,
-            } => {
-                let shift = shift_for_type(ty);
+            &AMode::RegScaledExtended { rn, rm, extendop } => {
+                let shift = shift_for_type(size_bytes);
                 let size = match extendop {
                     ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32,
                     _ => OperandSize::Size64,
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index b02fe200e..df840e5b1 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -15,13 +15,14 @@ use crate::trace;
 pub fn mem_finalize(
     sink: Option<&mut MachBuffer<Inst>>,
     mem: &AMode,
+    access_ty: Type,
     state: &EmitState,
 ) -> (SmallVec<[Inst; 4]>, AMode) {
     match mem {
-        &AMode::RegOffset { off, ty, .. }
-        | &AMode::SPOffset { off, ty }
-        | &AMode::FPOffset { off, ty }
-        | &AMode::NominalSPOffset { off, ty } => {
+        &AMode::RegOffset { off, .. }
+        | &AMode::SPOffset { off }
+        | &AMode::FPOffset { off }
+        | &AMode::NominalSPOffset { off } => {
             let basereg = match mem {
                 &AMode::RegOffset { rn, .. } => rn,
                 &AMode::SPOffset { .. } | &AMode::NominalSPOffset { .. } => stack_reg(),
@@ -45,7 +46,7 @@ pub fn mem_finalize(
             if let Some(simm9) = SImm9::maybe_from_i64(off) {
                 let mem = AMode::Unscaled { rn: basereg, simm9 };
                 (smallvec![], mem)
-            } else if let Some(uimm12) = UImm12Scaled::maybe_from_i64(off, ty) {
+            } else if let Some(uimm12) = UImm12Scaled::maybe_from_i64(off, access_ty) {
                 let mem = AMode::UnsignedOffset {
                     rn: basereg,
                     uimm12,
@@ -979,7 +980,8 @@ impl MachInstEmit for Inst {
             | &Inst::FpuLoad128 { rd, ref mem, flags } => {
                 let rd = allocs.next_writable(rd);
                 let mem = mem.with_allocs(&mut allocs);
-                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, state);
+                let access_ty = self.mem_type().unwrap();
+                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, access_ty, state);
 
                 for inst in mem_insts.into_iter() {
                     inst.emit(&[], sink, emit_info, state);
@@ -991,17 +993,17 @@ impl MachInstEmit for Inst {
                 // This is the base opcode (top 10 bits) for the "unscaled
                 // immediate" form (Unscaled). Other addressing modes will OR in
                 // other values for bits 24/25 (bits 1/2 of this constant).
-                let (op, bits) = match self {
-                    &Inst::ULoad8 { .. } => (0b0011100001, 8),
-                    &Inst::SLoad8 { .. } => (0b0011100010, 8),
-                    &Inst::ULoad16 { .. } => (0b0111100001, 16),
-                    &Inst::SLoad16 { .. } => (0b0111100010, 16),
-                    &Inst::ULoad32 { .. } => (0b1011100001, 32),
-                    &Inst::SLoad32 { .. } => (0b1011100010, 32),
-                    &Inst::ULoad64 { .. } => (0b1111100001, 64),
-                    &Inst::FpuLoad32 { .. } => (0b1011110001, 32),
-                    &Inst::FpuLoad64 { .. } => (0b1111110001, 64),
-                    &Inst::FpuLoad128 { .. } => (0b0011110011, 128),
+                let op = match self {
+                    Inst::ULoad8 { .. } => 0b0011100001,
+                    Inst::SLoad8 { .. } => 0b0011100010,
+                    Inst::ULoad16 { .. } => 0b0111100001,
+                    Inst::SLoad16 { .. } => 0b0111100010,
+                    Inst::ULoad32 { .. } => 0b1011100001,
+                    Inst::SLoad32 { .. } => 0b1011100010,
+                    Inst::ULoad64 { .. } => 0b1111100001,
+                    Inst::FpuLoad32 { .. } => 0b1011110001,
+                    Inst::FpuLoad64 { .. } => 0b1111110001,
+                    Inst::FpuLoad128 { .. } => 0b0011110011,
                     _ => unreachable!(),
                 };
 
@@ -1017,9 +1019,6 @@ impl MachInstEmit for Inst {
                     }
                     &AMode::UnsignedOffset { rn, uimm12 } => {
                         let reg = allocs.next(rn);
-                        if uimm12.value() != 0 {
-                            assert_eq!(bits, ty_bits(uimm12.scale_ty()));
-                        }
                         sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
                     }
                     &AMode::RegReg { rn, rm } => {
@@ -1029,11 +1028,9 @@ impl MachInstEmit for Inst {
                             op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
                         ));
                     }
-                    &AMode::RegScaled { rn, rm, ty }
-                    | &AMode::RegScaledExtended { rn, rm, ty, .. } => {
+                    &AMode::RegScaled { rn, rm } | &AMode::RegScaledExtended { rn, rm, .. } => {
                         let r1 = allocs.next(rn);
                         let r2 = allocs.next(rm);
-                        assert_eq!(bits, ty_bits(ty));
                         let extendop = match &mem {
                             &AMode::RegScaled { .. } => None,
                             &AMode::RegScaledExtended { extendop, .. } => Some(extendop),
@@ -1123,20 +1120,21 @@ impl MachInstEmit for Inst {
             | &Inst::FpuStore128 { rd, ref mem, flags } => {
                 let rd = allocs.next(rd);
                 let mem = mem.with_allocs(&mut allocs);
-                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, state);
+                let access_ty = self.mem_type().unwrap();
+                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, access_ty, state);
 
                 for inst in mem_insts.into_iter() {
                     inst.emit(&[], sink, emit_info, state);
                 }
 
-                let (op, bits) = match self {
-                    &Inst::Store8 { .. } => (0b0011100000, 8),
-                    &Inst::Store16 { .. } => (0b0111100000, 16),
-                    &Inst::Store32 { .. } => (0b1011100000, 32),
-                    &Inst::Store64 { .. } => (0b1111100000, 64),
-                    &Inst::FpuStore32 { .. } => (0b1011110000, 32),
-                    &Inst::FpuStore64 { .. } => (0b1111110000, 64),
-                    &Inst::FpuStore128 { .. } => (0b0011110010, 128),
+                let op = match self {
+                    Inst::Store8 { .. } => 0b0011100000,
+                    Inst::Store16 { .. } => 0b0111100000,
+                    Inst::Store32 { .. } => 0b1011100000,
+                    Inst::Store64 { .. } => 0b1111100000,
+                    Inst::FpuStore32 { .. } => 0b1011110000,
+                    Inst::FpuStore64 { .. } => 0b1111110000,
+                    Inst::FpuStore128 { .. } => 0b0011110010,
                     _ => unreachable!(),
                 };
 
@@ -1152,9 +1150,6 @@ impl MachInstEmit for Inst {
                     }
                     &AMode::UnsignedOffset { rn, uimm12 } => {
                         let reg = allocs.next(rn);
-                        if uimm12.value() != 0 {
-                            assert_eq!(bits, ty_bits(uimm12.scale_ty()));
-                        }
                         sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
                     }
                     &AMode::RegReg { rn, rm } => {
@@ -1164,7 +1159,7 @@ impl MachInstEmit for Inst {
                             op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
                         ));
                     }
-                    &AMode::RegScaled { rn, rm, .. } | &AMode::RegScaledExtended { rn, rm, .. } => {
+                    &AMode::RegScaled { rn, rm } | &AMode::RegScaledExtended { rn, rm, .. } => {
                         let r1 = allocs.next(rn);
                         let r2 = allocs.next(rm);
                         let extendop = match &mem {
@@ -3341,7 +3336,6 @@ impl MachInstEmit for Inst {
                     mem: AMode::reg_plus_reg_scaled_extended(
                         rtmp1.to_reg(),
                         rtmp2.to_reg(),
-                        I32,
                         ExtendOp::UXTW,
                     ),
                     flags: MemFlags::trusted(),
@@ -3436,7 +3430,7 @@ impl MachInstEmit for Inst {
             &Inst::LoadAddr { rd, ref mem } => {
                 let rd = allocs.next_writable(rd);
                 let mem = mem.with_allocs(&mut allocs);
-                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, state);
+                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, I8, state);
                 for inst in mem_insts.into_iter() {
                     inst.emit(&[], sink, emit_info, state);
                 }
@@ -3874,7 +3868,6 @@ fn emit_return_call_common_sequence(
             rd: tmp2,
             mem: AMode::SPOffset {
                 off: i64::from(i * 8),
-                ty: types::I64,
             },
             flags: ir::MemFlags::trusted(),
         }
@@ -3886,7 +3879,6 @@ fn emit_return_call_common_sequence(
             rd: tmp2.to_reg(),
             mem: AMode::FPOffset {
                 off: fp_to_callee_sp + i64::from(i * 8),
-                ty: types::I64,
             },
             flags: ir::MemFlags::trusted(),
         }
diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rs
index cd8a2aa76..b87209dda 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs
@@ -220,9 +220,9 @@ impl SImm9 {
 #[derive(Clone, Copy, Debug)]
 pub struct UImm12Scaled {
     /// The value.
-    pub value: u16,
+    value: u16,
     /// multiplied by the size of this type
-    pub scale_ty: Type,
+    scale_ty: Type,
 }
 
 impl UImm12Scaled {
@@ -257,11 +257,6 @@ impl UImm12Scaled {
     pub fn value(&self) -> u32 {
         self.value as u32
     }
-
-    /// The value type which is the scaling base.
-    pub fn scale_ty(&self) -> Type {
-        self.scale_ty
-    }
 }
 
 /// A shifted immediate value in 'imm12' format: supports 12 bits, shifted
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 82fdccab8..e1447d60a 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -367,6 +367,32 @@ impl Inst {
             }
         }
     }
+
+    /// What type does this load or store instruction access in memory? When
+    /// uimm12 encoding is used, the size of this type is the amount that
+    /// immediate offsets are scaled by.
+    pub fn mem_type(&self) -> Option<Type> {
+        match self {
+            Inst::ULoad8 { .. } => Some(I8),
+            Inst::SLoad8 { .. } => Some(I8),
+            Inst::ULoad16 { .. } => Some(I16),
+            Inst::SLoad16 { .. } => Some(I16),
+            Inst::ULoad32 { .. } => Some(I32),
+            Inst::SLoad32 { .. } => Some(I32),
+            Inst::ULoad64 { .. } => Some(I64),
+            Inst::FpuLoad32 { .. } => Some(F32),
+            Inst::FpuLoad64 { .. } => Some(F64),
+            Inst::FpuLoad128 { .. } => Some(I8X16),
+            Inst::Store8 { .. } => Some(I8),
+            Inst::Store16 { .. } => Some(I16),
+            Inst::Store32 { .. } => Some(I32),
+            Inst::Store64 { .. } => Some(I64),
+            Inst::FpuStore32 { .. } => Some(F32),
+            Inst::FpuStore64 { .. } => Some(F64),
+            Inst::FpuStore128 { .. } => Some(I8X16),
+            _ => None,
+        }
+    }
 }
 
 //=============================================================================
@@ -1211,8 +1237,8 @@ impl MachInst for Inst {
 //=============================================================================
 // Pretty-printing of instructions.
 
-fn mem_finalize_for_show(mem: &AMode, state: &EmitState) -> (String, AMode) {
-    let (mem_insts, mem) = mem_finalize(None, mem, state);
+fn mem_finalize_for_show(mem: &AMode, access_ty: Type, state: &EmitState) -> (String, String) {
+    let (mem_insts, mem) = mem_finalize(None, mem, access_ty, state);
     let mut mem_str = mem_insts
         .into_iter()
         .map(|inst| {
@@ -1224,6 +1250,7 @@ fn mem_finalize_for_show(mem: &AMode, state: &EmitState) -> (String, AMode) {
         mem_str += " ; ";
     }
 
+    let mem = mem.pretty_print(access_ty.bytes() as u8, &mut AllocationConsumer::new(&[]));
     (mem_str, mem)
 }
 
@@ -1411,8 +1438,8 @@ impl Inst {
 
                 let rd = pretty_print_ireg(rd.to_reg(), size, allocs);
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
 
                 format!("{}{} {}, {}", mem_str, op, rd, mem)
             }
@@ -1438,8 +1465,8 @@ impl Inst {
 
                 let rd = pretty_print_ireg(rd, size, allocs);
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
 
                 format!("{}{} {}, {}", mem_str, op, rd, mem)
             }
@@ -1822,45 +1849,45 @@ impl Inst {
             &Inst::FpuLoad32 { rd, ref mem, .. } => {
                 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32, allocs);
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
                 format!("{}ldr {}, {}", mem_str, rd, mem)
             }
             &Inst::FpuLoad64 { rd, ref mem, .. } => {
                 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64, allocs);
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
                 format!("{}ldr {}, {}", mem_str, rd, mem)
             }
             &Inst::FpuLoad128 { rd, ref mem, .. } => {
                 let rd = pretty_print_reg(rd.to_reg(), allocs);
                 let rd = "q".to_string() + &rd[1..];
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
                 format!("{}ldr {}, {}", mem_str, rd, mem)
             }
             &Inst::FpuStore32 { rd, ref mem, .. } => {
                 let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size32, allocs);
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
                 format!("{}str {}, {}", mem_str, rd, mem)
             }
             &Inst::FpuStore64 { rd, ref mem, .. } => {
                 let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size64, allocs);
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
                 format!("{}str {}, {}", mem_str, rd, mem)
             }
             &Inst::FpuStore128 { rd, ref mem, .. } => {
                 let rd = pretty_print_reg(rd, allocs);
                 let rd = "q".to_string() + &rd[1..];
                 let mem = mem.with_allocs(allocs);
-                let (mem_str, mem) = mem_finalize_for_show(&mem, state);
-                let mem = mem.pretty_print_default();
+                let access_ty = self.mem_type().unwrap();
+                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
                 format!("{}str {}, {}", mem_str, rd, mem)
             }
             &Inst::FpuLoadP64 {
@@ -2771,7 +2798,7 @@ impl Inst {
                 // of the existing legalization framework).
                 let rd = allocs.next_writable(rd);
                 let mem = mem.with_allocs(allocs);
-                let (mem_insts, mem) = mem_finalize(None, &mem, state);
+                let (mem_insts, mem) = mem_finalize(None, &mem, I8, state);
                 let mut ret = String::new();
                 for inst in mem_insts.into_iter() {
                     ret.push_str(
diff --git a/cranelift/codegen/src/isa/aarch64/pcc.rs b/cranelift/codegen/src/isa/aarch64/pcc.rs
index e17daffd4..cdc6ca42e 100644
--- a/cranelift/codegen/src/isa/aarch64/pcc.rs
+++ b/cranelift/codegen/src/isa/aarch64/pcc.rs
@@ -38,7 +38,8 @@ pub(crate) fn check(
     inst_idx: InsnIndex,
     state: &mut FactFlowState,
 ) -> PccResult<()> {
-    trace!("Checking facts on inst: {:?}", vcode[inst_idx]);
+    let inst = &vcode[inst_idx];
+    trace!("Checking facts on inst: {:?}", inst);
 
     // We only persist flag state for one instruction, because we
     // can't exhaustively enumerate all flags-effecting ops; so take
@@ -47,28 +48,29 @@ pub(crate) fn check(
     let cmp_flags = state.cmp_flags.take();
     trace!(" * with cmp_flags = {cmp_flags:?}");
 
-    match vcode[inst_idx] {
+    match *inst {
         Inst::Args { .. } => {
             // Defs on the args have "axiomatic facts": we trust the
             // ABI code to pass through the values unharmed, so the
             // facts given to us in the CLIF should still be true.
             Ok(())
         }
-        Inst::ULoad8 { rd, ref mem, flags } | Inst::SLoad8 { rd, ref mem, flags } => {
-            check_load(ctx, Some(rd.to_reg()), flags, mem, vcode, I8)
+        Inst::ULoad8 { rd, ref mem, flags }
+        | Inst::SLoad8 { rd, ref mem, flags }
+        | Inst::ULoad16 { rd, ref mem, flags }
+        | Inst::SLoad16 { rd, ref mem, flags }
+        | Inst::ULoad32 { rd, ref mem, flags }
+        | Inst::SLoad32 { rd, ref mem, flags }
+        | Inst::ULoad64 { rd, ref mem, flags } => {
+            let access_ty = inst.mem_type().unwrap();
+            check_load(ctx, Some(rd.to_reg()), flags, mem, vcode, access_ty)
         }
-        Inst::ULoad16 { rd, ref mem, flags } | Inst::SLoad16 { rd, ref mem, flags } => {
-            check_load(ctx, Some(rd.to_reg()), flags, mem, vcode, I16)
+        Inst::FpuLoad32 { ref mem, flags, .. }
+        | Inst::FpuLoad64 { ref mem, flags, .. }
+        | Inst::FpuLoad128 { ref mem, flags, .. } => {
+            let access_ty = inst.mem_type().unwrap();
+            check_load(ctx, None, flags, mem, vcode, access_ty)
         }
-        Inst::ULoad32 { rd, ref mem, flags } | Inst::SLoad32 { rd, ref mem, flags } => {
-            check_load(ctx, Some(rd.to_reg()), flags, mem, vcode, I32)
-        }
-        Inst::ULoad64 { rd, ref mem, flags } => {
-            check_load(ctx, Some(rd.to_reg()), flags, mem, vcode, I64)
-        }
-        Inst::FpuLoad32 { ref mem, flags, .. } => check_load(ctx, None, flags, mem, vcode, F32),
-        Inst::FpuLoad64 { ref mem, flags, .. } => check_load(ctx, None, flags, mem, vcode, F64),
-        Inst::FpuLoad128 { ref mem, flags, .. } => check_load(ctx, None, flags, mem, vcode, I8X16),
         Inst::LoadP64 { ref mem, flags, .. } => check_load_pair(ctx, flags, mem, vcode, 16),
         Inst::FpuLoadP64 { ref mem, flags, .. } => check_load_pair(ctx, flags, mem, vcode, 16),
         Inst::FpuLoadP128 { ref mem, flags, .. } => check_load_pair(ctx, flags, mem, vcode, 32),
@@ -82,14 +84,18 @@ pub(crate) fn check(
             ..
         } => check_load_addr(ctx, flags, rn, vcode, access_ty),
 
-        Inst::Store8 { rd, ref mem, flags } => check_store(ctx, Some(rd), flags, mem, vcode, I8),
-        Inst::Store16 { rd, ref mem, flags } => check_store(ctx, Some(rd), flags, mem, vcode, I16),
-        Inst::Store32 { rd, ref mem, flags } => check_store(ctx, Some(rd), flags, mem, vcode, I32),
-        Inst::Store64 { rd, ref mem, flags } => check_store(ctx, Some(rd), flags, mem, vcode, I64),
-        Inst::FpuStore32 { ref mem, flags, .. } => check_store(ctx, None, flags, mem, vcode, F32),
-        Inst::FpuStore64 { ref mem, flags, .. } => check_store(ctx, None, flags, mem, vcode, F64),
-        Inst::FpuStore128 { ref mem, flags, .. } => {
-            check_store(ctx, None, flags, mem, vcode, I8X16)
+        Inst::Store8 { rd, ref mem, flags }
+        | Inst::Store16 { rd, ref mem, flags }
+        | Inst::Store32 { rd, ref mem, flags }
+        | Inst::Store64 { rd, ref mem, flags } => {
+            let access_ty = inst.mem_type().unwrap();
+            check_store(ctx, Some(rd), flags, mem, vcode, access_ty)
+        }
+        Inst::FpuStore32 { ref mem, flags, .. }
+        | Inst::FpuStore64 { ref mem, flags, .. }
+        | Inst::FpuStore128 { ref mem, flags, .. } => {
+            let access_ty = inst.mem_type().unwrap();
+            check_store(ctx, None, flags, mem, vcode, access_ty)
         }
         Inst::StoreP64 { ref mem, flags, .. } => check_store_pair(ctx, flags, mem, vcode, 16),
         Inst::FpuStoreP64 { ref mem, flags, .. } => check_store_pair(ctx, flags, mem, vcode, 16),
@@ -444,19 +450,14 @@ fn check_addr<'a>(
             trace!("rn = {rn:?} rm = {rm:?} sum = {sum:?}");
             check(&sum, ty)
         }
-        &AMode::RegScaled { rn, rm, ty } => {
+        &AMode::RegScaled { rn, rm } => {
             let rn = get_fact_or_default(vcode, rn, 64);
             let rm = get_fact_or_default(vcode, rm, 64);
             let rm_scaled = fail_if_missing(ctx.scale(&rm, 64, ty.bytes()))?;
             let sum = fail_if_missing(ctx.add(&rn, &rm_scaled, 64))?;
             check(&sum, ty)
         }
-        &AMode::RegScaledExtended {
-            rn,
-            rm,
-            ty,
-            extendop,
-        } => {
+        &AMode::RegScaledExtended { rn, rm, extendop } => {
             let rn = get_fact_or_default(vcode, rn, 64);
             let rm = get_fact_or_default(vcode, rm, 64);
             let rm_extended = fail_if_missing(extend_fact(ctx, &rm, extendop))?;
@@ -485,7 +486,7 @@ fn check_addr<'a>(
             // 32760 (= 4095 * 8) for I64s. The `UImm12Scaled` type
             // stores the *scaled* value, so we don't need to multiply
             // (again) by the type's size here.
-            let offset: u64 = uimm12.value.into();
+            let offset: u64 = uimm12.value().into();
             // This `unwrap()` will always succeed because the value
             // will always be positive and much smaller than
             // `i64::MAX` (see above).
diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs
index a497c72e1..dca2cda68 100644
--- a/cranelift/codegen/src/isa/riscv64/abi.rs
+++ b/cranelift/codegen/src/isa/riscv64/abi.rs
@@ -293,7 +293,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
         insts
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Inst {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Inst {
         Inst::LoadAddr {
             rd: into_reg,
             mem: mem.into(),
@@ -301,12 +301,12 @@ impl ABIMachineSpec for Riscv64MachineDeps {
     }
 
     fn gen_load_base_offset(into_reg: Writable<Reg>, base: Reg, offset: i32, ty: Type) -> Inst {
-        let mem = AMode::RegOffset(base, offset as i64, ty);
+        let mem = AMode::RegOffset(base, offset as i64);
         Inst::gen_load(into_reg, mem, ty, MemFlags::trusted())
     }
 
     fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst {
-        let mem = AMode::RegOffset(base, offset as i64, ty);
+        let mem = AMode::RegOffset(base, offset as i64);
         Inst::gen_store(mem, from_reg, ty, MemFlags::trusted())
     }
 
@@ -359,13 +359,13 @@ impl ABIMachineSpec for Riscv64MachineDeps {
             // mv   fp,sp        ;; set fp to sp.
             insts.extend(Self::gen_sp_reg_adjust(-16));
             insts.push(Inst::gen_store(
-                AMode::SPOffset(8, I64),
+                AMode::SPOffset(8),
                 link_reg(),
                 I64,
                 MemFlags::trusted(),
             ));
             insts.push(Inst::gen_store(
-                AMode::SPOffset(0, I64),
+                AMode::SPOffset(0),
                 fp_reg(),
                 I64,
                 MemFlags::trusted(),
@@ -399,13 +399,13 @@ impl ABIMachineSpec for Riscv64MachineDeps {
         if frame_layout.setup_area_size > 0 {
             insts.push(Inst::gen_load(
                 writable_link_reg(),
-                AMode::SPOffset(8, I64),
+                AMode::SPOffset(8),
                 I64,
                 MemFlags::trusted(),
             ));
             insts.push(Inst::gen_load(
                 writable_fp_reg(),
-                AMode::SPOffset(0, I64),
+                AMode::SPOffset(0),
                 I64,
                 MemFlags::trusted(),
             ));
@@ -488,7 +488,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
                     });
                 }
                 insts.push(Inst::gen_store(
-                    AMode::SPOffset(-(cur_offset as i64), ty),
+                    AMode::SPOffset(-(cur_offset as i64)),
                     real_reg_to_reg(reg.to_reg()),
                     ty,
                     MemFlags::trusted(),
@@ -521,7 +521,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
             };
             insts.push(Inst::gen_load(
                 Writable::from_reg(real_reg_to_reg(reg.to_reg())),
-                AMode::SPOffset(-cur_offset, ty),
+                AMode::SPOffset(-cur_offset),
                 ty,
                 MemFlags::trusted(),
             ));
@@ -1096,7 +1096,7 @@ impl Riscv64MachineDeps {
             });
 
             insts.push(Inst::gen_store(
-                AMode::SPOffset(0, I8),
+                AMode::SPOffset(0),
                 zero_reg(),
                 I32,
                 MemFlags::trusted(),
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 3cc710196..b77e66922 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -2280,19 +2280,19 @@
     (writable_reg_to_reg sum)))
 
 ;; Generates a AMode that points to a register plus an offset.
-(decl gen_reg_offset_amode (Reg i64 Type) AMode)
+(decl gen_reg_offset_amode (Reg i64) AMode)
 (extern constructor gen_reg_offset_amode gen_reg_offset_amode)
 
 ;; Generates a AMode that an offset from the stack pointer.
-(decl gen_sp_offset_amode (i64 Type) AMode)
+(decl gen_sp_offset_amode (i64) AMode)
 (extern constructor gen_sp_offset_amode gen_sp_offset_amode)
 
 ;; Generates a AMode that an offset from the frame pointer.
-(decl gen_fp_offset_amode (i64 Type) AMode)
+(decl gen_fp_offset_amode (i64) AMode)
 (extern constructor gen_fp_offset_amode gen_fp_offset_amode)
 
 ;; Generates an AMode that points to a stack slot + offset.
-(decl gen_stack_slot_amode (StackSlot i64 Type) AMode)
+(decl gen_stack_slot_amode (StackSlot i64) AMode)
 (extern constructor gen_stack_slot_amode gen_stack_slot_amode)
 
 ;; Generates a AMode that points to a constant in the constant pool.
@@ -2302,41 +2302,41 @@
 
 
 ;; Tries to match a Value + Offset into an AMode
-(decl amode (Value i32 Type) AMode)
-(rule 0 (amode addr offset ty) (amode_inner addr offset ty))
+(decl amode (Value i32) AMode)
+(rule 0 (amode addr offset) (amode_inner addr offset))
 
 ;; If we are adding a constant offset with an iadd we can instead make that
 ;; offset part of the amode offset.
 ;;
 ;; We can't recurse into `amode` again since that could cause stack overflows.
 ;; See: https://github.com/bytecodealliance/wasmtime/pull/6968
-(rule 1 (amode (iadd addr (iconst (simm32 y))) offset ty)
+(rule 1 (amode (iadd addr (iconst (simm32 y))) offset)
   (if-let new_offset (s32_add_fallible y offset))
-  (amode_inner addr new_offset ty))
-(rule 2 (amode (iadd (iconst (simm32 x)) addr) offset ty)
+  (amode_inner addr new_offset))
+(rule 2 (amode (iadd (iconst (simm32 x)) addr) offset)
   (if-let new_offset (s32_add_fallible x offset))
-  (amode_inner addr new_offset ty))
+  (amode_inner addr new_offset))
 
 
 ;; These are the normal rules for generating an AMode.
-(decl amode_inner (Value i32 Type) AMode)
+(decl amode_inner (Value i32) AMode)
 
 ;; In the simplest case we just lower into a Reg+Offset
-(rule 0 (amode_inner r @ (value_type (ty_addr64 _)) offset ty)
-  (gen_reg_offset_amode r offset ty))
+(rule 0 (amode_inner r @ (value_type (ty_addr64 _)) offset)
+  (gen_reg_offset_amode r offset))
 
 ;; If the value is a `get_frame_pointer`, we can just use the offset from that.
-(rule 1 (amode_inner (get_frame_pointer) offset ty)
-  (gen_fp_offset_amode offset ty))
+(rule 1 (amode_inner (get_frame_pointer) offset)
+  (gen_fp_offset_amode offset))
 
 ;; If the value is a `get_stack_pointer`, we can just use the offset from that.
-(rule 1 (amode_inner (get_stack_pointer) offset ty)
-  (gen_sp_offset_amode offset ty))
+(rule 1 (amode_inner (get_stack_pointer) offset)
+  (gen_sp_offset_amode offset))
 
 ;; Similarly if the value is a `stack_addr` we can also turn that into an sp offset.
-(rule 1 (amode_inner (stack_addr ss ss_offset) amode_offset ty)
+(rule 1 (amode_inner (stack_addr ss ss_offset) amode_offset)
   (if-let combined_offset (s32_add_fallible ss_offset amode_offset))
-  (gen_stack_slot_amode ss combined_offset ty))
+  (gen_stack_slot_amode ss combined_offset))
 
 
 
diff --git a/cranelift/codegen/src/isa/riscv64/inst/args.rs b/cranelift/codegen/src/isa/riscv64/inst/args.rs
index f90168f54..4f8c7beed 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/args.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/args.rs
@@ -80,12 +80,12 @@ newtype_of_reg!(VReg, WritableVReg, |reg| reg.class() == RegClass::Vector);
 pub enum AMode {
     /// Arbitrary offset from a register. Converted to generation of large
     /// offsets with multiple instructions as necessary during code emission.
-    RegOffset(Reg, i64, Type),
+    RegOffset(Reg, i64),
     /// Offset from the stack pointer.
-    SPOffset(i64, Type),
+    SPOffset(i64),
 
     /// Offset from the frame pointer.
-    FPOffset(i64, Type),
+    FPOffset(i64),
 
     /// Offset from the "nominal stack pointer", which is where the real SP is
     /// just after stack and spill slots are allocated in the function prologue.
@@ -99,7 +99,7 @@ pub enum AMode {
     /// SP" is where the actual SP is after the function prologue and before
     /// clobber pushes. See the diagram in the documentation for
     /// [crate::isa::riscv64::abi](the ABI module) for more details.
-    NominalSPOffset(i64, Type),
+    NominalSPOffset(i64),
 
     /// A reference to a constant which is placed outside of the function's
     /// body, typically at the end.
@@ -112,7 +112,7 @@ pub enum AMode {
 impl AMode {
     pub(crate) fn with_allocs(self, allocs: &mut AllocationConsumer<'_>) -> Self {
         match self {
-            AMode::RegOffset(reg, offset, ty) => AMode::RegOffset(allocs.next(reg), offset, ty),
+            AMode::RegOffset(reg, offset) => AMode::RegOffset(allocs.next(reg), offset),
             AMode::SPOffset(..)
             | AMode::FPOffset(..)
             | AMode::NominalSPOffset(..)
@@ -146,10 +146,10 @@ impl AMode {
 
     pub(crate) fn get_offset_with_state(&self, state: &EmitState) -> i64 {
         match self {
-            &AMode::NominalSPOffset(offset, _) => offset + state.virtual_sp_offset,
-            &AMode::RegOffset(_, offset, ..) => offset,
-            &AMode::SPOffset(offset, _) => offset,
-            &AMode::FPOffset(offset, _) => offset,
+            &AMode::NominalSPOffset(offset) => offset + state.virtual_sp_offset,
+            &AMode::RegOffset(_, offset) => offset,
+            &AMode::SPOffset(offset) => offset,
+            &AMode::FPOffset(offset) => offset,
             &AMode::Const(_) | &AMode::Label(_) => 0,
         }
     }
@@ -200,9 +200,9 @@ impl Into<AMode> for StackAMode {
     fn into(self) -> AMode {
         match self {
             // Argument area begins after saved lr + fp.
-            StackAMode::ArgOffset(offset, ty) => AMode::FPOffset(offset + 16, ty),
-            StackAMode::SPOffset(offset, ty) => AMode::SPOffset(offset, ty),
-            StackAMode::NominalSPOffset(offset, ty) => AMode::NominalSPOffset(offset, ty),
+            StackAMode::IncomingArg(offset) => AMode::FPOffset(offset + 16),
+            StackAMode::OutgoingArg(offset) => AMode::SPOffset(offset),
+            StackAMode::Slot(offset) => AMode::NominalSPOffset(offset),
         }
     }
 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index c4f32c58b..06578d9e5 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -1951,7 +1951,7 @@ impl Inst {
                         rd,
                         op: LoadOP::Ld,
                         flags: MemFlags::trusted(),
-                        from: AMode::RegOffset(rd.to_reg(), 0, I64),
+                        from: AMode::RegOffset(rd.to_reg(), 0),
                     }
                     .emit_uncompressed(sink, emit_info, state, start_off);
                 } else {
@@ -2083,7 +2083,7 @@ impl Inst {
                     rd: rd,
                     op: LoadOP::from_type(ty),
                     flags: MemFlags::new(),
-                    from: AMode::RegOffset(p, 0, ty),
+                    from: AMode::RegOffset(p, 0),
                 }
                 .emit(&[], sink, emit_info, state);
                 Inst::Fence {
@@ -2099,7 +2099,7 @@ impl Inst {
                 }
                 .emit(&[], sink, emit_info, state);
                 Inst::Store {
-                    to: AMode::RegOffset(p, 0, ty),
+                    to: AMode::RegOffset(p, 0),
                     op: StoreOP::from_type(ty),
                     flags: MemFlags::new(),
                     src,
@@ -2599,7 +2599,7 @@ impl Inst {
                 }
                 .emit(&[], sink, emit_info, state);
                 Inst::Store {
-                    to: AMode::RegOffset(spilltmp_reg2(), 0, I8),
+                    to: AMode::RegOffset(spilltmp_reg2(), 0),
                     op: StoreOP::Sb,
                     flags: MemFlags::new(),
                     src: zero_reg(),
@@ -3404,12 +3404,12 @@ fn emit_return_call_common_sequence(
 
     Inst::gen_load(
         writable_link_reg(),
-        AMode::FPOffset(8, I64),
+        AMode::FPOffset(8),
         I64,
         MemFlags::trusted(),
     )
     .emit(&[], sink, emit_info, state);
-    Inst::gen_load(tmp1, AMode::FPOffset(0, I64), I64, MemFlags::trusted()).emit(
+    Inst::gen_load(tmp1, AMode::FPOffset(0), I64, MemFlags::trusted()).emit(
         &[],
         sink,
         emit_info,
@@ -3422,7 +3422,7 @@ fn emit_return_call_common_sequence(
         // space.
         Inst::gen_load(
             tmp2,
-            AMode::SPOffset(i64::from(i * 8), types::I64),
+            AMode::SPOffset(i64::from(i * 8)),
             types::I64,
             ir::MemFlags::trusted(),
         )
@@ -3431,7 +3431,7 @@ fn emit_return_call_common_sequence(
         // Store it to its final destination on the stack, overwriting our
         // current frame.
         Inst::gen_store(
-            AMode::FPOffset(fp_to_callee_sp + i64::from(i * 8), types::I64),
+            AMode::FPOffset(fp_to_callee_sp + i64::from(i * 8)),
             tmp2.to_reg(),
             types::I64,
             ir::MemFlags::trusted(),
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 5891cbbe0..a0abd498d 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -1979,41 +1979,41 @@
 
 ;;;;;  Rules for `uload8`;;;;;;;;;
 (rule (lower (uload8 flags addr offset))
-  (gen_load (amode addr offset $I8) (LoadOP.Lbu) flags))
+  (gen_load (amode addr offset) (LoadOP.Lbu) flags))
 
 ;;;;;  Rules for `sload8`;;;;;;;;;
 (rule (lower (sload8 flags addr offset))
-  (gen_load (amode addr offset $I8) (LoadOP.Lb) flags))
+  (gen_load (amode addr offset) (LoadOP.Lb) flags))
 
 ;;;;;  Rules for `uload16`;;;;;;;;;
 (rule (lower (uload16 flags addr offset))
-  (gen_load (amode addr offset $I16) (LoadOP.Lhu) flags))
+  (gen_load (amode addr offset) (LoadOP.Lhu) flags))
 
 ;;;;;  Rules for `iload16`;;;;;;;;;
 (rule (lower (sload16 flags addr offset))
-  (gen_load (amode addr offset $I16) (LoadOP.Lh) flags))
+  (gen_load (amode addr offset) (LoadOP.Lh) flags))
 
 ;;;;;  Rules for `uload32`;;;;;;;;;
 (rule (lower (uload32 flags addr offset))
-  (gen_load (amode addr offset $I32) (LoadOP.Lwu) flags))
+  (gen_load (amode addr offset) (LoadOP.Lwu) flags))
 
 ;;;;;  Rules for `sload32`;;;;;;;;;
 (rule (lower (sload32 flags addr offset))
-  (gen_load (amode addr offset $I32) (LoadOP.Lw) flags))
+  (gen_load (amode addr offset) (LoadOP.Lw) flags))
 
 ;;;;;  Rules for `load`;;;;;;;;;
 (rule (lower (has_type ty (load flags addr offset)))
-  (gen_load (amode addr offset ty) (load_op ty) flags))
+  (gen_load (amode addr offset) (load_op ty) flags))
 
 (rule 1 (lower (has_type $I128 (load flags addr offset)))
   (if-let offset_plus_8 (s32_add_fallible offset 8))
-  (let ((lo XReg (gen_load (amode addr offset $I64) (LoadOP.Ld) flags))
-        (hi XReg (gen_load (amode addr offset_plus_8 $I64) (LoadOP.Ld) flags)))
+  (let ((lo XReg (gen_load (amode addr offset) (LoadOP.Ld) flags))
+        (hi XReg (gen_load (amode addr offset_plus_8) (LoadOP.Ld) flags)))
     (value_regs lo hi)))
 
 (rule 2 (lower (has_type (ty_vec_fits_in_register ty) (load flags addr offset)))
   (let ((eew VecElementWidth (element_width_from_type ty))
-        (amode AMode (amode addr offset ty)))
+        (amode AMode (amode addr offset)))
     (vec_load eew (VecAMode.UnitStride amode) flags (unmasked) ty)))
 
 ;;;;;  Rules for Load + Extend Combos ;;;;;;;;;
@@ -2039,52 +2039,52 @@
 
 ;;;;;  Rules for `uload8x8`;;;;;;;;;;
 (rule (lower (has_type (ty_vec_fits_in_register ty @ $I16X8) (uload8x8 flags addr offset)))
-  (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset ty)))
+  (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)))
 
 ;;;;;  Rules for `uload16x4`;;;;;;;;;
 (rule (lower (has_type (ty_vec_fits_in_register ty @ $I32X4) (uload16x4 flags addr offset)))
-  (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset ty)))
+  (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)))
 
 ;;;;;  Rules for `uload32x2`;;;;;;;;;
 (rule (lower (has_type (ty_vec_fits_in_register ty @ $I64X2) (uload32x2 flags addr offset)))
-  (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset ty)))
+  (gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)))
 
 ;;;;;  Rules for `sload8x8`;;;;;;;;;;
 (rule (lower (has_type (ty_vec_fits_in_register ty @ $I16X8) (sload8x8 flags addr offset)))
-  (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset ty)))
+  (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)))
 
 ;;;;;  Rules for `sload16x4`;;;;;;;;;
 (rule (lower (has_type (ty_vec_fits_in_register ty @ $I32X4) (sload16x4 flags addr offset)))
-  (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset ty)))
+  (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)))
 
 ;;;;;  Rules for `sload32x2`;;;;;;;;;
 (rule (lower (has_type (ty_vec_fits_in_register ty @ $I64X2) (sload32x2 flags addr offset)))
-  (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset ty)))
+  (gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)))
 
 ;;;;;  Rules for `istore8`;;;;;;;;;
 (rule (lower (istore8 flags src addr offset))
-  (gen_store (amode addr offset $I8) (StoreOP.Sb) flags src))
+  (gen_store (amode addr offset) (StoreOP.Sb) flags src))
 
 ;;;;;  Rules for `istore16`;;;;;;;;;
 (rule (lower (istore16 flags src addr offset))
-  (gen_store (amode addr offset $I16) (StoreOP.Sh) flags src))
+  (gen_store (amode addr offset) (StoreOP.Sh) flags src))
 
 ;;;;;  Rules for `istore32`;;;;;;;;;
 (rule (lower (istore32 flags src addr offset))
-  (gen_store (amode addr offset $I32) (StoreOP.Sw) flags src))
+  (gen_store (amode addr offset) (StoreOP.Sw) flags src))
 
 ;;;;;  Rules for `store`;;;;;;;;;
 (rule (lower (store flags src @ (value_type ty) addr offset))
-  (gen_store (amode addr offset ty) (store_op ty) flags src))
+  (gen_store (amode addr offset) (store_op ty) flags src))
 
 (rule 1 (lower (store flags src @ (value_type $I128) addr offset))
   (if-let offset_plus_8 (s32_add_fallible offset 8))
-  (let ((_ InstOutput (gen_store (amode addr offset $I64) (StoreOP.Sd) flags (value_regs_get src 0))))
-    (gen_store (amode addr offset_plus_8 $I64) (StoreOP.Sd) flags (value_regs_get src 1))))
+  (let ((_ InstOutput (gen_store (amode addr offset) (StoreOP.Sd) flags (value_regs_get src 0))))
+    (gen_store (amode addr offset_plus_8) (StoreOP.Sd) flags (value_regs_get src 1))))
 
 (rule 2 (lower (store flags src @ (value_type (ty_vec_fits_in_register ty)) addr offset))
   (let ((eew VecElementWidth (element_width_from_type ty))
-        (amode AMode (amode addr offset ty)))
+        (amode AMode (amode addr offset)))
     (vec_store eew (VecAMode.UnitStride amode) src flags (unmasked) ty)))
 
 
diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
index 261a8f435..065b74631 100644
--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
@@ -177,7 +177,7 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
                 rd: tmp,
                 op: LoadOP::Ld,
                 flags: MemFlags::trusted(),
-                from: AMode::FPOffset(8, I64),
+                from: AMode::FPOffset(8),
             });
             tmp.to_reg()
         } else {
@@ -385,24 +385,24 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
         self.backend.isa_flags.has_zbs()
     }
 
-    fn gen_reg_offset_amode(&mut self, base: Reg, offset: i64, ty: Type) -> AMode {
-        AMode::RegOffset(base, offset, ty)
+    fn gen_reg_offset_amode(&mut self, base: Reg, offset: i64) -> AMode {
+        AMode::RegOffset(base, offset)
     }
 
-    fn gen_sp_offset_amode(&mut self, offset: i64, ty: Type) -> AMode {
-        AMode::SPOffset(offset, ty)
+    fn gen_sp_offset_amode(&mut self, offset: i64) -> AMode {
+        AMode::SPOffset(offset)
     }
 
-    fn gen_fp_offset_amode(&mut self, offset: i64, ty: Type) -> AMode {
-        AMode::FPOffset(offset, ty)
+    fn gen_fp_offset_amode(&mut self, offset: i64) -> AMode {
+        AMode::FPOffset(offset)
     }
 
-    fn gen_stack_slot_amode(&mut self, ss: StackSlot, offset: i64, ty: Type) -> AMode {
+    fn gen_stack_slot_amode(&mut self, ss: StackSlot, offset: i64) -> AMode {
         // Offset from beginning of stackslot area, which is at nominal SP (see
         // [MemArg::NominalSPOffset] for more details on nominal SP tracking).
         let stack_off = self.lower_ctx.abi().sized_stackslot_offsets()[ss] as i64;
         let sp_off: i64 = stack_off + offset;
-        AMode::NominalSPOffset(sp_off, ty)
+        AMode::NominalSPOffset(sp_off)
     }
 
     fn gen_const_amode(&mut self, c: VCodeConstant) -> AMode {
diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs
index b5fb077bf..55ebf6b2f 100644
--- a/cranelift/codegen/src/isa/s390x/abi.rs
+++ b/cranelift/codegen/src/isa/s390x/abi.rs
@@ -192,9 +192,9 @@ impl Into<MemArg> for StackAMode {
     fn into(self) -> MemArg {
         match self {
             // Argument area always begins at the initial SP.
-            StackAMode::ArgOffset(off, _ty) => MemArg::InitialSPOffset { off },
-            StackAMode::NominalSPOffset(off, _ty) => MemArg::NominalSPOffset { off },
-            StackAMode::SPOffset(off, _ty) => {
+            StackAMode::IncomingArg(off) => MemArg::InitialSPOffset { off },
+            StackAMode::Slot(off) => MemArg::NominalSPOffset { off },
+            StackAMode::OutgoingArg(off) => {
                 MemArg::reg_plus_off(stack_reg(), off, MemFlags::trusted())
             }
         }
@@ -495,7 +495,7 @@ impl ABIMachineSpec for S390xMachineDeps {
         insts
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Inst {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Inst {
         let mem = mem.into();
         Inst::LoadAddr { rd: into_reg, mem }
     }
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index b3eaea14c..a4c36a523 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -478,7 +478,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         ]
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Self::I {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Self::I {
         let mem: SyntheticAmode = mem.into();
         Inst::lea(mem, into_reg)
     }
@@ -1018,27 +1018,26 @@ impl From<StackAMode> for SyntheticAmode {
         // We enforce a 128 MB stack-frame size limit above, so these
         // `expect()`s should never fail.
         match amode {
-            StackAMode::ArgOffset(off, _ty) => {
-                let off =
-                    i32::try_from(off + 16) // frame pointer + return address
-                        .expect(
-                            "Offset in ArgOffset is greater than 2GB; should hit impl limit first",
-                        );
+            StackAMode::IncomingArg(off) => {
+                let off = i32::try_from(off + 16) // frame pointer + return address
+                    .expect(
+                        "Offset in IncomingArg is greater than 2GB; should hit impl limit first",
+                    );
                 SyntheticAmode::Real(Amode::ImmReg {
                     simm32: off,
                     base: regs::rbp(),
                     flags: MemFlags::trusted(),
                 })
             }
-            StackAMode::NominalSPOffset(off, _ty) => {
-                let off = i32::try_from(off).expect(
-                    "Offset in NominalSPOffset is greater than 2GB; should hit impl limit first",
-                );
+            StackAMode::Slot(off) => {
+                let off = i32::try_from(off)
+                    .expect("Offset in Slot is greater than 2GB; should hit impl limit first");
                 SyntheticAmode::nominal_sp_offset(off)
             }
-            StackAMode::SPOffset(off, _ty) => {
-                let off = i32::try_from(off)
-                    .expect("Offset in SPOffset is greater than 2GB; should hit impl limit first");
+            StackAMode::OutgoingArg(off) => {
+                let off = i32::try_from(off).expect(
+                    "Offset in OutgoingArg is greater than 2GB; should hit impl limit first",
+                );
                 SyntheticAmode::Real(Amode::ImmReg {
                     simm32: off,
                     base: regs::rsp(),
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 64f738053..e180d35bb 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -1428,6 +1428,7 @@
 (decl pure partial all_ones_or_all_zeros (Value) bool)
 (rule (all_ones_or_all_zeros (and (icmp _ _ _) (value_type (multi_lane _ _)))) $true)
 (rule (all_ones_or_all_zeros (and (fcmp _ _ _) (value_type (multi_lane _ _)))) $true)
+(rule (all_ones_or_all_zeros (and (bitcast _ (fcmp _ _ _)) (value_type (multi_lane _ _)))) $true)
 (rule (all_ones_or_all_zeros (vconst (vconst_all_ones_or_all_zeros))) $true)
 
 (decl pure vconst_all_ones_or_all_zeros () Constant)
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index 3f195baaa..bb65431a2 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -281,15 +281,12 @@ pub enum ArgsOrRets {
 /// appropriate addressing mode.
 #[derive(Clone, Copy, Debug)]
 pub enum StackAMode {
-    /// Offset into the current frame's argument area, possibly making use of a
-    /// specific type for a scaled indexing operation.
-    ArgOffset(i64, ir::Type),
-    /// Offset from the nominal stack pointer, possibly making use of a specific
-    /// type for a scaled indexing operation.
-    NominalSPOffset(i64, ir::Type),
-    /// Offset from the real stack pointer, possibly making use of a specific
-    /// type for a scaled indexing operation.
-    SPOffset(i64, ir::Type),
+    /// Offset into the current frame's argument area.
+    IncomingArg(i64),
+    /// Offset within the stack slots in the current frame.
+    Slot(i64),
+    /// Offset into the callee frame's argument area.
+    OutgoingArg(i64),
 }
 
 /// Trait implemented by machine-specific backend to represent ISA flags.
@@ -447,7 +444,7 @@ pub trait ABIMachineSpec {
 
     /// Generate an instruction to compute an address of a stack slot (FP- or
     /// SP-based offset).
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, ty: Type) -> Self::I;
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Self::I;
 
     /// Get a fixed register to use to compute a stack limit. This is needed for
     /// certain sequences generated after the register allocator has already
@@ -1475,7 +1472,7 @@ impl<M: ABIMachineSpec> Callee<M> {
                             ty
                         };
                     insts.push(M::gen_load_stack(
-                        StackAMode::ArgOffset(offset, ty),
+                        StackAMode::IncomingArg(offset),
                         *into_reg,
                         ty,
                     ));
@@ -1500,9 +1497,8 @@ impl<M: ABIMachineSpec> Callee<M> {
                 } else {
                     // Buffer address is implicitly defined by the ABI.
                     insts.push(M::gen_get_stack_addr(
-                        StackAMode::ArgOffset(offset, I8),
+                        StackAMode::IncomingArg(offset),
                         into_reg,
-                        I8,
                     ));
                 }
             }
@@ -1523,7 +1519,7 @@ impl<M: ABIMachineSpec> Callee<M> {
                         // This was allocated in the `init` routine.
                         let addr_reg = self.arg_temp_reg[idx].unwrap();
                         insts.push(M::gen_load_stack(
-                            StackAMode::ArgOffset(offset, ty),
+                            StackAMode::IncomingArg(offset),
                             addr_reg,
                             ty,
                         ));
@@ -1694,17 +1690,13 @@ impl<M: ABIMachineSpec> Callee<M> {
         // [MemArg::NominalSPOffset] for more details on nominal SP tracking).
         let stack_off = self.sized_stackslots[slot] as i64;
         let sp_off: i64 = stack_off + (offset as i64);
-        M::gen_get_stack_addr(StackAMode::NominalSPOffset(sp_off, I8), into_reg, I8)
+        M::gen_get_stack_addr(StackAMode::Slot(sp_off), into_reg)
     }
 
     /// Produce an instruction that computes a dynamic stackslot address.
     pub fn dynamic_stackslot_addr(&self, slot: DynamicStackSlot, into_reg: Writable<Reg>) -> M::I {
         let stack_off = self.dynamic_stackslots[slot] as i64;
-        M::gen_get_stack_addr(
-            StackAMode::NominalSPOffset(stack_off, I64X2XN),
-            into_reg,
-            I64X2XN,
-        )
+        M::gen_get_stack_addr(StackAMode::Slot(stack_off), into_reg)
     }
 
     /// Get an `args` pseudo-inst, if any, that should appear at the
@@ -1977,7 +1969,7 @@ impl<M: ABIMachineSpec> Callee<M> {
         let sp_off = self.get_spillslot_offset(to_slot);
         trace!("gen_spill: {from_reg:?} into slot {to_slot:?} at offset {sp_off}");
 
-        let from = StackAMode::NominalSPOffset(sp_off, ty);
+        let from = StackAMode::Slot(sp_off);
         <M>::gen_store_stack(from, Reg::from(from_reg), ty)
     }
 
@@ -1989,7 +1981,7 @@ impl<M: ABIMachineSpec> Callee<M> {
         let sp_off = self.get_spillslot_offset(from_slot);
         trace!("gen_reload: {to_reg:?} from slot {from_slot:?} at offset {sp_off}");
 
-        let from = StackAMode::NominalSPOffset(sp_off, ty);
+        let from = StackAMode::Slot(sp_off);
         <M>::gen_load_stack(from, to_reg.map(Reg::from), ty)
     }
 }
@@ -2242,9 +2234,8 @@ impl<M: ABIMachineSpec> CallSite<M> {
                 let src_ptr = from_regs.only_reg().unwrap();
                 let dst_ptr = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
                 ctx.emit(M::gen_get_stack_addr(
-                    StackAMode::SPOffset(offset, I8),
+                    StackAMode::OutgoingArg(offset),
                     dst_ptr,
-                    I8,
                 ));
                 // Emit a memcpy from `src_ptr` to `dst_ptr` of `size` bytes.
                 // N.B.: because we process StructArg params *first*, this is
@@ -2283,9 +2274,9 @@ impl<M: ABIMachineSpec> CallSite<M> {
                             "tail calls require frame pointers to be enabled"
                         );
 
-                        StackAMode::ArgOffset(offset, ty)
+                        StackAMode::IncomingArg(offset)
                     } else {
-                        StackAMode::SPOffset(offset, ty)
+                        StackAMode::OutgoingArg(offset)
                     };
                     ctx.emit(M::gen_store_stack(amode, vreg, ty))
                 }
@@ -2391,9 +2382,9 @@ impl<M: ABIMachineSpec> CallSite<M> {
             } => {
                 assert_eq!(from_regs.len(), 1);
                 let vreg = from_regs.regs()[0];
-                let amode = StackAMode::SPOffset(offset, ty);
+                let amode = StackAMode::OutgoingArg(offset);
                 let tmp = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
-                ctx.emit(M::gen_get_stack_addr(amode, tmp, ty));
+                ctx.emit(M::gen_get_stack_addr(amode, tmp));
                 let tmp = tmp.to_reg();
                 ctx.emit(M::gen_store_base_offset(tmp, 0, vreg, ty));
                 let loc = match pointer {
@@ -2504,7 +2495,7 @@ impl<M: ABIMachineSpec> CallSite<M> {
                                 sig_data.sized_stack_arg_space()
                             };
                             insts.push(M::gen_load_stack(
-                                StackAMode::SPOffset(offset + ret_area_base, ty),
+                                StackAMode::OutgoingArg(offset + ret_area_base),
                                 *into_reg,
                                 ty,
                             ));
@@ -2541,9 +2532,8 @@ impl<M: ABIMachineSpec> CallSite<M> {
             let rd = ctx.alloc_tmp(word_type).only_reg().unwrap();
             let ret_area_base = ctx.sigs()[self.sig].sized_stack_arg_space();
             ctx.emit(M::gen_get_stack_addr(
-                StackAMode::SPOffset(ret_area_base, I8),
+                StackAMode::OutgoingArg(ret_area_base),
                 rd,
-                I8,
             ));
             let moves = self.gen_arg(ctx, i.into(), ValueRegs::one(rd.to_reg()));
             self.emit_arg_moves(ctx, moves);
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index 8abe64a68..a18adea65 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -1675,7 +1675,7 @@ impl<I: VCodeInst> MachBuffer<I> {
                 (start_offset, end_offset)
             }
         };
-        trace!("Adding stack map for offsets {start:#x}..{end:#x}");
+        trace!("Adding stack map for offsets {start:#x}..{end:#x}: {stack_map:?}");
         self.stack_maps.push(MachStackMap {
             offset: start,
             offset_end: end,
diff --git a/cranelift/codegen/src/nan_canonicalization.rs b/cranelift/codegen/src/nan_canonicalization.rs
index c1bc16f77..5be41857a 100644
--- a/cranelift/codegen/src/nan_canonicalization.rs
+++ b/cranelift/codegen/src/nan_canonicalization.rs
@@ -5,7 +5,7 @@
 use crate::cursor::{Cursor, FuncCursor};
 use crate::ir::condcodes::FloatCC;
 use crate::ir::immediates::{Ieee32, Ieee64};
-use crate::ir::types;
+use crate::ir::types::{self};
 use crate::ir::{Function, Inst, InstBuilder, InstructionData, Opcode, Value};
 use crate::opts::MemFlags;
 use crate::timing;
@@ -15,13 +15,13 @@ static CANON_32BIT_NAN: u32 = 0b01111111110000000000000000000000;
 static CANON_64BIT_NAN: u64 = 0b0111111111111000000000000000000000000000000000000000000000000000;
 
 /// Perform the NaN canonicalization pass.
-pub fn do_nan_canonicalization(func: &mut Function) {
+pub fn do_nan_canonicalization(func: &mut Function, has_vector_support: bool) {
     let _tt = timing::canonicalize_nans();
     let mut pos = FuncCursor::new(func);
     while let Some(_block) = pos.next_block() {
         while let Some(inst) = pos.next_inst() {
             if is_fp_arith(&mut pos, inst) {
-                add_nan_canon_seq(&mut pos, inst);
+                add_nan_canon_seq(&mut pos, inst, has_vector_support);
             }
         }
     }
@@ -57,7 +57,7 @@ fn is_fp_arith(pos: &mut FuncCursor, inst: Inst) -> bool {
 }
 
 /// Append a sequence of canonicalizing instructions after the given instruction.
-fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) {
+fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst, has_vector_support: bool) {
     // Select the instruction result, result type. Replace the instruction
     // result and step forward before inserting the canonicalization sequence.
     let val = pos.func.dfg.first_result(inst);
@@ -65,16 +65,28 @@ fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) {
     let new_res = pos.func.dfg.replace_result(val, val_type);
     let _next_inst = pos.next_inst().expect("block missing terminator!");
 
-    // Insert a comparison instruction, to check if `inst_res` is NaN. Select
-    // the canonical NaN value if `val` is NaN, assign the result to `inst`.
-    let is_nan = pos.ins().fcmp(FloatCC::NotEqual, new_res, new_res);
+    // Insert a comparison instruction, to check if `inst_res` is NaN (comparing
+    // against NaN is always unordered). Select the canonical NaN value if `val`
+    // is NaN, assign the result to `inst`.
+    let comparison = FloatCC::Unordered;
 
+    let vectorized_scalar_select = |pos: &mut FuncCursor, canon_nan: Value, ty: types::Type| {
+        let canon_nan = pos.ins().scalar_to_vector(ty, canon_nan);
+        let new_res = pos.ins().scalar_to_vector(ty, new_res);
+        let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
+        let is_nan = pos.ins().bitcast(ty, MemFlags::new(), is_nan);
+        let simd_result = pos.ins().bitselect(is_nan, canon_nan, new_res);
+        pos.ins().with_result(val).extractlane(simd_result, 0);
+    };
     let scalar_select = |pos: &mut FuncCursor, canon_nan: Value| {
+        let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
         pos.ins()
             .with_result(val)
             .select(is_nan, canon_nan, new_res);
     };
+
     let vector_select = |pos: &mut FuncCursor, canon_nan: Value| {
+        let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
         let is_nan = pos.ins().bitcast(val_type, MemFlags::new(), is_nan);
         pos.ins()
             .with_result(val)
@@ -84,11 +96,19 @@ fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) {
     match val_type {
         types::F32 => {
             let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
-            scalar_select(pos, canon_nan);
+            if has_vector_support {
+                vectorized_scalar_select(pos, canon_nan, types::F32X4);
+            } else {
+                scalar_select(pos, canon_nan);
+            }
         }
         types::F64 => {
             let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN));
-            scalar_select(pos, canon_nan);
+            if has_vector_support {
+                vectorized_scalar_select(pos, canon_nan, types::F64X2);
+            } else {
+                scalar_select(pos, canon_nan);
+            }
         }
         types::F32X4 => {
             let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
diff --git a/crates/component-macro/src/bindgen.rs b/crates/component-macro/src/bindgen.rs
index 5bfd968b3..d0aee83a0 100644
--- a/crates/component-macro/src/bindgen.rs
+++ b/crates/component-macro/src/bindgen.rs
@@ -7,7 +7,7 @@ use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
 use syn::parse::{Error, Parse, ParseStream, Result};
 use syn::punctuated::Punctuated;
 use syn::{braced, token, Token};
-use wasmtime_wit_bindgen::{AsyncConfig, Opts, Ownership, TrappableError};
+use wasmtime_wit_bindgen::{AsyncConfig, Opts, Ownership, TrappableError, TrappableImports};
 use wit_parser::{PackageId, Resolve, UnresolvedPackage, WorldId};
 
 pub struct Config {
@@ -107,6 +107,7 @@ impl Parse for Config {
                         opts.async_ = val;
                     }
                     Opt::TrappableErrorType(val) => opts.trappable_error_type = val,
+                    Opt::TrappableImports(val) => opts.trappable_imports = val,
                     Opt::Ownership(val) => opts.ownership = val,
                     Opt::Interfaces(s) => {
                         if inline.is_some() {
@@ -201,6 +202,7 @@ mod kw {
     syn::custom_keyword!(with);
     syn::custom_keyword!(except_imports);
     syn::custom_keyword!(only_imports);
+    syn::custom_keyword!(trappable_imports);
 }
 
 enum Opt {
@@ -213,6 +215,7 @@ enum Opt {
     Ownership(Ownership),
     Interfaces(syn::LitStr),
     With(HashMap<String, String>),
+    TrappableImports(TrappableImports),
 }
 
 impl Parse for Opt {
@@ -330,6 +333,22 @@ impl Parse for Opt {
             let fields: Punctuated<(String, String), Token![,]> =
                 contents.parse_terminated(with_field_parse, Token![,])?;
             Ok(Opt::With(HashMap::from_iter(fields)))
+        } else if l.peek(kw::trappable_imports) {
+            input.parse::<kw::trappable_imports>()?;
+            input.parse::<Token![:]>()?;
+            let config = if input.peek(syn::LitBool) {
+                match input.parse::<syn::LitBool>()?.value {
+                    true => TrappableImports::All,
+                    false => TrappableImports::None,
+                }
+            } else {
+                let contents;
+                syn::bracketed!(contents in input);
+                let fields: Punctuated<syn::LitStr, Token![,]> =
+                    contents.parse_terminated(Parse::parse, Token![,])?;
+                TrappableImports::Only(fields.iter().map(|s| s.value()).collect())
+            };
+            Ok(Opt::TrappableImports(config))
         } else {
             Err(l.error())
         }
diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs
index 3334b1736..873105b34 100644
--- a/crates/environ/src/component/translate.rs
+++ b/crates/environ/src/component/translate.rs
@@ -4,6 +4,7 @@ use crate::{
     EntityIndex, ModuleEnvironment, ModuleTranslation, ModuleTypesBuilder, PrimaryMap, Tunables,
     TypeConvert, WasmHeapType, WasmValType,
 };
+use anyhow::anyhow;
 use anyhow::{bail, Result};
 use indexmap::IndexMap;
 use std::collections::HashMap;
@@ -534,7 +535,18 @@ impl<'a, 'data> Translator<'a, 'data> {
                     self.validator,
                     self.types.module_types_builder(),
                 )
-                .translate(parser, &component[range.start..range.end])?;
+                .translate(
+                    parser,
+                    component.get(range.start..range.end).ok_or_else(|| {
+                        anyhow!(
+                            "section range {}..{} is out of bounds (bound = {})",
+                            range.start,
+                            range.end,
+                            component.len()
+                        )
+                        .context("wasm component contains an invalid module section")
+                    })?,
+                )?;
                 let static_idx = self.static_modules.push(translation);
                 self.result
                     .initializers
diff --git a/crates/runtime/src/gc/enabled/drc.rs b/crates/runtime/src/gc/enabled/drc.rs
index 6987984da..56bba9838 100644
--- a/crates/runtime/src/gc/enabled/drc.rs
+++ b/crates/runtime/src/gc/enabled/drc.rs
@@ -160,6 +160,12 @@ impl DrcHeap {
 
         let drc_ref = drc_ref(gc_ref);
         let header = self.index_mut(&drc_ref);
+        debug_assert_ne!(
+            *header.ref_count.get_mut(),
+            0,
+            "{:#p} is supposedly live; should have nonzero ref count",
+            *gc_ref
+        );
         *header.ref_count.get_mut() += 1;
         log::trace!(
             "increment {:#p} ref count -> {}",
@@ -179,6 +185,12 @@ impl DrcHeap {
 
         let drc_ref = drc_ref(gc_ref);
         let header = self.index_mut(drc_ref);
+        debug_assert_ne!(
+            *header.ref_count.get_mut(),
+            0,
+            "{:#p} is supposedly live; should have nonzero ref count",
+            *gc_ref
+        );
         *header.ref_count.get_mut() -= 1;
         log::trace!(
             "decrement {:#p} ref count -> {}",
@@ -261,6 +273,13 @@ impl DrcHeap {
                 continue;
             }
 
+            debug_assert_ne!(
+                *self.index_mut(drc_ref(&gc_ref)).ref_count.get_mut(),
+                0,
+                "{gc_ref:#p} is on the Wasm stack and therefore should be held \
+                 by the activations table; should have nonzero ref count",
+            );
+
             log::trace!("Found GC reference on the stack: {:#p}", gc_ref);
             let is_new = self
                 .activations_table
@@ -272,26 +291,76 @@ impl DrcHeap {
         }
     }
 
-    /// Sweep the bump allocation table after we've discovered our precise stack
-    /// roots.
-    fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) {
-        // Sweep our bump chunk.
+    fn iter_bump_chunk(&mut self) -> impl Iterator<Item = VMGcRef> + '_ {
         let num_filled = self.activations_table.num_filled_in_bump_chunk();
+        self.activations_table
+            .alloc
+            .chunk
+            .iter_mut()
+            .take(num_filled)
+            .map(|slot| {
+                let r64 = *slot.get_mut();
+                VMGcRef::from_r64(r64)
+                    .expect("valid r64")
+                    .expect("non-null")
+            })
+    }
+
+    #[inline(never)]
+    #[cold]
+    fn log_gc_ref_set(prefix: &str, items: impl Iterator<Item = VMGcRef>) {
+        assert!(log::log_enabled!(log::Level::Trace));
+        let mut set = "{".to_string();
+        let mut any = false;
+        for gc_ref in items {
+            any = true;
+            set += &format!("\n  {gc_ref:#p},");
+        }
+        if any {
+            set.push('\n');
+        }
+        set.push('}');
+        log::trace!("{prefix}: {set}");
+    }
+
+    fn drain_bump_chunk(&mut self, mut f: impl FnMut(&mut Self, VMGcRef)) {
+        let num_filled = self.activations_table.num_filled_in_bump_chunk();
+
+        // Temporarily take the allocation out of `self` to avoid conflicting
+        // borrows.
         let mut alloc = mem::take(&mut self.activations_table.alloc);
         for slot in alloc.chunk.iter_mut().take(num_filled) {
             let r64 = mem::take(slot.get_mut());
             let gc_ref = VMGcRef::from_r64(r64)
                 .expect("valid r64")
                 .expect("non-null");
-            self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref);
-
+            f(self, gc_ref);
             *slot.get_mut() = 0;
         }
+
         debug_assert!(
-            alloc.chunk.iter().all(|slot| unsafe { *slot.get() == 0 }),
-            "after sweeping the bump chunk, all slots should be empty"
+            alloc.chunk.iter_mut().all(|slot| *slot.get_mut() == 0),
+            "after sweeping the bump chunk, all slots should be empty",
         );
 
+        debug_assert!(self.activations_table.alloc.chunk.is_empty());
+        self.activations_table.alloc = alloc;
+    }
+
+    /// Sweep the bump allocation table after we've discovered our precise stack
+    /// roots.
+    fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) {
+        if log::log_enabled!(log::Level::Trace) {
+            Self::log_gc_ref_set("bump chunk before sweeping", self.iter_bump_chunk());
+        }
+
+        // Sweep our bump chunk.
+        log::trace!("Begin sweeping bump chunk");
+        self.drain_bump_chunk(|heap, gc_ref| {
+            heap.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref);
+        });
+        log::trace!("Done sweeping bump chunk");
+
         if self.activations_table.alloc.chunk.is_empty() {
             // If this is the first collection, then the bump chunk is empty
             // since we lazily allocate it. Force that lazy allocation now so we
@@ -302,6 +371,16 @@ impl DrcHeap {
             self.activations_table.alloc.reset();
         }
 
+        if log::log_enabled!(log::Level::Trace) {
+            Self::log_gc_ref_set(
+                "hash set before sweeping",
+                self.activations_table
+                    .over_approximated_stack_roots
+                    .iter()
+                    .map(|r| r.unchecked_copy()),
+            );
+        }
+
         // The current `precise_stack_roots` becomes our new over-appoximated
         // set for the next GC cycle.
         mem::swap(
@@ -315,14 +394,26 @@ impl DrcHeap {
         // Note that this may run arbitrary code as we run gc_ref
         // destructors. Because of our `&mut` borrow above on this table,
         // though, we're guaranteed that nothing will touch this table.
+        log::trace!("Begin sweeping hash set");
         let mut precise_stack_roots = mem::take(&mut self.activations_table.precise_stack_roots);
         for gc_ref in precise_stack_roots.drain() {
             self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref);
         }
+        log::trace!("Done sweeping hash set");
 
         // Make sure to replace the `precise_stack_roots` so that we reuse any
         // allocated capacity.
         self.activations_table.precise_stack_roots = precise_stack_roots;
+
+        if log::log_enabled!(log::Level::Trace) {
+            Self::log_gc_ref_set(
+                "hash set after sweeping",
+                self.activations_table
+                    .over_approximated_stack_roots
+                    .iter()
+                    .map(|r| r.unchecked_copy()),
+            );
+        }
     }
 }
 
diff --git a/crates/runtime/src/gc/gc_runtime.rs b/crates/runtime/src/gc/gc_runtime.rs
index e3cbff94f..1d2741dc5 100644
--- a/crates/runtime/src/gc/gc_runtime.rs
+++ b/crates/runtime/src/gc/gc_runtime.rs
@@ -311,19 +311,22 @@ enum RawGcRoot {
 
 impl GcRootsList {
     /// Add a GC root that is inside a Wasm stack frame to this list.
-    //
-    // TODO: we shouldn't need to differentiate between on-stack GC roots and
-    // others, and both should just be `*mut VMGcRef`, but because Cranelift
-    // doesn't support `r32` on 64-bit platforms yet, we have to use `r64` for
-    // roots on the Wasm stack instead.
     #[inline]
-    pub fn add_wasm_stack_root(&mut self, ptr_to_root: SendSyncPtr<u64>) {
+    pub unsafe fn add_wasm_stack_root(&mut self, ptr_to_root: SendSyncPtr<u64>) {
+        log::trace!(
+            "Adding Wasm stack root: {:#p}",
+            VMGcRef::from_r64(*ptr_to_root.as_ref()).unwrap().unwrap()
+        );
         self.0.push(RawGcRoot::Stack(ptr_to_root));
     }
 
     /// Add a GC root to this list.
     #[inline]
-    pub fn add_root(&mut self, ptr_to_root: SendSyncPtr<VMGcRef>) {
+    pub unsafe fn add_root(&mut self, ptr_to_root: SendSyncPtr<VMGcRef>) {
+        log::trace!(
+            "Adding non-stack root: {:#p}",
+            ptr_to_root.as_ref().unchecked_copy()
+        );
         self.0.push(RawGcRoot::NonStack(ptr_to_root))
     }
 
diff --git a/crates/runtime/src/gc/host_data.rs b/crates/runtime/src/gc/host_data.rs
index 933935bed..13161b508 100644
--- a/crates/runtime/src/gc/host_data.rs
+++ b/crates/runtime/src/gc/host_data.rs
@@ -39,11 +39,14 @@ impl ExternRefHostDataTable {
     /// Allocate a new `externref` host data value.
     pub fn alloc(&mut self, value: Box<dyn Any + Send + Sync>) -> ExternRefHostDataId {
         let id = self.slab.alloc(value);
-        ExternRefHostDataId(id)
+        let id = ExternRefHostDataId(id);
+        log::trace!("allocated new externref host data: {id:?}");
+        id
     }
 
     /// Deallocate an `externref` host data value.
     pub fn dealloc(&mut self, id: ExternRefHostDataId) -> Box<dyn Any + Send + Sync> {
+        log::trace!("deallocated externref host data: {id:?}");
         self.slab.dealloc(id.0)
     }
 
diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs
index 1095948ba..c2074eeee 100644
--- a/crates/runtime/src/libcalls.rs
+++ b/crates/runtime/src/libcalls.rs
@@ -380,6 +380,7 @@ unsafe fn drop_gc_ref(instance: &mut Instance, gc_ref: *mut u8) {
     let gc_ref = VMGcRef::from_r64(u64::try_from(gc_ref as usize).unwrap())
         .expect("valid r64")
         .expect("non-null VMGcRef");
+    log::trace!("libcalls::drop_gc_ref({gc_ref:?})");
     (*instance.store()).gc_store().drop_gc_ref(gc_ref);
 }
 
diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs
index 4a0ea6019..f138d1a37 100644
--- a/crates/runtime/src/table.rs
+++ b/crates/runtime/src/table.rs
@@ -467,16 +467,16 @@ impl Table {
             TableElement::FuncRef(f) => {
                 self.funcrefs_mut()[start..end].fill(TaggedFuncRef::from(f));
             }
-            TableElement::GcRef(e) => {
+            TableElement::GcRef(r) => {
                 // Clone the init GC reference into each table slot.
                 for slot in &mut self.gc_refs_mut()[start..end] {
-                    gc_store.write_gc_ref(slot, e.as_ref());
+                    gc_store.write_gc_ref(slot, r.as_ref());
                 }
 
                 // Drop the init GC reference, since we aren't holding onto this
                 // reference anymore, only the clones in the table.
-                if let Some(e) = e {
-                    gc_store.drop_gc_ref(e);
+                if let Some(r) = r {
+                    gc_store.drop_gc_ref(r);
                 }
             }
             TableElement::UninitFunc => {
diff --git a/crates/wasi-http/src/lib.rs b/crates/wasi-http/src/lib.rs
index 29419d745..da72cecfa 100644
--- a/crates/wasi-http/src/lib.rs
+++ b/crates/wasi-http/src/lib.rs
@@ -20,6 +20,7 @@ pub mod bindings {
         ",
         tracing: true,
         async: false,
+        trappable_imports: true,
         with: {
             // Upstream package dependencies
             "wasi:io": wasmtime_wasi::bindings::io,
diff --git a/crates/wasi-nn/src/wit.rs b/crates/wasi-nn/src/wit.rs
index 95bfd1dac..dd9c9cc08 100644
--- a/crates/wasi-nn/src/wit.rs
+++ b/crates/wasi-nn/src/wit.rs
@@ -20,7 +20,11 @@ use std::{error::Error, fmt, hash::Hash, str::FromStr};
 
 /// Generate the traits and types from the `wasi-nn` WIT specification.
 mod gen_ {
-    wasmtime::component::bindgen!("ml" in "spec/wit/wasi-nn.wit");
+    wasmtime::component::bindgen!({
+        world: "ml",
+        path: "spec/wit/wasi-nn.wit",
+        trappable_imports: true,
+    });
 }
 use gen_::wasi::nn as gen; // Shortcut to the module containing the types we need.
 
diff --git a/crates/wasi/src/bindings.rs b/crates/wasi/src/bindings.rs
index 57c08329a..737301403 100644
--- a/crates/wasi/src/bindings.rs
+++ b/crates/wasi/src/bindings.rs
@@ -14,6 +14,7 @@ pub mod sync {
                 "wasi:io/streams/stream-error" => StreamError,
                 "wasi:filesystem/types/error-code" => FsError,
             },
+            trappable_imports: true,
             with: {
                 // These interfaces come from the outer module, as it's
                 // sync/async agnostic.
@@ -118,6 +119,7 @@ mod async_io {
         path: "wit",
         world: "wasi:cli/command",
         tracing: true,
+        trappable_imports: true,
         async: {
             // Only these functions are `async` and everything else is sync
             // meaning that it basically doesn't need to block. These functions
diff --git a/crates/wasmtime/src/runtime/component/mod.rs b/crates/wasmtime/src/runtime/component/mod.rs
index 9418ede3b..43ac2f710 100644
--- a/crates/wasmtime/src/runtime/component/mod.rs
+++ b/crates/wasmtime/src/runtime/component/mod.rs
@@ -209,8 +209,8 @@ pub(crate) use self::store::ComponentStoreData;
 /// impl HelloWorldImports for MyState {
 ///     // Note the `Result` return value here where `Ok` is returned back to
 ///     // the component and `Err` will raise a trap.
-///     fn name(&mut self) -> wasmtime::Result<String> {
-///         Ok(self.name.clone())
+///     fn name(&mut self) -> String {
+///         self.name.clone()
 ///     }
 /// }
 ///
@@ -315,13 +315,13 @@ pub(crate) use self::store::ComponentStoreData;
 ///
 /// // Note that the trait here is per-interface and within a submodule now.
 /// impl Host for MyState {
-///     fn gen_random_integer(&mut self) -> wasmtime::Result<u32> {
+///     fn gen_random_integer(&mut self) -> u32 {
 /// #       panic!();
 /// #       #[cfg(FALSE)]
-///         Ok(rand::thread_rng().gen())
+///         rand::thread_rng().gen()
 ///     }
 ///
-///     fn sha256(&mut self, bytes: Vec<u8>) -> wasmtime::Result<String> {
+///     fn sha256(&mut self, bytes: Vec<u8>) -> String {
 ///         // ...
 /// #       panic!()
 ///     }
@@ -432,6 +432,21 @@ pub(crate) use self::store::ComponentStoreData;
 ///         only_imports: ["foo", "bar"],
 ///     },
 ///
+///     // This option is used to indicate whether imports can trap.
+///     //
+///     // Imports that may trap have their return types wrapped in
+///     // `wasmtime::Result<T>` where the `Err` variant indicates that a
+///     // trap will be raised in the guest.
+///     //
+///     // By default imports cannot trap and the return value is the return
+///     // value from the WIT bindings itself. This value can be set to `true`
+///     // to indicate that any import can trap. This value can also be set to
+///     // an array-of-strings to indicate that only a set list of imports
+///     // can trap.
+///     trappable_imports: false,             // no imports can trap (default)
+///     // trappable_imports: true,           // all imports can trap
+///     // trappable_imports: ["foo", "bar"], // only these can trap
+///
 ///     // This can be used to translate WIT return values of the form
 ///     // `result<T, error-type>` into `Result<T, RustErrorType>` in Rust.
 ///     // Users must define `RustErrorType` and the `Host` trait for the
@@ -440,7 +455,8 @@ pub(crate) use self::store::ComponentStoreData;
 ///     // into `wasmtime::Result<ErrorType>`. This conversion can either
 ///     // return the raw WIT error (`ErrorType` here) or a trap.
 ///     //
-///     // By default this option is not specified.
+///     // By default this option is not specified. This option only takes
+///     // effect when `trappable_imports` is set for some imports.
 ///     trappable_error_type: {
 ///         "wasi:io/streams/stream-error" => RustErrorType,
 ///     },
diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs
index 8276e6b98..dc58268d2 100644
--- a/crates/wasmtime/src/runtime/externals/global.rs
+++ b/crates/wasmtime/src/runtime/externals/global.rs
@@ -213,7 +213,9 @@ impl Global {
             if let Some(gc_ref) = unsafe { (*store[self.0].definition).as_gc_ref() } {
                 let gc_ref = NonNull::from(gc_ref);
                 let gc_ref = SendSyncPtr::new(gc_ref);
-                gc_roots_list.add_root(gc_ref);
+                unsafe {
+                    gc_roots_list.add_root(gc_ref);
+                }
             }
         }
     }
diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs
index 145723a36..0adc6ad94 100644
--- a/crates/wasmtime/src/runtime/externals/table.rs
+++ b/crates/wasmtime/src/runtime/externals/table.rs
@@ -387,7 +387,9 @@ impl Table {
             if let Some(gc_ref) = gc_ref {
                 let gc_ref = NonNull::from(gc_ref);
                 let gc_ref = SendSyncPtr::new(gc_ref);
-                gc_roots_list.add_root(gc_ref);
+                unsafe {
+                    gc_roots_list.add_root(gc_ref);
+                }
             }
         }
     }
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index 06193a529..293765e0a 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -370,12 +370,21 @@ struct LifoRoot {
 
 impl RootSet {
     pub(crate) fn trace_roots(&mut self, gc_roots_list: &mut GcRootsList) {
+        log::trace!("Begin trace user LIFO roots");
         for root in &mut self.lifo_roots {
-            gc_roots_list.add_root((&mut root.gc_ref).into());
+            unsafe {
+                gc_roots_list.add_root((&mut root.gc_ref).into());
+            }
         }
+        log::trace!("End trace user LIFO roots");
+
+        log::trace!("Begin trace user manual roots");
         for (_id, root) in self.manually_rooted.iter_mut() {
-            gc_roots_list.add_root(root.into());
+            unsafe {
+                gc_roots_list.add_root(root.into());
+            }
         }
+        log::trace!("End trace user manual roots");
     }
 
     /// Enter a LIFO rooting scope.
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index f54c5846f..c21273fab 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1691,8 +1691,11 @@ impl StoreOpaque {
                     .expect("we should never use the high 32 bits of an r64");
 
                 if gc_ref.is_some() {
-                    gc_roots_list
-                        .add_wasm_stack_root(SendSyncPtr::new(NonNull::new(stack_slot).unwrap()));
+                    unsafe {
+                        gc_roots_list.add_wasm_stack_root(SendSyncPtr::new(
+                            NonNull::new(stack_slot).unwrap(),
+                        ));
+                    }
                 }
             }
 
diff --git a/crates/wit-bindgen/src/lib.rs b/crates/wit-bindgen/src/lib.rs
index b07ac4c4d..5d51c583b 100644
--- a/crates/wit-bindgen/src/lib.rs
+++ b/crates/wit-bindgen/src/lib.rs
@@ -104,6 +104,9 @@ pub struct Opts {
     /// Whether or not to generate code for only the interfaces of this wit file or not.
     pub only_interfaces: bool,
 
+    /// Configuration of which imports are allowed to generate a trap.
+    pub trappable_imports: TrappableImports,
+
     /// Remapping of interface names to rust module names.
     /// TODO: is there a better type to use for the value of this map?
     pub with: HashMap<String, String>,
@@ -154,6 +157,27 @@ impl AsyncConfig {
     }
 }
 
+#[derive(Default, Debug, Clone)]
+pub enum TrappableImports {
+    /// No imports are allowed to trap.
+    #[default]
+    None,
+    /// All imports may trap.
+    All,
+    /// Only the specified set of functions may trap.
+    Only(HashSet<String>),
+}
+
+impl TrappableImports {
+    fn can_trap(&self, f: &Function) -> bool {
+        match self {
+            TrappableImports::None => false,
+            TrappableImports::All => true,
+            TrappableImports::Only(set) => set.contains(&f.name),
+        }
+    }
+}
+
 impl Opts {
     pub fn generate(&self, resolve: &Resolve, world: WorldId) -> String {
         let mut r = Wasmtime::default();
@@ -1853,7 +1877,13 @@ impl<'a> InterfaceGenerator<'a> {
             );
         }
 
-        if let Some((_, err, _)) = self.special_case_trappable_error(&func.results) {
+        if !self.gen.opts.trappable_imports.can_trap(&func) {
+            if func.results.iter_types().len() == 1 {
+                uwrite!(self.src, "Ok((r,))\n");
+            } else {
+                uwrite!(self.src, "Ok(r)\n");
+            }
+        } else if let Some((_, err, _)) = self.special_case_trappable_error(&func.results) {
             let err = &self.resolve.types[resolve_type_definition_id(self.resolve, err)];
             let err_name = err.name.as_ref().unwrap();
             let owner = match err.owner {
@@ -1905,7 +1935,11 @@ impl<'a> InterfaceGenerator<'a> {
         self.push_str(")");
         self.push_str(" -> ");
 
-        if let Some((r, _id, error_typename)) = self.special_case_trappable_error(&func.results) {
+        if !self.gen.opts.trappable_imports.can_trap(func) {
+            self.print_result_ty(&func.results, TypeMode::Owned);
+        } else if let Some((r, _id, error_typename)) =
+            self.special_case_trappable_error(&func.results)
+        {
             // Functions which have a single result `result<ok,err>` get special
             // cased to use the host_wasmtime_rust::Error<err>, making it possible
             // for them to trap or use `?` to propogate their errors
diff --git a/examples/component/main.rs b/examples/component/main.rs
index 459764664..f70fc3819 100644
--- a/examples/component/main.rs
+++ b/examples/component/main.rs
@@ -13,8 +13,8 @@ struct HostComponent;
 
 // Implmentation of the host interface defined in the wit file.
 impl host::Host for HostComponent {
-    fn multiply(&mut self, a: f32, b: f32) -> wasmtime::Result<f32> {
-        Ok(a * b)
+    fn multiply(&mut self, a: f32, b: f32) -> f32 {
+        a * b
     }
 }
 
diff --git a/src/commands/explore.rs b/src/commands/explore.rs
index 465a1d342..6aaa004e3 100644
--- a/src/commands/explore.rs
+++ b/src/commands/explore.rs
@@ -2,7 +2,7 @@
 
 use anyhow::{Context, Result};
 use clap::Parser;
-use std::path::PathBuf;
+use std::{borrow::Cow, path::PathBuf};
 use wasmtime_cli_flags::CommonOptions;
 
 /// Explore the compilation of a WebAssembly module to native code.
@@ -32,8 +32,15 @@ impl ExploreCommand {
 
         let config = self.common.config(self.target.as_deref())?;
 
-        let wasm = std::fs::read(&self.module)
-            .with_context(|| format!("failed to read Wasm module: {}", self.module.display()))?;
+        let bytes =
+            Cow::Owned(std::fs::read(&self.module).with_context(|| {
+                format!("failed to read Wasm module: {}", self.module.display())
+            })?);
+        #[cfg(feature = "wat")]
+        let bytes = wat::parse_bytes(&bytes).map_err(|mut e| {
+            e.set_path(&self.module);
+            e
+        })?;
 
         let output = self
             .output
@@ -43,7 +50,7 @@ impl ExploreCommand {
             .with_context(|| format!("failed to create file: {}", output.display()))?;
         let mut output_file = std::io::BufWriter::new(output_file);
 
-        wasmtime_explorer::generate(&config, self.target.as_deref(), &wasm, &mut output_file)?;
+        wasmtime_explorer::generate(&config, self.target.as_deref(), &bytes, &mut output_file)?;
         println!("Exploration written to {}", output.display());
         Ok(())
     }
diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rs
index 63f1d31a2..377bbcb72 100644
--- a/winch/codegen/src/codegen/env.rs
+++ b/winch/codegen/src/codegen/env.rs
@@ -127,6 +127,8 @@ pub struct FuncEnv<'a, 'translation: 'a, 'data: 'translation, P: PtrSize> {
     ptr_type: WasmValType,
     /// Whether or not to enable Spectre mitigation on heap bounds checks.
     heap_access_spectre_mitigation: bool,
+    /// Whether or not to enable Spectre mitigation on table element accesses.
+    table_access_spectre_mitigation: bool,
     name_map: PrimaryMap<UserExternalNameRef, UserExternalName>,
     name_intern: HashMap<UserExternalName, UserExternalNameRef>,
 }
@@ -158,6 +160,7 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
             resolved_globals: HashMap::new(),
             ptr_type,
             heap_access_spectre_mitigation: isa.flags().enable_heap_access_spectre_mitigation(),
+            table_access_spectre_mitigation: isa.flags().enable_table_access_spectre_mitigation(),
             builtins,
             name_map: Default::default(),
             name_intern: Default::default(),
@@ -314,6 +317,12 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
         self.heap_access_spectre_mitigation
     }
 
+    /// Returns true if Spectre mitigations are enabled for table element
+    /// accesses.
+    pub fn table_access_spectre_mitigation(&self) -> bool {
+        self.table_access_spectre_mitigation
+    }
+
     pub(crate) fn callee_sig<'b, A>(&'b mut self, callee: &'b Callee) -> &'b ABISig
     where
         A: ABI,
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 1ba4e9867..3017dc812 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -2,8 +2,10 @@ use crate::{
     abi::{vmctx, ABIOperand, ABISig, RetArea, ABI},
     codegen::BlockSig,
     isa::reg::Reg,
-    masm::{ExtendKind, IntCmpKind, MacroAssembler, OperandSize, RegImm, SPOffset, TrapCode},
-    stack::TypedReg,
+    masm::{
+        ExtendKind, IntCmpKind, MacroAssembler, OperandSize, RegImm, SPOffset, ShiftKind, TrapCode,
+    },
+    stack::{TypedReg, Val},
 };
 use anyhow::Result;
 use smallvec::SmallVec;
@@ -12,7 +14,7 @@ use wasmparser::{
 };
 use wasmtime_environ::{
     GlobalIndex, MemoryIndex, PtrSize, TableIndex, TypeIndex, WasmHeapType, WasmValType,
-    FUNCREF_MASK,
+    FUNCREF_MASK, WASM_PAGE_SIZE,
 };
 
 use cranelift_codegen::{
@@ -475,9 +477,7 @@ where
         let index = self.context.pop_to_reg(self.masm, None);
         let base = self.context.any_gpr(self.masm);
 
-        let elem_addr =
-            self.masm
-                .table_elem_address(index.into(), base, &table_data, &mut self.context);
+        let elem_addr = self.emit_compute_table_elem_addr(index.into(), base, &table_data);
         self.masm.load_ptr(elem_addr, elem_value);
         // Free the register used as base, once we have loaded the element
         // address into the element value register.
@@ -745,6 +745,132 @@ where
         }
         self.context.free_reg(src);
     }
+
+    /// Loads the address of the table element at a given index. Returns the
+    /// address of the table element using the provided register as base.
+    pub fn emit_compute_table_elem_addr(
+        &mut self,
+        index: Reg,
+        base: Reg,
+        table_data: &TableData,
+    ) -> M::Address {
+        let scratch = <M::ABI as ABI>::scratch_reg();
+        let bound = self.context.any_gpr(self.masm);
+        let tmp = self.context.any_gpr(self.masm);
+        let ptr_size: OperandSize = self.env.ptr_type().into();
+
+        if let Some(offset) = table_data.import_from {
+            // If the table data declares a particular offset base,
+            // load the address into a register to further use it as
+            // the table address.
+            self.masm.load_ptr(self.masm.address_at_vmctx(offset), base);
+        } else {
+            // Else, simply move the vmctx register into the addr register as
+            // the base to calculate the table address.
+            self.masm.mov(vmctx!(M).into(), base, ptr_size);
+        };
+
+        // OOB check.
+        let bound_addr = self
+            .masm
+            .address_at_reg(base, table_data.current_elems_offset);
+        let bound_size = table_data.current_elements_size;
+        self.masm.load(bound_addr, bound, bound_size.into());
+        self.masm.cmp(bound.into(), index, bound_size);
+        self.masm
+            .trapif(IntCmpKind::GeU, TrapCode::TableOutOfBounds);
+
+        // Move the index into the scratch register to calcualte the table
+        // element address.
+        // Moving the value of the index register to the scratch register
+        // also avoids overwriting the context of the index register.
+        self.masm.mov(index.into(), scratch, bound_size);
+        self.masm.mul(
+            scratch,
+            scratch,
+            RegImm::i32(table_data.element_size.bytes() as i32),
+            table_data.element_size,
+        );
+        self.masm
+            .load_ptr(self.masm.address_at_reg(base, table_data.offset), base);
+        // Copy the value of the table base into a temporary register
+        // so that we can use it later in case of a misspeculation.
+        self.masm.mov(base.into(), tmp, ptr_size);
+        // Calculate the address of the table element.
+        self.masm.add(base, base, scratch.into(), ptr_size);
+        if self.env.table_access_spectre_mitigation() {
+            // Perform a bounds check and override the value of the
+            // table element address in case the index is out of bounds.
+            self.masm.cmp(bound.into(), index, OperandSize::S32);
+            self.masm.cmov(tmp, base, IntCmpKind::GeU, ptr_size);
+        }
+        self.context.free_reg(bound);
+        self.context.free_reg(tmp);
+        self.masm.address_at_reg(base, 0)
+    }
+
+    /// Retrieves the size of the table, pushing the result to the value stack.
+    pub fn emit_compute_table_size(&mut self, table_data: &TableData) {
+        let scratch = <M::ABI as ABI>::scratch_reg();
+        let size = self.context.any_gpr(self.masm);
+        let ptr_size: OperandSize = self.env.ptr_type().into();
+
+        if let Some(offset) = table_data.import_from {
+            self.masm
+                .load_ptr(self.masm.address_at_vmctx(offset), scratch);
+        } else {
+            self.masm.mov(vmctx!(M).into(), scratch, ptr_size);
+        };
+
+        let size_addr = self
+            .masm
+            .address_at_reg(scratch, table_data.current_elems_offset);
+        self.masm
+            .load(size_addr, size, table_data.current_elements_size.into());
+
+        self.context.stack.push(TypedReg::i32(size).into());
+    }
+
+    /// Retrieves the size of the memory, pushing the result to the value stack.
+    pub fn emit_compute_memory_size(&mut self, heap_data: &HeapData) {
+        let size_reg = self.context.any_gpr(self.masm);
+        let scratch = <M::ABI as ABI>::scratch_reg();
+
+        let base = if let Some(offset) = heap_data.import_from {
+            self.masm
+                .load_ptr(self.masm.address_at_vmctx(offset), scratch);
+            scratch
+        } else {
+            vmctx!(M)
+        };
+
+        let size_addr = self
+            .masm
+            .address_at_reg(base, heap_data.current_length_offset);
+        self.masm.load_ptr(size_addr, size_reg);
+        // Prepare the stack to emit a shift to get the size in pages rather
+        // than in bytes.
+        self.context
+            .stack
+            .push(TypedReg::new(heap_data.ty, size_reg).into());
+
+        // Since the page size is a power-of-two, verify that 2^16, equals the
+        // defined constant. This is mostly a safeguard in case the constant
+        // value ever changes.
+        let pow = 16;
+        debug_assert_eq!(2u32.pow(pow), WASM_PAGE_SIZE);
+
+        // Ensure that the constant is correctly typed according to the heap
+        // type to reduce register pressure when emitting the shift operation.
+        match heap_data.ty {
+            WasmValType::I32 => self.context.stack.push(Val::i32(pow as i32)),
+            WasmValType::I64 => self.context.stack.push(Val::i64(pow as i64)),
+            _ => unreachable!(),
+        }
+
+        self.masm
+            .shift(&mut self.context, ShiftKind::ShrU, heap_data.ty.into());
+    }
 }
 
 /// Returns the index of the [`ControlStackFrame`] for the given
diff --git a/winch/codegen/src/isa/aarch64/address.rs b/winch/codegen/src/isa/aarch64/address.rs
index 11521e5ce..02b1680d6 100644
--- a/winch/codegen/src/isa/aarch64/address.rs
+++ b/winch/codegen/src/isa/aarch64/address.rs
@@ -137,7 +137,6 @@ impl TryFrom<Address> for AMode {
             Offset { base, offset } => Ok(AMode::RegOffset {
                 rn: base.into(),
                 off: offset,
-                ty: types::I64,
             }),
         }
     }
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 8f2d20513..0a106d582 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -1,7 +1,7 @@
 use super::{abi::Aarch64ABI, address::Address, asm::Assembler, regs};
 use crate::{
     abi::{self, local::LocalSlot},
-    codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv, HeapData, TableData},
+    codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv},
     isa::reg::Reg,
     masm::{
         CalleeKind, DivKind, ExtendKind, FloatCmpKind, Imm as I, IntCmpKind,
@@ -111,24 +111,6 @@ impl Masm for MacroAssembler {
         Address::offset(reg, offset as i64)
     }
 
-    fn table_elem_address(
-        &mut self,
-        _index: Reg,
-        _base: Reg,
-        _table_data: &TableData,
-        _context: &mut CodeGenContext,
-    ) -> Self::Address {
-        todo!()
-    }
-
-    fn table_size(&mut self, _table_data: &TableData, _context: &mut CodeGenContext) {
-        todo!()
-    }
-
-    fn memory_size(&mut self, _heap_data: &HeapData, _context: &mut CodeGenContext) {
-        todo!()
-    }
-
     fn address_from_sp(&self, _offset: SPOffset) -> Self::Address {
         todo!()
     }
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index 4b68dde4a..2300253b7 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -11,13 +11,12 @@ use crate::masm::{
 };
 use crate::{
     abi::{self, align_to, calculate_frame_adjustment, LocalSlot},
-    codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv, HeapData, TableData},
+    codegen::{ptr_type_from_ptr_size, CodeGenContext, FuncEnv},
     stack::Val,
 };
 use crate::{
     abi::{vmctx, ABI},
     masm::{SPOffset, StackSlot},
-    stack::TypedReg,
 };
 use crate::{
     isa::reg::{Reg, RegClass},
@@ -34,7 +33,7 @@ use cranelift_codegen::{
     settings, Final, MachBufferFinalized, MachLabel,
 };
 
-use wasmtime_environ::{PtrSize, WasmValType, WASM_PAGE_SIZE};
+use wasmtime_environ::{PtrSize, WasmValType};
 
 /// x64 MacroAssembler.
 pub(crate) struct MacroAssembler {
@@ -180,141 +179,6 @@ impl Masm for MacroAssembler {
         Address::offset(reg, offset)
     }
 
-    fn table_elem_address(
-        &mut self,
-        index: Reg,
-        ptr_base: Reg,
-        table_data: &TableData,
-        context: &mut CodeGenContext,
-    ) -> Self::Address {
-        let scratch = regs::scratch();
-        let bound = context.any_gpr(self);
-        let tmp = context.any_gpr(self);
-
-        if let Some(offset) = table_data.import_from {
-            // If the table data declares a particular offset base,
-            // load the address into a register to further use it as
-            // the table address.
-            self.asm.movzx_mr(
-                &self.address_at_vmctx(offset),
-                ptr_base,
-                self.ptr_size.into(),
-                TRUSTED_FLAGS,
-            );
-        } else {
-            // Else, simply move the vmctx register into the addr register as
-            // the base to calculate the table address.
-            self.asm.mov_rr(vmctx!(Self), ptr_base, self.ptr_size);
-        };
-
-        // OOB check.
-        let bound_addr = self.address_at_reg(ptr_base, table_data.current_elems_offset);
-        let bound_size = table_data.current_elements_size;
-        self.asm
-            .movzx_mr(&bound_addr, bound, bound_size.into(), TRUSTED_FLAGS);
-        self.asm.cmp_rr(bound, index, bound_size);
-        self.asm.trapif(IntCmpKind::GeU, TrapCode::TableOutOfBounds);
-
-        // Move the index into the scratch register to calcualte the table
-        // element address.
-        // Moving the value of the index register to the scratch register
-        // also avoids overwriting the context of the index register.
-        self.asm.mov_rr(index, scratch, bound_size);
-        self.asm.mul_ir(
-            table_data.element_size.bytes() as i32,
-            scratch,
-            table_data.element_size,
-        );
-        self.asm.movzx_mr(
-            &self.address_at_reg(ptr_base, table_data.offset),
-            ptr_base,
-            self.ptr_size.into(),
-            TRUSTED_FLAGS,
-        );
-        // Copy the value of the table base into a temporary register
-        // so that we can use it later in case of a misspeculation.
-        self.asm.mov_rr(ptr_base, tmp, self.ptr_size);
-        // Calculate the address of the table element.
-        self.asm.add_rr(scratch, ptr_base, self.ptr_size);
-        if self.shared_flags.enable_table_access_spectre_mitigation() {
-            // Perform a bounds check and override the value of the
-            // table element address in case the index is out of bounds.
-            self.asm.cmp_rr(bound, index, OperandSize::S32);
-            self.asm.cmov(tmp, ptr_base, IntCmpKind::GeU, self.ptr_size);
-        }
-        context.free_reg(bound);
-        context.free_reg(tmp);
-        self.address_at_reg(ptr_base, 0)
-    }
-
-    fn table_size(&mut self, table_data: &TableData, context: &mut CodeGenContext) {
-        let scratch = regs::scratch();
-        let size = context.any_gpr(self);
-
-        if let Some(offset) = table_data.import_from {
-            self.asm.movzx_mr(
-                &self.address_at_vmctx(offset),
-                scratch,
-                self.ptr_size.into(),
-                TRUSTED_FLAGS,
-            );
-        } else {
-            self.asm.mov_rr(vmctx!(Self), scratch, self.ptr_size);
-        };
-
-        let size_addr = Address::offset(scratch, table_data.current_elems_offset);
-        self.asm.movzx_mr(
-            &size_addr,
-            size,
-            table_data.current_elements_size.into(),
-            TRUSTED_FLAGS,
-        );
-
-        context.stack.push(TypedReg::i32(size).into());
-    }
-
-    fn memory_size(&mut self, heap_data: &HeapData, context: &mut CodeGenContext) {
-        let size_reg = context.any_gpr(self);
-        let scratch = regs::scratch();
-
-        let base = if let Some(offset) = heap_data.import_from {
-            self.asm.movzx_mr(
-                &self.address_at_vmctx(offset),
-                scratch,
-                self.ptr_size.into(),
-                TRUSTED_FLAGS,
-            );
-            scratch
-        } else {
-            vmctx!(Self)
-        };
-
-        let size_addr = Address::offset(base, heap_data.current_length_offset);
-        self.asm
-            .movzx_mr(&size_addr, size_reg, self.ptr_size.into(), TRUSTED_FLAGS);
-        // Prepare the stack to emit a shift to get the size in pages rather
-        // than in bytes.
-        context
-            .stack
-            .push(TypedReg::new(heap_data.ty, size_reg).into());
-
-        // Since the page size is a power-of-two, verify that 2^16, equals the
-        // defined constant. This is mostly a safeguard in case the constant
-        // value ever changes.
-        let pow = 16;
-        debug_assert_eq!(2u32.pow(pow), WASM_PAGE_SIZE);
-
-        // Ensure that the constant is correctly typed according to the heap
-        // type to reduce register pressure when emitting the shift operation.
-        match heap_data.ty {
-            WasmValType::I32 => context.stack.push(Val::i32(pow as i32)),
-            WasmValType::I64 => context.stack.push(Val::i64(pow as i64)),
-            _ => unreachable!(),
-        }
-
-        self.shift(context, ShiftKind::ShrU, heap_data.ty.into());
-    }
-
     fn address_from_sp(&self, offset: SPOffset) -> Self::Address {
         Address::offset(regs::rsp(), self.sp_offset - offset.as_u32())
     }
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index e04cf5cda..5b86dc8fd 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -1,5 +1,5 @@
 use crate::abi::{self, align_to, LocalSlot};
-use crate::codegen::{CodeGenContext, FuncEnv, HeapData, TableData};
+use crate::codegen::{CodeGenContext, FuncEnv};
 use crate::isa::reg::Reg;
 use cranelift_codegen::{
     binemit::CodeOffset,
@@ -503,22 +503,6 @@ pub(crate) trait MacroAssembler {
     /// Get the address of a local slot.
     fn local_address(&mut self, local: &LocalSlot) -> Self::Address;
 
-    /// Loads the address of the table element at a given index. Returns the
-    /// address of the table element using the provided register as base.
-    fn table_elem_address(
-        &mut self,
-        index: Reg,
-        base: Reg,
-        table_data: &TableData,
-        context: &mut CodeGenContext,
-    ) -> Self::Address;
-
-    /// Retrieves the size of the table, pushing the result to the value stack.
-    fn table_size(&mut self, table_data: &TableData, context: &mut CodeGenContext);
-
-    /// Retrieves the size of the memory, pushing the result to the value stack.
-    fn memory_size(&mut self, heap_data: &HeapData, context: &mut CodeGenContext);
-
     /// Constructs an address with an offset that is relative to the
     /// current position of the stack pointer (e.g. [sp + (sp_offset -
     /// offset)].
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index 688e5b4bf..c02abb763 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -1469,7 +1469,7 @@ where
     fn visit_table_size(&mut self, table: u32) {
         let table_index = TableIndex::from_u32(table);
         let table_data = self.env.resolve_table_data(table_index);
-        self.masm.table_size(&table_data, &mut self.context);
+        self.emit_compute_table_size(&table_data);
     }
 
     fn visit_table_fill(&mut self, table: u32) {
@@ -1505,13 +1505,8 @@ where
                     let value = self.context.pop_to_reg(self.masm, None);
                     let index = self.context.pop_to_reg(self.masm, None);
                     let base = self.context.any_gpr(self.masm);
-                    let elem_addr = self.masm.table_elem_address(
-                        index.into(),
-                        base,
-                        &table_data,
-                        &mut self.context,
-                    );
-
+                    let elem_addr =
+                        self.emit_compute_table_elem_addr(index.into(), base, &table_data);
                     // Set the initialized bit.
                     self.masm.or(
                         value.into(),
@@ -1605,7 +1600,7 @@ where
 
     fn visit_memory_size(&mut self, mem: u32, _: u8) {
         let heap = self.env.resolve_heap(MemoryIndex::from_u32(mem));
-        self.masm.memory_size(&heap, &mut self.context);
+        self.emit_compute_memory_size(&heap);
     }
 
     fn visit_memory_grow(&mut self, mem: u32, _: u8) {


Test patch:
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
index 3a0a81324..d470ed901 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
@@ -1551,7 +1551,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(2),
                 rm: xreg(3),
-                ty: I16,
             },
             flags: MemFlags::trusted(),
         },
@@ -1588,7 +1587,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(20),
                 rm: xreg(20),
-                ty: I16,
             },
             flags: MemFlags::trusted(),
         },
@@ -1625,7 +1623,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(2),
                 rm: xreg(12),
-                ty: I32,
             },
             flags: MemFlags::trusted(),
         },
@@ -1662,7 +1659,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(5),
                 rm: xreg(1),
-                ty: I32,
             },
             flags: MemFlags::trusted(),
         },
@@ -1735,7 +1731,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(2),
                 rm: xreg(3),
-                ty: I64,
             },
             flags: MemFlags::trusted(),
         },
@@ -1748,7 +1743,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaledExtended {
                 rn: xreg(2),
                 rm: xreg(3),
-                ty: I64,
                 extendop: ExtendOp::SXTW,
             },
             flags: MemFlags::trusted(),
@@ -1805,7 +1799,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::ULoad64 {
             rd: writable_xreg(1),
-            mem: AMode::FPOffset { off: 32768, ty: I8 },
+            mem: AMode::FPOffset { off: 32768 },
             flags: MemFlags::trusted(),
         },
         "100090D2A1EB70F8",
@@ -1814,10 +1808,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::ULoad64 {
             rd: writable_xreg(1),
-            mem: AMode::FPOffset {
-                off: -32768,
-                ty: I8,
-            },
+            mem: AMode::FPOffset { off: -32768 },
             flags: MemFlags::trusted(),
         },
         "F0FF8F92A1EB70F8",
@@ -1826,10 +1817,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::ULoad64 {
             rd: writable_xreg(1),
-            mem: AMode::FPOffset {
-                off: 1048576,
-                ty: I8,
-            }, // 2^20
+            mem: AMode::FPOffset { off: 1048576 }, // 2^20
             flags: MemFlags::trusted(),
         },
         "1002A0D2A1EB70F8",
@@ -1838,10 +1826,7 @@ fn test_aarch64_binemit() {
     insns.push((
         Inst::ULoad64 {
             rd: writable_xreg(1),
-            mem: AMode::FPOffset {
-                off: 1048576 + 1,
-                ty: I8,
-            }, // 2^20 + 1
+            mem: AMode::FPOffset { off: 1048576 + 1 }, // 2^20 + 1
             flags: MemFlags::trusted(),
         },
         "300080521002A072A1EB70F8",
@@ -1854,7 +1839,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegOffset {
                 rn: xreg(7),
                 off: 8,
-                ty: I64,
             },
             flags: MemFlags::trusted(),
         },
@@ -1868,7 +1852,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegOffset {
                 rn: xreg(7),
                 off: 1024,
-                ty: I64,
             },
             flags: MemFlags::trusted(),
         },
@@ -1882,7 +1865,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegOffset {
                 rn: xreg(7),
                 off: 1048576,
-                ty: I64,
             },
             flags: MemFlags::trusted(),
         },
@@ -2004,7 +1986,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(2),
                 rm: xreg(3),
-                ty: I64,
             },
             flags: MemFlags::trusted(),
         },
@@ -2017,7 +1998,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaledExtended {
                 rn: xreg(2),
                 rm: xreg(3),
-                ty: I64,
                 extendop: ExtendOp::UXTW,
             },
             flags: MemFlags::trusted(),
@@ -6692,7 +6672,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(8),
                 rm: xreg(9),
-                ty: F32,
             },
             flags: MemFlags::trusted(),
         },
@@ -6706,7 +6685,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(8),
                 rm: xreg(9),
-                ty: F64,
             },
             flags: MemFlags::trusted(),
         },
@@ -6720,7 +6698,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(8),
                 rm: xreg(9),
-                ty: I128,
             },
             flags: MemFlags::trusted(),
         },
@@ -6770,7 +6747,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(8),
                 rm: xreg(9),
-                ty: F32,
             },
             flags: MemFlags::trusted(),
         },
@@ -6784,7 +6760,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(8),
                 rm: xreg(9),
-                ty: F64,
             },
             flags: MemFlags::trusted(),
         },
@@ -6798,7 +6773,6 @@ fn test_aarch64_binemit() {
             mem: AMode::RegScaled {
                 rn: xreg(8),
                 rm: xreg(9),
-                ty: I128,
             },
             flags: MemFlags::trusted(),
         },
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs b/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs
index 6949043ff..6a258db30 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit_tests.rs
@@ -631,7 +631,7 @@ fn test_riscv64_binemit() {
             rd: writable_a0(),
             op: LoadOP::Lb,
             flags: MemFlags::new(),
-            from: AMode::RegOffset(a1(), 100, I8),
+            from: AMode::RegOffset(a1(), 100),
         },
         "lb a0,100(a1)",
         0x6458503,
@@ -641,7 +641,7 @@ fn test_riscv64_binemit() {
             rd: writable_a0(),
             op: LoadOP::Lh,
             flags: MemFlags::new(),
-            from: AMode::RegOffset(a1(), 100, I16),
+            from: AMode::RegOffset(a1(), 100),
         },
         "lh a0,100(a1)",
         0x6459503,
@@ -652,7 +652,7 @@ fn test_riscv64_binemit() {
             rd: writable_a0(),
             op: LoadOP::Lw,
             flags: MemFlags::new(),
-            from: AMode::RegOffset(a1(), 100, I32),
+            from: AMode::RegOffset(a1(), 100),
         },
         "lw a0,100(a1)",
         0x645a503,
@@ -663,7 +663,7 @@ fn test_riscv64_binemit() {
             rd: writable_a0(),
             op: LoadOP::Ld,
             flags: MemFlags::new(),
-            from: AMode::RegOffset(a1(), 100, I64),
+            from: AMode::RegOffset(a1(), 100),
         },
         "ld a0,100(a1)",
         0x645b503,
@@ -673,7 +673,7 @@ fn test_riscv64_binemit() {
             rd: Writable::from_reg(fa0()),
             op: LoadOP::Flw,
             flags: MemFlags::new(),
-            from: AMode::RegOffset(a1(), 100, I64),
+            from: AMode::RegOffset(a1(), 100),
         },
         "flw fa0,100(a1)",
         0x645a507,
@@ -684,14 +684,14 @@ fn test_riscv64_binemit() {
             rd: Writable::from_reg(fa0()),
             op: LoadOP::Fld,
             flags: MemFlags::new(),
-            from: AMode::RegOffset(a1(), 100, I64),
+            from: AMode::RegOffset(a1(), 100),
         },
         "fld fa0,100(a1)",
         0x645b507,
     ));
     insns.push(TestUnit::new(
         Inst::Store {
-            to: AMode::SPOffset(100, I8),
+            to: AMode::SPOffset(100),
             op: StoreOP::Sb,
             flags: MemFlags::new(),
             src: a0(),
@@ -701,7 +701,7 @@ fn test_riscv64_binemit() {
     ));
     insns.push(TestUnit::new(
         Inst::Store {
-            to: AMode::SPOffset(100, I16),
+            to: AMode::SPOffset(100),
             op: StoreOP::Sh,
             flags: MemFlags::new(),
             src: a0(),
@@ -711,7 +711,7 @@ fn test_riscv64_binemit() {
     ));
     insns.push(TestUnit::new(
         Inst::Store {
-            to: AMode::SPOffset(100, I32),
+            to: AMode::SPOffset(100),
             op: StoreOP::Sw,
             flags: MemFlags::new(),
             src: a0(),
@@ -721,7 +721,7 @@ fn test_riscv64_binemit() {
     ));
     insns.push(TestUnit::new(
         Inst::Store {
-            to: AMode::SPOffset(100, I64),
+            to: AMode::SPOffset(100),
             op: StoreOP::Sd,
             flags: MemFlags::new(),
             src: a0(),
@@ -731,7 +731,7 @@ fn test_riscv64_binemit() {
     ));
     insns.push(TestUnit::new(
         Inst::Store {
-            to: AMode::SPOffset(100, I64),
+            to: AMode::SPOffset(100),
             op: StoreOP::Fsw,
             flags: MemFlags::new(),
             src: fa0(),
@@ -741,7 +741,7 @@ fn test_riscv64_binemit() {
     ));
     insns.push(TestUnit::new(
         Inst::Store {
-            to: AMode::SPOffset(100, I64),
+            to: AMode::SPOffset(100),
             op: StoreOP::Fsd,
             flags: MemFlags::new(),
             src: fa0(),
diff --git a/cranelift/filetests/filetests/egraph/bitcast-ref.clif b/cranelift/filetests/filetests/egraph/bitcast-ref.clif
new file mode 100644
index 000000000..e6bb67818
--- /dev/null
+++ b/cranelift/filetests/filetests/egraph/bitcast-ref.clif
@@ -0,0 +1,71 @@
+test optimize precise-output
+set opt_level=speed
+target x86_64
+
+function %bitcast_from_r64(i64 vmctx, r64) -> i32 fast {
+    sig0 = (i32) fast
+    fn0 = colocated %foo sig0
+
+block0(v0: i64, v1: r64):
+    v2 = bitcast.i64 v1
+    v3 = ireduce.i32 v2
+
+    call fn0(v3)
+
+    ;; We MUST NOT dedupe the bitcast *from* the reference type, as that would
+    ;; make `v1` no longer live across the `call`, which would mean that it
+    ;; would not show up in the `call`'s safepoint's stack map, which could
+    ;; result in the collector reclaiming an object too early, which is basically
+    ;; a use-after-free bug.
+    v4 = bitcast.i64 v1
+    v5 = ireduce.i32 v4
+
+    return v5
+}
+
+; function %bitcast_from_r64(i64 vmctx, r64) -> i32 fast {
+;     sig0 = (i32) fast
+;     fn0 = colocated %foo sig0
+;
+; block0(v0: i64, v1: r64):
+;     v2 = bitcast.i64 v1
+;     v3 = ireduce.i32 v2
+;     call fn0(v3)
+;     v4 = bitcast.i64 v1
+;     v5 = ireduce.i32 v4
+;     return v5
+; }
+
+function %bitcast_to_r64(i64 vmctx, i32) -> r64 fast {
+    sig0 = (r64) fast
+    fn0 = colocated %foo sig0
+
+block0(v0: i64, v1: i32):
+    v2 = uextend.i64 v1
+    v3 = bitcast.r64 v2
+
+    call fn0(v3)
+
+    ;; On the other hand, it is technically fine to dedupe bitcasts *to*
+    ;; reference types. Doing so extends the live range of the GC reference --
+    ;; potentially adding it to more stack maps than it otherwise would have
+    ;; been in, which means it might unnecessarily survive a GC it otherwise
+    ;; wouldn't have -- but that is okay. Shrinking live ranges of GC
+    ;; references, and removing them from stack maps they otherwise would have
+    ;; been in, is the problematic transformation.
+    v4 = uextend.i64 v1
+    v5 = bitcast.r64 v4
+
+    return v5
+}
+
+; function %bitcast_to_r64(i64 vmctx, i32) -> r64 fast {
+;     sig0 = (r64) fast
+;     fn0 = colocated %foo sig0
+;
+; block0(v0: i64, v1: i32):
+;     v2 = uextend.i64 v1
+;     v3 = bitcast.r64 v2
+;     call fn0(v3)
+;     return v3
+; }
diff --git a/cranelift/filetests/filetests/isa/aarch64/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/aarch64/nan-canonicalization.clif
new file mode 100644
index 000000000..8ffffcdd2
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/nan-canonicalization.clif
@@ -0,0 +1,140 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target x86_64 sse41
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addps   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r10d
+;   movd    %r10d, %xmm7
+;   shufps  $0, %xmm7, const(0), %xmm7
+;   movdqa  %xmm0, %xmm1
+;   cmpps   $3, %xmm1, %xmm0, %xmm1
+;   movdqa  %xmm0, %xmm2
+;   movdqa  %xmm1, %xmm0
+;   movdqa  %xmm2, %xmm1
+;   pblendvb %xmm1, %xmm7, %xmm1
+;   movdqa  %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addps %xmm1, %xmm0
+;   movl $0x7fc00000, %r10d
+;   movd %r10d, %xmm7
+;   shufps $0, 0x26(%rip), %xmm7
+;   movdqa %xmm0, %xmm1
+;   cmpunordps %xmm0, %xmm1
+;   movdqa %xmm0, %xmm2
+;   movdqa %xmm1, %xmm0
+;   movdqa %xmm2, %xmm1
+;   pblendvb %xmm0, %xmm7, %xmm1
+;   movdqa %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   sarb $0, (%rdi)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addsd   %xmm0, %xmm1, %xmm0
+;   movabsq $9221120237041090560, %r9
+;   movq    %r9, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmppd   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addsd %xmm1, %xmm0
+;   movabsq $0x7ff8000000000000, %r9
+;   movq %r9, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordpd %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addss   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r9d
+;   movd    %r9d, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmpps   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addss %xmm1, %xmm0
+;   movl $0x7fc00000, %r9d
+;   movd %r9d, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordps %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization-has_v.clif b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization-has_v.clif
new file mode 100644
index 000000000..0a9a82457
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization-has_v.clif
@@ -0,0 +1,160 @@
+test compile precise-output
+set enable_nan_canonicalization=true
+target riscv64 has_v
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+; block0:
+;   vle8.v v9,16(fp) #avl=16, #vtype=(e8, m1, ta, ma)
+;   vle8.v v11,32(fp) #avl=16, #vtype=(e8, m1, ta, ma)
+;   vfadd.vv v13,v9,v11 #avl=4, #vtype=(e32, m1, ta, ma)
+;   lui a1,523264
+;   fmv.w.x fa2,a1
+;   vfmv.v.f v14,fa2 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v10,v13,v13 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v12,v13,v13 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmor.mm v0,v10,v12 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmerge.vvm v8,v13,v14,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vse8.v v8,0(a0) #avl=16, #vtype=(e8, m1, ta, ma)
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+; block1: ; offset 0x10
+;   .byte 0x57, 0x70, 0x08, 0xcc
+;   addi t6, s0, 0x10
+;   .byte 0x87, 0x84, 0x0f, 0x02
+;   addi t6, s0, 0x20
+;   .byte 0x87, 0x85, 0x0f, 0x02
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0xd7, 0x96, 0x95, 0x02
+;   lui a1, 0x7fc00
+;   fmv.w.x fa2, a1
+;   .byte 0x57, 0x57, 0x06, 0x5e
+;   .byte 0x57, 0x95, 0xd6, 0x72
+;   .byte 0x57, 0x96, 0xd6, 0x72
+;   .byte 0x57, 0x20, 0xa6, 0x6a
+;   .byte 0x57, 0x04, 0xd7, 0x5c
+;   .byte 0x57, 0x70, 0x08, 0xcc
+;   .byte 0x27, 0x04, 0x05, 0x02
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.d fa1,fa0,fa1,rne
+;   lui a5,4095
+;   slli a1,a5,39
+;   fmv.d.x fa3,a1
+;   vmv.v.x v8,zero #avl=2, #vtype=(e64, m1, ta, ma)
+;   vfmv.s.f v10,fa3 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v14,v8,v10,v0.t #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmv.v.x v8,zero #avl=2, #vtype=(e64, m1, ta, ma)
+;   vfmv.s.f v10,fa1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v15,v8,v10,v0.t #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmfne.vv v8,v15,v15 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmfne.vv v10,v15,v15 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmor.mm v0,v8,v10 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v8,v15,v14,v0.t #avl=2, #vtype=(e64, m1, ta, ma)
+;   vfmv.f.s fa0,v8 #avl=2, #vtype=(e64, m1, ta, ma)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.d fa1, fa0, fa1, rne
+;   lui a5, 0xfff
+;   slli a1, a5, 0x27
+;   fmv.d.x fa3, a1
+;   .byte 0x57, 0x70, 0x81, 0xcd
+;   .byte 0x57, 0x44, 0x00, 0x5e
+;   .byte 0x57, 0xd5, 0x06, 0x42
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0x57, 0x07, 0x85, 0x5c
+;   .byte 0x57, 0x44, 0x00, 0x5e
+;   .byte 0x57, 0xd5, 0x05, 0x42
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0xd7, 0x07, 0x85, 0x5c
+;   .byte 0x57, 0x94, 0xf7, 0x72
+;   .byte 0x57, 0x95, 0xf7, 0x72
+;   .byte 0x57, 0x20, 0x85, 0x6a
+;   .byte 0x57, 0x04, 0xf7, 0x5c
+;   .byte 0x57, 0x15, 0x80, 0x42
+;   ret
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.s fa0,fa0,fa1,rne
+;   lui a5,523264
+;   fmv.w.x fa1,a5
+;   vmv.v.x v15,zero #avl=4, #vtype=(e32, m1, ta, ma)
+;   vfmv.s.f v9,fa1 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v13,v15,v9,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmv.v.x v15,zero #avl=4, #vtype=(e32, m1, ta, ma)
+;   vfmv.s.f v9,fa0 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v14,v15,v9,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v15,v14,v14 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v9,v14,v14 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmor.mm v0,v15,v9 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmerge.vvm v15,v14,v13,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vfmv.f.s fa0,v15 #avl=4, #vtype=(e32, m1, ta, ma)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.s fa0, fa0, fa1, rne
+;   lui a5, 0x7fc00
+;   fmv.w.x fa1, a5
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0xd7, 0x47, 0x00, 0x5e
+;   .byte 0xd7, 0xd4, 0x05, 0x42
+;   .byte 0x57, 0x70, 0x81, 0xcd
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0xd7, 0x86, 0xf4, 0x5c
+;   .byte 0xd7, 0x47, 0x00, 0x5e
+;   .byte 0xd7, 0x54, 0x05, 0x42
+;   .byte 0x57, 0x70, 0x81, 0xcd
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0x57, 0x87, 0xf4, 0x5c
+;   .byte 0xd7, 0x17, 0xe7, 0x72
+;   .byte 0xd7, 0x14, 0xe7, 0x72
+;   .byte 0x57, 0xa0, 0xf4, 0x6a
+;   .byte 0xd7, 0x87, 0xe6, 0x5c
+;   .byte 0x57, 0x15, 0xf0, 0x42
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization.clif
new file mode 100644
index 000000000..8f74386af
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization.clif
@@ -0,0 +1,66 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target riscv64
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.d fa4,fa0,fa1,rne
+;   lui a2,4095
+;   slli a4,a2,39
+;   fmv.d.x fa0,a4
+;   feq.d a1,fa4,fa4
+;   feq.d a3,fa4,fa4
+;   and a5,a1,a3
+;   select fa0,fa0,fa4##condition=(a5 eq zero)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.d fa4, fa0, fa1, rne
+;   lui a2, 0xfff
+;   slli a4, a2, 0x27
+;   fmv.d.x fa0, a4
+;   feq.d a1, fa4, fa4
+;   feq.d a3, fa4, fa4
+;   and a5, a1, a3
+;   beqz a5, 8
+;   fmv.d fa0, fa4
+;   ret
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.s fa3,fa0,fa1,rne
+;   lui a2,523264
+;   fmv.w.x fa4,a2
+;   feq.s a0,fa3,fa3
+;   feq.s a2,fa3,fa3
+;   and a4,a0,a2
+;   select fa0,fa4,fa3##condition=(a4 eq zero)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.s fa3, fa0, fa1, rne
+;   lui a2, 0x7fc00
+;   fmv.w.x fa4, a2
+;   feq.s a0, fa3, fa3
+;   feq.s a2, fa3, fa3
+;   and a4, a0, a2
+;   bnez a4, 0xc
+;   fmv.d fa0, fa4
+;   j 8
+;   fmv.d fa0, fa3
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/s390x/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/s390x/nan-canonicalization.clif
new file mode 100644
index 000000000..f2619125d
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/nan-canonicalization.clif
@@ -0,0 +1,112 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target s390x
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   vfasb %v17, %v24, %v25
+;   bras %r1, 8 ; data.f32 NaN ; vlef %v18, 0(%r1), 0
+;   vrepf %v18, %v18, 0
+;   vfchesb %v7, %v17, %v17
+;   vfchesb %v19, %v17, %v17
+;   vno %v19, %v7, %v19
+;   vsel %v24, %v18, %v17, %v19
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   vfasb %v17, %v24, %v25
+;   bras %r1, 0xe
+;   su %f12, 0
+;   vlef %v18, 0(%r1), 0
+;   vrepf %v18, %v18, 0
+;   vfchesb %v7, %v17, %v17
+;   vfchesb %v19, %v17, %v17
+;   vno %v19, %v7, %v19
+;   vsel %v24, %v18, %v17, %v19
+;   br %r14
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   wfadb %v21, %f0, %f2
+;   bras %r1, 12 ; data.f64 NaN ; vleg %v22, 0(%r1), 0
+;   vgbm %v20, 0
+;   vpdi %v22, %v22, %v20, 0
+;   vgbm %v20, 0
+;   vpdi %v23, %v21, %v20, 0
+;   vfchedb %v19, %v23, %v23
+;   vfchedb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepg %v0, %v21, 0
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   wfadb %v21, %f0, %f2
+;   bras %r1, 0x12
+;   su %f15, 0(%r8)
+;   .byte 0x00, 0x00
+;   .byte 0x00, 0x00
+;   vleg %v22, 0(%r1), 0
+;   vzero %v20
+;   vpdi %v22, %v22, %v20, 0
+;   vzero %v20
+;   vpdi %v23, %v21, %v20, 0
+;   vfchedb %v19, %v23, %v23
+;   vfchedb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepg %v0, %v21, 0
+;   br %r14
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   wfasb %v21, %f0, %f2
+;   bras %r1, 8 ; data.f32 NaN ; vlef %v22, 0(%r1), 0
+;   vgbm %v20, 61440
+;   vn %v22, %v22, %v20
+;   vgbm %v20, 61440
+;   vn %v23, %v21, %v20
+;   vfchesb %v19, %v23, %v23
+;   vfchesb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepf %v0, %v21, 0
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   wfasb %v21, %f0, %f2
+;   bras %r1, 0xe
+;   su %f12, 0
+;   vlef %v22, 0(%r1), 0
+;   vgbm %v20, 0xf000
+;   vn %v22, %v22, %v20
+;   vgbm %v20, 0xf000
+;   vn %v23, %v21, %v20
+;   vfchesb %v19, %v23, %v23
+;   vfchesb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepf %v0, %v21, 0
+;   br %r14
+
diff --git a/cranelift/filetests/filetests/isa/x64/nan-canonicalization-sse41.clif b/cranelift/filetests/filetests/isa/x64/nan-canonicalization-sse41.clif
new file mode 100644
index 000000000..b29003307
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/nan-canonicalization-sse41.clif
@@ -0,0 +1,140 @@
+test compile precise-output
+set enable_nan_canonicalization=true
+target x86_64 sse41
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addps   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r10d
+;   movd    %r10d, %xmm7
+;   shufps  $0, %xmm7, const(0), %xmm7
+;   movdqa  %xmm0, %xmm1
+;   cmpps   $3, %xmm1, %xmm0, %xmm1
+;   movdqa  %xmm0, %xmm2
+;   movdqa  %xmm1, %xmm0
+;   movdqa  %xmm2, %xmm1
+;   pblendvb %xmm1, %xmm7, %xmm1
+;   movdqa  %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addps %xmm1, %xmm0
+;   movl $0x7fc00000, %r10d
+;   movd %r10d, %xmm7
+;   shufps $0, 0x26(%rip), %xmm7
+;   movdqa %xmm0, %xmm1
+;   cmpunordps %xmm0, %xmm1
+;   movdqa %xmm0, %xmm2
+;   movdqa %xmm1, %xmm0
+;   movdqa %xmm2, %xmm1
+;   pblendvb %xmm0, %xmm7, %xmm1
+;   movdqa %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   sarb $0, (%rdi)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addsd   %xmm0, %xmm1, %xmm0
+;   movabsq $9221120237041090560, %r9
+;   movq    %r9, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmppd   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addsd %xmm1, %xmm0
+;   movabsq $0x7ff8000000000000, %r9
+;   movq %r9, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordpd %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addss   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r9d
+;   movd    %r9d, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmpps   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addss %xmm1, %xmm0
+;   movl $0x7fc00000, %r9d
+;   movd %r9d, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordps %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/x64/nan-canonicalization.clif
new file mode 100644
index 000000000..7454b82fa
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/nan-canonicalization.clif
@@ -0,0 +1,131 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target x86_64
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addps   %xmm0, %xmm1, %xmm0
+;   movdqa  %xmm0, %xmm1
+;   movl    $2143289344, %esi
+;   movd    %esi, %xmm5
+;   shufps  $0, %xmm5, const(0), %xmm5
+;   cmpps   $3, %xmm0, %xmm1, %xmm0
+;   andps   %xmm5, %xmm0, %xmm5
+;   andnps  %xmm0, %xmm1, %xmm0
+;   orps    %xmm0, %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addps %xmm1, %xmm0
+;   movdqa %xmm0, %xmm1
+;   movl $0x7fc00000, %esi
+;   movd %esi, %xmm5
+;   shufps $0, 0x14(%rip), %xmm5
+;   cmpunordps %xmm1, %xmm0
+;   andps %xmm0, %xmm5
+;   andnps %xmm1, %xmm0
+;   orps %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   sarb $0, (%rdi)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addsd   %xmm0, %xmm1, %xmm0
+;   movdqa  %xmm0, %xmm7
+;   movabsq $9221120237041090560, %r11
+;   movq    %r11, %xmm5
+;   cmppd   $3, %xmm0, %xmm7, %xmm0
+;   andpd   %xmm5, %xmm0, %xmm5
+;   andnpd  %xmm0, %xmm7, %xmm0
+;   orpd    %xmm0, %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addsd %xmm1, %xmm0
+;   movdqa %xmm0, %xmm7
+;   movabsq $0x7ff8000000000000, %r11
+;   movq %r11, %xmm5
+;   cmpunordpd %xmm7, %xmm0
+;   andpd %xmm0, %xmm5
+;   andnpd %xmm7, %xmm0
+;   orpd %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addss   %xmm0, %xmm1, %xmm0
+;   movdqa  %xmm0, %xmm7
+;   movl    $2143289344, %r11d
+;   movd    %r11d, %xmm5
+;   cmpps   $3, %xmm0, %xmm7, %xmm0
+;   andps   %xmm5, %xmm0, %xmm5
+;   andnps  %xmm0, %xmm7, %xmm0
+;   orps    %xmm0, %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addss %xmm1, %xmm0
+;   movdqa %xmm0, %xmm7
+;   movl $0x7fc00000, %r11d
+;   movd %r11d, %xmm5
+;   cmpunordps %xmm7, %xmm0
+;   andps %xmm0, %xmm5
+;   andnps %xmm7, %xmm0
+;   orps %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/crates/component-macro/tests/codegen.rs b/crates/component-macro/tests/codegen.rs
index a292d8004..76e228203 100644
--- a/crates/component-macro/tests/codegen.rs
+++ b/crates/component-macro/tests/codegen.rs
@@ -254,7 +254,7 @@ mod with_works_with_hierarchy {
         struct X;
 
         impl FooImports for X {
-            fn x(&mut self) -> wasmtime::Result<super::bindings::foo::foo::a::T> {
+            fn x(&mut self) -> super::bindings::foo::foo::a::T {
                 loop {}
             }
         }
@@ -294,7 +294,7 @@ mod with_works_with_hierarchy {
         struct X;
 
         impl FooImports for X {
-            fn x(&mut self) -> wasmtime::Result<super::bindings::foo::foo::a::T> {
+            fn x(&mut self) -> super::bindings::foo::foo::a::T {
                 loop {}
             }
         }
@@ -334,9 +334,189 @@ mod with_works_with_hierarchy {
         struct X;
 
         impl FooImports for X {
-            fn x(&mut self) -> wasmtime::Result<super::bindings::foo::foo::a::T> {
+            fn x(&mut self) -> super::bindings::foo::foo::a::T {
                 loop {}
             }
         }
     }
 }
+
+mod trappable_imports {
+    mod none {
+        wasmtime::component::bindgen!({
+            inline: "
+                package foo:foo;
+
+                world foo {
+                    import foo: func();
+                }
+            ",
+            trappable_imports: false,
+        });
+        struct X;
+
+        impl FooImports for X {
+            fn foo(&mut self) {}
+        }
+    }
+
+    mod all {
+        wasmtime::component::bindgen!({
+            inline: "
+                package foo:foo;
+
+                world foo {
+                    import foo: func();
+                }
+            ",
+            trappable_imports: true,
+        });
+        struct X;
+
+        impl FooImports for X {
+            fn foo(&mut self) -> wasmtime::Result<()> {
+                Ok(())
+            }
+        }
+    }
+
+    mod some {
+        wasmtime::component::bindgen!({
+            inline: "
+                package foo:foo;
+
+                world foo {
+                    import foo: func();
+                    import bar: func();
+                }
+            ",
+            trappable_imports: ["foo"],
+        });
+        struct X;
+
+        impl FooImports for X {
+            fn foo(&mut self) -> wasmtime::Result<()> {
+                Ok(())
+            }
+            fn bar(&mut self) {}
+        }
+    }
+
+    mod across_interfaces {
+        use wasmtime::component::Resource;
+
+        wasmtime::component::bindgen!({
+            inline: "
+                package foo:foo;
+
+                interface a {
+                    foo: func();
+                    bar: func();
+
+                    resource r {
+                        constructor();
+                        foo: func();
+                        bar: static func();
+                    }
+                }
+
+                world foo {
+                    import a;
+                    import foo: func();
+                    import bar: func();
+                    import i: interface {
+                        foo: func();
+                        bar: func();
+                    }
+
+                }
+            ",
+            trappable_imports: ["foo"],
+            with: { "foo:foo/a/r": R },
+        });
+
+        struct X;
+        pub struct R;
+
+        impl FooImports for X {
+            fn foo(&mut self) -> wasmtime::Result<()> {
+                Ok(())
+            }
+            fn bar(&mut self) {}
+        }
+
+        impl i::Host for X {
+            fn foo(&mut self) -> wasmtime::Result<()> {
+                Ok(())
+            }
+            fn bar(&mut self) {}
+        }
+
+        impl foo::foo::a::Host for X {
+            fn foo(&mut self) -> wasmtime::Result<()> {
+                Ok(())
+            }
+            fn bar(&mut self) {}
+        }
+
+        impl foo::foo::a::HostR for X {
+            fn new(&mut self) -> Resource<R> {
+                loop {}
+            }
+            fn foo(&mut self, _: Resource<R>) {}
+            fn bar(&mut self) {}
+            fn drop(&mut self, _: Resource<R>) -> wasmtime::Result<()> {
+                Ok(())
+            }
+        }
+    }
+
+    mod resources {
+        use wasmtime::component::Resource;
+
+        wasmtime::component::bindgen!({
+            inline: "
+                package foo:foo;
+
+                interface a {
+                    resource r {
+                        constructor();
+                        foo: func();
+                        bar: static func();
+                    }
+                }
+
+                world foo {
+                    import a;
+
+                }
+            ",
+            trappable_imports: [
+                "[constructor]r",
+                "[method]r.foo",
+                "[static]r.bar",
+            ],
+            with: { "foo:foo/a/r": R },
+        });
+
+        struct X;
+        pub struct R;
+
+        impl foo::foo::a::Host for X {}
+
+        impl foo::foo::a::HostR for X {
+            fn new(&mut self) -> wasmtime::Result<Resource<R>> {
+                loop {}
+            }
+            fn foo(&mut self, _: Resource<R>) -> wasmtime::Result<()> {
+                Ok(())
+            }
+            fn bar(&mut self) -> wasmtime::Result<()> {
+                Ok(())
+            }
+            fn drop(&mut self, _: Resource<R>) -> wasmtime::Result<()> {
+                Ok(())
+            }
+        }
+    }
+}
diff --git a/tests/all/component_model/aot.rs b/tests/all/component_model/aot.rs
index b94e8f085..676de39d1 100644
--- a/tests/all/component_model/aot.rs
+++ b/tests/all/component_model/aot.rs
@@ -164,3 +164,46 @@ fn reflect_resource_import() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn truncated_component_binaries_dont_panic() -> Result<()> {
+    let engine = super::engine();
+
+    let binary = wat::parse_str(
+        r#"
+        (component
+            (import "a" (core module $m0
+                (import "" "" (func))
+            ))
+
+            (core module $m1
+                (func (export ""))
+            )
+            (core instance $i1 (instantiate (module $m1)))
+            (func $f (canon lift (core func $i1 "f")))
+
+            (component $c1
+                (import "f" (func))
+                (core module $m2
+                    (func (export "g"))
+                )
+                (core instance $i2 (instantiate $m2))
+                (func (export "g")
+                    (canon lift (core func $i2 "g"))
+                )
+            )
+            (instance $i3 (instantiate $c1 (with "f" (func $f))))
+            (func (export "g") (alias export $i3 "g"))
+        )
+        "#,
+    )?;
+
+    // Check that if we feed each truncation of the component binary into
+    // `Component::new` we don't get any panics.
+    for i in 1..binary.len() - 1 {
+        let _ = Component::from_binary(&engine, &binary[0..i]);
+    }
+
+    Ok(())
+}
diff --git a/tests/all/component_model/bindgen.rs b/tests/all/component_model/bindgen.rs
index 1c0b713a1..764836491 100644
--- a/tests/all/component_model/bindgen.rs
+++ b/tests/all/component_model/bindgen.rs
@@ -105,9 +105,8 @@ mod one_import {
         }
 
         impl foo::Host for MyImports {
-            fn foo(&mut self) -> Result<()> {
+            fn foo(&mut self) {
                 self.hit = true;
-                Ok(())
             }
         }
 
@@ -181,9 +180,9 @@ mod resources_at_world_level {
         }
 
         impl HostX for MyImports {
-            fn new(&mut self) -> Result<Resource<X>> {
+            fn new(&mut self) -> Resource<X> {
                 self.ctor_hit = true;
-                Ok(Resource::new_own(80))
+                Resource::new_own(80)
             }
 
             fn drop(&mut self, val: Resource<X>) -> Result<()> {
@@ -286,9 +285,9 @@ mod resources_at_interface_level {
         use foo::foo::def::X;
 
         impl foo::foo::def::HostX for MyImports {
-            fn new(&mut self) -> Result<Resource<X>> {
+            fn new(&mut self) -> Resource<X> {
                 self.ctor_hit = true;
-                Ok(Resource::new_own(80))
+                Resource::new_own(80)
             }
 
             fn drop(&mut self, val: Resource<X>) -> Result<()> {
@@ -337,13 +336,9 @@ mod async_config {
 
     #[async_trait::async_trait]
     impl T1Imports for T {
-        async fn x(&mut self) -> Result<()> {
-            Ok(())
-        }
+        async fn x(&mut self) {}
 
-        async fn y(&mut self) -> Result<()> {
-            Ok(())
-        }
+        async fn y(&mut self) {}
     }
 
     async fn _test_t1(t1: &T1, store: &mut Store<()>) {
@@ -367,13 +362,9 @@ mod async_config {
 
     #[async_trait::async_trait]
     impl T2Imports for T {
-        fn x(&mut self) -> Result<()> {
-            Ok(())
-        }
+        fn x(&mut self) {}
 
-        async fn y(&mut self) -> Result<()> {
-            Ok(())
-        }
+        async fn y(&mut self) {}
     }
 
     async fn _test_t2(t2: &T2, store: &mut Store<()>) {
@@ -397,13 +388,9 @@ mod async_config {
 
     #[async_trait::async_trait]
     impl T3Imports for T {
-        async fn x(&mut self) -> Result<()> {
-            Ok(())
-        }
+        async fn x(&mut self) {}
 
-        fn y(&mut self) -> Result<()> {
-            Ok(())
-        }
+        fn y(&mut self) {}
     }
 
     async fn _test_t3(t3: &T3, store: &mut Store<()>) {
@@ -468,11 +455,11 @@ mod exported_resources {
     }
 
     impl a::HostX for MyImports {
-        fn new(&mut self) -> Result<Resource<a::X>> {
+        fn new(&mut self) -> Resource<a::X> {
             let rep = self.next_a_x;
             self.next_a_x += 1;
             self.hostcalls.push(Hostcall::NewA);
-            Ok(Resource::new_own(rep))
+            Resource::new_own(rep)
         }
 
         fn drop(&mut self, val: Resource<a::X>) -> Result<()> {
diff --git a/tests/all/component_model/bindgen/results.rs b/tests/all/component_model/bindgen/results.rs
index 29f60fe8b..dfb361523 100644
--- a/tests/all/component_model/bindgen/results.rs
+++ b/tests/all/component_model/bindgen/results.rs
@@ -17,6 +17,7 @@ mod empty_error {
 
             export empty-error: func(a: float64) -> result<float64>;
         }",
+        trappable_imports: true,
     });
 
     #[test]
@@ -118,6 +119,7 @@ mod string_error {
 
             export string-error: func(a: float64) -> result<float64, string>;
         }",
+        trappable_imports: true,
     });
 
     #[test]
@@ -238,7 +240,8 @@ mod enum_error {
                 enum-error: func(a: float64) -> result<float64, e1>;
             }
         }",
-        trappable_error_type: { "inline:inline/imports/e1" => TrappableE1 }
+        trappable_error_type: { "inline:inline/imports/e1" => TrappableE1 },
+        trappable_imports: true,
     });
 
     // You can create concrete trap types which make it all the way out to the
@@ -418,7 +421,8 @@ mod record_error {
                 record-error: func(a: float64) -> result<float64, e2>;
             }
         }",
-        trappable_error_type: { "inline:inline/imports/e2" => TrappableE2 }
+        trappable_error_type: { "inline:inline/imports/e2" => TrappableE2 },
+        trappable_imports: true,
     });
 
     pub enum TrappableE2 {
@@ -589,7 +593,8 @@ mod variant_error {
                 variant-error: func(a: float64) -> result<float64, e3>;
             }
         }",
-        trappable_error_type: { "inline:inline/imports/e3" => TrappableE3 }
+        trappable_error_type: { "inline:inline/imports/e3" => TrappableE3 },
+        trappable_imports: true,
     });
 
     pub enum TrappableE3 {
@@ -784,7 +789,8 @@ mod multiple_interfaces_error {
                 enum-error: func(a: float64) -> result<float64, e1>;
             }
         }",
-        trappable_error_type: { "inline:inline/types/e1" => TrappableE1 }
+        trappable_error_type: { "inline:inline/types/e1" => TrappableE1 },
+        trappable_imports: true,
     });
 
     pub enum TrappableE1 {
@@ -967,6 +973,7 @@ mod with_remapping {
             import imports: interface {
                 empty-error: func(a: float64) -> result<float64>;
             }",
+            trappable_imports: true,
         });
     }
 
@@ -983,6 +990,7 @@ mod with_remapping {
         with: {
             "imports": interfaces::imports,
         },
+        trappable_imports: true,
     });
 
     #[test]