Repository: bytecodealliance/wasmtime
Pull request #9137 (2024-08-16 20:55:33)

 Don't force `Multiple` on multi-result instructions 
This commit is a result of [discussion on Zulip][Zulip] and is
attempting to fix an issue where some 128-bit instructions aren't fully
benefitting from load sinking on x64. At a high level 128-bit
addition isn't able to sink loads into instructions for halves of the
128-bit operation. At a lower level the reason for this is that
currently all operands of a multiple-result instruction are considered
multiply-used (as each result could be used) which prevents load
sinking.

Operations on 128-bit integers may be coupled with `isplit` afterwards
which is a multi-result instruction. This then means that the `Multiple`
state flows backwards to the 128-bit operation and all its operands,
including whatever is necessary to produce the individual components of
each 128-bit integer.

The fix in this commit is to introduce the concept of a "root"
instruction from the perspective of the calculation of `ValueUseState`.
In other words `ValueUseState` is no longer an accurate picture of the
function as a whole, but only the parts of the function rooted at a
"root" instruction. This is currently defined as multi-result
instructions meaning that `isplit` for example is a root instruction.
This is coupled with documentation/changes to
`get_value_as_source_or_const` to never allow looking through root
instructions (or considering them a `UniqueUse`).

This commit additionally updates some documentation in a few places and
refactors some usage of `get_value_as_source_or_const` to use other
helpers instead to reduce callers of `get_value_as_source_or_const` and
who to audit when modifying this function.

[Zulip]: https://bytecodealliance.zulipchat.com/#narrow/stream/217117-cranelift/topic/ValueUseState.3A.3AMultiple.20and.20multi-result.20instructions/near/462833578

github-actions[bot]: #### Subscribe to Label Action

cc @cfallin, @fitzgen

<details>
This issue or pull request has been labeled: "cranelift", "cranelift:area:machinst", "cranelift:area:x64", "isle"

Thus the following users have been cc'd because of the following labels:

* cfallin: isle
* fitzgen: isle

To subscribe or unsubscribe from this label, edit the <code>.github/subscribe-to-label.json</code> configuration file.

[Learn more.](https://github.com/bytecodealliance/subscribe-to-label-action)
</details>

Patch:
diff --git a/Cargo.lock b/Cargo.lock
index e010c2011..d4cba3dc3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3651,6 +3651,7 @@ dependencies = [
  "gimli",
  "log",
  "object",
+ "smallvec",
  "target-lexicon",
  "thiserror",
  "wasmparser",
diff --git a/cranelift/codegen/meta/src/isa/riscv64.rs b/cranelift/codegen/meta/src/isa/riscv64.rs
index d11fbf951..355a9125a 100644
--- a/cranelift/codegen/meta/src/isa/riscv64.rs
+++ b/cranelift/codegen/meta/src/isa/riscv64.rs
@@ -64,6 +64,13 @@ pub(crate) fn define() -> TargetIsa {
         false,
     );
 
+    let _has_zfh = setting.add_bool(
+        "has_zfh",
+        "has extension Zfh?",
+        "Zfh: Half-Precision Floating-Point Instructions",
+        false,
+    );
+
     let _has_v = setting.add_bool(
         "has_v",
         "has extension V?",
diff --git a/cranelift/codegen/src/ir/user_stack_maps.rs b/cranelift/codegen/src/ir/user_stack_maps.rs
index 48ae47c82..7378c3747 100644
--- a/cranelift/codegen/src/ir/user_stack_maps.rs
+++ b/cranelift/codegen/src/ir/user_stack_maps.rs
@@ -67,12 +67,20 @@ pub struct UserStackMapEntry {
     derive(serde_derive::Deserialize, serde_derive::Serialize)
 )]
 pub struct UserStackMap {
+    // Offsets into the frame's sized stack slots that are GC references, by type.
     by_type: SmallVec<[(ir::Type, CompoundBitSet); 1]>,
+
+    // The offset of the sized stack slots, from SP, for this stack map's
+    // associated PC.
+    //
+    // This is initially `None` upon construction during lowering, but filled in
+    // after regalloc during emission when we have the precise frame layout.
+    sp_to_sized_stack_slots: Option<u32>,
 }
 
 impl UserStackMap {
     /// Coalesce the given entries into a new `UserStackMap`.
-    pub fn new(
+    pub(crate) fn new(
         entries: &[UserStackMapEntry],
         stack_slot_offsets: &PrimaryMap<ir::StackSlot, u32>,
     ) -> Self {
@@ -96,6 +104,35 @@ impl UserStackMap {
             by_type[index].1.insert(offset);
         }
 
-        UserStackMap { by_type }
+        UserStackMap {
+            by_type,
+            sp_to_sized_stack_slots: None,
+        }
+    }
+
+    /// Finalize this stack map by filling in the SP-to-stack-slots offset.
+    pub(crate) fn finalize(&mut self, sp_to_sized_stack_slots: u32) {
+        debug_assert!(self.sp_to_sized_stack_slots.is_none());
+        self.sp_to_sized_stack_slots = Some(sp_to_sized_stack_slots);
+    }
+
+    /// Iterate over the entries in this stack map.
+    ///
+    /// Yields pairs of the type of GC reference that is at the offset, and the
+    /// offset from SP. If a pair `(i64, 0x42)` is yielded, for example, then
+    /// when execution is at this stack map's associated PC, `SP + 0x42` is a
+    /// pointer to an `i64`, and that `i64` is a live GC reference.
+    pub fn entries(&self) -> impl Iterator<Item = (ir::Type, u32)> + '_ {
+        let sp_to_sized_stack_slots = self.sp_to_sized_stack_slots.expect(
+            "`sp_to_sized_stack_slots` should have been filled in before this stack map was used",
+        );
+        self.by_type.iter().flat_map(move |(ty, bitset)| {
+            bitset.iter().map(move |slot_offset| {
+                (
+                    *ty,
+                    sp_to_sized_stack_slots + u32::try_from(slot_offset).unwrap(),
+                )
+            })
+        })
     }
 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 111a68ca4..d345cd064 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -449,6 +449,7 @@
   (Lhu)
   (Lwu)
   (Ld)
+  (Flh)
   (Flw)
   (Fld)
 ))
@@ -458,6 +459,7 @@
   (Sh)
   (Sw)
   (Sd)
+  (Fsh)
   (Fsw)
   (Fsd)
 ))
@@ -1004,6 +1006,9 @@
 (decl pure has_zfa () bool)
 (extern constructor has_zfa has_zfa)
 
+(decl pure has_zfh () bool)
+(extern constructor has_zfh has_zfh)
+
 (decl pure has_zbkb () bool)
 (extern constructor has_zbkb has_zbkb)
 
@@ -1364,6 +1369,10 @@
 (decl rv_fnmsub (Type FRM FReg FReg FReg) FReg)
 (rule (rv_fnmsub ty frm rs1 rs2 rs3) (fpu_rrrr (FpuOPRRRR.Fnmsub) ty frm rs1 rs2 rs3))
 
+;; Helper for emitting the `fmv.x.h` instruction.
+(decl rv_fmvxh (FReg) XReg)
+(rule (rv_fmvxh r) (fpu_rr_int (FpuOPRR.FmvXFmt) $F16 (FRM.RNE) r))
+
 ;; Helper for emitting the `fmv.x.w` instruction.
 (decl rv_fmvxw (FReg) XReg)
 (rule (rv_fmvxw r) (fpu_rr_int (FpuOPRR.FmvXFmt) $F32 (FRM.RNE) r))
@@ -1372,6 +1381,10 @@
 (decl rv_fmvxd (FReg) XReg)
 (rule (rv_fmvxd r) (fpu_rr_int (FpuOPRR.FmvXFmt) $F64 (FRM.RNE) r))
 
+;; Helper for emitting the `fmv.h.x` instruction.
+(decl rv_fmvhx (XReg) FReg)
+(rule (rv_fmvhx r) (fpu_rr (FpuOPRR.FmvFmtX) $F16 (FRM.RNE) r))
+
 ;; Helper for emitting the `fmv.w.x` instruction.
 (decl rv_fmvwx (XReg) FReg)
 (rule (rv_fmvwx r) (fpu_rr (FpuOPRR.FmvFmtX) $F32 (FRM.RNE) r))
@@ -1829,11 +1842,12 @@
 
 ;; Special-case 0.0 for floats to use the `(zero_reg)` directly.
 ;; See #7162 for why this doesn't fall out of the rules below.
-(rule 8 (imm $F32 0) (gen_bitcast (zero_reg) $I32 $F32))
-(rule 8 (imm $F64 0) (gen_bitcast (zero_reg) $I64 $F64))
+(rule 9 (imm (ty_supported_float $F16) 0) (gen_bitcast (zero_reg) $I16 $F16))
+(rule 9 (imm (ty_supported_float $F32) 0) (gen_bitcast (zero_reg) $I32 $F32))
+(rule 9 (imm (ty_supported_float $F64) 0) (gen_bitcast (zero_reg) $I64 $F64))
 
 ;; If Zfa is enabled, we can load certain constants with the `fli` instruction.
-(rule 7 (imm (ty_supported_float ty) imm)
+(rule 8 (imm (ty_supported_float (ty_32_or_64 ty)) imm)
   (if-let $true (has_zfa))
   (if-let const (fli_constant_from_u64 ty imm))
   (rv_fli ty const))
@@ -1843,14 +1857,15 @@
 ;;
 ;; For f64's this saves one instruction, and for f32's it avoids
 ;; having to allocate an integer register, reducing integer register pressure.
-(rule 6 (imm (ty_supported_float ty) imm)
+(rule 7 (imm (ty_supported_float (ty_32_or_64 ty)) imm)
   (if-let $true (has_zfa))
   (if-let const (fli_constant_from_negated_u64 ty imm))
   (rv_fneg ty (rv_fli ty const)))
 
 ;; Otherwise floats get loaded as integers and then moved into an F register.
-(rule 5 (imm $F32 c) (gen_bitcast (imm $I32 c) $I32 $F32))
-(rule 5 (imm $F64 c) (gen_bitcast (imm $I64 c) $I64 $F64))
+(rule 6 (imm (ty_supported_float $F16) c) (gen_bitcast (imm $I16 c) $I16 $F16))
+(rule 6 (imm (ty_supported_float $F32) c) (gen_bitcast (imm $I32 c) $I32 $F32))
+(rule 6 (imm (ty_supported_float $F64) c) (gen_bitcast (imm $I64 c) $I64 $F64))
 
 ;; Refs get loaded as integers.
 (rule 5 (imm $R32 c) (imm $I32 c))
@@ -2422,6 +2437,7 @@
 (decl load_op_reg_type (LoadOP) Type)
 (rule 1 (load_op_reg_type (LoadOP.Fld)) $F64)
 (rule 1 (load_op_reg_type (LoadOP.Flw)) $F64)
+(rule 1 (load_op_reg_type (LoadOP.Flh)) $F64)
 (rule 0 (load_op_reg_type _) $I64)
 
 ;; Helper constructor to build a load instruction.
@@ -2908,12 +2924,18 @@
 ;; Generates a bitcast instruction.
 ;; Args are: src, src_ty, dst_ty
 (decl gen_bitcast (Reg Type Type) Reg)
-(rule 5 (gen_bitcast r (ty_supported_float src_ty) (ty_supported_vec _)) (rv_vfmv_sf r src_ty))
-(rule 4 (gen_bitcast r (ty_int_ref_scalar_64 src_ty) (ty_supported_vec _)) (rv_vmv_sx r src_ty))
-(rule 3 (gen_bitcast r (ty_supported_vec _) (ty_supported_float dst_ty)) (rv_vfmv_fs r dst_ty))
+
+;; To support FP16 vfmv.* we need to check for the `zvfh` isa flag, which we currently don't
+;; support, so restrict the floating point types to 32/64 bits.
+(rule 5 (gen_bitcast r (ty_supported_float (ty_32_or_64 src_ty)) (ty_supported_vec _)) (rv_vfmv_sf r src_ty))
+(rule 4 (gen_bitcast r (ty_supported_vec _) (ty_supported_float (ty_32_or_64 dst_ty))) (rv_vfmv_fs r dst_ty))
+
+(rule 3 (gen_bitcast r (ty_int_ref_scalar_64 src_ty) (ty_supported_vec _)) (rv_vmv_sx r src_ty))
 (rule 2 (gen_bitcast r (ty_supported_vec _) (ty_int_ref_scalar_64 dst_ty)) (rv_vmv_xs r dst_ty))
+(rule 1 (gen_bitcast r $F16 $I16) (rv_fmvxh r))
 (rule 1 (gen_bitcast r $F32 $I32) (rv_fmvxw r))
 (rule 1 (gen_bitcast r $F64 $I64) (rv_fmvxd r))
+(rule 1 (gen_bitcast r $I16 $F16) (rv_fmvhx r))
 (rule 1 (gen_bitcast r $I32 $F32) (rv_fmvwx r))
 (rule 1 (gen_bitcast r $I64 $F64) (rv_fmvdx r))
 (rule 1 (gen_bitcast r $I64 $R64) (copy_reg r $I64))
diff --git a/cranelift/codegen/src/isa/riscv64/inst/args.rs b/cranelift/codegen/src/isa/riscv64/inst/args.rs
index f3a276e0d..60123aeb8 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/args.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/args.rs
@@ -1170,22 +1170,22 @@ impl LoadOP {
             Self::Lhu => "lhu",
             Self::Lwu => "lwu",
             Self::Ld => "ld",
+            Self::Flh => "flh",
             Self::Flw => "flw",
             Self::Fld => "fld",
         }
     }
 
-    pub(crate) fn from_type(t: Type) -> Self {
-        if t.is_float() {
-            return if t == F32 { Self::Flw } else { Self::Fld };
-        }
-        match t {
-            R32 => Self::Lwu,
-            R64 | I64 => Self::Ld,
-
+    pub(crate) fn from_type(ty: Type) -> Self {
+        match ty {
+            F16 => Self::Flh,
+            F32 => Self::Flw,
+            F64 => Self::Fld,
             I8 => Self::Lb,
             I16 => Self::Lh,
             I32 => Self::Lw,
+            R32 => Self::Lwu,
+            R64 | I64 => Self::Ld,
             _ => unreachable!(),
         }
     }
@@ -1193,7 +1193,7 @@ impl LoadOP {
     pub(crate) fn size(&self) -> i64 {
         match self {
             Self::Lb | Self::Lbu => 1,
-            Self::Lh | Self::Lhu => 2,
+            Self::Lh | Self::Lhu | Self::Flh => 2,
             Self::Lw | Self::Lwu | Self::Flw => 4,
             Self::Ld | Self::Fld => 8,
         }
@@ -1204,7 +1204,7 @@ impl LoadOP {
             Self::Lb | Self::Lh | Self::Lw | Self::Lbu | Self::Lhu | Self::Lwu | Self::Ld => {
                 0b0000011
             }
-            Self::Flw | Self::Fld => 0b0000111,
+            Self::Flh | Self::Flw | Self::Fld => 0b0000111,
         }
     }
     pub(crate) fn funct3(self) -> u32 {
@@ -1216,6 +1216,7 @@ impl LoadOP {
             Self::Lbu => 0b100,
             Self::Lhu => 0b101,
             Self::Ld => 0b011,
+            Self::Flh => 0b001,
             Self::Flw => 0b010,
             Self::Fld => 0b011,
         }
@@ -1229,19 +1230,20 @@ impl StoreOP {
             Self::Sh => "sh",
             Self::Sw => "sw",
             Self::Sd => "sd",
+            Self::Fsh => "fsh",
             Self::Fsw => "fsw",
             Self::Fsd => "fsd",
         }
     }
-    pub(crate) fn from_type(t: Type) -> Self {
-        if t.is_float() {
-            return if t == F32 { Self::Fsw } else { Self::Fsd };
-        }
-        match t.bits() {
-            1 | 8 => Self::Sb,
-            16 => Self::Sh,
-            32 => Self::Sw,
-            64 => Self::Sd,
+    pub(crate) fn from_type(ty: Type) -> Self {
+        match ty {
+            F16 => Self::Fsh,
+            F32 => Self::Fsw,
+            F64 => Self::Fsd,
+            I8 => Self::Sb,
+            I16 => Self::Sh,
+            I32 | R32 => Self::Sw,
+            I64 | R64 => Self::Sd,
             _ => unreachable!(),
         }
     }
@@ -1249,7 +1251,7 @@ impl StoreOP {
     pub(crate) fn size(&self) -> i64 {
         match self {
             Self::Sb => 1,
-            Self::Sh => 2,
+            Self::Sh | Self::Fsh => 2,
             Self::Sw | Self::Fsw => 4,
             Self::Sd | Self::Fsd => 8,
         }
@@ -1258,7 +1260,7 @@ impl StoreOP {
     pub(crate) fn op_code(self) -> u32 {
         match self {
             Self::Sb | Self::Sh | Self::Sw | Self::Sd => 0b0100011,
-            Self::Fsw | Self::Fsd => 0b0100111,
+            Self::Fsh | Self::Fsw | Self::Fsd => 0b0100111,
         }
     }
     pub(crate) fn funct3(self) -> u32 {
@@ -1267,6 +1269,7 @@ impl StoreOP {
             Self::Sh => 0b001,
             Self::Sw => 0b010,
             Self::Sd => 0b011,
+            Self::Fsh => 0b001,
             Self::Fsw => 0b010,
             Self::Fsd => 0b011,
         }
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
index 71fe4bc70..70ca1d019 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -807,6 +807,7 @@ impl MachInst for Inst {
             I64 => Ok((&[RegClass::Int], &[I64])),
             R32 => panic!("32-bit reftype pointer should never be seen on riscv64"),
             R64 => Ok((&[RegClass::Int], &[R64])),
+            F16 => Ok((&[RegClass::Float], &[F16])),
             F32 => Ok((&[RegClass::Float], &[F32])),
             F64 => Ok((&[RegClass::Float], &[F64])),
             I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
@@ -1429,6 +1430,7 @@ impl Inst {
                 let rd = format_reg(rd.to_reg());
 
                 let op = match ty {
+                    F16 => "fmv.h",
                     F32 => "fmv.s",
                     F64 => "fmv.d",
                     ty if ty.is_vector() => "vmv1r.v",
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 6b249698b..7e3328af7 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -14,6 +14,11 @@
 (rule (lower (has_type (ty_supported_vec ty) (vconst n)))
   (gen_constant ty (const_to_vconst n)))
 
+;;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (f16const (u16_from_ieee16 n)))
+  (imm $F16 n))
+
 ;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (f32const (u32_from_ieee32 n)))
diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
index e1e79e075..e4d516e22 100644
--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
@@ -194,6 +194,8 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
             // So are references
             ty if ty.is_ref() => true,
             // Floating point types depend on certain extensions
+            F16 => self.backend.isa_flags.has_zfh(),
+            // F32 depends on the F extension
             F32 => self.backend.isa_flags.has_f(),
             // F64 depends on the D extension
             F64 => self.backend.isa_flags.has_d(),
@@ -207,10 +209,16 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
                 true
             }
 
-            // If the vector type has floating point lanes, then we only support it for
-            // 32 or 64 bit lanes with the base extension
+            // If the vector type has floating point lanes then the spec states:
+            //
+            // Vector instructions where any floating-point vector operand’s EEW is not a
+            // supported floating-point type width (which includes when FLEN < SEW) are reserved.
+            //
+            // So we also have to check if we support the scalar version of the type.
             ty if self.ty_vec_fits_in_register(ty).is_some()
                 && lane_type.is_float()
+                && self.ty_supported(lane_type).is_some()
+                // Additionally the base V spec only supports 32 and 64 bit floating point types.
                 && (lane_type.bits() == 32 || lane_type.bits() == 64) =>
             {
                 true
@@ -455,6 +463,10 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
         self.backend.isa_flags.has_zfa()
     }
 
+    fn has_zfh(&mut self) -> bool {
+        self.backend.isa_flags.has_zfh()
+    }
+
     fn has_zbkb(&mut self) -> bool {
         self.backend.isa_flags.has_zbkb()
     }
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs
index f205fe74f..a3812a6d9 100644
--- a/cranelift/codegen/src/isa/s390x/lower/isle.rs
+++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs
@@ -1049,11 +1049,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> {
 
     #[inline]
     fn sinkable_inst(&mut self, val: Value) -> Option<Inst> {
-        let input = self.lower_ctx.get_value_as_source_or_const(val);
-        if let Some((inst, 0)) = input.inst.as_inst() {
-            return Some(inst);
-        }
-        None
+        self.is_sinkable_inst(val)
     }
 
     #[inline]
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 34bfb93f9..d05950a3c 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -331,8 +331,7 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
     }
 
     fn sinkable_load(&mut self, val: Value) -> Option<SinkableLoad> {
-        let input = self.lower_ctx.get_value_as_source_or_const(val);
-        if let InputSourceInst::UniqueUse(inst, 0) = input.inst {
+        if let Some(inst) = self.is_sinkable_inst(val) {
             if let Some((addr_input, offset)) =
                 is_mergeable_load(self.lower_ctx, inst, MergeableLoadSize::Min32)
             {
@@ -347,8 +346,7 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
     }
 
     fn sinkable_load_exact(&mut self, val: Value) -> Option<SinkableLoad> {
-        let input = self.lower_ctx.get_value_as_source_or_const(val);
-        if let InputSourceInst::UniqueUse(inst, 0) = input.inst {
+        if let Some(inst) = self.is_sinkable_inst(val) {
             if let Some((addr_input, offset)) =
                 is_mergeable_load(self.lower_ctx, inst, MergeableLoadSize::Exact)
             {
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index aa09c5c9e..904e3f4b7 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -70,24 +70,24 @@
 //! SP at function entry ----->  | return address            |
 //!                              +---------------------------+
 //! FP after prologue -------->  | FP (pushed by prologue)   |
-//!                              +---------------------------+   -----
-//!                              |          ...              |     |
-//!                              | clobbered callee-saves    |     |
-//! unwind-frame base -------->  | (pushed by prologue)      |     |
-//!                              +---------------------------+     |
-//!                              |          ...              |     |
-//!                              | spill slots               |     |
-//!                              | (accessed via SP)         |   active
-//!                              |          ...              |    size
-//!                              | stack slots               |     |
-//!                              | (accessed via SP)         |     |
-//!                              | (alloc'd by prologue)     |     |
-//!                              +---------------------------+     |
-//!                              | [alignment as needed]     |     |
-//!                              |          ...              |     |
-//!                              | args for largest call     |     |
-//! SP ----------------------->  | (alloc'd by prologue)     |     |
-//!                              +===========================+   -----
+//!                              +---------------------------+           -----
+//!                              |          ...              |             |
+//!                              | clobbered callee-saves    |             |
+//! unwind-frame base -------->  | (pushed by prologue)      |             |
+//!                              +---------------------------+   -----     |
+//!                              |          ...              |     |       |
+//!                              | spill slots               |     |       |
+//!                              | (accessed via SP)         |   fixed   active
+//!                              |          ...              |   frame    size
+//!                              | stack slots               |  storage    |
+//!                              | (accessed via SP)         |    size     |
+//!                              | (alloc'd by prologue)     |     |       |
+//!                              +---------------------------+   -----     |
+//!                              | [alignment as needed]     |             |
+//!                              |          ...              |             |
+//!                              | args for largest call     |             |
+//! SP ----------------------->  | (alloc'd by prologue)     |             |
+//!                              +===========================+           -----
 //!
 //!   (low address)
 //! ```
@@ -986,6 +986,11 @@ impl FrameLayout {
     pub fn active_size(&self) -> u32 {
         self.outgoing_args_size + self.fixed_frame_storage_size + self.clobber_size
     }
+
+    /// Get the offset from the SP to the sized stack slots area.
+    pub fn sp_to_sized_stack_slots(&self) -> u32 {
+        self.outgoing_args_size
+    }
 }
 
 /// ABI object for a function body.
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index 738a7f24a..82a6e6add 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -1693,7 +1693,7 @@ impl<I: VCodeInst> MachBuffer<I> {
         &mut self,
         emit_state: &I::State,
         return_addr: CodeOffset,
-        stack_map: ir::UserStackMap,
+        mut stack_map: ir::UserStackMap,
     ) {
         let span = emit_state.frame_layout().active_size();
         trace!("Adding user stack map @ {return_addr:#x} spanning {span} bytes: {stack_map:?}");
@@ -1707,6 +1707,7 @@ impl<I: VCodeInst> MachBuffer<I> {
             return_addr,
         );
 
+        stack_map.finalize(emit_state.frame_layout().sp_to_sized_stack_slots());
         self.user_stack_maps.push((return_addr, span, stack_map));
     }
 }
@@ -1777,6 +1778,16 @@ impl<T: CompilePhase> MachBufferFinalized<T> {
         mem::take(&mut self.stack_maps)
     }
 
+    /// Ge tthe user stack map metadata for this code.
+    pub fn user_stack_maps(&self) -> &[(CodeOffset, u32, ir::UserStackMap)] {
+        &self.user_stack_maps
+    }
+
+    /// Take this buffer's user strack map metadata.
+    pub fn take_user_stack_maps(&mut self) -> SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]> {
+        mem::take(&mut self.user_stack_maps)
+    }
+
     /// Get the list of call sites for this code.
     pub fn call_sites(&self) -> &[MachCallSite] {
         &self.call_sites[..]
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index fa6e8265d..86cda5d3e 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -295,15 +295,36 @@ pub struct Lower<'func, I: VCodeInst> {
 /// actually merged" point. Instead, we compute a
 /// transitive-uniqueness. That is what this enum represents.
 ///
-/// To define it plainly: a value is `Unused` if no references exist
-/// to it; `Once` if only one other op refers to it, *and* that other
-/// op is `Unused` or `Once`; and `Multiple` otherwise. In other
-/// words, `Multiple` is contagious: even if an op's result value is
-/// directly used only once in the CLIF, that value is `Multiple` if
-/// the op that uses it is itself used multiple times (hence could be
-/// codegen'd multiple times). In brief, this analysis tells us
-/// whether, if every op merged all of its operand tree, a given op
-/// could be codegen'd in more than one place.
+/// There is one final caveat as well to the result of this analysis.  Notably,
+/// we define some instructions to be "root" instructions, which means that we
+/// assume they will always be codegen'd at the root of a matching tree, and not
+/// matched. (This comes with the caveat that we actually enforce this property
+/// by making them "opaque" to subtree matching in
+/// `get_value_as_source_or_const`). Because they will always be codegen'd once,
+/// they in some sense "reset" multiplicity: these root instructions can be used
+/// many times, but because their result(s) are only computed once, they only
+/// use their inputs once.
+///
+/// We currently define all multi-result instructions to be "root" instructions,
+/// because it is too complex to reason about matching through them, and they
+/// cause too-coarse-grained approximation of multiplicity otherwise: the
+/// analysis would have to assume (as it used to!) that they are always
+/// multiply-used, simply because they have multiple outputs even if those
+/// outputs are used only once.
+///
+/// In the future we could define other instructions to be "root" instructions
+/// as well, if we make the corresponding change to get_value_as_source_or_const
+/// as well.
+///
+/// To define `ValueUseState` more plainly: a value is `Unused` if no references
+/// exist to it; `Once` if only one other op refers to it, *and* that other op
+/// is `Unused` or `Once`; and `Multiple` otherwise. In other words, `Multiple`
+/// is contagious (except through root instructions): even if an op's result
+/// value is directly used only once in the CLIF, that value is `Multiple` if
+/// the op that uses it is itself used multiple times (hence could be codegen'd
+/// multiple times). In brief, this analysis tells us whether, if every op
+/// merged all of its operand tree, a given op could be codegen'd in more than
+/// one place.
 ///
 /// To compute this, we first consider direct uses. At this point
 /// `Unused` answers are correct, `Multiple` answers are correct, but
@@ -452,7 +473,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             block_end_colors[bb] = InstColor::new(cur_color);
         }
 
-        let value_ir_uses = Self::compute_use_states(f, sret_param);
+        let value_ir_uses = compute_use_states(f, sret_param);
 
         Ok(Lower {
             f,
@@ -482,120 +503,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         self.vcode.sigs_mut()
     }
 
-    /// Pre-analysis: compute `value_ir_uses`. See comment on
-    /// `ValueUseState` for a description of what this analysis
-    /// computes.
-    fn compute_use_states(
-        f: &Function,
-        sret_param: Option<Value>,
-    ) -> SecondaryMap<Value, ValueUseState> {
-        // We perform the analysis without recursion, so we don't
-        // overflow the stack on long chains of ops in the input.
-        //
-        // This is sort of a hybrid of a "shallow use-count" pass and
-        // a DFS. We iterate over all instructions and mark their args
-        // as used. However when we increment a use-count to
-        // "Multiple" we push its args onto the stack and do a DFS,
-        // immediately marking the whole dependency tree as
-        // Multiple. Doing both (shallow use-counting over all insts,
-        // and deep Multiple propagation) lets us trim both
-        // traversals, stopping recursion when a node is already at
-        // the appropriate state.
-        //
-        // In particular, note that the *coarsening* into {Unused,
-        // Once, Multiple} is part of what makes this pass more
-        // efficient than a full indirect-use-counting pass.
-
-        let mut value_ir_uses = SecondaryMap::with_default(ValueUseState::Unused);
-
-        if let Some(sret_param) = sret_param {
-            // There's an implicit use of the struct-return parameter in each
-            // copy of the function epilogue, which we count here.
-            value_ir_uses[sret_param] = ValueUseState::Multiple;
-        }
-
-        // Stack of iterators over Values as we do DFS to mark
-        // Multiple-state subtrees. The iterator type is whatever is
-        // returned by `uses` below.
-        let mut stack: SmallVec<[_; 16]> = smallvec![];
-
-        // Find the args for the inst corresponding to the given value.
-        let uses = |value| {
-            trace!(" -> pushing args for {} onto stack", value);
-            if let ValueDef::Result(src_inst, _) = f.dfg.value_def(value) {
-                Some(f.dfg.inst_values(src_inst))
-            } else {
-                None
-            }
-        };
-
-        // Do a DFS through `value_ir_uses` to mark a subtree as
-        // Multiple.
-        for inst in f
-            .layout
-            .blocks()
-            .flat_map(|block| f.layout.block_insts(block))
-        {
-            // If this inst produces multiple values, we must mark all
-            // of its args as Multiple, because otherwise two uses
-            // could come in as Once on our two different results.
-            let force_multiple = f.dfg.inst_results(inst).len() > 1;
-
-            // Iterate over all values used by all instructions, noting an
-            // additional use on each operand.
-            for arg in f.dfg.inst_values(inst) {
-                debug_assert!(f.dfg.value_is_real(arg));
-                let old = value_ir_uses[arg];
-                if force_multiple {
-                    trace!(
-                        "forcing arg {} to Multiple because of multiple results of user inst",
-                        arg
-                    );
-                    value_ir_uses[arg] = ValueUseState::Multiple;
-                } else {
-                    value_ir_uses[arg].inc();
-                }
-                let new = value_ir_uses[arg];
-                trace!("arg {} used, old state {:?}, new {:?}", arg, old, new);
-
-                // On transition to Multiple, do DFS.
-                if old == ValueUseState::Multiple || new != ValueUseState::Multiple {
-                    continue;
-                }
-                if let Some(iter) = uses(arg) {
-                    stack.push(iter);
-                }
-                while let Some(iter) = stack.last_mut() {
-                    if let Some(value) = iter.next() {
-                        debug_assert!(f.dfg.value_is_real(value));
-                        trace!(" -> DFS reaches {}", value);
-                        if value_ir_uses[value] == ValueUseState::Multiple {
-                            // Truncate DFS here: no need to go further,
-                            // as whole subtree must already be Multiple.
-                            // With debug asserts, check one level of
-                            // that invariant at least.
-                            debug_assert!(uses(value).into_iter().flatten().all(|arg| {
-                                debug_assert!(f.dfg.value_is_real(arg));
-                                value_ir_uses[arg] == ValueUseState::Multiple
-                            }));
-                            continue;
-                        }
-                        value_ir_uses[value] = ValueUseState::Multiple;
-                        trace!(" -> became Multiple");
-                        if let Some(iter) = uses(value) {
-                            stack.push(iter);
-                        }
-                    } else {
-                        // Empty iterator, discard.
-                        stack.pop();
-                    }
-                }
-            }
-        }
-
-        value_ir_uses
-    }
-
     fn gen_arg_setup(&mut self) {
         if let Some(entry_bb) = self.f.layout.entry_block() {
             trace!(
@@ -1137,6 +1044,137 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
     }
 }
 
+/// Pre-analysis: compute `value_ir_uses`. See comment on
+/// `ValueUseState` for a description of what this analysis
+/// computes.
+fn compute_use_states(
+    f: &Function,
+    sret_param: Option<Value>,
+) -> SecondaryMap<Value, ValueUseState> {
+    // We perform the analysis without recursion, so we don't
+    // overflow the stack on long chains of ops in the input.
+    //
+    // This is sort of a hybrid of a "shallow use-count" pass and
+    // a DFS. We iterate over all instructions and mark their args
+    // as used. However when we increment a use-count to
+    // "Multiple" we push its args onto the stack and do a DFS,
+    // immediately marking the whole dependency tree as
+    // Multiple. Doing both (shallow use-counting over all insts,
+    // and deep Multiple propagation) lets us trim both
+    // traversals, stopping recursion when a node is already at
+    // the appropriate state.
+    //
+    // In particular, note that the *coarsening* into {Unused,
+    // Once, Multiple} is part of what makes this pass more
+    // efficient than a full indirect-use-counting pass.
+
+    let mut value_ir_uses = SecondaryMap::with_default(ValueUseState::Unused);
+
+    if let Some(sret_param) = sret_param {
+        // There's an implicit use of the struct-return parameter in each
+        // copy of the function epilogue, which we count here.
+        value_ir_uses[sret_param] = ValueUseState::Multiple;
+    }
+
+    // Stack of iterators over Values as we do DFS to mark
+    // Multiple-state subtrees. The iterator type is whatever is
+    // returned by `uses` below.
+    let mut stack: SmallVec<[_; 16]> = smallvec![];
+
+    // Find the args for the inst corresponding to the given value.
+    //
+    // Note that "root" instructions are skipped here. This means that multiple
+    // uses of any result of a multi-result instruction are not considered
+    // multiple uses of the operands of a multi-result instruction. This
+    // requires tight coupling with `get_value_as_source_or_const` above which
+    // is the consumer of the map that this function is producing.
+    let uses = |value| {
+        trace!(" -> pushing args for {} onto stack", value);
+        if let ValueDef::Result(src_inst, _) = f.dfg.value_def(value) {
+            if is_value_use_root(f, src_inst) {
+                None
+            } else {
+                Some(f.dfg.inst_values(src_inst))
+            }
+        } else {
+            None
+        }
+    };
+
+    // Do a DFS through `value_ir_uses` to mark a subtree as
+    // Multiple.
+    for inst in f
+        .layout
+        .blocks()
+        .flat_map(|block| f.layout.block_insts(block))
+    {
+        // Iterate over all values used by all instructions, noting an
+        // additional use on each operand.
+        for arg in f.dfg.inst_values(inst) {
+            debug_assert!(f.dfg.value_is_real(arg));
+            let old = value_ir_uses[arg];
+            value_ir_uses[arg].inc();
+            let new = value_ir_uses[arg];
+            trace!("arg {} used, old state {:?}, new {:?}", arg, old, new);
+
+            // On transition to Multiple, do DFS.
+            if old == ValueUseState::Multiple || new != ValueUseState::Multiple {
+                continue;
+            }
+            if let Some(iter) = uses(arg) {
+                stack.push(iter);
+            }
+            while let Some(iter) = stack.last_mut() {
+                if let Some(value) = iter.next() {
+                    debug_assert!(f.dfg.value_is_real(value));
+                    trace!(" -> DFS reaches {}", value);
+                    if value_ir_uses[value] == ValueUseState::Multiple {
+                        // Truncate DFS here: no need to go further,
+                        // as whole subtree must already be Multiple.
+                        // With debug asserts, check one level of
+                        // that invariant at least.
+                        debug_assert!(uses(value).into_iter().flatten().all(|arg| {
+                            debug_assert!(f.dfg.value_is_real(arg));
+                            value_ir_uses[arg] == ValueUseState::Multiple
+                        }));
+                        continue;
+                    }
+                    value_ir_uses[value] = ValueUseState::Multiple;
+                    trace!(" -> became Multiple");
+                    if let Some(iter) = uses(value) {
+                        stack.push(iter);
+                    }
+                } else {
+                    // Empty iterator, discard.
+                    stack.pop();
+                }
+            }
+        }
+    }
+
+    value_ir_uses
+}
+
+/// Definition of a "root" instruction for the calculation of `ValueUseState`.
+///
+/// This function calculates whether `inst` is considered a "root" for value-use
+/// information. This concept is used to forcibly prevent looking-through the
+/// instruction during `get_value_as_source_or_const` as it additionally
+/// prevents propagating `Multiple`-used results of the `inst` here to the
+/// operands of the instruction.
+///
+/// Currently this is defined as multi-result instructions. That means that
+/// lowerings are never allowed to look through a multi-result instruction to
+/// generate patterns. Note that this isn't possible in ISLE today anyway so
+/// this isn't currently much of a loss.
+///
+/// The main purpose of this function is to prevent the operands of a
+/// multi-result instruction from being forcibly considered `Multiple`-used
+/// regardless of circumstances.
+fn is_value_use_root(f: &Function, inst: Inst) -> bool {
+    f.dfg.inst_results(inst).len() > 1
+}
+
 /// Function-level queries.
 impl<'func, I: VCodeInst> Lower<'func, I> {
     pub fn dfg(&self) -> &DataFlowGraph {
@@ -1267,14 +1305,31 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         val
     }
 
-    /// Like `get_input_as_source_or_const` but with a `Value`.
+    /// Resolves a particular input of an instruction to the `Value` that it is
+    /// represented with.
+    ///
+    /// For more information see [`Lower::get_value_as_source_or_const`].
     pub fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput {
         let val = self.input_as_value(ir_inst, idx);
         self.get_value_as_source_or_const(val)
     }
 
-    /// Resolves a particular input of an instruction to the `Value` that it is
-    /// represented with.
+    /// Resolves a `Value` definition to the source instruction it came from
+    /// plus whether it's a unique-use of that instruction.
+    ///
+    /// This function is the workhorse of pattern-matching in ISLE which enables
+    /// combining multiple instructions together. This is used implicitly in
+    /// patterns such as `(iadd x (iconst y))` where this function is used to
+    /// extract the `(iconst y)` operand.
+    ///
+    /// At its core this function is a wrapper around
+    /// [`DataFlowGraph::value_def`]. This function applies a filter on top of
+    /// that, however, to determine when it is actually safe to "look through"
+    /// the `val` definition here and view the underlying instruction. This
+    /// protects against duplicating side effects, such as loads, for example.
+    ///
+    /// Internally this uses the data computed from `compute_use_states` along
+    /// with other instruction properties to know what to return.
     pub fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput {
         trace!(
             "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}",
@@ -1283,9 +1338,21 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             self.cur_scan_entry_color,
         );
         let inst = match self.f.dfg.value_def(val) {
-            // OK to merge source instruction if (i) we have a source
-            // instruction, and:
-            // - It has no side-effects, OR
+            // OK to merge source instruction if we have a source
+            // instruction, and one of these two conditions hold:
+            //
+            // - It has no side-effects and this instruction is not a "value-use
+            //   root" instruction. Instructions which are considered "roots"
+            //   for value-use calculations do not have accurate information
+            //   known about the `ValueUseState` of their operands. This is
+            //   currently done for multi-result instructions to prevent a use
+            //   of each result from forcing all operands of the multi-result
+            //   instruction to also be `Multiple`. This in turn means that the
+            //   `ValueUseState` for operands of a "root" instruction to be a
+            //   lie if pattern matching were to look through the multi-result
+            //   instruction. As a result the "look through this instruction"
+            //   logic only succeeds if it's not a root instruction.
+            //
             // - It has a side-effect, has one output value, that one
             //   output has only one use, directly or indirectly (so
             //   cannot be duplicated -- see comment on
@@ -1308,10 +1375,21 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
                 let src_side_effect = has_lowering_side_effect(self.f, src_inst);
                 trace!(" -> src inst {}", src_inst);
                 trace!(" -> has lowering side effect: {}", src_side_effect);
-                if !src_side_effect {
-                    // Pure instruction: always possible to
-                    // sink. Let's determine whether we are the only
-                    // user or not.
+                if is_value_use_root(self.f, src_inst) {
+                    // If this instruction is a "root instruction" then it's
+                    // required that we can't look through it to see the
+                    // definition. This means that the `ValueUseState` for the
+                    // operands of this result assume that this instruction is
+                    // generated exactly once which might get violated were we
+                    // to allow looking through it.
+                    trace!(" -> is a root instruction");
+                    InputSourceInst::None
+                } else if !src_side_effect {
+                    // Otherwise if this instruction has no side effects and the
+                    // value is used only once then we can look through it with
+                    // a "unique" tag. A non-unique `Use` can be shown for other
+                    // values ensuring consumers know how it's computed but that
+                    // it's not available to omit.
                     if self.value_ir_uses[val] == ValueUseState::Once {
                         InputSourceInst::UniqueUse(src_inst, result_idx)
                     } else {
@@ -1460,3 +1538,53 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::ValueUseState;
+    use crate::cursor::{Cursor, FuncCursor};
+    use crate::ir::types;
+    use crate::ir::{Function, InstBuilder};
+
+    #[test]
+    fn multi_result_use_once() {
+        let mut func = Function::new();
+        let block0 = func.dfg.make_block();
+        let mut pos = FuncCursor::new(&mut func);
+        pos.insert_block(block0);
+        let v1 = pos.ins().iconst(types::I64, 0);
+        let v2 = pos.ins().iconst(types::I64, 1);
+        let v3 = pos.ins().iconcat(v1, v2);
+        let (v4, v5) = pos.ins().isplit(v3);
+        pos.ins().return_(&[v4, v5]);
+        let func = pos.func;
+
+        let uses = super::compute_use_states(&func, None);
+        assert_eq!(uses[v1], ValueUseState::Once);
+        assert_eq!(uses[v2], ValueUseState::Once);
+        assert_eq!(uses[v3], ValueUseState::Once);
+        assert_eq!(uses[v4], ValueUseState::Once);
+        assert_eq!(uses[v5], ValueUseState::Once);
+    }
+
+    #[test]
+    fn results_used_twice_but_not_operands() {
+        let mut func = Function::new();
+        let block0 = func.dfg.make_block();
+        let mut pos = FuncCursor::new(&mut func);
+        pos.insert_block(block0);
+        let v1 = pos.ins().iconst(types::I64, 0);
+        let v2 = pos.ins().iconst(types::I64, 1);
+        let v3 = pos.ins().iconcat(v1, v2);
+        let (v4, v5) = pos.ins().isplit(v3);
+        pos.ins().return_(&[v4, v4]);
+        let func = pos.func;
+
+        let uses = super::compute_use_states(&func, None);
+        assert_eq!(uses[v1], ValueUseState::Once);
+        assert_eq!(uses[v2], ValueUseState::Once);
+        assert_eq!(uses[v3], ValueUseState::Once);
+        assert_eq!(uses[v4], ValueUseState::Multiple);
+        assert_eq!(uses[v5], ValueUseState::Unused);
+    }
+}
diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs
index 8d3f9983f..125be6a77 100644
--- a/cranelift/frontend/src/frontend.rs
+++ b/cranelift/frontend/src/frontend.rs
@@ -438,6 +438,7 @@ impl<'a> FunctionBuilder<'a> {
     /// Panics if the variable's type is larger than 16 bytes or if this
     /// variable has not been declared yet.
     pub fn declare_var_needs_stack_map(&mut self, var: Variable) {
+        log::trace!("declare_var_needs_stack_map({var:?})");
         let ty = self.func_ctx.types[var];
         assert!(ty != types::INVALID);
         assert!(ty.bytes() <= 16);
@@ -560,6 +561,8 @@ impl<'a> FunctionBuilder<'a> {
     ///
     /// Panics if `val` is larger than 16 bytes.
     pub fn declare_value_needs_stack_map(&mut self, val: Value) {
+        log::trace!("declare_value_needs_stack_map({val:?})");
+
         // We rely on these properties in `insert_safepoint_spills`.
         let size = self.func.dfg.value_type(val).bytes();
         assert!(size <= 16);
diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs
index 71ef60569..7bb017563 100644
--- a/cranelift/wasm/src/code_translator.rs
+++ b/cranelift/wasm/src/code_translator.rs
@@ -180,10 +180,8 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
                     flags.set_alias_region(Some(ir::AliasRegion::Table));
                     builder.ins().load(ty, flags, addr, offset)
                 }
-                GlobalVariable::Custom => environ.translate_custom_global_get(
-                    builder.cursor(),
-                    GlobalIndex::from_u32(*global_index),
-                )?,
+                GlobalVariable::Custom => environ
+                    .translate_custom_global_get(builder, GlobalIndex::from_u32(*global_index))?,
             };
             state.push1(val);
         }
@@ -207,7 +205,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
                 GlobalVariable::Custom => {
                     let val = state.pop1();
                     environ.translate_custom_global_set(
-                        builder.cursor(),
+                        builder,
                         GlobalIndex::from_u32(*global_index),
                         val,
                     )?;
diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs
index 7fea12696..61c272dbc 100644
--- a/cranelift/wasm/src/environ/spec.rs
+++ b/cranelift/wasm/src/environ/spec.rs
@@ -69,18 +69,9 @@ pub trait TargetEnvironment: TypeConvert {
     /// Get the Cranelift reference type to use for the given Wasm reference
     /// type.
     ///
-    /// By default, this returns `R64` for 64-bit architectures and `R32` for
-    /// 32-bit architectures. If you override this, then you should also
-    /// override `FuncEnvironment::{translate_ref_null, translate_ref_is_null}`
-    /// as well.
-    fn reference_type(&self, ty: WasmHeapType) -> ir::Type {
-        let _ = ty;
-        match self.pointer_type() {
-            ir::types::I32 => ir::types::R32,
-            ir::types::I64 => ir::types::R64,
-            _ => panic!("unsupported pointer type"),
-        }
-    }
+    /// Returns a pair of the CLIF reference type to use and a boolean that
+    /// describes whether the value should be included in GC stack maps or not.
+    fn reference_type(&self, ty: WasmHeapType) -> (ir::Type, bool);
 }
 
 /// Environment affecting the translation of a single WebAssembly function.
@@ -95,6 +86,20 @@ pub trait FuncEnvironment: TargetEnvironment {
         signature.params[index].purpose == ir::ArgumentPurpose::Normal
     }
 
+    /// Does the given parameter require inclusion in stack maps?
+    fn param_needs_stack_map(&self, signature: &ir::Signature, index: usize) -> bool;
+
+    /// Does the given result require inclusion in stack maps?
+    fn sig_ref_result_needs_stack_map(&self, sig_ref: ir::SigRef, index: usize) -> bool;
+
+    /// Does the given result require inclusion in stack maps?
+    fn func_ref_result_needs_stack_map(
+        &self,
+        func: &ir::Function,
+        func_ref: ir::FuncRef,
+        index: usize,
+    ) -> bool;
+
     /// Is the given return of the given function a wasm-level parameter, as
     /// opposed to a hidden parameter added for use by the implementation?
     fn is_wasm_return(&self, signature: &ir::Signature, index: usize) -> bool {
@@ -424,39 +429,11 @@ pub trait FuncEnvironment: TargetEnvironment {
     fn translate_elem_drop(&mut self, pos: FuncCursor, seg_index: u32) -> WasmResult<()>;
 
     /// Translate a `ref.null T` WebAssembly instruction.
-    ///
-    /// By default, translates into a null reference type.
-    ///
-    /// Override this if you don't use Cranelift reference types for all Wasm
-    /// reference types (e.g. you use a raw pointer for `funcref`s) or if the
-    /// null sentinel is not a null reference type pointer for your type. If you
-    /// override this method, then you should also override
-    /// `translate_ref_is_null` as well.
-    fn translate_ref_null(
-        &mut self,
-        mut pos: FuncCursor,
-        ty: WasmHeapType,
-    ) -> WasmResult<ir::Value> {
-        let _ = ty;
-        Ok(pos.ins().null(self.reference_type(ty)))
-    }
+    fn translate_ref_null(&mut self, pos: FuncCursor, ty: WasmHeapType) -> WasmResult<ir::Value>;
 
     /// Translate a `ref.is_null` WebAssembly instruction.
-    ///
-    /// By default, assumes that `value` is a Cranelift reference type, and that
-    /// a null Cranelift reference type is the null value for all Wasm reference
-    /// types.
-    ///
-    /// If you override this method, you probably also want to override
-    /// `translate_ref_null` as well.
-    fn translate_ref_is_null(
-        &mut self,
-        mut pos: FuncCursor,
-        value: ir::Value,
-    ) -> WasmResult<ir::Value> {
-        let is_null = pos.ins().is_null(value);
-        Ok(pos.ins().uextend(ir::types::I32, is_null))
-    }
+    fn translate_ref_is_null(&mut self, pos: FuncCursor, value: ir::Value)
+        -> WasmResult<ir::Value>;
 
     /// Translate a `ref.func` WebAssembly instruction.
     fn translate_ref_func(
@@ -469,7 +446,7 @@ pub trait FuncEnvironment: TargetEnvironment {
     /// that is custom.
     fn translate_custom_global_get(
         &mut self,
-        pos: FuncCursor,
+        builder: &mut FunctionBuilder,
         global_index: GlobalIndex,
     ) -> WasmResult<ir::Value>;
 
@@ -477,7 +454,7 @@ pub trait FuncEnvironment: TargetEnvironment {
     /// that is custom.
     fn translate_custom_global_set(
         &mut self,
-        pos: FuncCursor,
+        builder: &mut FunctionBuilder,
         global_index: GlobalIndex,
         val: ir::Value,
     ) -> WasmResult<()>;
diff --git a/cranelift/wasm/src/func_translator.rs b/cranelift/wasm/src/func_translator.rs
index 916dfc60d..d55d7fb38 100644
--- a/cranelift/wasm/src/func_translator.rs
+++ b/cranelift/wasm/src/func_translator.rs
@@ -116,6 +116,10 @@ fn declare_wasm_parameters<FE: FuncEnvironment + ?Sized>(
             builder.declare_var(local, param_type.value_type);
             next_local += 1;
 
+            if environ.param_needs_stack_map(&builder.func.signature, i) {
+                builder.declare_var_needs_stack_map(local);
+            }
+
             let param_value = builder.block_params(entry_block)[i];
             builder.def_var(local, param_value);
         }
@@ -167,45 +171,53 @@ fn declare_locals<FE: FuncEnvironment + ?Sized>(
 ) -> WasmResult<()> {
     // All locals are initialized to 0.
     use wasmparser::ValType::*;
-    let (ty, init) = match wasm_type {
+    let (ty, init, needs_stack_map) = match wasm_type {
         I32 => (
             ir::types::I32,
             Some(builder.ins().iconst(ir::types::I32, 0)),
+            false,
         ),
         I64 => (
             ir::types::I64,
             Some(builder.ins().iconst(ir::types::I64, 0)),
+            false,
         ),
         F32 => (
             ir::types::F32,
             Some(builder.ins().f32const(ir::immediates::Ieee32::with_bits(0))),
+            false,
         ),
         F64 => (
             ir::types::F64,
             Some(builder.ins().f64const(ir::immediates::Ieee64::with_bits(0))),
+            false,
         ),
         V128 => {
             let constant_handle = builder.func.dfg.constants.insert([0; 16].to_vec().into());
             (
                 ir::types::I8X16,
                 Some(builder.ins().vconst(ir::types::I8X16, constant_handle)),
+                false,
             )
         }
         Ref(rt) => {
             let hty = environ.convert_heap_type(rt.heap_type());
-            let ty = environ.reference_type(hty);
+            let (ty, needs_stack_map) = environ.reference_type(hty);
             let init = if rt.is_nullable() {
                 Some(environ.translate_ref_null(builder.cursor(), hty)?)
             } else {
                 None
             };
-            (ty, init)
+            (ty, init, needs_stack_map)
         }
     };
 
     for _ in 0..count {
         let local = Variable::new(*next_local);
         builder.declare_var(local, ty);
+        if needs_stack_map {
+            builder.declare_var_needs_stack_map(local);
+        }
         if let Some(init) = init {
             builder.def_var(local, init);
             builder.set_val_label(init, ValueLabel::new(*next_local));
diff --git a/cranelift/wasm/src/translation_utils.rs b/cranelift/wasm/src/translation_utils.rs
index bfdba3bc4..588cfa5c3 100644
--- a/cranelift/wasm/src/translation_utils.rs
+++ b/cranelift/wasm/src/translation_utils.rs
@@ -64,7 +64,11 @@ pub fn block_with_params<PE: TargetEnvironment + ?Sized>(
             }
             wasmparser::ValType::Ref(rt) => {
                 let hty = environ.convert_heap_type(rt.heap_type());
-                builder.append_block_param(block, environ.reference_type(hty));
+                let (ty, needs_stack_map) = environ.reference_type(hty);
+                let val = builder.append_block_param(block, ty);
+                if needs_stack_map {
+                    builder.declare_value_needs_stack_map(val);
+                }
             }
             wasmparser::ValType::V128 => {
                 builder.append_block_param(block, ir::types::I8X16);
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
index f172d2541..8a7bb6dc9 100644
--- a/crates/cli-flags/src/lib.rs
+++ b/crates/cli-flags/src/lib.rs
@@ -115,6 +115,14 @@ wasmtime_option_group! {
         /// The maximum runtime size of each linear memory in the pooling
         /// allocator, in bytes.
         pub pooling_max_memory_size: Option<usize>,
+
+        /// The maximum table elements for any table defined in a module when
+        /// using the pooling allocator.
+        pub pooling_table_elements: Option<u32>,
+
+        /// The maximum size, in bytes, allocated for a core instance's metadata
+        /// when using the pooling allocator.
+        pub pooling_max_core_instance_size: Option<usize>,
     }
 
     enum Optimize {
@@ -609,6 +617,12 @@ impl CommonOptions {
                     if let Some(limit) = self.opts.pooling_total_tables {
                         cfg.total_tables(limit);
                     }
+                    if let Some(limit) = self.opts.pooling_table_elements {
+                        cfg.table_elements(limit);
+                    }
+                    if let Some(limit) = self.opts.pooling_max_core_instance_size {
+                        cfg.max_core_instance_size(limit);
+                    }
                     match_feature! {
                         ["async" : self.opts.pooling_total_stacks]
                         limit => cfg.total_stacks(limit),
diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml
index 51cf98393..0d7caa626 100644
--- a/crates/cranelift/Cargo.toml
+++ b/crates/cranelift/Cargo.toml
@@ -28,6 +28,7 @@ wasmparser = { workspace = true }
 target-lexicon = { workspace = true }
 gimli = { workspace = true, features = ['std'] }
 object = { workspace = true, features = ['write', 'std'] }
+smallvec = { workspace = true }
 thiserror = { workspace = true }
 cfg-if = { workspace = true }
 wasmtime-versioned-export-macros = { workspace = true }
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 3e048b664..a49d019b7 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -4,14 +4,15 @@ use crate::DEBUG_ASSERT_TRAP_CODE;
 use crate::{array_call_signature, CompiledFunction, ModuleTextBuilder};
 use crate::{builder::LinkOptions, wasm_call_signature, BuiltinFunctionSignatures};
 use anyhow::{Context as _, Result};
+use cranelift_codegen::binemit::CodeOffset;
+use cranelift_codegen::bitset::CompoundBitSet;
 use cranelift_codegen::ir::{self, InstBuilder, MemFlags, UserExternalName, UserFuncName, Value};
 use cranelift_codegen::isa::{
     unwind::{UnwindInfo, UnwindInfoKind},
     OwnedTargetIsa, TargetIsa,
 };
 use cranelift_codegen::print_errors::pretty_error;
-use cranelift_codegen::Context;
-use cranelift_codegen::{CompiledCode, MachStackMap};
+use cranelift_codegen::{CompiledCode, Context};
 use cranelift_entity::PrimaryMap;
 use cranelift_frontend::FunctionBuilder;
 use cranelift_wasm::{DefinedFuncIndex, FuncTranslator, WasmFuncType, WasmValType};
@@ -147,8 +148,14 @@ impl wasmtime_environ::Compiler for Compiler {
             context.func.collect_debug_info();
         }
 
-        let mut func_env =
-            FuncEnvironment::new(isa, translation, types, &self.tunables, self.wmemcheck);
+        let mut func_env = FuncEnvironment::new(
+            isa,
+            translation,
+            types,
+            &self.tunables,
+            self.wmemcheck,
+            wasm_func_ty,
+        );
 
         // The `stack_limit` global value below is the implementation of stack
         // overflow checks in Wasmtime.
@@ -866,7 +873,7 @@ impl FunctionCompiler<'_> {
         }
 
         let stack_maps =
-            mach_stack_maps_to_stack_maps(compiled_code.buffer.take_stack_maps().into_iter());
+            clif_to_env_stack_maps(compiled_code.buffer.take_user_stack_maps().into_iter());
         compiled_function
             .set_sized_stack_slots(std::mem::take(&mut context.func.sized_stack_slots));
         self.compiler.contexts.lock().unwrap().push(self.cx);
@@ -881,23 +888,24 @@ impl FunctionCompiler<'_> {
     }
 }
 
-fn mach_stack_maps_to_stack_maps(
-    mach_stack_maps: impl ExactSizeIterator<Item = MachStackMap>,
+/// Convert from Cranelift's representation of a stack map to Wasmtime's
+/// compiler-agnostic representation.
+fn clif_to_env_stack_maps(
+    clif_stack_maps: impl ExactSizeIterator<Item = (CodeOffset, u32, ir::UserStackMap)>,
 ) -> Vec<StackMapInformation> {
-    // This is converting from Cranelift's representation of a stack map to
-    // Wasmtime's representation. They happen to align today but that may
-    // not always be true in the future.
-    let mut stack_maps = Vec::with_capacity(mach_stack_maps.len());
-    for MachStackMap {
-        offset_end,
-        stack_map,
-        ..
-    } in mach_stack_maps
-    {
-        let mapped_words = stack_map.mapped_words();
-        let stack_map = wasmtime_environ::StackMap::new(mapped_words, stack_map.into_bitset());
+    let mut stack_maps = Vec::with_capacity(clif_stack_maps.len());
+    for (code_offset, mapped_bytes, stack_map) in clif_stack_maps {
+        let mut bitset = CompoundBitSet::new();
+        for (ty, offset) in stack_map.entries() {
+            assert_eq!(ty, ir::types::I32);
+            bitset.insert(usize::try_from(offset).unwrap());
+        }
+        if bitset.is_empty() {
+            continue;
+        }
+        let stack_map = wasmtime_environ::StackMap::new(mapped_bytes, bitset);
         stack_maps.push(StackMapInformation {
-            code_offset: offset_end,
+            code_offset,
             stack_map,
         });
     }
diff --git a/crates/cranelift/src/debug/transform/attr.rs b/crates/cranelift/src/debug/transform/attr.rs
index 8b54c7200..5982100b8 100644
--- a/crates/cranelift/src/debug/transform/attr.rs
+++ b/crates/cranelift/src/debug/transform/attr.rs
@@ -42,7 +42,6 @@ pub(crate) fn clone_die_attributes<'a, R>(
     current_scope_id: write::UnitEntryId,
     subprogram_range_builder: Option<RangeInfoBuilder>,
     scope_ranges: Option<&Vec<(u64, u64)>>,
-    cu_low_pc: u64,
     out_strings: &mut write::StringTable,
     pending_die_refs: &mut PendingUnitRefs,
     pending_di_refs: &mut PendingDebugInfoRefs,
@@ -60,7 +59,7 @@ where
         // FIXME for CU: currently address_transform operate on a single
         // function range, and when CU spans multiple ranges the
         // transformation may be incomplete.
-        RangeInfoBuilder::from(dwarf, unit, entry, cu_low_pc)?
+        RangeInfoBuilder::from(dwarf, unit, entry)?
     };
     range_info.build(addr_tr, out_unit, current_scope_id);
 
@@ -92,7 +91,7 @@ where
                 write::AttributeValue::Address(addr)
             }
             AttributeValue::DebugAddrIndex(i) => {
-                let u = dwarf.debug_addr.get_address(4, unit.addr_base, i)?;
+                let u = dwarf.address(unit, i)?;
                 let addr = addr_tr.translate(u).unwrap_or(write::Address::Constant(0));
                 write::AttributeValue::Address(addr)
             }
@@ -142,7 +141,7 @@ where
             }
             AttributeValue::RangeListsRef(_) | AttributeValue::DebugRngListsIndex(_) => {
                 let r = dwarf.attr_ranges_offset(unit, attr_value)?.unwrap();
-                let range_info = RangeInfoBuilder::from_ranges_ref(dwarf, unit, r, cu_low_pc)?;
+                let range_info = RangeInfoBuilder::from_ranges_ref(dwarf, unit, r)?;
                 let range_list_id = range_info.build_ranges(addr_tr, &mut out_unit.ranges);
                 write::AttributeValue::RangeListRef(range_list_id)
             }
@@ -307,27 +306,3 @@ where
     }
     Ok(())
 }
-
-pub(crate) fn clone_attr_string<R>(
-    attr_value: &AttributeValue<R>,
-    form: gimli::DwForm,
-    unit: &Unit<R, R::Offset>,
-    dwarf: &gimli::Dwarf<R>,
-    out_strings: &mut write::StringTable,
-) -> Result<write::LineString, Error>
-where
-    R: Reader,
-{
-    let content = dwarf
-        .attr_string(unit, attr_value.clone())?
-        .to_string_lossy()?
-        .into_owned();
-    Ok(match form {
-        gimli::DW_FORM_strp => {
-            let id = out_strings.add(content);
-            write::LineString::StringRef(id)
-        }
-        gimli::DW_FORM_string => write::LineString::String(content.into()),
-        _ => bail!("DW_FORM_line_strp or other not supported"),
-    })
-}
diff --git a/crates/cranelift/src/debug/transform/line_program.rs b/crates/cranelift/src/debug/transform/line_program.rs
index 715428aee..e9bea8be4 100644
--- a/crates/cranelift/src/debug/transform/line_program.rs
+++ b/crates/cranelift/src/debug/transform/line_program.rs
@@ -1,11 +1,7 @@
 use super::address_transform::AddressTransform;
-use super::attr::clone_attr_string;
 use super::{Reader, TransformError};
-use anyhow::{Context, Error};
-use gimli::{
-    write, AttributeValue::DebugLineRef, DebugLine, DebugLineOffset, DebugStr,
-    DebuggingInformationEntry, LineEncoding, Unit,
-};
+use anyhow::{bail, Error};
+use gimli::{write, DebugLineOffset, LineEncoding, Unit};
 use wasmtime_environ::DefinedFuncIndex;
 
 #[derive(Debug)]
@@ -41,65 +37,18 @@ enum ReadLineProgramState {
 
 pub(crate) fn clone_line_program<R>(
     dwarf: &gimli::Dwarf<R>,
-    skeleton_dwarf: &gimli::Dwarf<R>,
     unit: &Unit<R, R::Offset>,
-    root: &DebuggingInformationEntry<R>,
-    skeleton_die: Option<&DebuggingInformationEntry<R>>,
+    comp_name: Option<R>,
     addr_tr: &AddressTransform,
     out_encoding: gimli::Encoding,
-    debug_str: &DebugStr<R>,
-    debug_line: &DebugLine<R>,
     out_strings: &mut write::StringTable,
 ) -> Result<(write::LineProgram, DebugLineOffset, Vec<write::FileId>, u64), Error>
 where
     R: Reader,
 {
-    // Where are the "location" attributes
-    let (location_die, location_dwarf) = match skeleton_die {
-        Some(die) => (die, skeleton_dwarf),
-        _ => (root, dwarf),
-    };
-
-    let offset = match location_die.attr_value(gimli::DW_AT_stmt_list)? {
-        Some(DebugLineRef(offset)) => offset,
-        Some(gimli::AttributeValue::SecOffset(offset)) => DebugLineOffset(offset),
-        _ => {
-            return Err(TransformError("Debug line offset is not found").into());
-        }
-    };
-    let comp_dir = location_die.attr_value(gimli::DW_AT_comp_dir)?;
-    let comp_name = root.attr_value(gimli::DW_AT_name)?;
-    let out_comp_dir = match &comp_dir {
-        Some(comp_dir) => Some(clone_attr_string(
-            comp_dir,
-            gimli::DW_FORM_strp,
-            unit,
-            location_dwarf,
-            out_strings,
-        )?),
-        None => None,
-    };
-    let out_comp_name = match comp_name {
-        Some(_) => clone_attr_string(
-            comp_name
-                .as_ref()
-                .context("failed to read DW_AT_name attribute")?,
-            gimli::DW_FORM_strp,
-            unit,
-            dwarf,
-            out_strings,
-        )?,
-        _ => gimli::write::LineString::String("missing DW_AT_name attribute".into()),
-    };
-
-    let program = debug_line.program(
-        offset,
-        unit.header.address_size(),
-        comp_dir.and_then(|val| val.string_value(&debug_str)),
-        comp_name.and_then(|val| val.string_value(&debug_str)),
-    );
-    if let Ok(program) = program {
+    if let Some(program) = unit.line_program.clone() {
         let header = program.header();
+        let offset = header.offset();
         let file_index_base = if header.version() < 5 { 1 } else { 0 };
         assert!(header.version() <= 5, "not supported 6");
         let line_encoding = LineEncoding {
@@ -109,21 +58,32 @@ where
             line_base: header.line_base(),
             line_range: header.line_range(),
         };
+        let out_comp_dir = match header.directory(0) {
+            Some(comp_dir) => clone_line_string(
+                dwarf.attr_string(unit, comp_dir)?,
+                gimli::DW_FORM_strp,
+                out_strings,
+            )?,
+            None => write::LineString::String(Vec::new()),
+        };
+        let out_comp_name = match comp_name {
+            Some(comp_name) => clone_line_string(comp_name, gimli::DW_FORM_strp, out_strings)?,
+            None => write::LineString::String(Vec::new()),
+        };
+
         let mut out_program = write::LineProgram::new(
             out_encoding,
             line_encoding,
-            out_comp_dir.unwrap_or_else(|| write::LineString::String(Vec::new())),
+            out_comp_dir,
             out_comp_name,
             None,
         );
         let mut dirs = Vec::new();
         dirs.push(out_program.default_directory());
         for dir_attr in header.include_directories() {
-            let dir_id = out_program.add_directory(clone_attr_string(
-                dir_attr,
+            let dir_id = out_program.add_directory(clone_line_string(
+                dwarf.attr_string(unit, dir_attr.clone())?,
                 gimli::DW_FORM_string,
-                unit,
-                location_dwarf,
                 out_strings,
             )?);
             dirs.push(dir_id);
@@ -135,11 +95,9 @@ where
             let dir_index = file_entry.directory_index() + directory_index_correction;
             let dir_id = dirs[dir_index as usize];
             let file_id = out_program.add_file(
-                clone_attr_string(
-                    &file_entry.path_name(),
+                clone_line_string(
+                    dwarf.attr_string(unit, file_entry.path_name())?,
                     gimli::DW_FORM_string,
-                    unit,
-                    location_dwarf,
                     out_strings,
                 )?,
                 dir_id,
@@ -285,3 +243,22 @@ where
         Err(TransformError("Valid line program not found").into())
     }
 }
+
+fn clone_line_string<R>(
+    value: R,
+    form: gimli::DwForm,
+    out_strings: &mut write::StringTable,
+) -> Result<write::LineString, Error>
+where
+    R: Reader,
+{
+    let content = value.to_string_lossy()?.into_owned();
+    Ok(match form {
+        gimli::DW_FORM_strp => {
+            let id = out_strings.add(content);
+            write::LineString::StringRef(id)
+        }
+        gimli::DW_FORM_string => write::LineString::String(content.into()),
+        _ => bail!("DW_FORM_line_strp or other not supported"),
+    })
+}
diff --git a/crates/cranelift/src/debug/transform/range_info_builder.rs b/crates/cranelift/src/debug/transform/range_info_builder.rs
index 3f4138757..526c83e61 100644
--- a/crates/cranelift/src/debug/transform/range_info_builder.rs
+++ b/crates/cranelift/src/debug/transform/range_info_builder.rs
@@ -16,14 +16,13 @@ impl RangeInfoBuilder {
         dwarf: &gimli::Dwarf<R>,
         unit: &Unit<R, R::Offset>,
         entry: &DebuggingInformationEntry<R>,
-        cu_low_pc: u64,
     ) -> Result<Self, Error>
     where
         R: Reader,
     {
         if let Some(AttributeValue::RangeListsRef(r)) = entry.attr_value(gimli::DW_AT_ranges)? {
             let r = dwarf.ranges_offset_from_raw(unit, r);
-            return RangeInfoBuilder::from_ranges_ref(dwarf, unit, r, cu_low_pc);
+            return RangeInfoBuilder::from_ranges_ref(dwarf, unit, r);
         };
 
         let low_pc =
@@ -32,7 +31,7 @@ impl RangeInfoBuilder {
             } else if let Some(AttributeValue::DebugAddrIndex(i)) =
                 entry.attr_value(gimli::DW_AT_low_pc)?
             {
-                dwarf.debug_addr.get_address(4, unit.addr_base, i)?
+                dwarf.address(unit, i)?
             } else {
                 return Ok(RangeInfoBuilder::Undefined);
             };
@@ -50,19 +49,11 @@ impl RangeInfoBuilder {
         dwarf: &gimli::Dwarf<R>,
         unit: &Unit<R, R::Offset>,
         ranges: RangeListsOffset,
-        cu_low_pc: u64,
     ) -> Result<Self, Error>
     where
         R: Reader,
     {
-        let unit_encoding = unit.encoding();
-        let mut ranges = dwarf.ranges.ranges(
-            ranges,
-            unit_encoding,
-            cu_low_pc,
-            &dwarf.debug_addr,
-            unit.addr_base,
-        )?;
+        let mut ranges = dwarf.ranges(unit, ranges)?;
         let mut result = Vec::new();
         while let Some(range) = ranges.next()? {
             if range.begin >= range.end {
@@ -83,30 +74,22 @@ impl RangeInfoBuilder {
         unit: &Unit<R, R::Offset>,
         entry: &DebuggingInformationEntry<R>,
         addr_tr: &AddressTransform,
-        cu_low_pc: u64,
     ) -> Result<Self, Error>
     where
         R: Reader,
     {
-        let unit_encoding = unit.encoding();
         let addr =
             if let Some(AttributeValue::Addr(addr)) = entry.attr_value(gimli::DW_AT_low_pc)? {
                 addr
             } else if let Some(AttributeValue::DebugAddrIndex(i)) =
                 entry.attr_value(gimli::DW_AT_low_pc)?
             {
-                dwarf.debug_addr.get_address(4, unit.addr_base, i)?
+                dwarf.address(unit, i)?
             } else if let Some(AttributeValue::RangeListsRef(r)) =
                 entry.attr_value(gimli::DW_AT_ranges)?
             {
                 let r = dwarf.ranges_offset_from_raw(unit, r);
-                let mut ranges = dwarf.ranges.ranges(
-                    r,
-                    unit_encoding,
-                    cu_low_pc,
-                    &dwarf.debug_addr,
-                    unit.addr_base,
-                )?;
+                let mut ranges = dwarf.ranges(unit, r)?;
                 if let Some(range) = ranges.next()? {
                     range.begin
                 } else {
diff --git a/crates/cranelift/src/debug/transform/unit.rs b/crates/cranelift/src/debug/transform/unit.rs
index 9610f286d..8689566db 100644
--- a/crates/cranelift/src/debug/transform/unit.rs
+++ b/crates/cranelift/src/debug/transform/unit.rs
@@ -262,17 +262,6 @@ pub(crate) fn clone_unit(
     let mut pending_di_refs = PendingDebugInfoRefs::new();
     let mut stack = Vec::new();
 
-    let mut skeleton_die = None;
-
-    // Get entries in outer scope to avoid borrowing on short lived temporary.
-    let mut skeleton_entries = skeleton_unit.entries();
-    if split_unit.is_some() {
-        // From the spec, a skeleton unit has no children so we can assume the first, and only, entry is the DW_TAG_skeleton_unit (https://dwarfstd.org/doc/DWARF5.pdf).
-        if let Some(die_tuple) = skeleton_entries.next_dfs()? {
-            skeleton_die = Some(die_tuple.1);
-        }
-    }
-
     let skeleton_dwarf = &compilation.translations[module].debuginfo.dwarf;
     let memory_offset = &compilation.module_memory_offsets[module];
 
@@ -280,20 +269,16 @@ pub(crate) fn clone_unit(
     let dwarf = split_dwarf.unwrap_or(skeleton_dwarf);
     let unit = split_unit.unwrap_or(skeleton_unit);
     let mut entries = unit.entries();
-    let (mut comp_unit, unit_id, file_map, file_index_base, cu_low_pc, wp_die_id, vmctx_die_id) =
+    let (mut comp_unit, unit_id, file_map, file_index_base, wp_die_id, vmctx_die_id) =
         if let Some((depth_delta, entry)) = entries.next_dfs()? {
             assert_eq!(depth_delta, 0);
             let (out_line_program, debug_line_offset, file_map, file_index_base) =
                 clone_line_program(
-                    dwarf,
                     skeleton_dwarf,
-                    unit,
-                    entry,
-                    skeleton_die,
+                    skeleton_unit,
+                    unit.name,
                     addr_tr,
                     out_encoding,
-                    &skeleton_dwarf.debug_str,
-                    &skeleton_dwarf.debug_line,
                     out_strings,
                 )?;
 
@@ -303,7 +288,6 @@ pub(crate) fn clone_unit(
 
                 let root_id = comp_unit.root();
                 die_ref_map.insert(entry.offset(), root_id);
-                let cu_low_pc = unit.low_pc;
 
                 clone_die_attributes(
                     dwarf,
@@ -315,7 +299,6 @@ pub(crate) fn clone_unit(
                     root_id,
                     None,
                     None,
-                    cu_low_pc,
                     out_strings,
                     &mut pending_die_refs,
                     &mut pending_di_refs,
@@ -332,7 +315,6 @@ pub(crate) fn clone_unit(
                     unit_id,
                     file_map,
                     file_index_base,
-                    cu_low_pc,
                     wp_die_id,
                     vmctx_die_id,
                 )
@@ -387,7 +369,7 @@ pub(crate) fn clone_unit(
         current_value_range.update(new_stack_len);
         let range_builder = if entry.tag() == gimli::DW_TAG_subprogram {
             let range_builder =
-                RangeInfoBuilder::from_subprogram_die(dwarf, &unit, entry, addr_tr, cu_low_pc)?;
+                RangeInfoBuilder::from_subprogram_die(dwarf, &unit, entry, addr_tr)?;
             if let RangeInfoBuilder::Function(func) = range_builder {
                 let frame_info = compilation.function_frame_info(module, func);
                 current_value_range.push(new_stack_len, frame_info);
@@ -403,7 +385,7 @@ pub(crate) fn clone_unit(
             let high_pc = entry.attr_value(gimli::DW_AT_high_pc)?;
             let ranges = entry.attr_value(gimli::DW_AT_ranges)?;
             if high_pc.is_some() || ranges.is_some() {
-                let range_builder = RangeInfoBuilder::from(dwarf, &unit, entry, cu_low_pc)?;
+                let range_builder = RangeInfoBuilder::from(dwarf, &unit, entry)?;
                 current_scope_ranges.push(new_stack_len, range_builder.get_ranges(addr_tr));
                 Some(range_builder)
             } else {
@@ -468,7 +450,6 @@ pub(crate) fn clone_unit(
             die_id,
             range_builder,
             current_scope_ranges.top(),
-            cu_low_pc,
             out_strings,
             &mut pending_die_refs,
             &mut pending_di_refs,
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index ad6eda0f8..38ef9c928 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -1,10 +1,10 @@
 use crate::{gc, BuiltinFunctionSignatures};
 use cranelift_codegen::cursor::FuncCursor;
-use cranelift_codegen::ir;
-use cranelift_codegen::ir::condcodes::*;
+use cranelift_codegen::ir::condcodes::IntCC;
 use cranelift_codegen::ir::immediates::{Imm64, Offset32};
 use cranelift_codegen::ir::pcc::Fact;
 use cranelift_codegen::ir::types::*;
+use cranelift_codegen::ir::{self, types};
 use cranelift_codegen::ir::{ArgumentPurpose, Function, InstBuilder, MemFlags};
 use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa};
 use cranelift_entity::packed_option::ReservedValue;
@@ -12,10 +12,11 @@ use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap};
 use cranelift_frontend::FunctionBuilder;
 use cranelift_frontend::Variable;
 use cranelift_wasm::{
-    EngineOrModuleTypeIndex, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, Heap,
-    HeapData, HeapStyle, MemoryIndex, TableData, TableIndex, TableSize, TargetEnvironment,
-    TypeIndex, WasmHeapTopType, WasmHeapType, WasmResult,
+    EngineOrModuleTypeIndex, FuncEnvironment as _, FuncIndex, FuncTranslationState, GlobalIndex,
+    GlobalVariable, Heap, HeapData, HeapStyle, MemoryIndex, TableData, TableIndex, TableSize,
+    TargetEnvironment, TypeIndex, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmResult,
 };
+use smallvec::SmallVec;
 use std::mem;
 use wasmparser::Operator;
 use wasmtime_environ::{
@@ -84,6 +85,8 @@ pub struct FuncEnvironment<'module_environment> {
     isa: &'module_environment (dyn TargetIsa + 'module_environment),
     module: &'module_environment Module,
     types: &'module_environment ModuleTypesBuilder,
+    wasm_func_ty: &'module_environment WasmFuncType,
+    sig_ref_to_ty: SecondaryMap<ir::SigRef, Option<&'module_environment WasmFuncType>>,
 
     #[cfg(feature = "wmemcheck")]
     translation: &'module_environment ModuleTranslation<'module_environment>,
@@ -147,6 +150,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         types: &'module_environment ModuleTypesBuilder,
         tunables: &'module_environment Tunables,
         wmemcheck: bool,
+        wasm_func_ty: &'module_environment WasmFuncType,
     ) -> Self {
         let builtin_functions = BuiltinFunctions::new(isa);
 
@@ -158,6 +162,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             isa,
             module: &translation.module,
             types,
+            wasm_func_ty,
+            sig_ref_to_ty: SecondaryMap::default(),
+
             heaps: PrimaryMap::default(),
             tables: SecondaryMap::default(),
             vmctx: None,
@@ -781,7 +788,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             // For GC-managed references, tables store `Option<VMGcRef>`s.
             ir::types::I32.bytes()
         } else {
-            self.reference_type(table.wasm_ty.heap_type).bytes()
+            self.reference_type(table.wasm_ty.heap_type).0.bytes()
         };
 
         let base_gv = func.create_global_value(ir::GlobalValueData::Load {
@@ -998,32 +1005,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         });
         (pointee, mt)
     }
-
-    fn i31_ref_to_unshifted_value(&self, pos: &mut FuncCursor, i31ref: ir::Value) -> ir::Value {
-        let ref_ty = self.reference_type(WasmHeapType::I31);
-        debug_assert_eq!(pos.func.dfg.value_type(i31ref), ref_ty);
-
-        let is_null = pos.ins().is_null(i31ref);
-        pos.ins().trapnz(is_null, ir::TrapCode::NullI31Ref);
-
-        let val = pos.ins().bitcast(ref_ty.as_int(), MemFlags::new(), i31ref);
-
-        if cfg!(debug_assertions) {
-            let is_i31_ref = pos
-                .ins()
-                .band_imm(val, i64::from(crate::I31_REF_DISCRIMINANT));
-            pos.ins().trapz(
-                is_i31_ref,
-                ir::TrapCode::User(crate::DEBUG_ASSERT_TRAP_CODE),
-            );
-        }
-
-        match ref_ty.bytes() {
-            8 => pos.ins().ireduce(ir::types::I32, val),
-            4 => val,
-            _ => unreachable!(),
-        }
-    }
 }
 
 struct Call<'a, 'func, 'module_env> {
@@ -1433,7 +1414,24 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         if self.tail {
             self.builder.ins().return_call(callee, args)
         } else {
-            self.builder.ins().call(callee, args)
+            let inst = self.builder.ins().call(callee, args);
+            let results: SmallVec<[_; 4]> = self
+                .builder
+                .func
+                .dfg
+                .inst_results(inst)
+                .iter()
+                .copied()
+                .collect();
+            for (i, val) in results.into_iter().enumerate() {
+                if self
+                    .env
+                    .func_ref_result_needs_stack_map(&self.builder.func, callee, i)
+                {
+                    self.builder.declare_value_needs_stack_map(val);
+                }
+            }
+            inst
         }
     }
 
@@ -1448,7 +1446,21 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
                 .ins()
                 .return_call_indirect(sig_ref, func_addr, args)
         } else {
-            self.builder.ins().call_indirect(sig_ref, func_addr, args)
+            let inst = self.builder.ins().call_indirect(sig_ref, func_addr, args);
+            let results: SmallVec<[_; 4]> = self
+                .builder
+                .func
+                .dfg
+                .inst_results(inst)
+                .iter()
+                .copied()
+                .collect();
+            for (i, val) in results.into_iter().enumerate() {
+                if self.env.sig_ref_result_needs_stack_map(sig_ref, i) {
+                    self.builder.declare_value_needs_stack_map(val);
+                }
+            }
+            inst
         }
     }
 }
@@ -1469,8 +1481,13 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm
         self.isa.frontend_config()
     }
 
-    fn reference_type(&self, ty: WasmHeapType) -> ir::Type {
-        crate::reference_type(ty, self.pointer_type())
+    fn reference_type(&self, wasm_ty: WasmHeapType) -> (ir::Type, bool) {
+        let ty = crate::reference_type(wasm_ty, self.pointer_type());
+        let needs_stack_map = match wasm_ty.top() {
+            WasmHeapTopType::Extern | WasmHeapTopType::Any => true,
+            WasmHeapTopType::Func => false,
+        };
+        (ty, needs_stack_map)
     }
 
     fn heap_access_spectre_mitigation(&self) -> bool {
@@ -1493,6 +1510,31 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         index >= 2
     }
 
+    fn param_needs_stack_map(&self, _signature: &ir::Signature, index: usize) -> bool {
+        // Skip the caller and callee vmctx.
+        if index < 2 {
+            return false;
+        }
+
+        self.wasm_func_ty.params()[index - 2].is_vmgcref_type_and_not_i31()
+    }
+
+    fn sig_ref_result_needs_stack_map(&self, sig_ref: ir::SigRef, index: usize) -> bool {
+        let wasm_func_ty = self.sig_ref_to_ty[sig_ref].as_ref().unwrap();
+        wasm_func_ty.returns()[index].is_vmgcref_type_and_not_i31()
+    }
+
+    fn func_ref_result_needs_stack_map(
+        &self,
+        func: &ir::Function,
+        func_ref: ir::FuncRef,
+        index: usize,
+    ) -> bool {
+        let sig_ref = func.dfg.ext_funcs[func_ref].signature;
+        let wasm_func_ty = self.sig_ref_to_ty[sig_ref].as_ref().unwrap();
+        wasm_func_ty.returns()[index].is_vmgcref_type_and_not_i31()
+    }
+
     fn after_locals(&mut self, num_locals: usize) {
         self.fuel_var = Variable::new(num_locals);
         self.epoch_deadline_var = Variable::new(num_locals + 1);
@@ -1677,18 +1719,14 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
     }
 
     fn translate_ref_i31(&mut self, mut pos: FuncCursor, val: ir::Value) -> WasmResult<ir::Value> {
+        debug_assert_eq!(pos.func.dfg.value_type(val), ir::types::I32);
         let shifted = pos.ins().ishl_imm(val, 1);
         let tagged = pos
             .ins()
             .bor_imm(shifted, i64::from(crate::I31_REF_DISCRIMINANT));
-        let ref_ty = self.reference_type(WasmHeapType::I31);
-        let extended = if ref_ty.bytes() > 4 {
-            pos.ins().uextend(ref_ty.as_int(), tagged)
-        } else {
-            tagged
-        };
-        let i31ref = pos.ins().bitcast(ref_ty, MemFlags::new(), extended);
-        Ok(i31ref)
+        let (ref_ty, _needs_stack_map) = self.reference_type(WasmHeapType::I31);
+        debug_assert_eq!(ref_ty, ir::types::I32);
+        Ok(tagged)
     }
 
     fn translate_i31_get_s(
@@ -1696,9 +1734,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         mut pos: FuncCursor,
         i31ref: ir::Value,
     ) -> WasmResult<ir::Value> {
-        let val = self.i31_ref_to_unshifted_value(&mut pos, i31ref);
-        let shifted = pos.ins().sshr_imm(val, 1);
-        Ok(shifted)
+        Ok(pos.ins().sshr_imm(i31ref, 1))
     }
 
     fn translate_i31_get_u(
@@ -1706,9 +1742,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         mut pos: FuncCursor,
         i31ref: ir::Value,
     ) -> WasmResult<ir::Value> {
-        let val = self.i31_ref_to_unshifted_value(&mut pos, i31ref);
-        let shifted = pos.ins().ushr_imm(val, 1);
-        Ok(shifted)
+        Ok(pos.ins().ushr_imm(i31ref, 1))
     }
 
     fn translate_ref_null(
@@ -1718,9 +1752,8 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
     ) -> WasmResult<ir::Value> {
         Ok(match ht.top() {
             WasmHeapTopType::Func => pos.ins().iconst(self.pointer_type(), 0),
-            WasmHeapTopType::Any | WasmHeapTopType::Extern => {
-                pos.ins().null(self.reference_type(ht))
-            }
+            // NB: null GC references don't need to be in stack maps.
+            WasmHeapTopType::Any | WasmHeapTopType::Extern => pos.ins().iconst(types::I32, 0),
         })
     }
 
@@ -1729,18 +1762,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         mut pos: cranelift_codegen::cursor::FuncCursor,
         value: ir::Value,
     ) -> WasmResult<ir::Value> {
-        let bool_is_null = match pos.func.dfg.value_type(value) {
-            // `externref`
-            ty if ty.is_ref() => pos.ins().is_null(value),
-            // `funcref`
-            ty if ty == self.pointer_type() => {
-                pos.ins()
-                    .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0)
-            }
-            _ => unreachable!(),
-        };
-
-        Ok(pos.ins().uextend(ir::types::I32, bool_is_null))
+        let byte_is_null =
+            pos.ins()
+                .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0);
+        Ok(pos.ins().uextend(ir::types::I32, byte_is_null))
     }
 
     fn translate_ref_func(
@@ -1758,7 +1783,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
     fn translate_custom_global_get(
         &mut self,
-        mut pos: cranelift_codegen::cursor::FuncCursor<'_>,
+        builder: &mut FunctionBuilder,
         index: cranelift_wasm::GlobalIndex,
     ) -> WasmResult<ir::Value> {
         let ty = self.module.globals[index].wasm_ty;
@@ -1769,19 +1794,23 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
         // TODO: use `GcCompiler::translate_read_gc_reference` for GC-reference
         // globals instead of a libcall.
-        let libcall = gc::gc_ref_global_get_builtin(ty, self, &mut pos.func)?;
+        let libcall = gc::gc_ref_global_get_builtin(ty, self, &mut builder.func)?;
 
-        let vmctx = self.vmctx_val(&mut pos);
+        let vmctx = self.vmctx_val(&mut builder.cursor());
 
-        let global_index_arg = pos.ins().iconst(I32, index.as_u32() as i64);
-        let call_inst = pos.ins().call(libcall, &[vmctx, global_index_arg]);
+        let global_index_arg = builder.ins().iconst(I32, index.as_u32() as i64);
+        let call_inst = builder.ins().call(libcall, &[vmctx, global_index_arg]);
 
-        Ok(pos.func.dfg.first_result(call_inst))
+        let val = builder.func.dfg.first_result(call_inst);
+        if ty.is_vmgcref_type_and_not_i31() {
+            builder.declare_value_needs_stack_map(val);
+        }
+        Ok(val)
     }
 
     fn translate_custom_global_set(
         &mut self,
-        mut pos: cranelift_codegen::cursor::FuncCursor<'_>,
+        builder: &mut FunctionBuilder,
         index: cranelift_wasm::GlobalIndex,
         value: ir::Value,
     ) -> WasmResult<()> {
@@ -1793,12 +1822,14 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
         // TODO: use `GcCompiler::translate_write_gc_reference` for GC-reference
         // globals instead of a libcall.
-        let libcall = gc::gc_ref_global_set_builtin(ty, self, &mut pos.func)?;
+        let libcall = gc::gc_ref_global_set_builtin(ty, self, &mut builder.func)?;
 
-        let vmctx = self.vmctx_val(&mut pos);
+        let vmctx = self.vmctx_val(&mut builder.cursor());
 
-        let global_index_arg = pos.ins().iconst(I32, index.as_u32() as i64);
-        pos.ins().call(libcall, &[vmctx, global_index_arg, value]);
+        let global_index_arg = builder.ins().iconst(I32, index.as_u32() as i64);
+        builder
+            .ins()
+            .call(libcall, &[vmctx, global_index_arg, value]);
 
         Ok(())
     }
@@ -2077,10 +2108,12 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         func: &mut ir::Function,
         index: TypeIndex,
     ) -> WasmResult<ir::SigRef> {
-        let index = self.module.types[index];
-        let sig =
-            crate::wasm_call_signature(self.isa, self.types[index].unwrap_func(), &self.tunables);
-        Ok(func.import_signature(sig))
+        let interned_index = self.module.types[index];
+        let wasm_func_ty = self.types[interned_index].unwrap_func();
+        let sig = crate::wasm_call_signature(self.isa, wasm_func_ty, &self.tunables);
+        let sig_ref = func.import_signature(sig);
+        self.sig_ref_to_ty[sig_ref] = Some(wasm_func_ty);
+        Ok(sig_ref)
     }
 
     fn make_direct_func(
@@ -2089,9 +2122,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         index: FuncIndex,
     ) -> WasmResult<ir::FuncRef> {
         let sig = self.module.functions[index].signature;
-        let sig =
-            crate::wasm_call_signature(self.isa, self.types[sig].unwrap_func(), &self.tunables);
+        let wasm_func_ty = self.types[sig].unwrap_func();
+        let sig = crate::wasm_call_signature(self.isa, wasm_func_ty, &self.tunables);
         let signature = func.import_signature(sig);
+        self.sig_ref_to_ty[signature] = Some(wasm_func_ty);
         let name =
             ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName {
                 namespace: crate::NS_WASM_FUNC,
@@ -2101,19 +2135,19 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
             name,
             signature,
 
-            // The value of this flag determines the codegen for calls to this
-            // function. If this flag is `false` then absolute relocations will
+            // the value of this flag determines the codegen for calls to this
+            // function. if this flag is `false` then absolute relocations will
             // be generated for references to the function, which requires
-            // load-time relocation resolution. If this flag is set to `true`
+            // load-time relocation resolution. if this flag is set to `true`
             // then relative relocations are emitted which can be resolved at
             // object-link-time, just after all functions are compiled.
             //
-            // This flag is set to `true` for functions defined in the object
+            // this flag is set to `true` for functions defined in the object
             // we'll be defining in this compilation unit, or everything local
-            // to the wasm module. This means that between functions in a wasm
-            // module there's relative calls encoded. All calls external to a
+            // to the wasm module. this means that between functions in a wasm
+            // module there's relative calls encoded. all calls external to a
             // wasm module (e.g. imports or libcalls) are either encoded through
-            // the `VMContext` as relative jumps (hence no relocations) or
+            // the `vmcontext` as relative jumps (hence no relocations) or
             // they're libcalls with absolute relocations.
             colocated: self.module.defined_func_index(index).is_some(),
         }))
diff --git a/crates/cranelift/src/gc.rs b/crates/cranelift/src/gc.rs
index 698aaa98f..5df95e4fc 100644
--- a/crates/cranelift/src/gc.rs
+++ b/crates/cranelift/src/gc.rs
@@ -25,14 +25,11 @@ pub fn gc_compiler(func_env: &FuncEnvironment<'_>) -> Box<dyn GcCompiler> {
     imp::gc_compiler(func_env)
 }
 
-/// Load a `*mut VMGcRef` into a virtual register of the function environment's
-/// reference type, aka `r{32,64}`, without any GC barriers.
+/// Load a `*mut VMGcRef` into a virtual register, without any GC barriers.
 ///
-/// Note that a `VMGcRef` is always 4-bytes large, even when targeting
-/// 64-bit architectures.
-///
-/// Because Cranelift doesn't support using `r32` with 64-bit targets, this
-/// means that the loaded value may need to be extended.
+/// The resulting value is an instance of the function environment's type for
+/// GC-managed references, aka `i32`. Note that a `VMGcRef` is always 4-bytes
+/// large, even when targeting 64-bit architectures.
 pub fn unbarriered_load_gc_ref(
     func_env: &FuncEnvironment<'_>,
     builder: &mut FunctionBuilder<'_>,
@@ -47,15 +44,9 @@ pub fn unbarriered_load_gc_ref(
 ///
 /// `dst` is a `*mut VMGcRef`.
 ///
-/// `gc_ref` is an instance of the function environment's reference type, aka
-/// `r{32,64}`.
-///
-/// Note that a `VMGcRef` is always 4-bytes large, even when targeting
-/// 64-bit architectures.
-///
-/// Because Cranelift doesn't support using `r32` with 64-bit targets, we keep
-/// `VMGcRef`s on the Wasm stack as `r64`s with the top half unset, and this
-/// means that `value` may need to be truncated.
+/// `gc_ref` is an instance of the function environment's type for GC-managed
+/// references, aka `i32`. Note that a `VMGcRef` is always 4-bytes large, even
+/// when targeting 64-bit architectures.
 pub fn unbarriered_store_gc_ref(
     func_env: &FuncEnvironment<'_>,
     builder: &mut FunctionBuilder<'_>,
@@ -144,7 +135,7 @@ pub trait GcCompiler {
     /// * `flags`: The memory flags that should be used when accessing `src`.
     ///
     /// This method should return the cloned GC reference (an instance of
-    /// `VMGcRef`) of type `r64`.
+    /// `VMGcRef`) of type `i32`.
     fn translate_read_gc_reference(
         &mut self,
         func_env: &mut FuncEnvironment<'_>,
@@ -183,7 +174,7 @@ pub trait GcCompiler {
     ///   itself or a `*mut VMGcHeader`!
     ///
     /// * `new_val`: The new value that should be written into `dst`. This is a
-    ///   `VMGcRef` of Cranelift type `r64`; not a `*mut VMGcRef`.
+    ///   `VMGcRef` of Cranelift type `i32`; not a `*mut VMGcRef`.
     ///
     /// * `flags`: The memory flags that should be used when accessing `dst`.
     fn translate_write_gc_reference(
diff --git a/crates/cranelift/src/gc/enabled.rs b/crates/cranelift/src/gc/enabled.rs
index e610e57a2..81d0abb67 100644
--- a/crates/cranelift/src/gc/enabled.rs
+++ b/crates/cranelift/src/gc/enabled.rs
@@ -1,6 +1,6 @@
 use super::GcCompiler;
 use crate::func_environ::FuncEnvironment;
-use cranelift_codegen::ir::{self, InstBuilder};
+use cranelift_codegen::ir::{self, condcodes::IntCC, InstBuilder};
 use cranelift_frontend::FunctionBuilder;
 use cranelift_wasm::{
     TargetEnvironment, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmValType,
@@ -19,15 +19,13 @@ pub fn unbarriered_load_gc_ref(
     ptr_to_gc_ref: ir::Value,
     flags: ir::MemFlags,
 ) -> WasmResult<ir::Value> {
-    let ref_ty = func_env.reference_type(ty);
+    debug_assert!({
+        let (_, needs_stack_map) = func_env.reference_type(ty);
+        needs_stack_map
+    });
     let gc_ref = builder.ins().load(ir::types::I32, flags, ptr_to_gc_ref, 0);
-    let gc_ref = if ref_ty.bytes() > ir::types::I32.bytes() {
-        builder.ins().uextend(ref_ty.as_int(), gc_ref)
-    } else {
-        assert_eq!(ref_ty.bytes(), ir::types::I32.bytes());
-        gc_ref
-    };
-    Ok(builder.ins().bitcast(ref_ty, ir::MemFlags::new(), gc_ref))
+    builder.declare_value_needs_stack_map(gc_ref);
+    Ok(gc_ref)
 }
 
 pub fn unbarriered_store_gc_ref(
@@ -38,16 +36,10 @@ pub fn unbarriered_store_gc_ref(
     gc_ref: ir::Value,
     flags: ir::MemFlags,
 ) -> WasmResult<()> {
-    let ref_ty = func_env.reference_type(ty);
-    let gc_ref = builder
-        .ins()
-        .bitcast(ref_ty.as_int(), ir::MemFlags::new(), gc_ref);
-    let gc_ref = if ref_ty.bytes() > ir::types::I32.bytes() {
-        builder.ins().ireduce(ir::types::I32, gc_ref)
-    } else {
-        assert_eq!(ref_ty.bytes(), ir::types::I32.bytes());
-        gc_ref
-    };
+    debug_assert!({
+        let (_, needs_stack_map) = func_env.reference_type(ty);
+        needs_stack_map
+    });
     builder.ins().store(flags, gc_ref, dst, 0);
     Ok(())
 }
@@ -121,9 +113,12 @@ impl FuncEnvironment<'_> {
         let pointer_type = self.pointer_type();
         let (base, bound) = self.get_gc_heap_base_bound(builder);
 
-        let index = builder
-            .ins()
-            .bitcast(pointer_type, ir::MemFlags::new(), gc_ref);
+        debug_assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32);
+        let index = match pointer_type {
+            ir::types::I32 => gc_ref,
+            ir::types::I64 => builder.ins().uextend(ir::types::I64, gc_ref),
+            _ => unreachable!(),
+        };
 
         let offset = builder
             .ins()
@@ -178,36 +173,29 @@ impl FuncEnvironment<'_> {
             WasmHeapTopType::Extern | WasmHeapTopType::Func => false,
         };
 
-        let ptr_ty = self.pointer_type();
-
         match (ty.nullable, might_be_i31) {
             // This GC reference statically cannot be null nor an i31. (Let
             // Cranelift's optimizer const-propagate this value and erase any
             // unnecessary control flow resulting from branching on this value.)
-            (false, false) => builder.ins().iconst(ptr_ty, 0),
+            (false, false) => builder.ins().iconst(ir::types::I32, 0),
 
             // This GC reference is always non-null, but might be an i31.
-            (false, true) => {
-                // TODO: support bitwise operations directly on `r{32,64}` types
-                // so we don't need this bitcast.
-                let raw = builder.ins().bitcast(ptr_ty, ir::MemFlags::new(), gc_ref);
-                builder.ins().band_imm(raw, I31_DISCRIMINANT as i64)
-            }
+            (false, true) => builder.ins().band_imm(gc_ref, I31_DISCRIMINANT as i64),
 
             // This GC reference might be null, but can never be an i31.
-            (true, false) => builder.ins().is_null(gc_ref),
+            (true, false) => builder.ins().icmp_imm(IntCC::Equal, gc_ref, 0),
 
             // Fully general case: this GC reference could be either null or an
             // i31.
             (true, true) => {
-                // TODO: support bitwise operations directly on `r{32,64}` types
-                // so we don't need this bitcast.
-                let raw = builder.ins().bitcast(ptr_ty, ir::MemFlags::new(), gc_ref);
                 // Mask for checking whether any bits are set, other than the
                 // `i31ref` discriminant, which should not be set. This folds
                 // the null and i31ref checks together into a single `band`.
-                let mask = builder.ins().iconst(ptr_ty, NON_NULL_NON_I31_MASK as i64);
-                let is_non_null_and_non_i31 = builder.ins().band(raw, mask);
+                let mask = builder.ins().iconst(
+                    ir::types::I32,
+                    (NON_NULL_NON_I31_MASK & u32::MAX as u64) as i64,
+                );
+                let is_non_null_and_non_i31 = builder.ins().band(gc_ref, mask);
                 builder
                     .ins()
                     .icmp_imm(ir::condcodes::IntCC::Equal, is_non_null_and_non_i31, 0)
@@ -318,14 +306,14 @@ impl GcCompiler for DrcCompiler {
     ) -> WasmResult<ir::Value> {
         assert!(ty.is_vmgcref_type_and_not_i31());
 
-        let reference_type = func_env.reference_type(ty.heap_type);
+        let (reference_type, needs_stack_map) = func_env.reference_type(ty.heap_type);
+        debug_assert!(needs_stack_map);
 
-        // Special case for references to uninhabited bottom types: either
-        // the reference must either be nullable and we can just eagerly
-        // return null, or we are in dynamically unreachable code and should
-        // just trap.
+        // Special case for references to uninhabited bottom types: the
+        // reference must either be nullable and we can just eagerly return
+        // null, or we are in dynamically unreachable code and should just trap.
         if let WasmHeapType::None = ty.heap_type {
-            let null = builder.ins().null(reference_type);
+            let null = builder.ins().iconst(reference_type, 0);
             if !ty.nullable {
                 // NB: Don't use an unconditional trap instruction, since that
                 // is a block terminator, and we still need to integrate with
@@ -461,7 +449,8 @@ impl GcCompiler for DrcCompiler {
     ) -> WasmResult<()> {
         assert!(ty.is_vmgcref_type_and_not_i31());
 
-        let ref_ty = func_env.reference_type(ty.heap_type);
+        let (ref_ty, needs_stack_map) = func_env.reference_type(ty.heap_type);
+        debug_assert!(needs_stack_map);
 
         // Special case for references to uninhabited bottom types: either the
         // reference must either be nullable and we can just eagerly store null
@@ -624,13 +613,7 @@ impl GcCompiler for DrcCompiler {
         builder.seal_block(drop_old_val_block);
         let drop_gc_ref_libcall = func_env.builtin_functions.drop_gc_ref(builder.func);
         let vmctx = func_env.vmctx_val(&mut builder.cursor());
-        let old_val_pointer =
-            builder
-                .ins()
-                .bitcast(func_env.pointer_type(), ir::MemFlags::new(), old_val);
-        builder
-            .ins()
-            .call(drop_gc_ref_libcall, &[vmctx, old_val_pointer]);
+        builder.ins().call(drop_gc_ref_libcall, &[vmctx, old_val]);
         builder.ins().jump(continue_block, &[]);
 
         // Block to store the new ref count back to `old_val` for when
diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs
index 65f243c18..edc4ab687 100644
--- a/crates/cranelift/src/lib.rs
+++ b/crates/cranelift/src/lib.rs
@@ -190,11 +190,7 @@ fn wasm_call_signature(
 fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type {
     match wasm_ht.top() {
         WasmHeapTopType::Func => pointer_type,
-        WasmHeapTopType::Any | WasmHeapTopType::Extern => match pointer_type {
-            ir::types::I32 => ir::types::R32,
-            ir::types::I64 => ir::types::R64,
-            _ => panic!("unsupported pointer type"),
-        },
+        WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32,
     }
 }
 
@@ -354,13 +350,10 @@ impl BuiltinFunctionSignatures {
     fn new(isa: &dyn TargetIsa) -> Self {
         Self {
             pointer_type: isa.pointer_type(),
-            #[cfg(feature = "gc")]
-            reference_type: match isa.pointer_type() {
-                ir::types::I32 => ir::types::R32,
-                ir::types::I64 => ir::types::R64,
-                _ => panic!(),
-            },
             call_conv: CallConv::triple_default(isa.triple()),
+
+            #[cfg(feature = "gc")]
+            reference_type: ir::types::I32,
         }
     }
 
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index 422172e15..e9411ac9f 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -67,11 +67,11 @@ macro_rules! foreach_builtin_function {
             update_mem_size(vmctx: vmctx, num_bytes: i32);
 
             // Drop a non-stack GC reference (eg an overwritten table entry)
-            // once it will no longer be used again. (Note: `val` is not a
+            // once it will no longer be used again. (Note: `val` is not of type
             // `reference` because it needn't appear in any stack maps, as it
             // must not be live after this call.)
             #[cfg(feature = "gc")]
-            drop_gc_ref(vmctx: vmctx, val: pointer);
+            drop_gc_ref(vmctx: vmctx, val: i32);
 
             // Do a GC, treating the optional `root` as a GC root and returning
             // the updated `root` (so that, in the case of moving collectors,
diff --git a/crates/environ/src/stack_map.rs b/crates/environ/src/stack_map.rs
index 023d6b0ec..2737e2642 100644
--- a/crates/environ/src/stack_map.rs
+++ b/crates/environ/src/stack_map.rs
@@ -9,23 +9,52 @@ use serde_derive::{Deserialize, Serialize};
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StackMap {
     bits: CompoundBitSet,
-    mapped_words: u32,
+    frame_size: u32,
 }
 
 impl StackMap {
     /// Creates a new `StackMap`, typically from a preexisting
     /// `binemit::StackMap`.
-    pub fn new(mapped_words: u32, bits: CompoundBitSet) -> StackMap {
-        StackMap { bits, mapped_words }
+    pub fn new(frame_size: u32, bits: CompoundBitSet) -> StackMap {
+        StackMap { bits, frame_size }
     }
 
-    /// Returns a specified bit.
-    pub fn get_bit(&self, bit_index: usize) -> bool {
-        self.bits.contains(bit_index)
+    /// Returns the byte size of this stack map's frame.
+    pub fn frame_size(&self) -> u32 {
+        self.frame_size
     }
 
-    /// Returns the number of words represented by this stack map.
-    pub fn mapped_words(&self) -> u32 {
-        self.mapped_words
+    /// Given a frame pointer, get the stack pointer.
+    ///
+    /// # Safety
+    ///
+    /// The `fp` must be the frame pointer at the code offset that this stack
+    /// map is associated with.
+    pub unsafe fn sp(&self, fp: *mut usize) -> *mut usize {
+        let frame_size = usize::try_from(self.frame_size).unwrap();
+        fp.byte_sub(frame_size)
+    }
+
+    /// Given the stack pointer, get a reference to each live GC reference in
+    /// the stack frame.
+    ///
+    /// # Safety
+    ///
+    /// The `sp` must be the stack pointer at the code offset that this stack
+    /// map is associated with.
+    pub unsafe fn live_gc_refs(&self, sp: *mut usize) -> impl Iterator<Item = *mut u32> + '_ {
+        self.bits.iter().map(move |i| {
+            log::trace!("Live GC ref in frame at frame offset {:#x}", i);
+            let ptr_to_gc_ref = sp.byte_add(i);
+
+            // Assert that the pointer is inside this stack map's frame.
+            assert!({
+                let delta = ptr_to_gc_ref as usize - sp as usize;
+                let frame_size = usize::try_from(self.frame_size).unwrap();
+                delta < frame_size
+            });
+
+            ptr_to_gc_ref.cast::<u32>()
+        })
     }
 }
diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs
index 14251cdd1..d28ebfd3c 100644
--- a/crates/fuzzing/src/oracles.rs
+++ b/crates/fuzzing/src/oracles.rs
@@ -744,6 +744,11 @@ pub fn table_ops(
 
         {
             let mut scope = RootScope::new(&mut store);
+
+            log::info!(
+                "table_ops: begin allocating {} externref arguments",
+                ops.num_globals
+            );
             let args: Vec<_> = (0..ops.num_params)
                 .map(|_| {
                     Ok(Val::ExternRef(Some(ExternRef::new(
@@ -752,11 +757,16 @@ pub fn table_ops(
                     )?)))
                 })
                 .collect::<Result<_>>()?;
+            log::info!(
+                "table_ops: end allocating {} externref arguments",
+                ops.num_globals
+            );
 
             // The generated function should always return a trap. The only two
             // valid traps are table-out-of-bounds which happens through `table.get`
             // and `table.set` generated or an out-of-fuel trap. Otherwise any other
             // error is unexpected and should fail fuzzing.
+            log::info!("table_ops: calling into Wasm `run` function");
             let trap = run
                 .call(&mut scope, &args, &mut [])
                 .unwrap_err()
diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs
index 8ec0ccdc6..1b396001d 100644
--- a/crates/wasmtime/src/runtime/externals/global.rs
+++ b/crates/wasmtime/src/runtime/externals/global.rs
@@ -224,7 +224,7 @@ impl Global {
                 let gc_ref = NonNull::from(gc_ref);
                 let gc_ref = SendSyncPtr::new(gc_ref);
                 unsafe {
-                    gc_roots_list.add_root(gc_ref);
+                    gc_roots_list.add_root(gc_ref, "Wasm global");
                 }
             }
         }
diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs
index 1decf6b5c..bea26c42a 100644
--- a/crates/wasmtime/src/runtime/externals/table.rs
+++ b/crates/wasmtime/src/runtime/externals/table.rs
@@ -388,7 +388,7 @@ impl Table {
                 let gc_ref = NonNull::from(gc_ref);
                 let gc_ref = SendSyncPtr::new(gc_ref);
                 unsafe {
-                    gc_roots_list.add_root(gc_ref);
+                    gc_roots_list.add_root(gc_ref, "Wasm table element");
                 }
             }
         }
diff --git a/crates/wasmtime/src/runtime/gc/enabled/i31.rs b/crates/wasmtime/src/runtime/gc/enabled/i31.rs
index 488e3581d..6b739513d 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/i31.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/i31.rs
@@ -241,22 +241,14 @@ unsafe impl WasmTy for I31 {
     }
 
     fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
-        let r64 = VMGcRef::from_i31(self.into()).into_r64();
-        let anyref = u32::try_from(r64).unwrap();
-        ptr.write(ValRaw::anyref(anyref));
+        let gc_ref = VMGcRef::from_i31(self.into()).as_raw_u32();
+        ptr.write(ValRaw::anyref(gc_ref));
         Ok(())
     }
 
     unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
         let raw = ptr.get_anyref();
-        if cfg!(debug_assertions) {
-            let gc_ref = VMGcRef::from_raw_u32(raw).unwrap();
-            assert!(gc_ref.is_i31());
-        }
-        let r64 = u64::from(raw);
-        let gc_ref = VMGcRef::from_r64(r64)
-            .expect("valid r64")
-            .expect("non-null");
+        let gc_ref = VMGcRef::from_raw_u32(raw).expect("non-null");
         gc_ref.unwrap_i31().into()
     }
 }
@@ -292,8 +284,8 @@ unsafe impl WasmTy for Option<I31> {
     }
 
     unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
-        let r64 = u64::from(ptr.get_anyref());
-        let gc_ref = VMGcRef::from_r64(r64).expect("valid r64")?;
+        let raw = ptr.get_anyref();
+        let gc_ref = VMGcRef::from_raw_u32(raw)?;
         Some(I31(gc_ref.unwrap_i31()))
     }
 }
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index eb798602d..bcdee4159 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -381,7 +381,7 @@ impl RootSet {
         log::trace!("Begin trace user LIFO roots");
         for root in &mut self.lifo_roots {
             unsafe {
-                gc_roots_list.add_root((&mut root.gc_ref).into());
+                gc_roots_list.add_root((&mut root.gc_ref).into(), "user LIFO root");
             }
         }
         log::trace!("End trace user LIFO roots");
@@ -389,7 +389,7 @@ impl RootSet {
         log::trace!("Begin trace user manual roots");
         for (_id, root) in self.manually_rooted.iter_mut() {
             unsafe {
-                gc_roots_list.add_root(root.into());
+                gc_roots_list.add_root(root.into(), "user manual root");
             }
         }
         log::trace!("End trace user manual roots");
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index e3dbe894c..0efefef4e 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1701,11 +1701,12 @@ impl StoreOpaque {
             let pc = frame.pc();
             debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames");
 
-            let fp = frame.fp();
+            let fp = frame.fp() as *mut usize;
             debug_assert!(
-                fp != 0,
+                !fp.is_null(),
                 "we should always get a valid frame pointer for Wasm frames"
             );
+
             let module_info = self
                 .modules()
                 .lookup(pc)
@@ -1719,31 +1720,16 @@ impl StoreOpaque {
                 }
             };
             log::trace!(
-                "We have a stack map that maps {} words in this Wasm frame",
-                stack_map.mapped_words()
+                "We have a stack map that maps {} bytes in this Wasm frame",
+                stack_map.frame_size()
             );
 
-            let sp = fp - stack_map.mapped_words() as usize * mem::size_of::<usize>();
-
-            for i in 0..(stack_map.mapped_words() as usize) {
-                // Stack maps have one bit per word in the frame, and the
-                // zero^th bit is the *lowest* addressed word in the frame,
-                // i.e. the closest to the SP. So to get the `i`^th word in
-                // this frame, we add `i * sizeof(word)` to the SP.
-                let stack_slot = sp + i * mem::size_of::<usize>();
-                let stack_slot = stack_slot as *mut u64;
-
-                if !stack_map.get_bit(i) {
-                    log::trace!("Stack slot @ {stack_slot:p} does not contain gc_refs");
-                    continue;
-                }
-
-                let gc_ref = unsafe { core::ptr::read(stack_slot) };
-                log::trace!("Stack slot @ {stack_slot:p} = {gc_ref:#x}");
-
-                let gc_ref = VMGcRef::from_r64(gc_ref)
-                    .expect("we should never use the high 32 bits of an r64");
+            let sp = unsafe { stack_map.sp(fp) };
+            for stack_slot in unsafe { stack_map.live_gc_refs(sp) } {
+                let raw: u32 = unsafe { core::ptr::read(stack_slot) };
+                log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}");
 
+                let gc_ref = VMGcRef::from_raw_u32(raw);
                 if gc_ref.is_some() {
                     unsafe {
                         gc_roots_list.add_wasm_stack_root(SendSyncPtr::new(
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index e3d85f646..d714c1337 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -128,6 +128,7 @@ impl GcStore {
     /// Hook to call whenever a GC reference is about to be exposed to Wasm.
     pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) {
         if !gc_ref.is_i31() {
+            log::trace!("exposing GC ref to Wasm: {gc_ref:p}");
             self.gc_heap.expose_gc_ref_to_wasm(gc_ref);
         }
     }
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index c52478de2..9b7086ea3 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -369,7 +369,7 @@ impl DrcHeap {
                 gc_ref.is_i31() || activations_table_set.contains(&gc_ref),
                 "every on-stack gc_ref inside a Wasm frame should \
                  have an entry in the VMGcRefActivationsTable; \
-                 {gc_ref:?} is not in the table",
+                 {gc_ref:#p} is not in the table",
             );
             if gc_ref.is_i31() {
                 continue;
@@ -401,10 +401,8 @@ impl DrcHeap {
             .iter_mut()
             .take(num_filled)
             .map(|slot| {
-                let r64 = *slot.get_mut();
-                VMGcRef::from_r64(r64)
-                    .expect("valid r64")
-                    .expect("non-null")
+                let raw = *slot.get_mut();
+                VMGcRef::from_raw_u32(raw).expect("non-null")
             })
     }
 
@@ -432,10 +430,8 @@ impl DrcHeap {
         // borrows.
         let mut alloc = mem::take(&mut self.activations_table.alloc);
         for slot in alloc.chunk.iter_mut().take(num_filled) {
-            let r64 = mem::take(slot.get_mut());
-            let gc_ref = VMGcRef::from_r64(r64)
-                .expect("valid r64")
-                .expect("non-null");
+            let raw = mem::take(slot.get_mut());
+            let gc_ref = VMGcRef::from_raw_u32(raw).expect("non-null");
             f(self, gc_ref);
             *slot.get_mut() = 0;
         }
@@ -777,7 +773,7 @@ impl<'a> GarbageCollection<'a> for DrcCollection<'a> {
 /// The type of `VMGcRefActivationsTable`'s bump region's elements.
 ///
 /// These are written to by Wasm.
-type TableElem = UnsafeCell<u64>;
+type TableElem = UnsafeCell<u32>;
 
 /// A table that over-approximizes the set of `VMGcRef`s that any Wasm
 /// activation on this thread is currently using.
@@ -943,7 +939,7 @@ impl VMGcRefActivationsTable {
                 0,
                 "slots >= the `next` bump finger are always `None`"
             );
-            ptr::write(next.as_ptr(), UnsafeCell::new(gc_ref.into_r64()));
+            ptr::write(next.as_ptr(), UnsafeCell::new(gc_ref.as_raw_u32()));
 
             let next = NonNull::new_unchecked(next.as_ptr().add(1));
             debug_assert!(next <= self.alloc.end);
@@ -982,7 +978,7 @@ impl VMGcRefActivationsTable {
         // filled-in slots.
         let num_filled = self.num_filled_in_bump_chunk();
         for slot in self.alloc.chunk.iter().take(num_filled) {
-            if let Some(elem) = VMGcRef::from_r64(unsafe { *slot.get() }).expect("valid r64") {
+            if let Some(elem) = VMGcRef::from_raw_u32(unsafe { *slot.get() }) {
                 f(&elem);
             }
         }
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs b/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs
index 33905e3cf..4f2a1d73b 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs
@@ -233,22 +233,6 @@ impl VMGcRef {
         VMGcRef::from_raw_non_zero_u32(non_zero)
     }
 
-    /// Create a new `VMGcRef` from a raw `r64` value from Cranelift.
-    ///
-    /// Returns an error if `raw` cannot be losslessly converted from a `u64`
-    /// into a `u32`.
-    ///
-    /// Returns `Ok(None)` if `raw` is zero (aka a "null" `VMGcRef`).
-    ///
-    /// This method only exists because we can't currently use Cranelift's `r32`
-    /// type on 64-bit platforms. We should instead have a `from_r32` method.
-    pub fn from_r64(raw: u64) -> Result<Option<Self>> {
-        let raw = u32::try_from(raw & (u32::MAX as u64))
-            .err2anyhow()
-            .with_context(|| format!("invalid r64: {raw:#x} cannot be converted into a u32"))?;
-        Ok(Self::from_raw_u32(raw))
-    }
-
     /// Copy this `VMGcRef` without running the GC's clone barriers.
     ///
     /// Prefer calling `clone(&mut GcStore)` instead! This is mostly an internal
@@ -278,24 +262,6 @@ impl VMGcRef {
         self.0.get()
     }
 
-    /// Get this GC reference as a raw `r64` value for passing to Cranelift.
-    ///
-    /// This method only exists because we can't currently use Cranelift's `r32`
-    /// type on 64-bit platforms. We should instead be able to pass `VMGcRef`
-    /// into compiled code directly.
-    pub fn into_r64(self) -> u64 {
-        u64::from(self.0.get())
-    }
-
-    /// Get this GC reference as a raw `r64` value for passing to Cranelift.
-    ///
-    /// This method only exists because we can't currently use Cranelift's `r32`
-    /// type on 64-bit platforms. We should instead be able to pass `VMGcRef`
-    /// into compiled code directly.
-    pub fn as_r64(&self) -> u64 {
-        u64::from(self.0.get())
-    }
-
     /// Creates a typed GC reference from `self`, checking that `self` actually
     /// is a `T`.
     ///
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
index e82fdba81..ee42134aa 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
@@ -451,28 +451,30 @@ pub struct GcRootsList(Vec<RawGcRoot>);
 //    such that it is easily reusable across GCs is in the store itself. But the
 //    contents of the roots list (when it is non-empty, during GCs) borrow from
 //    the store, which creates self-references.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug)]
 enum RawGcRoot {
-    Stack(SendSyncPtr<u64>),
+    Stack(SendSyncPtr<u32>),
     NonStack(SendSyncPtr<VMGcRef>),
 }
 
 impl GcRootsList {
     /// Add a GC root that is inside a Wasm stack frame to this list.
     #[inline]
-    pub unsafe fn add_wasm_stack_root(&mut self, ptr_to_root: SendSyncPtr<u64>) {
+    pub unsafe fn add_wasm_stack_root(&mut self, ptr_to_root: SendSyncPtr<u32>) {
         log::trace!(
-            "Adding Wasm stack root: {:#p}",
-            VMGcRef::from_r64(*ptr_to_root.as_ref()).unwrap().unwrap()
+            "Adding Wasm stack root: {:#p} -> {:#p}",
+            ptr_to_root,
+            VMGcRef::from_raw_u32(*ptr_to_root.as_ref()).unwrap()
         );
+        debug_assert!(VMGcRef::from_raw_u32(*ptr_to_root.as_ref()).is_some());
         self.0.push(RawGcRoot::Stack(ptr_to_root));
     }
 
     /// Add a GC root to this list.
     #[inline]
-    pub unsafe fn add_root(&mut self, ptr_to_root: SendSyncPtr<VMGcRef>) {
+    pub unsafe fn add_root(&mut self, ptr_to_root: SendSyncPtr<VMGcRef>, why: &str) {
         log::trace!(
-            "Adding non-stack root: {:#p}",
+            "Adding non-stack root: {why}: {:#p}",
             ptr_to_root.as_ref().unchecked_copy()
         );
         self.0.push(RawGcRoot::NonStack(ptr_to_root))
@@ -530,6 +532,7 @@ impl<'a> Iterator for GcRootsIter<'a> {
 ///
 /// Collector implementations should update the `VMGcRef` if they move the
 /// `VMGcRef`'s referent during the course of a GC.
+#[derive(Debug)]
 pub struct GcRoot<'a> {
     raw: RawGcRoot,
     _phantom: marker::PhantomData<&'a mut VMGcRef>,
@@ -550,10 +553,8 @@ impl GcRoot<'_> {
         match self.raw {
             RawGcRoot::NonStack(ptr) => unsafe { ptr::read(ptr.as_ptr()) },
             RawGcRoot::Stack(ptr) => unsafe {
-                let r64 = ptr::read(ptr.as_ptr());
-                VMGcRef::from_r64(r64)
-                    .expect("valid r64")
-                    .expect("non-null")
+                let raw: u32 = ptr::read(ptr.as_ptr());
+                VMGcRef::from_raw_u32(raw).expect("non-null")
             },
         }
     }
@@ -571,8 +572,7 @@ impl GcRoot<'_> {
                 ptr::write(ptr.as_ptr(), new_ref);
             },
             RawGcRoot::Stack(ptr) => unsafe {
-                let r64 = new_ref.into_r64();
-                ptr::write(ptr.as_ptr(), r64);
+                ptr::write(ptr.as_ptr(), new_ref.as_raw_u32());
             },
         }
     }
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index a514e9381..207048809 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -139,7 +139,7 @@ pub mod raw {
 
         (@ty i32) => (u32);
         (@ty i64) => (u64);
-        (@ty reference) => (*mut u8);
+        (@ty reference) => (u32);
         (@ty pointer) => (*mut u8);
         (@ty vmctx) => (*mut VMContext);
     }
@@ -201,23 +201,18 @@ fn memory32_grow(
     Ok(result as *mut _)
 }
 
-// Implementation of `table.grow`.
-unsafe fn table_grow(
+/// Implementation of `table.grow` for `funcref` tables.
+unsafe fn table_grow_func_ref(
     instance: &mut Instance,
     table_index: u32,
     delta: u32,
-    // NB: we don't know whether this is a pointer to a `VMFuncRef` or is an
-    // `r64` that represents a `VMGcRef` until we look at the table type.
     init_value: *mut u8,
 ) -> Result<u32> {
     let table_index = TableIndex::from_u32(table_index);
 
     let element = match instance.table_element_type(table_index) {
         TableElementType::Func => (init_value as *mut VMFuncRef).into(),
-        TableElementType::GcRef => VMGcRef::from_r64(u64::try_from(init_value as usize).unwrap())
-            .unwrap()
-            .map(|r| (*instance.store()).gc_store().clone_gc_ref(&r))
-            .into(),
+        TableElementType::GcRef => unreachable!(),
     };
 
     Ok(match instance.table_grow(table_index, delta, element)? {
@@ -226,18 +221,34 @@ unsafe fn table_grow(
     })
 }
 
-use table_grow as table_grow_func_ref;
-
+/// Implementation of `table.grow` for GC-reference tables.
 #[cfg(feature = "gc")]
-use table_grow as table_grow_gc_ref;
+unsafe fn table_grow_gc_ref(
+    instance: &mut Instance,
+    table_index: u32,
+    delta: u32,
+    init_value: u32,
+) -> Result<u32> {
+    let table_index = TableIndex::from_u32(table_index);
+
+    let element = match instance.table_element_type(table_index) {
+        TableElementType::Func => unreachable!(),
+        TableElementType::GcRef => VMGcRef::from_raw_u32(init_value)
+            .map(|r| (*instance.store()).gc_store().clone_gc_ref(&r))
+            .into(),
+    };
+
+    Ok(match instance.table_grow(table_index, delta, element)? {
+        Some(r) => r,
+        None => (-1_i32).unsigned(),
+    })
+}
 
-// Implementation of `table.fill`.
-unsafe fn table_fill(
+/// Implementation of `table.fill` for `funcref`s.
+unsafe fn table_fill_func_ref(
     instance: &mut Instance,
     table_index: u32,
     dst: u32,
-    // NB: we don't know whether this is an `r64` that represents a `VMGcRef` or
-    // a pointer to a `VMFuncRef` until we look at the table's element type.
     val: *mut u8,
     len: u32,
 ) -> Result<(), Trap> {
@@ -248,21 +259,31 @@ unsafe fn table_fill(
             let val = val.cast::<VMFuncRef>();
             table.fill((*instance.store()).gc_store(), dst, val.into(), len)
         }
+        TableElementType::GcRef => unreachable!(),
+    }
+}
 
+#[cfg(feature = "gc")]
+unsafe fn table_fill_gc_ref(
+    instance: &mut Instance,
+    table_index: u32,
+    dst: u32,
+    val: u32,
+    len: u32,
+) -> Result<(), Trap> {
+    let table_index = TableIndex::from_u32(table_index);
+    let table = &mut *instance.get_table(table_index);
+    match table.element_type() {
+        TableElementType::Func => unreachable!(),
         TableElementType::GcRef => {
             let gc_store = (*instance.store()).gc_store();
-            let gc_ref = VMGcRef::from_r64(u64::try_from(val as usize).unwrap()).unwrap();
+            let gc_ref = VMGcRef::from_raw_u32(val);
             let gc_ref = gc_ref.map(|r| gc_store.clone_gc_ref(&r));
             table.fill(gc_store, dst, gc_ref.into(), len)
         }
     }
 }
 
-use table_fill as table_fill_func_ref;
-
-#[cfg(feature = "gc")]
-use table_fill as table_fill_gc_ref;
-
 // Implementation of `table.copy`.
 unsafe fn table_copy(
     instance: &mut Instance,
@@ -372,22 +393,19 @@ unsafe fn table_get_lazy_init_func_ref(
     elem.into_func_ref_asserting_initialized().cast()
 }
 
-// Drop a GC reference.
+/// Drop a GC reference.
 #[cfg(feature = "gc")]
-unsafe fn drop_gc_ref(instance: &mut Instance, gc_ref: *mut u8) {
-    let gc_ref = VMGcRef::from_r64(u64::try_from(gc_ref as usize).unwrap())
-        .expect("valid r64")
-        .expect("non-null VMGcRef");
-    log::trace!("libcalls::drop_gc_ref({gc_ref:?})");
+unsafe fn drop_gc_ref(instance: &mut Instance, gc_ref: u32) {
+    log::trace!("libcalls::drop_gc_ref({gc_ref:#x})");
+    let gc_ref = VMGcRef::from_raw_u32(gc_ref).expect("non-null VMGcRef");
     (*instance.store()).gc_store().drop_gc_ref(gc_ref);
 }
 
-// Do a GC, keeping `gc_ref` rooted and returning the updated `gc_ref`
-// reference.
+/// Do a GC, keeping `gc_ref` rooted and returning the updated `gc_ref`
+/// reference.
 #[cfg(feature = "gc")]
-unsafe fn gc(instance: &mut Instance, gc_ref: *mut u8) -> Result<*mut u8> {
-    let gc_ref = u64::try_from(gc_ref as usize).unwrap();
-    let gc_ref = VMGcRef::from_r64(gc_ref).expect("valid r64");
+unsafe fn gc(instance: &mut Instance, gc_ref: u32) -> Result<u32> {
+    let gc_ref = VMGcRef::from_raw_u32(gc_ref);
     let gc_ref = gc_ref.map(|r| (*instance.store()).gc_store().clone_gc_ref(&r));
 
     if let Some(gc_ref) = &gc_ref {
@@ -404,18 +422,18 @@ unsafe fn gc(instance: &mut Instance, gc_ref: *mut u8) -> Result<*mut u8> {
     }
 
     match (*instance.store()).gc(gc_ref)? {
-        None => Ok(core::ptr::null_mut()),
+        None => Ok(0),
         Some(r) => {
-            let r64 = r.as_r64();
+            let raw = r.as_raw_u32();
             (*instance.store()).gc_store().expose_gc_ref_to_wasm(r);
-            Ok(usize::try_from(r64).unwrap() as *mut u8)
+            Ok(raw)
         }
     }
 }
 
-// Perform a Wasm `global.get` for GC reference globals.
+/// Perform a Wasm `global.get` for GC reference globals.
 #[cfg(feature = "gc")]
-unsafe fn gc_ref_global_get(instance: &mut Instance, index: u32) -> Result<*mut u8> {
+unsafe fn gc_ref_global_get(instance: &mut Instance, index: u32) -> Result<u32> {
     use core::num::NonZeroUsize;
 
     let index = wasmtime_environ::GlobalIndex::from_u32(index);
@@ -430,22 +448,22 @@ unsafe fn gc_ref_global_get(instance: &mut Instance, index: u32) -> Result<*mut
     }
 
     match (*global).as_gc_ref() {
-        None => Ok(core::ptr::null_mut()),
+        None => Ok(0),
         Some(gc_ref) => {
             let gc_ref = gc_store.clone_gc_ref(gc_ref);
-            let ret = usize::try_from(gc_ref.as_r64()).unwrap() as *mut u8;
+            let ret = gc_ref.as_raw_u32();
             gc_store.expose_gc_ref_to_wasm(gc_ref);
             Ok(ret)
         }
     }
 }
 
-// Perform a Wasm `global.set` for GC reference globals.
+/// Perform a Wasm `global.set` for GC reference globals.
 #[cfg(feature = "gc")]
-unsafe fn gc_ref_global_set(instance: &mut Instance, index: u32, gc_ref: *mut u8) {
+unsafe fn gc_ref_global_set(instance: &mut Instance, index: u32, gc_ref: u32) {
     let index = wasmtime_environ::GlobalIndex::from_u32(index);
     let global = instance.defined_or_imported_global_ptr(index);
-    let gc_ref = VMGcRef::from_r64(u64::try_from(gc_ref as usize).unwrap()).expect("valid r64");
+    let gc_ref = VMGcRef::from_raw_u32(gc_ref);
     let gc_store = (*instance.store()).gc_store();
     (*global).write_gc_ref(gc_store, gc_ref.as_ref());
 }
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 27a058c05..77c2aa91d 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -732,7 +732,7 @@ macro_rules! define_builtin_array {
 
     (@ty i32) => (u32);
     (@ty i64) => (u64);
-    (@ty reference) => (*mut u8);
+    (@ty reference) => (u32);
     (@ty pointer) => (*mut u8);
     (@ty vmctx) => (*mut VMContext);
 }


Test patch:
diff --git a/ci/build-test-matrix.js b/ci/build-test-matrix.js
index 170f6334c..b1c0b73c7 100644
--- a/ci/build-test-matrix.js
+++ b/ci/build-test-matrix.js
@@ -125,7 +125,7 @@ const FULL_MATRIX = [
     "target": "riscv64gc-unknown-linux-gnu",
     "gcc_package": "gcc-riscv64-linux-gnu",
     "gcc": "riscv64-linux-gnu-gcc",
-    "qemu": "qemu-riscv64 -cpu rv64,v=true,vlen=256,vext_spec=v1.0,Zfa=true,zba=true,zbb=true,zbc=true,zbs=true,zbkb=true,zcb=true,x-zicond=true -L /usr/riscv64-linux-gnu",
+    "qemu": "qemu-riscv64 -cpu rv64,v=true,vlen=256,vext_spec=v1.0,Zfa=true,Zfh=true,zba=true,zbb=true,zbc=true,zbs=true,zbkb=true,zcb=true,x-zicond=true -L /usr/riscv64-linux-gnu",
     "qemu_target": "riscv64-linux-user",
     "name": "Test Linux riscv64",
     "filter": "linux-riscv64",
diff --git a/cranelift/filetests/filetests/isa/aarch64/user_stack_maps.clif b/cranelift/filetests/filetests/isa/aarch64/user_stack_maps.clif
index 5c1d93b62..1d4efdf44 100644
--- a/cranelift/filetests/filetests/isa/aarch64/user_stack_maps.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/user_stack_maps.clif
@@ -52,7 +52,7 @@ block0:
 ;   mov x22, x0
 ;   mov x0, x24
 ;   bl 0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})], sp_to_sized_stack_slots: None }
 ;   mov x12, sp
 ;   mov x0, x19
 ;   str w0, [x12]
@@ -62,13 +62,13 @@ block0:
 ;   mov x22, x0
 ;   mov x0, x24
 ;   bl 0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})], sp_to_sized_stack_slots: None }
 ;   mov x15, sp
 ;   mov x0, x22
 ;   str w0, [x15]
 ;   mov x0, x19
 ;   bl 0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})], sp_to_sized_stack_slots: None }
 ;   mov x0, x22
 ;   bl 0
 ;   add sp, sp, #16
@@ -167,7 +167,7 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
 ;   str d1, [x1]
 ;   str q1, [sp, #112]
 ;   bl 0
-;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})], sp_to_sized_stack_slots: None }
 ;   mov x0, x23
 ;   mov x1, x20
 ;   mov x2, x21
diff --git a/cranelift/filetests/filetests/isa/riscv64/f16-bitcast.clif b/cranelift/filetests/filetests/isa/riscv64/f16-bitcast.clif
new file mode 100644
index 000000000..f7cf54546
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/f16-bitcast.clif
@@ -0,0 +1,35 @@
+test compile precise-output
+target riscv64 has_zfh
+
+function %bitcast_i16_f16(i16) -> f16 fast {
+block0(v0: i16):
+    v1 = bitcast.f16 v0
+    return v1
+}
+
+; VCode:
+; block0:
+;   fmv.h.x fa0,a0
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0x05, 0xf4
+;   ret
+
+function %bitcast_f16_i16(f16) -> i16 fast {
+block0(v0: f16):
+    v1 = bitcast.i16 v0
+    return v1
+}
+
+; VCode:
+; block0:
+;   fmv.x.h a0,fa0
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0x05, 0xe4
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/f16-fma.clif b/cranelift/filetests/filetests/isa/riscv64/f16-fma.clif
new file mode 100644
index 000000000..7e3c4c575
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/f16-fma.clif
@@ -0,0 +1,74 @@
+test compile precise-output
+set unwind_info=false
+target riscv64 has_zfh
+
+
+function %fmadd(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+  v3 = fma v0, v1, v2
+  return v3
+}
+
+; VCode:
+; block0:
+;   fmadd.h fa0,fa0,fa1,fa2,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x43, 0x05, 0xb5, 0x64
+;   ret
+
+
+
+function %fmsub(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+    v3 = fneg v2
+    v4 = fma v0, v1, v3
+    return v4
+}
+
+; VCode:
+; block0:
+;   fmsub.h fa0,fa0,fa1,fa2,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x47, 0x05, 0xb5, 0x64
+;   ret
+
+function %fnmsub(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   fnmsub.h fa0,fa0,fa1,fa2,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x4b, 0x05, 0xb5, 0x64
+;   ret
+
+function %fnmadd(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+    v3 = fneg v0
+    v4 = fneg v2
+    v5 = fma v3, v1, v4
+    return v5
+}
+
+; VCode:
+; block0:
+;   fnmadd.h fa0,fa0,fa1,fa2,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x4f, 0x05, 0xb5, 0x64
+;   ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/f16-memory.clif b/cranelift/filetests/filetests/isa/riscv64/f16-memory.clif
new file mode 100644
index 000000000..166771fc0
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/f16-memory.clif
@@ -0,0 +1,125 @@
+test compile precise-output
+target riscv64 has_zfh
+
+function %f16_load(i16) -> f16 {
+    ss0 = explicit_slot 4
+
+block0(v0: i16):
+    stack_store.i16 v0, ss0+2
+    
+    v1 = stack_load.f16 ss0+2
+    return v1
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+;   addi sp,sp,-16
+; block0:
+;   sh a0,2(slot)
+;   flh fa0,2(slot)
+;   addi sp,sp,16
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+;   addi sp, sp, -0x10
+; block1: ; offset 0x14
+;   sh a0, 2(sp)
+;   .byte 0x07, 0x15, 0x21, 0x00
+;   addi sp, sp, 0x10
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
+function %f16_store(f16) -> i16 {
+    ss0 = explicit_slot 4
+
+block0(v0: f16):
+    stack_store.f16 v0, ss0+2
+    
+    v1 = stack_load.i16 ss0+2
+    return v1
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+;   addi sp,sp,-16
+; block0:
+;   fsh fa0,2(slot)
+;   lh a0,2(slot)
+;   addi sp,sp,16
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+;   addi sp, sp, -0x10
+; block1: ; offset 0x14
+;   .byte 0x27, 0x11, 0xa1, 0x00
+;   lh a0, 2(sp)
+;   addi sp, sp, 0x10
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
+function %f16_roundtrip(f16) -> f16 {
+    ss0 = explicit_slot 2
+
+block0(v0: f16):
+    stack_store.f16 v0, ss0
+    v1 = stack_load.f16 ss0
+    return v1
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+;   addi sp,sp,-16
+; block0:
+;   fsh fa0,0(slot)
+;   flh fa0,0(slot)
+;   addi sp,sp,16
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+;   addi sp, sp, -0x10
+; block1: ; offset 0x14
+;   .byte 0x27, 0x10, 0xa1, 0x00
+;   .byte 0x07, 0x15, 0x01, 0x00
+;   addi sp, sp, 0x10
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/f16-select.clif b/cranelift/filetests/filetests/isa/riscv64/f16-select.clif
new file mode 100644
index 000000000..02e57425b
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/f16-select.clif
@@ -0,0 +1,103 @@
+test compile precise-output
+target riscv64 has_zfh
+
+function %select_icmp_i8_f16(i8, f16, f16) -> f16 {
+block0(v0: i8, v1: f16, v2: f16):
+  v3 = iconst.i8 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+
+; VCode:
+; block0:
+;   li a1,42
+;   andi a5,a0,255
+;   andi a1,a1,255
+;   select fa0,fa0,fa1##condition=(a5 eq a1)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi a1, zero, 0x2a
+;   andi a5, a0, 0xff
+;   andi a1, a1, 0xff
+;   beq a5, a1, 8
+;   fmv.d fa0, fa1
+;   ret
+
+function %select_icmp_i16_f16(i16, f16, f16) -> f16 {
+block0(v0: i16, v1: f16, v2: f16):
+  v3 = iconst.i16 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+
+; VCode:
+; block0:
+;   li a3,42
+;   slli a5,a0,48
+;   srai a1,a5,48
+;   slli a3,a3,48
+;   srai a5,a3,48
+;   select fa0,fa0,fa1##condition=(a1 eq a5)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi a3, zero, 0x2a
+;   slli a5, a0, 0x30
+;   srai a1, a5, 0x30
+;   slli a3, a3, 0x30
+;   srai a5, a3, 0x30
+;   beq a1, a5, 8
+;   fmv.d fa0, fa1
+;   ret
+
+function %select_icmp_i32_f16(i32, f16, f16) -> f16 {
+block0(v0: i32, v1: f16, v2: f16):
+  v3 = iconst.i32 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+
+; VCode:
+; block0:
+;   li a1,42
+;   sext.w a5,a0
+;   sext.w a1,a1
+;   select fa0,fa0,fa1##condition=(a5 eq a1)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi a1, zero, 0x2a
+;   sext.w a5, a0
+;   sext.w a1, a1
+;   beq a5, a1, 8
+;   fmv.d fa0, fa1
+;   ret
+
+function %select_icmp_i64_f16(i64, f16, f16) -> f16 {
+block0(v0: i64, v1: f16, v2: f16):
+  v3 = iconst.i64 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+
+; VCode:
+; block0:
+;   li a5,42
+;   select fa0,fa0,fa1##condition=(a0 eq a5)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi a5, zero, 0x2a
+;   beq a0, a5, 8
+;   fmv.d fa0, fa1
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/f16.clif b/cranelift/filetests/filetests/isa/riscv64/f16.clif
new file mode 100644
index 000000000..8727ddfdd
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/f16.clif
@@ -0,0 +1,131 @@
+test compile precise-output
+set unwind_info=false
+target riscv64 has_zfh
+
+function %fadd(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fadd v0, v1
+  return v2
+}
+
+; VCode:
+; block0:
+;   fadd.h fa0,fa0,fa1,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0xb5, 0x04
+;   ret
+
+function %fsub(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fsub v0, v1
+  return v2
+}
+
+; VCode:
+; block0:
+;   fsub.h fa0,fa0,fa1,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0xb5, 0x0c
+;   ret
+
+function %fmul(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fmul v0, v1
+  return v2
+}
+
+; VCode:
+; block0:
+;   fmul.h fa0,fa0,fa1,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0xb5, 0x14
+;   ret
+
+function %fdiv(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fdiv v0, v1
+  return v2
+}
+
+; VCode:
+; block0:
+;   fdiv.h fa0,fa0,fa1,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0xb5, 0x1c
+;   ret
+
+function %fsqrt(f16) -> f16 {
+block0(v0: f16):
+  v1 = sqrt v0
+  return v1
+}
+
+; VCode:
+; block0:
+;   fsqrt.h fa0,fa0,rne
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0x05, 0x5c
+;   ret
+
+function %fabs(f16) -> f16 {
+block0(v0: f16):
+  v1 = fabs v0
+  return v1
+}
+
+; VCode:
+; block0:
+;   fabs.h fa0,fa0
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x25, 0xa5, 0x24
+;   ret
+
+function %fneg_h(f16) -> f16 {
+block0(v0: f16):
+  v1 = fneg v0
+  return v1
+}
+
+; VCode:
+; block0:
+;   fneg.h fa0,fa0
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x15, 0xa5, 0x24
+;   ret
+
+function %fsignj_h(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fcopysign v0, v1
+  return v2
+}
+
+; VCode:
+; block0:
+;   fsgnj.h fa0,fa0,fa1
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0xb5, 0x24
+;   ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/f16const.clif b/cranelift/filetests/filetests/isa/riscv64/f16const.clif
new file mode 100644
index 000000000..1ee3d9366
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/f16const.clif
@@ -0,0 +1,39 @@
+test compile precise-output
+target riscv64 has_zfh
+
+function %ret_0() -> f16 {
+block0():
+  v0 = f16const 0.0
+  return v0
+}
+
+; VCode:
+; block0:
+;   fmv.h.x fa0,zero
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   .byte 0x53, 0x05, 0x00, 0xf4
+;   ret
+
+function %ret_1() -> f16 {
+block0():
+  v0 = f16const 0x1.0
+  return v0
+}
+
+; VCode:
+; block0:
+;   lui a0,4
+;   addi a2,a0,-1024
+;   fmv.h.x fa0,a2
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   lui a0, 4
+;   addi a2, a0, -0x400
+;   .byte 0x53, 0x05, 0x06, 0xf4
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/user_stack_maps.clif b/cranelift/filetests/filetests/isa/riscv64/user_stack_maps.clif
index b265508b9..5b46be979 100644
--- a/cranelift/filetests/filetests/isa/riscv64/user_stack_maps.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/user_stack_maps.clif
@@ -52,7 +52,7 @@ block0:
 ;   mv s2,a2
 ;   mv a0,s3
 ;   call userextname0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})], sp_to_sized_stack_slots: None }
 ;   mv a0,s1
 ;   sw a0,0(slot)
 ;   mv a0,s2
@@ -60,12 +60,12 @@ block0:
 ;   mv s2,a0
 ;   mv a0,s3
 ;   call userextname0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})], sp_to_sized_stack_slots: None }
 ;   mv a0,s2
 ;   sw a0,0(slot)
 ;   mv a0,s1
 ;   call userextname0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})], sp_to_sized_stack_slots: None }
 ;   mv a0,s2
 ;   call userextname0
 ;   ld s1,40(sp)
@@ -173,7 +173,7 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
 ;   fsd fa1,32(slot)
 ;   fmv.d fs0,fa1
 ;   call userextname0
-;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})], sp_to_sized_stack_slots: None }
 ;   mv a2,s2
 ;   mv a4,s10
 ;   sw a2,0(a4)
diff --git a/cranelift/filetests/filetests/isa/s390x/user_stack_maps.clif b/cranelift/filetests/filetests/isa/s390x/user_stack_maps.clif
index 911a93f61..9ee6c7c1f 100644
--- a/cranelift/filetests/filetests/isa/s390x/user_stack_maps.clif
+++ b/cranelift/filetests/filetests/isa/s390x/user_stack_maps.clif
@@ -49,19 +49,19 @@ block0:
 ;   mvhi 0(%r4), 2
 ;   lgr %r2, %r11
 ;   brasl %r14, userextname0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})], sp_to_sized_stack_slots: None }
 ;   la %r2, 160(%r15)
 ;   mvhi 0(%r2), 1
 ;   la %r3, 164(%r15)
 ;   mvhi 0(%r3), 2
 ;   lgr %r2, %r11
 ;   brasl %r14, userextname0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})], sp_to_sized_stack_slots: None }
 ;   la %r5, 160(%r15)
 ;   mvhi 0(%r5), 2
 ;   lgr %r2, %r7
 ;   brasl %r14, userextname0
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})], sp_to_sized_stack_slots: None }
 ;   lgr %r2, %r9
 ;   brasl %r14, userextname0
 ;   lmg %r7, %r15, 232(%r15)
@@ -152,7 +152,7 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
 ;   std %f2, 0(%r3)
 ;   vst %v2, 272(%r15)
 ;   brasl %r14, userextname0
-;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})], sp_to_sized_stack_slots: None }
 ;   lgr %r2, %r11
 ;   lgr %r3, %r9
 ;   lgr %r4, %r7
diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif
index 0571e6880..ec3c9b7ed 100644
--- a/cranelift/filetests/filetests/isa/x64/i128.clif
+++ b/cranelift/filetests/filetests/isa/x64/i128.clif
@@ -2034,3 +2034,38 @@ block0(v0: i64, v1: i64):
 ;   popq %rbp
 ;   retq
 
+function %sink_load_of_i128_half(i64, i64) -> i64, i64 {
+block0(v0: i64, v1: i64):
+    v10 = load.i64 v0
+    v18 = uextend.i128 v1
+    v20 = uextend.i128 v10
+    v14 = iadd v18, v20
+    v15, v16 = isplit v14
+    return v15, v16
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   xorq    %rdx, %rdx, %rdx
+;   movq    %rsi, %rax
+;   addq    %rax, 0(%rdi), %rax
+;   adcq    %rdx, $0, %rdx
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   xorq %rdx, %rdx
+;   movq %rsi, %rax
+;   addq (%rdi), %rax ; trap: heap_oob
+;   adcq $0, %rdx
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/user_stack_maps.clif b/cranelift/filetests/filetests/isa/x64/user_stack_maps.clif
index bddc779df..dc258661b 100644
--- a/cranelift/filetests/filetests/isa/x64/user_stack_maps.clif
+++ b/cranelift/filetests/filetests/isa/x64/user_stack_maps.clif
@@ -53,19 +53,19 @@ block0:
 ;   movl    $2, 0(%rdi)
 ;   movq    %r15, %rdi
 ;   call    User(userextname0)
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})], sp_to_sized_stack_slots: None }
 ;   lea     rsp(0 + virtual offset), %rcx
 ;   movl    $1, 0(%rcx)
 ;   lea     rsp(4 + virtual offset), %rdx
 ;   movl    $2, 0(%rdx)
 ;   movq    %r15, %rdi
 ;   call    User(userextname0)
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})], sp_to_sized_stack_slots: None }
 ;   lea     rsp(0 + virtual offset), %r9
 ;   movl    $2, 0(%r9)
 ;   movq    %rbx, %rdi
 ;   call    User(userextname0)
-;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})], sp_to_sized_stack_slots: None }
 ;   movq    %r13, %rdi
 ;   call    User(userextname0)
 ;   movq    16(%rsp), %rbx
@@ -169,7 +169,7 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
 ;   movsd   %xmm1, 0(%rsi)
 ;   movdqu  %xmm1, rsp(112 + virtual offset)
 ;   call    User(userextname0)
-;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})], sp_to_sized_stack_slots: None }
 ;   movq    %r12, %rdx
 ;   movq    %r13, %r8
 ;   movl    %edx, 0(%r8)
diff --git a/cranelift/filetests/filetests/runtests/bitcast-f16-f128.clif b/cranelift/filetests/filetests/runtests/f128-bitcast.clif
similarity index 65%
rename from cranelift/filetests/filetests/runtests/bitcast-f16-f128.clif
rename to cranelift/filetests/filetests/runtests/f128-bitcast.clif
index e7503bc97..7d9ba7639 100644
--- a/cranelift/filetests/filetests/runtests/bitcast-f16-f128.clif
+++ b/cranelift/filetests/filetests/runtests/f128-bitcast.clif
@@ -5,21 +5,6 @@ target x86_64
 target aarch64
 target aarch64 has_fp16
 
-function %bitcast_i16_f16(i16) -> f16 fast {
-block0(v0: i16):
-    v1 = bitcast.f16 v0
-    return v1
-}
-; run: %bitcast_i16_f16(0x0000) == 0x0.0
-; run: %bitcast_i16_f16(0x0001) == 0x0.004p-14
-; run: %bitcast_i16_f16(0x7c00) == Inf
-; run: %bitcast_i16_f16(0xfe00) == -NaN
-; run: %bitcast_i16_f16(0x7c01) == +sNaN:0x1
-; run: %bitcast_i16_f16(0x3c00) == 0x1.0
-; run: %bitcast_i16_f16(0x47fe) == 0x7.fe
-; run: %bitcast_i16_f16(0xf0e0) == -0x1.380p13
-; run: %bitcast_i16_f16(0xabcd) == -0x1.f34p-5
-
 function %bitcast_i128_f128(i128) -> f128 fast {
 block0(v0: i128):
     v1 = bitcast.f128 v0
@@ -35,21 +20,6 @@ block0(v0: i128):
 ; run: %bitcast_i128_f128(0xf0e0d0c0b0a090807060504030201000) == -0x1.d0c0b0a090807060504030201000p12513
 ; run: %bitcast_i128_f128(0xabcdef01234567890123456789abcdef) == -0x1.ef01234567890123456789abcdefp-5170
 
-function %bitcast_f16_i16(f16) -> i16 fast {
-block0(v0: f16):
-    v1 = bitcast.i16 v0
-    return v1
-}
-; run: %bitcast_f16_i16(0x0.0) == 0x0000
-; run: %bitcast_f16_i16(0x0.004p-14) == 0x0001
-; run: %bitcast_f16_i16(Inf) == 0x7c00
-; run: %bitcast_f16_i16(-NaN) == 0xfe00
-; run: %bitcast_f16_i16(+sNaN:0x1) == 0x7c01
-; run: %bitcast_f16_i16(0x1.0) == 0x3c00
-; run: %bitcast_f16_i16(0x7.fe) == 0x47fe
-; run: %bitcast_f16_i16(-0x1.3c0p13) == 0xf0f0
-; run: %bitcast_f16_i16(-0x1.f34p-5) == 0xabcd
-
 function %bitcast_f128_i128(f128) -> i128 fast {
 block0(v0: f128):
     v1 = bitcast.i128 v0
diff --git a/cranelift/filetests/filetests/runtests/select-f16-f128.clif b/cranelift/filetests/filetests/runtests/f128-select.clif
similarity index 56%
rename from cranelift/filetests/filetests/runtests/select-f16-f128.clif
rename to cranelift/filetests/filetests/runtests/f128-select.clif
index beebd78fc..2593988f8 100644
--- a/cranelift/filetests/filetests/runtests/select-f16-f128.clif
+++ b/cranelift/filetests/filetests/runtests/f128-select.clif
@@ -5,23 +5,6 @@ target x86_64
 target aarch64
 target aarch64 has_fp16
 
-function %select_icmp_i8_f16(i8, f16, f16) -> f16 {
-block0(v0: i8, v1: f16, v2: f16):
-  v3 = iconst.i8 42
-  v4 = icmp eq v0, v3
-  v5 = select.f16 v4, v1, v2
-  return v5
-}
-; run: %select_icmp_i8_f16(42, 0x0.0, 0x1.0) == 0x0.0
-; run: %select_icmp_i8_f16(10, 0x0.0, 0x1.0) == 0x1.0
-; run: %select_icmp_i8_f16(42, +Inf, -Inf) == +Inf
-; run: %select_icmp_i8_f16(10, +Inf, -Inf) == -Inf
-; run: %select_icmp_i8_f16(42, +NaN, -NaN) == +NaN
-; run: %select_icmp_i8_f16(10, +NaN, -NaN) == -NaN
-; run: %select_icmp_i8_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
-; run: %select_icmp_i8_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
-
-
 function %select_icmp_i8_f128(i8, f128, f128) -> f128 {
 block0(v0: i8, v1: f128, v2: f128):
   v3 = iconst.i8 42
@@ -38,24 +21,6 @@ block0(v0: i8, v1: f128, v2: f128):
 ; run: %select_icmp_i8_f128(42, 0x0.8000000000000000000000000000p-16382, -0x0.8000000000000000000000000000p-16382) == 0x0.8000000000000000000000000000p-16382
 ; run: %select_icmp_i8_f128(10, 0x0.8000000000000000000000000000p-16382, -0x0.8000000000000000000000000000p-16382) == -0x0.8000000000000000000000000000p-16382
 
-
-function %select_icmp_i16_f16(i16, f16, f16) -> f16 {
-block0(v0: i16, v1: f16, v2: f16):
-  v3 = iconst.i16 42
-  v4 = icmp eq v0, v3
-  v5 = select.f16 v4, v1, v2
-  return v5
-}
-; run: %select_icmp_i16_f16(42, 0x0.0, 0x1.0) == 0x0.0
-; run: %select_icmp_i16_f16(10, 0x0.0, 0x1.0) == 0x1.0
-; run: %select_icmp_i16_f16(42, +Inf, -Inf) == +Inf
-; run: %select_icmp_i16_f16(10, +Inf, -Inf) == -Inf
-; run: %select_icmp_i16_f16(42, +NaN, -NaN) == +NaN
-; run: %select_icmp_i16_f16(10, +NaN, -NaN) == -NaN
-; run: %select_icmp_i16_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
-; run: %select_icmp_i16_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
-
-
 function %select_icmp_i16_f128(i16, f128, f128) -> f128 {
 block0(v0: i16, v1: f128, v2: f128):
   v3 = iconst.i16 42
@@ -73,23 +38,6 @@ block0(v0: i16, v1: f128, v2: f128):
 ; run: %select_icmp_i16_f128(10, 0x0.8000000000000000000000000000p-16382, -0x0.8000000000000000000000000000p-16382) == -0x0.8000000000000000000000000000p-16382
 
 
-function %select_icmp_i32_f16(i32, f16, f16) -> f16 {
-block0(v0: i32, v1: f16, v2: f16):
-  v3 = iconst.i32 42
-  v4 = icmp eq v0, v3
-  v5 = select.f16 v4, v1, v2
-  return v5
-}
-; run: %select_icmp_i32_f16(42, 0x0.0, 0x1.0) == 0x0.0
-; run: %select_icmp_i32_f16(10, 0x0.0, 0x1.0) == 0x1.0
-; run: %select_icmp_i32_f16(42, +Inf, -Inf) == +Inf
-; run: %select_icmp_i32_f16(10, +Inf, -Inf) == -Inf
-; run: %select_icmp_i32_f16(42, +NaN, -NaN) == +NaN
-; run: %select_icmp_i32_f16(10, +NaN, -NaN) == -NaN
-; run: %select_icmp_i32_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
-; run: %select_icmp_i32_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
-
-
 function %select_icmp_i32_f128(i32, f128, f128) -> f128 {
 block0(v0: i32, v1: f128, v2: f128):
   v3 = iconst.i32 42
@@ -106,24 +54,6 @@ block0(v0: i32, v1: f128, v2: f128):
 ; run: %select_icmp_i32_f128(42, 0x0.8000000000000000000000000000p-16382, -0x0.8000000000000000000000000000p-16382) == 0x0.8000000000000000000000000000p-16382
 ; run: %select_icmp_i32_f128(10, 0x0.8000000000000000000000000000p-16382, -0x0.8000000000000000000000000000p-16382) == -0x0.8000000000000000000000000000p-16382
 
-
-function %select_icmp_i64_f16(i64, f16, f16) -> f16 {
-block0(v0: i64, v1: f16, v2: f16):
-  v3 = iconst.i64 42
-  v4 = icmp eq v0, v3
-  v5 = select.f16 v4, v1, v2
-  return v5
-}
-; run: %select_icmp_i64_f16(42, 0x0.0, 0x1.0) == 0x0.0
-; run: %select_icmp_i64_f16(10, 0x0.0, 0x1.0) == 0x1.0
-; run: %select_icmp_i64_f16(42, +Inf, -Inf) == +Inf
-; run: %select_icmp_i64_f16(10, +Inf, -Inf) == -Inf
-; run: %select_icmp_i64_f16(42, +NaN, -NaN) == +NaN
-; run: %select_icmp_i64_f16(10, +NaN, -NaN) == -NaN
-; run: %select_icmp_i64_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
-; run: %select_icmp_i64_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
-
-
 function %select_icmp_i64_f128(i64, f128, f128) -> f128 {
 block0(v0: i64, v1: f128, v2: f128):
   v3 = iconst.i64 42
diff --git a/cranelift/filetests/filetests/runtests/f16-arithmetic.clif b/cranelift/filetests/filetests/runtests/f16-arithmetic.clif
new file mode 100644
index 000000000..e1c80f9f3
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/f16-arithmetic.clif
@@ -0,0 +1,77 @@
+test run
+target riscv64 has_zfh
+
+function %fadd(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fadd v0, v1
+  return v2
+}
+; run: %fadd(0x0.0, 0x0.0) == 0x0.0
+; run: %fadd(0x1.5, 0x2.9) == 0x1.fp1
+
+function %fsub(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fsub v0, v1
+  return v2
+}
+; run: %fsub(0x0.5, 0x1.0) == -0x1.600p-1
+; run: %fsub(0x1.5, 0x2.9) == -0x1.400p0
+
+
+function %fmul(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fmul v0, v1
+  return v2
+}
+; run: %fmul(0x0.5, 0x1.0) == 0x1.400p-2
+; run: %fmul(0x1.5, 0x2.9) == 0x1.ae8p1
+
+function %fdiv(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fdiv v0, v1
+  return v2
+}
+; run: %fdiv(0x0.5, 0x1.0) == 0x1.400p-2
+; run: %fdiv(0x1.5, 0x2.9) == 0x1.064p-1
+
+function %sqrt(f16) -> f16 {
+block0(v0: f16):
+  v1 = sqrt v0
+  return v1
+}
+; run: %sqrt(0x9.0) == 0x3.0
+; run: %sqrt(0x0.0) == 0x0.0
+; run: %sqrt(-0x0.0) == -0x0.0
+; run: %sqrt(+Inf) == +Inf
+
+function %fabs(f16) -> f16 {
+block0(v0: f16):
+  v1 = fabs v0
+  return v1
+}
+; run: %fabs(0x9.0) == 0x9.0
+; run: %fabs(-0x9.0) == 0x9.0
+; run: %fabs(0x0.0) == 0x0.0
+; run: %fabs(-0x0.0) == 0x0.0
+
+function %fneg(f16) -> f16 {
+block0(v0: f16):
+  v1 = fneg v0
+  return v1
+}
+; run: %fneg(0x9.0) == -0x9.0
+; run: %fneg(-0x9.0) == 0x9.0
+; run: %fneg(0x0.0) == -0x0.0
+; run: %fneg(-0x0.0) == 0x0.0
+
+function %fcopysign(f16, f16) -> f16 {
+block0(v0: f16, v1: f16):
+  v2 = fcopysign v0, v1
+  return v2
+}
+; run: %fcopysign(0x9.0, 0x9.0) == 0x9.0
+; run: %fcopysign(-0x9.0, 0x9.0) == 0x9.0
+; run: %fcopysign(0x9.0, -0x9.0) == -0x9.0
+; run: %fcopysign(-0x9.0, -0x9.0) == -0x9.0
+; run: %fcopysign(0x0.0, -0x0.0) == -0x0.0
+; run: %fcopysign(-0x0.0, 0x0.0) == 0x0.0
diff --git a/cranelift/filetests/filetests/runtests/f16-bitcast.clif b/cranelift/filetests/filetests/runtests/f16-bitcast.clif
new file mode 100644
index 000000000..214ed4088
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/f16-bitcast.clif
@@ -0,0 +1,35 @@
+test interpret
+test run
+set enable_llvm_abi_extensions
+target x86_64
+target riscv64 has_zfh
+
+function %bitcast_i16_f16(i16) -> f16 fast {
+block0(v0: i16):
+    v1 = bitcast.f16 v0
+    return v1
+}
+; run: %bitcast_i16_f16(0x0000) == 0x0.0
+; run: %bitcast_i16_f16(0x0001) == 0x0.004p-14
+; run: %bitcast_i16_f16(0x7c00) == Inf
+; run: %bitcast_i16_f16(0xfe00) == -NaN
+; run: %bitcast_i16_f16(0x7c01) == +sNaN:0x1
+; run: %bitcast_i16_f16(0x3c00) == 0x1.0
+; run: %bitcast_i16_f16(0x47fe) == 0x7.fe
+; run: %bitcast_i16_f16(0xf0e0) == -0x1.380p13
+; run: %bitcast_i16_f16(0xabcd) == -0x1.f34p-5
+
+function %bitcast_f16_i16(f16) -> i16 fast {
+block0(v0: f16):
+    v1 = bitcast.i16 v0
+    return v1
+}
+; run: %bitcast_f16_i16(0x0.0) == 0x0000
+; run: %bitcast_f16_i16(0x0.004p-14) == 0x0001
+; run: %bitcast_f16_i16(Inf) == 0x7c00
+; run: %bitcast_f16_i16(-NaN) == 0xfe00
+; run: %bitcast_f16_i16(+sNaN:0x1) == 0x7c01
+; run: %bitcast_f16_i16(0x1.0) == 0x3c00
+; run: %bitcast_f16_i16(0x7.fe) == 0x47fe
+; run: %bitcast_f16_i16(-0x1.3c0p13) == 0xf0f0
+; run: %bitcast_f16_i16(-0x1.f34p-5) == 0xabcd
diff --git a/cranelift/filetests/filetests/runtests/f16-fma.clif b/cranelift/filetests/filetests/runtests/f16-fma.clif
new file mode 100644
index 000000000..0a4959df6
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/f16-fma.clif
@@ -0,0 +1,38 @@
+test run
+target riscv64 has_zfh
+
+function %fmadd(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+  v3 = fma v0, v1, v2
+  return v3
+}
+; run: %fmadd(0x7.0, 0x8.0, 0x9.0) == 0x1.040p6
+
+
+
+function %fmsub(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+    v3 = fneg v2
+    v4 = fma v0, v1, v3
+    return v4
+}
+; run: %fmsub(0x7.0, 0x8.0, 0x9.0) == 0x1.780p5
+
+
+function %fnmsub(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+; run: %fnmsub(0x7.0, 0x8.0, 0x9.0) == -0x1.780p5
+
+
+function %fnmadd(f16, f16, f16) -> f16 {
+block0(v0: f16, v1: f16, v2: f16):
+    v3 = fneg v0
+    v4 = fneg v2
+    v5 = fma v3, v1, v4
+    return v5
+}
+; run: %fnmadd(0x7.0, 0x8.0, 0x9.0) == -0x1.040p6
diff --git a/cranelift/filetests/filetests/runtests/f16-memory.clif b/cranelift/filetests/filetests/runtests/f16-memory.clif
new file mode 100644
index 000000000..60418f705
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/f16-memory.clif
@@ -0,0 +1,48 @@
+test interpret
+test run
+set enable_llvm_abi_extensions
+target x86_64
+target riscv64 has_zfh
+
+function %f16_load(i16) -> f16 {
+    ss0 = explicit_slot 4
+
+block0(v0: i16):
+    stack_store.i16 v0, ss0+2
+    
+    v1 = stack_load.f16 ss0+2
+    return v1
+}
+; run: %f16_load(0) == 0x0.0
+; run: %f16_load(15680) == 0x1.5
+; run: %f16_load(32256)  == NaN
+; run: %f16_load(31744)  == Inf
+
+
+function %f16_store(f16) -> i16 {
+    ss0 = explicit_slot 4
+
+block0(v0: f16):
+    stack_store.f16 v0, ss0+2
+    
+    v1 = stack_load.i16 ss0+2
+    return v1
+}
+; run: %f16_store(0x0.0) == 0
+; run: %f16_store(0x1.5) == 15680
+; run: %f16_store(NaN) == 32256
+; run: %f16_store(Inf) == 31744
+
+
+function %f16_roundtrip(f16) -> f16 {
+    ss0 = explicit_slot 2
+
+block0(v0: f16):
+    stack_store.f16 v0, ss0
+    v1 = stack_load.f16 ss0
+    return v1
+}
+; run: %f16_roundtrip(0x0.0) == 0x0.0
+; run: %f16_roundtrip(0x1.5) == 0x1.5
+; run: %f16_roundtrip(NaN) == NaN
+; run: %f16_roundtrip(Inf) == Inf
\ No newline at end of file
diff --git a/cranelift/filetests/filetests/runtests/f16-select.clif b/cranelift/filetests/filetests/runtests/f16-select.clif
new file mode 100644
index 000000000..8d9ad4f32
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/f16-select.clif
@@ -0,0 +1,70 @@
+test interpret
+test run
+set enable_llvm_abi_extensions
+target x86_64
+target riscv64 has_zfh
+
+function %select_icmp_i8_f16(i8, f16, f16) -> f16 {
+block0(v0: i8, v1: f16, v2: f16):
+  v3 = iconst.i8 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+; run: %select_icmp_i8_f16(42, 0x0.0, 0x1.0) == 0x0.0
+; run: %select_icmp_i8_f16(10, 0x0.0, 0x1.0) == 0x1.0
+; run: %select_icmp_i8_f16(42, +Inf, -Inf) == +Inf
+; run: %select_icmp_i8_f16(10, +Inf, -Inf) == -Inf
+; run: %select_icmp_i8_f16(42, +NaN, -NaN) == +NaN
+; run: %select_icmp_i8_f16(10, +NaN, -NaN) == -NaN
+; run: %select_icmp_i8_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
+; run: %select_icmp_i8_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
+
+function %select_icmp_i16_f16(i16, f16, f16) -> f16 {
+block0(v0: i16, v1: f16, v2: f16):
+  v3 = iconst.i16 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+; run: %select_icmp_i16_f16(42, 0x0.0, 0x1.0) == 0x0.0
+; run: %select_icmp_i16_f16(10, 0x0.0, 0x1.0) == 0x1.0
+; run: %select_icmp_i16_f16(42, +Inf, -Inf) == +Inf
+; run: %select_icmp_i16_f16(10, +Inf, -Inf) == -Inf
+; run: %select_icmp_i16_f16(42, +NaN, -NaN) == +NaN
+; run: %select_icmp_i16_f16(10, +NaN, -NaN) == -NaN
+; run: %select_icmp_i16_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
+; run: %select_icmp_i16_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
+
+function %select_icmp_i32_f16(i32, f16, f16) -> f16 {
+block0(v0: i32, v1: f16, v2: f16):
+  v3 = iconst.i32 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+; run: %select_icmp_i32_f16(42, 0x0.0, 0x1.0) == 0x0.0
+; run: %select_icmp_i32_f16(10, 0x0.0, 0x1.0) == 0x1.0
+; run: %select_icmp_i32_f16(42, +Inf, -Inf) == +Inf
+; run: %select_icmp_i32_f16(10, +Inf, -Inf) == -Inf
+; run: %select_icmp_i32_f16(42, +NaN, -NaN) == +NaN
+; run: %select_icmp_i32_f16(10, +NaN, -NaN) == -NaN
+; run: %select_icmp_i32_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
+; run: %select_icmp_i32_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
+
+
+function %select_icmp_i64_f16(i64, f16, f16) -> f16 {
+block0(v0: i64, v1: f16, v2: f16):
+  v3 = iconst.i64 42
+  v4 = icmp eq v0, v3
+  v5 = select.f16 v4, v1, v2
+  return v5
+}
+; run: %select_icmp_i64_f16(42, 0x0.0, 0x1.0) == 0x0.0
+; run: %select_icmp_i64_f16(10, 0x0.0, 0x1.0) == 0x1.0
+; run: %select_icmp_i64_f16(42, +Inf, -Inf) == +Inf
+; run: %select_icmp_i64_f16(10, +Inf, -Inf) == -Inf
+; run: %select_icmp_i64_f16(42, +NaN, -NaN) == +NaN
+; run: %select_icmp_i64_f16(10, +NaN, -NaN) == -NaN
+; run: %select_icmp_i64_f16(42, 0x0.800p-14, -0x0.800p-14) == 0x0.800p-14
+; run: %select_icmp_i64_f16(10, 0x0.800p-14, -0x0.800p-14) == -0x0.800p-14
diff --git a/cranelift/filetests/filetests/runtests/f16const.clif b/cranelift/filetests/filetests/runtests/f16const.clif
index 0382b6087..f5ea902a3 100644
--- a/cranelift/filetests/filetests/runtests/f16const.clif
+++ b/cranelift/filetests/filetests/runtests/f16const.clif
@@ -4,6 +4,7 @@ set enable_llvm_abi_extensions
 target x86_64
 target aarch64
 target aarch64 has_fp16
+target riscv64 has_zfh
 
 
 ;; These values are special for RISC-V since it has a dedicated
diff --git a/tests/disas/nullref.wat b/tests/disas/nullref.wat
index b31217d23..b298809d8 100644
--- a/tests/disas/nullref.wat
+++ b/tests/disas/nullref.wat
@@ -12,33 +12,33 @@
 	)
 )
 
-;; function u0:0(i64 vmctx, i64) -> r64 tail {
+;; function u0:0(i64 vmctx, i64) -> i32 tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
-;; @0019                               v3 = null.r64 
-;; @001b                               jump block1(v3)
+;; @0019                               v3 = iconst.i32 0
+;; @001b                               jump block1(v3)  ; v3 = 0
 ;;
-;;                                 block1(v2: r64):
+;;                                 block1(v2: i32):
 ;; @001b                               return v2
 ;; }
 ;;
-;; function u0:1(i64 vmctx, i64) -> r64 tail {
+;; function u0:1(i64 vmctx, i64) -> i32 tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
-;; @0020                               v4 = null.r64 
-;; @0022                               jump block2(v4)
+;; @0020                               v4 = iconst.i32 0
+;; @0022                               jump block2(v4)  ; v4 = 0
 ;;
-;;                                 block2(v3: r64):
+;;                                 block2(v3: i32):
 ;; @0023                               jump block1(v3)
 ;;
-;;                                 block1(v2: r64):
+;;                                 block1(v2: i32):
 ;; @0023                               return v2
 ;; }
diff --git a/tests/disas/ref-func-0.wat b/tests/disas/ref-func-0.wat
index 3e0980dad..d0dcc9d94 100644
--- a/tests/disas/ref-func-0.wat
+++ b/tests/disas/ref-func-0.wat
@@ -13,12 +13,13 @@
   (global (export "funcref-imported") funcref (ref.func $imported))
   (global (export "funcref-local") funcref (ref.func $local)))
 
-;; function u0:1(i64 vmctx, i64) -> r64, r64, i64, i64 tail {
+;; function u0:1(i64 vmctx, i64) -> i32, i32, i64, i64 tail {
+;;     ss0 = explicit_slot 4, align = 4
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
-;;     sig0 = (i64 vmctx, i32 uext) -> r64 system_v
+;;     sig0 = (i64 vmctx, i32 uext) -> i32 system_v
 ;;     fn0 = colocated u1:27 sig0
 ;;     stack_limit = gv2
 ;;
@@ -26,15 +27,17 @@
 ;; @008f                               v6 = global_value.i64 gv3
 ;; @008f                               v7 = iconst.i32 0
 ;; @008f                               v8 = call fn0(v6, v7)  ; v7 = 0
+;;                                     stack_store v8, ss0
 ;; @0091                               v9 = global_value.i64 gv3
 ;; @0091                               v10 = iconst.i32 1
-;; @0091                               v11 = call fn0(v9, v10)  ; v10 = 1
+;; @0091                               v11 = call fn0(v9, v10), stack_map=[i32 @ ss0+0]  ; v10 = 1
 ;; @0093                               v12 = global_value.i64 gv3
 ;; @0093                               v13 = load.i64 notrap aligned table v12+144
 ;; @0095                               v14 = global_value.i64 gv3
 ;; @0095                               v15 = load.i64 notrap aligned table v14+160
-;; @0097                               jump block1(v8, v11, v13, v15)
+;;                                     v16 = stack_load.i32 ss0
+;; @0097                               jump block1(v16, v11, v13, v15)
 ;;
-;;                                 block1(v2: r64, v3: r64, v4: i64, v5: i64):
+;;                                 block1(v2: i32, v3: i32, v4: i64, v5: i64):
 ;; @0097                               return v2, v3, v4, v5
 ;; }
diff --git a/tests/disas/select.wat b/tests/disas/select.wat
index 3fd576ca6..f618353ad 100644
--- a/tests/disas/select.wat
+++ b/tests/disas/select.wat
@@ -37,35 +37,35 @@
 ;; @002a                               return v2
 ;; }
 ;;
-;; function u0:1(i64 vmctx, i64) -> r64 tail {
+;; function u0:1(i64 vmctx, i64) -> i32 tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64):
-;; @002d                               v3 = null.r64 
-;; @002f                               v4 = null.r64 
+;; @002d                               v3 = iconst.i32 0
+;; @002f                               v4 = iconst.i32 0
 ;; @0031                               v5 = iconst.i32 1
-;; @0033                               v6 = select v5, v3, v4  ; v5 = 1
+;; @0033                               v6 = select v5, v3, v4  ; v5 = 1, v3 = 0, v4 = 0
 ;; @0036                               jump block1(v6)
 ;;
-;;                                 block1(v2: r64):
+;;                                 block1(v2: i32):
 ;; @0036                               return v2
 ;; }
 ;;
-;; function u0:2(i64 vmctx, i64, r64) -> r64 tail {
+;; function u0:2(i64 vmctx, i64, i32) -> i32 tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     stack_limit = gv2
 ;;
-;;                                 block0(v0: i64, v1: i64, v2: r64):
-;; @0039                               v4 = null.r64 
+;;                                 block0(v0: i64, v1: i64, v2: i32):
+;; @0039                               v4 = iconst.i32 0
 ;; @003d                               v5 = iconst.i32 1
-;; @003f                               v6 = select v5, v4, v2  ; v5 = 1
+;; @003f                               v6 = select v5, v4, v2  ; v5 = 1, v4 = 0
 ;; @0042                               jump block1(v6)
 ;;
-;;                                 block1(v3: r64):
+;;                                 block1(v3: i32):
 ;; @0042                               return v3
 ;; }
diff --git a/tests/disas/table-get-fixed-size.wat b/tests/disas/table-get-fixed-size.wat
index 0419bba37..4024ed37c 100644
--- a/tests/disas/table-get-fixed-size.wat
+++ b/tests/disas/table-get-fixed-size.wat
@@ -15,13 +15,14 @@
     local.get 0
     table.get 0))
 
-;; function u0:0(i64 vmctx, i64) -> r64 tail {
+;; function u0:0(i64 vmctx, i64) -> i32 tail {
+;;     ss0 = explicit_slot 4, align = 4
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, r64) -> r64 system_v
+;;     sig0 = (i64 vmctx, i32) -> i32 system_v
 ;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
@@ -31,83 +32,97 @@
 ;; @0054                               v5 = icmp uge v3, v4  ; v3 = 0, v4 = 7
 ;; @0054                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0054                               v7 = load.i64 notrap aligned readonly v0+88
-;;                                     v47 = iconst.i64 2
-;; @0054                               v8 = ishl v6, v47  ; v47 = 2
+;;                                     v51 = iconst.i64 2
+;; @0054                               v8 = ishl v6, v51  ; v51 = 2
 ;; @0054                               v9 = iadd v7, v8
 ;; @0054                               v10 = iconst.i64 0
 ;; @0054                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @0054                               v12 = load.i32 table_oob aligned table v11
-;; @0054                               v13 = uextend.i64 v12
-;; @0054                               v14 = bitcast.r64 v13
-;; @0054                               v15 = is_null v14
-;; @0054                               brif v15, block5, block2
+;;                                     v52 = stack_addr.i64 ss0
+;;                                     store notrap v12, v52
+;;                                     v53 = stack_addr.i64 ss0
+;;                                     v49 = load.i32 notrap v53
+;;                                     v54 = iconst.i32 0
+;; @0054                               v13 = icmp eq v49, v54  ; v54 = 0
+;; @0054                               brif v13, block5, block2
 ;;
 ;;                                 block2:
-;; @0054                               v17 = load.i64 notrap aligned v0+56
-;; @0054                               v18 = load.i64 notrap aligned v17
-;; @0054                               v19 = load.i64 notrap aligned v17+8
-;; @0054                               v20 = icmp eq v18, v19
-;; @0054                               brif v20, block3, block4
+;; @0054                               v15 = load.i64 notrap aligned v0+56
+;; @0054                               v16 = load.i64 notrap aligned v15
+;; @0054                               v17 = load.i64 notrap aligned v15+8
+;; @0054                               v18 = icmp eq v16, v17
+;; @0054                               brif v18, block3, block4
 ;;
 ;;                                 block4:
-;; @0054                               v22 = load.i64 notrap aligned readonly v0+40
-;; @0054                               v23 = load.i64 notrap aligned readonly v0+48
-;; @0054                               v24 = bitcast.i64 v14
+;; @0054                               v20 = load.i64 notrap aligned readonly v0+40
+;; @0054                               v21 = load.i64 notrap aligned readonly v0+48
+;;                                     v55 = stack_addr.i64 ss0
+;;                                     v48 = load.i32 notrap v55
+;; @0054                               v22 = uextend.i64 v48
+;; @0054                               v23 = iconst.i64 8
+;; @0054                               v24 = uadd_overflow_trap v22, v23, user65535  ; v23 = 8
 ;; @0054                               v25 = iconst.i64 8
 ;; @0054                               v26 = uadd_overflow_trap v24, v25, user65535  ; v25 = 8
-;; @0054                               v27 = iconst.i64 8
-;; @0054                               v28 = uadd_overflow_trap v26, v27, user65535  ; v27 = 8
-;; @0054                               v29 = icmp ult v28, v23
-;; @0054                               brif v29, block7, block6
+;; @0054                               v27 = icmp ult v26, v21
+;; @0054                               brif v27, block7, block6
 ;;
 ;;                                 block6 cold:
 ;; @0054                               trap user65535
 ;;
 ;;                                 block7:
-;; @0054                               v30 = iadd.i64 v22, v26
-;; @0054                               v31 = load.i64 notrap aligned v30
-;;                                     v48 = iconst.i64 1
-;; @0054                               v32 = iadd v31, v48  ; v48 = 1
-;; @0054                               v34 = load.i64 notrap aligned readonly v0+40
-;; @0054                               v35 = load.i64 notrap aligned readonly v0+48
-;; @0054                               v36 = bitcast.i64 v14
+;; @0054                               v28 = iadd.i64 v20, v24
+;; @0054                               v29 = load.i64 notrap aligned v28
+;;                                     v56 = iconst.i64 1
+;; @0054                               v30 = iadd v29, v56  ; v56 = 1
+;; @0054                               v32 = load.i64 notrap aligned readonly v0+40
+;; @0054                               v33 = load.i64 notrap aligned readonly v0+48
+;;                                     v57 = stack_addr.i64 ss0
+;;                                     v47 = load.i32 notrap v57
+;; @0054                               v34 = uextend.i64 v47
+;; @0054                               v35 = iconst.i64 8
+;; @0054                               v36 = uadd_overflow_trap v34, v35, user65535  ; v35 = 8
 ;; @0054                               v37 = iconst.i64 8
 ;; @0054                               v38 = uadd_overflow_trap v36, v37, user65535  ; v37 = 8
-;; @0054                               v39 = iconst.i64 8
-;; @0054                               v40 = uadd_overflow_trap v38, v39, user65535  ; v39 = 8
-;; @0054                               v41 = icmp ult v40, v35
-;; @0054                               brif v41, block9, block8
+;; @0054                               v39 = icmp ult v38, v33
+;; @0054                               brif v39, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @0054                               trap user65535
 ;;
 ;;                                 block9:
-;; @0054                               v42 = iadd.i64 v34, v38
-;; @0054                               store.i64 notrap aligned v32, v42
-;; @0054                               store.r64 notrap aligned v14, v18
-;;                                     v49 = iconst.i64 8
-;; @0054                               v43 = iadd.i64 v18, v49  ; v49 = 8
-;; @0054                               store notrap aligned v43, v17
+;; @0054                               v40 = iadd.i64 v32, v36
+;; @0054                               store.i64 notrap aligned v30, v40
+;;                                     v58 = stack_addr.i64 ss0
+;;                                     v46 = load.i32 notrap v58
+;; @0054                               store notrap aligned v46, v16
+;;                                     v59 = iconst.i64 4
+;; @0054                               v41 = iadd.i64 v16, v59  ; v59 = 4
+;; @0054                               store notrap aligned v41, v15
 ;; @0054                               jump block5
 ;;
 ;;                                 block3 cold:
-;; @0054                               v45 = call fn0(v0, v14)
+;;                                     v60 = stack_addr.i64 ss0
+;;                                     v45 = load.i32 notrap v60
+;; @0054                               v43 = call fn0(v0, v45), stack_map=[i32 @ ss0+0]
 ;; @0054                               jump block5
 ;;
 ;;                                 block5:
+;;                                     v61 = stack_addr.i64 ss0
+;;                                     v44 = load.i32 notrap v61
 ;; @0056                               jump block1
 ;;
 ;;                                 block1:
-;; @0056                               return v14
+;; @0056                               return v44
 ;; }
 ;;
-;; function u0:1(i64 vmctx, i64, i32) -> r64 tail {
+;; function u0:1(i64 vmctx, i64, i32) -> i32 tail {
+;;     ss0 = explicit_slot 4, align = 4
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, r64) -> r64 system_v
+;;     sig0 = (i64 vmctx, i32) -> i32 system_v
 ;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
@@ -116,72 +131,85 @@
 ;; @005b                               v5 = icmp uge v2, v4  ; v4 = 7
 ;; @005b                               v6 = uextend.i64 v2
 ;; @005b                               v7 = load.i64 notrap aligned readonly v0+88
-;;                                     v47 = iconst.i64 2
-;; @005b                               v8 = ishl v6, v47  ; v47 = 2
+;;                                     v51 = iconst.i64 2
+;; @005b                               v8 = ishl v6, v51  ; v51 = 2
 ;; @005b                               v9 = iadd v7, v8
 ;; @005b                               v10 = iconst.i64 0
 ;; @005b                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @005b                               v12 = load.i32 table_oob aligned table v11
-;; @005b                               v13 = uextend.i64 v12
-;; @005b                               v14 = bitcast.r64 v13
-;; @005b                               v15 = is_null v14
-;; @005b                               brif v15, block5, block2
+;;                                     v52 = stack_addr.i64 ss0
+;;                                     store notrap v12, v52
+;;                                     v53 = stack_addr.i64 ss0
+;;                                     v49 = load.i32 notrap v53
+;;                                     v54 = iconst.i32 0
+;; @005b                               v13 = icmp eq v49, v54  ; v54 = 0
+;; @005b                               brif v13, block5, block2
 ;;
 ;;                                 block2:
-;; @005b                               v17 = load.i64 notrap aligned v0+56
-;; @005b                               v18 = load.i64 notrap aligned v17
-;; @005b                               v19 = load.i64 notrap aligned v17+8
-;; @005b                               v20 = icmp eq v18, v19
-;; @005b                               brif v20, block3, block4
+;; @005b                               v15 = load.i64 notrap aligned v0+56
+;; @005b                               v16 = load.i64 notrap aligned v15
+;; @005b                               v17 = load.i64 notrap aligned v15+8
+;; @005b                               v18 = icmp eq v16, v17
+;; @005b                               brif v18, block3, block4
 ;;
 ;;                                 block4:
-;; @005b                               v22 = load.i64 notrap aligned readonly v0+40
-;; @005b                               v23 = load.i64 notrap aligned readonly v0+48
-;; @005b                               v24 = bitcast.i64 v14
+;; @005b                               v20 = load.i64 notrap aligned readonly v0+40
+;; @005b                               v21 = load.i64 notrap aligned readonly v0+48
+;;                                     v55 = stack_addr.i64 ss0
+;;                                     v48 = load.i32 notrap v55
+;; @005b                               v22 = uextend.i64 v48
+;; @005b                               v23 = iconst.i64 8
+;; @005b                               v24 = uadd_overflow_trap v22, v23, user65535  ; v23 = 8
 ;; @005b                               v25 = iconst.i64 8
 ;; @005b                               v26 = uadd_overflow_trap v24, v25, user65535  ; v25 = 8
-;; @005b                               v27 = iconst.i64 8
-;; @005b                               v28 = uadd_overflow_trap v26, v27, user65535  ; v27 = 8
-;; @005b                               v29 = icmp ult v28, v23
-;; @005b                               brif v29, block7, block6
+;; @005b                               v27 = icmp ult v26, v21
+;; @005b                               brif v27, block7, block6
 ;;
 ;;                                 block6 cold:
 ;; @005b                               trap user65535
 ;;
 ;;                                 block7:
-;; @005b                               v30 = iadd.i64 v22, v26
-;; @005b                               v31 = load.i64 notrap aligned v30
-;;                                     v48 = iconst.i64 1
-;; @005b                               v32 = iadd v31, v48  ; v48 = 1
-;; @005b                               v34 = load.i64 notrap aligned readonly v0+40
-;; @005b                               v35 = load.i64 notrap aligned readonly v0+48
-;; @005b                               v36 = bitcast.i64 v14
+;; @005b                               v28 = iadd.i64 v20, v24
+;; @005b                               v29 = load.i64 notrap aligned v28
+;;                                     v56 = iconst.i64 1
+;; @005b                               v30 = iadd v29, v56  ; v56 = 1
+;; @005b                               v32 = load.i64 notrap aligned readonly v0+40
+;; @005b                               v33 = load.i64 notrap aligned readonly v0+48
+;;                                     v57 = stack_addr.i64 ss0
+;;                                     v47 = load.i32 notrap v57
+;; @005b                               v34 = uextend.i64 v47
+;; @005b                               v35 = iconst.i64 8
+;; @005b                               v36 = uadd_overflow_trap v34, v35, user65535  ; v35 = 8
 ;; @005b                               v37 = iconst.i64 8
 ;; @005b                               v38 = uadd_overflow_trap v36, v37, user65535  ; v37 = 8
-;; @005b                               v39 = iconst.i64 8
-;; @005b                               v40 = uadd_overflow_trap v38, v39, user65535  ; v39 = 8
-;; @005b                               v41 = icmp ult v40, v35
-;; @005b                               brif v41, block9, block8
+;; @005b                               v39 = icmp ult v38, v33
+;; @005b                               brif v39, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @005b                               trap user65535
 ;;
 ;;                                 block9:
-;; @005b                               v42 = iadd.i64 v34, v38
-;; @005b                               store.i64 notrap aligned v32, v42
-;; @005b                               store.r64 notrap aligned v14, v18
-;;                                     v49 = iconst.i64 8
-;; @005b                               v43 = iadd.i64 v18, v49  ; v49 = 8
-;; @005b                               store notrap aligned v43, v17
+;; @005b                               v40 = iadd.i64 v32, v36
+;; @005b                               store.i64 notrap aligned v30, v40
+;;                                     v58 = stack_addr.i64 ss0
+;;                                     v46 = load.i32 notrap v58
+;; @005b                               store notrap aligned v46, v16
+;;                                     v59 = iconst.i64 4
+;; @005b                               v41 = iadd.i64 v16, v59  ; v59 = 4
+;; @005b                               store notrap aligned v41, v15
 ;; @005b                               jump block5
 ;;
 ;;                                 block3 cold:
-;; @005b                               v45 = call fn0(v0, v14)
+;;                                     v60 = stack_addr.i64 ss0
+;;                                     v45 = load.i32 notrap v60
+;; @005b                               v43 = call fn0(v0, v45), stack_map=[i32 @ ss0+0]
 ;; @005b                               jump block5
 ;;
 ;;                                 block5:
+;;                                     v61 = stack_addr.i64 ss0
+;;                                     v44 = load.i32 notrap v61
 ;; @005d                               jump block1
 ;;
 ;;                                 block1:
-;; @005d                               return v14
+;; @005d                               return v44
 ;; }
diff --git a/tests/disas/table-get.wat b/tests/disas/table-get.wat
index de6aa74c4..2a2207b1a 100644
--- a/tests/disas/table-get.wat
+++ b/tests/disas/table-get.wat
@@ -14,14 +14,15 @@
     local.get 0
     table.get 0))
 
-;; function u0:0(i64 vmctx, i64) -> r64 tail {
+;; function u0:0(i64 vmctx, i64) -> i32 tail {
+;;     ss0 = explicit_slot 4, align = 4
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+88
 ;;     gv5 = load.i32 notrap aligned gv3+96
-;;     sig0 = (i64 vmctx, r64) -> r64 system_v
+;;     sig0 = (i64 vmctx, i32) -> i32 system_v
 ;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
@@ -31,84 +32,98 @@
 ;; @0053                               v5 = icmp uge v3, v4  ; v3 = 0
 ;; @0053                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0053                               v7 = load.i64 notrap aligned v0+88
-;;                                     v48 = iconst.i64 2
-;; @0053                               v8 = ishl v6, v48  ; v48 = 2
+;;                                     v52 = iconst.i64 2
+;; @0053                               v8 = ishl v6, v52  ; v52 = 2
 ;; @0053                               v9 = iadd v7, v8
 ;; @0053                               v10 = iconst.i64 0
 ;; @0053                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @0053                               v12 = load.i32 table_oob aligned table v11
-;; @0053                               v13 = uextend.i64 v12
-;; @0053                               v14 = bitcast.r64 v13
-;; @0053                               v15 = is_null v14
-;; @0053                               brif v15, block5, block2
+;;                                     v53 = stack_addr.i64 ss0
+;;                                     store notrap v12, v53
+;;                                     v54 = stack_addr.i64 ss0
+;;                                     v49 = load.i32 notrap v54
+;;                                     v55 = iconst.i32 0
+;; @0053                               v13 = icmp eq v49, v55  ; v55 = 0
+;; @0053                               brif v13, block5, block2
 ;;
 ;;                                 block2:
-;; @0053                               v17 = load.i64 notrap aligned v0+56
-;; @0053                               v18 = load.i64 notrap aligned v17
-;; @0053                               v19 = load.i64 notrap aligned v17+8
-;; @0053                               v20 = icmp eq v18, v19
-;; @0053                               brif v20, block3, block4
+;; @0053                               v15 = load.i64 notrap aligned v0+56
+;; @0053                               v16 = load.i64 notrap aligned v15
+;; @0053                               v17 = load.i64 notrap aligned v15+8
+;; @0053                               v18 = icmp eq v16, v17
+;; @0053                               brif v18, block3, block4
 ;;
 ;;                                 block4:
-;; @0053                               v22 = load.i64 notrap aligned readonly v0+40
-;; @0053                               v23 = load.i64 notrap aligned readonly v0+48
-;; @0053                               v24 = bitcast.i64 v14
+;; @0053                               v20 = load.i64 notrap aligned readonly v0+40
+;; @0053                               v21 = load.i64 notrap aligned readonly v0+48
+;;                                     v56 = stack_addr.i64 ss0
+;;                                     v48 = load.i32 notrap v56
+;; @0053                               v22 = uextend.i64 v48
+;; @0053                               v23 = iconst.i64 8
+;; @0053                               v24 = uadd_overflow_trap v22, v23, user65535  ; v23 = 8
 ;; @0053                               v25 = iconst.i64 8
 ;; @0053                               v26 = uadd_overflow_trap v24, v25, user65535  ; v25 = 8
-;; @0053                               v27 = iconst.i64 8
-;; @0053                               v28 = uadd_overflow_trap v26, v27, user65535  ; v27 = 8
-;; @0053                               v29 = icmp ult v28, v23
-;; @0053                               brif v29, block7, block6
+;; @0053                               v27 = icmp ult v26, v21
+;; @0053                               brif v27, block7, block6
 ;;
 ;;                                 block6 cold:
 ;; @0053                               trap user65535
 ;;
 ;;                                 block7:
-;; @0053                               v30 = iadd.i64 v22, v26
-;; @0053                               v31 = load.i64 notrap aligned v30
-;;                                     v49 = iconst.i64 1
-;; @0053                               v32 = iadd v31, v49  ; v49 = 1
-;; @0053                               v34 = load.i64 notrap aligned readonly v0+40
-;; @0053                               v35 = load.i64 notrap aligned readonly v0+48
-;; @0053                               v36 = bitcast.i64 v14
+;; @0053                               v28 = iadd.i64 v20, v24
+;; @0053                               v29 = load.i64 notrap aligned v28
+;;                                     v57 = iconst.i64 1
+;; @0053                               v30 = iadd v29, v57  ; v57 = 1
+;; @0053                               v32 = load.i64 notrap aligned readonly v0+40
+;; @0053                               v33 = load.i64 notrap aligned readonly v0+48
+;;                                     v58 = stack_addr.i64 ss0
+;;                                     v47 = load.i32 notrap v58
+;; @0053                               v34 = uextend.i64 v47
+;; @0053                               v35 = iconst.i64 8
+;; @0053                               v36 = uadd_overflow_trap v34, v35, user65535  ; v35 = 8
 ;; @0053                               v37 = iconst.i64 8
 ;; @0053                               v38 = uadd_overflow_trap v36, v37, user65535  ; v37 = 8
-;; @0053                               v39 = iconst.i64 8
-;; @0053                               v40 = uadd_overflow_trap v38, v39, user65535  ; v39 = 8
-;; @0053                               v41 = icmp ult v40, v35
-;; @0053                               brif v41, block9, block8
+;; @0053                               v39 = icmp ult v38, v33
+;; @0053                               brif v39, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @0053                               trap user65535
 ;;
 ;;                                 block9:
-;; @0053                               v42 = iadd.i64 v34, v38
-;; @0053                               store.i64 notrap aligned v32, v42
-;; @0053                               store.r64 notrap aligned v14, v18
-;;                                     v50 = iconst.i64 8
-;; @0053                               v43 = iadd.i64 v18, v50  ; v50 = 8
-;; @0053                               store notrap aligned v43, v17
+;; @0053                               v40 = iadd.i64 v32, v36
+;; @0053                               store.i64 notrap aligned v30, v40
+;;                                     v59 = stack_addr.i64 ss0
+;;                                     v46 = load.i32 notrap v59
+;; @0053                               store notrap aligned v46, v16
+;;                                     v60 = iconst.i64 4
+;; @0053                               v41 = iadd.i64 v16, v60  ; v60 = 4
+;; @0053                               store notrap aligned v41, v15
 ;; @0053                               jump block5
 ;;
 ;;                                 block3 cold:
-;; @0053                               v45 = call fn0(v0, v14)
+;;                                     v61 = stack_addr.i64 ss0
+;;                                     v45 = load.i32 notrap v61
+;; @0053                               v43 = call fn0(v0, v45), stack_map=[i32 @ ss0+0]
 ;; @0053                               jump block5
 ;;
 ;;                                 block5:
+;;                                     v62 = stack_addr.i64 ss0
+;;                                     v44 = load.i32 notrap v62
 ;; @0055                               jump block1
 ;;
 ;;                                 block1:
-;; @0055                               return v14
+;; @0055                               return v44
 ;; }
 ;;
-;; function u0:1(i64 vmctx, i64, i32) -> r64 tail {
+;; function u0:1(i64 vmctx, i64, i32) -> i32 tail {
+;;     ss0 = explicit_slot 4, align = 4
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+88
 ;;     gv5 = load.i32 notrap aligned gv3+96
-;;     sig0 = (i64 vmctx, r64) -> r64 system_v
+;;     sig0 = (i64 vmctx, i32) -> i32 system_v
 ;;     fn0 = colocated u1:26 sig0
 ;;     stack_limit = gv2
 ;;
@@ -117,72 +132,85 @@
 ;; @005a                               v5 = icmp uge v2, v4
 ;; @005a                               v6 = uextend.i64 v2
 ;; @005a                               v7 = load.i64 notrap aligned v0+88
-;;                                     v48 = iconst.i64 2
-;; @005a                               v8 = ishl v6, v48  ; v48 = 2
+;;                                     v52 = iconst.i64 2
+;; @005a                               v8 = ishl v6, v52  ; v52 = 2
 ;; @005a                               v9 = iadd v7, v8
 ;; @005a                               v10 = iconst.i64 0
 ;; @005a                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @005a                               v12 = load.i32 table_oob aligned table v11
-;; @005a                               v13 = uextend.i64 v12
-;; @005a                               v14 = bitcast.r64 v13
-;; @005a                               v15 = is_null v14
-;; @005a                               brif v15, block5, block2
+;;                                     v53 = stack_addr.i64 ss0
+;;                                     store notrap v12, v53
+;;                                     v54 = stack_addr.i64 ss0
+;;                                     v49 = load.i32 notrap v54
+;;                                     v55 = iconst.i32 0
+;; @005a                               v13 = icmp eq v49, v55  ; v55 = 0
+;; @005a                               brif v13, block5, block2
 ;;
 ;;                                 block2:
-;; @005a                               v17 = load.i64 notrap aligned v0+56
-;; @005a                               v18 = load.i64 notrap aligned v17
-;; @005a                               v19 = load.i64 notrap aligned v17+8
-;; @005a                               v20 = icmp eq v18, v19
-;; @005a                               brif v20, block3, block4
+;; @005a                               v15 = load.i64 notrap aligned v0+56
+;; @005a                               v16 = load.i64 notrap aligned v15
+;; @005a                               v17 = load.i64 notrap aligned v15+8
+;; @005a                               v18 = icmp eq v16, v17
+;; @005a                               brif v18, block3, block4
 ;;
 ;;                                 block4:
-;; @005a                               v22 = load.i64 notrap aligned readonly v0+40
-;; @005a                               v23 = load.i64 notrap aligned readonly v0+48
-;; @005a                               v24 = bitcast.i64 v14
+;; @005a                               v20 = load.i64 notrap aligned readonly v0+40
+;; @005a                               v21 = load.i64 notrap aligned readonly v0+48
+;;                                     v56 = stack_addr.i64 ss0
+;;                                     v48 = load.i32 notrap v56
+;; @005a                               v22 = uextend.i64 v48
+;; @005a                               v23 = iconst.i64 8
+;; @005a                               v24 = uadd_overflow_trap v22, v23, user65535  ; v23 = 8
 ;; @005a                               v25 = iconst.i64 8
 ;; @005a                               v26 = uadd_overflow_trap v24, v25, user65535  ; v25 = 8
-;; @005a                               v27 = iconst.i64 8
-;; @005a                               v28 = uadd_overflow_trap v26, v27, user65535  ; v27 = 8
-;; @005a                               v29 = icmp ult v28, v23
-;; @005a                               brif v29, block7, block6
+;; @005a                               v27 = icmp ult v26, v21
+;; @005a                               brif v27, block7, block6
 ;;
 ;;                                 block6 cold:
 ;; @005a                               trap user65535
 ;;
 ;;                                 block7:
-;; @005a                               v30 = iadd.i64 v22, v26
-;; @005a                               v31 = load.i64 notrap aligned v30
-;;                                     v49 = iconst.i64 1
-;; @005a                               v32 = iadd v31, v49  ; v49 = 1
-;; @005a                               v34 = load.i64 notrap aligned readonly v0+40
-;; @005a                               v35 = load.i64 notrap aligned readonly v0+48
-;; @005a                               v36 = bitcast.i64 v14
+;; @005a                               v28 = iadd.i64 v20, v24
+;; @005a                               v29 = load.i64 notrap aligned v28
+;;                                     v57 = iconst.i64 1
+;; @005a                               v30 = iadd v29, v57  ; v57 = 1
+;; @005a                               v32 = load.i64 notrap aligned readonly v0+40
+;; @005a                               v33 = load.i64 notrap aligned readonly v0+48
+;;                                     v58 = stack_addr.i64 ss0
+;;                                     v47 = load.i32 notrap v58
+;; @005a                               v34 = uextend.i64 v47
+;; @005a                               v35 = iconst.i64 8
+;; @005a                               v36 = uadd_overflow_trap v34, v35, user65535  ; v35 = 8
 ;; @005a                               v37 = iconst.i64 8
 ;; @005a                               v38 = uadd_overflow_trap v36, v37, user65535  ; v37 = 8
-;; @005a                               v39 = iconst.i64 8
-;; @005a                               v40 = uadd_overflow_trap v38, v39, user65535  ; v39 = 8
-;; @005a                               v41 = icmp ult v40, v35
-;; @005a                               brif v41, block9, block8
+;; @005a                               v39 = icmp ult v38, v33
+;; @005a                               brif v39, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @005a                               trap user65535
 ;;
 ;;                                 block9:
-;; @005a                               v42 = iadd.i64 v34, v38
-;; @005a                               store.i64 notrap aligned v32, v42
-;; @005a                               store.r64 notrap aligned v14, v18
-;;                                     v50 = iconst.i64 8
-;; @005a                               v43 = iadd.i64 v18, v50  ; v50 = 8
-;; @005a                               store notrap aligned v43, v17
+;; @005a                               v40 = iadd.i64 v32, v36
+;; @005a                               store.i64 notrap aligned v30, v40
+;;                                     v59 = stack_addr.i64 ss0
+;;                                     v46 = load.i32 notrap v59
+;; @005a                               store notrap aligned v46, v16
+;;                                     v60 = iconst.i64 4
+;; @005a                               v41 = iadd.i64 v16, v60  ; v60 = 4
+;; @005a                               store notrap aligned v41, v15
 ;; @005a                               jump block5
 ;;
 ;;                                 block3 cold:
-;; @005a                               v45 = call fn0(v0, v14)
+;;                                     v61 = stack_addr.i64 ss0
+;;                                     v45 = load.i32 notrap v61
+;; @005a                               v43 = call fn0(v0, v45), stack_map=[i32 @ ss0+0]
 ;; @005a                               jump block5
 ;;
 ;;                                 block5:
+;;                                     v62 = stack_addr.i64 ss0
+;;                                     v44 = load.i32 notrap v62
 ;; @005c                               jump block1
 ;;
 ;;                                 block1:
-;; @005c                               return v14
+;; @005c                               return v44
 ;; }
diff --git a/tests/disas/table-set-fixed-size.wat b/tests/disas/table-set-fixed-size.wat
index 2349f675b..0360fc327 100644
--- a/tests/disas/table-set-fixed-size.wat
+++ b/tests/disas/table-set-fixed-size.wat
@@ -16,122 +16,119 @@
     local.get 0
     local.get 1
     table.set 0))
-;; function u0:0(i64 vmctx, i64, r64) tail {
+;; function u0:0(i64 vmctx, i64, i32) tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) system_v
+;;     sig0 = (i64 vmctx, i32 uext) system_v
 ;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
-;;                                 block0(v0: i64, v1: i64, v2: r64):
+;;                                 block0(v0: i64, v1: i64, v2: i32):
 ;; @0052                               v3 = iconst.i32 0
 ;; @0056                               v4 = iconst.i32 7
 ;; @0056                               v5 = icmp uge v3, v4  ; v3 = 0, v4 = 7
 ;; @0056                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0056                               v7 = load.i64 notrap aligned readonly v0+88
-;;                                     v67 = iconst.i64 2
-;; @0056                               v8 = ishl v6, v67  ; v67 = 2
+;;                                     v62 = iconst.i64 2
+;; @0056                               v8 = ishl v6, v62  ; v62 = 2
 ;; @0056                               v9 = iadd v7, v8
 ;; @0056                               v10 = iconst.i64 0
 ;; @0056                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @0056                               v12 = load.i32 table_oob aligned table v11
-;; @0056                               v13 = uextend.i64 v12
-;; @0056                               v14 = bitcast.r64 v13
-;; @0056                               v15 = is_null v2
-;; @0056                               brif v15, block3, block2
+;;                                     v63 = iconst.i32 0
+;; @0056                               v13 = icmp eq v2, v63  ; v63 = 0
+;; @0056                               brif v13, block3, block2
 ;;
 ;;                                 block2:
-;; @0056                               v17 = load.i64 notrap aligned readonly v0+40
-;; @0056                               v18 = load.i64 notrap aligned readonly v0+48
-;; @0056                               v19 = bitcast.i64 v2
+;; @0056                               v15 = load.i64 notrap aligned readonly v0+40
+;; @0056                               v16 = load.i64 notrap aligned readonly v0+48
+;; @0056                               v17 = uextend.i64 v2
+;; @0056                               v18 = iconst.i64 8
+;; @0056                               v19 = uadd_overflow_trap v17, v18, user65535  ; v18 = 8
 ;; @0056                               v20 = iconst.i64 8
 ;; @0056                               v21 = uadd_overflow_trap v19, v20, user65535  ; v20 = 8
-;; @0056                               v22 = iconst.i64 8
-;; @0056                               v23 = uadd_overflow_trap v21, v22, user65535  ; v22 = 8
-;; @0056                               v24 = icmp ult v23, v18
-;; @0056                               brif v24, block9, block8
+;; @0056                               v22 = icmp ult v21, v16
+;; @0056                               brif v22, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @0056                               trap user65535
 ;;
 ;;                                 block9:
-;; @0056                               v25 = iadd.i64 v17, v21
-;; @0056                               v26 = load.i64 notrap aligned v25
-;;                                     v68 = iconst.i64 1
-;; @0056                               v27 = iadd v26, v68  ; v68 = 1
-;; @0056                               v29 = load.i64 notrap aligned readonly v0+40
-;; @0056                               v30 = load.i64 notrap aligned readonly v0+48
-;; @0056                               v31 = bitcast.i64 v2
+;; @0056                               v23 = iadd.i64 v15, v19
+;; @0056                               v24 = load.i64 notrap aligned v23
+;;                                     v64 = iconst.i64 1
+;; @0056                               v25 = iadd v24, v64  ; v64 = 1
+;; @0056                               v27 = load.i64 notrap aligned readonly v0+40
+;; @0056                               v28 = load.i64 notrap aligned readonly v0+48
+;; @0056                               v29 = uextend.i64 v2
+;; @0056                               v30 = iconst.i64 8
+;; @0056                               v31 = uadd_overflow_trap v29, v30, user65535  ; v30 = 8
 ;; @0056                               v32 = iconst.i64 8
 ;; @0056                               v33 = uadd_overflow_trap v31, v32, user65535  ; v32 = 8
-;; @0056                               v34 = iconst.i64 8
-;; @0056                               v35 = uadd_overflow_trap v33, v34, user65535  ; v34 = 8
-;; @0056                               v36 = icmp ult v35, v30
-;; @0056                               brif v36, block11, block10
+;; @0056                               v34 = icmp ult v33, v28
+;; @0056                               brif v34, block11, block10
 ;;
 ;;                                 block10 cold:
 ;; @0056                               trap user65535
 ;;
 ;;                                 block11:
-;; @0056                               v37 = iadd.i64 v29, v33
-;; @0056                               store.i64 notrap aligned v27, v37
+;; @0056                               v35 = iadd.i64 v27, v31
+;; @0056                               store.i64 notrap aligned v25, v35
 ;; @0056                               jump block3
 ;;
 ;;                                 block3:
-;; @0056                               v38 = bitcast.i64 v2
-;; @0056                               v39 = ireduce.i32 v38
-;; @0056                               store table_oob aligned table v39, v11
-;; @0056                               v40 = is_null.r64 v14
-;; @0056                               brif v40, block7, block4
+;; @0056                               store.i32 table_oob aligned table v2, v11
+;;                                     v65 = iconst.i32 0
+;; @0056                               v36 = icmp.i32 eq v12, v65  ; v65 = 0
+;; @0056                               brif v36, block7, block4
 ;;
 ;;                                 block4:
-;; @0056                               v42 = load.i64 notrap aligned readonly v0+40
-;; @0056                               v43 = load.i64 notrap aligned readonly v0+48
-;; @0056                               v44 = bitcast.i64 v14
-;; @0056                               v45 = iconst.i64 8
-;; @0056                               v46 = uadd_overflow_trap v44, v45, user65535  ; v45 = 8
-;; @0056                               v47 = iconst.i64 8
-;; @0056                               v48 = uadd_overflow_trap v46, v47, user65535  ; v47 = 8
-;; @0056                               v49 = icmp ult v48, v43
-;; @0056                               brif v49, block13, block12
+;; @0056                               v38 = load.i64 notrap aligned readonly v0+40
+;; @0056                               v39 = load.i64 notrap aligned readonly v0+48
+;; @0056                               v40 = uextend.i64 v12
+;; @0056                               v41 = iconst.i64 8
+;; @0056                               v42 = uadd_overflow_trap v40, v41, user65535  ; v41 = 8
+;; @0056                               v43 = iconst.i64 8
+;; @0056                               v44 = uadd_overflow_trap v42, v43, user65535  ; v43 = 8
+;; @0056                               v45 = icmp ult v44, v39
+;; @0056                               brif v45, block13, block12
 ;;
 ;;                                 block12 cold:
 ;; @0056                               trap user65535
 ;;
 ;;                                 block13:
-;; @0056                               v50 = iadd.i64 v42, v46
-;; @0056                               v51 = load.i64 notrap aligned v50
-;;                                     v69 = iconst.i64 -1
-;; @0056                               v52 = iadd v51, v69  ; v69 = -1
-;;                                     v70 = iconst.i64 0
-;; @0056                               v53 = icmp eq v52, v70  ; v70 = 0
-;; @0056                               brif v53, block5, block6
+;; @0056                               v46 = iadd.i64 v38, v42
+;; @0056                               v47 = load.i64 notrap aligned v46
+;;                                     v66 = iconst.i64 -1
+;; @0056                               v48 = iadd v47, v66  ; v66 = -1
+;;                                     v67 = iconst.i64 0
+;; @0056                               v49 = icmp eq v48, v67  ; v67 = 0
+;; @0056                               brif v49, block5, block6
 ;;
 ;;                                 block5 cold:
-;; @0056                               v55 = bitcast.i64 v14
-;; @0056                               call fn0(v0, v55)
+;; @0056                               call fn0(v0, v12)
 ;; @0056                               jump block7
 ;;
 ;;                                 block6:
-;; @0056                               v57 = load.i64 notrap aligned readonly v0+40
-;; @0056                               v58 = load.i64 notrap aligned readonly v0+48
-;; @0056                               v59 = bitcast.i64 v14
-;; @0056                               v60 = iconst.i64 8
-;; @0056                               v61 = uadd_overflow_trap v59, v60, user65535  ; v60 = 8
-;; @0056                               v62 = iconst.i64 8
-;; @0056                               v63 = uadd_overflow_trap v61, v62, user65535  ; v62 = 8
-;; @0056                               v64 = icmp ult v63, v58
-;; @0056                               brif v64, block15, block14
+;; @0056                               v52 = load.i64 notrap aligned readonly v0+40
+;; @0056                               v53 = load.i64 notrap aligned readonly v0+48
+;; @0056                               v54 = uextend.i64 v12
+;; @0056                               v55 = iconst.i64 8
+;; @0056                               v56 = uadd_overflow_trap v54, v55, user65535  ; v55 = 8
+;; @0056                               v57 = iconst.i64 8
+;; @0056                               v58 = uadd_overflow_trap v56, v57, user65535  ; v57 = 8
+;; @0056                               v59 = icmp ult v58, v53
+;; @0056                               brif v59, block15, block14
 ;;
 ;;                                 block14 cold:
 ;; @0056                               trap user65535
 ;;
 ;;                                 block15:
-;; @0056                               v65 = iadd.i64 v57, v61
-;; @0056                               store.i64 notrap aligned v52, v65
+;; @0056                               v60 = iadd.i64 v52, v56
+;; @0056                               store.i64 notrap aligned v48, v60
 ;; @0056                               jump block7
 ;;
 ;;                                 block7:
@@ -141,121 +138,118 @@
 ;; @0058                               return
 ;; }
 ;;
-;; function u0:1(i64 vmctx, i64, i32, r64) tail {
+;; function u0:1(i64 vmctx, i64, i32, i32) tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) system_v
+;;     sig0 = (i64 vmctx, i32 uext) system_v
 ;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
-;;                                 block0(v0: i64, v1: i64, v2: i32, v3: r64):
+;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32):
 ;; @005f                               v4 = iconst.i32 7
 ;; @005f                               v5 = icmp uge v2, v4  ; v4 = 7
 ;; @005f                               v6 = uextend.i64 v2
 ;; @005f                               v7 = load.i64 notrap aligned readonly v0+88
-;;                                     v67 = iconst.i64 2
-;; @005f                               v8 = ishl v6, v67  ; v67 = 2
+;;                                     v62 = iconst.i64 2
+;; @005f                               v8 = ishl v6, v62  ; v62 = 2
 ;; @005f                               v9 = iadd v7, v8
 ;; @005f                               v10 = iconst.i64 0
 ;; @005f                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @005f                               v12 = load.i32 table_oob aligned table v11
-;; @005f                               v13 = uextend.i64 v12
-;; @005f                               v14 = bitcast.r64 v13
-;; @005f                               v15 = is_null v3
-;; @005f                               brif v15, block3, block2
+;;                                     v63 = iconst.i32 0
+;; @005f                               v13 = icmp eq v3, v63  ; v63 = 0
+;; @005f                               brif v13, block3, block2
 ;;
 ;;                                 block2:
-;; @005f                               v17 = load.i64 notrap aligned readonly v0+40
-;; @005f                               v18 = load.i64 notrap aligned readonly v0+48
-;; @005f                               v19 = bitcast.i64 v3
+;; @005f                               v15 = load.i64 notrap aligned readonly v0+40
+;; @005f                               v16 = load.i64 notrap aligned readonly v0+48
+;; @005f                               v17 = uextend.i64 v3
+;; @005f                               v18 = iconst.i64 8
+;; @005f                               v19 = uadd_overflow_trap v17, v18, user65535  ; v18 = 8
 ;; @005f                               v20 = iconst.i64 8
 ;; @005f                               v21 = uadd_overflow_trap v19, v20, user65535  ; v20 = 8
-;; @005f                               v22 = iconst.i64 8
-;; @005f                               v23 = uadd_overflow_trap v21, v22, user65535  ; v22 = 8
-;; @005f                               v24 = icmp ult v23, v18
-;; @005f                               brif v24, block9, block8
+;; @005f                               v22 = icmp ult v21, v16
+;; @005f                               brif v22, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @005f                               trap user65535
 ;;
 ;;                                 block9:
-;; @005f                               v25 = iadd.i64 v17, v21
-;; @005f                               v26 = load.i64 notrap aligned v25
-;;                                     v68 = iconst.i64 1
-;; @005f                               v27 = iadd v26, v68  ; v68 = 1
-;; @005f                               v29 = load.i64 notrap aligned readonly v0+40
-;; @005f                               v30 = load.i64 notrap aligned readonly v0+48
-;; @005f                               v31 = bitcast.i64 v3
+;; @005f                               v23 = iadd.i64 v15, v19
+;; @005f                               v24 = load.i64 notrap aligned v23
+;;                                     v64 = iconst.i64 1
+;; @005f                               v25 = iadd v24, v64  ; v64 = 1
+;; @005f                               v27 = load.i64 notrap aligned readonly v0+40
+;; @005f                               v28 = load.i64 notrap aligned readonly v0+48
+;; @005f                               v29 = uextend.i64 v3
+;; @005f                               v30 = iconst.i64 8
+;; @005f                               v31 = uadd_overflow_trap v29, v30, user65535  ; v30 = 8
 ;; @005f                               v32 = iconst.i64 8
 ;; @005f                               v33 = uadd_overflow_trap v31, v32, user65535  ; v32 = 8
-;; @005f                               v34 = iconst.i64 8
-;; @005f                               v35 = uadd_overflow_trap v33, v34, user65535  ; v34 = 8
-;; @005f                               v36 = icmp ult v35, v30
-;; @005f                               brif v36, block11, block10
+;; @005f                               v34 = icmp ult v33, v28
+;; @005f                               brif v34, block11, block10
 ;;
 ;;                                 block10 cold:
 ;; @005f                               trap user65535
 ;;
 ;;                                 block11:
-;; @005f                               v37 = iadd.i64 v29, v33
-;; @005f                               store.i64 notrap aligned v27, v37
+;; @005f                               v35 = iadd.i64 v27, v31
+;; @005f                               store.i64 notrap aligned v25, v35
 ;; @005f                               jump block3
 ;;
 ;;                                 block3:
-;; @005f                               v38 = bitcast.i64 v3
-;; @005f                               v39 = ireduce.i32 v38
-;; @005f                               store table_oob aligned table v39, v11
-;; @005f                               v40 = is_null.r64 v14
-;; @005f                               brif v40, block7, block4
+;; @005f                               store.i32 table_oob aligned table v3, v11
+;;                                     v65 = iconst.i32 0
+;; @005f                               v36 = icmp.i32 eq v12, v65  ; v65 = 0
+;; @005f                               brif v36, block7, block4
 ;;
 ;;                                 block4:
-;; @005f                               v42 = load.i64 notrap aligned readonly v0+40
-;; @005f                               v43 = load.i64 notrap aligned readonly v0+48
-;; @005f                               v44 = bitcast.i64 v14
-;; @005f                               v45 = iconst.i64 8
-;; @005f                               v46 = uadd_overflow_trap v44, v45, user65535  ; v45 = 8
-;; @005f                               v47 = iconst.i64 8
-;; @005f                               v48 = uadd_overflow_trap v46, v47, user65535  ; v47 = 8
-;; @005f                               v49 = icmp ult v48, v43
-;; @005f                               brif v49, block13, block12
+;; @005f                               v38 = load.i64 notrap aligned readonly v0+40
+;; @005f                               v39 = load.i64 notrap aligned readonly v0+48
+;; @005f                               v40 = uextend.i64 v12
+;; @005f                               v41 = iconst.i64 8
+;; @005f                               v42 = uadd_overflow_trap v40, v41, user65535  ; v41 = 8
+;; @005f                               v43 = iconst.i64 8
+;; @005f                               v44 = uadd_overflow_trap v42, v43, user65535  ; v43 = 8
+;; @005f                               v45 = icmp ult v44, v39
+;; @005f                               brif v45, block13, block12
 ;;
 ;;                                 block12 cold:
 ;; @005f                               trap user65535
 ;;
 ;;                                 block13:
-;; @005f                               v50 = iadd.i64 v42, v46
-;; @005f                               v51 = load.i64 notrap aligned v50
-;;                                     v69 = iconst.i64 -1
-;; @005f                               v52 = iadd v51, v69  ; v69 = -1
-;;                                     v70 = iconst.i64 0
-;; @005f                               v53 = icmp eq v52, v70  ; v70 = 0
-;; @005f                               brif v53, block5, block6
+;; @005f                               v46 = iadd.i64 v38, v42
+;; @005f                               v47 = load.i64 notrap aligned v46
+;;                                     v66 = iconst.i64 -1
+;; @005f                               v48 = iadd v47, v66  ; v66 = -1
+;;                                     v67 = iconst.i64 0
+;; @005f                               v49 = icmp eq v48, v67  ; v67 = 0
+;; @005f                               brif v49, block5, block6
 ;;
 ;;                                 block5 cold:
-;; @005f                               v55 = bitcast.i64 v14
-;; @005f                               call fn0(v0, v55)
+;; @005f                               call fn0(v0, v12)
 ;; @005f                               jump block7
 ;;
 ;;                                 block6:
-;; @005f                               v57 = load.i64 notrap aligned readonly v0+40
-;; @005f                               v58 = load.i64 notrap aligned readonly v0+48
-;; @005f                               v59 = bitcast.i64 v14
-;; @005f                               v60 = iconst.i64 8
-;; @005f                               v61 = uadd_overflow_trap v59, v60, user65535  ; v60 = 8
-;; @005f                               v62 = iconst.i64 8
-;; @005f                               v63 = uadd_overflow_trap v61, v62, user65535  ; v62 = 8
-;; @005f                               v64 = icmp ult v63, v58
-;; @005f                               brif v64, block15, block14
+;; @005f                               v52 = load.i64 notrap aligned readonly v0+40
+;; @005f                               v53 = load.i64 notrap aligned readonly v0+48
+;; @005f                               v54 = uextend.i64 v12
+;; @005f                               v55 = iconst.i64 8
+;; @005f                               v56 = uadd_overflow_trap v54, v55, user65535  ; v55 = 8
+;; @005f                               v57 = iconst.i64 8
+;; @005f                               v58 = uadd_overflow_trap v56, v57, user65535  ; v57 = 8
+;; @005f                               v59 = icmp ult v58, v53
+;; @005f                               brif v59, block15, block14
 ;;
 ;;                                 block14 cold:
 ;; @005f                               trap user65535
 ;;
 ;;                                 block15:
-;; @005f                               v65 = iadd.i64 v57, v61
-;; @005f                               store.i64 notrap aligned v52, v65
+;; @005f                               v60 = iadd.i64 v52, v56
+;; @005f                               store.i64 notrap aligned v48, v60
 ;; @005f                               jump block7
 ;;
 ;;                                 block7:
diff --git a/tests/disas/table-set.wat b/tests/disas/table-set.wat
index cebd63daa..54029fe7c 100644
--- a/tests/disas/table-set.wat
+++ b/tests/disas/table-set.wat
@@ -16,123 +16,120 @@
     local.get 1
     table.set 0))
 
-;; function u0:0(i64 vmctx, i64, r64) tail {
+;; function u0:0(i64 vmctx, i64, i32) tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+88
 ;;     gv5 = load.i32 notrap aligned gv3+96
-;;     sig0 = (i64 vmctx, i64) system_v
+;;     sig0 = (i64 vmctx, i32 uext) system_v
 ;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
-;;                                 block0(v0: i64, v1: i64, v2: r64):
+;;                                 block0(v0: i64, v1: i64, v2: i32):
 ;; @0051                               v3 = iconst.i32 0
 ;; @0055                               v4 = load.i32 notrap aligned v0+96
 ;; @0055                               v5 = icmp uge v3, v4  ; v3 = 0
 ;; @0055                               v6 = uextend.i64 v3  ; v3 = 0
 ;; @0055                               v7 = load.i64 notrap aligned v0+88
-;;                                     v68 = iconst.i64 2
-;; @0055                               v8 = ishl v6, v68  ; v68 = 2
+;;                                     v63 = iconst.i64 2
+;; @0055                               v8 = ishl v6, v63  ; v63 = 2
 ;; @0055                               v9 = iadd v7, v8
 ;; @0055                               v10 = iconst.i64 0
 ;; @0055                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @0055                               v12 = load.i32 table_oob aligned table v11
-;; @0055                               v13 = uextend.i64 v12
-;; @0055                               v14 = bitcast.r64 v13
-;; @0055                               v15 = is_null v2
-;; @0055                               brif v15, block3, block2
+;;                                     v64 = iconst.i32 0
+;; @0055                               v13 = icmp eq v2, v64  ; v64 = 0
+;; @0055                               brif v13, block3, block2
 ;;
 ;;                                 block2:
-;; @0055                               v17 = load.i64 notrap aligned readonly v0+40
-;; @0055                               v18 = load.i64 notrap aligned readonly v0+48
-;; @0055                               v19 = bitcast.i64 v2
+;; @0055                               v15 = load.i64 notrap aligned readonly v0+40
+;; @0055                               v16 = load.i64 notrap aligned readonly v0+48
+;; @0055                               v17 = uextend.i64 v2
+;; @0055                               v18 = iconst.i64 8
+;; @0055                               v19 = uadd_overflow_trap v17, v18, user65535  ; v18 = 8
 ;; @0055                               v20 = iconst.i64 8
 ;; @0055                               v21 = uadd_overflow_trap v19, v20, user65535  ; v20 = 8
-;; @0055                               v22 = iconst.i64 8
-;; @0055                               v23 = uadd_overflow_trap v21, v22, user65535  ; v22 = 8
-;; @0055                               v24 = icmp ult v23, v18
-;; @0055                               brif v24, block9, block8
+;; @0055                               v22 = icmp ult v21, v16
+;; @0055                               brif v22, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @0055                               trap user65535
 ;;
 ;;                                 block9:
-;; @0055                               v25 = iadd.i64 v17, v21
-;; @0055                               v26 = load.i64 notrap aligned v25
-;;                                     v69 = iconst.i64 1
-;; @0055                               v27 = iadd v26, v69  ; v69 = 1
-;; @0055                               v29 = load.i64 notrap aligned readonly v0+40
-;; @0055                               v30 = load.i64 notrap aligned readonly v0+48
-;; @0055                               v31 = bitcast.i64 v2
+;; @0055                               v23 = iadd.i64 v15, v19
+;; @0055                               v24 = load.i64 notrap aligned v23
+;;                                     v65 = iconst.i64 1
+;; @0055                               v25 = iadd v24, v65  ; v65 = 1
+;; @0055                               v27 = load.i64 notrap aligned readonly v0+40
+;; @0055                               v28 = load.i64 notrap aligned readonly v0+48
+;; @0055                               v29 = uextend.i64 v2
+;; @0055                               v30 = iconst.i64 8
+;; @0055                               v31 = uadd_overflow_trap v29, v30, user65535  ; v30 = 8
 ;; @0055                               v32 = iconst.i64 8
 ;; @0055                               v33 = uadd_overflow_trap v31, v32, user65535  ; v32 = 8
-;; @0055                               v34 = iconst.i64 8
-;; @0055                               v35 = uadd_overflow_trap v33, v34, user65535  ; v34 = 8
-;; @0055                               v36 = icmp ult v35, v30
-;; @0055                               brif v36, block11, block10
+;; @0055                               v34 = icmp ult v33, v28
+;; @0055                               brif v34, block11, block10
 ;;
 ;;                                 block10 cold:
 ;; @0055                               trap user65535
 ;;
 ;;                                 block11:
-;; @0055                               v37 = iadd.i64 v29, v33
-;; @0055                               store.i64 notrap aligned v27, v37
+;; @0055                               v35 = iadd.i64 v27, v31
+;; @0055                               store.i64 notrap aligned v25, v35
 ;; @0055                               jump block3
 ;;
 ;;                                 block3:
-;; @0055                               v38 = bitcast.i64 v2
-;; @0055                               v39 = ireduce.i32 v38
-;; @0055                               store table_oob aligned table v39, v11
-;; @0055                               v40 = is_null.r64 v14
-;; @0055                               brif v40, block7, block4
+;; @0055                               store.i32 table_oob aligned table v2, v11
+;;                                     v66 = iconst.i32 0
+;; @0055                               v36 = icmp.i32 eq v12, v66  ; v66 = 0
+;; @0055                               brif v36, block7, block4
 ;;
 ;;                                 block4:
-;; @0055                               v42 = load.i64 notrap aligned readonly v0+40
-;; @0055                               v43 = load.i64 notrap aligned readonly v0+48
-;; @0055                               v44 = bitcast.i64 v14
-;; @0055                               v45 = iconst.i64 8
-;; @0055                               v46 = uadd_overflow_trap v44, v45, user65535  ; v45 = 8
-;; @0055                               v47 = iconst.i64 8
-;; @0055                               v48 = uadd_overflow_trap v46, v47, user65535  ; v47 = 8
-;; @0055                               v49 = icmp ult v48, v43
-;; @0055                               brif v49, block13, block12
+;; @0055                               v38 = load.i64 notrap aligned readonly v0+40
+;; @0055                               v39 = load.i64 notrap aligned readonly v0+48
+;; @0055                               v40 = uextend.i64 v12
+;; @0055                               v41 = iconst.i64 8
+;; @0055                               v42 = uadd_overflow_trap v40, v41, user65535  ; v41 = 8
+;; @0055                               v43 = iconst.i64 8
+;; @0055                               v44 = uadd_overflow_trap v42, v43, user65535  ; v43 = 8
+;; @0055                               v45 = icmp ult v44, v39
+;; @0055                               brif v45, block13, block12
 ;;
 ;;                                 block12 cold:
 ;; @0055                               trap user65535
 ;;
 ;;                                 block13:
-;; @0055                               v50 = iadd.i64 v42, v46
-;; @0055                               v51 = load.i64 notrap aligned v50
-;;                                     v70 = iconst.i64 -1
-;; @0055                               v52 = iadd v51, v70  ; v70 = -1
-;;                                     v71 = iconst.i64 0
-;; @0055                               v53 = icmp eq v52, v71  ; v71 = 0
-;; @0055                               brif v53, block5, block6
+;; @0055                               v46 = iadd.i64 v38, v42
+;; @0055                               v47 = load.i64 notrap aligned v46
+;;                                     v67 = iconst.i64 -1
+;; @0055                               v48 = iadd v47, v67  ; v67 = -1
+;;                                     v68 = iconst.i64 0
+;; @0055                               v49 = icmp eq v48, v68  ; v68 = 0
+;; @0055                               brif v49, block5, block6
 ;;
 ;;                                 block5 cold:
-;; @0055                               v55 = bitcast.i64 v14
-;; @0055                               call fn0(v0, v55)
+;; @0055                               call fn0(v0, v12)
 ;; @0055                               jump block7
 ;;
 ;;                                 block6:
-;; @0055                               v57 = load.i64 notrap aligned readonly v0+40
-;; @0055                               v58 = load.i64 notrap aligned readonly v0+48
-;; @0055                               v59 = bitcast.i64 v14
-;; @0055                               v60 = iconst.i64 8
-;; @0055                               v61 = uadd_overflow_trap v59, v60, user65535  ; v60 = 8
-;; @0055                               v62 = iconst.i64 8
-;; @0055                               v63 = uadd_overflow_trap v61, v62, user65535  ; v62 = 8
-;; @0055                               v64 = icmp ult v63, v58
-;; @0055                               brif v64, block15, block14
+;; @0055                               v52 = load.i64 notrap aligned readonly v0+40
+;; @0055                               v53 = load.i64 notrap aligned readonly v0+48
+;; @0055                               v54 = uextend.i64 v12
+;; @0055                               v55 = iconst.i64 8
+;; @0055                               v56 = uadd_overflow_trap v54, v55, user65535  ; v55 = 8
+;; @0055                               v57 = iconst.i64 8
+;; @0055                               v58 = uadd_overflow_trap v56, v57, user65535  ; v57 = 8
+;; @0055                               v59 = icmp ult v58, v53
+;; @0055                               brif v59, block15, block14
 ;;
 ;;                                 block14 cold:
 ;; @0055                               trap user65535
 ;;
 ;;                                 block15:
-;; @0055                               v65 = iadd.i64 v57, v61
-;; @0055                               store.i64 notrap aligned v52, v65
+;; @0055                               v60 = iadd.i64 v52, v56
+;; @0055                               store.i64 notrap aligned v48, v60
 ;; @0055                               jump block7
 ;;
 ;;                                 block7:
@@ -142,122 +139,119 @@
 ;; @0057                               return
 ;; }
 ;;
-;; function u0:1(i64 vmctx, i64, i32, r64) tail {
+;; function u0:1(i64 vmctx, i64, i32, i32) tail {
 ;;     gv0 = vmctx
 ;;     gv1 = load.i64 notrap aligned readonly gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned gv3+88
 ;;     gv5 = load.i32 notrap aligned gv3+96
-;;     sig0 = (i64 vmctx, i64) system_v
+;;     sig0 = (i64 vmctx, i32 uext) system_v
 ;;     fn0 = colocated u1:25 sig0
 ;;     stack_limit = gv2
 ;;
-;;                                 block0(v0: i64, v1: i64, v2: i32, v3: r64):
+;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32):
 ;; @005e                               v4 = load.i32 notrap aligned v0+96
 ;; @005e                               v5 = icmp uge v2, v4
 ;; @005e                               v6 = uextend.i64 v2
 ;; @005e                               v7 = load.i64 notrap aligned v0+88
-;;                                     v68 = iconst.i64 2
-;; @005e                               v8 = ishl v6, v68  ; v68 = 2
+;;                                     v63 = iconst.i64 2
+;; @005e                               v8 = ishl v6, v63  ; v63 = 2
 ;; @005e                               v9 = iadd v7, v8
 ;; @005e                               v10 = iconst.i64 0
 ;; @005e                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
 ;; @005e                               v12 = load.i32 table_oob aligned table v11
-;; @005e                               v13 = uextend.i64 v12
-;; @005e                               v14 = bitcast.r64 v13
-;; @005e                               v15 = is_null v3
-;; @005e                               brif v15, block3, block2
+;;                                     v64 = iconst.i32 0
+;; @005e                               v13 = icmp eq v3, v64  ; v64 = 0
+;; @005e                               brif v13, block3, block2
 ;;
 ;;                                 block2:
-;; @005e                               v17 = load.i64 notrap aligned readonly v0+40
-;; @005e                               v18 = load.i64 notrap aligned readonly v0+48
-;; @005e                               v19 = bitcast.i64 v3
+;; @005e                               v15 = load.i64 notrap aligned readonly v0+40
+;; @005e                               v16 = load.i64 notrap aligned readonly v0+48
+;; @005e                               v17 = uextend.i64 v3
+;; @005e                               v18 = iconst.i64 8
+;; @005e                               v19 = uadd_overflow_trap v17, v18, user65535  ; v18 = 8
 ;; @005e                               v20 = iconst.i64 8
 ;; @005e                               v21 = uadd_overflow_trap v19, v20, user65535  ; v20 = 8
-;; @005e                               v22 = iconst.i64 8
-;; @005e                               v23 = uadd_overflow_trap v21, v22, user65535  ; v22 = 8
-;; @005e                               v24 = icmp ult v23, v18
-;; @005e                               brif v24, block9, block8
+;; @005e                               v22 = icmp ult v21, v16
+;; @005e                               brif v22, block9, block8
 ;;
 ;;                                 block8 cold:
 ;; @005e                               trap user65535
 ;;
 ;;                                 block9:
-;; @005e                               v25 = iadd.i64 v17, v21
-;; @005e                               v26 = load.i64 notrap aligned v25
-;;                                     v69 = iconst.i64 1
-;; @005e                               v27 = iadd v26, v69  ; v69 = 1
-;; @005e                               v29 = load.i64 notrap aligned readonly v0+40
-;; @005e                               v30 = load.i64 notrap aligned readonly v0+48
-;; @005e                               v31 = bitcast.i64 v3
+;; @005e                               v23 = iadd.i64 v15, v19
+;; @005e                               v24 = load.i64 notrap aligned v23
+;;                                     v65 = iconst.i64 1
+;; @005e                               v25 = iadd v24, v65  ; v65 = 1
+;; @005e                               v27 = load.i64 notrap aligned readonly v0+40
+;; @005e                               v28 = load.i64 notrap aligned readonly v0+48
+;; @005e                               v29 = uextend.i64 v3
+;; @005e                               v30 = iconst.i64 8
+;; @005e                               v31 = uadd_overflow_trap v29, v30, user65535  ; v30 = 8
 ;; @005e                               v32 = iconst.i64 8
 ;; @005e                               v33 = uadd_overflow_trap v31, v32, user65535  ; v32 = 8
-;; @005e                               v34 = iconst.i64 8
-;; @005e                               v35 = uadd_overflow_trap v33, v34, user65535  ; v34 = 8
-;; @005e                               v36 = icmp ult v35, v30
-;; @005e                               brif v36, block11, block10
+;; @005e                               v34 = icmp ult v33, v28
+;; @005e                               brif v34, block11, block10
 ;;
 ;;                                 block10 cold:
 ;; @005e                               trap user65535
 ;;
 ;;                                 block11:
-;; @005e                               v37 = iadd.i64 v29, v33
-;; @005e                               store.i64 notrap aligned v27, v37
+;; @005e                               v35 = iadd.i64 v27, v31
+;; @005e                               store.i64 notrap aligned v25, v35
 ;; @005e                               jump block3
 ;;
 ;;                                 block3:
-;; @005e                               v38 = bitcast.i64 v3
-;; @005e                               v39 = ireduce.i32 v38
-;; @005e                               store table_oob aligned table v39, v11
-;; @005e                               v40 = is_null.r64 v14
-;; @005e                               brif v40, block7, block4
+;; @005e                               store.i32 table_oob aligned table v3, v11
+;;                                     v66 = iconst.i32 0
+;; @005e                               v36 = icmp.i32 eq v12, v66  ; v66 = 0
+;; @005e                               brif v36, block7, block4
 ;;
 ;;                                 block4:
-;; @005e                               v42 = load.i64 notrap aligned readonly v0+40
-;; @005e                               v43 = load.i64 notrap aligned readonly v0+48
-;; @005e                               v44 = bitcast.i64 v14
-;; @005e                               v45 = iconst.i64 8
-;; @005e                               v46 = uadd_overflow_trap v44, v45, user65535  ; v45 = 8
-;; @005e                               v47 = iconst.i64 8
-;; @005e                               v48 = uadd_overflow_trap v46, v47, user65535  ; v47 = 8
-;; @005e                               v49 = icmp ult v48, v43
-;; @005e                               brif v49, block13, block12
+;; @005e                               v38 = load.i64 notrap aligned readonly v0+40
+;; @005e                               v39 = load.i64 notrap aligned readonly v0+48
+;; @005e                               v40 = uextend.i64 v12
+;; @005e                               v41 = iconst.i64 8
+;; @005e                               v42 = uadd_overflow_trap v40, v41, user65535  ; v41 = 8
+;; @005e                               v43 = iconst.i64 8
+;; @005e                               v44 = uadd_overflow_trap v42, v43, user65535  ; v43 = 8
+;; @005e                               v45 = icmp ult v44, v39
+;; @005e                               brif v45, block13, block12
 ;;
 ;;                                 block12 cold:
 ;; @005e                               trap user65535
 ;;
 ;;                                 block13:
-;; @005e                               v50 = iadd.i64 v42, v46
-;; @005e                               v51 = load.i64 notrap aligned v50
-;;                                     v70 = iconst.i64 -1
-;; @005e                               v52 = iadd v51, v70  ; v70 = -1
-;;                                     v71 = iconst.i64 0
-;; @005e                               v53 = icmp eq v52, v71  ; v71 = 0
-;; @005e                               brif v53, block5, block6
+;; @005e                               v46 = iadd.i64 v38, v42
+;; @005e                               v47 = load.i64 notrap aligned v46
+;;                                     v67 = iconst.i64 -1
+;; @005e                               v48 = iadd v47, v67  ; v67 = -1
+;;                                     v68 = iconst.i64 0
+;; @005e                               v49 = icmp eq v48, v68  ; v68 = 0
+;; @005e                               brif v49, block5, block6
 ;;
 ;;                                 block5 cold:
-;; @005e                               v55 = bitcast.i64 v14
-;; @005e                               call fn0(v0, v55)
+;; @005e                               call fn0(v0, v12)
 ;; @005e                               jump block7
 ;;
 ;;                                 block6:
-;; @005e                               v57 = load.i64 notrap aligned readonly v0+40
-;; @005e                               v58 = load.i64 notrap aligned readonly v0+48
-;; @005e                               v59 = bitcast.i64 v14
-;; @005e                               v60 = iconst.i64 8
-;; @005e                               v61 = uadd_overflow_trap v59, v60, user65535  ; v60 = 8
-;; @005e                               v62 = iconst.i64 8
-;; @005e                               v63 = uadd_overflow_trap v61, v62, user65535  ; v62 = 8
-;; @005e                               v64 = icmp ult v63, v58
-;; @005e                               brif v64, block15, block14
+;; @005e                               v52 = load.i64 notrap aligned readonly v0+40
+;; @005e                               v53 = load.i64 notrap aligned readonly v0+48
+;; @005e                               v54 = uextend.i64 v12
+;; @005e                               v55 = iconst.i64 8
+;; @005e                               v56 = uadd_overflow_trap v54, v55, user65535  ; v55 = 8
+;; @005e                               v57 = iconst.i64 8
+;; @005e                               v58 = uadd_overflow_trap v56, v57, user65535  ; v57 = 8
+;; @005e                               v59 = icmp ult v58, v53
+;; @005e                               brif v59, block15, block14
 ;;
 ;;                                 block14 cold:
 ;; @005e                               trap user65535
 ;;
 ;;                                 block15:
-;; @005e                               v65 = iadd.i64 v57, v61
-;; @005e                               store.i64 notrap aligned v52, v65
+;; @005e                               v60 = iadd.i64 v52, v56
+;; @005e                               store.i64 notrap aligned v48, v60
 ;; @005e                               jump block7
 ;;
 ;;                                 block7: