Repository: bytecodealliance/wasmtime
Pull request #8978 (2024-07-19 18:21:50)

`cranelift-frontend`: Support moving GCs with user stack maps
This refactors the way that user stack maps work such that they are compatible with moving GCs. We now keep every GC value in a stack slot, spill to the stack slot on the value's definition, and reload from the stack slot on every use. This means that we can generate a bunch of redundant loads, but we let the mid-end's alias analysis clean that up. This has the added benefit of reducing the length of live ranges for GC values and also means we don't need to proactively spill every single live GC values at each safepoint, because we know they are already spilled. This latter bit actually helps us avoid an accidentally quadratic issue with many, long, overlapping live ranges where we would do `O(n^2)` spills for a series of safepoints that keep creating more GC refs.

We also implement two further optimizations:

1. We lazily allocate slots for live GC values, which means that if a GC value is not ever live across a safepoint, then we do never allocate a stack slot for it and never spill it to the stack slot. If we didn't do this, then frames would be larger and we'd have a dead store to the stack slot that would otherwise require the mid-end to grow a dead-store-elimination pass.

2. We keep a free list of available slots that will never be used again, and we reuse slots from here when possible. This means that a chain of non-overlapping live ranges, each of which still needs to appear in some safepoint's stack map, will all reuse the same stack slot, keeping frames from being bloated.

Finally, this commit also introduces some logs for the liveness analysis to ease future debugging.

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->



Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 3576a2b48..6d29899f4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -674,6 +674,7 @@ name = "cranelift-frontend"
 version = "0.111.0"
 dependencies = [
  "cranelift-codegen",
+ "env_logger",
  "hashbrown 0.14.3",
  "log",
  "similar",
diff --git a/Cargo.toml b/Cargo.toml
index 5b94be104..130227eae 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,7 +15,7 @@ rust-version.workspace = true
 
 [package.metadata.binstall]
 pkg-url = "{repo}/releases/download/v{version}/wasmtime-v{version}-{target-arch}-{target-family}{archive-suffix}"
-bin-dir = "wasmtime-{version}-{target-arch}-{target-family}/{bin}{binary-ext}"
+bin-dir = "wasmtime-v{version}-{target-arch}-{target-family}/{bin}{binary-ext}"
 pkg-fmt = "txz"
 [package.metadata.binstall.overrides.x86_64-apple-darwin]
 pkg-url = "{repo}/releases/download/v{version}/wasmtime-v{version}-{target-arch}-macos{archive-suffix}"
diff --git a/cranelift/bitset/Cargo.toml b/cranelift/bitset/Cargo.toml
index 0731e450b..3401f6705 100644
--- a/cranelift/bitset/Cargo.toml
+++ b/cranelift/bitset/Cargo.toml
@@ -6,7 +6,6 @@ description = "Various bitset stuff for use inside Cranelift"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift-bitset"
 repository = "https://github.com/bytecodealliance/wasmtime"
-readme = "README.md"
 edition.workspace = true
 
 [lints]
diff --git a/cranelift/codegen/src/ir/immediates.rs b/cranelift/codegen/src/ir/immediates.rs
index ffc212e41..24982c644 100644
--- a/cranelift/codegen/src/ir/immediates.rs
+++ b/cranelift/codegen/src/ir/immediates.rs
@@ -915,6 +915,11 @@ impl Ieee32 {
         self.as_f32().is_nan()
     }
 
+    /// Returns `None` if `self` is a NaN and `Some(self)` otherwise.
+    pub fn non_nan(self) -> Option<Self> {
+        Some(self).filter(|f| !f.is_nan())
+    }
+
     /// Converts Self to a rust f32
     pub fn as_f32(self) -> f32 {
         f32::from_bits(self.0)
@@ -927,17 +932,21 @@ impl Ieee32 {
 
     /// Computes the absolute value of self.
     pub fn abs(self) -> Self {
-        Self::with_float(self.as_f32().abs())
+        Self(self.0 & !(1u32 << 31))
     }
 
     /// Returns a number composed of the magnitude of self and the sign of sign.
     pub fn copysign(self, sign: Self) -> Self {
-        Self::with_float(self.as_f32().copysign(sign.as_f32()))
+        if self.is_negative() == sign.is_negative() {
+            self
+        } else {
+            self.neg()
+        }
     }
 
     /// Returns true if self has a negative sign, including -0.0, NaNs with negative sign bit and negative infinity.
     pub fn is_negative(&self) -> bool {
-        self.as_f32().is_sign_negative()
+        self.0 & (1 << 31) != 0
     }
 
     /// Returns true if self is positive or negative zero
@@ -963,17 +972,7 @@ impl Ieee32 {
     /// Returns the nearest integer to `self`. Rounds half-way cases to the number
     /// with an even least significant digit.
     pub fn round_ties_even(self) -> Self {
-        // TODO: Replace with the native implementation once
-        // https://github.com/rust-lang/rust/issues/96710 is stabilized
-        let toint_32: f32 = 1.0 / f32::EPSILON;
-
-        let f = self.as_f32();
-        let e = self.0 >> 23 & 0xff;
-        if e >= 0x7f_u32 + 23 {
-            self
-        } else {
-            Self::with_float((f.abs() + toint_32 - toint_32).copysign(f))
-        }
+        Self::with_float(self.as_f32().round_ties_even())
     }
 }
 
@@ -1017,7 +1016,7 @@ impl Neg for Ieee32 {
     type Output = Ieee32;
 
     fn neg(self) -> Self::Output {
-        Self::with_float(self.as_f32().neg())
+        Self(self.0 ^ (1 << 31))
     }
 }
 
@@ -1133,6 +1132,11 @@ impl Ieee64 {
         self.as_f64().is_nan()
     }
 
+    /// Returns `None` if `self` is a NaN and `Some(self)` otherwise.
+    pub fn non_nan(self) -> Option<Self> {
+        Some(self).filter(|f| !f.is_nan())
+    }
+
     /// Converts Self to a rust f64
     pub fn as_f64(self) -> f64 {
         f64::from_bits(self.0)
@@ -1145,17 +1149,21 @@ impl Ieee64 {
 
     /// Computes the absolute value of self.
     pub fn abs(self) -> Self {
-        Self::with_float(self.as_f64().abs())
+        Self(self.0 & !(1u64 << 63))
     }
 
     /// Returns a number composed of the magnitude of self and the sign of sign.
     pub fn copysign(self, sign: Self) -> Self {
-        Self::with_float(self.as_f64().copysign(sign.as_f64()))
+        if self.is_negative() == sign.is_negative() {
+            self
+        } else {
+            self.neg()
+        }
     }
 
     /// Returns true if self has a negative sign, including -0.0, NaNs with negative sign bit and negative infinity.
     pub fn is_negative(&self) -> bool {
-        self.as_f64().is_sign_negative()
+        self.0 & (1 << 63) != 0
     }
 
     /// Returns true if self is positive or negative zero
@@ -1181,17 +1189,7 @@ impl Ieee64 {
     /// Returns the nearest integer to `self`. Rounds half-way cases to the number
     /// with an even least significant digit.
     pub fn round_ties_even(self) -> Self {
-        // TODO: Replace with the native implementation once
-        // https://github.com/rust-lang/rust/issues/96710 is stabilized
-        let toint_64: f64 = 1.0 / f64::EPSILON;
-
-        let f = self.as_f64();
-        let e = self.0 >> 52 & 0x7ff_u64;
-        if e >= 0x3ff_u64 + 52 {
-            self
-        } else {
-            Self::with_float((f.abs() + toint_64 - toint_64).copysign(f))
-        }
+        Self::with_float(self.as_f64().round_ties_even())
     }
 }
 
@@ -1241,7 +1239,7 @@ impl Neg for Ieee64 {
     type Output = Ieee64;
 
     fn neg(self) -> Self::Output {
-        Self::with_float(self.as_f64().neg())
+        Self(self.0 ^ (1 << 63))
     }
 }
 
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 25534059d..5c5ea82c1 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -165,9 +165,9 @@ pub(crate) fn emit(
             let mut rex = RexFlags::from(*size);
             let (opcode_r, opcode_m, subopcode_i) = match op {
                 AluRmiROpcode::Add => (0x01, 0x03, 0),
-                AluRmiROpcode::Adc => (0x11, 0x03, 0),
+                AluRmiROpcode::Adc => (0x11, 0x03, 2),
                 AluRmiROpcode::Sub => (0x29, 0x2B, 5),
-                AluRmiROpcode::Sbb => (0x19, 0x2B, 5),
+                AluRmiROpcode::Sbb => (0x19, 0x2B, 3),
                 AluRmiROpcode::And => (0x21, 0x23, 4),
                 AluRmiROpcode::Or => (0x09, 0x0B, 1),
                 AluRmiROpcode::Xor => (0x31, 0x33, 6),
diff --git a/cranelift/codegen/src/isle_prelude.rs b/cranelift/codegen/src/isle_prelude.rs
index f36c4bdb0..530cfb139 100644
--- a/cranelift/codegen/src/isle_prelude.rs
+++ b/cranelift/codegen/src/isle_prelude.rs
@@ -950,6 +950,62 @@ macro_rules! isle_common_prelude_methods {
             a.copysign(b)
         }
 
+        fn f32_add(&mut self, lhs: Ieee32, rhs: Ieee32) -> Option<Ieee32> {
+            (lhs + rhs).non_nan()
+        }
+
+        fn f32_sub(&mut self, lhs: Ieee32, rhs: Ieee32) -> Option<Ieee32> {
+            (lhs - rhs).non_nan()
+        }
+
+        fn f32_mul(&mut self, lhs: Ieee32, rhs: Ieee32) -> Option<Ieee32> {
+            (lhs * rhs).non_nan()
+        }
+
+        fn f32_div(&mut self, lhs: Ieee32, rhs: Ieee32) -> Option<Ieee32> {
+            (lhs / rhs).non_nan()
+        }
+
+        fn f32_sqrt(&mut self, n: Ieee32) -> Option<Ieee32> {
+            n.sqrt().non_nan()
+        }
+
+        fn f32_ceil(&mut self, n: Ieee32) -> Option<Ieee32> {
+            n.ceil().non_nan()
+        }
+
+        fn f32_floor(&mut self, n: Ieee32) -> Option<Ieee32> {
+            n.floor().non_nan()
+        }
+
+        fn f32_trunc(&mut self, n: Ieee32) -> Option<Ieee32> {
+            n.trunc().non_nan()
+        }
+
+        fn f32_nearest(&mut self, n: Ieee32) -> Option<Ieee32> {
+            n.round_ties_even().non_nan()
+        }
+
+        fn f32_min(&mut self, a: Ieee32, b: Ieee32) -> Option<Ieee32> {
+            if a.is_nan() || b.is_nan() {
+                None
+            } else if a <= b {
+                Some(a)
+            } else {
+                Some(b)
+            }
+        }
+
+        fn f32_max(&mut self, a: Ieee32, b: Ieee32) -> Option<Ieee32> {
+            if a.is_nan() || b.is_nan() {
+                None
+            } else if a >= b {
+                Some(a)
+            } else {
+                Some(b)
+            }
+        }
+
         fn f32_neg(&mut self, n: Ieee32) -> Ieee32 {
             n.neg()
         }
@@ -962,6 +1018,62 @@ macro_rules! isle_common_prelude_methods {
             a.copysign(b)
         }
 
+        fn f64_add(&mut self, lhs: Ieee64, rhs: Ieee64) -> Option<Ieee64> {
+            (lhs + rhs).non_nan()
+        }
+
+        fn f64_sub(&mut self, lhs: Ieee64, rhs: Ieee64) -> Option<Ieee64> {
+            (lhs - rhs).non_nan()
+        }
+
+        fn f64_mul(&mut self, lhs: Ieee64, rhs: Ieee64) -> Option<Ieee64> {
+            (lhs * rhs).non_nan()
+        }
+
+        fn f64_div(&mut self, lhs: Ieee64, rhs: Ieee64) -> Option<Ieee64> {
+            (lhs / rhs).non_nan()
+        }
+
+        fn f64_sqrt(&mut self, n: Ieee64) -> Option<Ieee64> {
+            n.sqrt().non_nan()
+        }
+
+        fn f64_ceil(&mut self, n: Ieee64) -> Option<Ieee64> {
+            n.ceil().non_nan()
+        }
+
+        fn f64_floor(&mut self, n: Ieee64) -> Option<Ieee64> {
+            n.floor().non_nan()
+        }
+
+        fn f64_trunc(&mut self, n: Ieee64) -> Option<Ieee64> {
+            n.trunc().non_nan()
+        }
+
+        fn f64_nearest(&mut self, n: Ieee64) -> Option<Ieee64> {
+            n.round_ties_even().non_nan()
+        }
+
+        fn f64_min(&mut self, a: Ieee64, b: Ieee64) -> Option<Ieee64> {
+            if a.is_nan() || b.is_nan() {
+                None
+            } else if a <= b {
+                Some(a)
+            } else {
+                Some(b)
+            }
+        }
+
+        fn f64_max(&mut self, a: Ieee64, b: Ieee64) -> Option<Ieee64> {
+            if a.is_nan() || b.is_nan() {
+                None
+            } else if a >= b {
+                Some(a)
+            } else {
+                Some(b)
+            }
+        }
+
         fn f64_neg(&mut self, n: Ieee64) -> Ieee64 {
             n.neg()
         }
diff --git a/cranelift/codegen/src/opts/cprop.isle b/cranelift/codegen/src/opts/cprop.isle
index 9b31b0e7e..a9685be75 100644
--- a/cranelift/codegen/src/opts/cprop.isle
+++ b/cranelift/codegen/src/opts/cprop.isle
@@ -182,8 +182,6 @@
       (if-let $true (u64_lt (i64_as_u64 (i64_neg k)) (i64_as_u64 k)))
       (iadd ty x (iconst ty (imm64_masked ty (i64_as_u64 (i64_neg k))))))
 
-;; TODO: fadd, fsub, fmul, fdiv, fneg, fabs
-
 ;; A splat of a constant can become a direct `vconst` with the appropriate bit
 ;; pattern.
 (rule (simplify (splat dst (iconst $I8 n)))
@@ -280,7 +278,92 @@
 (decl pure u64_bswap64 (u64) u64)
 (extern constructor u64_bswap64 u64_bswap64)
 
-;; Constant fold bitwise float operations (fneg/fabs/fcopysign)
+;; Constant fold float operations
+;; Note: With the exception of fabs, fneg and copysign,
+;; constant folding is only performed when the result of
+;; an instruction isn't NaN. We want the NaN bit patterns
+;; produced by an instruction to be consistent, and
+;; compile-time evaluation in a cross-compilation scenario
+;; risks producing different NaN bit patterns than the target
+;; would have at run-time.
+;; TODO: fcmp, fma, demote, promote, to-int ops
+(rule (simplify (fadd $F32 (f32const $F32 lhs) (f32const $F32 rhs)))
+      (if-let r (f32_add lhs rhs))
+      (subsume (f32const $F32 r)))
+(rule (simplify (fadd $F64 (f64const $F64 lhs) (f64const $F64 rhs)))
+      (if-let r (f64_add lhs rhs))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (fsub $F32 (f32const $F32 lhs) (f32const $F32 rhs)))
+      (if-let r (f32_sub lhs rhs))
+      (subsume (f32const $F32 r)))
+(rule (simplify (fsub $F64 (f64const $F64 lhs) (f64const $F64 rhs)))
+      (if-let r (f64_sub lhs rhs))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (fmul $F32 (f32const $F32 lhs) (f32const $F32 rhs)))
+      (if-let r (f32_mul lhs rhs))
+      (subsume (f32const $F32 r)))
+(rule (simplify (fmul $F64 (f64const $F64 lhs) (f64const $F64 rhs)))
+      (if-let r (f64_mul lhs rhs))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (fdiv $F32 (f32const $F32 lhs) (f32const $F32 rhs)))
+      (if-let r (f32_div lhs rhs))
+      (subsume (f32const $F32 r)))
+(rule (simplify (fdiv $F64 (f64const $F64 lhs) (f64const $F64 rhs)))
+      (if-let r (f64_div lhs rhs))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (sqrt $F32 (f32const $F32 n)))
+      (if-let r (f32_sqrt n))
+      (subsume (f32const $F32 r)))
+(rule (simplify (sqrt $F64 (f64const $F64 n)))
+      (if-let r (f64_sqrt n))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (ceil $F32 (f32const $F32 n)))
+      (if-let r (f32_ceil n))
+      (subsume (f32const $F32 r)))
+(rule (simplify (ceil $F64 (f64const $F64 n)))
+      (if-let r (f64_ceil n))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (floor $F32 (f32const $F32 n)))
+      (if-let r (f32_floor n))
+      (subsume (f32const $F32 r)))
+(rule (simplify (floor $F64 (f64const $F64 n)))
+      (if-let r (f64_floor n))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (trunc $F32 (f32const $F32 n)))
+      (if-let r (f32_trunc n))
+      (subsume (f32const $F32 r)))
+(rule (simplify (trunc $F64 (f64const $F64 n)))
+      (if-let r (f64_trunc n))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (nearest $F32 (f32const $F32 n)))
+      (if-let r (f32_nearest n))
+      (subsume (f32const $F32 r)))
+(rule (simplify (nearest $F64 (f64const $F64 n)))
+      (if-let r (f64_nearest n))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (fmin $F32 (f32const $F32 n) (f32const $F32 m)))
+      (if-let r (f32_min n m))
+      (subsume (f32const $F32 r)))
+(rule (simplify (fmin $F64 (f64const $F64 n) (f64const $F64 m)))
+      (if-let r (f64_min n m))
+      (subsume (f64const $F64 r)))
+
+(rule (simplify (fmax $F32 (f32const $F32 n) (f32const $F32 m)))
+      (if-let r (f32_max n m))
+      (subsume (f32const $F32 r)))
+(rule (simplify (fmax $F64 (f64const $F64 n) (f64const $F64 m)))
+      (if-let r (f64_max n m))
+      (subsume (f64const $F64 r)))
+
 (rule (simplify (fneg $F16 (f16const $F16 n)))
       (subsume (f16const $F16 (f16_neg n))))
 (rule (simplify (fneg $F32 (f32const $F32 n)))
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index 95f31bfdb..233ab899e 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -245,12 +245,56 @@
 (extern constructor f16_abs f16_abs)
 (decl pure f16_copysign (Ieee16 Ieee16) Ieee16)
 (extern constructor f16_copysign f16_copysign)
+(decl pure partial f32_add (Ieee32 Ieee32) Ieee32)
+(extern constructor f32_add f32_add)
+(decl pure partial f32_sub (Ieee32 Ieee32) Ieee32)
+(extern constructor f32_sub f32_sub)
+(decl pure partial f32_mul (Ieee32 Ieee32) Ieee32)
+(extern constructor f32_mul f32_mul)
+(decl pure partial f32_div (Ieee32 Ieee32) Ieee32)
+(extern constructor f32_div f32_div)
+(decl pure partial f32_sqrt (Ieee32) Ieee32)
+(extern constructor f32_sqrt f32_sqrt)
+(decl pure partial f32_ceil (Ieee32) Ieee32)
+(extern constructor f32_ceil f32_ceil)
+(decl pure partial f32_floor (Ieee32) Ieee32)
+(extern constructor f32_floor f32_floor)
+(decl pure partial f32_trunc (Ieee32) Ieee32)
+(extern constructor f32_trunc f32_trunc)
+(decl pure partial f32_nearest (Ieee32) Ieee32)
+(extern constructor f32_nearest f32_nearest)
+(decl pure partial f32_min (Ieee32 Ieee32) Ieee32)
+(extern constructor f32_min f32_min)
+(decl pure partial f32_max (Ieee32 Ieee32) Ieee32)
+(extern constructor f32_max f32_max)
 (decl pure f32_neg (Ieee32) Ieee32)
 (extern constructor f32_neg f32_neg)
 (decl pure f32_abs (Ieee32) Ieee32)
 (extern constructor f32_abs f32_abs)
 (decl pure f32_copysign (Ieee32 Ieee32) Ieee32)
 (extern constructor f32_copysign f32_copysign)
+(decl pure partial f64_add (Ieee64 Ieee64) Ieee64)
+(extern constructor f64_add f64_add)
+(decl pure partial f64_sub (Ieee64 Ieee64) Ieee64)
+(extern constructor f64_sub f64_sub)
+(decl pure partial f64_mul (Ieee64 Ieee64) Ieee64)
+(extern constructor f64_mul f64_mul)
+(decl pure partial f64_div (Ieee64 Ieee64) Ieee64)
+(extern constructor f64_div f64_div)
+(decl pure partial f64_sqrt (Ieee64) Ieee64)
+(extern constructor f64_sqrt f64_sqrt)
+(decl pure partial f64_ceil (Ieee64) Ieee64)
+(extern constructor f64_ceil f64_ceil)
+(decl pure partial f64_floor (Ieee64) Ieee64)
+(extern constructor f64_floor f64_floor)
+(decl pure partial f64_trunc (Ieee64) Ieee64)
+(extern constructor f64_trunc f64_trunc)
+(decl pure partial f64_nearest (Ieee64) Ieee64)
+(extern constructor f64_nearest f64_nearest)
+(decl pure partial f64_min (Ieee64 Ieee64) Ieee64)
+(extern constructor f64_min f64_min)
+(decl pure partial f64_max (Ieee64 Ieee64) Ieee64)
+(extern constructor f64_max f64_max)
 (decl pure f64_neg (Ieee64) Ieee64)
 (extern constructor f64_neg f64_neg)
 (decl pure f64_abs (Ieee64) Ieee64)
diff --git a/cranelift/frontend/Cargo.toml b/cranelift/frontend/Cargo.toml
index 3155defa0..51b207537 100644
--- a/cranelift/frontend/Cargo.toml
+++ b/cranelift/frontend/Cargo.toml
@@ -21,6 +21,7 @@ hashbrown = { workspace = true, optional = true }
 smallvec = { workspace = true }
 
 [dev-dependencies]
+env_logger = { workspace = true }
 similar = { workspace = true }
 cranelift-codegen = { workspace = true, features = ['x86'] }
 
diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs
index ac0909e98..ed77944d8 100644
--- a/cranelift/frontend/src/frontend.rs
+++ b/cranelift/frontend/src/frontend.rs
@@ -2,8 +2,9 @@
 use crate::ssa::{SSABuilder, SideEffects};
 use crate::variable::Variable;
 use alloc::collections::BTreeSet;
+use alloc::vec::Vec;
 use core::fmt::{self, Debug};
-use cranelift_codegen::cursor::{Cursor, FuncCursor};
+use cranelift_codegen::cursor::{Cursor, CursorPosition, FuncCursor};
 use cranelift_codegen::entity::{EntityRef, EntitySet, SecondaryMap};
 use cranelift_codegen::ir;
 use cranelift_codegen::ir::condcodes::IntCC;
@@ -272,8 +273,6 @@ impl fmt::Display for DefVariableError {
     }
 }
 
-const LOG2_SIZE_CAPACITY: usize = (16u8.ilog2() as usize) + 1;
-
 /// This module allows you to create a function in Cranelift IR in a straightforward way, hiding
 /// all the complexity of its internal representation.
 ///
@@ -504,6 +503,11 @@ impl<'a> FunctionBuilder<'a> {
             return Err(DefVariableError::TypeMismatch(var, val));
         }
 
+        // If `var` needs inclusion in stack maps, then `val` does too.
+        if self.func_ctx.stack_map_vars.contains(var) {
+            self.declare_value_needs_stack_map(val);
+        }
+
         self.func_ctx.ssa.def_var(var, val, self.position.unwrap());
         Ok(())
     }
@@ -704,45 +708,46 @@ impl<'a> FunctionBuilder<'a> {
     /// Third, and finally, we spill the live needs-stack-map values at each
     /// safepoint into those stack slots.
     fn insert_safepoint_spills(&mut self) {
-        // Find all the GC values that are live across each safepoint.
-        let (safepoints, max_vals_in_stack_map_by_log2_size) =
-            self.find_live_stack_map_values_at_each_safepoint();
+        log::trace!(
+            "before inserting safepoint spills and reloads:\n{}",
+            self.func.display()
+        );
 
-        // Create the stack slots to spill them into.
-        let stack_slots = self.create_safepoint_slots(max_vals_in_stack_map_by_log2_size);
+        // Find all the GC values that are live across each safepoint and add
+        // stack map entries for them.
+        let stack_slots = self.find_live_stack_map_values_at_each_safepoint();
 
         // Insert spills to our new stack slots before each safepoint
         // instruction.
-        self.insert_safepoint_spills_to_stack_slots(safepoints, stack_slots);
+        self.insert_safepoint_spills_and_reloads(&stack_slots);
+
+        log::trace!(
+            "after inserting safepoint spills and reloads:\n{}",
+            self.func.display()
+        );
     }
 
     /// Find the live GC references for each safepoint instruction in this
     /// function.
     ///
-    /// Returns a pair of
-    ///
-    /// 1. A map from each safepoint instruction to the set of GC references
-    ///    that are live across it
-    ///
-    /// 2. The maximum number of values we need to store in a stack map at the
-    ///    same time, bucketed by their type's size. This array is indexed by
-    ///    the log2 of the type's size.
+    /// Returns a map from each safepoint instruction to the set of GC
+    /// references that are live across it
     fn find_live_stack_map_values_at_each_safepoint(
         &mut self,
-    ) -> (
-        crate::HashMap<ir::Inst, SmallVec<[ir::Value; 4]>>,
-        [usize; LOG2_SIZE_CAPACITY],
-    ) {
-        // A map from each safepoint to the set of GC references that are live
-        // across it.
-        let mut safepoints: crate::HashMap<ir::Inst, SmallVec<[ir::Value; 4]>> =
-            crate::HashMap::new();
-
-        // The maximum number of values we need to store in a stack map at the
-        // same time, bucketed by their type's size. This array is indexed by
-        // the log2 of the type's size. We do not support recording values whose
-        // size is greater than 16 in stack maps.
-        let mut max_vals_in_stack_map_by_log2_size = [0; LOG2_SIZE_CAPACITY];
+    ) -> crate::HashMap<ir::Value, ir::StackSlot> {
+        // A mapping from each needs-stack-map value that is live across some
+        // safepoint to the stack slot that it resides within. Note that if a
+        // needs-stack-map value is never live across a safepoint, then we won't
+        // ever add it to this map, it can remain in a virtual register for the
+        // duration of its lifetime, and we won't replace all its uses with
+        // reloads and all that stuff.
+        let mut stack_slots: crate::HashMap<ir::Value, ir::StackSlot> = Default::default();
+
+        // A map from size/align to free stack slots that are not being used
+        // anymore. This allows us to reuse stack slots across multiple values
+        // helps cut down on the ultimate size of our stack frames.
+        let mut free_stack_slots: crate::HashMap<u32, SmallVec<[ir::StackSlot; 4]>> =
+            Default::default();
 
         // The set of needs-stack-maps values that are currently live in our
         // traversal.
@@ -760,9 +765,8 @@ impl<'a> FunctionBuilder<'a> {
         // 1. The definition of a value removes it from our `live` set. Values
         //    are not live before they are defined.
         //
-        // 2. When we see any instruction that requires a safepoint (aka
-        //    non-tail calls) we record the current live set of needs-stack-map
-        //    values.
+        // 2. When we see any instruction that is a safepoint (aka non-tail
+        //    calls) we record the current live set of needs-stack-map values.
         //
         //    We ignore tail calls because this caller and its frame won't exist
         //    by the time the callee is executing and potentially triggers a GC;
@@ -791,28 +795,117 @@ impl<'a> FunctionBuilder<'a> {
         //    simplest thing. Besides, none of our mid-end optimization passes
         //    have run at this point in time yet, so there probably isn't much,
         //    if any, dead code.
-        for block in self.func_ctx.dfs.post_order_iter(&self.func) {
+
+        // Helper for (1)
+        let process_def = |func: &Function,
+                           stack_slots: &crate::HashMap<_, _>,
+                           free_stack_slots: &mut crate::HashMap<u32, SmallVec<_>>,
+                           live: &mut BTreeSet<ir::Value>,
+                           val: ir::Value| {
+            log::trace!("stack map liveness:   defining {val:?}, removing it from the live set");
+            live.remove(&val);
+
+            // This value's stack slot, if any, is now available for reuse.
+            if let Some(slot) = stack_slots.get(&val) {
+                log::trace!("stack map liveness:     returning {slot:?} to the free list");
+                let ty = func.dfg.value_type(val);
+                free_stack_slots.entry(ty.bytes()).or_default().push(*slot);
+            }
+        };
+
+        // Helper for (2)
+        let process_safepoint = |func: &mut Function,
+                                 stack_slots: &mut crate::HashMap<Value, StackSlot>,
+                                 free_stack_slots: &mut crate::HashMap<u32, SmallVec<_>>,
+                                 live: &BTreeSet<_>,
+                                 inst: Inst| {
+            log::trace!(
+                "stack map liveness:   found safepoint: {inst:?}: {}",
+                func.dfg.display_inst(inst)
+            );
+            log::trace!("stack map liveness:     live set = {live:?}");
+
+            for val in live {
+                let ty = func.dfg.value_type(*val);
+                let slot = *stack_slots.entry(*val).or_insert_with(|| {
+                    log::trace!("stack map liveness:     {val:?} needs a stack slot");
+                    let size = func.dfg.value_type(*val).bytes();
+                    free_stack_slots
+                        .get_mut(&size)
+                        .and_then(|list| list.pop().inspect(|slot| {
+                            log::trace!(
+                                "stack map liveness:       reusing free stack slot {slot:?} for {val:?}"
+                            )
+                        }))
+                        .unwrap_or_else(|| {
+                            debug_assert!(size.is_power_of_two());
+                            let log2_size = size.ilog2();
+                            let slot = func.create_sized_stack_slot(ir::StackSlotData::new(
+                                ir::StackSlotKind::ExplicitSlot,
+                                size,
+                                log2_size.try_into().unwrap(),
+                            ));
+                            log::trace!(
+                                "stack map liveness:       created new stack slot {slot:?} for {val:?}"
+                            );
+                            slot
+                        })
+                });
+                func.dfg.append_user_stack_map_entry(
+                    inst,
+                    ir::UserStackMapEntry {
+                        ty,
+                        slot,
+                        offset: 0,
+                    },
+                );
+            }
+        };
+
+        // Helper for (3)
+        let process_use = |func: &Function, live: &mut BTreeSet<_>, inst: Inst, val: Value| {
+            if live.insert(val) {
+                log::trace!(
+                    "stack map liveness:   found use of {val:?}, marking it live: {inst:?}: {}",
+                    func.dfg.display_inst(inst)
+                );
+            }
+        };
+
+        for block in self
+            .func_ctx
+            .dfs
+            .post_order_iter(&self.func)
+            // We have to `collect` here to release the borrow on `self.func` so
+            // we can add the stack map entries below.
+            .collect::<Vec<_>>()
+        {
+            log::trace!("stack map liveness: traversing {block:?}");
             let mut option_inst = self.func.layout.last_inst(block);
             while let Some(inst) = option_inst {
                 // (1) Remove values defined by this instruction from the `live`
                 // set.
                 for val in self.func.dfg.inst_results(inst) {
-                    live.remove(val);
+                    process_def(
+                        &self.func,
+                        &stack_slots,
+                        &mut free_stack_slots,
+                        &mut live,
+                        *val,
+                    );
                 }
 
-                // (2) If this instruction is a call, then we need to add a
-                // safepoint to record any values in `live`.
+                // (2) If this instruction is a safepoint, then we need to add
+                // stack map entries to record the values in `live`.
                 let opcode = self.func.dfg.insts[inst].opcode();
                 if opcode.is_call() && !opcode.is_return() {
-                    let mut live: SmallVec<[_; 4]> = live.iter().copied().collect();
-                    for chunk in live_vals_by_size(&self.func.dfg, &mut live) {
-                        let index = log2_size(&self.func.dfg, chunk[0]);
-                        max_vals_in_stack_map_by_log2_size[index] =
-                            core::cmp::max(max_vals_in_stack_map_by_log2_size[index], chunk.len());
-                    }
-
-                    let old_val = safepoints.insert(inst, live);
-                    debug_assert!(old_val.is_none());
+                    process_safepoint(
+                        &mut self.func,
+                        &mut stack_slots,
+                        &mut free_stack_slots,
+                        &live,
+                        inst,
+                    );
                 }
 
                 // (3) Add all needs-stack-map values that are operands to this
@@ -821,7 +914,7 @@ impl<'a> FunctionBuilder<'a> {
                 for val in self.func.dfg.inst_values(inst) {
                     let val = self.func.dfg.resolve_aliases(val);
                     if self.func_ctx.stack_map_values.contains(val) {
-                        live.insert(val);
+                        process_use(&self.func, &mut live, inst, val);
                     }
                 }
 
@@ -831,74 +924,90 @@ impl<'a> FunctionBuilder<'a> {
             // After we've processed this block's instructions, remove its
             // parameters from the live set. This is part of step (1).
             for val in self.func.dfg.block_params(block) {
-                live.remove(val);
+                process_def(
+                    &self.func,
+                    &stack_slots,
+                    &mut free_stack_slots,
+                    &mut live,
+                    *val,
+                );
             }
         }
 
-        (safepoints, max_vals_in_stack_map_by_log2_size)
+        stack_slots
     }
 
-    /// Create a stack slot for each size of needs-stack-map value.
+    /// This function does a forwards pass over the IR and does two things:
     ///
-    /// These slots are arrays capable of holding the maximum number of
-    /// same-sized values that must appear in the same stack map at the same
-    /// time.
+    /// 1. Insert spills to a needs-stack-map value's associated stack slot just
+    ///    after its definition.
     ///
-    /// The resulting array of stack slots is indexed by the log2 of the type
-    /// size.
-    fn create_safepoint_slots(
+    /// 2. Replace all uses of the needs-stack-map value with loads from that
+    ///    stack slot. This will introduce many redundant loads, but the alias
+    ///    analysis pass in the mid-end should clean most of these up when not
+    ///    actually needed.
+    fn insert_safepoint_spills_and_reloads(
         &mut self,
-        max_vals_in_stack_map_by_log2_size: [usize; LOG2_SIZE_CAPACITY],
-    ) -> [PackedOption<ir::StackSlot>; LOG2_SIZE_CAPACITY] {
-        let mut stack_slots = [PackedOption::<ir::StackSlot>::default(); LOG2_SIZE_CAPACITY];
-        for (log2_size, capacity) in max_vals_in_stack_map_by_log2_size.into_iter().enumerate() {
-            if capacity == 0 {
-                continue;
-            }
-            let size = 1usize << log2_size;
-            let slot = self.func.create_sized_stack_slot(ir::StackSlotData::new(
-                ir::StackSlotKind::ExplicitSlot,
-                u32::try_from(size * capacity).unwrap(),
-                u8::try_from(log2_size).unwrap(),
-            ));
-            stack_slots[log2_size] = Some(slot).into();
-        }
-        stack_slots
-    }
-
-    /// Insert spills to the given stack slots before each safepoint
-    /// instruction.
-    fn insert_safepoint_spills_to_stack_slots(
-        &mut self,
-        safepoints: crate::HashMap<ir::Inst, SmallVec<[ir::Value; 4]>>,
-        stack_slots: [PackedOption<ir::StackSlot>; LOG2_SIZE_CAPACITY],
+        stack_slots: &crate::HashMap<ir::Value, ir::StackSlot>,
     ) {
-        let mut cursor = FuncCursor::new(self.func);
-        for (inst, live_vals) in safepoints {
-            cursor = cursor.at_inst(inst);
+        let mut pos = FuncCursor::new(self.func);
+        let mut vals: SmallVec<[_; 8]> = Default::default();
 
-            // The offset within each stack slot for the next spill to that
+        while let Some(block) = pos.next_block() {
+            // Spill needs-stack-map values defined by block parameters to their
             // associated stack slot.
-            let mut stack_slot_offsets = [0; LOG2_SIZE_CAPACITY];
-
-            for val in live_vals {
-                let ty = cursor.func.dfg.value_type(val);
-                let size_of_val = ty.bytes();
+            vals.extend_from_slice(pos.func.dfg.block_params(block));
+            pos.next_inst();
+            let mut spilled_any = false;
+            for val in vals.drain(..) {
+                if let Some(slot) = stack_slots.get(&val) {
+                    pos.ins().stack_store(val, *slot, 0);
+                    spilled_any = true;
+                }
+            }
 
-                let index = log2_size(&cursor.func.dfg, val);
-                let slot = stack_slots[index].unwrap();
+            // The cursor needs to point just *before* the first original
+            // instruction in the block that we didn't introduce just above, so
+            // that when we loop over `next_inst()` below we are processing the
+            // block's original instructions. If we inserted spills, then it
+            // already does point there. If we did not insert spills, then it is
+            // currently pointing at the first original instruction, but that
+            // means that the upcoming `pos.next_inst()` call will skip over the
+            // first original instruction, so in this case we need to back up
+            // the cursor.
+            if !spilled_any {
+                pos = pos.at_position(CursorPosition::Before(block));
+            }
 
-                let offset = stack_slot_offsets[index];
-                stack_slot_offsets[index] += size_of_val;
+            while let Some(mut inst) = pos.next_inst() {
+                // Replace all uses of needs-stack-map values with loads from
+                // the value's associated stack slot.
+                vals.extend(pos.func.dfg.inst_values(inst));
+                let mut replaced_any = false;
+                for val in &mut vals {
+                    if let Some(slot) = stack_slots.get(val) {
+                        replaced_any = true;
+                        let ty = pos.func.dfg.value_type(*val);
+                        *val = pos.ins().stack_load(ty, *slot, 0);
+                    }
+                }
+                if replaced_any {
+                    pos.func.dfg.overwrite_inst_values(inst, vals.drain(..));
+                } else {
+                    vals.clear();
+                }
 
-                cursor
-                    .ins()
-                    .stack_store(val, slot, i32::try_from(offset).unwrap());
+                // If this instruction defines a needs-stack-map value, then
+                // spill it to its stack slot.
+                pos = pos.after_inst(inst);
+                vals.extend_from_slice(pos.func.dfg.inst_results(inst));
+                for val in vals.drain(..) {
+                    if let Some(slot) = stack_slots.get(&val) {
+                        inst = pos.ins().stack_store(val, *slot, 0);
+                    }
+                }
 
-                cursor
-                    .func
-                    .dfg
-                    .append_user_stack_map_entry(inst, ir::UserStackMapEntry { ty, slot, offset });
+                pos = pos.at_inst(inst);
             }
         }
     }
@@ -952,25 +1061,6 @@ impl<'a> FunctionBuilder<'a> {
     }
 }
 
-/// Sort `live` by size and return an iterable of subslices grouped by size.
-fn live_vals_by_size<'a, 'b>(
-    dfg: &'a ir::DataFlowGraph,
-    live: &'b mut [ir::Value],
-) -> impl Iterator<Item = &'b [ir::Value]>
-where
-    'a: 'b,
-{
-    live.sort_by_key(|val| dfg.value_type(*val).bytes());
-    live.chunk_by(|a, b| dfg.value_type(*a).bytes() == dfg.value_type(*b).bytes())
-}
-
-/// Get `log2(sizeof(val))` as a `usize`.
-fn log2_size(dfg: &ir::DataFlowGraph, val: ir::Value) -> usize {
-    let size = dfg.value_type(val).bytes();
-    debug_assert!(size.is_power_of_two());
-    usize::try_from(size.ilog2()).unwrap()
-}
-
 /// All the functions documented in the previous block are write-only and help you build a valid
 /// Cranelift IR functions via multiple debug asserts. However, you might need to improve the
 /// performance of your translation perform more complex transformations to your Cranelift IR
@@ -2221,18 +2311,20 @@ block0:
             func.display().to_string().trim(),
             r#"
 function %sample(i32, i32) system_v {
-    ss0 = explicit_slot 8, align = 4
+    ss0 = explicit_slot 4, align = 4
+    ss1 = explicit_slot 4, align = 4
     sig0 = (i32) system_v
     fn0 = colocated u0:0 sig0
 
 block0(v0: i32, v1: i32):
     stack_store v0, ss0
-    stack_store v1, ss0+4
-    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4]
-    jump block0(v0, v1)
-}
-            "#
-            .trim()
+    stack_store v1, ss1
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]
+    v2 = stack_load.i32 ss0
+    v3 = stack_load.i32 ss1
+    jump block0(v2, v3)
+}            "#
+                .trim()
         );
     }
 
@@ -2261,17 +2353,18 @@ block0(v0: i32, v1: i32):
 
         // At each `call` we are losing one more value as no longer live, so
         // each stack map should be one smaller than the last. `v3` is never
-        // live, so should never appear in a stack map. Note that a value that
-        // is an argument to the call, but is not live after the call, should
-        // not appear in the stack map. This is why `v0` appears in the first
-        // call's stack map, but not the second call's stack map.
+        // live across a safepoint, so should never appear in a stack map. Note
+        // that a value that is an argument to the call, but is not live after
+        // the call, should not appear in the stack map. This is why `v0`
+        // appears in the first call's stack map, but not the second call's
+        // stack map.
         //
         //     block0:
         //       v0 = needs stack map
         //       v1 = needs stack map
         //       v2 = needs stack map
         //       v3 = needs stack map
-        //       call $foo(v0)
+        //       call $foo(v3)
         //       call $foo(v0)
         //       call $foo(v1)
         //       call $foo(v2)
@@ -2287,7 +2380,7 @@ block0(v0: i32, v1: i32):
         builder.declare_value_needs_stack_map(v2);
         let v3 = builder.ins().iconst(ir::types::I32, 3);
         builder.declare_value_needs_stack_map(v3);
-        builder.ins().call(func_ref, &[v0]);
+        builder.ins().call(func_ref, &[v3]);
         builder.ins().call(func_ref, &[v0]);
         builder.ins().call(func_ref, &[v1]);
         builder.ins().call(func_ref, &[v2]);
@@ -2300,25 +2393,27 @@ block0(v0: i32, v1: i32):
             func.display().to_string().trim(),
             r#"
 function %sample() system_v {
-    ss0 = explicit_slot 12, align = 4
+    ss0 = explicit_slot 4, align = 4
+    ss1 = explicit_slot 4, align = 4
+    ss2 = explicit_slot 4, align = 4
     sig0 = (i32) system_v
     fn0 = colocated u0:0 sig0
 
 block0:
     v0 = iconst.i32 0
+    stack_store v0, ss2  ; v0 = 0
     v1 = iconst.i32 1
+    stack_store v1, ss1  ; v1 = 1
     v2 = iconst.i32 2
-    v3 = iconst.i32 3
-    stack_store v0, ss0  ; v0 = 0
-    stack_store v1, ss0+4  ; v1 = 1
-    stack_store v2, ss0+8  ; v2 = 2
-    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4, i32 @ ss0+8]  ; v0 = 0
-    stack_store v1, ss0  ; v1 = 1
-    stack_store v2, ss0+4  ; v2 = 2
-    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4]  ; v0 = 0
     stack_store v2, ss0  ; v2 = 2
-    call fn0(v1), stack_map=[i32 @ ss0+0]  ; v1 = 1
-    call fn0(v2)  ; v2 = 2
+    v3 = iconst.i32 3
+    call fn0(v3), stack_map=[i32 @ ss2+0, i32 @ ss1+0, i32 @ ss0+0]  ; v3 = 3
+    v4 = stack_load.i32 ss2
+    call fn0(v4), stack_map=[i32 @ ss1+0, i32 @ ss0+0]
+    v5 = stack_load.i32 ss1
+    call fn0(v5), stack_map=[i32 @ ss0+0]
+    v6 = stack_load.i32 ss0
+    call fn0(v6)
     return
 }
             "#
@@ -2580,14 +2675,15 @@ function u0:0(i32) system_v {
 
 block0(v0: i32):
     v1 = iconst.i64 0x1234_5678
+    stack_store v1, ss0  ; v1 = 0x1234_5678
     brif v0, block1, block2
 
 block1:
-    v2 = iadd_imm.i64 v1, 0  ; v1 = 0x1234_5678
+    v3 = stack_load.i64 ss0
+    v2 = iadd_imm v3, 0
     return
 
 block2:
-    stack_store.i64 v1, ss0  ; v1 = 0x1234_5678
     call fn0(), stack_map=[i64 @ ss0+0]
     return
 }
@@ -2826,6 +2922,8 @@ block2:
         builder.ins().jump(block3, &[v3, v3]);
 
         builder.switch_to_block(block3);
+        builder.append_block_param(block3, ir::types::I64);
+        builder.append_block_param(block3, ir::types::I64);
         builder.ins().call(func_ref, &[]);
         // NB: Our simplistic liveness analysis conservatively treats any use of
         // a value as keeping it live, regardless if the use has side effects or
@@ -2842,7 +2940,8 @@ block2:
             func.display().to_string().trim(),
             r#"
 function %sample(i32) system_v {
-    ss0 = explicit_slot 16, align = 8
+    ss0 = explicit_slot 8, align = 8
+    ss1 = explicit_slot 8, align = 8
     sig0 = () system_v
     fn0 = colocated u0:0 sig0
 
@@ -2851,23 +2950,27 @@ block0(v0: i32):
 
 block1:
     v1 = iconst.i64 1
-    v2 = iconst.i64 2
     stack_store v1, ss0  ; v1 = 1
-    stack_store v2, ss0+8  ; v2 = 2
-    call fn0(), stack_map=[i64 @ ss0+0, i64 @ ss0+8]
-    jump block3(v1, v2)  ; v1 = 1, v2 = 2
+    v2 = iconst.i64 2
+    stack_store v2, ss1  ; v2 = 2
+    call fn0(), stack_map=[i64 @ ss0+0, i64 @ ss1+0]
+    v8 = stack_load.i64 ss0
+    v9 = stack_load.i64 ss1
+    jump block3(v8, v9)
 
 block2:
     v3 = iconst.i64 3
-    v4 = iconst.i64 4
     stack_store v3, ss0  ; v3 = 3
+    v4 = iconst.i64 4
     call fn0(), stack_map=[i64 @ ss0+0]
-    jump block3(v3, v3)  ; v3 = 3, v3 = 3
+    v10 = stack_load.i64 ss0
+    v11 = stack_load.i64 ss0
+    jump block3(v10, v11)
 
-block3:
-    stack_store.i64 v1, ss0  ; v1 = 1
+block3(v5: i64, v6: i64):
     call fn0(), stack_map=[i64 @ ss0+0]
-    v5 = iadd_imm.i64 v1, 0  ; v1 = 1
+    v12 = stack_load.i64 ss0
+    v7 = iadd_imm v12, 0
     return
 }
             "#
@@ -2939,9 +3042,13 @@ block3:
 function %sample(i8, i16, i32, i64, i128, f32, f64, i8x16, i16x8) -> i8, i16, i32, i64, i128, f32, f64, i8x16, i16x8 system_v {
     ss0 = explicit_slot 1
     ss1 = explicit_slot 2, align = 2
-    ss2 = explicit_slot 8, align = 4
-    ss3 = explicit_slot 16, align = 8
-    ss4 = explicit_slot 48, align = 16
+    ss2 = explicit_slot 4, align = 4
+    ss3 = explicit_slot 8, align = 8
+    ss4 = explicit_slot 16, align = 16
+    ss5 = explicit_slot 4, align = 4
+    ss6 = explicit_slot 8, align = 8
+    ss7 = explicit_slot 16, align = 16
+    ss8 = explicit_slot 16, align = 16
     sig0 = () system_v
     fn0 = colocated u0:0 sig0
 
@@ -2949,14 +3056,308 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: i128, v5: f32, v6: f64, v7: i8x16,
     stack_store v0, ss0
     stack_store v1, ss1
     stack_store v2, ss2
-    stack_store v5, ss2+4
     stack_store v3, ss3
-    stack_store v6, ss3+8
     stack_store v4, ss4
-    stack_store v7, ss4+16
-    stack_store v8, ss4+32
-    call fn0(), stack_map=[i8 @ ss0+0, i16 @ ss1+0, i32 @ ss2+0, f32 @ ss2+4, i64 @ ss3+0, f64 @ ss3+8, i128 @ ss4+0, i8x16 @ ss4+16, i16x8 @ ss4+32]
-    return v0, v1, v2, v3, v4, v5, v6, v7, v8
+    stack_store v5, ss5
+    stack_store v6, ss6
+    stack_store v7, ss7
+    stack_store v8, ss8
+    call fn0(), stack_map=[i8 @ ss0+0, i16 @ ss1+0, i32 @ ss2+0, i64 @ ss3+0, i128 @ ss4+0, f32 @ ss5+0, f64 @ ss6+0, i8x16 @ ss7+0, i16x8 @ ss8+0]
+    v9 = stack_load.i8 ss0
+    v10 = stack_load.i16 ss1
+    v11 = stack_load.i32 ss2
+    v12 = stack_load.i64 ss3
+    v13 = stack_load.i128 ss4
+    v14 = stack_load.f32 ss5
+    v15 = stack_load.f64 ss6
+    v16 = stack_load.i8x16 ss7
+    v17 = stack_load.i16x8 ss8
+    return v9, v10, v11, v12, v13, v14, v15, v16, v17
+}
+            "#
+            .trim()
+        );
+    }
+
+    #[test]
+    fn series_of_non_overlapping_live_ranges_needs_stack_map() {
+        let sig = Signature::new(CallConv::SystemV);
+
+        let mut fn_ctx = FunctionBuilderContext::new();
+        let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
+        let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
+
+        let name = builder
+            .func
+            .declare_imported_user_function(ir::UserExternalName {
+                namespace: 0,
+                index: 0,
+            });
+        let signature = builder
+            .func
+            .import_signature(Signature::new(CallConv::SystemV));
+        let foo_func_ref = builder.import_function(ir::ExtFuncData {
+            name: ir::ExternalName::user(name),
+            signature,
+            colocated: true,
+        });
+
+        let name = builder
+            .func
+            .declare_imported_user_function(ir::UserExternalName {
+                namespace: 0,
+                index: 1,
+            });
+        let mut sig = Signature::new(CallConv::SystemV);
+        sig.params.push(AbiParam::new(ir::types::I32));
+        let signature = builder.func.import_signature(sig);
+        let consume_func_ref = builder.import_function(ir::ExtFuncData {
+            name: ir::ExternalName::user(name),
+            signature,
+            colocated: true,
+        });
+
+        // Create a series of needs-stack-map values that do not have
+        // overlapping live ranges, but which do appear in stack maps for calls
+        // to `$foo`:
+        //
+        //     block0:
+        //       v0 = needs stack map
+        //       call $foo()
+        //       call consume(v0)
+        //       v1 = needs stack map
+        //       call $foo()
+        //       call consume(v1)
+        //       v2 = needs stack map
+        //       call $foo()
+        //       call consume(v2)
+        //       v3 = needs stack map
+        //       call $foo()
+        //       call consume(v3)
+        //       return
+        let block0 = builder.create_block();
+        builder.append_block_params_for_function_params(block0);
+        builder.switch_to_block(block0);
+        let v0 = builder.ins().iconst(ir::types::I32, 0);
+        builder.declare_value_needs_stack_map(v0);
+        builder.ins().call(foo_func_ref, &[]);
+        builder.ins().call(consume_func_ref, &[v0]);
+        let v1 = builder.ins().iconst(ir::types::I32, 1);
+        builder.declare_value_needs_stack_map(v1);
+        builder.ins().call(foo_func_ref, &[]);
+        builder.ins().call(consume_func_ref, &[v1]);
+        let v2 = builder.ins().iconst(ir::types::I32, 2);
+        builder.declare_value_needs_stack_map(v2);
+        builder.ins().call(foo_func_ref, &[]);
+        builder.ins().call(consume_func_ref, &[v2]);
+        let v3 = builder.ins().iconst(ir::types::I32, 3);
+        builder.declare_value_needs_stack_map(v3);
+        builder.ins().call(foo_func_ref, &[]);
+        builder.ins().call(consume_func_ref, &[v3]);
+        builder.ins().return_(&[]);
+        builder.seal_all_blocks();
+        builder.finalize();
+
+        eprintln!("Actual = {}", func.display());
+        assert_eq!(
+            func.display().to_string().trim(),
+            r#"
+function %sample() system_v {
+    ss0 = explicit_slot 4, align = 4
+    sig0 = () system_v
+    sig1 = (i32) system_v
+    fn0 = colocated u0:0 sig0
+    fn1 = colocated u0:1 sig1
+
+block0:
+    v0 = iconst.i32 0
+    stack_store v0, ss0  ; v0 = 0
+    call fn0(), stack_map=[i32 @ ss0+0]
+    v4 = stack_load.i32 ss0
+    call fn1(v4)
+    v1 = iconst.i32 1
+    stack_store v1, ss0  ; v1 = 1
+    call fn0(), stack_map=[i32 @ ss0+0]
+    v5 = stack_load.i32 ss0
+    call fn1(v5)
+    v2 = iconst.i32 2
+    stack_store v2, ss0  ; v2 = 2
+    call fn0(), stack_map=[i32 @ ss0+0]
+    v6 = stack_load.i32 ss0
+    call fn1(v6)
+    v3 = iconst.i32 3
+    stack_store v3, ss0  ; v3 = 3
+    call fn0(), stack_map=[i32 @ ss0+0]
+    v7 = stack_load.i32 ss0
+    call fn1(v7)
+    return
+}
+            "#
+            .trim()
+        );
+    }
+
+    #[test]
+    fn vars_block_params_and_needs_stack_map() {
+        let _ = env_logger::try_init();
+
+        let mut sig = Signature::new(CallConv::SystemV);
+        sig.params.push(AbiParam::new(ir::types::I32));
+        sig.returns.push(AbiParam::new(ir::types::I32));
+
+        let mut fn_ctx = FunctionBuilderContext::new();
+        let mut func = Function::with_name_signature(UserFuncName::testcase("sample"), sig);
+        let mut builder = FunctionBuilder::new(&mut func, &mut fn_ctx);
+
+        let name = builder
+            .func
+            .declare_imported_user_function(ir::UserExternalName {
+                namespace: 0,
+                index: 0,
+            });
+        let mut sig = Signature::new(CallConv::SystemV);
+        sig.params.push(AbiParam::new(ir::types::I32));
+        let signature = builder.func.import_signature(sig);
+        let func_ref = builder.import_function(ir::ExtFuncData {
+            name: ir::ExternalName::user(name),
+            signature,
+            colocated: true,
+        });
+
+        // Use a variable, create a control flow diamond so that the variable
+        // forces a block parameter on the control join point, and make sure
+        // that we get stack maps for all the appropriate uses of the variable
+        // in all blocks, as well as that we are reusing stack slots for each of
+        // the values.
+        //
+        //                        block0:
+        //                          x := needs stack map
+        //                          call $foo(x)
+        //                          br_if v0, block1, block2
+        //
+        //
+        //     block1:                                     block2:
+        //       call $foo(x)                                call $foo(x)
+        //       call $foo(x)                                call $foo(x)
+        //       x := new needs stack map                    x := new needs stack map
+        //       call $foo(x)                                call $foo(x)
+        //       jump block3                                 jump block3
+        //
+        //
+        //                        block3:
+        //                          call $foo(x)
+        //                          return x
+
+        let x = Variable::from_u32(0);
+        builder.declare_var(x, ir::types::I32);
+        builder.declare_var_needs_stack_map(x);
+
+        let block0 = builder.create_block();
+        let block1 = builder.create_block();
+        let block2 = builder.create_block();
+        let block3 = builder.create_block();
+
+        builder.append_block_params_for_function_params(block0);
+        builder.switch_to_block(block0);
+        let v0 = builder.func.dfg.block_params(block0)[0];
+        let val = builder.ins().iconst(ir::types::I32, 42);
+        builder.def_var(x, val);
+        {
+            let x = builder.use_var(x);
+            builder.ins().call(func_ref, &[x]);
+        }
+        builder.ins().brif(v0, block1, &[], block2, &[]);
+
+        builder.switch_to_block(block1);
+        {
+            let x = builder.use_var(x);
+            builder.ins().call(func_ref, &[x]);
+            builder.ins().call(func_ref, &[x]);
+        }
+        let val = builder.ins().iconst(ir::types::I32, 36);
+        builder.def_var(x, val);
+        {
+            let x = builder.use_var(x);
+            builder.ins().call(func_ref, &[x]);
+        }
+        builder.ins().jump(block3, &[]);
+
+        builder.switch_to_block(block2);
+        {
+            let x = builder.use_var(x);
+            builder.ins().call(func_ref, &[x]);
+            builder.ins().call(func_ref, &[x]);
+        }
+        let val = builder.ins().iconst(ir::types::I32, 36);
+        builder.def_var(x, val);
+        {
+            let x = builder.use_var(x);
+            builder.ins().call(func_ref, &[x]);
+        }
+        builder.ins().jump(block3, &[]);
+
+        builder.switch_to_block(block3);
+        let x = builder.use_var(x);
+        builder.ins().call(func_ref, &[x]);
+        builder.ins().return_(&[x]);
+
+        builder.seal_all_blocks();
+        builder.finalize();
+
+        eprintln!("Actual = {}", func.display());
+
+        // Because our liveness analysis is very simple, and visit blocks in the
+        // order 3->1->2->0, we see uses of `v2` in block1 and mark it live
+        // across all of block2 because we haven't reached the def in block0
+        // yet, even though it isn't technically live out of block2, only live
+        // in. This means that it shows up in the stack map for block2's second
+        // call to `foo()` when it technically needn't, and additionally means
+        // that we have two stack slots instead of a single one below. This
+        // could all be improved and cleaned up by improving the liveness
+        // analysis.
+        assert_eq!(
+            func.display().to_string().trim(),
+            r#"
+function %sample(i32) -> i32 system_v {
+    ss0 = explicit_slot 4, align = 4
+    ss1 = explicit_slot 4, align = 4
+    sig0 = (i32) system_v
+    fn0 = colocated u0:0 sig0
+
+block0(v0: i32):
+    v1 = iconst.i32 42
+    v2 -> v1
+    v4 -> v1
+    stack_store v1, ss0  ; v1 = 42
+    v7 = stack_load.i32 ss0
+    call fn0(v7), stack_map=[i32 @ ss0+0]
+    brif v0, block1, block2
+
+block1:
+    call fn0(v2), stack_map=[i32 @ ss0+0]  ; v2 = 42
+    call fn0(v2)  ; v2 = 42
+    v3 = iconst.i32 36
+    stack_store v3, ss0  ; v3 = 36
+    v8 = stack_load.i32 ss0
+    call fn0(v8), stack_map=[i32 @ ss0+0]
+    v9 = stack_load.i32 ss0
+    jump block3(v9)
+
+block2:
+    call fn0(v4), stack_map=[i32 @ ss0+0]  ; v4 = 42
+    call fn0(v4), stack_map=[i32 @ ss0+0]  ; v4 = 42
+    v5 = iconst.i32 36
+    stack_store v5, ss1  ; v5 = 36
+    v10 = stack_load.i32 ss1
+    call fn0(v10), stack_map=[i32 @ ss0+0, i32 @ ss1+0]
+    v11 = stack_load.i32 ss1
+    jump block3(v11)
+
+block3(v6: i32):
+    stack_store v6, ss0
+    call fn0(v6), stack_map=[i32 @ ss0+0]
+    v12 = stack_load.i32 ss0
+    return v12
 }
             "#
             .trim()
@@ -3021,7 +3422,8 @@ function %sample(i32) -> i32 system_v {
 block0(v0: i32):
     stack_store v0, ss0
     call fn0(), stack_map=[i32 @ ss0+0]
-    return v0
+    v1 = stack_load.i32 ss0
+    return v1
 }
             "#
             .trim()
diff --git a/crates/wasi-http/src/body.rs b/crates/wasi-http/src/body.rs
index 427b527ba..5f16f4239 100644
--- a/crates/wasi-http/src/body.rs
+++ b/crates/wasi-http/src/body.rs
@@ -23,6 +23,7 @@ pub type HyperIncomingBody = BoxBody<Bytes, types::ErrorCode>;
 pub type HyperOutgoingBody = BoxBody<Bytes, types::ErrorCode>;
 
 /// The concrete type behind a `was:http/types/incoming-body` resource.
+#[derive(Debug)]
 pub struct HostIncomingBody {
     body: IncomingBodyState,
     /// An optional worker task to keep alive while this body is being read.
@@ -72,6 +73,7 @@ impl HostIncomingBody {
 }
 
 /// Internal state of a [`HostIncomingBody`].
+#[derive(Debug)]
 enum IncomingBodyState {
     /// The body is stored here meaning that within `HostIncomingBody` the
     /// `take_stream` method can be called for example.
@@ -84,6 +86,7 @@ enum IncomingBodyState {
 }
 
 /// Small wrapper around [`HyperIncomingBody`] which adds a timeout to every frame.
+#[derive(Debug)]
 struct BodyWithTimeout {
     /// Underlying stream that frames are coming from.
     inner: HyperIncomingBody,
@@ -146,6 +149,7 @@ impl Body for BodyWithTimeout {
 
 /// Message sent when a `HostIncomingBodyStream` is done to the
 /// `HostFutureTrailers` state.
+#[derive(Debug)]
 enum StreamEnd {
     /// The body wasn't completely read and was dropped early. May still have
     /// trailers, but requires reading more frames.
@@ -158,6 +162,7 @@ enum StreamEnd {
 
 /// The concrete type behind the `wasi:io/streams/input-stream` resource returned
 /// by `wasi:http/types/incoming-body`'s `stream` method.
+#[derive(Debug)]
 pub struct HostIncomingBodyStream {
     state: IncomingBodyStreamState,
     buffer: Bytes,
@@ -209,6 +214,7 @@ impl HostIncomingBodyStream {
     }
 }
 
+#[derive(Debug)]
 enum IncomingBodyStreamState {
     /// The body is currently open for reading and present here.
     ///
@@ -293,6 +299,7 @@ impl Drop for HostIncomingBodyStream {
 }
 
 /// The concrete type behind a `wasi:http/types/future-trailers` resource.
+#[derive(Debug)]
 pub enum HostFutureTrailers {
     /// Trailers aren't here yet.
     ///
@@ -378,7 +385,7 @@ impl Subscribe for HostFutureTrailers {
     }
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 struct WrittenState {
     expected: u64,
     written: Arc<std::sync::atomic::AtomicU64>,
@@ -540,6 +547,7 @@ impl HostOutgoingBody {
 }
 
 /// Message sent to end the `[HostOutgoingBody]` stream.
+#[derive(Debug)]
 enum FinishMessage {
     Finished,
     Trailers(hyper::HeaderMap),
@@ -566,6 +574,7 @@ impl StreamContext {
 }
 
 /// Provides a [`HostOutputStream`] impl from a [`tokio::sync::mpsc::Sender`].
+#[derive(Debug)]
 struct BodyWriteStream {
     context: StreamContext,
     writer: mpsc::Sender<Bytes>,
diff --git a/crates/wasi-http/src/io.rs b/crates/wasi-http/src/io.rs
index e8446c075..962bd58f2 100644
--- a/crates/wasi-http/src/io.rs
+++ b/crates/wasi-http/src/io.rs
@@ -8,6 +8,7 @@ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
 
 /// A type that wraps any type implementing [`tokio::io::AsyncRead`] and [`tokio::io::AsyncWrite`]
 /// and itself implements [`hyper::rt::Read`] and [`hyper::rt::Write`].
+#[derive(Debug)]
 pub struct TokioIo<T> {
     inner: T,
 }
diff --git a/crates/wasi-http/src/types.rs b/crates/wasi-http/src/types.rs
index 4f22444d3..4821e65c2 100644
--- a/crates/wasi-http/src/types.rs
+++ b/crates/wasi-http/src/types.rs
@@ -21,6 +21,7 @@ use wasmtime::component::{Resource, ResourceTable};
 use wasmtime_wasi::{runtime::AbortOnDropJoinHandle, Subscribe};
 
 /// Capture the state necessary for use in the wasi-http API implementation.
+#[derive(Debug)]
 pub struct WasiHttpCtx {
     _priv: (),
 }
@@ -486,6 +487,7 @@ impl TryInto<http::Method> for types::Method {
 }
 
 /// The concrete type behind a `wasi:http/types/incoming-request` resource.
+#[derive(Debug)]
 pub struct HostIncomingRequest {
     pub(crate) parts: http::request::Parts,
     pub(crate) scheme: Scheme,
@@ -561,6 +563,7 @@ impl TryFrom<HostOutgoingResponse> for hyper::Response<HyperOutgoingBody> {
 }
 
 /// The concrete type behind a `wasi:http/types/outgoing-request` resource.
+#[derive(Debug)]
 pub struct HostOutgoingRequest {
     /// The method of the request.
     pub method: Method,
@@ -577,7 +580,7 @@ pub struct HostOutgoingRequest {
 }
 
 /// The concrete type behind a `wasi:http/types/request-options` resource.
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct HostRequestOptions {
     /// How long to wait for a connection to be established.
     pub connect_timeout: Option<std::time::Duration>,
@@ -588,6 +591,7 @@ pub struct HostRequestOptions {
 }
 
 /// The concrete type behind a `wasi:http/types/incoming-response` resource.
+#[derive(Debug)]
 pub struct HostIncomingResponse {
     /// The response status
     pub status: u16,
@@ -598,6 +602,7 @@ pub struct HostIncomingResponse {
 }
 
 /// The concrete type behind a `wasi:http/types/fields` resource.
+#[derive(Debug)]
 pub enum HostFields {
     /// A reference to the fields of a parent entry.
     Ref {
@@ -626,6 +631,7 @@ pub type FutureIncomingResponseHandle =
     AbortOnDropJoinHandle<anyhow::Result<Result<IncomingResponse, types::ErrorCode>>>;
 
 /// A response that is in the process of being received.
+#[derive(Debug)]
 pub struct IncomingResponse {
     /// The response itself.
     pub resp: hyper::Response<HyperIncomingBody>,
@@ -636,6 +642,7 @@ pub struct IncomingResponse {
 }
 
 /// The concrete type behind a `wasi:http/types/future-incoming-response` resource.
+#[derive(Debug)]
 pub enum HostFutureIncomingResponse {
     /// A pending response
     Pending(FutureIncomingResponseHandle),
diff --git a/crates/wasi-threads/Cargo.toml b/crates/wasi-threads/Cargo.toml
index 84b055385..467ce1e6f 100644
--- a/crates/wasi-threads/Cargo.toml
+++ b/crates/wasi-threads/Cargo.toml
@@ -5,7 +5,7 @@ authors.workspace = true
 description = "Wasmtime implementation of the wasi-threads API"
 documentation = "https://docs.rs/wasmtime-wasi-nn"
 license = "Apache-2.0 WITH LLVM-exception"
-categories = ["wasm", "parallelism", "threads"]
+categories = ["wasm"]
 keywords = ["webassembly", "wasm", "neural-network"]
 repository = "https://github.com/bytecodealliance/wasmtime"
 readme = "README.md"
diff --git a/crates/wasi/src/runtime.rs b/crates/wasi/src/runtime.rs
index b92509810..da41faa38 100644
--- a/crates/wasi/src/runtime.rs
+++ b/crates/wasi/src/runtime.rs
@@ -37,6 +37,7 @@ pub(crate) static RUNTIME: once_cell::sync::Lazy<tokio::runtime::Runtime> =
 ///
 /// This behavior makes it easier to tie a worker task to the lifetime of a Resource
 /// by keeping this handle owned by the Resource.
+#[derive(Debug)]
 pub struct AbortOnDropJoinHandle<T>(tokio::task::JoinHandle<T>);
 impl<T> Drop for AbortOnDropJoinHandle<T> {
     fn drop(&mut self) {
diff --git a/docs/stability-tiers.md b/docs/stability-tiers.md
index a038e296e..b294270ef 100644
--- a/docs/stability-tiers.md
+++ b/docs/stability-tiers.md
@@ -96,11 +96,13 @@ For explanations of what each tier means see below.
 | WebAssembly Proposal | [`gc`]                            | Complete implementation     |
 | WASI Proposal        | [`wasi-nn`]                       | More expansive CI testing   |
 | WASI Proposal        | [`wasi-threads`]                  | More CI, unstable proposal  |
+| WASI Proposal        | [`wasi-runtime-config`]           | unstable proposal           |
 | *misc*               | Non-Wasmtime Cranelift usage [^1] | CI testing, full-time maintainer |
 | *misc*               | DWARF debugging [^2]              | CI testing, full-time maintainer, improved quality |
 
 [`wasi-nn`]: https://github.com/WebAssembly/wasi-nn
 [`wasi-threads`]: https://github.com/WebAssembly/wasi-threads
+[`wasi-runtime-config`]: https://github.com/WebAssembly/wasi-runtime-config
 [`gc`]: https://github.com/WebAssembly/gc
 
 [^1]: This is intended to encompass features that Cranelift supports as a
@@ -159,7 +161,6 @@ features to figure out how best to implement them and at least move them to Tier
 * [WASI proposal: `wasi-message-queue`](https://github.com/WebAssembly/wasi-message-queue)
 * [WASI proposal: `wasi-parallel`](https://github.com/WebAssembly/wasi-parallel)
 * [WASI proposal: `wasi-pubsub`](https://github.com/WebAssembly/wasi-pubsub)
-* [WASI proposal: `wasi-runtime-config`](https://github.com/WebAssembly/wasi-runtime-config)
 * [WASI proposal: `wasi-sql`](https://github.com/WebAssembly/wasi-sql)
 * [WASI proposal: `wasi-url`](https://github.com/WebAssembly/wasi-url)
 
diff --git a/winch/codegen/src/abi/mod.rs b/winch/codegen/src/abi/mod.rs
index b5f46ee63..e591f7e8f 100644
--- a/winch/codegen/src/abi/mod.rs
+++ b/winch/codegen/src/abi/mod.rs
@@ -67,10 +67,22 @@ pub(super) enum ParamsOrReturns {
 /// Macro to get the pinned register holding the [VMContext].
 macro_rules! vmctx {
     ($m:ident) => {
-        <$m::ABI as ABI>::vmctx_reg()
+        <$m::ABI as $crate::abi::ABI>::vmctx_reg()
     };
 }
 
+/// Macro to get the designated general purpose scratch register or the
+/// designated scratch register for the given type.
+macro_rules! scratch {
+    ($m:ident) => {
+        <$m::ABI as $crate::abi::ABI>::scratch_reg()
+    };
+    ($m:ident, $wasm_type:expr) => {
+        <$m::ABI as $crate::abi::ABI>::scratch_for($wasm_type)
+    };
+}
+
+pub(crate) use scratch;
 pub(crate) use vmctx;
 
 /// Constructs an [ABISig] using Winch's ABI.
diff --git a/winch/codegen/src/codegen/bounds.rs b/winch/codegen/src/codegen/bounds.rs
index c1fb6d395..0411a031e 100644
--- a/winch/codegen/src/codegen/bounds.rs
+++ b/winch/codegen/src/codegen/bounds.rs
@@ -3,7 +3,7 @@
 //! recommended when working on this area of Winch.
 use super::env::{HeapData, HeapStyle};
 use crate::{
-    abi::{vmctx, ABI},
+    abi::{scratch, vmctx},
     codegen::CodeGenContext,
     isa::reg::Reg,
     masm::{IntCmpKind, MacroAssembler, OperandSize, RegImm, TrapCode},
@@ -96,7 +96,7 @@ where
             masm.mov(RegImm::i64(max_size as i64), dst, ptr_size)
         }
         (_, HeapStyle::Dynamic) => {
-            let scratch = <M::ABI as ABI>::scratch_reg();
+            let scratch = scratch!(M);
             let base = if let Some(offset) = heap.import_from {
                 let addr = masm.address_at_vmctx(offset);
                 masm.load_ptr(addr, scratch);
@@ -199,7 +199,7 @@ pub(crate) fn load_heap_addr_unchecked<M>(
     let base = if let Some(offset) = heap.import_from {
         // If the WebAssembly memory is imported, load the address into
         // the scratch register.
-        let scratch = <M::ABI as ABI>::scratch_reg();
+        let scratch = scratch!(M);
         masm.load_ptr(masm.address_at_vmctx(offset), scratch);
         scratch
     } else {
diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs
index 84680e897..50d501db9 100644
--- a/winch/codegen/src/codegen/call.rs
+++ b/winch/codegen/src/codegen/call.rs
@@ -57,7 +57,7 @@
 //! └──────────────────────────────────────────────────┘ ------> Stack pointer when emitting the call
 
 use crate::{
-    abi::{vmctx, ABIOperand, ABISig, RetArea, ABI},
+    abi::{scratch, vmctx, ABIOperand, ABISig, RetArea},
     codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext},
     masm::{
         CalleeKind, ContextArgs, MacroAssembler, MemMoveDirection, OperandSize, SPOffset,
@@ -299,7 +299,7 @@ impl FnCall {
                 &ABIOperand::Stack { ty, offset, .. } => {
                     let addr = masm.address_at_sp(SPOffset::from_u32(offset));
                     let size: OperandSize = ty.into();
-                    let scratch = <M::ABI as ABI>::scratch_for(&ty);
+                    let scratch = scratch!(M, &ty);
                     context.move_val_to_reg(val, scratch, masm);
                     masm.store(scratch.into(), addr, size);
                 }
@@ -319,7 +319,7 @@ impl FnCall {
                     let slot = masm.address_at_sp(SPOffset::from_u32(offset));
                     // Don't rely on `ABI::scratch_for` as we always use
                     // an int register as the return pointer.
-                    let scratch = <M::ABI as ABI>::scratch_reg();
+                    let scratch = scratch!(M);
                     masm.load_addr(addr, scratch, ty.into());
                     masm.store(scratch.into(), slot, ty.into());
                 }
diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rs
index f9a3ead31..49804899f 100644
--- a/winch/codegen/src/codegen/context.rs
+++ b/winch/codegen/src/codegen/context.rs
@@ -2,7 +2,7 @@ use wasmtime_environ::{VMOffsets, WasmHeapType, WasmValType};
 
 use super::ControlStackFrame;
 use crate::{
-    abi::{vmctx, ABIOperand, ABIResults, RetArea, ABI},
+    abi::{scratch, vmctx, ABIOperand, ABIResults, RetArea},
     frame::Frame,
     isa::reg::RegClass,
     masm::{MacroAssembler, OperandSize, RegImm, SPOffset, ShiftKind, StackSlot},
@@ -185,13 +185,13 @@ impl<'a> CodeGenContext<'a> {
             Val::F64(v) => masm.store(RegImm::f64(v.bits()), addr, size),
             Val::Local(local) => {
                 let slot = self.frame.get_wasm_local(local.index);
-                let scratch = <M::ABI as ABI>::scratch_reg();
+                let scratch = scratch!(M);
                 let local_addr = masm.local_address(&slot);
                 masm.load(local_addr, scratch, size);
                 masm.store(scratch.into(), addr, size);
             }
             Val::Memory(_) => {
-                let scratch = <M::ABI as ABI>::scratch_reg();
+                let scratch = scratch!(M);
                 masm.pop(scratch, size);
                 masm.store(scratch.into(), addr, size);
             }
@@ -576,7 +576,7 @@ impl<'a> CodeGenContext<'a> {
             Val::Local(local) => {
                 let slot = frame.get_wasm_local(local.index);
                 let addr = masm.local_address(&slot);
-                let scratch = <M::ABI as ABI>::scratch_for(&slot.ty);
+                let scratch = scratch!(M, &slot.ty);
                 masm.load(addr, scratch, slot.ty.into());
                 let stack_slot = masm.push(scratch, slot.ty.into());
                 *v = Val::mem(slot.ty, stack_slot);
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 8ab22e69a..76e85c4ca 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -1,5 +1,5 @@
 use crate::{
-    abi::{vmctx, ABIOperand, ABISig, RetArea, ABI},
+    abi::{scratch, vmctx, ABIOperand, ABISig, RetArea},
     codegen::BlockSig,
     isa::reg::Reg,
     masm::{
@@ -329,7 +329,7 @@ where
             .checked_mul(sig_index_bytes.into())
             .unwrap();
         let signatures_base_offset = self.env.vmoffsets.ptr.vmctx_type_ids_array();
-        let scratch = <M::ABI as ABI>::scratch_reg();
+        let scratch = scratch!(M);
         let funcref_sig_offset = self.env.vmoffsets.ptr.vm_func_ref_type_index();
 
         // Load the signatures address into the scratch register.
@@ -443,7 +443,7 @@ where
 
         let addr = if data.imported {
             let global_base = self.masm.address_at_reg(vmctx!(M), data.offset);
-            let scratch = <M::ABI as ABI>::scratch_reg();
+            let scratch = scratch!(M);
             self.masm.load_ptr(global_base, scratch);
             self.masm.address_at_reg(scratch, 0)
         } else {
@@ -754,7 +754,7 @@ where
         base: Reg,
         table_data: &TableData,
     ) -> M::Address {
-        let scratch = <M::ABI as ABI>::scratch_reg();
+        let scratch = scratch!(M);
         let bound = self.context.any_gpr(self.masm);
         let tmp = self.context.any_gpr(self.masm);
         let ptr_size: OperandSize = self.env.ptr_type().into();
@@ -811,7 +811,7 @@ where
 
     /// Retrieves the size of the table, pushing the result to the value stack.
     pub fn emit_compute_table_size(&mut self, table_data: &TableData) {
-        let scratch = <M::ABI as ABI>::scratch_reg();
+        let scratch = scratch!(M);
         let size = self.context.any_gpr(self.masm);
         let ptr_size: OperandSize = self.env.ptr_type().into();
 
@@ -834,7 +834,7 @@ where
     /// Retrieves the size of the memory, pushing the result to the value stack.
     pub fn emit_compute_memory_size(&mut self, heap_data: &HeapData) {
         let size_reg = self.context.any_gpr(self.masm);
-        let scratch = <M::ABI as ABI>::scratch_reg();
+        let scratch = scratch!(M);
 
         let base = if let Some(offset) = heap_data.import_from {
             self.masm
diff --git a/winch/codegen/src/isa/x64/regs.rs b/winch/codegen/src/isa/x64/regs.rs
index 69e796246..0118f8b97 100644
--- a/winch/codegen/src/isa/x64/regs.rs
+++ b/winch/codegen/src/isa/x64/regs.rs
@@ -99,18 +99,6 @@ pub(crate) fn scratch() -> Reg {
     r11()
 }
 
-/// This register is used as a scratch register, in the context of trampolines only,
-/// where we assume that callee-saved registers are given the correct handling
-/// according to the system ABI. r12 is chosen given that it's a callee-saved,
-/// non-argument register.
-///
-/// In the context of all other internal functions, this register is not excluded
-/// from register allocation, so no extra assumptions should be made regarding
-/// its availability.
-pub(crate) fn argv() -> Reg {
-    r12()
-}
-
 fn fpr(enc: u8) -> Reg {
     Reg::new(PReg::new(enc as usize, RegClass::Float))
 }
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index eeb73b132..7f7f7d0ed 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -1,4 +1,4 @@
-use crate::abi::{self, align_to, LocalSlot};
+use crate::abi::{self, align_to, scratch, LocalSlot};
 use crate::codegen::{CodeGenContext, FuncEnv};
 use crate::isa::reg::Reg;
 use cranelift_codegen::{
@@ -592,7 +592,7 @@ pub(crate) trait MacroAssembler {
         debug_assert!(bytes % 4 == 0);
         let mut remaining = bytes;
         let word_bytes = <Self::ABI as abi::ABI>::word_bytes();
-        let scratch = <Self::ABI as abi::ABI>::scratch_reg();
+        let scratch = scratch!(Self);
 
         let mut dst_offs = dst.as_u32() - bytes;
         let mut src_offs = src.as_u32() - bytes;
@@ -869,7 +869,7 @@ pub(crate) trait MacroAssembler {
             // Add an upper bound to this generation;
             // given a considerably large amount of slots
             // this will be inefficient.
-            let zero = <Self::ABI as abi::ABI>::scratch_reg();
+            let zero = scratch!(Self);
             self.zero(zero);
             let zero = RegImm::reg(zero);
 


Test patch:
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 2b53fae95..5dd3dd505 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -1632,6 +1632,36 @@ fn test_x64_emit() {
         "4520FF",
         "andb    %r15b, %r15b, %r15b",
     ));
+    insns.push((
+        Inst::alu_rmi_r(
+            OperandSize::Size32,
+            AluRmiROpcode::Sbb,
+            RegMemImm::reg(r14),
+            w_r15,
+        ),
+        "4519F7",
+        "sbbl    %r15d, %r14d, %r15d",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(
+            OperandSize::Size64,
+            AluRmiROpcode::Sbb,
+            RegMemImm::imm(0),
+            w_r15,
+        ),
+        "4983DF00",
+        "sbbq    %r15, $0, %r15",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(
+            OperandSize::Size64,
+            AluRmiROpcode::Adc,
+            RegMemImm::imm(0),
+            w_r15,
+        ),
+        "4983D700",
+        "adcq    %r15, $0, %r15",
+    ));
 
     // ========================================================
     // AluRM
diff --git a/cranelift/filetests/filetests/egraph/cprop.clif b/cranelift/filetests/filetests/egraph/cprop.clif
index 70d026f41..f1ca491f4 100644
--- a/cranelift/filetests/filetests/egraph/cprop.clif
+++ b/cranelift/filetests/filetests/egraph/cprop.clif
@@ -344,6 +344,122 @@ block0:
 ; check: v4 = f16const -NaN
 ; check: return v4  ; v4 = -NaN
 
+function %f32_fadd() -> f32 {
+block0:
+    v1 = f32const 0x1.fp2
+    v2 = f32const 0x1.9p3
+    v3 = fadd v1, v2
+    return v3
+}
+
+; check: v4 = f32const 0x1.440000p4
+; check: return v4  ; v4 = 0x1.440000p4
+
+function %f32_fsub() -> f32 {
+block0:
+    v1 = f32const 0x1.fp2
+    v2 = f32const 0x1.9p3
+    v3 = fsub v1, v2
+    return v3
+}
+
+; check: v4 = f32const -0x1.300000p2
+; check: return v4  ; v4 = -0x1.300000p2
+
+function %f32_fmul() -> f32 {
+block0:
+    v1 = f32const 0x1.fp2
+    v2 = f32const 0x1.9p3
+    v3 = fmul v1, v2
+    return v3
+}
+
+; check: v4 = f32const 0x1.838000p6
+; check: return v4  ; v4 = 0x1.838000p6
+
+function %f32_fdiv() -> f32 {
+block0:
+    v1 = f32const 0x1.9p5
+    v2 = f32const 0x1.9p3
+    v3 = fdiv v1, v2
+    return v3
+}
+
+; check: v4 = f32const 0x1.000000p2
+; check: return v4  ; v4 = 0x1.000000p2
+
+function %f32_sqrt() -> f32 {
+block0:
+    v1 = f32const 0x1.9p4
+    v2 = sqrt v1
+    return v2
+}
+
+; check: v3 = f32const 0x1.400000p2
+; check: return v3  ; v3 = 0x1.400000p2
+
+function %f32_ceil() -> f32 {
+block0:
+    v1 = f32const -0x1.9p3
+    v2 = ceil v1
+    return v2
+}
+
+; check: v3 = f32const -0x1.800000p3
+; check: return v3  ; v3 = -0x1.800000p3
+
+function %f32_floor() -> f32 {
+block0:
+    v1 = f32const -0x1.9p3
+    v2 = floor v1
+    return v2
+}
+
+; check: v3 = f32const -0x1.a00000p3
+; check: return v3  ; v3 = -0x1.a00000p3
+
+function %f32_trunc() -> f32 {
+block0:
+    v1 = f32const 0x1.9p3
+    v2 = trunc v1
+    return v2
+}
+
+; check: v3 = f32const 0x1.800000p3
+; check: return v3  ; v3 = 0x1.800000p3
+
+function %f32_nearest() -> f32 {
+block0:
+    v1 = f32const 0x1.9p3
+    v2 = nearest v1
+    return v2
+}
+
+; check: v3 = f32const 0x1.800000p3
+; check: return v3  ; v3 = 0x1.800000p3
+
+function %f32_fmin() -> f32 {
+block0:
+    v1 = f32const 0x1.5p6
+    v2 = f32const 0x1.5p7
+    v3 = fmin v2, v1
+    return v3
+}
+
+; check: v4 = f32const 0x1.500000p6
+; check: return v4  ; v4 = 0x1.500000p6
+
+function %f32_fmax() -> f32 {
+block0:
+    v1 = f32const 0x1.5p6
+    v2 = f32const 0x1.5p7
+    v3 = fmax v2, v1
+    return v3
+}
+
+; check: v4 = f32const 0x1.500000p7
+; check: return v4  ; v4 = 0x1.500000p7
+
 function %f32_fneg() -> f32 {
 block0:
     v1 = f32const 0.0
@@ -375,6 +491,122 @@ block0:
 ; check: v4 = f32const -NaN
 ; check: return v4  ; v4 = -NaN
 
+function %f64_fadd() -> f64 {
+block0:
+    v1 = f64const 0x1.fp2
+    v2 = f64const 0x1.9p3
+    v3 = fadd v1, v2
+    return v3
+}
+
+; check: v4 = f64const 0x1.4400000000000p4
+; check: return v4  ; v4 = 0x1.4400000000000p4
+
+function %f64_fsub() -> f64 {
+block0:
+    v1 = f64const 0x1.fp2
+    v2 = f64const 0x1.9p3
+    v3 = fsub v1, v2
+    return v3
+}
+
+; check: v4 = f64const -0x1.3000000000000p2
+; check: return v4  ; v4 = -0x1.3000000000000p2
+
+function %f64_fmul() -> f64 {
+block0:
+    v1 = f64const 0x1.fp2
+    v2 = f64const 0x1.9p3
+    v3 = fmul v1, v2
+    return v3
+}
+
+; check: v4 = f64const 0x1.8380000000000p6
+; check: return v4  ; v4 = 0x1.8380000000000p6
+
+function %f64_fdiv() -> f64 {
+block0:
+    v1 = f64const 0x1.9p5
+    v2 = f64const 0x1.9p3
+    v3 = fdiv v1, v2
+    return v3
+}
+
+; check: v4 = f64const 0x1.0000000000000p2
+; check: return v4  ; v4 = 0x1.0000000000000p2
+
+function %f64_sqrt() -> f64 {
+block0:
+    v1 = f64const 0x1.9p4
+    v2 = sqrt v1
+    return v2
+}
+
+; check: v3 = f64const 0x1.4000000000000p2
+; check: return v3  ; v3 = 0x1.4000000000000p2
+
+function %f64_ceil() -> f64 {
+block0:
+    v1 = f64const 0x1.9p3
+    v2 = ceil v1
+    return v2
+}
+
+; check: v3 = f64const 0x1.a000000000000p3
+; check: return v3  ; v3 = 0x1.a000000000000p3
+
+function %f64_floor() -> f64 {
+block0:
+    v1 = f64const 0x1.9p3
+    v2 = floor v1
+    return v2
+}
+
+; check: v3 = f64const 0x1.8000000000000p3
+; check: return v3  ; v3 = 0x1.8000000000000p3
+
+function %f64_trunc() -> f64 {
+block0:
+    v1 = f64const 0x1.9p3
+    v2 = trunc v1
+    return v2
+}
+
+; check: v3 = f64const 0x1.8000000000000p3
+; check: return v3  ; v3 = 0x1.8000000000000p3
+
+function %f64_nearest() -> f64 {
+block0:
+    v1 = f64const 0x1.9p3
+    v2 = nearest v1
+    return v2
+}
+
+; check: v3 = f64const 0x1.8000000000000p3
+; check: return v3  ; v3 = 0x1.8000000000000p3
+
+function %f64_fmin() -> f64 {
+block0:
+    v1 = f64const -0x1.5p6
+    v2 = f64const -0x1.5p7
+    v3 = fmin v2, v1
+    return v3
+}
+
+; check: v4 = f64const -0x1.5000000000000p7
+; check: return v4  ; v4 = -0x1.5000000000000p7
+
+function %f64_fmax() -> f64 {
+block0:
+    v1 = f64const -0x1.5p6
+    v2 = f64const -0x1.5p7
+    v3 = fmax v2, v1
+    return v3
+}
+
+; check: v4 = f64const -0x1.5000000000000p6
+; check: return v4  ; v4 = -0x1.5000000000000p6
+
 function %f64_fneg() -> f64 {
 block0:
     v1 = f64const 0.0