Repository: bytecodealliance/wasmtime
Pull request #8876 (2024-06-26 19:16:48)

Cranelift: Take user stack maps through lowering and emission
Previously, user stack maps were inserted by the frontend and preserved in the mid-end. This commit takes them from the mid-end CLIF into the backend vcode, and then from that vcode into the finalized mach buffer during emission.

During lowering, we compile the `UserStackMapEntry`s into packed `UserStackMap`s. This is the appropriate moment in time to do that coalescing, packing, and compiling because the stack map entries are immutable from this point on.

Additionally, we include user stack maps in the `Debug` and disassembly implementations for vcode, just after their associated safepoint instructions. This allows us to see the stack maps we are generating when debugging, as well as write filetests that check we are generating the expected stack maps for the correct instructions.

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->



Patch:
diff --git a/cranelift/codegen/meta/src/cdsl/types.rs b/cranelift/codegen/meta/src/cdsl/types.rs
index 0ff486609..28b23132d 100644
--- a/cranelift/codegen/meta/src/cdsl/types.rs
+++ b/cranelift/codegen/meta/src/cdsl/types.rs
@@ -136,6 +136,12 @@ impl LaneType {
     /// Return a string containing the documentation comment for this lane type.
     pub fn doc(self) -> String {
         match self {
+            LaneType::Float(shared_types::Float::F16) => String::from(
+                "A 16-bit floating point type represented in the IEEE 754-2008
+                *binary16* interchange format. This corresponds to the :c:type:`_Float16`
+                type in most C implementations.
+                WARNING: f16 support is a work-in-progress and is incomplete",
+            ),
             LaneType::Float(shared_types::Float::F32) => String::from(
                 "A 32-bit floating point type represented in the IEEE 754-2008
                 *binary32* interchange format. This corresponds to the :c:type:`float`
@@ -146,6 +152,12 @@ impl LaneType {
                 *binary64* interchange format. This corresponds to the :c:type:`double`
                 type in most C implementations.",
             ),
+            LaneType::Float(shared_types::Float::F128) => String::from(
+                "A 128-bit floating point type represented in the IEEE 754-2008
+                *binary128* interchange format. This corresponds to the :c:type:`_Float128`
+                type in most C implementations.
+                WARNING: f128 support is a work-in-progress and is incomplete",
+            ),
             LaneType::Int(_) if self.lane_bits() < 32 => format!(
                 "An integer type with {} bits.
                 WARNING: arithmetic on {}bit integers is incomplete",
@@ -168,13 +180,15 @@ impl LaneType {
     pub fn number(self) -> u16 {
         constants::LANE_BASE
             + match self {
-                LaneType::Int(shared_types::Int::I8) => 6,
-                LaneType::Int(shared_types::Int::I16) => 7,
-                LaneType::Int(shared_types::Int::I32) => 8,
-                LaneType::Int(shared_types::Int::I64) => 9,
-                LaneType::Int(shared_types::Int::I128) => 10,
-                LaneType::Float(shared_types::Float::F32) => 11,
-                LaneType::Float(shared_types::Float::F64) => 12,
+                LaneType::Int(shared_types::Int::I8) => 4,
+                LaneType::Int(shared_types::Int::I16) => 5,
+                LaneType::Int(shared_types::Int::I32) => 6,
+                LaneType::Int(shared_types::Int::I64) => 7,
+                LaneType::Int(shared_types::Int::I128) => 8,
+                LaneType::Float(shared_types::Float::F16) => 9,
+                LaneType::Float(shared_types::Float::F32) => 10,
+                LaneType::Float(shared_types::Float::F64) => 11,
+                LaneType::Float(shared_types::Float::F128) => 12,
             }
     }
 
@@ -191,8 +205,10 @@ impl LaneType {
 
     pub fn float_from_bits(num_bits: u16) -> LaneType {
         LaneType::Float(match num_bits {
+            16 => shared_types::Float::F16,
             32 => shared_types::Float::F32,
             64 => shared_types::Float::F64,
+            128 => shared_types::Float::F128,
             _ => unreachable!("unexpected num bits for float"),
         })
     }
diff --git a/cranelift/codegen/meta/src/cdsl/typevar.rs b/cranelift/codegen/meta/src/cdsl/typevar.rs
index da0e92cd8..d0a331369 100644
--- a/cranelift/codegen/meta/src/cdsl/typevar.rs
+++ b/cranelift/codegen/meta/src/cdsl/typevar.rs
@@ -9,7 +9,7 @@ use crate::cdsl::types::{LaneType, ReferenceType, ValueType};
 
 const MAX_LANES: u16 = 256;
 const MAX_BITS: u16 = 128;
-const MAX_FLOAT_BITS: u16 = 64;
+const MAX_FLOAT_BITS: u16 = 128;
 
 /// Type variables can be used in place of concrete types when defining
 /// instructions. This makes the instructions *polymorphic*.
@@ -159,7 +159,7 @@ impl TypeVar {
                     "can't halve all integer types"
                 );
                 assert!(
-                    ts.floats.is_empty() || *ts.floats.iter().min().unwrap() > 32,
+                    ts.floats.is_empty() || *ts.floats.iter().min().unwrap() > 16,
                     "can't halve all float types"
                 );
             }
@@ -179,7 +179,7 @@ impl TypeVar {
                     "can't halve all integer types"
                 );
                 assert!(
-                    ts.floats.is_empty() || *ts.floats.iter().min().unwrap() > 32,
+                    ts.floats.is_empty() || *ts.floats.iter().min().unwrap() > 16,
                     "can't halve all float types"
                 );
                 assert!(
@@ -464,7 +464,7 @@ impl TypeSet {
     fn half_width(&self) -> TypeSet {
         let mut copy = self.clone();
         copy.ints = NumSet::from_iter(self.ints.iter().filter(|&&x| x > 8).map(|&x| x / 2));
-        copy.floats = NumSet::from_iter(self.floats.iter().filter(|&&x| x > 32).map(|&x| x / 2));
+        copy.floats = NumSet::from_iter(self.floats.iter().filter(|&&x| x > 16).map(|&x| x / 2));
         copy
     }
 
@@ -643,7 +643,7 @@ impl TypeSetBuilder {
             range_to_set(self.simd_lanes.to_range(min_lanes..MAX_LANES, Some(1))),
             range_to_set(self.dynamic_simd_lanes.to_range(2..MAX_LANES, None)),
             range_to_set(self.ints.to_range(8..MAX_BITS, None)),
-            range_to_set(self.floats.to_range(32..64, None)),
+            range_to_set(self.floats.to_range(16..MAX_FLOAT_BITS, None)),
             range_to_set(self.refs.to_range(32..64, None)),
         )
     }
@@ -711,7 +711,7 @@ fn test_typevar_builder() {
 
     let type_set = TypeSetBuilder::new().floats(Interval::All).build();
     assert_eq!(type_set.lanes, num_set![1]);
-    assert_eq!(type_set.floats, num_set![32, 64]);
+    assert_eq!(type_set.floats, num_set![16, 32, 64, 128]);
     assert!(type_set.ints.is_empty());
 
     let type_set = TypeSetBuilder::new()
@@ -720,7 +720,7 @@ fn test_typevar_builder() {
         .includes_scalars(false)
         .build();
     assert_eq!(type_set.lanes, num_set![2, 4, 8, 16, 32, 64, 128, 256]);
-    assert_eq!(type_set.floats, num_set![32, 64]);
+    assert_eq!(type_set.floats, num_set![16, 32, 64, 128]);
     assert!(type_set.ints.is_empty());
 
     let type_set = TypeSetBuilder::new()
@@ -729,7 +729,7 @@ fn test_typevar_builder() {
         .includes_scalars(true)
         .build();
     assert_eq!(type_set.lanes, num_set![1, 2, 4, 8, 16, 32, 64, 128, 256]);
-    assert_eq!(type_set.floats, num_set![32, 64]);
+    assert_eq!(type_set.floats, num_set![16, 32, 64, 128]);
     assert!(type_set.ints.is_empty());
 
     let type_set = TypeSetBuilder::new()
@@ -738,7 +738,7 @@ fn test_typevar_builder() {
         .includes_scalars(false)
         .build();
     assert_eq!(type_set.lanes, num_set![2, 4, 8, 16, 32, 64, 128, 256]);
-    assert_eq!(type_set.floats, num_set![32, 64]);
+    assert_eq!(type_set.floats, num_set![16, 32, 64, 128]);
     assert!(type_set.dynamic_lanes.is_empty());
     assert!(type_set.ints.is_empty());
 
@@ -753,7 +753,7 @@ fn test_typevar_builder() {
         num_set![2, 4, 8, 16, 32, 64, 128, 256]
     );
     assert_eq!(type_set.ints, num_set![8, 16, 32, 64, 128]);
-    assert_eq!(type_set.floats, num_set![32, 64]);
+    assert_eq!(type_set.floats, num_set![16, 32, 64, 128]);
     assert_eq!(type_set.lanes, num_set![1]);
 
     let type_set = TypeSetBuilder::new()
@@ -765,7 +765,7 @@ fn test_typevar_builder() {
         type_set.dynamic_lanes,
         num_set![2, 4, 8, 16, 32, 64, 128, 256]
     );
-    assert_eq!(type_set.floats, num_set![32, 64]);
+    assert_eq!(type_set.floats, num_set![16, 32, 64, 128]);
     assert_eq!(type_set.lanes, num_set![1]);
     assert!(type_set.ints.is_empty());
 
@@ -871,12 +871,12 @@ fn test_forward_images() {
         TypeSetBuilder::new().ints(8..16).build()
     );
     assert_eq!(
-        TypeSetBuilder::new().floats(32..32).build().half_width(),
+        TypeSetBuilder::new().floats(16..16).build().half_width(),
         empty_set
     );
     assert_eq!(
-        TypeSetBuilder::new().floats(32..64).build().half_width(),
-        TypeSetBuilder::new().floats(32..32).build()
+        TypeSetBuilder::new().floats(32..128).build().half_width(),
+        TypeSetBuilder::new().floats(16..64).build()
     );
 
     // Double width.
@@ -893,8 +893,8 @@ fn test_forward_images() {
         TypeSetBuilder::new().floats(64..64).build()
     );
     assert_eq!(
-        TypeSetBuilder::new().floats(32..64).build().double_width(),
-        TypeSetBuilder::new().floats(64..64).build()
+        TypeSetBuilder::new().floats(16..64).build().double_width(),
+        TypeSetBuilder::new().floats(32..128).build()
     );
 }
 
diff --git a/cranelift/codegen/meta/src/shared/types.rs b/cranelift/codegen/meta/src/shared/types.rs
index 33efd1080..3bd2e2d55 100644
--- a/cranelift/codegen/meta/src/shared/types.rs
+++ b/cranelift/codegen/meta/src/shared/types.rs
@@ -43,8 +43,10 @@ impl Iterator for IntIterator {
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub(crate) enum Float {
+    F16 = 16,
     F32 = 32,
     F64 = 64,
+    F128 = 128,
 }
 
 /// Iterator through the variants of the Float enum.
@@ -63,8 +65,10 @@ impl Iterator for FloatIterator {
     type Item = Float;
     fn next(&mut self) -> Option<Self::Item> {
         let res = match self.index {
-            0 => Some(Float::F32),
-            1 => Some(Float::F64),
+            0 => Some(Float::F16),
+            1 => Some(Float::F32),
+            2 => Some(Float::F64),
+            3 => Some(Float::F128),
             _ => return None,
         };
         self.index += 1;
@@ -122,8 +126,10 @@ mod iter_tests {
     #[test]
     fn float_iter_works() {
         let mut float_iter = FloatIterator::new();
+        assert_eq!(float_iter.next(), Some(Float::F16));
         assert_eq!(float_iter.next(), Some(Float::F32));
         assert_eq!(float_iter.next(), Some(Float::F64));
+        assert_eq!(float_iter.next(), Some(Float::F128));
         assert_eq!(float_iter.next(), None);
     }
 
diff --git a/cranelift/codegen/src/data_value.rs b/cranelift/codegen/src/data_value.rs
index 25eb7a401..e2feb4946 100644
--- a/cranelift/codegen/src/data_value.rs
+++ b/cranelift/codegen/src/data_value.rs
@@ -1,7 +1,8 @@
 //! This module gives users to instantiate values that Cranelift understands. These values are used,
 //! for example, during interpretation and for wrapping immediates.
-use crate::ir::immediates::{Ieee32, Ieee64, Offset32};
+use crate::ir::immediates::{Ieee128, Ieee16, Ieee32, Ieee64, Offset32};
 use crate::ir::{types, ConstantData, Type};
+use core::cmp::Ordering;
 use core::fmt::{self, Display, Formatter};
 
 /// Represent a data value. Where [Value] is an SSA reference, [DataValue] is the type + value
@@ -16,8 +17,10 @@ pub enum DataValue {
     I32(i32),
     I64(i64),
     I128(i128),
+    F16(Ieee16),
     F32(Ieee32),
     F64(Ieee64),
+    F128(Ieee128),
     V128([u8; 16]),
     V64([u8; 8]),
 }
@@ -36,10 +39,14 @@ impl PartialEq for DataValue {
             (I64(_), _) => false,
             (I128(l), I128(r)) => l == r,
             (I128(_), _) => false,
+            (F16(l), F16(r)) => l.partial_cmp(&r) == Some(Ordering::Equal),
+            (F16(_), _) => false,
             (F32(l), F32(r)) => l.as_f32() == r.as_f32(),
             (F32(_), _) => false,
             (F64(l), F64(r)) => l.as_f64() == r.as_f64(),
             (F64(_), _) => false,
+            (F128(l), F128(r)) => l.partial_cmp(&r) == Some(Ordering::Equal),
+            (F128(_), _) => false,
             (V128(l), V128(r)) => l == r,
             (V128(_), _) => false,
             (V64(l), V64(r)) => l == r,
@@ -70,8 +77,10 @@ impl DataValue {
             DataValue::I32(_) => types::I32,
             DataValue::I64(_) => types::I64,
             DataValue::I128(_) => types::I128,
+            DataValue::F16(_) => types::F16,
             DataValue::F32(_) => types::F32,
             DataValue::F64(_) => types::F64,
+            DataValue::F128(_) => types::F128,
             DataValue::V128(_) => types::I8X16, // A default type.
             DataValue::V64(_) => types::I8X8,   // A default type.
         }
@@ -92,8 +101,10 @@ impl DataValue {
             DataValue::I32(i) => DataValue::I32(i.swap_bytes()),
             DataValue::I64(i) => DataValue::I64(i.swap_bytes()),
             DataValue::I128(i) => DataValue::I128(i.swap_bytes()),
+            DataValue::F16(f) => DataValue::F16(Ieee16::with_bits(f.bits().swap_bytes())),
             DataValue::F32(f) => DataValue::F32(Ieee32::with_bits(f.bits().swap_bytes())),
             DataValue::F64(f) => DataValue::F64(Ieee64::with_bits(f.bits().swap_bytes())),
+            DataValue::F128(f) => DataValue::F128(Ieee128::with_bits(f.bits().swap_bytes())),
             DataValue::V128(mut v) => {
                 v.reverse();
                 DataValue::V128(v)
@@ -135,8 +146,10 @@ impl DataValue {
             DataValue::I32(i) => dst[..4].copy_from_slice(&i.to_ne_bytes()[..]),
             DataValue::I64(i) => dst[..8].copy_from_slice(&i.to_ne_bytes()[..]),
             DataValue::I128(i) => dst[..16].copy_from_slice(&i.to_ne_bytes()[..]),
+            DataValue::F16(f) => dst[..2].copy_from_slice(&f.bits().to_ne_bytes()[..]),
             DataValue::F32(f) => dst[..4].copy_from_slice(&f.bits().to_ne_bytes()[..]),
             DataValue::F64(f) => dst[..8].copy_from_slice(&f.bits().to_ne_bytes()[..]),
+            DataValue::F128(f) => dst[..16].copy_from_slice(&f.bits().to_ne_bytes()[..]),
             DataValue::V128(v) => dst[..16].copy_from_slice(&v[..]),
             DataValue::V64(v) => dst[..8].copy_from_slice(&v[..]),
         };
@@ -172,12 +185,18 @@ impl DataValue {
             types::I32 => DataValue::I32(i32::from_ne_bytes(src[..4].try_into().unwrap())),
             types::I64 => DataValue::I64(i64::from_ne_bytes(src[..8].try_into().unwrap())),
             types::I128 => DataValue::I128(i128::from_ne_bytes(src[..16].try_into().unwrap())),
+            types::F16 => DataValue::F16(Ieee16::with_bits(u16::from_ne_bytes(
+                src[..2].try_into().unwrap(),
+            ))),
             types::F32 => DataValue::F32(Ieee32::with_bits(u32::from_ne_bytes(
                 src[..4].try_into().unwrap(),
             ))),
             types::F64 => DataValue::F64(Ieee64::with_bits(u64::from_ne_bytes(
                 src[..8].try_into().unwrap(),
             ))),
+            types::F128 => DataValue::F128(Ieee128::with_bits(u128::from_ne_bytes(
+                src[..16].try_into().unwrap(),
+            ))),
             _ if ty.is_vector() => {
                 if ty.bytes() == 16 {
                     DataValue::V128(src[..16].try_into().unwrap())
@@ -233,8 +252,10 @@ impl DataValue {
             // We need to bit compare the floats to ensure that we produce the correct values
             // on NaN's. The test suite expects to assert the precise bit pattern on NaN's or
             // works around it in the tests themselves.
+            (DataValue::F16(a), DataValue::F16(b)) => a.bits() == b.bits(),
             (DataValue::F32(a), DataValue::F32(b)) => a.bits() == b.bits(),
             (DataValue::F64(a), DataValue::F64(b)) => a.bits() == b.bits(),
+            (DataValue::F128(a), DataValue::F128(b)) => a.bits() == b.bits(),
 
             // We don't need to worry about F32x4 / F64x2 Since we compare V128 which is already the
             // raw bytes anyway
@@ -305,8 +326,10 @@ build_conversion_impl!(i16, I16, I16);
 build_conversion_impl!(i32, I32, I32);
 build_conversion_impl!(i64, I64, I64);
 build_conversion_impl!(i128, I128, I128);
+build_conversion_impl!(Ieee16, F16, F16);
 build_conversion_impl!(Ieee32, F32, F32);
 build_conversion_impl!(Ieee64, F64, F64);
+build_conversion_impl!(Ieee128, F128, F128);
 build_conversion_impl!([u8; 16], V128, I8X16);
 build_conversion_impl!([u8; 8], V64, I8X8);
 impl From<Offset32> for DataValue {
@@ -324,8 +347,10 @@ impl Display for DataValue {
             DataValue::I64(dv) => write!(f, "{}", dv),
             DataValue::I128(dv) => write!(f, "{}", dv),
             // The Ieee* wrappers here print the expected syntax.
+            DataValue::F16(dv) => write!(f, "{}", dv),
             DataValue::F32(dv) => write!(f, "{}", dv),
             DataValue::F64(dv) => write!(f, "{}", dv),
+            DataValue::F128(dv) => write!(f, "{}", dv),
             // Again, for syntax consistency, use ConstantData, which in this case displays as hex.
             DataValue::V128(dv) => write!(f, "{}", ConstantData::from(&dv[..])),
             DataValue::V64(dv) => write!(f, "{}", ConstantData::from(&dv[..])),
diff --git a/cranelift/codegen/src/ir/immediates.rs b/cranelift/codegen/src/ir/immediates.rs
index 153f331b9..c5f5d7bc6 100644
--- a/cranelift/codegen/src/ir/immediates.rs
+++ b/cranelift/codegen/src/ir/immediates.rs
@@ -468,6 +468,21 @@ impl FromStr for Offset32 {
     }
 }
 
+/// An IEEE binary16 immediate floating point value, represented as a u16
+/// containing the bit pattern.
+///
+/// We specifically avoid using a f16 here since some architectures may silently alter floats.
+/// See: <https://github.com/bytecodealliance/wasmtime/pull/2251#discussion_r498508646>
+///
+/// The [PartialEq] and [Hash] implementations are over the underlying bit pattern, but
+/// [PartialOrd] respects IEEE754 semantics.
+///
+/// All bit patterns are allowed.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[repr(C)]
+pub struct Ieee16(u16);
+
 /// An IEEE binary32 immediate floating point value, represented as a u32
 /// containing the bit pattern.
 ///
@@ -498,6 +513,21 @@ pub struct Ieee32(u32);
 #[repr(C)]
 pub struct Ieee64(u64);
 
+/// An IEEE binary128 immediate floating point value, represented as a u128
+/// containing the bit pattern.
+///
+/// We specifically avoid using a f128 here since some architectures may silently alter floats.
+/// See: <https://github.com/bytecodealliance/wasmtime/pull/2251#discussion_r498508646>
+///
+/// The [PartialEq] and [Hash] implementations are over the underlying bit pattern, but
+/// [PartialOrd] respects IEEE754 semantics.
+///
+/// All bit patterns are allowed.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[repr(C)]
+pub struct Ieee128(u128);
+
 /// Format a floating point number in a way that is reasonably human-readable, and that can be
 /// converted back to binary without any rounding issues. The hexadecimal formatting of normal and
 /// subnormal numbers is compatible with C99 and the `printf "%a"` format specifier. The NaN and Inf
@@ -508,13 +538,13 @@ pub struct Ieee64(u64);
 /// w - exponent field width in bits
 /// t - trailing significand field width in bits
 ///
-fn format_float(bits: u64, w: u8, t: u8, f: &mut Formatter) -> fmt::Result {
+fn format_float(bits: u128, w: u8, t: u8, f: &mut Formatter) -> fmt::Result {
     debug_assert!(w > 0 && w <= 16, "Invalid exponent range");
-    debug_assert!(1 + w + t <= 64, "Too large IEEE format for u64");
+    debug_assert!(1 + w + t <= 128, "Too large IEEE format for u128");
     debug_assert!((t + w + 1).is_power_of_two(), "Unexpected IEEE format size");
 
-    let max_e_bits = (1u64 << w) - 1;
-    let t_bits = bits & ((1u64 << t) - 1); // Trailing significand.
+    let max_e_bits = (1u128 << w) - 1;
+    let t_bits = bits & ((1u128 << t) - 1); // Trailing significand.
     let e_bits = (bits >> t) & max_e_bits; // Biased exponent.
     let sign_bit = (bits >> (w + t)) & 1;
 
@@ -583,13 +613,13 @@ fn format_float(bits: u64, w: u8, t: u8, f: &mut Formatter) -> fmt::Result {
 /// w - exponent field width in bits
 /// t - trailing significand field width in bits
 ///
-fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
+fn parse_float(s: &str, w: u8, t: u8) -> Result<u128, &'static str> {
     debug_assert!(w > 0 && w <= 16, "Invalid exponent range");
-    debug_assert!(1 + w + t <= 64, "Too large IEEE format for u64");
+    debug_assert!(1 + w + t <= 128, "Too large IEEE format for u128");
     debug_assert!((t + w + 1).is_power_of_two(), "Unexpected IEEE format size");
 
     let (sign_bit, s2) = if s.starts_with('-') {
-        (1u64 << (t + w), &s[1..])
+        (1u128 << (t + w), &s[1..])
     } else if s.starts_with('+') {
         (0, &s[1..])
     } else {
@@ -597,8 +627,8 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
     };
 
     if !s2.starts_with("0x") {
-        let max_e_bits = ((1u64 << w) - 1) << t;
-        let quiet_bit = 1u64 << (t - 1);
+        let max_e_bits = ((1u128 << w) - 1) << t;
+        let quiet_bit = 1u128 << (t - 1);
 
         // The only decimal encoding allowed is 0.
         if s2 == "0.0" {
@@ -615,7 +645,7 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
         }
         if s2.starts_with("NaN:0x") {
             // Quiet NaN with payload.
-            return match u64::from_str_radix(&s2[6..], 16) {
+            return match u128::from_str_radix(&s2[6..], 16) {
                 Ok(payload) if payload < quiet_bit => {
                     Ok(sign_bit | max_e_bits | quiet_bit | payload)
                 }
@@ -624,7 +654,7 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
         }
         if s2.starts_with("sNaN:0x") {
             // Signaling NaN with payload.
-            return match u64::from_str_radix(&s2[7..], 16) {
+            return match u128::from_str_radix(&s2[7..], 16) {
                 Ok(payload) if 0 < payload && payload < quiet_bit => {
                     Ok(sign_bit | max_e_bits | payload)
                 }
@@ -638,7 +668,7 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
 
     let mut digits = 0u8;
     let mut digits_before_period: Option<u8> = None;
-    let mut significand = 0u64;
+    let mut significand = 0u128;
     let mut exponent = 0i32;
 
     for (idx, ch) in s3.char_indices() {
@@ -665,10 +695,10 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
             _ => match ch.to_digit(16) {
                 Some(digit) => {
                     digits += 1;
-                    if digits > 16 {
+                    if digits > 32 {
                         return Err("Too many digits");
                     }
-                    significand = (significand << 4) | u64::from(digit);
+                    significand = (significand << 4) | u128::from(digit);
                 }
                 None => return Err("Invalid character"),
             },
@@ -691,10 +721,10 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
     };
 
     // Normalize the significand and exponent.
-    let significant_bits = (64 - significand.leading_zeros()) as u8;
+    let significant_bits = (128 - significand.leading_zeros()) as u8;
     if significant_bits > t + 1 {
         let adjust = significant_bits - (t + 1);
-        if significand & ((1u64 << adjust) - 1) != 0 {
+        if significand & ((1u128 << adjust) - 1) != 0 {
             return Err("Too many significant bits");
         }
         // Adjust significand down.
@@ -718,13 +748,13 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
         Err("Magnitude too large")
     } else if exponent > 0 {
         // This is a normal number.
-        let e_bits = (exponent as u64) << t;
+        let e_bits = (exponent as u128) << t;
         Ok(sign_bit | e_bits | t_bits)
     } else if 1 - exponent <= i32::from(t) {
         // This is a subnormal number: e = 0, t = significand bits.
         // Renormalize significand for exponent = 1.
         let adjust = 1 - exponent;
-        if significand & ((1u64 << adjust) - 1) != 0 {
+        if significand & ((1u128 << adjust) - 1) != 0 {
             Err("Subnormal underflow")
         } else {
             significand >>= adjust;
@@ -735,6 +765,84 @@ fn parse_float(s: &str, w: u8, t: u8) -> Result<u64, &'static str> {
     }
 }
 
+impl Ieee16 {
+    const SIGNIFICAND_BITS: u8 = 10;
+    const EXPONENT_BITS: u8 = 5;
+
+    /// Create a new `Ieee16` containing the bits of `x`.
+    pub fn with_bits(x: u16) -> Self {
+        Self(x)
+    }
+
+    /// Get the bitwise representation.
+    pub fn bits(self) -> u16 {
+        self.0
+    }
+
+    /// Returns true if self is positive or negative zero
+    pub fn is_zero(&self) -> bool {
+        self.partial_cmp(&Self::with_bits(0)) == Some(Ordering::Equal)
+    }
+}
+
+impl PartialOrd for Ieee16 {
+    fn partial_cmp(&self, rhs: &Self) -> Option<Ordering> {
+        // FIXME(#8312): Use Rust `f16` comparisons once `f16` support is stabalised.
+        let significand_mask = u16::MAX >> (Self::EXPONENT_BITS + 1);
+        let sign_mask = 1 << (Self::EXPONENT_BITS + Self::SIGNIFICAND_BITS);
+        let exponent_mask = !sign_mask & !significand_mask;
+
+        let lhs_abs = self.bits() & !sign_mask;
+        let rhs_abs = rhs.bits() & !sign_mask;
+        if (lhs_abs & exponent_mask == exponent_mask && lhs_abs & significand_mask != 0)
+            && (rhs_abs & exponent_mask == exponent_mask && rhs_abs & significand_mask != 0)
+        {
+            // One of the floats is a NaN.
+            return None;
+        }
+        if lhs_abs == 0 && rhs_abs == 0 {
+            // Zeros are always equal regardless of sign.
+            return Some(Ordering::Equal);
+        }
+        let lhs_positive = self.bits() & sign_mask == 0;
+        let rhs_positive = rhs.bits() & sign_mask == 0;
+        if lhs_positive != rhs_positive {
+            // Different signs: negative < positive
+            return lhs_positive.partial_cmp(&rhs_positive);
+        }
+        // Finite or infinity will order correctly with an integer comparison of the bits.
+        if lhs_positive {
+            lhs_abs.partial_cmp(&rhs_abs)
+        } else {
+            // Reverse the comparison when both floats are negative.
+            rhs_abs.partial_cmp(&lhs_abs)
+        }
+    }
+}
+
+impl Display for Ieee16 {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        let bits: u16 = self.0;
+        format_float(
+            u128::from(bits),
+            Self::EXPONENT_BITS,
+            Self::SIGNIFICAND_BITS,
+            f,
+        )
+    }
+}
+
+impl FromStr for Ieee16 {
+    type Err = &'static str;
+
+    fn from_str(s: &str) -> Result<Self, &'static str> {
+        match parse_float(s, Self::EXPONENT_BITS, Self::SIGNIFICAND_BITS) {
+            Ok(b) => Ok(Self(b as u16)),
+            Err(s) => Err(s),
+        }
+    }
+}
+
 impl Ieee32 {
     /// Create a new `Ieee32` containing the bits of `x`.
     pub fn with_bits(x: u32) -> Self {
@@ -853,7 +961,7 @@ impl PartialOrd for Ieee32 {
 impl Display for Ieee32 {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         let bits: u32 = self.0;
-        format_float(u64::from(bits), 8, 23, f)
+        format_float(u128::from(bits), 8, 23, f)
     }
 }
 
@@ -1071,7 +1179,7 @@ impl PartialOrd for Ieee64 {
 impl Display for Ieee64 {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         let bits: u64 = self.0;
-        format_float(bits, 11, 52, f)
+        format_float(u128::from(bits), 11, 52, f)
     }
 }
 
@@ -1080,7 +1188,7 @@ impl FromStr for Ieee64 {
 
     fn from_str(s: &str) -> Result<Self, &'static str> {
         match parse_float(s, 11, 52) {
-            Ok(b) => Ok(Self(b)),
+            Ok(b) => Ok(Self(b as u64)),
             Err(s) => Err(s),
         }
     }
@@ -1176,6 +1284,79 @@ impl Not for Ieee64 {
     }
 }
 
+impl Ieee128 {
+    const SIGNIFICAND_BITS: u8 = 112;
+    const EXPONENT_BITS: u8 = 15;
+
+    /// Create a new `Ieee128` containing the bits of `x`.
+    pub fn with_bits(x: u128) -> Self {
+        Self(x)
+    }
+
+    /// Get the bitwise representation.
+    pub fn bits(self) -> u128 {
+        self.0
+    }
+
+    /// Returns true if self is positive or negative zero
+    pub fn is_zero(&self) -> bool {
+        self.partial_cmp(&Self::with_bits(0)) == Some(Ordering::Equal)
+    }
+}
+
+impl PartialOrd for Ieee128 {
+    fn partial_cmp(&self, rhs: &Self) -> Option<Ordering> {
+        // FIXME(#8312): Use Rust `f128` comparisons once `f128` support is stabalised.
+        let significand_mask = u128::MAX >> (Self::EXPONENT_BITS + 1);
+        let sign_mask = 1 << (Self::EXPONENT_BITS + Self::SIGNIFICAND_BITS);
+        let exponent_mask = !sign_mask & !significand_mask;
+
+        let lhs_abs = self.bits() & !sign_mask;
+        let rhs_abs = rhs.bits() & !sign_mask;
+        if (lhs_abs & exponent_mask == exponent_mask && lhs_abs & significand_mask != 0)
+            && (rhs_abs & exponent_mask == exponent_mask && rhs_abs & significand_mask != 0)
+        {
+            // One of the floats is a NaN.
+            return None;
+        }
+        if lhs_abs == 0 && rhs_abs == 0 {
+            // Zeros are always equal regardless of sign.
+            return Some(Ordering::Equal);
+        }
+        let lhs_positive = self.bits() & sign_mask == 0;
+        let rhs_positive = rhs.bits() & sign_mask == 0;
+        if lhs_positive != rhs_positive {
+            // Different signs: negative < positive
+            return lhs_positive.partial_cmp(&rhs_positive);
+        }
+        // Finite or infinity will order correctly with an integer comparison of the bits.
+        if lhs_positive {
+            lhs_abs.partial_cmp(&rhs_abs)
+        } else {
+            // Reverse the comparison when both floats are negative.
+            rhs_abs.partial_cmp(&lhs_abs)
+        }
+    }
+}
+
+impl Display for Ieee128 {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        let bits: u128 = self.0;
+        format_float(bits, Self::EXPONENT_BITS, Self::SIGNIFICAND_BITS, f)
+    }
+}
+
+impl FromStr for Ieee128 {
+    type Err = &'static str;
+
+    fn from_str(s: &str) -> Result<Self, &'static str> {
+        match parse_float(s, Self::EXPONENT_BITS, Self::SIGNIFICAND_BITS) {
+            Ok(b) => Ok(Self(b)),
+            Err(s) => Err(s),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1346,6 +1527,134 @@ mod tests {
         parse_err::<Offset32>("+0x8000_0000", "Offset out of range");
     }
 
+    #[test]
+    fn format_ieee16() {
+        assert_eq!(Ieee16::with_bits(0).to_string(), "0.0"); // 0.0
+        assert_eq!(Ieee16::with_bits(0x8000).to_string(), "-0.0"); // -0.0
+        assert_eq!(Ieee16::with_bits(0x3c00).to_string(), "0x1.000p0"); // 1.0
+        assert_eq!(Ieee16::with_bits(0x3e00).to_string(), "0x1.800p0"); // 1.5
+        assert_eq!(Ieee16::with_bits(0x3800).to_string(), "0x1.000p-1"); // 0.5
+        assert_eq!(
+            Ieee16::with_bits(0x1400).to_string(), // `f16::EPSILON`
+            "0x1.000p-10"
+        );
+        assert_eq!(
+            Ieee16::with_bits(0xfbff).to_string(), // `f16::MIN`
+            "-0x1.ffcp15"
+        );
+        assert_eq!(
+            Ieee16::with_bits(0x7bff).to_string(), // `f16::MAX`
+            "0x1.ffcp15"
+        );
+        // Smallest positive normal number.
+        assert_eq!(
+            Ieee16::with_bits(0x0400).to_string(), // `f16::MIN_POSITIVE`
+            "0x1.000p-14"
+        );
+        // Subnormals.
+        assert_eq!(
+            Ieee16::with_bits(0x0200).to_string(), // `f16::MIN_POSITIVE / 2.0`
+            "0x0.800p-14"
+        );
+        assert_eq!(
+            Ieee16::with_bits(0x0001).to_string(), // `f16::MIN_POSITIVE * f16::EPSILON`
+            "0x0.004p-14"
+        );
+        assert_eq!(
+            Ieee16::with_bits(0x7c00).to_string(), // `f16::INFINITY`
+            "+Inf"
+        );
+        assert_eq!(
+            Ieee16::with_bits(0xfc00).to_string(), // `f16::NEG_INFINITY`
+            "-Inf"
+        );
+        assert_eq!(
+            Ieee16::with_bits(0x7e00).to_string(), // `f16::NAN`
+            "+NaN"
+        );
+        assert_eq!(
+            Ieee16::with_bits(0xfe00).to_string(), // `-f16::NAN`
+            "-NaN"
+        );
+        // Construct some qNaNs with payloads.
+        assert_eq!(Ieee16(0x7e01).to_string(), "+NaN:0x1");
+        assert_eq!(Ieee16(0x7f01).to_string(), "+NaN:0x101");
+        // Signaling NaNs.
+        assert_eq!(Ieee16(0x7c01).to_string(), "+sNaN:0x1");
+        assert_eq!(Ieee16(0x7d01).to_string(), "+sNaN:0x101");
+    }
+
+    #[test]
+    fn parse_ieee16() {
+        parse_ok::<Ieee16>("0.0", "0.0");
+        parse_ok::<Ieee16>("+0.0", "0.0");
+        parse_ok::<Ieee16>("-0.0", "-0.0");
+        parse_ok::<Ieee16>("0x0", "0.0");
+        parse_ok::<Ieee16>("0x0.0", "0.0");
+        parse_ok::<Ieee16>("0x.0", "0.0");
+        parse_ok::<Ieee16>("0x0.", "0.0");
+        parse_ok::<Ieee16>("0x1", "0x1.000p0");
+        parse_ok::<Ieee16>("+0x1", "0x1.000p0");
+        parse_ok::<Ieee16>("-0x1", "-0x1.000p0");
+        parse_ok::<Ieee16>("0x10", "0x1.000p4");
+        parse_ok::<Ieee16>("0x10.0", "0x1.000p4");
+        parse_err::<Ieee16>("0.", "Float must be hexadecimal");
+        parse_err::<Ieee16>(".0", "Float must be hexadecimal");
+        parse_err::<Ieee16>("0", "Float must be hexadecimal");
+        parse_err::<Ieee16>("-0", "Float must be hexadecimal");
+        parse_err::<Ieee16>(".", "Float must be hexadecimal");
+        parse_err::<Ieee16>("", "Float must be hexadecimal");
+        parse_err::<Ieee16>("-", "Float must be hexadecimal");
+        parse_err::<Ieee16>("0x", "No digits");
+        parse_err::<Ieee16>("0x..", "Multiple radix points");
+
+        // Check significant bits.
+        parse_ok::<Ieee16>("0x0.ffe", "0x1.ffcp-1");
+        parse_ok::<Ieee16>("0x1.ffc", "0x1.ffcp0");
+        parse_ok::<Ieee16>("0x3.ff8", "0x1.ffcp1");
+        parse_ok::<Ieee16>("0x7.ff", "0x1.ffcp2");
+        parse_ok::<Ieee16>("0xf.fe", "0x1.ffcp3");
+        parse_err::<Ieee16>("0x1.ffe", "Too many significant bits");
+        parse_err::<Ieee16>("0x1.ffc00000000000000000000000000000", "Too many digits");
+
+        // Exponents.
+        parse_ok::<Ieee16>("0x1p3", "0x1.000p3");
+        parse_ok::<Ieee16>("0x1p-3", "0x1.000p-3");
+        parse_ok::<Ieee16>("0x1.0p3", "0x1.000p3");
+        parse_ok::<Ieee16>("0x2.0p3", "0x1.000p4");
+        parse_ok::<Ieee16>("0x1.0p15", "0x1.000p15");
+        parse_ok::<Ieee16>("0x1.0p-14", "0x1.000p-14");
+        parse_ok::<Ieee16>("0x0.1p-10", "0x1.000p-14");
+        parse_err::<Ieee16>("0x2.0p15", "Magnitude too large");
+
+        // Subnormals.
+        parse_ok::<Ieee16>("0x1.0p-15", "0x0.800p-14");
+        parse_ok::<Ieee16>("0x1.0p-24", "0x0.004p-14");
+        parse_ok::<Ieee16>("0x0.004p-14", "0x0.004p-14");
+        parse_err::<Ieee16>("0x0.102p-14", "Subnormal underflow");
+        parse_err::<Ieee16>("0x1.8p-24", "Subnormal underflow");
+        parse_err::<Ieee16>("0x1.0p-25", "Magnitude too small");
+
+        // NaNs and Infs.
+        parse_ok::<Ieee16>("Inf", "+Inf");
+        parse_ok::<Ieee16>("+Inf", "+Inf");
+        parse_ok::<Ieee16>("-Inf", "-Inf");
+        parse_ok::<Ieee16>("NaN", "+NaN");
+        parse_ok::<Ieee16>("+NaN", "+NaN");
+        parse_ok::<Ieee16>("-NaN", "-NaN");
+        parse_ok::<Ieee16>("NaN:0x0", "+NaN");
+        parse_err::<Ieee16>("NaN:", "Float must be hexadecimal");
+        parse_err::<Ieee16>("NaN:0", "Float must be hexadecimal");
+        parse_err::<Ieee16>("NaN:0x", "Invalid NaN payload");
+        parse_ok::<Ieee16>("NaN:0x001", "+NaN:0x1");
+        parse_ok::<Ieee16>("NaN:0x101", "+NaN:0x101");
+        parse_err::<Ieee16>("NaN:0x301", "Invalid NaN payload");
+        parse_ok::<Ieee16>("sNaN:0x1", "+sNaN:0x1");
+        parse_err::<Ieee16>("sNaN:0x0", "Invalid sNaN payload");
+        parse_ok::<Ieee16>("sNaN:0x101", "+sNaN:0x101");
+        parse_err::<Ieee16>("sNaN:0x301", "Invalid sNaN payload");
+    }
+
     #[test]
     fn format_ieee32() {
         assert_eq!(Ieee32::with_float(0.0).to_string(), "0.0");
@@ -1416,7 +1725,7 @@ mod tests {
         parse_ok::<Ieee32>("0x7.fffff8", "0x1.fffffep2");
         parse_ok::<Ieee32>("0xf.fffff0", "0x1.fffffep3");
         parse_err::<Ieee32>("0x1.ffffff", "Too many significant bits");
-        parse_err::<Ieee32>("0x1.fffffe0000000000", "Too many digits");
+        parse_err::<Ieee32>("0x1.fffffe00000000000000000000000000", "Too many digits");
 
         // Exponents.
         parse_ok::<Ieee32>("0x1p3", "0x1.000000p3");
@@ -1556,7 +1865,7 @@ mod tests {
         parse_ok::<Ieee64>("0x7.ffffffffffffc", "0x1.fffffffffffffp2");
         parse_ok::<Ieee64>("0xf.ffffffffffff8", "0x1.fffffffffffffp3");
         parse_err::<Ieee64>("0x3.fffffffffffff", "Too many significant bits");
-        parse_err::<Ieee64>("0x001.fffffe00000000", "Too many digits");
+        parse_err::<Ieee64>("0x001.fffffe000000000000000000000000", "Too many digits");
 
         // Exponents.
         parse_ok::<Ieee64>("0x1p3", "0x1.0000000000000p3");
@@ -1613,4 +1922,188 @@ mod tests {
             });
         }
     }
+
+    #[test]
+    fn format_ieee128() {
+        assert_eq!(
+            Ieee128::with_bits(0x00000000000000000000000000000000).to_string(), // 0.0
+            "0.0"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x80000000000000000000000000000000).to_string(), // -0.0
+            "-0.0"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x3fff0000000000000000000000000000).to_string(), // 1.0
+            "0x1.0000000000000000000000000000p0"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x3fff8000000000000000000000000000).to_string(), // 1.5
+            "0x1.8000000000000000000000000000p0"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x3ffe0000000000000000000000000000).to_string(), // 0.5
+            "0x1.0000000000000000000000000000p-1"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x3f8f0000000000000000000000000000).to_string(), // `f128::EPSILON`
+            "0x1.0000000000000000000000000000p-112"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0xfffeffffffffffffffffffffffffffff).to_string(), // `f128::MIN`
+            "-0x1.ffffffffffffffffffffffffffffp16383"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x7ffeffffffffffffffffffffffffffff).to_string(), // `f128::MAX`
+            "0x1.ffffffffffffffffffffffffffffp16383"
+        );
+        // Smallest positive normal number.
+        assert_eq!(
+            Ieee128::with_bits(0x00010000000000000000000000000000).to_string(), // `f128::MIN_POSITIVE`
+            "0x1.0000000000000000000000000000p-16382"
+        );
+        // Subnormals.
+        assert_eq!(
+            Ieee128::with_bits(0x00008000000000000000000000000000).to_string(), // `f128::MIN_POSITIVE / 2.0`
+            "0x0.8000000000000000000000000000p-16382"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x00000000000000000000000000000001).to_string(), // `f128::MIN_POSITIVE * f128::EPSILON`
+            "0x0.0000000000000000000000000001p-16382"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x7fff0000000000000000000000000000).to_string(), // `f128::INFINITY`
+            "+Inf"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0xffff0000000000000000000000000000).to_string(), // `f128::NEG_INFINITY`
+            "-Inf"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0x7fff8000000000000000000000000000).to_string(), // `f128::NAN`
+            "+NaN"
+        );
+        assert_eq!(
+            Ieee128::with_bits(0xffff8000000000000000000000000000).to_string(), // `-f128::NAN`
+            "-NaN"
+        );
+        // Construct some qNaNs with payloads.
+        assert_eq!(
+            Ieee128(0x7fff8000000000000000000000000001).to_string(),
+            "+NaN:0x1"
+        );
+        assert_eq!(
+            Ieee128(0x7fffc000000000000000000000000001).to_string(),
+            "+NaN:0x4000000000000000000000000001"
+        );
+        // Signaling NaNs.
+        assert_eq!(
+            Ieee128(0x7fff0000000000000000000000000001).to_string(),
+            "+sNaN:0x1"
+        );
+        assert_eq!(
+            Ieee128(0x7fff4000000000000000000000000001).to_string(),
+            "+sNaN:0x4000000000000000000000000001"
+        );
+    }
+
+    #[test]
+    fn parse_ieee128() {
+        parse_ok::<Ieee128>("0.0", "0.0");
+        parse_ok::<Ieee128>("-0.0", "-0.0");
+        parse_ok::<Ieee128>("0x0", "0.0");
+        parse_ok::<Ieee128>("0x0.0", "0.0");
+        parse_ok::<Ieee128>("0x.0", "0.0");
+        parse_ok::<Ieee128>("0x0.", "0.0");
+        parse_ok::<Ieee128>("0x1", "0x1.0000000000000000000000000000p0");
+        parse_ok::<Ieee128>("-0x1", "-0x1.0000000000000000000000000000p0");
+        parse_ok::<Ieee128>("0x10", "0x1.0000000000000000000000000000p4");
+        parse_ok::<Ieee128>("0x10.0", "0x1.0000000000000000000000000000p4");
+        parse_err::<Ieee128>("0.", "Float must be hexadecimal");
+        parse_err::<Ieee128>(".0", "Float must be hexadecimal");
+        parse_err::<Ieee128>("0", "Float must be hexadecimal");
+        parse_err::<Ieee128>("-0", "Float must be hexadecimal");
+        parse_err::<Ieee128>(".", "Float must be hexadecimal");
+        parse_err::<Ieee128>("", "Float must be hexadecimal");
+        parse_err::<Ieee128>("-", "Float must be hexadecimal");
+        parse_err::<Ieee128>("0x", "No digits");
+        parse_err::<Ieee128>("0x..", "Multiple radix points");
+
+        // Check significant bits.
+        parse_ok::<Ieee128>(
+            "0x0.ffffffffffffffffffffffffffff8",
+            "0x1.ffffffffffffffffffffffffffffp-1",
+        );
+        parse_ok::<Ieee128>(
+            "0x1.ffffffffffffffffffffffffffff",
+            "0x1.ffffffffffffffffffffffffffffp0",
+        );
+        parse_ok::<Ieee128>(
+            "0x3.fffffffffffffffffffffffffffe",
+            "0x1.ffffffffffffffffffffffffffffp1",
+        );
+        parse_ok::<Ieee128>(
+            "0x7.fffffffffffffffffffffffffffc",
+            "0x1.ffffffffffffffffffffffffffffp2",
+        );
+        parse_ok::<Ieee128>(
+            "0xf.fffffffffffffffffffffffffff8",
+            "0x1.ffffffffffffffffffffffffffffp3",
+        );
+        parse_err::<Ieee128>(
+            "0x3.ffffffffffffffffffffffffffff",
+            "Too many significant bits",
+        );
+        parse_err::<Ieee128>("0x001.fffffe000000000000000000000000", "Too many digits");
+
+        // Exponents.
+        parse_ok::<Ieee128>("0x1p3", "0x1.0000000000000000000000000000p3");
+        parse_ok::<Ieee128>("0x1p-3", "0x1.0000000000000000000000000000p-3");
+        parse_ok::<Ieee128>("0x1.0p3", "0x1.0000000000000000000000000000p3");
+        parse_ok::<Ieee128>("0x2.0p3", "0x1.0000000000000000000000000000p4");
+        parse_ok::<Ieee128>("0x1.0p16383", "0x1.0000000000000000000000000000p16383");
+        parse_ok::<Ieee128>("0x1.0p-16382", "0x1.0000000000000000000000000000p-16382");
+        parse_ok::<Ieee128>("0x0.1p-16378", "0x1.0000000000000000000000000000p-16382");
+        parse_err::<Ieee128>("0x2.0p16383", "Magnitude too large");
+
+        // Subnormals.
+        parse_ok::<Ieee128>("0x1.0p-16383", "0x0.8000000000000000000000000000p-16382");
+        parse_ok::<Ieee128>("0x1.0p-16494", "0x0.0000000000000000000000000001p-16382");
+        parse_ok::<Ieee128>(
+            "0x0.0000000000000000000000000001p-16382",
+            "0x0.0000000000000000000000000001p-16382",
+        );
+        parse_err::<Ieee128>(
+            "0x0.10000000000000000000000000008p-16382",
+            "Subnormal underflow",
+        );
+        parse_err::<Ieee128>("0x1.8p-16494", "Subnormal underflow");
+        parse_err::<Ieee128>("0x1.0p-16495", "Magnitude too small");
+
+        // NaNs and Infs.
+        parse_ok::<Ieee128>("Inf", "+Inf");
+        parse_ok::<Ieee128>("-Inf", "-Inf");
+        parse_ok::<Ieee128>("NaN", "+NaN");
+        parse_ok::<Ieee128>("-NaN", "-NaN");
+        parse_ok::<Ieee128>("NaN:0x0", "+NaN");
+        parse_err::<Ieee128>("NaN:", "Float must be hexadecimal");
+        parse_err::<Ieee128>("NaN:0", "Float must be hexadecimal");
+        parse_err::<Ieee128>("NaN:0x", "Invalid NaN payload");
+        parse_ok::<Ieee128>("NaN:0x000001", "+NaN:0x1");
+        parse_ok::<Ieee128>(
+            "NaN:0x4000000000000000000000000001",
+            "+NaN:0x4000000000000000000000000001",
+        );
+        parse_err::<Ieee128>("NaN:0x8000000000000000000000000001", "Invalid NaN payload");
+        parse_ok::<Ieee128>("sNaN:0x1", "+sNaN:0x1");
+        parse_err::<Ieee128>("sNaN:0x0", "Invalid sNaN payload");
+        parse_ok::<Ieee128>(
+            "sNaN:0x4000000000000000000000000001",
+            "+sNaN:0x4000000000000000000000000001",
+        );
+        parse_err::<Ieee128>(
+            "sNaN:0x8000000000000000000000000001",
+            "Invalid sNaN payload",
+        );
+    }
 }
diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs
index fd5208070..6d483aba3 100644
--- a/cranelift/codegen/src/ir/instructions.rs
+++ b/cranelift/codegen/src/ir/instructions.rs
@@ -794,8 +794,8 @@ impl OperandConstraint {
                     tys.ints = BitSet8::from_range(3, ctrl_type_bits as u8);
                 } else if ctrl_type.is_float() {
                     // The upper bound in from_range is exclusive, and we want to exclude the
-                    // control type to construct the interval of [F32, ctrl_type).
-                    tys.floats = BitSet8::from_range(5, ctrl_type_bits as u8);
+                    // control type to construct the interval of [F16, ctrl_type).
+                    tys.floats = BitSet8::from_range(4, ctrl_type_bits as u8);
                 } else {
                     panic!("The Narrower constraint only operates on floats or ints");
                 }
@@ -822,10 +822,11 @@ impl OperandConstraint {
                         tys.ints = BitSet8::from_range(lower_bound, 8);
                     }
                 } else if ctrl_type.is_float() {
-                    // The interval should include all float types wider than `ctrl_type`, so we
-                    // use `2^7` as the upper bound, and add one to the bits of `ctrl_type` to
-                    // define the interval `(ctrl_type, F64]`.
-                    tys.floats = BitSet8::from_range(ctrl_type_bits as u8 + 1, 7);
+                    // Same as above but for `tys.floats`, as the largest float type is F128.
+                    let lower_bound = ctrl_type_bits as u8 + 1;
+                    if lower_bound < BitSet8::capacity() {
+                        tys.floats = BitSet8::from_range(lower_bound, 8);
+                    }
                 } else {
                     panic!("The Wider constraint only operates on floats or ints");
                 }
@@ -964,7 +965,9 @@ mod tests {
         assert!(vts.contains(I64));
         assert!(vts.contains(I32X4));
         assert!(vts.contains(I32X4XN));
+        assert!(!vts.contains(F16));
         assert!(!vts.contains(F32));
+        assert!(!vts.contains(F128));
         assert!(vts.contains(R32));
         assert!(vts.contains(R64));
         assert_eq!(vts.example().to_string(), "i32");
diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs
index b17e2b111..e6f082d70 100644
--- a/cranelift/codegen/src/ir/mod.rs
+++ b/cranelift/codegen/src/ir/mod.rs
@@ -65,7 +65,7 @@ pub use crate::ir::stackslot::{
 };
 pub use crate::ir::trapcode::TrapCode;
 pub use crate::ir::types::Type;
-pub use crate::ir::user_stack_maps::UserStackMapEntry;
+pub use crate::ir::user_stack_maps::{UserStackMap, UserStackMapEntry};
 
 use crate::entity::{entity_impl, PrimaryMap, SecondaryMap};
 
diff --git a/cranelift/codegen/src/ir/types.rs b/cranelift/codegen/src/ir/types.rs
index 928a28526..a62fffa61 100644
--- a/cranelift/codegen/src/ir/types.rs
+++ b/cranelift/codegen/src/ir/types.rs
@@ -14,7 +14,7 @@ use target_lexicon::{PointerWidth, Triple};
 ///
 /// Basic integer types: `I8`, `I16`, `I32`, `I64`, and `I128`. These types are sign-agnostic.
 ///
-/// Basic floating point types: `F32` and `F64`. IEEE single and double precision.
+/// Basic floating point types: `F16`, `F32`, `F64`, and `F128`. IEEE half, single, double, and quadruple precision.
 ///
 /// SIMD vector types have power-of-two lanes, up to 256. Lanes can be any int/float type.
 ///
@@ -56,10 +56,10 @@ impl Type {
     pub fn log2_lane_bits(self) -> u32 {
         match self.lane_type() {
             I8 => 3,
-            I16 => 4,
+            I16 | F16 => 4,
             I32 | F32 | R32 => 5,
             I64 | F64 | R64 => 6,
-            I128 => 7,
+            I128 | F128 => 7,
             _ => 0,
         }
     }
@@ -68,10 +68,10 @@ impl Type {
     pub fn lane_bits(self) -> u32 {
         match self.lane_type() {
             I8 => 8,
-            I16 => 16,
+            I16 | F16 => 16,
             I32 | F32 | R32 => 32,
             I64 | F64 | R64 => 64,
-            I128 => 128,
+            I128 | F128 => 128,
             _ => 0,
         }
     }
@@ -136,11 +136,11 @@ impl Type {
         // Replace the low 4 bits with the boolean version, preserve the high 4 bits.
         self.replace_lanes(match self.lane_type() {
             I8 => I8,
-            I16 => I16,
+            I16 | F16 => I16,
             I32 | F32 => I32,
             I64 | F64 => I64,
             R32 | R64 => panic!("Reference types are not truthy"),
-            I128 => I128,
+            I128 | F128 => I128,
             _ => I8,
         })
     }
@@ -161,10 +161,10 @@ impl Type {
     pub fn as_int(self) -> Self {
         self.replace_lanes(match self.lane_type() {
             I8 => I8,
-            I16 => I16,
+            I16 | F16 => I16,
             I32 | F32 | R32 => I32,
             I64 | F64 | R64 => I64,
-            I128 => I128,
+            I128 | F128 => I128,
             _ => unimplemented!(),
         })
     }
@@ -177,7 +177,9 @@ impl Type {
             I32 => I16,
             I64 => I32,
             I128 => I64,
+            F32 => F16,
             F64 => F32,
+            F128 => F64,
             _ => return None,
         }))
     }
@@ -190,7 +192,9 @@ impl Type {
             I16 => I32,
             I32 => I64,
             I64 => I128,
+            F16 => F32,
             F32 => F64,
+            F64 => F128,
             _ => return None,
         }))
     }
@@ -235,7 +239,7 @@ impl Type {
     /// Is this a scalar floating point type?
     pub fn is_float(self) -> bool {
         match self {
-            F32 | F64 => true,
+            F16 | F32 | F64 | F128 => true,
             _ => false,
         }
     }
@@ -486,7 +490,9 @@ mod tests {
         assert_eq!(I64, I64.lane_type());
         assert_eq!(I128, I128.lane_type());
         assert_eq!(F32, F32.lane_type());
+        assert_eq!(F16, F16.lane_type());
         assert_eq!(F64, F64.lane_type());
+        assert_eq!(F128, F128.lane_type());
         assert_eq!(I32, I32X4.lane_type());
         assert_eq!(F64, F64X2.lane_type());
         assert_eq!(R32, R32.lane_type());
@@ -498,8 +504,10 @@ mod tests {
         assert_eq!(I32.lane_bits(), 32);
         assert_eq!(I64.lane_bits(), 64);
         assert_eq!(I128.lane_bits(), 128);
+        assert_eq!(F16.lane_bits(), 16);
         assert_eq!(F32.lane_bits(), 32);
         assert_eq!(F64.lane_bits(), 64);
+        assert_eq!(F128.lane_bits(), 128);
         assert_eq!(R32.lane_bits(), 32);
         assert_eq!(R64.lane_bits(), 64);
     }
@@ -514,8 +522,10 @@ mod tests {
         assert_eq!(I32X4.half_width(), Some(I16X4));
         assert_eq!(I64.half_width(), Some(I32));
         assert_eq!(I128.half_width(), Some(I64));
-        assert_eq!(F32.half_width(), None);
+        assert_eq!(F16.half_width(), None);
+        assert_eq!(F32.half_width(), Some(F16));
         assert_eq!(F64.half_width(), Some(F32));
+        assert_eq!(F128.half_width(), Some(F64));
 
         assert_eq!(INVALID.double_width(), None);
         assert_eq!(I8.double_width(), Some(I16));
@@ -524,8 +534,10 @@ mod tests {
         assert_eq!(I32X4.double_width(), Some(I64X4));
         assert_eq!(I64.double_width(), Some(I128));
         assert_eq!(I128.double_width(), None);
+        assert_eq!(F16.double_width(), Some(F32));
         assert_eq!(F32.double_width(), Some(F64));
-        assert_eq!(F64.double_width(), None);
+        assert_eq!(F64.double_width(), Some(F128));
+        assert_eq!(F128.double_width(), None);
     }
 
     #[test]
@@ -558,14 +570,18 @@ mod tests {
         // Conversions to and from vectors.
         assert_eq!(I8.by(16).unwrap().vector_to_dynamic(), Some(I8X16XN));
         assert_eq!(I16.by(8).unwrap().vector_to_dynamic(), Some(I16X8XN));
+        assert_eq!(F16.by(8).unwrap().vector_to_dynamic(), Some(F16X8XN));
         assert_eq!(I32.by(4).unwrap().vector_to_dynamic(), Some(I32X4XN));
         assert_eq!(F32.by(4).unwrap().vector_to_dynamic(), Some(F32X4XN));
         assert_eq!(F64.by(2).unwrap().vector_to_dynamic(), Some(F64X2XN));
         assert_eq!(I128.by(2).unwrap().vector_to_dynamic(), Some(I128X2XN));
+        assert_eq!(F128.by(2).unwrap().vector_to_dynamic(), Some(F128X2XN));
 
         assert_eq!(I128X2XN.dynamic_to_vector(), Some(I128X2));
+        assert_eq!(F16X4XN.dynamic_to_vector(), Some(F16X4));
         assert_eq!(F32X4XN.dynamic_to_vector(), Some(F32X4));
         assert_eq!(F64X4XN.dynamic_to_vector(), Some(F64X4));
+        assert_eq!(F128X4XN.dynamic_to_vector(), Some(F128X4));
         assert_eq!(I32X2XN.dynamic_to_vector(), Some(I32X2));
         assert_eq!(I32X8XN.dynamic_to_vector(), Some(I32X8));
         assert_eq!(I16X16XN.dynamic_to_vector(), Some(I16X16));
diff --git a/cranelift/codegen/src/ir/user_stack_maps.rs b/cranelift/codegen/src/ir/user_stack_maps.rs
index 4f9daf910..48ae47c82 100644
--- a/cranelift/codegen/src/ir/user_stack_maps.rs
+++ b/cranelift/codegen/src/ir/user_stack_maps.rs
@@ -29,13 +29,19 @@
 //! contrast to the old system and its `r64` values).
 
 use crate::ir;
+use cranelift_bitset::CompoundBitSet;
+use cranelift_entity::PrimaryMap;
 use smallvec::SmallVec;
 
 pub(crate) type UserStackMapEntryVec = SmallVec<[UserStackMapEntry; 4]>;
 
-/// A stack map entry describes a GC-managed value and its location at a
-/// particular instruction.
-#[derive(Clone, PartialEq, Hash)]
+/// A stack map entry describes a single GC-managed value and its location on
+/// the stack.
+///
+/// A stack map entry is associated with a particular instruction, and that
+/// instruction must be a safepoint. The GC-managed value must be stored in the
+/// described location across this entry's instruction.
+#[derive(Clone, Debug, PartialEq, Hash)]
 #[cfg_attr(
     feature = "enable-serde",
     derive(serde_derive::Serialize, serde_derive::Deserialize)
@@ -50,3 +56,46 @@ pub struct UserStackMapEntry {
     /// The offset within the stack slot where this entry's value can be found.
     pub offset: u32,
 }
+
+/// A compiled stack map, describing the location of many GC-managed values.
+///
+/// A stack map is associated with a particular instruction, and that
+/// instruction is a safepoint.
+#[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(
+    feature = "enable-serde",
+    derive(serde_derive::Deserialize, serde_derive::Serialize)
+)]
+pub struct UserStackMap {
+    by_type: SmallVec<[(ir::Type, CompoundBitSet); 1]>,
+}
+
+impl UserStackMap {
+    /// Coalesce the given entries into a new `UserStackMap`.
+    pub fn new(
+        entries: &[UserStackMapEntry],
+        stack_slot_offsets: &PrimaryMap<ir::StackSlot, u32>,
+    ) -> Self {
+        let mut by_type = SmallVec::<[(ir::Type, CompoundBitSet); 1]>::default();
+
+        for entry in entries {
+            let offset = stack_slot_offsets[entry.slot] + entry.offset;
+            let offset = usize::try_from(offset).unwrap();
+
+            // Don't bother trying to avoid an `O(n)` search here: `n` is
+            // basically always one in practice; even if it isn't, there aren't
+            // that many different CLIF types.
+            let index = by_type
+                .iter()
+                .position(|(ty, _)| *ty == entry.ty)
+                .unwrap_or_else(|| {
+                    by_type.push((entry.ty, CompoundBitSet::with_capacity(offset + 1)));
+                    by_type.len() - 1
+                });
+
+            by_type[index].1.insert(offset);
+        }
+
+        UserStackMap { by_type }
+    }
+}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 42afe51d4..c1f91b07a 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -651,11 +651,18 @@ fn enc_asimd_mod_imm(rd: Writable<Reg>, q_op: u32, cmode: u32, imm: u8) -> u32 {
 /// State carried between emissions of a sequence of instructions.
 #[derive(Default, Clone, Debug)]
 pub struct EmitState {
-    /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
+    /// Safepoint stack map for upcoming instruction, as provided to
+    /// `pre_safepoint()`.
     stack_map: Option<StackMap>,
+
+    /// The user stack map for the upcoming instruction, as provided to
+    /// `pre_safepoint()`.
+    user_stack_map: Option<ir::UserStackMap>,
+
     /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and
     /// optimized away at compiletime. See [cranelift_control].
     ctrl_plane: ControlPlane,
+
     frame_layout: FrameLayout,
 }
 
@@ -663,13 +670,19 @@ impl MachInstEmitState<Inst> for EmitState {
     fn new(abi: &Callee<AArch64MachineDeps>, ctrl_plane: ControlPlane) -> Self {
         EmitState {
             stack_map: None,
+            user_stack_map: None,
             ctrl_plane,
             frame_layout: abi.frame_layout().clone(),
         }
     }
 
-    fn pre_safepoint(&mut self, stack_map: StackMap) {
-        self.stack_map = Some(stack_map);
+    fn pre_safepoint(
+        &mut self,
+        stack_map: Option<StackMap>,
+        user_stack_map: Option<ir::UserStackMap>,
+    ) {
+        self.stack_map = stack_map;
+        self.user_stack_map = user_stack_map;
     }
 
     fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
@@ -686,8 +699,8 @@ impl MachInstEmitState<Inst> for EmitState {
 }
 
 impl EmitState {
-    fn take_stack_map(&mut self) -> Option<StackMap> {
-        self.stack_map.take()
+    fn take_stack_map(&mut self) -> (Option<StackMap>, Option<ir::UserStackMap>) {
+        (self.stack_map.take(), self.user_stack_map.take())
     }
 
     fn clear_post_insn(&mut self) {
@@ -2921,11 +2934,16 @@ impl MachInstEmit for Inst {
                 }
             }
             &Inst::Call { ref info } => {
-                if let Some(s) = state.take_stack_map() {
+                let (stack_map, user_stack_map) = state.take_stack_map();
+                if let Some(s) = stack_map {
                     sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
                 }
                 sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);
                 sink.put4(enc_jump26(0b100101, 0));
+                if let Some(s) = user_stack_map {
+                    let offset = sink.cur_offset();
+                    sink.push_user_stack_map(state, offset, s);
+                }
                 if info.opcode.is_call() {
                     sink.add_call_site(info.opcode);
                 }
@@ -2939,11 +2957,16 @@ impl MachInstEmit for Inst {
                 }
             }
             &Inst::CallInd { ref info } => {
-                if let Some(s) = state.take_stack_map() {
+                let (stack_map, user_stack_map) = state.take_stack_map();
+                if let Some(s) = stack_map {
                     sink.add_stack_map(StackMapExtent::UpcomingBytes(4), s);
                 }
                 let rn = info.rn;
                 sink.put4(0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(rn) << 5));
+                if let Some(s) = user_stack_map {
+                    let offset = sink.cur_offset();
+                    sink.push_user_stack_map(state, offset, s);
+                }
                 if info.opcode.is_call() {
                     sink.add_call_site(info.opcode);
                 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index 90186fb9f..fb63c355f 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -46,20 +46,28 @@ pub enum EmitVState {
 /// State carried between emissions of a sequence of instructions.
 #[derive(Default, Clone, Debug)]
 pub struct EmitState {
-    /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
+    /// Safepoint stack map for upcoming instruction, as provided to
+    /// `pre_safepoint()`.
     stack_map: Option<StackMap>,
+
+    /// The user stack map for the upcoming instruction, as provided to
+    /// `pre_safepoint()`.
+    user_stack_map: Option<ir::UserStackMap>,
+
     /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and
     /// optimized away at compiletime. See [cranelift_control].
     ctrl_plane: ControlPlane,
+
     /// Vector State
     /// Controls the current state of the vector unit at the emission point.
     vstate: EmitVState,
+
     frame_layout: FrameLayout,
 }
 
 impl EmitState {
-    fn take_stack_map(&mut self) -> Option<StackMap> {
-        self.stack_map.take()
+    fn take_stack_map(&mut self) -> (Option<StackMap>, Option<ir::UserStackMap>) {
+        (self.stack_map.take(), self.user_stack_map.take())
     }
 }
 
@@ -70,14 +78,20 @@ impl MachInstEmitState<Inst> for EmitState {
     ) -> Self {
         EmitState {
             stack_map: None,
+            user_stack_map: None,
             ctrl_plane,
             vstate: EmitVState::Unknown,
             frame_layout: abi.frame_layout().clone(),
         }
     }
 
-    fn pre_safepoint(&mut self, stack_map: StackMap) {
-        self.stack_map = Some(stack_map);
+    fn pre_safepoint(
+        &mut self,
+        stack_map: Option<StackMap>,
+        user_stack_map: Option<ir::UserStackMap>,
+    ) {
+        self.stack_map = stack_map;
+        self.user_stack_map = user_stack_map;
     }
 
     fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
@@ -1134,13 +1148,21 @@ impl Inst {
                     sink.add_call_site(info.opcode);
                 }
                 sink.add_reloc(Reloc::RiscvCallPlt, &info.dest, 0);
-                if let Some(s) = state.take_stack_map() {
+
+                let (stack_map, user_stack_map) = state.take_stack_map();
+                if let Some(s) = stack_map {
                     sink.add_stack_map(StackMapExtent::UpcomingBytes(8), s);
                 }
+
                 Inst::construct_auipc_and_jalr(Some(writable_link_reg()), writable_link_reg(), 0)
                     .into_iter()
                     .for_each(|i| i.emit_uncompressed(sink, emit_info, state, start_off));
 
+                if let Some(s) = user_stack_map {
+                    let offset = sink.cur_offset();
+                    sink.push_user_stack_map(state, offset, s);
+                }
+
                 let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap();
                 if callee_pop_size > 0 {
                     for inst in Riscv64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) {
@@ -1158,9 +1180,14 @@ impl Inst {
                 }
                 .emit(sink, emit_info, state);
 
-                if let Some(s) = state.take_stack_map() {
+                let (stack_map, user_stack_map) = state.take_stack_map();
+                if let Some(s) = stack_map {
                     sink.add_stack_map(StackMapExtent::StartedAtOffset(start_offset), s);
                 }
+                if let Some(s) = user_stack_map {
+                    let offset = sink.cur_offset();
+                    sink.push_user_stack_map(state, offset, s);
+                }
 
                 if info.opcode.is_call() {
                     sink.add_call_site(info.opcode);
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index b98e064ad..5c72b887e 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -1,7 +1,7 @@
 //! S390x ISA: binary code emission.
 
 use crate::binemit::StackMap;
-use crate::ir::{MemFlags, TrapCode};
+use crate::ir::{self, MemFlags, TrapCode};
 use crate::isa::s390x::inst::*;
 use crate::isa::s390x::settings as s390x_settings;
 use cranelift_control::ControlPlane;
@@ -1306,11 +1306,19 @@ fn put_with_trap(sink: &mut MachBuffer<Inst>, enc: &[u8], trap_code: TrapCode) {
 #[derive(Default, Clone, Debug)]
 pub struct EmitState {
     pub(crate) initial_sp_offset: i64,
-    /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
+
+    /// Safepoint stack map for upcoming instruction, as provided to
+    /// `pre_safepoint()`.
     stack_map: Option<StackMap>,
+
+    /// The user stack map for the upcoming instruction, as provided to
+    /// `pre_safepoint()`.
+    user_stack_map: Option<ir::UserStackMap>,
+
     /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and
     /// optimized away at compiletime. See [cranelift_control].
     ctrl_plane: ControlPlane,
+
     frame_layout: FrameLayout,
 }
 
@@ -1319,13 +1327,19 @@ impl MachInstEmitState<Inst> for EmitState {
         EmitState {
             initial_sp_offset: abi.frame_size() as i64,
             stack_map: None,
+            user_stack_map: None,
             ctrl_plane,
             frame_layout: abi.frame_layout().clone(),
         }
     }
 
-    fn pre_safepoint(&mut self, stack_map: StackMap) {
-        self.stack_map = Some(stack_map);
+    fn pre_safepoint(
+        &mut self,
+        stack_map: Option<StackMap>,
+        user_stack_map: Option<ir::UserStackMap>,
+    ) {
+        self.stack_map = stack_map;
+        self.user_stack_map = user_stack_map;
     }
 
     fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
@@ -1342,8 +1356,8 @@ impl MachInstEmitState<Inst> for EmitState {
 }
 
 impl EmitState {
-    fn take_stack_map(&mut self) -> Option<StackMap> {
-        self.stack_map.take()
+    fn take_stack_map(&mut self) -> (Option<StackMap>, Option<ir::UserStackMap>) {
+        (self.stack_map.take(), self.user_stack_map.take())
     }
 
     fn clear_post_insn(&mut self) {
@@ -3243,9 +3257,15 @@ impl Inst {
                     _ => unreachable!(),
                 }
 
-                if let Some(s) = state.take_stack_map() {
+                let (stack_map, user_stack_map) = state.take_stack_map();
+                if let Some(s) = stack_map {
                     sink.add_stack_map(StackMapExtent::UpcomingBytes(6), s);
                 }
+                if let Some(s) = user_stack_map {
+                    let offset = sink.cur_offset() + 6;
+                    sink.push_user_stack_map(state, offset, s);
+                }
+
                 put(sink, &enc_ril_b(opcode, link.to_reg(), 0));
                 if info.opcode.is_call() {
                     sink.add_call_site(info.opcode);
@@ -3255,10 +3275,16 @@ impl Inst {
                 debug_assert_eq!(link.to_reg(), gpr(14));
                 let rn = info.rn;
 
-                let opcode = 0x0d; // BASR
-                if let Some(s) = state.take_stack_map() {
+                let (stack_map, user_stack_map) = state.take_stack_map();
+                if let Some(s) = stack_map {
                     sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
                 }
+                if let Some(s) = user_stack_map {
+                    let offset = sink.cur_offset() + 2;
+                    sink.push_user_stack_map(state, offset, s);
+                }
+
+                let opcode = 0x0d; // BASR
                 put(sink, &enc_rr(opcode, link.to_reg(), rn));
                 if info.opcode.is_call() {
                     sink.add_call_site(info.opcode);
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index abfcc4477..3e0d44059 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1599,9 +1599,15 @@ pub(crate) fn emit(
             opcode,
             info: call_info,
         } => {
-            if let Some(s) = state.take_stack_map() {
+            let (stack_map, user_stack_map) = state.take_stack_map();
+            if let Some(s) = stack_map {
                 sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s);
             }
+            if let Some(s) = user_stack_map {
+                let offset = sink.cur_offset() + 5;
+                sink.push_user_stack_map(state, offset, s);
+            }
+
             sink.put1(0xE8);
             // The addend adjusts for the difference between the end of the instruction and the
             // beginning of the immediate field.
@@ -1696,9 +1702,16 @@ pub(crate) fn emit(
                     );
                 }
             }
-            if let Some(s) = state.take_stack_map() {
+
+            let (stack_map, user_stack_map) = state.take_stack_map();
+            if let Some(s) = stack_map {
                 sink.add_stack_map(StackMapExtent::StartedAtOffset(start_offset), s);
             }
+            if let Some(s) = user_stack_map {
+                let offset = sink.cur_offset();
+                sink.push_user_stack_map(state, offset, s);
+            }
+
             if opcode.is_call() {
                 sink.add_call_site(*opcode);
             }
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_state.rs b/cranelift/codegen/src/isa/x64/inst/emit_state.rs
index 21f01dcc0..9cca7d789 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_state.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_state.rs
@@ -1,11 +1,18 @@
 use super::*;
+use crate::ir;
 use cranelift_control::ControlPlane;
 
 /// State carried between emissions of a sequence of instructions.
 #[derive(Default, Clone, Debug)]
 pub struct EmitState {
-    /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
+    /// Safepoint stack map for upcoming instruction, as provided to
+    /// `pre_safepoint()`.
     stack_map: Option<StackMap>,
+
+    /// The user stack map for the upcoming instruction, as provided to
+    /// `pre_safepoint()`.
+    user_stack_map: Option<ir::UserStackMap>,
+
     /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and
     /// optimized away at compiletime. See [cranelift_control].
     ctrl_plane: ControlPlane,
@@ -19,13 +26,19 @@ impl MachInstEmitState<Inst> for EmitState {
     fn new(abi: &Callee<X64ABIMachineSpec>, ctrl_plane: ControlPlane) -> Self {
         EmitState {
             stack_map: None,
+            user_stack_map: None,
             ctrl_plane,
             frame_layout: abi.frame_layout().clone(),
         }
     }
 
-    fn pre_safepoint(&mut self, stack_map: StackMap) {
-        self.stack_map = Some(stack_map);
+    fn pre_safepoint(
+        &mut self,
+        stack_map: Option<StackMap>,
+        user_stack_map: Option<ir::UserStackMap>,
+    ) {
+        self.stack_map = stack_map;
+        self.user_stack_map = user_stack_map;
     }
 
     fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
@@ -42,8 +55,8 @@ impl MachInstEmitState<Inst> for EmitState {
 }
 
 impl EmitState {
-    pub(crate) fn take_stack_map(&mut self) -> Option<StackMap> {
-        self.stack_map.take()
+    pub(crate) fn take_stack_map(&mut self) -> (Option<StackMap>, Option<ir::UserStackMap>) {
+        (self.stack_map.take(), self.user_stack_map.take())
     }
 
     pub(crate) fn clear_post_insn(&mut self) {
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index 713fcec25..fd9555b7c 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -59,8 +59,10 @@
 //!
 //! ```plain
 //!   (high address)
-//!
-//!                              +---------------------------+
+//!                              |          ...              |
+//!                              | caller frames             |
+//!                              |          ...              |
+//!                              +===========================+
 //!                              |          ...              |
 //!                              | stack args                |
 //! Canonical Frame Address -->  | (accessed via FP)         |
@@ -68,24 +70,24 @@
 //! SP at function entry ----->  | return address            |
 //!                              +---------------------------+
 //! FP after prologue -------->  | FP (pushed by prologue)   |
-//!                              +---------------------------+
-//!                              |          ...              |
-//!                              | clobbered callee-saves    |
-//! unwind-frame base -------->  | (pushed by prologue)      |
-//!                              +---------------------------+
-//!                              |          ...              |
-//!                              | spill slots               |
-//!                              | (accessed via SP)         |
-//!                              |          ...              |
-//!                              | stack slots               |
-//!                              | (accessed via SP)         |
-//!                              | (alloc'd by prologue)     |
-//!                              +---------------------------+
-//!                              | [alignment as needed]     |
-//!                              |          ...              |
-//!                              | args for largest call     |
-//! SP ----------------------->  | (alloc'd by prologue)     |
-//!                              +---------------------------+
+//!                              +---------------------------+   -----
+//!                              |          ...              |     |
+//!                              | clobbered callee-saves    |     |
+//! unwind-frame base -------->  | (pushed by prologue)      |     |
+//!                              +---------------------------+     |
+//!                              |          ...              |     |
+//!                              | spill slots               |     |
+//!                              | (accessed via SP)         |   active
+//!                              |          ...              |    size
+//!                              | stack slots               |     |
+//!                              | (accessed via SP)         |     |
+//!                              | (alloc'd by prologue)     |     |
+//!                              +---------------------------+     |
+//!                              | [alignment as needed]     |     |
+//!                              |          ...              |     |
+//!                              | args for largest call     |     |
+//! SP ----------------------->  | (alloc'd by prologue)     |     |
+//!                              +===========================+   -----
 //!
 //!   (low address)
 //! ```
@@ -1012,6 +1014,12 @@ impl FrameLayout {
         debug_assert!(floats.iter().all(|r| r.to_reg().class() == RegClass::Float));
         (ints, floats)
     }
+
+    /// The size of FP to SP while the frame is active (not during prologue
+    /// setup or epilogue tear down).
+    pub fn active_size(&self) -> u32 {
+        self.outgoing_args_size + self.fixed_frame_storage_size + self.clobber_size
+    }
 }
 
 /// ABI object for a function body.
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index c4ffbb2f8..c02b8596d 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -178,6 +178,7 @@ use crate::machinst::{
     BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst,
 };
 use crate::trace;
+use crate::{ir, MachInstEmitState};
 use crate::{timing, VCodeConstantData};
 use cranelift_control::ControlPlane;
 use cranelift_entity::{entity_impl, PrimaryMap};
@@ -250,6 +251,11 @@ pub struct MachBuffer<I: VCodeInst> {
     srclocs: SmallVec<[MachSrcLoc<Stencil>; 64]>,
     /// Any stack maps referring to this code.
     stack_maps: SmallVec<[MachStackMap; 8]>,
+    /// Any user stack maps for this code.
+    ///
+    /// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted
+    /// by code offset, and each stack map covers `span` bytes on the stack.
+    user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,
     /// Any unwind info at a given location.
     unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,
     /// The current source location in progress (after `start_srcloc()` and
@@ -329,6 +335,7 @@ impl MachBufferFinalized<Stencil> {
                 .map(|srcloc| srcloc.apply_base_srcloc(base_srcloc))
                 .collect(),
             stack_maps: self.stack_maps,
+            user_stack_maps: self.user_stack_maps,
             unwind_info: self.unwind_info,
             alignment: self.alignment,
         }
@@ -357,9 +364,14 @@ pub struct MachBufferFinalized<T: CompilePhase> {
     pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>,
     /// Any stack maps referring to this code.
     pub(crate) stack_maps: SmallVec<[MachStackMap; 8]>,
+    /// Any user stack maps for this code.
+    ///
+    /// Each entry is an `(offset, span, stack_map)` triple. Entries are sorted
+    /// by code offset, and each stack map covers `span` bytes on the stack.
+    pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>,
     /// Any unwind info at a given location.
     pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>,
-    /// The requireed alignment of this buffer
+    /// The required alignment of this buffer.
     pub alignment: u32,
 }
 
@@ -447,6 +459,7 @@ impl<I: VCodeInst> MachBuffer<I> {
             call_sites: SmallVec::new(),
             srclocs: SmallVec::new(),
             stack_maps: SmallVec::new(),
+            user_stack_maps: SmallVec::new(),
             unwind_info: SmallVec::new(),
             cur_srcloc: None,
             label_offsets: SmallVec::new(),
@@ -1532,6 +1545,7 @@ impl<I: VCodeInst> MachBuffer<I> {
             call_sites: self.call_sites,
             srclocs,
             stack_maps: self.stack_maps,
+            user_stack_maps: self.user_stack_maps,
             unwind_info: self.unwind_info,
             alignment,
         }
@@ -1667,6 +1681,36 @@ impl<I: VCodeInst> MachBuffer<I> {
             stack_map,
         });
     }
+
+    /// Push a user stack map onto this buffer.
+    ///
+    /// The stack map is associated with the given `return_addr` code
+    /// offset. This must be the PC for the instruction just *after* this stack
+    /// map's associated instruction. For example in the sequence `call $foo;
+    /// add r8, rax`, the `return_addr` must be the offset of the start of the
+    /// `add` instruction.
+    ///
+    /// Stack maps must be pushed in sorted `return_addr` order.
+    pub fn push_user_stack_map(
+        &mut self,
+        emit_state: &I::State,
+        return_addr: CodeOffset,
+        stack_map: ir::UserStackMap,
+    ) {
+        let span = emit_state.frame_layout().active_size();
+        trace!("Adding user stack map @ {return_addr:#x} spanning {span} bytes: {stack_map:?}");
+
+        debug_assert!(
+            self.user_stack_maps
+                .last()
+                .map_or(true, |(prev_addr, _, _)| *prev_addr < return_addr),
+            "pushed stack maps out of order: {} is not less than {}",
+            self.user_stack_maps.last().unwrap().0,
+            return_addr,
+        );
+
+        self.user_stack_maps.push((return_addr, span, stack_map));
+    }
 }
 
 impl<T: CompilePhase> MachBufferFinalized<T> {
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index ea2be76d4..32db947a2 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -14,9 +14,9 @@ use crate::ir::{
     Value, ValueDef, ValueLabelAssignments, ValueLabelStart,
 };
 use crate::machinst::{
-    writable_value_regs, BlockIndex, BlockLoweringOrder, Callee, InsnIndex, LoweredBlock,
-    MachLabel, Reg, SigSet, VCode, VCodeBuilder, VCodeConstant, VCodeConstantData, VCodeConstants,
-    VCodeInst, ValueRegs, Writable,
+    writable_value_regs, BackwardsInsnIndex, BlockIndex, BlockLoweringOrder, Callee, InsnIndex,
+    LoweredBlock, MachLabel, Reg, SigSet, VCode, VCodeBuilder, VCodeConstant, VCodeConstantData,
+    VCodeConstants, VCodeInst, ValueRegs, Writable,
 };
 use crate::settings::Flags;
 use crate::{trace, CodegenResult};
@@ -485,8 +485,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
     /// Pre-analysis: compute `value_ir_uses`. See comment on
     /// `ValueUseState` for a description of what this analysis
     /// computes.
-    fn compute_use_states<'a>(
-        f: &'a Function,
+    fn compute_use_states(
+        f: &Function,
         sret_param: Option<Value>,
     ) -> SecondaryMap<Value, ValueUseState> {
         // We perform the analysis without recursion, so we don't
@@ -803,9 +803,39 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
                 }
             }
 
+            let start = self.vcode.vcode.num_insts();
             let loc = self.srcloc(inst);
             self.finish_ir_inst(loc);
 
+            // If the instruction had a user stack map, forward it from the CLIF
+            // to the vcode.
+            if let Some(entries) = self.f.dfg.user_stack_map_entries(inst) {
+                let end = self.vcode.vcode.num_insts();
+                debug_assert!(end > start);
+                debug_assert_eq!(
+                    (start..end)
+                        .filter(|i| self.vcode.vcode[InsnIndex::new(*i)].is_safepoint())
+                        .count(),
+                    1
+                );
+                for i in start..end {
+                    let iix = InsnIndex::new(i);
+                    if self.vcode.vcode[iix].is_safepoint() {
+                        trace!(
+                            "Adding user stack map from clif\n\n\
+                                 {inst:?} `{}`\n\n\
+                             to vcode\n\n\
+                                 {iix:?} `{}`",
+                            self.f.dfg.display_inst(inst),
+                            &self.vcode.vcode[iix].pretty_print_inst(&mut Default::default()),
+                        );
+                        self.vcode
+                            .add_user_stack_map(BackwardsInsnIndex::new(iix.index()), entries);
+                        break;
+                    }
+                }
+            }
+
             // maybe insert random instruction
             if ctrl_plane.get_decision() {
                 if ctrl_plane.get_decision() {
diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs
index bf15ec462..af87c3053 100644
--- a/cranelift/codegen/src/machinst/mod.rs
+++ b/cranelift/codegen/src/machinst/mod.rs
@@ -45,8 +45,9 @@
 //! ```
 
 use crate::binemit::{Addend, CodeInfo, CodeOffset, Reloc, StackMap};
-use crate::ir::function::FunctionParameters;
-use crate::ir::{DynamicStackSlot, RelSourceLoc, StackSlot, Type};
+use crate::ir::{
+    self, function::FunctionParameters, DynamicStackSlot, RelSourceLoc, StackSlot, Type,
+};
 use crate::isa::FunctionAlignment;
 use crate::result::CodegenResult;
 use crate::settings;
@@ -284,10 +285,13 @@ pub enum MachTerminator {
 pub trait MachInstEmit: MachInst {
     /// Persistent state carried across `emit` invocations.
     type State: MachInstEmitState<Self>;
+
     /// Constant information used in `emit` invocations.
     type Info;
+
     /// Emit the instruction.
     fn emit(&self, code: &mut MachBuffer<Self>, info: &Self::Info, state: &mut Self::State);
+
     /// Pretty-print the instruction.
     fn pretty_print_inst(&self, state: &mut Self::State) -> String;
 }
@@ -297,20 +301,29 @@ pub trait MachInstEmit: MachInst {
 pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {
     /// Create a new emission state given the ABI object.
     fn new(abi: &Callee<I::ABIMachineSpec>, ctrl_plane: ControlPlane) -> Self;
+
     /// Update the emission state before emitting an instruction that is a
     /// safepoint.
-    fn pre_safepoint(&mut self, _stack_map: StackMap) {}
+    fn pre_safepoint(
+        &mut self,
+        stack_map: Option<StackMap>,
+        user_stack_map: Option<ir::UserStackMap>,
+    );
+
     /// The emission state holds ownership of a control plane, so it doesn't
     /// have to be passed around explicitly too much. `ctrl_plane_mut` may
     /// be used if temporary access to the control plane is needed by some
     /// other function that doesn't have access to the emission state.
     fn ctrl_plane_mut(&mut self) -> &mut ControlPlane;
+
     /// Used to continue using a control plane after the emission state is
     /// not needed anymore.
     fn take_ctrl_plane(self) -> ControlPlane;
+
     /// A hook that triggers when first emitting a new block.
     /// It is guaranteed to be called before any instructions are emitted.
     fn on_new_block(&mut self) {}
+
     /// The [`FrameLayout`] for the function currently being compiled.
     fn frame_layout(&self) -> &FrameLayout;
 }
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs
index 7a971be74..48a45c90e 100644
--- a/cranelift/codegen/src/machinst/vcode.rs
+++ b/cranelift/codegen/src/machinst/vcode.rs
@@ -40,6 +40,33 @@ use std::fmt;
 /// Index referring to an instruction in VCode.
 pub type InsnIndex = regalloc2::Inst;
 
+/// Extension trait for `InsnIndex` to allow conversion to a
+/// `BackwardsInsnIndex`.
+trait ToBackwardsInsnIndex {
+    fn to_backwards_insn_index(&self, num_insts: usize) -> BackwardsInsnIndex;
+}
+
+impl ToBackwardsInsnIndex for InsnIndex {
+    fn to_backwards_insn_index(&self, num_insts: usize) -> BackwardsInsnIndex {
+        BackwardsInsnIndex::new(num_insts - self.index() - 1)
+    }
+}
+
+/// An index referring to an instruction in the VCode when it is backwards,
+/// during VCode construction.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[cfg_attr(
+    feature = "enable-serde",
+    derive(::serde::Serialize, ::serde::Deserialize)
+)]
+pub struct BackwardsInsnIndex(InsnIndex);
+
+impl BackwardsInsnIndex {
+    pub fn new(i: usize) -> Self {
+        BackwardsInsnIndex(InsnIndex::new(i))
+    }
+}
+
 /// Index referring to a basic block in VCode.
 pub type BlockIndex = regalloc2::Block;
 
@@ -67,6 +94,14 @@ pub struct VCode<I: VCodeInst> {
     /// Lowered machine instructions in order corresponding to the original IR.
     insts: Vec<I>,
 
+    /// A map from backwards instruction index to the user stack map for that
+    /// instruction.
+    ///
+    /// This is a sparse side table that only has entries for instructions that
+    /// are safepoints, and only for a subset of those that have an associated
+    /// user stack map.
+    user_stack_maps: FxHashMap<BackwardsInsnIndex, ir::UserStackMap>,
+
     /// Operands: pre-regalloc references to virtual registers with
     /// constraints, in one flattened array. This allows the regalloc
     /// to efficiently access all operands without requiring expensive
@@ -251,7 +286,7 @@ impl<I: VCodeInst> VCodeBuilder<I> {
         block_order: BlockLoweringOrder,
         constants: VCodeConstants,
         direction: VCodeBuildDirection,
-    ) -> VCodeBuilder<I> {
+    ) -> Self {
         let vcode = VCode::new(sigs, abi, emit_info, block_order, constants);
 
         VCodeBuilder {
@@ -565,6 +600,17 @@ impl<I: VCodeInst> VCodeBuilder<I> {
 
         self.vcode
     }
+
+    /// Add a user stack map for the associated instruction.
+    pub fn add_user_stack_map(
+        &mut self,
+        inst: BackwardsInsnIndex,
+        entries: &[ir::UserStackMapEntry],
+    ) {
+        let stack_map = ir::UserStackMap::new(entries, self.vcode.abi.sized_stackslot_offsets());
+        let old_entry = self.vcode.user_stack_maps.insert(inst, stack_map);
+        debug_assert!(old_entry.is_none());
+    }
 }
 
 /// Is this type a reference type?
@@ -582,12 +628,13 @@ impl<I: VCodeInst> VCode<I> {
         emit_info: I::Info,
         block_order: BlockLoweringOrder,
         constants: VCodeConstants,
-    ) -> VCode<I> {
+    ) -> Self {
         let n_blocks = block_order.lowered_order().len();
         VCode {
             sigs,
             vreg_types: vec![],
             insts: Vec::with_capacity(10 * n_blocks),
+            user_stack_maps: FxHashMap::default(),
             operands: Vec::with_capacity(30 * n_blocks),
             operand_ranges: Ranges::with_capacity(10 * n_blocks),
             clobbers: FxHashMap::default(),
@@ -864,7 +911,7 @@ impl<I: VCodeInst> VCode<I> {
 
                         // If this is a safepoint, compute a stack map
                         // and pass it to the emit state.
-                        if self.insts[iix.index()].is_safepoint() {
+                        let stack_map_disasm = if self.insts[iix.index()].is_safepoint() {
                             let mut safepoint_slots: SmallVec<[SpillSlot; 8]> = smallvec![];
                             // Find the contiguous range of
                             // (progpoint, allocation) safepoint slot
@@ -888,13 +935,36 @@ impl<I: VCodeInst> VCode<I> {
                                 let slot = alloc.as_stack().unwrap();
                                 safepoint_slots.push(slot);
                             }
-                            if !safepoint_slots.is_empty() {
-                                let stack_map = self
-                                    .abi
-                                    .spillslots_to_stack_map(&safepoint_slots[..], &state);
-                                state.pre_safepoint(stack_map);
-                            }
-                        }
+
+                            let stack_map = if safepoint_slots.is_empty() {
+                                None
+                            } else {
+                                Some(
+                                    self.abi
+                                        .spillslots_to_stack_map(&safepoint_slots[..], &state),
+                                )
+                            };
+
+                            let (user_stack_map, user_stack_map_disasm) = {
+                                // The `user_stack_maps` is keyed by reverse
+                                // instruction index, so we must flip the
+                                // index. We can't put this into a helper method
+                                // due to borrowck issues because parts of
+                                // `self` are borrowed mutably elsewhere in this
+                                // function.
+                                let index = iix.to_backwards_insn_index(self.num_insts());
+                                let user_stack_map = self.user_stack_maps.remove(&index);
+                                let user_stack_map_disasm =
+                                    user_stack_map.as_ref().map(|m| format!("  ; {m:?}"));
+                                (user_stack_map, user_stack_map_disasm)
+                            };
+
+                            state.pre_safepoint(stack_map, user_stack_map);
+
+                            user_stack_map_disasm
+                        } else {
+                            None
+                        };
 
                         // If the instruction we are about to emit is
                         // a return, place an epilogue at this point
@@ -932,6 +1002,10 @@ impl<I: VCodeInst> VCode<I> {
                                 &mut buffer,
                                 &mut state,
                             );
+                            if let Some(stack_map_disasm) = stack_map_disasm {
+                                disasm.push_str(&stack_map_disasm);
+                                disasm.push('\n');
+                            }
                         }
                     }
 
@@ -1014,6 +1088,12 @@ impl<I: VCodeInst> VCode<I> {
             }
         }
 
+        debug_assert!(
+            self.user_stack_maps.is_empty(),
+            "any stack maps should have been consumed by instruction emission, still have: {:#?}",
+            self.user_stack_maps,
+        );
+
         // Do any optimizations on branches at tail of buffer, as if we had
         // bound one last label.
         buffer.optimize_branches(ctrl_plane);
@@ -1224,6 +1304,12 @@ impl<I: VCodeInst> VCode<I> {
             .map(|o| o.vreg())
             .any(|vreg| self.facts[vreg.vreg()].is_some())
     }
+
+    /// Get the user stack map associated with the given forward instruction index.
+    pub fn get_user_stack_map(&self, inst: InsnIndex) -> Option<&ir::UserStackMap> {
+        let index = inst.to_backwards_insn_index(self.num_insts());
+        self.user_stack_maps.get(&index)
+    }
 }
 
 impl<I: VCodeInst> std::ops::Index<InsnIndex> for VCode<I> {
@@ -1385,6 +1471,9 @@ impl<I: VCodeInst> fmt::Debug for VCode<I> {
                         }
                     }
                 }
+                if let Some(user_stack_map) = self.get_user_stack_map(InsnIndex::new(inst)) {
+                    writeln!(f, "    {user_stack_map:?}")?;
+                }
             }
         }
 
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index bb9703bfc..01d0377e4 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -1419,7 +1419,7 @@ pub(crate) fn extractlanes(
 
     let iterations = match lane_type {
         types::I8 => 1,
-        types::I16 => 2,
+        types::I16 | types::F16 => 2,
         types::I32 | types::F32 => 4,
         types::I64 | types::F64 => 8,
         _ => unimplemented!("vectors with lanes wider than 64-bits are currently unsupported."),
@@ -1458,7 +1458,7 @@ fn vectorizelanes_all(x: &[DataValue], vector_type: types::Type) -> ValueResult<
     let lane_type = vector_type.lane_type();
     let iterations = match lane_type {
         types::I8 => 1,
-        types::I16 => 2,
+        types::I16 | types::F16 => 2,
         types::I32 | types::F32 => 4,
         types::I64 | types::F64 => 8,
         _ => unimplemented!("vectors with lanes wider than 64-bits are currently unsupported."),
diff --git a/cranelift/interpreter/src/value.rs b/cranelift/interpreter/src/value.rs
index 4689fe6a1..d72ee5714 100644
--- a/cranelift/interpreter/src/value.rs
+++ b/cranelift/interpreter/src/value.rs
@@ -5,7 +5,7 @@
 
 use core::fmt::{self, Display, Formatter};
 use cranelift_codegen::data_value::{DataValue, DataValueCastFailure};
-use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
+use cranelift_codegen::ir::immediates::{Ieee128, Ieee16, Ieee32, Ieee64};
 use cranelift_codegen::ir::{types, Type};
 use thiserror::Error;
 
@@ -318,7 +318,7 @@ impl DataValueExt for DataValue {
 
     fn is_float(&self) -> bool {
         match self {
-            DataValue::F32(_) | DataValue::F64(_) => true,
+            DataValue::F16(_) | DataValue::F32(_) | DataValue::F64(_) | DataValue::F128(_) => true,
             _ => false,
         }
     }
@@ -399,10 +399,14 @@ impl DataValueExt for DataValue {
                 (val, ty) if val.ty().is_int() && ty.is_int() => {
                     DataValue::from_integer(val.into_int_signed()?, ty)?
                 }
+                (DataValue::I16(n), types::F16) => DataValue::F16(Ieee16::with_bits(n as u16)),
                 (DataValue::I32(n), types::F32) => DataValue::F32(f32::from_bits(n as u32).into()),
                 (DataValue::I64(n), types::F64) => DataValue::F64(f64::from_bits(n as u64).into()),
+                (DataValue::I128(n), types::F128) => DataValue::F128(Ieee128::with_bits(n as u128)),
+                (DataValue::F16(n), types::I16) => DataValue::I16(n.bits() as i16),
                 (DataValue::F32(n), types::I32) => DataValue::I32(n.bits() as i32),
                 (DataValue::F64(n), types::I64) => DataValue::I64(n.bits() as i64),
+                (DataValue::F128(n), types::I128) => DataValue::I128(n.bits() as i128),
                 (DataValue::F32(n), types::F64) => DataValue::F64((n.as_f32() as f64).into()),
                 (dv, t) if (t.is_int() || t.is_float()) && dv.ty() == t => dv,
                 (dv, _) => unimplemented!("conversion: {} -> {:?}", dv.ty(), kind),
@@ -502,8 +506,10 @@ impl DataValueExt for DataValue {
             DataValue::I32(f) => Ok(*f == 0),
             DataValue::I64(f) => Ok(*f == 0),
             DataValue::I128(f) => Ok(*f == 0),
+            DataValue::F16(f) => Ok(f.is_zero()),
             DataValue::F32(f) => Ok(f.is_zero()),
             DataValue::F64(f) => Ok(f.is_zero()),
+            DataValue::F128(f) => Ok(f.is_zero()),
             DataValue::V64(_) | DataValue::V128(_) => {
                 Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty()))
             }
diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rs
index 0fec4fcd2..afdf3043b 100644
--- a/cranelift/reader/src/lexer.rs
+++ b/cranelift/reader/src/lexer.rs
@@ -369,8 +369,10 @@ impl<'a> Lexer<'a> {
             "i32" => types::I32,
             "i64" => types::I64,
             "i128" => types::I128,
+            "f16" => types::F16,
             "f32" => types::F32,
             "f64" => types::F64,
+            "f128" => types::F128,
             "r32" => types::R32,
             "r64" => types::R64,
             _ => return None,
@@ -626,7 +628,7 @@ mod tests {
     fn lex_identifiers() {
         let mut lex = Lexer::new(
             "v0 v00 vx01 block1234567890 block5234567890 v1x vx1 vxvx4 \
-             function0 function i8 i32x4 f32x5",
+             function0 function i8 i32x4 f32x5 f16 f128",
         );
         assert_eq!(
             lex.next(),
@@ -647,6 +649,8 @@ mod tests {
         assert_eq!(lex.next(), token(Token::Type(types::I8), 1));
         assert_eq!(lex.next(), token(Token::Type(types::I32X4), 1));
         assert_eq!(lex.next(), token(Token::Identifier("f32x5"), 1));
+        assert_eq!(lex.next(), token(Token::Type(types::F16), 1));
+        assert_eq!(lex.next(), token(Token::Type(types::F128), 1));
         assert_eq!(lex.next(), None);
     }
 
diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs
index 6068af8de..3783bb49e 100644
--- a/cranelift/reader/src/parser.rs
+++ b/cranelift/reader/src/parser.rs
@@ -10,7 +10,9 @@ use crate::testfile::{Comment, Details, Feature, TestFile};
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::entity::{EntityRef, PrimaryMap};
 use cranelift_codegen::ir::entities::{AnyEntity, DynamicType, MemoryType};
-use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, Uimm32, Uimm64};
+use cranelift_codegen::ir::immediates::{
+    Ieee128, Ieee16, Ieee32, Ieee64, Imm64, Offset32, Uimm32, Uimm64,
+};
 use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs};
 use cranelift_codegen::ir::pcc::{BaseExpr, Expr, Fact};
 use cranelift_codegen::ir::types;
@@ -855,6 +857,18 @@ impl<'a> Parser<'a> {
         Ok(Imm64::new(0))
     }
 
+    // Match and consume an Ieee16 immediate.
+    fn match_ieee16(&mut self, err_msg: &str) -> ParseResult<Ieee16> {
+        if let Some(Token::Float(text)) = self.token() {
+            self.consume();
+            // Lexer just gives us raw text that looks like a float.
+            // Parse it as an Ieee16 to check for the right number of digits and other issues.
+            text.parse().map_err(|e| self.error(e))
+        } else {
+            err!(self.loc, err_msg)
+        }
+    }
+
     // Match and consume an Ieee32 immediate.
     fn match_ieee32(&mut self, err_msg: &str) -> ParseResult<Ieee32> {
         if let Some(Token::Float(text)) = self.token() {
@@ -879,6 +893,18 @@ impl<'a> Parser<'a> {
         }
     }
 
+    // Match and consume an Ieee128 immediate.
+    fn match_ieee128(&mut self, err_msg: &str) -> ParseResult<Ieee128> {
+        if let Some(Token::Float(text)) = self.token() {
+            self.consume();
+            // Lexer just gives us raw text that looks like a float.
+            // Parse it as an Ieee128 to check for the right number of digits and other issues.
+            text.parse().map_err(|e| self.error(e))
+        } else {
+            err!(self.loc, err_msg)
+        }
+    }
+
     // Match and consume an enumerated immediate, like one of the condition codes.
     fn match_enum<T: FromStr>(&mut self, err_msg: &str) -> ParseResult<T> {
         if let Some(Token::Identifier(text)) = self.token() {
@@ -2793,8 +2819,10 @@ impl<'a> Parser<'a> {
             I32 => DataValue::from(self.match_imm32("expected an i32")?),
             I64 => DataValue::from(Into::<i64>::into(self.match_imm64("expected an i64")?)),
             I128 => DataValue::from(self.match_imm128("expected an i128")?),
+            F16 => DataValue::from(self.match_ieee16("expected an f16")?),
             F32 => DataValue::from(self.match_ieee32("expected an f32")?),
             F64 => DataValue::from(self.match_ieee64("expected an f64")?),
+            F128 => DataValue::from(self.match_ieee128("expected an f128")?),
             _ if (ty.is_vector() || ty.is_dynamic_vector()) => {
                 let as_vec = self.match_uimm128(ty)?.into_vec();
                 if as_vec.len() == 16 {
@@ -3254,12 +3282,13 @@ mod tests {
         assert_eq!(sig.returns.len(), 0);
         assert_eq!(sig.call_conv, CallConv::SystemV);
 
-        let sig2 = Parser::new("(i8 uext, f32, f64, i32 sret) -> i32 sext, f64 system_v")
-            .parse_signature()
-            .unwrap();
+        let sig2 =
+            Parser::new("(i8 uext, f16, f32, f64, f128, i32 sret) -> i32 sext, f64 system_v")
+                .parse_signature()
+                .unwrap();
         assert_eq!(
             sig2.to_string(),
-            "(i8 uext, f32, f64, i32 sret) -> i32 sext, f64 system_v"
+            "(i8 uext, f16, f32, f64, f128, i32 sret) -> i32 sext, f64 system_v"
         );
         assert_eq!(sig2.call_conv, CallConv::SystemV);
 
@@ -3865,8 +3894,13 @@ mod tests {
             "1512366032949150931280199141537564007"
         );
         assert_eq!(parse("1234567", I128).to_string(), "1234567");
+        assert_eq!(parse("0x16.1", F16).to_string(), "0x1.610p4");
         assert_eq!(parse("0x32.32", F32).to_string(), "0x1.919000p5");
         assert_eq!(parse("0x64.64", F64).to_string(), "0x1.9190000000000p6");
+        assert_eq!(
+            parse("0x128.128", F128).to_string(),
+            "0x1.2812800000000000000000000000p8"
+        );
         assert_eq!(
             parse("[0 1 2 3]", I32X4).to_string(),
             "0x00000003000000020000000100000000"
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
index 89f1472ba..a45e3fffa 100644
--- a/crates/cli-flags/src/lib.rs
+++ b/crates/cli-flags/src/lib.rs
@@ -115,17 +115,6 @@ wasmtime_option_group! {
         /// The maximum runtime size of each linear memory in the pooling
         /// allocator, in bytes.
         pub pooling_max_memory_size: Option<usize>,
-
-        /// Whether to enable call-indirect caching.
-        pub cache_call_indirects: Option<bool>,
-
-        /// The maximum call-indirect cache slot count.
-        ///
-        /// One slot is allocated per indirect callsite; if the module
-        /// has more indirect callsites than this limit, then the
-        /// first callsites in linear order in the code section, up to
-        /// the limit, will receive a cache slot.
-        pub max_call_indirect_cache_slots: Option<usize>,
     }
 
     enum Optimize {
@@ -576,12 +565,6 @@ impl CommonOptions {
         if let Some(enable) = self.opts.memory_init_cow {
             config.memory_init_cow(enable);
         }
-        if let Some(enable) = self.opts.cache_call_indirects {
-            config.cache_call_indirects(enable);
-        }
-        if let Some(max) = self.opts.max_call_indirect_cache_slots {
-            config.max_call_indirect_cache_slots(max);
-        }
 
         match_feature! {
             ["pooling-allocator" : self.opts.pooling_allocator.or(pooling_allocator_default)]
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 80075c648..c720cf18e 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -147,14 +147,8 @@ impl wasmtime_environ::Compiler for Compiler {
             context.func.collect_debug_info();
         }
 
-        let mut func_env = FuncEnvironment::new(
-            isa,
-            translation,
-            types,
-            &self.tunables,
-            self.wmemcheck,
-            input.call_indirect_start,
-        );
+        let mut func_env =
+            FuncEnvironment::new(isa, translation, types, &self.tunables, self.wmemcheck);
 
         // The `stack_limit` global value below is the implementation of stack
         // overflow checks in Wasmtime.
@@ -206,11 +200,7 @@ impl wasmtime_environ::Compiler for Compiler {
             flags: MemFlags::trusted(),
         });
         context.func.stack_limit = Some(stack_limit);
-        let FunctionBodyData {
-            validator,
-            body,
-            call_indirect_start: _,
-        } = input;
+        let FunctionBodyData { validator, body } = input;
         let mut validator =
             validator.into_validator(mem::take(&mut compiler.cx.validator_allocations));
         compiler.cx.func_translator.translate_body(
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 6ade601af..8ec068b39 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -12,9 +12,9 @@ use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap};
 use cranelift_frontend::FunctionBuilder;
 use cranelift_frontend::Variable;
 use cranelift_wasm::{
-    CallIndirectSiteIndex, EngineOrModuleTypeIndex, FuncIndex, FuncTranslationState, GlobalIndex,
-    GlobalVariable, Heap, HeapData, HeapStyle, MemoryIndex, TableData, TableIndex, TableSize,
-    TargetEnvironment, TypeIndex, WasmHeapTopType, WasmHeapType, WasmResult,
+    EngineOrModuleTypeIndex, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, Heap,
+    HeapData, HeapStyle, MemoryIndex, TableData, TableIndex, TableSize, TargetEnvironment,
+    TypeIndex, WasmHeapTopType, WasmHeapType, WasmResult,
 };
 use std::mem;
 use wasmparser::Operator;
@@ -138,9 +138,6 @@ pub struct FuncEnvironment<'module_environment> {
 
     #[cfg(feature = "wmemcheck")]
     wmemcheck: bool,
-
-    /// The current call-indirect-cache index.
-    pub call_indirect_index: usize,
 }
 
 impl<'module_environment> FuncEnvironment<'module_environment> {
@@ -150,7 +147,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         types: &'module_environment ModuleTypesBuilder,
         tunables: &'module_environment Tunables,
         wmemcheck: bool,
-        call_indirect_start: usize,
     ) -> Self {
         let builtin_functions = BuiltinFunctions::new(isa);
 
@@ -178,8 +174,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             // functions should consume at least some fuel.
             fuel_consumed: 1,
 
-            call_indirect_index: call_indirect_start,
-
             #[cfg(feature = "wmemcheck")]
             wmemcheck,
             #[cfg(feature = "wmemcheck")]
@@ -1030,37 +1024,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             _ => unreachable!(),
         }
     }
-
-    /// Allocate the next CallIndirectSiteIndex for indirect-target
-    /// caching purposes, if slots remain below the slot-count limit.
-    fn alloc_call_indirect_index(&mut self) -> Option<CallIndirectSiteIndex> {
-        // We need to check to see if we have reached the cache-slot
-        // limit.
-        //
-        // There are two kinds of limit behavior:
-        //
-        // 1. Our function's start-index is below the limit, but we
-        //    hit the limit in the middle of the function. We will
-        //    allocate slots up to the limit, then stop exactly when we
-        //    hit it.
-        //
-        // 2. Our function is beyond the limit-count of
-        //    `call_indirect`s. The counting prescan in
-        //    `ModuleEnvironment` that assigns start indices will
-        //    saturate at the limit, and this function's start index
-        //    will be exactly the limit, so we get zero slots and exit
-        //    immediately at every call to this function.
-        if self.call_indirect_index >= self.tunables.max_call_indirect_cache_slots {
-            return None;
-        }
-
-        let idx = CallIndirectSiteIndex::from_u32(
-            u32::try_from(self.call_indirect_index)
-                .expect("More than 2^32 callsites; should be limited by impl limits"),
-        );
-        self.call_indirect_index += 1;
-        Some(idx)
-    }
 }
 
 struct Call<'a, 'func, 'module_env> {
@@ -1172,68 +1135,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
     }
 
-    /// Get the address of the call-indirect cache slot for a given callsite.
-    pub fn call_indirect_cache_slot_addr(
-        &mut self,
-        call_site: CallIndirectSiteIndex,
-        vmctx: ir::Value,
-    ) -> ir::Value {
-        let offset = self.env.offsets.vmctx_call_indirect_cache(call_site);
-        self.builder.ins().iadd_imm(vmctx, i64::from(offset))
-    }
-
-    /// Load the cached index and code pointer for an indirect call.
-    ///
-    /// Generates IR like:
-    ///
-    /// ```ignore
-    /// v1 = load.i64 cache_ptr+0 ;; cached index (cache key)
-    /// v2 = load.i64 cache_ptr+8 ;; cached raw code pointer (cache value)
-    /// ```
-    ///
-    /// and returns `(index, code_ptr)` (e.g. from above, `(v1, v2)`).
-    fn load_cached_indirect_index_and_code_ptr(
-        &mut self,
-        cache_ptr: ir::Value,
-    ) -> (ir::Value, ir::Value) {
-        let cached_index = self.builder.ins().load(
-            I32,
-            MemFlags::trusted(),
-            cache_ptr,
-            Offset32::from(self.env.offsets.ptr.vmcall_indirect_cache_index()),
-        );
-        let cached_code_ptr = self.builder.ins().load(
-            self.env.pointer_type(),
-            MemFlags::trusted(),
-            cache_ptr,
-            Offset32::from(self.env.offsets.ptr.vmcall_indirect_cache_wasm_call()),
-        );
-
-        (cached_index, cached_code_ptr)
-    }
-
-    /// Update the indirect-call cache: store a new index and raw code
-    /// pointer in the slot for a given callsite.
-    fn store_cached_indirect_index_and_code_ptr(
-        &mut self,
-        cache_ptr: ir::Value,
-        index: ir::Value,
-        code_ptr: ir::Value,
-    ) {
-        self.builder.ins().store(
-            MemFlags::trusted(),
-            index,
-            cache_ptr,
-            Offset32::from(self.env.offsets.ptr.vmcall_indirect_cache_index()),
-        );
-        self.builder.ins().store(
-            MemFlags::trusted(),
-            code_ptr,
-            cache_ptr,
-            Offset32::from(self.env.offsets.ptr.vmcall_indirect_cache_wasm_call()),
-        );
-    }
-
     /// Do an indirect call through the given funcref table.
     pub fn indirect_call(
         mut self,
@@ -1243,126 +1144,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         callee: ir::Value,
         call_args: &[ir::Value],
     ) -> WasmResult<Option<ir::Inst>> {
-        // If we are performing call-indirect caching with this table, check the cache.
-        let caching = if self.env.tunables.cache_call_indirects {
-            let plan = &self.env.module.table_plans[table_index];
-            // We can do the indirect call caching optimization only
-            // if table elements will not change (no opcodes exist
-            // that could write the table, and table not exported),
-            // and if we can use the zero-index as a sentinenl for "no
-            // cache entry" (initial zeroed vmctx state).
-            !plan.written && !plan.non_null_zero
-        } else {
-            false
-        };
-
-        // Allocate a call-indirect cache slot if caching is
-        // enabled. Note that this still may be `None` if we run out
-        // of slots.
-        let call_site = if caching {
-            self.env.alloc_call_indirect_index()
-        } else {
-            None
-        };
-
-        let (code_ptr, callee_vmctx) = if let Some(call_site) = call_site {
-            // Get a local copy of `vmctx`.
-            let vmctx = self.env.vmctx(self.builder.func);
-            let vmctx = self
-                .builder
-                .ins()
-                .global_value(self.env.pointer_type(), vmctx);
-
-            // Get the address of the cache slot in the VMContext
-            // struct.
-            let slot = self.call_indirect_cache_slot_addr(call_site, vmctx);
-
-            // Create the following CFG and generate code with the following outline:
-            //
-            //   (load cached index and code pointer)
-            //   hit = icmp eq (cached index), (callee)
-            //   brif hit, call_block((cached code ptr), vmctx), miss_block
-            //
-            // miss_block:
-            //   (compute actual code pointer, with checks)
-            //   same_instance = icmp eq (callee vmctx), (vmctx)
-            //   brif same_instance update_block, call_block((actual code ptr), (callee vmctx))
-            //
-            // update_block:
-            //   (store actual index and actual code pointer)
-            //   jump call_block((actual code ptr), (callee vmctx))
-            //
-            // call_block(code_ptr, callee_vmctx):
-            //   (unchecked call-indirect sequence)
-
-            // Create two-level conditionals with CFG.
-            let current_block = self.builder.current_block().unwrap();
-            let miss_block = self.builder.create_block();
-            let call_block = self.builder.create_block();
-            let update_block = self.builder.create_block();
-
-            self.builder.insert_block_after(miss_block, current_block);
-            self.builder.insert_block_after(update_block, miss_block);
-            self.builder.insert_block_after(call_block, update_block);
-            self.builder.set_cold_block(miss_block);
-            self.builder.set_cold_block(update_block);
-
-            // Load cached values, check for hit, branch to
-            // call block or miss block.
-
-            let (cached_index, cached_code_ptr) =
-                self.load_cached_indirect_index_and_code_ptr(slot);
-            let hit = self.builder.ins().icmp(IntCC::Equal, cached_index, callee);
-            self.builder
-                .ins()
-                .brif(hit, call_block, &[cached_code_ptr, vmctx], miss_block, &[]);
-
-            // Miss block: compute actual callee code pointer and
-            // vmctx, and update cache if same-instance.
-
-            self.builder.seal_block(miss_block);
-            self.builder.switch_to_block(miss_block);
-
-            if let Some((code_ptr, callee_vmctx)) =
-                self.check_and_load_code_and_callee_vmctx(table_index, ty_index, callee, true)?
-            {
-                // If callee vmctx is equal to ours, update the cache.
-                let same_instance = self.builder.ins().icmp(IntCC::Equal, callee_vmctx, vmctx);
-
-                self.builder.ins().brif(
-                    same_instance,
-                    update_block,
-                    &[],
-                    call_block,
-                    &[code_ptr, callee_vmctx],
-                );
-
-                self.builder.seal_block(update_block);
-                self.builder.switch_to_block(update_block);
-
-                self.store_cached_indirect_index_and_code_ptr(slot, callee, code_ptr);
-                self.builder
-                    .ins()
-                    .jump(call_block, &[code_ptr, callee_vmctx]);
-            }
-
-            // Call block: do the call.
-
-            self.builder.seal_block(call_block);
-            self.builder.switch_to_block(call_block);
-
-            let code_ptr = self
-                .builder
-                .append_block_param(call_block, self.env.pointer_type());
-            let callee_vmctx = self
-                .builder
-                .append_block_param(call_block, self.env.pointer_type());
-            (code_ptr, callee_vmctx)
-        } else {
-            match self.check_and_load_code_and_callee_vmctx(table_index, ty_index, callee, false)? {
-                Some(pair) => pair,
-                None => return Ok(None),
-            }
+        let (code_ptr, callee_vmctx) = match self.check_and_load_code_and_callee_vmctx(
+            table_index,
+            ty_index,
+            callee,
+            false,
+        )? {
+            Some(pair) => pair,
+            None => return Ok(None),
         };
 
         self.unchecked_call_impl(sig_ref, code_ptr, callee_vmctx, call_args)
diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs
index 4548095b8..f8edcc845 100644
--- a/crates/environ/src/compile/module_environ.rs
+++ b/crates/environ/src/compile/module_environ.rs
@@ -11,7 +11,6 @@ use crate::{
 };
 use anyhow::{bail, Result};
 use cranelift_entity::packed_option::ReservedValue;
-use cranelift_entity::EntityRef;
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::mem;
@@ -19,8 +18,8 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use wasmparser::{
     types::Types, CustomSectionReader, DataKind, ElementItems, ElementKind, Encoding, ExternalKind,
-    FuncToValidate, FunctionBody, KnownCustom, NameSectionReader, Naming, Operator, Parser,
-    Payload, TypeRef, Validator, ValidatorResources,
+    FuncToValidate, FunctionBody, KnownCustom, NameSectionReader, Naming, Parser, Payload, TypeRef,
+    Validator, ValidatorResources,
 };
 use wasmtime_types::{ConstExpr, ConstOp, ModuleInternedTypeIndex, SizeOverflow, WasmHeapTopType};
 
@@ -116,8 +115,6 @@ pub struct FunctionBodyData<'a> {
     pub body: FunctionBody<'a>,
     /// Validator for the function body
     pub validator: FuncToValidate<ValidatorResources>,
-    /// The start index for call-indirects in this body.
-    pub call_indirect_start: usize,
 }
 
 #[derive(Debug, Default)]
@@ -436,9 +433,6 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
                         // this never gets past validation
                         ExternalKind::Tag => unreachable!(),
                     };
-                    if let EntityIndex::Table(table) = entity {
-                        self.flag_written_table(table);
-                    }
                     self.result
                         .module
                         .exports
@@ -504,10 +498,6 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
                             let (offset, escaped) = ConstExpr::from_wasmparser(offset_expr)?;
                             debug_assert!(escaped.is_empty());
 
-                            if !offset.provably_nonzero_i32() {
-                                self.flag_table_possibly_non_null_zero_element(table_index);
-                            }
-
                             self.result
                                 .module
                                 .table_initialization
@@ -547,8 +537,6 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
                     self.result.code_index + self.result.module.num_imported_funcs as u32;
                 let func_index = FuncIndex::from_u32(func_index);
 
-                let call_indirect_start = self.result.module.num_call_indirect_caches;
-
                 if self.tunables.generate_native_debuginfo {
                     let sig_index = self.result.module.functions[func_index].signature;
                     let sig = self.types[sig_index].unwrap_func();
@@ -567,12 +555,9 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
                             params: sig.params().into(),
                         });
                 }
-                self.prescan_code_section(&body)?;
-                self.result.function_body_inputs.push(FunctionBodyData {
-                    validator,
-                    body,
-                    call_indirect_start,
-                });
+                self.result
+                    .function_body_inputs
+                    .push(FunctionBodyData { validator, body });
                 self.result.code_index += 1;
             }
 
@@ -684,83 +669,6 @@ and for re-adding support for interface types you can see this issue:
         Ok(())
     }
 
-    /// Check various properties in function bodies in a "pre-pass" as
-    /// needed, before we actually generate code. Currently this is
-    /// used for:
-    ///
-    /// - Call-indirect caching: we need to know whether a table is
-    ///   "immutable", i.e., there are opcodes that could update its
-    ///   entries. If this is the case then the optimization isn't
-    ///   applicable. We can check this by simply scanning all functions
-    ///   for the relevant opcodes.
-    ///
-    ///   We also need to know how many `call_indirect` opcodes are in
-    ///   the whole module so that we know how large a `vmctx` struct
-    ///   to reserve and what its layout will be; and the starting
-    ///   index in this count for each function, so we can generate
-    ///   its code (with accesses to its own `call_indirect` callsite
-    ///   caches) in parallel.
-    fn prescan_code_section(&mut self, body: &FunctionBody<'_>) -> Result<()> {
-        if self.tunables.cache_call_indirects {
-            for op in body.get_operators_reader()? {
-                let op = op?;
-                match op {
-                    // Check whether a table may be mutated by any
-                    // opcode. (Note that we separately check for
-                    // table exports so we can detect mutations from
-                    // the outside; here we are only concerned with
-                    // mutations by our own module's code.)
-                    Operator::TableSet { table }
-                    | Operator::TableFill { table }
-                    | Operator::TableInit { table, .. }
-                    | Operator::TableCopy {
-                        dst_table: table, ..
-                    } => {
-                        // We haven't yet validated the body during
-                        // this pre-scan, so we need to check that
-                        // `dst_table` is in bounds. Ignore if not:
-                        // we'll catch the error later.
-                        let table = TableIndex::from_u32(table);
-                        if table.index() < self.result.module.table_plans.len() {
-                            self.flag_written_table(table);
-                        }
-                    }
-                    // Count the `call_indirect` sites so we can
-                    // assign them unique slots.
-                    //
-                    // We record the value of this counter as a
-                    // start-index as we start to scan each function,
-                    // and that function's compilation (which is
-                    // normally a separate parallel task) counts on
-                    // its own from that start index.
-                    Operator::CallIndirect { .. } => {
-                        self.result.module.num_call_indirect_caches += 1;
-
-                        // Cap the `num_call_indirect_caches` counter
-                        // at `max_call_indirect_cache_slots` so that
-                        // we don't allocate more than that amount of
-                        // space in the VMContext struct.
-                        //
-                        // Note that we also separately check against
-                        // this limit when emitting code for each
-                        // individual slot because we may cross the
-                        // limit in the middle of a function; also
-                        // once we hit the limit, the start-index for
-                        // each subsequent function will be saturated
-                        // at the limit.
-                        self.result.module.num_call_indirect_caches = core::cmp::min(
-                            self.result.module.num_call_indirect_caches,
-                            self.tunables.max_call_indirect_cache_slots,
-                        );
-                    }
-
-                    _ => {}
-                }
-            }
-        }
-        Ok(())
-    }
-
     fn register_custom_section(&mut self, section: &CustomSectionReader<'data>) {
         match section.as_known() {
             KnownCustom::Name(name) => {
@@ -886,14 +794,6 @@ and for re-adding support for interface types you can see this issue:
         self.result.module.num_escaped_funcs += 1;
     }
 
-    fn flag_written_table(&mut self, table: TableIndex) {
-        self.result.module.table_plans[table].written = true;
-    }
-
-    fn flag_table_possibly_non_null_zero_element(&mut self, table: TableIndex) {
-        self.result.module.table_plans[table].non_null_zero = true;
-    }
-
     /// Parses the Name section of the wasm module.
     fn name_section(&mut self, names: NameSectionReader<'data>) -> WasmResult<()> {
         for subsection in names {
diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs
index 724d0aad5..e983333c3 100644
--- a/crates/environ/src/module.rs
+++ b/crates/environ/src/module.rs
@@ -338,24 +338,13 @@ pub struct TablePlan {
     pub table: Table,
     /// Our chosen implementation style.
     pub style: TableStyle,
-    /// Whether the table is observed to be written or possibly
-    /// written: either by some opcode present in the code section, or
-    /// by the fact that the table is exported.
-    pub written: bool,
-    /// Whether this table may have a non-null zero element.
-    pub non_null_zero: bool,
 }
 
 impl TablePlan {
     /// Draw up a plan for implementing a `Table`.
     pub fn for_table(table: Table, tunables: &Tunables) -> Self {
         let style = TableStyle::for_table(table, tunables);
-        Self {
-            table,
-            style,
-            written: false,
-            non_null_zero: false,
-        }
+        Self { table, style }
     }
 }
 
diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs
index d32cd5bb7..c6fb03aa1 100644
--- a/crates/environ/src/tunables.rs
+++ b/crates/environ/src/tunables.rs
@@ -65,14 +65,6 @@ pub struct Tunables {
 
     /// Whether or not Wasm functions target the winch abi.
     pub winch_callable: bool,
-
-    /// Whether we implement a one-entry cache at each call_indirect
-    /// site.
-    pub cache_call_indirects: bool,
-
-    /// The maximum number of call-indirect cache slots that we will
-    /// allocate for one instance.
-    pub max_call_indirect_cache_slots: usize,
 }
 
 impl Tunables {
@@ -125,8 +117,6 @@ impl Tunables {
             relaxed_simd_deterministic: false,
             tail_callable: false,
             winch_callable: false,
-            cache_call_indirects: false,
-            max_call_indirect_cache_slots: 50_000,
         }
     }
 
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index d913f2d62..852c76d09 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -24,12 +24,11 @@
 //      owned_memories: [VMMemoryDefinition; module.num_owned_memories],
 //      globals: [VMGlobalDefinition; module.num_defined_globals],
 //      func_refs: [VMFuncRef; module.num_escaped_funcs],
-//      call_indirect_caches: [VMCallIndirectCache; module.num_call_indirect_caches],
 // }
 
 use crate::{
-    CallIndirectSiteIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex,
-    FuncRefIndex, GlobalIndex, MemoryIndex, Module, TableIndex,
+    DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, FuncRefIndex,
+    GlobalIndex, MemoryIndex, Module, TableIndex,
 };
 use cranelift_entity::packed_option::ReservedValue;
 use wasmtime_types::OwnedMemoryIndex;
@@ -74,8 +73,6 @@ pub struct VMOffsets<P> {
     /// The number of escaped functions in the module, the size of the func_refs
     /// array.
     pub num_escaped_funcs: u32,
-    /// The number of call_indirect cache entries in the cache array.
-    pub num_call_indirect_caches: u32,
 
     // precalculated offsets of various member fields
     imported_functions: u32,
@@ -87,7 +84,6 @@ pub struct VMOffsets<P> {
     owned_memories: u32,
     defined_globals: u32,
     defined_func_refs: u32,
-    call_indirect_caches: u32,
     size: u32,
 }
 
@@ -219,23 +215,6 @@ pub trait PtrSize {
         .unwrap()
     }
 
-    // Offsets within `VMCallIndirectCache`.
-
-    /// Return the offset of `VMCallIndirectCache::wasm_call`.
-    fn vmcall_indirect_cache_wasm_call(&self) -> u8 {
-        0
-    }
-
-    /// Return the offset of `VMCallIndirectCache::index`.
-    fn vmcall_indirect_cache_index(&self) -> u8 {
-        self.size()
-    }
-
-    /// Return the size of a `VMCallIndirectCache`.
-    fn size_of_vmcall_indirect_cache(&self) -> u8 {
-        2 * self.size()
-    }
-
     /// Return the offset to the `magic` value in this `VMContext`.
     #[inline]
     fn vmctx_magic(&self) -> u8 {
@@ -354,8 +333,6 @@ pub struct VMOffsetsFields<P> {
     /// The number of escaped functions in the module, the size of the function
     /// references array.
     pub num_escaped_funcs: u32,
-    /// The number of call_indirect cache entries in the cache array.
-    pub num_call_indirect_caches: u32,
 }
 
 impl<P: PtrSize> VMOffsets<P> {
@@ -382,7 +359,6 @@ impl<P: PtrSize> VMOffsets<P> {
             num_owned_memories,
             num_defined_globals: cast_to_u32(module.globals.len() - module.num_imported_globals),
             num_escaped_funcs: cast_to_u32(module.num_escaped_funcs),
-            num_call_indirect_caches: cast_to_u32(module.num_call_indirect_caches),
         })
     }
 
@@ -412,7 +388,6 @@ impl<P: PtrSize> VMOffsets<P> {
                     num_defined_memories: _,
                     num_owned_memories: _,
                     num_escaped_funcs: _,
-                    num_call_indirect_caches: _,
 
                     // used as the initial size below
                     size,
@@ -441,7 +416,6 @@ impl<P: PtrSize> VMOffsets<P> {
         }
 
         calculate_sizes! {
-            call_indirect_caches: "call_indirect caches",
             defined_func_refs: "module functions",
             defined_globals: "defined globals",
             owned_memories: "owned memories",
@@ -468,7 +442,6 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
             num_owned_memories: fields.num_owned_memories,
             num_defined_globals: fields.num_defined_globals,
             num_escaped_funcs: fields.num_escaped_funcs,
-            num_call_indirect_caches: fields.num_call_indirect_caches,
             imported_functions: 0,
             imported_tables: 0,
             imported_memories: 0,
@@ -478,7 +451,6 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
             owned_memories: 0,
             defined_globals: 0,
             defined_func_refs: 0,
-            call_indirect_caches: 0,
             size: 0,
         };
 
@@ -533,10 +505,6 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
                 ret.num_escaped_funcs,
                 ret.ptr.size_of_vm_func_ref(),
             ),
-            size(call_indirect_caches) = cmul(
-                ret.num_call_indirect_caches,
-                ret.ptr.size_of_vmcall_indirect_cache(),
-            ),
         }
 
         ret.size = next_field_offset;
@@ -727,12 +695,6 @@ impl<P: PtrSize> VMOffsets<P> {
         self.defined_func_refs
     }
 
-    /// The offset of the `call_indirect_caches` array.
-    #[inline]
-    pub fn vmctx_call_indirec_caches_begin(&self) -> u32 {
-        self.call_indirect_caches
-    }
-
     /// Return the size of the `VMContext` allocation.
     #[inline]
     pub fn size_of_vmctx(&self) -> u32 {
@@ -877,31 +839,6 @@ impl<P: PtrSize> VMOffsets<P> {
     pub fn vmctx_vmglobal_import_from(&self, index: GlobalIndex) -> u32 {
         self.vmctx_vmglobal_import(index) + u32::from(self.vmglobal_import_from())
     }
-
-    /// Return the offset to the `VMCallIndirectCache` for the given
-    /// call-indirect site.
-    #[inline]
-    pub fn vmctx_call_indirect_cache(&self, call_site: CallIndirectSiteIndex) -> u32 {
-        assert!(call_site.as_u32() < self.num_call_indirect_caches);
-        self.vmctx_call_indirec_caches_begin()
-            + call_site.as_u32() * u32::from(self.ptr.size_of_vmcall_indirect_cache())
-    }
-
-    /// Return the offset to the `wasm_call` field in `*const
-    /// VMCallIndirectCache` with call-site ID `call_site`.
-    #[inline]
-    pub fn vmctx_call_indirect_cache_wasm_call(&self, call_site: CallIndirectSiteIndex) -> u32 {
-        self.vmctx_call_indirect_cache(call_site)
-            + u32::from(self.ptr.vmcall_indirect_cache_wasm_call())
-    }
-
-    /// Return the offset to the `index` field in `*const
-    /// VMCallIndirectCache` with call-site ID `call_site`.
-    #[inline]
-    pub fn vmctx_call_indirect_cache_index(&self, call_site: CallIndirectSiteIndex) -> u32 {
-        self.vmctx_call_indirect_cache(call_site)
-            + u32::from(self.ptr.vmcall_indirect_cache_index())
-    }
 }
 
 /// Offsets for `VMDrcHeader`.
diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs
index 294613700..d6fd64824 100644
--- a/crates/fuzzing/src/generators/config.rs
+++ b/crates/fuzzing/src/generators/config.rs
@@ -180,9 +180,7 @@ impl Config {
                 self.wasmtime.memory_guaranteed_dense_image_size,
             ))
             .allocation_strategy(self.wasmtime.strategy.to_wasmtime())
-            .generate_address_map(self.wasmtime.generate_address_map)
-            .cache_call_indirects(self.wasmtime.cache_call_indirects)
-            .max_call_indirect_cache_slots(self.wasmtime.max_call_indirect_cache_slots);
+            .generate_address_map(self.wasmtime.generate_address_map);
 
         if !self.module_config.config.simd_enabled {
             cfg.wasm_relaxed_simd(false);
@@ -501,10 +499,6 @@ pub struct WasmtimeConfig {
     native_unwind_info: bool,
     /// Configuration for the compiler to use.
     pub compiler_strategy: CompilerStrategy,
-    /// Whether we enable indirect-call caching.
-    cache_call_indirects: bool,
-    /// The maximum number of call-indirect cache slots.
-    max_call_indirect_cache_slots: usize,
     table_lazy_init: bool,
 
     /// Whether or not fuzzing should enable PCC.
diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs
index 2fe7cb056..dd9946fb2 100644
--- a/crates/types/src/lib.rs
+++ b/crates/types/src/lib.rs
@@ -1146,12 +1146,6 @@ entity_impl!(TagIndex);
 pub struct StaticModuleIndex(u32);
 entity_impl!(StaticModuleIndex);
 
-/// Index of a `call_indirect` instruction in a module, used for
-/// caching that callsite's target in the VMContext.
-#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
-pub struct CallIndirectSiteIndex(u32);
-entity_impl!(CallIndirectSiteIndex);
-
 /// An index of an entity.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
 pub enum EntityIndex {
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 2e678f65a..3c4e49b00 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -151,8 +151,6 @@ struct ConfigTunables {
     debug_adapter_modules: Option<bool>,
     relaxed_simd_deterministic: Option<bool>,
     tail_callable: Option<bool>,
-    cache_call_indirects: Option<bool>,
-    max_call_indirect_cache_slots: Option<usize>,
 }
 
 /// User-provided configuration for the compiler.
@@ -986,57 +984,6 @@ impl Config {
         self
     }
 
-    /// Configures whether we enable the "indirect call cache" optimization.
-    ///
-    /// This feature adds, for each `call_indirect` instruction in a
-    /// Wasm module (i.e., a function-pointer call in guest code), a
-    /// one-entry cache that speeds up the translation from a table
-    /// index to the actual machine code. By default, the VM's
-    /// implementation of this translation requires several
-    /// indirections and checks (table bounds-check, function
-    /// signature-check, table lazy-initialization logic). The intent
-    /// of this feature is to speed up indirect calls substantially
-    /// when they are repeated frequently in hot code.
-    ///
-    /// While it accelerates repeated calls, this feature has the
-    /// potential to slow down instantiation slightly, because it adds
-    /// additional state (the cache storage -- usually 16 bytes per
-    /// `call_indirect` instruction for each instance) that has to be
-    /// initialized. In practice, we have not seen
-    /// measurable/statistically-significant impact from this, though.
-    ///
-    /// Until we have further experience with this feature, it will
-    /// remain off: it is `false` by default.
-    pub fn cache_call_indirects(&mut self, enable: bool) -> &mut Self {
-        self.tunables.cache_call_indirects = Some(enable);
-        self
-    }
-
-    /// Configures the "indirect call cache" maximum capacity.
-    ///
-    /// If the [`Config::cache_call_indirects`] configuration option
-    /// is enabled, the engine allocates "cache slots" directly in its
-    /// per-instance state struct for each `call_indirect` in the
-    /// module's code. We place a limit on this count in order to
-    /// avoid inflating the state too much with very large modules. If
-    /// a module exceeds the limit, the first `max` indirect
-    /// call-sites will still have a one-entry cache, but any indirect
-    /// call-sites beyond the limit (in linear order in the module's
-    /// code section) do not participate in the caching, as if the
-    /// option were turned off.
-    ///
-    /// There is also an internal hard cap to this limit:
-    /// configurations with `max` beyond `50_000` will effectively cap
-    /// the limit at `50_000`. This is so that instance state does not
-    /// become unreasonably large.
-    ///
-    /// This is `50_000` by default.
-    pub fn max_call_indirect_cache_slots(&mut self, max: usize) -> &mut Self {
-        const HARD_CAP: usize = 50_000; // See doc-comment above.
-        self.tunables.max_call_indirect_cache_slots = Some(core::cmp::min(max, HARD_CAP));
-        self
-    }
-
     /// Configures which compilation strategy will be used for wasm modules.
     ///
     /// This method can be used to configure which compiler is used for wasm
@@ -1849,8 +1796,6 @@ impl Config {
             debug_adapter_modules
             relaxed_simd_deterministic
             tail_callable
-            cache_call_indirects
-            max_call_indirect_cache_slots
         }
 
         // If we're going to compile with winch, we must use the winch calling convention.
diff --git a/crates/wasmtime/src/engine/serialization.rs b/crates/wasmtime/src/engine/serialization.rs
index 5da038859..f350f9286 100644
--- a/crates/wasmtime/src/engine/serialization.rs
+++ b/crates/wasmtime/src/engine/serialization.rs
@@ -360,8 +360,6 @@ impl Metadata<'_> {
             relaxed_simd_deterministic,
             tail_callable,
             winch_callable,
-            cache_call_indirects,
-            max_call_indirect_cache_slots,
 
             // This doesn't affect compilation, it's just a runtime setting.
             dynamic_memory_growth_reserve: _,
@@ -429,16 +427,6 @@ impl Metadata<'_> {
             other.winch_callable,
             "Winch calling convention",
         )?;
-        Self::check_bool(
-            cache_call_indirects,
-            other.cache_call_indirects,
-            "caching of call-indirect targets",
-        )?;
-        Self::check_int(
-            max_call_indirect_cache_slots,
-            other.max_call_indirect_cache_slots,
-            "maximum slot count for caching of call-indirect targets",
-        )?;
 
         Ok(())
     }
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index ce49c3474..a0bbcb08f 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -900,7 +900,6 @@ mod tests {
             num_owned_memories: 0,
             num_defined_globals: 0,
             num_escaped_funcs: 0,
-            num_call_indirect_caches: 0,
         });
 
         assert_eq!(
@@ -929,7 +928,6 @@ mod tests {
             num_owned_memories: 0,
             num_defined_globals: 0,
             num_escaped_funcs: 0,
-            num_call_indirect_caches: 0,
         });
         assert_eq!(
             offsets.vm_gc_ref_activation_table_next() as usize,
@@ -957,7 +955,6 @@ mod tests {
             num_owned_memories: 0,
             num_defined_globals: 0,
             num_escaped_funcs: 0,
-            num_call_indirect_caches: 0,
         });
         assert_eq!(
             offsets.vm_gc_ref_activation_table_end() as usize,
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index d9da984eb..073f6d80a 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -871,53 +871,6 @@ mod test_vmruntime_limits {
     }
 }
 
-/// One call-indirect cache entry.
-///
-/// It consists of the last observed function-pointer index, and the
-/// direct code pointer (with the same vmctx, i.e., in the same
-/// instance) to call if this index matches.
-#[derive(Debug, Clone)]
-#[allow(dead_code)] // not actually used in Rust runtime code; only in generated code.
-#[repr(C)]
-pub struct VMCallIndirectCache {
-    /// Function pointer for this funcref if being called via the Wasm
-    /// calling convention.
-    pub wasm_call: NonNull<VMWasmCallFunction>,
-
-    /// Table index corresponding to the above function pointer.
-    pub index: usize,
-    // If more elements are added here, remember to add offset_of tests below!
-}
-
-unsafe impl Send for VMCallIndirectCache {}
-unsafe impl Sync for VMCallIndirectCache {}
-
-#[cfg(test)]
-mod test_vm_call_indirect_cache {
-    use super::VMCallIndirectCache;
-    use core::mem::offset_of;
-    use std::mem::size_of;
-    use wasmtime_environ::{Module, PtrSize, VMOffsets};
-
-    #[test]
-    fn check_vm_call_indirect_cache_offsets() {
-        let module = Module::new();
-        let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module);
-        assert_eq!(
-            size_of::<VMCallIndirectCache>(),
-            usize::from(offsets.ptr.size_of_vmcall_indirect_cache())
-        );
-        assert_eq!(
-            offset_of!(VMCallIndirectCache, wasm_call),
-            usize::from(offsets.ptr.vmcall_indirect_cache_wasm_call())
-        );
-        assert_eq!(
-            offset_of!(VMCallIndirectCache, index),
-            usize::from(offsets.ptr.vmcall_indirect_cache_index())
-        );
-    }
-}
-
 /// The VM "context", which is pointed to by the `vmctx` arg in Cranelift.
 /// This has information about globals, memories, tables, and other runtime
 /// state associated with the current instance.


Test patch:
diff --git a/cranelift/filetests/filetests/isa/aarch64/user_stack_maps.clif b/cranelift/filetests/filetests/isa/aarch64/user_stack_maps.clif
new file mode 100644
index 000000000..5c1d93b62
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/user_stack_maps.clif
@@ -0,0 +1,221 @@
+test compile precise-output
+set unwind_info=false
+set enable_probestack=false
+target aarch64
+
+function %foo() system_v {
+    ss0 = explicit_slot 12, align = 4
+    sig0 = (i32) system_v
+    fn0 = colocated u0:0 sig0
+
+block0:
+    v0 = iconst.i32 0
+    v1 = iconst.i32 1
+    v2 = iconst.i32 2
+    v3 = iconst.i32 3
+
+    stack_store v0, ss0
+    stack_store v1, ss0+4
+    stack_store v2, ss0+8
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4, i32 @ ss0+8]
+
+    stack_store v1, ss0
+    stack_store v2, ss0+4
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4]
+
+    stack_store v2, ss0
+    call fn0(v1), stack_map=[i32 @ ss0+0]
+
+    call fn0(v2)
+
+    return
+}
+
+; VCode:
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   str x24, [sp, #-16]!
+;   stp x19, x22, [sp, #-16]!
+;   sub sp, sp, #16
+; block0:
+;   movz w9, #0
+;   movz w8, #1
+;   movz w0, #2
+;   mov x10, sp
+;   str w9, [x10]
+;   mov x24, x9
+;   add x9, sp, #4
+;   str w8, [x9]
+;   mov x19, x8
+;   add x10, sp, #8
+;   str w0, [x10]
+;   mov x22, x0
+;   mov x0, x24
+;   bl 0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   mov x12, sp
+;   mov x0, x19
+;   str w0, [x12]
+;   add x13, sp, #4
+;   mov x0, x22
+;   str w0, [x13]
+;   mov x22, x0
+;   mov x0, x24
+;   bl 0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   mov x15, sp
+;   mov x0, x22
+;   str w0, [x15]
+;   mov x0, x19
+;   bl 0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   mov x0, x22
+;   bl 0
+;   add sp, sp, #16
+;   ldp x19, x22, [sp], #16
+;   ldr x24, [sp], #16
+;   ldp fp, lr, [sp], #16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   stp x29, x30, [sp, #-0x10]!
+;   mov x29, sp
+;   str x24, [sp, #-0x10]!
+;   stp x19, x22, [sp, #-0x10]!
+;   sub sp, sp, #0x10
+; block1: ; offset 0x14
+;   mov w9, #0
+;   mov w8, #1
+;   mov w0, #2
+;   mov x10, sp
+;   str w9, [x10]
+;   mov x24, x9
+;   add x9, sp, #4
+;   str w8, [x9]
+;   mov x19, x8
+;   add x10, sp, #8
+;   str w0, [x10]
+;   mov x22, x0
+;   mov x0, x24
+;   bl #0x48 ; reloc_external Call u0:0 0
+;   mov x12, sp
+;   mov x0, x19
+;   str w0, [x12]
+;   add x13, sp, #4
+;   mov x0, x22
+;   str w0, [x13]
+;   mov x22, x0
+;   mov x0, x24
+;   bl #0x6c ; reloc_external Call u0:0 0
+;   mov x15, sp
+;   mov x0, x22
+;   str w0, [x15]
+;   mov x0, x19
+;   bl #0x80 ; reloc_external Call u0:0 0
+;   mov x0, x22
+;   bl #0x88 ; reloc_external Call u0:0 0
+;   add sp, sp, #0x10
+;   ldp x19, x22, [sp], #0x10
+;   ldr x24, [sp], #0x10
+;   ldp x29, x30, [sp], #0x10
+;   ret
+
+function %different_types(i8, i16, i32, i64, f32, f64) -> i8, i16, i32, i64, f32, f64 {
+    ss0 = explicit_slot 1
+    ss1 = explicit_slot 2, align = 2
+    ss2 = explicit_slot 8, align = 4
+    ss3 = explicit_slot 16, align = 8
+    ss4 = explicit_slot 48, align = 16
+    sig0 = () system_v
+    fn0 = colocated u0:0 sig0
+
+block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
+    stack_store v0, ss0
+    stack_store v1, ss1
+    stack_store v2, ss2
+    stack_store v4, ss2+4
+    stack_store v3, ss3
+    stack_store v5, ss3+8
+    call fn0(), stack_map=[i8 @ ss0+0, i16 @ ss1+0, i32 @ ss2+0, f32 @ ss2+4, i64 @ ss3+0, f64 @ ss3+8]
+    return v0, v1, v2, v3, v4, v5
+}
+
+; VCode:
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   stp x23, x27, [sp, #-16]!
+;   stp x20, x21, [sp, #-16]!
+;   sub sp, sp, #128
+; block0:
+;   mov x12, sp
+;   strb w0, [x12]
+;   mov x23, x0
+;   add x13, sp, #8
+;   strh w1, [x13]
+;   mov x20, x1
+;   add x14, sp, #16
+;   str w2, [x14]
+;   mov x21, x2
+;   add x15, sp, #20
+;   str s0, [x15]
+;   str q0, [sp, #96]
+;   add x0, sp, #24
+;   str x3, [x0]
+;   mov x27, x3
+;   add x1, sp, #32
+;   str d1, [x1]
+;   str q1, [sp, #112]
+;   bl 0
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   mov x0, x23
+;   mov x1, x20
+;   mov x2, x21
+;   mov x3, x27
+;   ldr q0, [sp, #96]
+;   ldr q1, [sp, #112]
+;   add sp, sp, #128
+;   ldp x20, x21, [sp], #16
+;   ldp x23, x27, [sp], #16
+;   ldp fp, lr, [sp], #16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   stp x29, x30, [sp, #-0x10]!
+;   mov x29, sp
+;   stp x23, x27, [sp, #-0x10]!
+;   stp x20, x21, [sp, #-0x10]!
+;   sub sp, sp, #0x80
+; block1: ; offset 0x14
+;   mov x12, sp
+;   strb w0, [x12]
+;   mov x23, x0
+;   add x13, sp, #8
+;   strh w1, [x13]
+;   mov x20, x1
+;   add x14, sp, #0x10
+;   str w2, [x14]
+;   mov x21, x2
+;   add x15, sp, #0x14
+;   str s0, [x15]
+;   stur q0, [sp, #0x60]
+;   add x0, sp, #0x18
+;   str x3, [x0]
+;   mov x27, x3
+;   add x1, sp, #0x20
+;   str d1, [x1]
+;   stur q1, [sp, #0x70]
+;   bl #0x5c ; reloc_external Call u0:0 0
+;   mov x0, x23
+;   mov x1, x20
+;   mov x2, x21
+;   mov x3, x27
+;   ldur q0, [sp, #0x60]
+;   ldur q1, [sp, #0x70]
+;   add sp, sp, #0x80
+;   ldp x20, x21, [sp], #0x10
+;   ldp x23, x27, [sp], #0x10
+;   ldp x29, x30, [sp], #0x10
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/user_stack_maps.clif b/cranelift/filetests/filetests/isa/riscv64/user_stack_maps.clif
new file mode 100644
index 000000000..b265508b9
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/user_stack_maps.clif
@@ -0,0 +1,250 @@
+test compile precise-output
+set unwind_info=false
+set enable_probestack=false
+target riscv64
+
+function %foo() system_v {
+    ss0 = explicit_slot 12, align = 4
+    sig0 = (i32) system_v
+    fn0 = colocated u0:0 sig0
+
+block0:
+    v0 = iconst.i32 0
+    v1 = iconst.i32 1
+    v2 = iconst.i32 2
+    v3 = iconst.i32 3
+
+    stack_store v0, ss0
+    stack_store v1, ss0+4
+    stack_store v2, ss0+8
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4, i32 @ ss0+8]
+
+    stack_store v1, ss0
+    stack_store v2, ss0+4
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4]
+
+    stack_store v2, ss0
+    call fn0(v1), stack_map=[i32 @ ss0+0]
+
+    call fn0(v2)
+
+    return
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+;   addi sp,sp,-48
+;   sd s1,40(sp)
+;   sd s2,32(sp)
+;   sd s3,24(sp)
+; block0:
+;   li a0,0
+;   mv s3,a0
+;   li a0,1
+;   li a2,2
+;   sw zero,0(slot)
+;   sw a0,4(slot)
+;   mv s1,a0
+;   sw a2,8(slot)
+;   mv s2,a2
+;   mv a0,s3
+;   call userextname0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   mv a0,s1
+;   sw a0,0(slot)
+;   mv a0,s2
+;   sw a0,4(slot)
+;   mv s2,a0
+;   mv a0,s3
+;   call userextname0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   mv a0,s2
+;   sw a0,0(slot)
+;   mv a0,s1
+;   call userextname0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   mv a0,s2
+;   call userextname0
+;   ld s1,40(sp)
+;   ld s2,32(sp)
+;   ld s3,24(sp)
+;   addi sp,sp,48
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+;   addi sp, sp, -0x30
+;   sd s1, 0x28(sp)
+;   sd s2, 0x20(sp)
+;   sd s3, 0x18(sp)
+; block1: ; offset 0x20
+;   mv a0, zero
+;   mv s3, a0
+;   addi a0, zero, 1
+;   addi a2, zero, 2
+;   sw zero, 0(sp)
+;   sw a0, 4(sp)
+;   mv s1, a0
+;   sw a2, 8(sp)
+;   mv s2, a2
+;   mv a0, s3
+;   auipc ra, 0 ; reloc_external RiscvCallPlt u0:0 0
+;   jalr ra
+;   mv a0, s1
+;   sw a0, 0(sp)
+;   mv a0, s2
+;   sw a0, 4(sp)
+;   mv s2, a0
+;   mv a0, s3
+;   auipc ra, 0 ; reloc_external RiscvCallPlt u0:0 0
+;   jalr ra
+;   mv a0, s2
+;   sw a0, 0(sp)
+;   mv a0, s1
+;   auipc ra, 0 ; reloc_external RiscvCallPlt u0:0 0
+;   jalr ra
+;   mv a0, s2
+;   auipc ra, 0 ; reloc_external RiscvCallPlt u0:0 0
+;   jalr ra
+;   ld s1, 0x28(sp)
+;   ld s2, 0x20(sp)
+;   ld s3, 0x18(sp)
+;   addi sp, sp, 0x30
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
+function %different_types(i8, i16, i32, i64, f32, f64) -> i8, i16, i32, i64, f32, f64 {
+    ss0 = explicit_slot 1
+    ss1 = explicit_slot 2, align = 2
+    ss2 = explicit_slot 8, align = 4
+    ss3 = explicit_slot 16, align = 8
+    ss4 = explicit_slot 48, align = 16
+    sig0 = () system_v
+    fn0 = colocated u0:0 sig0
+
+block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
+    stack_store v0, ss0
+    stack_store v1, ss1
+    stack_store v2, ss2
+    stack_store v4, ss2+4
+    stack_store v3, ss3
+    stack_store v5, ss3+8
+    call fn0(), stack_map=[i8 @ ss0+0, i16 @ ss1+0, i32 @ ss2+0, f32 @ ss2+4, i64 @ ss3+0, f64 @ ss3+8]
+    return v0, v1, v2, v3, v4, v5
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+;   addi sp,sp,-160
+;   sd s1,152(sp)
+;   sd s2,144(sp)
+;   sd s6,136(sp)
+;   sd s8,128(sp)
+;   sd s10,120(sp)
+;   fsd fs0,112(sp)
+;   fsd fs2,104(sp)
+; block0:
+;   mv s10,a4
+;   sb a0,0(slot)
+;   mv s8,a0
+;   sh a1,8(slot)
+;   mv s6,a1
+;   sw a2,16(slot)
+;   mv s2,a2
+;   fsw fa0,20(slot)
+;   fmv.d fs2,fa0
+;   sd a3,24(slot)
+;   mv s1,a3
+;   fsd fa1,32(slot)
+;   fmv.d fs0,fa1
+;   call userextname0
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   mv a2,s2
+;   mv a4,s10
+;   sw a2,0(a4)
+;   mv a3,s1
+;   sd a3,8(a4)
+;   mv a0,s8
+;   mv a1,s6
+;   fmv.d fa0,fs2
+;   fmv.d fa1,fs0
+;   ld s1,152(sp)
+;   ld s2,144(sp)
+;   ld s6,136(sp)
+;   ld s8,128(sp)
+;   ld s10,120(sp)
+;   fld fs0,112(sp)
+;   fld fs2,104(sp)
+;   addi sp,sp,160
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+;   addi sp, sp, -0xa0
+;   sd s1, 0x98(sp)
+;   sd s2, 0x90(sp)
+;   sd s6, 0x88(sp)
+;   sd s8, 0x80(sp)
+;   sd s10, 0x78(sp)
+;   fsd fs0, 0x70(sp)
+;   fsd fs2, 0x68(sp)
+; block1: ; offset 0x30
+;   mv s10, a4
+;   sb a0, 0(sp)
+;   mv s8, a0
+;   sh a1, 8(sp)
+;   mv s6, a1
+;   sw a2, 0x10(sp)
+;   mv s2, a2
+;   fsw fa0, 0x14(sp)
+;   fmv.d fs2, fa0
+;   sd a3, 0x18(sp)
+;   mv s1, a3
+;   fsd fa1, 0x20(sp)
+;   fmv.d fs0, fa1
+;   auipc ra, 0 ; reloc_external RiscvCallPlt u0:0 0
+;   jalr ra
+;   mv a2, s2
+;   mv a4, s10
+;   sw a2, 0(a4)
+;   mv a3, s1
+;   sd a3, 8(a4)
+;   mv a0, s8
+;   mv a1, s6
+;   fmv.d fa0, fs2
+;   fmv.d fa1, fs0
+;   ld s1, 0x98(sp)
+;   ld s2, 0x90(sp)
+;   ld s6, 0x88(sp)
+;   ld s8, 0x80(sp)
+;   ld s10, 0x78(sp)
+;   fld fs0, 0x70(sp)
+;   fld fs2, 0x68(sp)
+;   addi sp, sp, 0xa0
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/s390x/user_stack_maps.clif b/cranelift/filetests/filetests/isa/s390x/user_stack_maps.clif
new file mode 100644
index 000000000..911a93f61
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/user_stack_maps.clif
@@ -0,0 +1,221 @@
+test compile precise-output
+set unwind_info=false
+set enable_probestack=false
+target s390x
+
+function %foo() system_v {
+    ss0 = explicit_slot 12, align = 4
+    sig0 = (i32) system_v
+    fn0 = colocated u0:0 sig0
+
+block0:
+    v0 = iconst.i32 0
+    v1 = iconst.i32 1
+    v2 = iconst.i32 2
+    v3 = iconst.i32 3
+
+    stack_store v0, ss0
+    stack_store v1, ss0+4
+    stack_store v2, ss0+8
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4, i32 @ ss0+8]
+
+    stack_store v1, ss0
+    stack_store v2, ss0+4
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4]
+
+    stack_store v2, ss0
+    call fn0(v1), stack_map=[i32 @ ss0+0]
+
+    call fn0(v2)
+
+    return
+}
+
+; VCode:
+;   stmg %r7, %r15, 56(%r15)
+;   aghi %r15, -176
+; block0:
+;   lhi %r2, 0
+;   lgr %r11, %r2
+;   lhi %r2, 1
+;   lgr %r7, %r2
+;   lhi %r2, 2
+;   lgr %r9, %r2
+;   la %r2, 160(%r15)
+;   mvhi 0(%r2), 0
+;   la %r3, 164(%r15)
+;   mvhi 0(%r3), 1
+;   la %r4, 168(%r15)
+;   mvhi 0(%r4), 2
+;   lgr %r2, %r11
+;   brasl %r14, userextname0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   la %r2, 160(%r15)
+;   mvhi 0(%r2), 1
+;   la %r3, 164(%r15)
+;   mvhi 0(%r3), 2
+;   lgr %r2, %r11
+;   brasl %r14, userextname0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   la %r5, 160(%r15)
+;   mvhi 0(%r5), 2
+;   lgr %r2, %r7
+;   brasl %r14, userextname0
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   lgr %r2, %r9
+;   brasl %r14, userextname0
+;   lmg %r7, %r15, 232(%r15)
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   stmg %r7, %r15, 0x38(%r15)
+;   aghi %r15, -0xb0
+; block1: ; offset 0xa
+;   lhi %r2, 0
+;   lgr %r11, %r2
+;   lhi %r2, 1
+;   lgr %r7, %r2
+;   lhi %r2, 2
+;   lgr %r9, %r2
+;   la %r2, 0xa0(%r15)
+;   mvhi 0(%r2), 0
+;   la %r3, 0xa4(%r15)
+;   mvhi 0(%r3), 1
+;   la %r4, 0xa8(%r15)
+;   mvhi 0(%r4), 2
+;   lgr %r2, %r11
+;   brasl %r14, 0x44 ; reloc_external PLTRel32Dbl u0:0 2
+;   la %r2, 0xa0(%r15)
+;   mvhi 0(%r2), 1
+;   la %r3, 0xa4(%r15)
+;   mvhi 0(%r3), 2
+;   lgr %r2, %r11
+;   brasl %r14, 0x62 ; reloc_external PLTRel32Dbl u0:0 2
+;   la %r5, 0xa0(%r15)
+;   mvhi 0(%r5), 2
+;   lgr %r2, %r7
+;   brasl %r14, 0x76 ; reloc_external PLTRel32Dbl u0:0 2
+;   lgr %r2, %r9
+;   brasl %r14, 0x80 ; reloc_external PLTRel32Dbl u0:0 2
+;   lmg %r7, %r15, 0xe8(%r15)
+;   br %r14
+
+function %different_types(i8, i16, i32, i64, f32, f64) -> i8, i16, i32, i64, f32, f64 {
+    ss0 = explicit_slot 1
+    ss1 = explicit_slot 2, align = 2
+    ss2 = explicit_slot 8, align = 4
+    ss3 = explicit_slot 16, align = 8
+    ss4 = explicit_slot 48, align = 16
+    sig0 = () system_v
+    fn0 = colocated u0:0 sig0
+
+block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
+    stack_store v0, ss0
+    stack_store v1, ss1
+    stack_store v2, ss2
+    stack_store v4, ss2+4
+    stack_store v3, ss3
+    stack_store v5, ss3+8
+    call fn0(), stack_map=[i8 @ ss0+0, i16 @ ss1+0, i32 @ ss2+0, f32 @ ss2+4, i64 @ ss3+0, f64 @ ss3+8]
+    return v0, v1, v2, v3, v4, v5
+}
+
+; VCode:
+;   stmg %r6, %r15, 48(%r15)
+;   aghi %r15, -352
+;   std %f8, 288(%r15)
+;   std %f9, 296(%r15)
+;   std %f10, 304(%r15)
+;   std %f11, 312(%r15)
+;   std %f12, 320(%r15)
+;   std %f13, 328(%r15)
+;   std %f14, 336(%r15)
+;   std %f15, 344(%r15)
+; block0:
+;   la %r9, 160(%r15)
+;   stc %r2, 0(%r9)
+;   lgr %r11, %r2
+;   la %r2, 168(%r15)
+;   sth %r3, 0(%r2)
+;   lgr %r9, %r3
+;   la %r2, 176(%r15)
+;   st %r4, 0(%r2)
+;   lgr %r7, %r4
+;   la %r2, 180(%r15)
+;   ste %f0, 0(%r2)
+;   vst %v0, 256(%r15)
+;   la %r2, 184(%r15)
+;   stg %r5, 0(%r2)
+;   lgr %r6, %r5
+;   la %r3, 192(%r15)
+;   std %f2, 0(%r3)
+;   vst %v2, 272(%r15)
+;   brasl %r14, userextname0
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   lgr %r2, %r11
+;   lgr %r3, %r9
+;   lgr %r4, %r7
+;   lgr %r5, %r6
+;   vl %v0, 256(%r15)
+;   vl %v2, 272(%r15)
+;   ld %f8, 288(%r15)
+;   ld %f9, 296(%r15)
+;   ld %f10, 304(%r15)
+;   ld %f11, 312(%r15)
+;   ld %f12, 320(%r15)
+;   ld %f13, 328(%r15)
+;   ld %f14, 336(%r15)
+;   ld %f15, 344(%r15)
+;   lmg %r6, %r15, 400(%r15)
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   stmg %r6, %r15, 0x30(%r15)
+;   aghi %r15, -0x160
+;   std %f8, 0x120(%r15)
+;   std %f9, 0x128(%r15)
+;   std %f10, 0x130(%r15)
+;   std %f11, 0x138(%r15)
+;   std %f12, 0x140(%r15)
+;   std %f13, 0x148(%r15)
+;   std %f14, 0x150(%r15)
+;   std %f15, 0x158(%r15)
+; block1: ; offset 0x2a
+;   la %r9, 0xa0(%r15)
+;   stc %r2, 0(%r9)
+;   lgr %r11, %r2
+;   la %r2, 0xa8(%r15)
+;   sth %r3, 0(%r2)
+;   lgr %r9, %r3
+;   la %r2, 0xb0(%r15)
+;   st %r4, 0(%r2)
+;   lgr %r7, %r4
+;   la %r2, 0xb4(%r15)
+;   ste %f0, 0(%r2)
+;   vst %v0, 0x100(%r15)
+;   la %r2, 0xb8(%r15)
+;   stg %r5, 0(%r2)
+;   lgr %r6, %r5
+;   la %r3, 0xc0(%r15)
+;   std %f2, 0(%r3)
+;   vst %v2, 0x110(%r15)
+;   brasl %r14, 0x78 ; reloc_external PLTRel32Dbl u0:0 2
+;   lgr %r2, %r11
+;   lgr %r3, %r9
+;   lgr %r4, %r7
+;   lgr %r5, %r6
+;   vl %v0, 0x100(%r15)
+;   vl %v2, 0x110(%r15)
+;   ld %f8, 0x120(%r15)
+;   ld %f9, 0x128(%r15)
+;   ld %f10, 0x130(%r15)
+;   ld %f11, 0x138(%r15)
+;   ld %f12, 0x140(%r15)
+;   ld %f13, 0x148(%r15)
+;   ld %f14, 0x150(%r15)
+;   ld %f15, 0x158(%r15)
+;   lmg %r6, %r15, 0x190(%r15)
+;   br %r14
+
diff --git a/cranelift/filetests/filetests/isa/x64/user_stack_maps.clif b/cranelift/filetests/filetests/isa/x64/user_stack_maps.clif
new file mode 100644
index 000000000..bddc779df
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/user_stack_maps.clif
@@ -0,0 +1,241 @@
+test compile precise-output
+set unwind_info=false
+set enable_probestack=false
+target x86_64
+
+function %foo() system_v {
+    ss0 = explicit_slot 12, align = 4
+    sig0 = (i32) system_v
+    fn0 = colocated u0:0 sig0
+
+block0:
+    v0 = iconst.i32 0
+    v1 = iconst.i32 1
+    v2 = iconst.i32 2
+    v3 = iconst.i32 3
+
+    stack_store v0, ss0
+    stack_store v1, ss0+4
+    stack_store v2, ss0+8
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4, i32 @ ss0+8]
+
+    stack_store v1, ss0
+    stack_store v2, ss0+4
+    call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss0+4]
+
+    stack_store v2, ss0
+    call fn0(v1), stack_map=[i32 @ ss0+0]
+
+    call fn0(v2)
+
+    return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+;   subq    %rsp, $48, %rsp
+;   movq    %rbx, 16(%rsp)
+;   movq    %r13, 24(%rsp)
+;   movq    %r15, 32(%rsp)
+; block0:
+;   xorl    %edi, %edi, %edi
+;   movq    %rdi, %r15
+;   movl    $1, %edi
+;   movq    %rdi, %rbx
+;   movl    $2, %edi
+;   movq    %rdi, %r13
+;   lea     rsp(0 + virtual offset), %r11
+;   movl    $0, 0(%r11)
+;   lea     rsp(4 + virtual offset), %rsi
+;   movl    $1, 0(%rsi)
+;   lea     rsp(8 + virtual offset), %rdi
+;   movl    $2, 0(%rdi)
+;   movq    %r15, %rdi
+;   call    User(userextname0)
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4, 8})] }
+;   lea     rsp(0 + virtual offset), %rcx
+;   movl    $1, 0(%rcx)
+;   lea     rsp(4 + virtual offset), %rdx
+;   movl    $2, 0(%rdx)
+;   movq    %r15, %rdi
+;   call    User(userextname0)
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0, 4})] }
+;   lea     rsp(0 + virtual offset), %r9
+;   movl    $2, 0(%r9)
+;   movq    %rbx, %rdi
+;   call    User(userextname0)
+;   ; UserStackMap { by_type: [(types::I32, CompoundBitSet {0})] }
+;   movq    %r13, %rdi
+;   call    User(userextname0)
+;   movq    16(%rsp), %rbx
+;   movq    24(%rsp), %r13
+;   movq    32(%rsp), %r15
+;   addq    %rsp, $48, %rsp
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+;   subq $0x30, %rsp
+;   movq %rbx, 0x10(%rsp)
+;   movq %r13, 0x18(%rsp)
+;   movq %r15, 0x20(%rsp)
+; block1: ; offset 0x17
+;   xorl %edi, %edi
+;   movq %rdi, %r15
+;   movl $1, %edi
+;   movq %rdi, %rbx
+;   movl $2, %edi
+;   movq %rdi, %r13
+;   leaq (%rsp), %r11
+;   movl $0, (%r11)
+;   leaq 4(%rsp), %rsi
+;   movl $1, (%rsi)
+;   leaq 8(%rsp), %rdi
+;   movl $2, (%rdi)
+;   movq %r15, %rdi
+;   callq 0x55 ; reloc_external CallPCRel4 u0:0 -4
+;   leaq (%rsp), %rcx
+;   movl $1, (%rcx)
+;   leaq 4(%rsp), %rdx
+;   movl $2, (%rdx)
+;   movq %r15, %rdi
+;   callq 0x72 ; reloc_external CallPCRel4 u0:0 -4
+;   leaq (%rsp), %r9
+;   movl $2, (%r9)
+;   movq %rbx, %rdi
+;   callq 0x85 ; reloc_external CallPCRel4 u0:0 -4
+;   movq %r13, %rdi
+;   callq 0x8d ; reloc_external CallPCRel4 u0:0 -4
+;   movq 0x10(%rsp), %rbx
+;   movq 0x18(%rsp), %r13
+;   movq 0x20(%rsp), %r15
+;   addq $0x30, %rsp
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %different_types(i8, i16, i32, i64, f32, f64) -> i8, i16, i32, i64, f32, f64 {
+    ss0 = explicit_slot 1
+    ss1 = explicit_slot 2, align = 2
+    ss2 = explicit_slot 8, align = 4
+    ss3 = explicit_slot 16, align = 8
+    ss4 = explicit_slot 48, align = 16
+    sig0 = () system_v
+    fn0 = colocated u0:0 sig0
+
+block0(v0: i8, v1: i16, v2: i32, v3: i64, v4: f32, v5: f64):
+    stack_store v0, ss0
+    stack_store v1, ss1
+    stack_store v2, ss2
+    stack_store v4, ss2+4
+    stack_store v3, ss3
+    stack_store v5, ss3+8
+    call fn0(), stack_map=[i8 @ ss0+0, i16 @ ss1+0, i32 @ ss2+0, f32 @ ss2+4, i64 @ ss3+0, f64 @ ss3+8]
+    return v0, v1, v2, v3, v4, v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+;   subq    %rsp, $176, %rsp
+;   movq    %rbx, 128(%rsp)
+;   movq    %r12, 136(%rsp)
+;   movq    %r13, 144(%rsp)
+;   movq    %r14, 152(%rsp)
+;   movq    %r15, 160(%rsp)
+; block0:
+;   movq    %r8, %r13
+;   lea     rsp(0 + virtual offset), %r8
+;   movb    %dil, 0(%r8)
+;   movq    %rdi, %rbx
+;   lea     rsp(8 + virtual offset), %r8
+;   movw    %si, 0(%r8)
+;   movq    %rsi, %r14
+;   lea     rsp(16 + virtual offset), %r9
+;   movl    %edx, 0(%r9)
+;   movq    %rdx, %r12
+;   lea     rsp(20 + virtual offset), %r10
+;   movss   %xmm0, 0(%r10)
+;   movdqu  %xmm0, rsp(96 + virtual offset)
+;   lea     rsp(24 + virtual offset), %r11
+;   movq    %rcx, 0(%r11)
+;   movq    %rcx, %r15
+;   lea     rsp(32 + virtual offset), %rsi
+;   movsd   %xmm1, 0(%rsi)
+;   movdqu  %xmm1, rsp(112 + virtual offset)
+;   call    User(userextname0)
+;   ; UserStackMap { by_type: [(types::I8, CompoundBitSet {0}), (types::I16, CompoundBitSet {8}), (types::I32, CompoundBitSet {16}), (types::F32, CompoundBitSet {20}), (types::I64, CompoundBitSet {24}), (types::F64, CompoundBitSet {32})] }
+;   movq    %r12, %rdx
+;   movq    %r13, %r8
+;   movl    %edx, 0(%r8)
+;   movq    %r15, %rcx
+;   movq    %rcx, 8(%r8)
+;   movq    %rbx, %rax
+;   movq    %r14, %rdx
+;   movdqu  rsp(96 + virtual offset), %xmm0
+;   movdqu  rsp(112 + virtual offset), %xmm1
+;   movq    128(%rsp), %rbx
+;   movq    136(%rsp), %r12
+;   movq    144(%rsp), %r13
+;   movq    152(%rsp), %r14
+;   movq    160(%rsp), %r15
+;   addq    %rsp, $176, %rsp
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+;   subq $0xb0, %rsp
+;   movq %rbx, 0x80(%rsp)
+;   movq %r12, 0x88(%rsp)
+;   movq %r13, 0x90(%rsp)
+;   movq %r14, 0x98(%rsp)
+;   movq %r15, 0xa0(%rsp)
+; block1: ; offset 0x33
+;   movq %r8, %r13
+;   leaq (%rsp), %r8
+;   movb %dil, (%r8)
+;   movq %rdi, %rbx
+;   leaq 8(%rsp), %r8
+;   movw %si, (%r8)
+;   movq %rsi, %r14
+;   leaq 0x10(%rsp), %r9
+;   movl %edx, (%r9)
+;   movq %rdx, %r12
+;   leaq 0x14(%rsp), %r10
+;   movss %xmm0, (%r10)
+;   movdqu %xmm0, 0x60(%rsp)
+;   leaq 0x18(%rsp), %r11
+;   movq %rcx, (%r11)
+;   movq %rcx, %r15
+;   leaq 0x20(%rsp), %rsi
+;   movsd %xmm1, (%rsi)
+;   movdqu %xmm1, 0x70(%rsp)
+;   callq 0x86 ; reloc_external CallPCRel4 u0:0 -4
+;   movq %r12, %rdx
+;   movq %r13, %r8
+;   movl %edx, (%r8)
+;   movq %r15, %rcx
+;   movq %rcx, 8(%r8)
+;   movq %rbx, %rax
+;   movq %r14, %rdx
+;   movdqu 0x60(%rsp), %xmm0
+;   movdqu 0x70(%rsp), %xmm1
+;   movq 0x80(%rsp), %rbx
+;   movq 0x88(%rsp), %r12
+;   movq 0x90(%rsp), %r13
+;   movq 0x98(%rsp), %r14
+;   movq 0xa0(%rsp), %r15
+;   addq $0xb0, %rsp
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/runtests/bitcast-f16-f128.clif b/cranelift/filetests/filetests/runtests/bitcast-f16-f128.clif
new file mode 100644
index 000000000..5448ae481
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/bitcast-f16-f128.clif
@@ -0,0 +1,61 @@
+test interpret
+
+function %bitcast_i16_f16(i16) -> f16 fast {
+block0(v0: i16):
+    v1 = bitcast.f16 v0
+    return v1
+}
+; run: %bitcast_i16_f16(0x0000) == 0x0.0
+; run: %bitcast_i16_f16(0x0001) == 0x0.004p-14
+; run: %bitcast_i16_f16(0x7c00) == Inf
+; run: %bitcast_i16_f16(0xfe00) == -NaN
+; run: %bitcast_i16_f16(0x7c01) == +sNaN:0x1
+; run: %bitcast_i16_f16(0x3c00) == 0x1.0
+; run: %bitcast_i16_f16(0x47fe) == 0x7.fe
+; run: %bitcast_i16_f16(0xf0e0) == -0x1.380p13
+; run: %bitcast_i16_f16(0xabcd) == -0x1.f34p-5
+
+function %bitcast_i128_f128(i128) -> f128 fast {
+block0(v0: i128):
+    v1 = bitcast.f128 v0
+    return v1
+}
+; run: %bitcast_i128_f128(0x00000000000000000000000000000000) == 0x0.0
+; run: %bitcast_i128_f128(0x00000000000000000000000000000001) == 0x0.0000000000000000000000000001p-16382
+; run: %bitcast_i128_f128(0x7fff0000000000000000000000000000) == Inf
+; run: %bitcast_i128_f128(0xffff8000000000000000000000000000) == -NaN
+; run: %bitcast_i128_f128(0x7fff0000000000000000000000000001) == +sNaN:0x1
+; run: %bitcast_i128_f128(0x3fff0000000000000000000000000000) == 0x1.0
+; run: %bitcast_i128_f128(0x3fff7fef123456789abcdefcda456987) == 0x1.7fef123456789abcdefcda456987
+; run: %bitcast_i128_f128(0xf0e0d0c0b0a090807060504030201000) == -0x1.d0c0b0a090807060504030201000p12513
+; run: %bitcast_i128_f128(0xabcdef01234567890123456789abcdef) == -0x1.ef01234567890123456789abcdefp-5170
+
+function %bitcast_f16_i16(f16) -> i16 fast {
+block0(v0: f16):
+    v1 = bitcast.i16 v0
+    return v1
+}
+; run: %bitcast_f16_i16(0x0.0) == 0x0000
+; run: %bitcast_f16_i16(0x0.004p-14) == 0x0001
+; run: %bitcast_f16_i16(Inf) == 0x7c00
+; run: %bitcast_f16_i16(-NaN) == 0xfe00
+; run: %bitcast_f16_i16(+sNaN:0x1) == 0x7c01
+; run: %bitcast_f16_i16(0x1.0) == 0x3c00
+; run: %bitcast_f16_i16(0x7.fe) == 0x47fe
+; run: %bitcast_f16_i16(-0x1.3c0p13) == 0xf0f0
+; run: %bitcast_f16_i16(-0x1.f34p-5) == 0xabcd
+
+function %bitcast_f128_i128(f128) -> i128 fast {
+block0(v0: f128):
+    v1 = bitcast.i128 v0
+    return v1
+}
+; run: %bitcast_f128_i128(0x0.0) == 0x00000000000000000000000000000000
+; run: %bitcast_f128_i128(0x0.0000000000000000000000000001p-16382) == 0x00000000000000000000000000000001
+; run: %bitcast_f128_i128(Inf) == 0x7fff0000000000000000000000000000
+; run: %bitcast_f128_i128(-NaN) == 0xffff8000000000000000000000000000
+; run: %bitcast_f128_i128(+sNaN:0x1) == 0x7fff0000000000000000000000000001
+; run: %bitcast_f128_i128(0x1.0) == 0x3fff0000000000000000000000000000
+; run: %bitcast_f128_i128(0x1.7fef123456789abcdefcda456987) == 0x3fff7fef123456789abcdefcda456987
+; run: %bitcast_f128_i128(-0x1.d0c0b0a090807060504030201000p12513) == 0xf0e0d0c0b0a090807060504030201000
+; run: %bitcast_f128_i128(-0x1.ef01234567890123456789abcdefp-5170) == 0xabcdef01234567890123456789abcdef
diff --git a/tests/all/module.rs b/tests/all/module.rs
index eb658f69f..94a56acf5 100644
--- a/tests/all/module.rs
+++ b/tests/all/module.rs
@@ -251,53 +251,6 @@ fn compile_a_component() -> Result<()> {
     Ok(())
 }
 
-#[test]
-fn call_indirect_caching_and_memory64() -> Result<()> {
-    let mut config = Config::new();
-    config.wasm_memory64(true);
-    config.cache_call_indirects(true);
-    let engine = Engine::new(&config)?;
-    Module::new(
-        &engine,
-        "(module
-            (memory i64 1)
-            (func (param i64) (result i32)
-                local.get 0
-                i32.load offset=0x100000000
-            )
-        )",
-    )?;
-    Ok(())
-}
-
-#[test]
-fn call_indirect_caching_out_of_bounds_table_index() -> Result<()> {
-    let mut config = Config::new();
-    config.cache_call_indirects(true);
-    let engine = Engine::new(&config)?;
-    // Test an out-of-bounds table index: this is exposed to the prescan
-    // that call-indirect caching must perform during compilation, so we
-    // need to make sure the error is properly handled by the validation
-    // that comes later.
-    let err = Module::new(
-        &engine,
-        "(module
-            (func (param i32)
-                ref.null func
-                local.get 0
-                table.set 32  ;; out-of-bounds table index
-            )
-        )",
-    )
-    .unwrap_err();
-    let err = format!("{err:?}");
-    assert!(
-        err.contains("table index out of bounds"),
-        "bad error: {err}"
-    );
-    Ok(())
-}
-
 #[test]
 fn tail_call_defaults() -> Result<()> {
     let wasm_with_tail_calls = "(module (func $a return_call $a))";
diff --git a/tests/disas/indirect-call-caching-exclude-0-index.wat b/tests/disas/indirect-call-caching-exclude-0-index.wat
deleted file mode 100644
index c1446fb23..000000000
--- a/tests/disas/indirect-call-caching-exclude-0-index.wat
+++ /dev/null
@@ -1,107 +0,0 @@
-;;! target = "x86_64"
-;;! flags = [ "-Ocache-call-indirects=y" ]
-
-;; This test checks that we do *not* get the indirect-call caching optimization
-;; when it must not be used: in this case, because there is a non-null element
-;; at index 0 in the table (`0` is the default value for vmctx struct initialization
-;; and so is our "not yet cached" sentinel).
-
-(module
- (table 10 10 funcref)
-
- (func $f1 (result i32) i32.const 1)
- (func $f2 (result i32) i32.const 2)
- (func $f3 (result i32) i32.const 3)
-
- (func (export "call_it") (param i32) (result i32)
-  local.get 0
-  call_indirect (result i32))
-
- (elem (i32.const 0) func $f1 $f2 $f3))
-;; function u0:0(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @003f                               v3 = iconst.i32 1
-;; @0041                               jump block1(v3)  ; v3 = 1
-;;
-;;                                 block1(v2: i32):
-;; @0041                               return v2
-;; }
-;;
-;; function u0:1(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0044                               v3 = iconst.i32 2
-;; @0046                               jump block1(v3)  ; v3 = 2
-;;
-;;                                 block1(v2: i32):
-;; @0046                               return v2
-;; }
-;;
-;; function u0:2(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0049                               v3 = iconst.i32 3
-;; @004b                               jump block1(v3)  ; v3 = 3
-;;
-;;                                 block1(v2: i32):
-;; @004b                               return v2
-;; }
-;;
-;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) -> i32 tail
-;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
-;;     fn0 = colocated u1:9 sig1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64, v2: i32):
-;; @0050                               v4 = iconst.i32 10
-;; @0050                               v5 = icmp uge v2, v4  ; v4 = 10
-;; @0050                               v6 = uextend.i64 v2
-;; @0050                               v7 = global_value.i64 gv4
-;; @0050                               v8 = ishl_imm v6, 3
-;; @0050                               v9 = iadd v7, v8
-;; @0050                               v10 = iconst.i64 0
-;; @0050                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
-;; @0050                               v12 = load.i64 table_oob aligned table v11
-;; @0050                               v13 = band_imm v12, -2
-;; @0050                               brif v12, block3(v13), block2
-;;
-;;                                 block2 cold:
-;; @0050                               v15 = iconst.i32 0
-;; @0050                               v16 = global_value.i64 gv3
-;; @0050                               v17 = call fn0(v16, v15, v2)  ; v15 = 0
-;; @0050                               jump block3(v17)
-;;
-;;                                 block3(v14: i64):
-;; @0050                               v18 = global_value.i64 gv3
-;; @0050                               v19 = load.i64 notrap aligned readonly v18+80
-;; @0050                               v20 = load.i32 notrap aligned readonly v19
-;; @0050                               v21 = load.i32 icall_null aligned readonly v14+16
-;; @0050                               v22 = icmp eq v21, v20
-;; @0050                               trapz v22, bad_sig
-;; @0050                               v23 = load.i64 notrap aligned readonly v14+8
-;; @0050                               v24 = load.i64 notrap aligned readonly v14+24
-;; @0050                               v25 = call_indirect sig0, v23(v24, v0)
-;; @0053                               jump block1(v25)
-;;
-;;                                 block1(v3: i32):
-;; @0053                               return v3
-;; }
diff --git a/tests/disas/indirect-call-caching-exclude-table-export.wat b/tests/disas/indirect-call-caching-exclude-table-export.wat
deleted file mode 100644
index 1e7ee89f5..000000000
--- a/tests/disas/indirect-call-caching-exclude-table-export.wat
+++ /dev/null
@@ -1,106 +0,0 @@
-;;! target = "x86_64"
-;;! flags = [ "-Ocache-call-indirects=y" ]
-
-;; This test checks that we do *not* get the indirect-call caching optimization
-;; when it must not be used: in this case, because the table is exported so
-;; could be mutated (invalidating the cache, which we would not detect).
-
-(module
- (table (export "t") 10 10 funcref)
-
- (func $f1 (result i32) i32.const 1)
- (func $f2 (result i32) i32.const 2)
- (func $f3 (result i32) i32.const 3)
-
- (func (export "call_it") (param i32) (result i32)
-  local.get 0
-  call_indirect (result i32))
-
- (elem (i32.const 1) func $f1 $f2 $f3))
-;; function u0:0(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0043                               v3 = iconst.i32 1
-;; @0045                               jump block1(v3)  ; v3 = 1
-;;
-;;                                 block1(v2: i32):
-;; @0045                               return v2
-;; }
-;;
-;; function u0:1(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0048                               v3 = iconst.i32 2
-;; @004a                               jump block1(v3)  ; v3 = 2
-;;
-;;                                 block1(v2: i32):
-;; @004a                               return v2
-;; }
-;;
-;; function u0:2(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @004d                               v3 = iconst.i32 3
-;; @004f                               jump block1(v3)  ; v3 = 3
-;;
-;;                                 block1(v2: i32):
-;; @004f                               return v2
-;; }
-;;
-;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) -> i32 tail
-;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
-;;     fn0 = colocated u1:9 sig1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64, v2: i32):
-;; @0054                               v4 = iconst.i32 10
-;; @0054                               v5 = icmp uge v2, v4  ; v4 = 10
-;; @0054                               v6 = uextend.i64 v2
-;; @0054                               v7 = global_value.i64 gv4
-;; @0054                               v8 = ishl_imm v6, 3
-;; @0054                               v9 = iadd v7, v8
-;; @0054                               v10 = iconst.i64 0
-;; @0054                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
-;; @0054                               v12 = load.i64 table_oob aligned table v11
-;; @0054                               v13 = band_imm v12, -2
-;; @0054                               brif v12, block3(v13), block2
-;;
-;;                                 block2 cold:
-;; @0054                               v15 = iconst.i32 0
-;; @0054                               v16 = global_value.i64 gv3
-;; @0054                               v17 = call fn0(v16, v15, v2)  ; v15 = 0
-;; @0054                               jump block3(v17)
-;;
-;;                                 block3(v14: i64):
-;; @0054                               v18 = global_value.i64 gv3
-;; @0054                               v19 = load.i64 notrap aligned readonly v18+80
-;; @0054                               v20 = load.i32 notrap aligned readonly v19
-;; @0054                               v21 = load.i32 icall_null aligned readonly v14+16
-;; @0054                               v22 = icmp eq v21, v20
-;; @0054                               trapz v22, bad_sig
-;; @0054                               v23 = load.i64 notrap aligned readonly v14+8
-;; @0054                               v24 = load.i64 notrap aligned readonly v14+24
-;; @0054                               v25 = call_indirect sig0, v23(v24, v0)
-;; @0057                               jump block1(v25)
-;;
-;;                                 block1(v3: i32):
-;; @0057                               return v3
-;; }
diff --git a/tests/disas/indirect-call-caching-exclude-table-writes.wat b/tests/disas/indirect-call-caching-exclude-table-writes.wat
deleted file mode 100644
index 387da41b7..000000000
--- a/tests/disas/indirect-call-caching-exclude-table-writes.wat
+++ /dev/null
@@ -1,138 +0,0 @@
-;;! target = "x86_64"
-;;! flags = [ "-Ocache-call-indirects=y" ]
-
-;; This test checks that we do *not* get the indirect-call caching optimization
-;; when it must not be used: in this case, because the table is updated with a
-;; `table.set` instruction (invalidating the cache, which we would not detect).
-
-(module
- (table 10 10 funcref)
-
- (func $f1 (result i32) i32.const 1)
- (func $f2 (result i32) i32.const 2)
- (func $f3 (result i32) i32.const 3)
-
- (func (export "call_it") (param i32) (result i32)
-  local.get 0
-  call_indirect (result i32))
-
- (func (export "update_table")
-  i32.const 1
-  ref.null func
-  table.set)
-
- (elem (i32.const 1) func $f1 $f2 $f3))
-;; function u0:0(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0052                               v3 = iconst.i32 1
-;; @0054                               jump block1(v3)  ; v3 = 1
-;;
-;;                                 block1(v2: i32):
-;; @0054                               return v2
-;; }
-;;
-;; function u0:1(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0057                               v3 = iconst.i32 2
-;; @0059                               jump block1(v3)  ; v3 = 2
-;;
-;;                                 block1(v2: i32):
-;; @0059                               return v2
-;; }
-;;
-;; function u0:2(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @005c                               v3 = iconst.i32 3
-;; @005e                               jump block1(v3)  ; v3 = 3
-;;
-;;                                 block1(v2: i32):
-;; @005e                               return v2
-;; }
-;;
-;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) -> i32 tail
-;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
-;;     fn0 = colocated u1:9 sig1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64, v2: i32):
-;; @0063                               v4 = iconst.i32 10
-;; @0063                               v5 = icmp uge v2, v4  ; v4 = 10
-;; @0063                               v6 = uextend.i64 v2
-;; @0063                               v7 = global_value.i64 gv4
-;; @0063                               v8 = ishl_imm v6, 3
-;; @0063                               v9 = iadd v7, v8
-;; @0063                               v10 = iconst.i64 0
-;; @0063                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
-;; @0063                               v12 = load.i64 table_oob aligned table v11
-;; @0063                               v13 = band_imm v12, -2
-;; @0063                               brif v12, block3(v13), block2
-;;
-;;                                 block2 cold:
-;; @0063                               v15 = iconst.i32 0
-;; @0063                               v16 = global_value.i64 gv3
-;; @0063                               v17 = call fn0(v16, v15, v2)  ; v15 = 0
-;; @0063                               jump block3(v17)
-;;
-;;                                 block3(v14: i64):
-;; @0063                               v18 = global_value.i64 gv3
-;; @0063                               v19 = load.i64 notrap aligned readonly v18+80
-;; @0063                               v20 = load.i32 notrap aligned readonly v19
-;; @0063                               v21 = load.i32 icall_null aligned readonly v14+16
-;; @0063                               v22 = icmp eq v21, v20
-;; @0063                               trapz v22, bad_sig
-;; @0063                               v23 = load.i64 notrap aligned readonly v14+8
-;; @0063                               v24 = load.i64 notrap aligned readonly v14+24
-;; @0063                               v25 = call_indirect sig0, v23(v24, v0)
-;; @0066                               jump block1(v25)
-;;
-;;                                 block1(v3: i32):
-;; @0066                               return v3
-;; }
-;;
-;; function u0:4(i64 vmctx, i64) tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0069                               v2 = iconst.i32 1
-;; @006b                               v3 = iconst.i64 0
-;; @006d                               v4 = iconst.i32 10
-;; @006d                               v5 = icmp uge v2, v4  ; v2 = 1, v4 = 10
-;; @006d                               v6 = uextend.i64 v2  ; v2 = 1
-;; @006d                               v7 = global_value.i64 gv4
-;; @006d                               v8 = ishl_imm v6, 3
-;; @006d                               v9 = iadd v7, v8
-;; @006d                               v10 = iconst.i64 0
-;; @006d                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
-;; @006d                               v12 = bor_imm v3, 1  ; v3 = 0
-;; @006d                               store table_oob aligned table v12, v11
-;; @006f                               jump block1
-;;
-;;                                 block1:
-;; @006f                               return
-;; }
diff --git a/tests/disas/indirect-call-caching-slot-limit-1.wat b/tests/disas/indirect-call-caching-slot-limit-1.wat
deleted file mode 100644
index cedf4aaf4..000000000
--- a/tests/disas/indirect-call-caching-slot-limit-1.wat
+++ /dev/null
@@ -1,157 +0,0 @@
-;;! target = "x86_64"
-;;! flags = [ "-Ocache-call-indirects=y", "-Omax-call-indirect-cache-slots=2" ]
-
-;; This test checks that we properly bound the number of call-indirect
-;; cache slots. The first case (here) is when the limit falls in the
-;; middle of a function. We set the limit to 2 above; we have 3
-;; `call_indirect`s below; the last should not have caching code.
-;;
-;; In particular, below we see the cache probe sequence in block0
-;; (first) and block3 (second); but the third call, starting in
-;; block8, has no cache slot access and just performs the checks
-;; unconditionally.
-
-(module
- (table 10 10 funcref)
-
- (func (export "call_it") (param i32) (result i32)
-  local.get 0
-  call_indirect (result i32)
-  call_indirect (result i32)
-  call_indirect (result i32)))
-  
-;; function u0:0(i64 vmctx, i64, i32) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) -> i32 tail
-;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
-;;     fn0 = colocated u1:9 sig1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64, v2: i32):
-;; @0033                               v4 = global_value.i64 gv3
-;; @0033                               v5 = iadd_imm v4, 144
-;; @0033                               v6 = load.i32 notrap aligned v5+8
-;; @0033                               v7 = load.i64 notrap aligned v5
-;; @0033                               v8 = icmp eq v6, v2
-;; @0033                               brif v8, block3(v7, v4), block2
-;;
-;;                                 block2 cold:
-;; @0033                               v9 = iconst.i32 10
-;; @0033                               v10 = icmp.i32 uge v2, v9  ; v9 = 10
-;; @0033                               v11 = uextend.i64 v2
-;; @0033                               v12 = global_value.i64 gv4
-;; @0033                               v13 = ishl_imm v11, 3
-;; @0033                               v14 = iadd v12, v13
-;; @0033                               v15 = iconst.i64 0
-;; @0033                               v16 = select_spectre_guard v10, v15, v14  ; v15 = 0
-;; @0033                               v17 = load.i64 table_oob aligned table v16
-;; @0033                               v18 = band_imm v17, -2
-;; @0033                               brif v17, block6(v18), block5
-;;
-;;                                 block4 cold:
-;; @0033                               store.i32 notrap aligned v2, v5+8
-;; @0033                               store.i64 notrap aligned v28, v5
-;; @0033                               jump block3(v28, v29)
-;;
-;;                                 block3(v31: i64, v32: i64):
-;; @0033                               v33 = call_indirect sig0, v31(v32, v0)
-;; @0036                               v34 = global_value.i64 gv3
-;; @0036                               v35 = iadd_imm v34, 160
-;; @0036                               v36 = load.i32 notrap aligned v35+8
-;; @0036                               v37 = load.i64 notrap aligned v35
-;; @0036                               v38 = icmp eq v36, v33
-;; @0036                               brif v38, block8(v37, v34), block7
-;;
-;;                                 block7 cold:
-;; @0036                               v39 = iconst.i32 10
-;; @0036                               v40 = icmp.i32 uge v33, v39  ; v39 = 10
-;; @0036                               v41 = uextend.i64 v33
-;; @0036                               v42 = global_value.i64 gv4
-;; @0036                               v43 = ishl_imm v41, 3
-;; @0036                               v44 = iadd v42, v43
-;; @0036                               v45 = iconst.i64 0
-;; @0036                               v46 = select_spectre_guard v40, v45, v44  ; v45 = 0
-;; @0036                               v47 = load.i64 table_oob aligned table v46
-;; @0036                               v48 = band_imm v47, -2
-;; @0036                               brif v47, block11(v48), block10
-;;
-;;                                 block9 cold:
-;; @0036                               store.i32 notrap aligned v33, v35+8
-;; @0036                               store.i64 notrap aligned v58, v35
-;; @0036                               jump block8(v58, v59)
-;;
-;;                                 block8(v61: i64, v62: i64):
-;; @0036                               v63 = call_indirect sig0, v61(v62, v0)
-;; @0039                               v64 = iconst.i32 10
-;; @0039                               v65 = icmp uge v63, v64  ; v64 = 10
-;; @0039                               v66 = uextend.i64 v63
-;; @0039                               v67 = global_value.i64 gv4
-;; @0039                               v68 = ishl_imm v66, 3
-;; @0039                               v69 = iadd v67, v68
-;; @0039                               v70 = iconst.i64 0
-;; @0039                               v71 = select_spectre_guard v65, v70, v69  ; v70 = 0
-;; @0039                               v72 = load.i64 table_oob aligned table v71
-;; @0039                               v73 = band_imm v72, -2
-;; @0039                               brif v72, block13(v73), block12
-;;
-;;                                 block5 cold:
-;; @0033                               v20 = iconst.i32 0
-;; @0033                               v21 = global_value.i64 gv3
-;; @0033                               v22 = call fn0(v21, v20, v2)  ; v20 = 0
-;; @0033                               jump block6(v22)
-;;
-;;                                 block6(v19: i64) cold:
-;; @0033                               v23 = global_value.i64 gv3
-;; @0033                               v24 = load.i64 notrap aligned readonly v23+80
-;; @0033                               v25 = load.i32 notrap aligned readonly v24+4
-;; @0033                               v26 = load.i32 icall_null aligned readonly v19+16
-;; @0033                               v27 = icmp eq v26, v25
-;; @0033                               trapz v27, bad_sig
-;; @0033                               v28 = load.i64 notrap aligned readonly v19+8
-;; @0033                               v29 = load.i64 notrap aligned readonly v19+24
-;; @0033                               v30 = icmp eq v29, v4
-;; @0033                               brif v30, block4, block3(v28, v29)
-;;
-;;                                 block10 cold:
-;; @0036                               v50 = iconst.i32 0
-;; @0036                               v51 = global_value.i64 gv3
-;; @0036                               v52 = call fn0(v51, v50, v33)  ; v50 = 0
-;; @0036                               jump block11(v52)
-;;
-;;                                 block11(v49: i64) cold:
-;; @0036                               v53 = global_value.i64 gv3
-;; @0036                               v54 = load.i64 notrap aligned readonly v53+80
-;; @0036                               v55 = load.i32 notrap aligned readonly v54+4
-;; @0036                               v56 = load.i32 icall_null aligned readonly v49+16
-;; @0036                               v57 = icmp eq v56, v55
-;; @0036                               trapz v57, bad_sig
-;; @0036                               v58 = load.i64 notrap aligned readonly v49+8
-;; @0036                               v59 = load.i64 notrap aligned readonly v49+24
-;; @0036                               v60 = icmp eq v59, v34
-;; @0036                               brif v60, block9, block8(v58, v59)
-;;
-;;                                 block12 cold:
-;; @0039                               v75 = iconst.i32 0
-;; @0039                               v76 = global_value.i64 gv3
-;; @0039                               v77 = call fn0(v76, v75, v63)  ; v75 = 0
-;; @0039                               jump block13(v77)
-;;
-;;                                 block13(v74: i64):
-;; @0039                               v78 = global_value.i64 gv3
-;; @0039                               v79 = load.i64 notrap aligned readonly v78+80
-;; @0039                               v80 = load.i32 notrap aligned readonly v79+4
-;; @0039                               v81 = load.i32 icall_null aligned readonly v74+16
-;; @0039                               v82 = icmp eq v81, v80
-;; @0039                               trapz v82, bad_sig
-;; @0039                               v83 = load.i64 notrap aligned readonly v74+8
-;; @0039                               v84 = load.i64 notrap aligned readonly v74+24
-;; @0039                               v85 = call_indirect sig0, v83(v84, v0)
-;; @003c                               jump block1(v85)
-;;
-;;                                 block1(v3: i32):
-;; @003c                               return v3
-;; }
diff --git a/tests/disas/indirect-call-caching-slot-limit-2.wat b/tests/disas/indirect-call-caching-slot-limit-2.wat
deleted file mode 100644
index 719d5e68b..000000000
--- a/tests/disas/indirect-call-caching-slot-limit-2.wat
+++ /dev/null
@@ -1,180 +0,0 @@
-;;! target = "x86_64"
-;;! flags = [ "-Ocache-call-indirects=y", "-Omax-call-indirect-cache-slots=2" ]
-
-;; This test checks that we properly bound the number of call-indirect
-;; cache slots. The second case (here) is when the limit falls
-;; entirely before a function. We set the limit to 2 above; we have 2
-;; callsites in the first function; the second function should have no
-;; caching.
-;;
-;; In particular, below we see the cache probe sequence in block0
-;; (first) and block3 (second) in `u0:0` (`call_it`); but the call in
-;; the second function in `u0:1` (`call_it_2`), starting in block0 in
-;; that function, has no cache slot access and just performs the
-;; checks unconditionally.
-
-(module
- (table 10 10 funcref)
-
- (func (export "call_it") (param i32) (result i32)
-  local.get 0
-  call_indirect (result i32)
-  call_indirect (result i32))
- 
- (func (export "call_it_2") (param i32) (result i32)
-  local.get 0
-  call_indirect (result i32)))
-  
-;; function u0:0(i64 vmctx, i64, i32) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) -> i32 tail
-;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
-;;     fn0 = colocated u1:9 sig1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64, v2: i32):
-;; @0040                               v4 = global_value.i64 gv3
-;; @0040                               v5 = iadd_imm v4, 176
-;; @0040                               v6 = load.i32 notrap aligned v5+8
-;; @0040                               v7 = load.i64 notrap aligned v5
-;; @0040                               v8 = icmp eq v6, v2
-;; @0040                               brif v8, block3(v7, v4), block2
-;;
-;;                                 block2 cold:
-;; @0040                               v9 = iconst.i32 10
-;; @0040                               v10 = icmp.i32 uge v2, v9  ; v9 = 10
-;; @0040                               v11 = uextend.i64 v2
-;; @0040                               v12 = global_value.i64 gv4
-;; @0040                               v13 = ishl_imm v11, 3
-;; @0040                               v14 = iadd v12, v13
-;; @0040                               v15 = iconst.i64 0
-;; @0040                               v16 = select_spectre_guard v10, v15, v14  ; v15 = 0
-;; @0040                               v17 = load.i64 table_oob aligned table v16
-;; @0040                               v18 = band_imm v17, -2
-;; @0040                               brif v17, block6(v18), block5
-;;
-;;                                 block4 cold:
-;; @0040                               store.i32 notrap aligned v2, v5+8
-;; @0040                               store.i64 notrap aligned v28, v5
-;; @0040                               jump block3(v28, v29)
-;;
-;;                                 block3(v31: i64, v32: i64):
-;; @0040                               v33 = call_indirect sig0, v31(v32, v0)
-;; @0043                               v34 = global_value.i64 gv3
-;; @0043                               v35 = iadd_imm v34, 192
-;; @0043                               v36 = load.i32 notrap aligned v35+8
-;; @0043                               v37 = load.i64 notrap aligned v35
-;; @0043                               v38 = icmp eq v36, v33
-;; @0043                               brif v38, block8(v37, v34), block7
-;;
-;;                                 block7 cold:
-;; @0043                               v39 = iconst.i32 10
-;; @0043                               v40 = icmp.i32 uge v33, v39  ; v39 = 10
-;; @0043                               v41 = uextend.i64 v33
-;; @0043                               v42 = global_value.i64 gv4
-;; @0043                               v43 = ishl_imm v41, 3
-;; @0043                               v44 = iadd v42, v43
-;; @0043                               v45 = iconst.i64 0
-;; @0043                               v46 = select_spectre_guard v40, v45, v44  ; v45 = 0
-;; @0043                               v47 = load.i64 table_oob aligned table v46
-;; @0043                               v48 = band_imm v47, -2
-;; @0043                               brif v47, block11(v48), block10
-;;
-;;                                 block9 cold:
-;; @0043                               store.i32 notrap aligned v33, v35+8
-;; @0043                               store.i64 notrap aligned v58, v35
-;; @0043                               jump block8(v58, v59)
-;;
-;;                                 block8(v61: i64, v62: i64):
-;; @0043                               v63 = call_indirect sig0, v61(v62, v0)
-;; @0046                               jump block1(v63)
-;;
-;;                                 block5 cold:
-;; @0040                               v20 = iconst.i32 0
-;; @0040                               v21 = global_value.i64 gv3
-;; @0040                               v22 = call fn0(v21, v20, v2)  ; v20 = 0
-;; @0040                               jump block6(v22)
-;;
-;;                                 block6(v19: i64) cold:
-;; @0040                               v23 = global_value.i64 gv3
-;; @0040                               v24 = load.i64 notrap aligned readonly v23+80
-;; @0040                               v25 = load.i32 notrap aligned readonly v24+4
-;; @0040                               v26 = load.i32 icall_null aligned readonly v19+16
-;; @0040                               v27 = icmp eq v26, v25
-;; @0040                               trapz v27, bad_sig
-;; @0040                               v28 = load.i64 notrap aligned readonly v19+8
-;; @0040                               v29 = load.i64 notrap aligned readonly v19+24
-;; @0040                               v30 = icmp eq v29, v4
-;; @0040                               brif v30, block4, block3(v28, v29)
-;;
-;;                                 block10 cold:
-;; @0043                               v50 = iconst.i32 0
-;; @0043                               v51 = global_value.i64 gv3
-;; @0043                               v52 = call fn0(v51, v50, v33)  ; v50 = 0
-;; @0043                               jump block11(v52)
-;;
-;;                                 block11(v49: i64) cold:
-;; @0043                               v53 = global_value.i64 gv3
-;; @0043                               v54 = load.i64 notrap aligned readonly v53+80
-;; @0043                               v55 = load.i32 notrap aligned readonly v54+4
-;; @0043                               v56 = load.i32 icall_null aligned readonly v49+16
-;; @0043                               v57 = icmp eq v56, v55
-;; @0043                               trapz v57, bad_sig
-;; @0043                               v58 = load.i64 notrap aligned readonly v49+8
-;; @0043                               v59 = load.i64 notrap aligned readonly v49+24
-;; @0043                               v60 = icmp eq v59, v34
-;; @0043                               brif v60, block9, block8(v58, v59)
-;;
-;;                                 block1(v3: i32):
-;; @0046                               return v3
-;; }
-;;
-;; function u0:1(i64 vmctx, i64, i32) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) -> i32 tail
-;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
-;;     fn0 = colocated u1:9 sig1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64, v2: i32):
-;; @004b                               v4 = iconst.i32 10
-;; @004b                               v5 = icmp uge v2, v4  ; v4 = 10
-;; @004b                               v6 = uextend.i64 v2
-;; @004b                               v7 = global_value.i64 gv4
-;; @004b                               v8 = ishl_imm v6, 3
-;; @004b                               v9 = iadd v7, v8
-;; @004b                               v10 = iconst.i64 0
-;; @004b                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
-;; @004b                               v12 = load.i64 table_oob aligned table v11
-;; @004b                               v13 = band_imm v12, -2
-;; @004b                               brif v12, block3(v13), block2
-;;
-;;                                 block2 cold:
-;; @004b                               v15 = iconst.i32 0
-;; @004b                               v16 = global_value.i64 gv3
-;; @004b                               v17 = call fn0(v16, v15, v2)  ; v15 = 0
-;; @004b                               jump block3(v17)
-;;
-;;                                 block3(v14: i64):
-;; @004b                               v18 = global_value.i64 gv3
-;; @004b                               v19 = load.i64 notrap aligned readonly v18+80
-;; @004b                               v20 = load.i32 notrap aligned readonly v19+4
-;; @004b                               v21 = load.i32 icall_null aligned readonly v14+16
-;; @004b                               v22 = icmp eq v21, v20
-;; @004b                               trapz v22, bad_sig
-;; @004b                               v23 = load.i64 notrap aligned readonly v14+8
-;; @004b                               v24 = load.i64 notrap aligned readonly v14+24
-;; @004b                               v25 = call_indirect sig0, v23(v24, v0)
-;; @004e                               jump block1(v25)
-;;
-;;                                 block1(v3: i32):
-;; @004e                               return v3
-;; }
diff --git a/tests/disas/indirect-call-caching.wat b/tests/disas/indirect-call-caching.wat
deleted file mode 100644
index a32015292..000000000
--- a/tests/disas/indirect-call-caching.wat
+++ /dev/null
@@ -1,125 +0,0 @@
-;;! target = "x86_64"
-;;! flags = [ "-Ocache-call-indirects=y" ]
-
-;; This test checks that we get the indirect-call caching optimization
-;; where it should be applicable (immutable table, null 0-index).
-;;
-;; The key bit in the expectation below is the cached-index load (v6),
-;; compare (v7), branch, fastpath in block2/block4.
-
-(module
- (table 10 10 funcref)
-
- (func $f1 (result i32) i32.const 1)
- (func $f2 (result i32) i32.const 2)
- (func $f3 (result i32) i32.const 3)
-
- (func (export "call_it") (param i32) (result i32)
-  local.get 0
-  call_indirect (result i32))
-
- (elem (i32.const 1) func $f1 $f2 $f3))
-;; function u0:0(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @003f                               v3 = iconst.i32 1
-;; @0041                               jump block1(v3)  ; v3 = 1
-;;
-;;                                 block1(v2: i32):
-;; @0041                               return v2
-;; }
-;;
-;; function u0:1(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0044                               v3 = iconst.i32 2
-;; @0046                               jump block1(v3)  ; v3 = 2
-;;
-;;                                 block1(v2: i32):
-;; @0046                               return v2
-;; }
-;;
-;; function u0:2(i64 vmctx, i64) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64):
-;; @0049                               v3 = iconst.i32 3
-;; @004b                               jump block1(v3)  ; v3 = 3
-;;
-;;                                 block1(v2: i32):
-;; @004b                               return v2
-;; }
-;;
-;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
-;;     gv2 = load.i64 notrap aligned gv1
-;;     gv3 = vmctx
-;;     gv4 = load.i64 notrap aligned readonly gv3+88
-;;     sig0 = (i64 vmctx, i64) -> i32 tail
-;;     sig1 = (i64 vmctx, i32 uext, i32 uext) -> i64 system_v
-;;     fn0 = colocated u1:9 sig1
-;;     stack_limit = gv2
-;;
-;;                                 block0(v0: i64, v1: i64, v2: i32):
-;; @0050                               v4 = global_value.i64 gv3
-;; @0050                               v5 = iadd_imm v4, 240
-;; @0050                               v6 = load.i32 notrap aligned v5+8
-;; @0050                               v7 = load.i64 notrap aligned v5
-;; @0050                               v8 = icmp eq v6, v2
-;; @0050                               brif v8, block3(v7, v4), block2
-;;
-;;                                 block2 cold:
-;; @0050                               v9 = iconst.i32 10
-;; @0050                               v10 = icmp.i32 uge v2, v9  ; v9 = 10
-;; @0050                               v11 = uextend.i64 v2
-;; @0050                               v12 = global_value.i64 gv4
-;; @0050                               v13 = ishl_imm v11, 3
-;; @0050                               v14 = iadd v12, v13
-;; @0050                               v15 = iconst.i64 0
-;; @0050                               v16 = select_spectre_guard v10, v15, v14  ; v15 = 0
-;; @0050                               v17 = load.i64 table_oob aligned table v16
-;; @0050                               v18 = band_imm v17, -2
-;; @0050                               brif v17, block6(v18), block5
-;;
-;;                                 block4 cold:
-;; @0050                               store.i32 notrap aligned v2, v5+8
-;; @0050                               store.i64 notrap aligned v28, v5
-;; @0050                               jump block3(v28, v29)
-;;
-;;                                 block3(v31: i64, v32: i64):
-;; @0050                               v33 = call_indirect sig0, v31(v32, v0)
-;; @0053                               jump block1(v33)
-;;
-;;                                 block5 cold:
-;; @0050                               v20 = iconst.i32 0
-;; @0050                               v21 = global_value.i64 gv3
-;; @0050                               v22 = call fn0(v21, v20, v2)  ; v20 = 0
-;; @0050                               jump block6(v22)
-;;
-;;                                 block6(v19: i64) cold:
-;; @0050                               v23 = global_value.i64 gv3
-;; @0050                               v24 = load.i64 notrap aligned readonly v23+80
-;; @0050                               v25 = load.i32 notrap aligned readonly v24
-;; @0050                               v26 = load.i32 icall_null aligned readonly v19+16
-;; @0050                               v27 = icmp eq v26, v25
-;; @0050                               trapz v27, bad_sig
-;; @0050                               v28 = load.i64 notrap aligned readonly v19+8
-;; @0050                               v29 = load.i64 notrap aligned readonly v19+24
-;; @0050                               v30 = icmp eq v29, v4
-;; @0050                               brif v30, block4, block3(v28, v29)
-;;
-;;                                 block1(v3: i32):
-;; @0053                               return v3
-;; }