Repository: bytecodealliance/wasmtime
Pull request #5342 (2022-11-29 16:26:06)

Move precompiled module detection into wasmtime
Previously, wasmtime-cli checked the module to be loaded is
precompiled or not, by pre-opening the given file path to
check if the "\x7FELF" header exists.
This commit moves this branch into the `Module::from_trusted_file`,
which is only invoked with `--allow-precompiled` flag on CLI.

The initial motivation of the commit is, feeding a module to wasmtime
from piped inputs, is blocked by the pre-opening of the module.
The `Module::from_trusted_file`, assumes the --allow-precompiled flag
so there is no piped inputs, happily mmap-ing the module to test
if the header exists.
If --allow-precompiled is not supplied, the existing `Module::from_file`
will be used, without the additional header check as the precompiled
modules are intentionally not allowed on piped inputs for security measures.

One caveat of this approach is that the user may be confused if
he or she tries to execute a precompiled module without
--allow-precompiled, as wasmtime shows an 'input bytes aren't valid
utf-8' error, not directly getting what's going wrong.
So this commit includes a hack-ish workaround for this: If the error
on `Module::new` ends with "input bytes aren't valid utf-8" strings,
show a simple note to the standard error stream.

This might be a small hiccup on preparing i18n or changing error
format on the `wat` crate, but I feel comfortable (yet) this strategy
because the `wat` crate includes tests for the error messages, so one
would notice the breakage if the error message have changed.

Thanks to @jameysharp for suggesting this idea with a detailed guidance.

Note: this supercedes #5035.

github-actions[bot]: #### Subscribe to Label Action

cc @peterhuene

<details>
This issue or pull request has been labeled: "wasmtime:api"

Thus the following users have been cc'd because of the following labels:

* peterhuene: wasmtime:api

To subscribe or unsubscribe from this label, edit the <code>.github/subscribe-to-label.json</code> configuration file.

[Learn more.](https://github.com/bytecodealliance/subscribe-to-label-action)
</details>

Patch:
diff --git a/cranelift/codegen/meta/src/shared/settings.rs b/cranelift/codegen/meta/src/shared/settings.rs
index d006d3881..b9fbef55c 100644
--- a/cranelift/codegen/meta/src/shared/settings.rs
+++ b/cranelift/codegen/meta/src/shared/settings.rs
@@ -273,7 +273,7 @@ pub(crate) fn define() -> SettingGroup {
         "enable_probestack",
         "Enable the use of stack probes for supported calling conventions.",
         "",
-        true,
+        false,
     );
 
     settings.add_bool(
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
index be6d2d978..d57a804ef 100644
--- a/cranelift/codegen/src/isa/aarch64/abi.rs
+++ b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -631,14 +631,56 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
-    fn gen_probestack(_: u32) -> SmallInstVec<Self::I> {
+    fn gen_probestack(_insts: &mut SmallInstVec<Self::I>, _: u32) {
         // TODO: implement if we ever require stack probes on an AArch64 host
         // (unlikely unless Lucet is ported)
-        smallvec![]
+        unimplemented!("Stack probing is unimplemented on AArch64");
     }
 
-    fn gen_inline_probestack(_frame_size: u32, _guard_size: u32) -> SmallInstVec<Self::I> {
-        unimplemented!("Inline stack probing is unimplemented on AArch64");
+    fn gen_inline_probestack(insts: &mut SmallInstVec<Self::I>, frame_size: u32, guard_size: u32) {
+        // The stack probe loop currently takes 6 instructions and each inline
+        // probe takes 2 (ish, these numbers sort of depend on the constants).
+        // Set this to 3 to keep the max size of the probe to 6 instructions.
+        const PROBE_MAX_UNROLL: u32 = 3;
+
+        let probe_count = align_to(frame_size, guard_size) / guard_size;
+        if probe_count <= PROBE_MAX_UNROLL {
+            // When manually unrolling stick an instruction that stores 0 at a
+            // constant offset relative to the stack pointer. This will
+            // turn into something like `movn tmp, #n ; stur xzr [sp, tmp]`.
+            //
+            // Note that this may actually store beyond the stack size for the
+            // last item but that's ok since it's unused stack space and if
+            // that faults accidentally we're so close to faulting it shouldn't
+            // make too much difference to fault there.
+            insts.reserve(probe_count as usize);
+            for i in 0..probe_count {
+                let offset = (guard_size * (i + 1)) as i64;
+                insts.push(Self::gen_store_stack(
+                    StackAMode::SPOffset(-offset, I8),
+                    zero_reg(),
+                    I32,
+                ));
+            }
+        } else {
+            // The non-unrolled version uses two temporary registers. The
+            // `start` contains the current offset from sp and counts downwards
+            // during the loop by increments of `guard_size`. The `end` is
+            // the size of the frame and where we stop.
+            //
+            // Note that this emission is all post-regalloc so it should be ok
+            // to use the temporary registers here as input/output as the loop
+            // itself is not allowed to use the registers.
+            let start = writable_spilltmp_reg();
+            let end = writable_tmp2_reg();
+            insts.extend(Inst::load_constant(start, 0));
+            insts.extend(Inst::load_constant(end, frame_size.into()));
+            insts.push(Inst::StackProbeLoop {
+                start,
+                end: end.to_reg(),
+                step: Imm12::maybe_from_u64(guard_size.into()).unwrap(),
+            });
+        }
     }
 
     // Returns stack bytes used as well as instructions. Does not adjust
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 46e8e842d..b61ca529c 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -935,7 +935,16 @@
 
        ;; A dummy use, useful to keep a value alive.
        (DummyUse
-        (reg Reg))))
+        (reg Reg))
+
+       ;; Emits an inline stack probe loop.
+       ;;
+       ;; Note that this is emitted post-regalloc so `start` and `end` can be
+       ;; temporary registers such as the spilltmp and tmp2 registers. This also
+       ;; means that the internal codegen can't use these registers.
+       (StackProbeLoop (start WritableReg)
+                       (end Reg)
+                       (step Imm12))))
 
 ;; An ALU operation. This can be paired with several instruction formats
 ;; below (see `Inst`) in any combination.
@@ -1134,8 +1143,7 @@
         ;; [crate::isa::aarch64::abi](the ABI module) for more details.
         (NominalSPOffset
          (off i64)
-         (ty Type))
-))
+         (ty Type))))
 
 (type PairAMode extern (enum))
 (type FPUOpRI extern (enum))
@@ -2242,7 +2250,7 @@
       (let ((dst WritableReg (temp_writable_reg $I64)))
         (ConsumesFlags.ConsumesFlagsReturnsResultWithProducer (MInst.CSet dst cond) dst)))
 
-;; Helper for constructing `csetm` instructions. 
+;; Helper for constructing `csetm` instructions.
 (decl csetm (Cond) ConsumesFlags)
 (rule (csetm cond)
       (let ((dst WritableReg (temp_writable_reg $I64)))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index d2277b4ba..0280a4c00 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -63,21 +63,14 @@ pub fn mem_finalize(
                 (smallvec![], mem)
             } else {
                 let tmp = writable_spilltmp_reg();
-                let mut const_insts = Inst::load_constant(tmp, off as u64);
-                // N.B.: we must use AluRRRExtend because AluRRR uses the "shifted register" form
-                // (AluRRRShift) instead, which interprets register 31 as the zero reg, not SP. SP
-                // is a valid base (for SPOffset) which we must handle here.
-                // Also, SP needs to be the first arg, not second.
-                let add_inst = Inst::AluRRRExtend {
-                    alu_op: ALUOp::Add,
-                    size: OperandSize::Size64,
-                    rd: tmp,
-                    rn: basereg,
-                    rm: tmp.to_reg(),
-                    extendop: ExtendOp::UXTX,
-                };
-                const_insts.push(add_inst);
-                (const_insts, AMode::reg(tmp.to_reg()))
+                (
+                    Inst::load_constant(tmp, off as u64),
+                    AMode::RegExtended {
+                        rn: basereg,
+                        rm: tmp.to_reg(),
+                        extendop: ExtendOp::SXTX,
+                    },
+                )
             }
         }
 
@@ -3424,6 +3417,72 @@ impl MachInstEmit for Inst {
             }
 
             &Inst::DummyUse { .. } => {}
+
+            &Inst::StackProbeLoop { start, end, step } => {
+                assert!(emit_info.0.enable_probestack());
+                let start = allocs.next_writable(start);
+                let end = allocs.next(end);
+
+                // The loop generated here uses `start` as a counter register to
+                // count backwards until negating it exceeds `end`. In other
+                // words `start` is an offset from `sp` we're testing where
+                // `end` is the max size we need to test. The loop looks like:
+                //
+                //      loop_start:
+                //          sub start, start, #step
+                //          stur xzr, [sp, start]
+                //          cmn start, end
+                //          br.gt loop_start
+                //      loop_end:
+                //
+                // Note that this loop cannot use the spilltmp and tmp2
+                // registers as those are currently used as the input to this
+                // loop when generating the instruction. This means that some
+                // more flavorful address modes and lowerings need to be
+                // avoided.
+                //
+                // Perhaps someone more clever than I can figure out how to use
+                // `subs` or the like and skip the `cmn`, but I can't figure it
+                // out at this time.
+
+                let loop_start = sink.get_label();
+                sink.bind_label(loop_start);
+
+                Inst::AluRRImm12 {
+                    alu_op: ALUOp::Sub,
+                    size: OperandSize::Size64,
+                    rd: start,
+                    rn: start.to_reg(),
+                    imm12: step,
+                }
+                .emit(&[], sink, emit_info, state);
+                Inst::Store32 {
+                    rd: regs::zero_reg(),
+                    mem: AMode::RegReg {
+                        rn: regs::stack_reg(),
+                        rm: start.to_reg(),
+                    },
+                    flags: MemFlags::trusted(),
+                }
+                .emit(&[], sink, emit_info, state);
+                Inst::AluRRR {
+                    alu_op: ALUOp::AddS,
+                    size: OperandSize::Size64,
+                    rd: regs::writable_zero_reg(),
+                    rn: start.to_reg(),
+                    rm: end,
+                }
+                .emit(&[], sink, emit_info, state);
+
+                let loop_end = sink.get_label();
+                Inst::CondBr {
+                    taken: BranchTarget::Label(loop_start),
+                    not_taken: BranchTarget::Label(loop_end),
+                    kind: CondBrKind::Cond(Cond::Gt),
+                }
+                .emit(&[], sink, emit_info, state);
+                sink.bind_label(loop_end);
+            }
         }
 
         let end_off = sink.cur_offset();
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 2acef7e19..3ed883a49 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -1103,6 +1103,10 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
         &Inst::DummyUse { reg } => {
             collector.reg_use(reg);
         }
+        &Inst::StackProbeLoop { start, end, .. } => {
+            collector.reg_early_def(start);
+            collector.reg_use(end);
+        }
     }
 }
 
@@ -2887,6 +2891,12 @@ impl Inst {
                 let reg = pretty_print_reg(reg, allocs);
                 format!("dummy_use {}", reg)
             }
+            &Inst::StackProbeLoop { start, end, step } => {
+                let start = pretty_print_reg(start.to_reg(), allocs);
+                let end = pretty_print_reg(end, allocs);
+                let step = step.pretty_print(0, allocs);
+                format!("stack_probe_loop {start}, {end}, {step}")
+            }
         }
     }
 }
diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs
index 7aeee8ba5..995a52fbc 100644
--- a/cranelift/codegen/src/isa/riscv64/abi.rs
+++ b/cranelift/codegen/src/isa/riscv64/abi.rs
@@ -360,8 +360,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
         insts
     }
 
-    fn gen_probestack(frame_size: u32) -> SmallInstVec<Self::I> {
-        let mut insts = SmallVec::new();
+    fn gen_probestack(insts: &mut SmallInstVec<Self::I>, frame_size: u32) {
         insts.extend(Inst::load_constant_u32(writable_a0(), frame_size as u64));
         insts.push(Inst::Call {
             info: Box::new(CallInfo {
@@ -377,7 +376,6 @@ impl ABIMachineSpec for Riscv64MachineDeps {
                 caller_callconv: CallConv::SystemV,
             }),
         });
-        insts
     }
     // Returns stack bytes used as well as instructions. Does not adjust
     // nominal SP offset; abi_impl generic code will do that.
@@ -632,16 +630,16 @@ impl ABIMachineSpec for Riscv64MachineDeps {
         || fixed_frame_storage_size > 0
     }
 
-    fn gen_inline_probestack(frame_size: u32, guard_size: u32) -> SmallInstVec<Self::I> {
+    fn gen_inline_probestack(insts: &mut SmallInstVec<Self::I>, frame_size: u32, guard_size: u32) {
         // Unroll at most n consecutive probes, before falling back to using a loop
         const PROBE_MAX_UNROLL: u32 = 3;
         // Number of probes that we need to perform
         let probe_count = align_to(frame_size, guard_size) / guard_size;
 
         if probe_count <= PROBE_MAX_UNROLL {
-            Self::gen_probestack_unroll(guard_size, probe_count)
+            Self::gen_probestack_unroll(insts, guard_size, probe_count)
         } else {
-            Self::gen_probestack_loop(guard_size, probe_count)
+            Self::gen_probestack_loop(insts, guard_size, probe_count)
         }
     }
 }
@@ -697,8 +695,8 @@ fn compute_clobber_size(clobbers: &[Writable<RealReg>]) -> u32 {
 }
 
 impl Riscv64MachineDeps {
-    fn gen_probestack_unroll(guard_size: u32, probe_count: u32) -> SmallInstVec<Inst> {
-        let mut insts = SmallVec::with_capacity(probe_count as usize);
+    fn gen_probestack_unroll(insts: &mut SmallInstVec<Inst>, guard_size: u32, probe_count: u32) {
+        insts.reserve(probe_count as usize);
         for i in 0..probe_count {
             let offset = (guard_size * (i + 1)) as i64;
             insts.push(Self::gen_store_stack(
@@ -707,13 +705,13 @@ impl Riscv64MachineDeps {
                 I32,
             ));
         }
-        insts
     }
-    fn gen_probestack_loop(guard_size: u32, probe_count: u32) -> SmallInstVec<Inst> {
-        smallvec![Inst::StackProbeLoop {
+
+    fn gen_probestack_loop(insts: &mut SmallInstVec<Inst>, guard_size: u32, probe_count: u32) {
+        insts.push(Inst::StackProbeLoop {
             guard_size,
             probe_count,
             tmp: Writable::from_reg(x_reg(28)), // t3
-        }]
+        });
     }
 }
diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs
index 45e051f02..f123a4896 100644
--- a/cranelift/codegen/src/isa/s390x/abi.rs
+++ b/cranelift/codegen/src/isa/s390x/abi.rs
@@ -569,13 +569,17 @@ impl ABIMachineSpec for S390xMachineDeps {
         SmallVec::new()
     }
 
-    fn gen_probestack(_: u32) -> SmallInstVec<Self::I> {
+    fn gen_probestack(_insts: &mut SmallInstVec<Self::I>, _: u32) {
         // TODO: implement if we ever require stack probes on an s390x host
         // (unlikely unless Lucet is ported)
-        smallvec![]
+        unimplemented!("Stack probing is unimplemented on S390x");
     }
 
-    fn gen_inline_probestack(_frame_size: u32, _guard_size: u32) -> SmallInstVec<Self::I> {
+    fn gen_inline_probestack(
+        _insts: &mut SmallInstVec<Self::I>,
+        _frame_size: u32,
+        _guard_size: u32,
+    ) {
         unimplemented!("Inline stack probing is unimplemented on S390x");
     }
 
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index f0cd7b879..eb7c48bc5 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -104,7 +104,7 @@ pub fn mem_finalize(
             } else {
                 let tmp = writable_spilltmp_reg();
                 assert!(base != tmp.to_reg());
-                insts.extend(Inst::load_constant64(tmp, off as u64));
+                insts.extend(Inst::load_constant64(tmp, off as u64, |_| tmp));
                 MemArg::reg_plus_reg(base, tmp.to_reg(), mem.get_flags())
             }
         }
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 5caf862c1..41b34ea87 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -335,7 +335,11 @@ impl Inst {
     }
 
     /// Create an instruction that loads a 64-bit integer constant.
-    pub fn load_constant64(rd: Writable<Reg>, value: u64) -> SmallVec<[Inst; 4]> {
+    pub fn load_constant64<F: FnMut(Type) -> Writable<Reg>>(
+        rd: Writable<Reg>,
+        value: u64,
+        mut alloc_tmp: F,
+    ) -> SmallVec<[Inst; 4]> {
         if let Ok(imm) = i16::try_from(value as i64) {
             // 16-bit signed immediate
             smallvec![Inst::Mov64SImm16 { rd, imm }]
@@ -353,12 +357,13 @@ impl Inst {
             let hi = value & 0xffff_ffff_0000_0000u64;
             let lo = value & 0x0000_0000_ffff_ffffu64;
 
+            let hi_rd = alloc_tmp(types::I64);
             if let Some(imm) = UImm16Shifted::maybe_from_u64(hi) {
                 // 16-bit shifted immediate
-                insts.push(Inst::Mov64UImm16Shifted { rd, imm });
+                insts.push(Inst::Mov64UImm16Shifted { rd: hi_rd, imm });
             } else if let Some(imm) = UImm32Shifted::maybe_from_u64(hi) {
                 // 32-bit shifted immediate
-                insts.push(Inst::Mov64UImm32Shifted { rd, imm });
+                insts.push(Inst::Mov64UImm32Shifted { rd: hi_rd, imm });
             } else {
                 unreachable!();
             }
@@ -367,14 +372,14 @@ impl Inst {
                 // 16-bit shifted immediate
                 insts.push(Inst::Insert64UImm16Shifted {
                     rd,
-                    ri: rd.to_reg(),
+                    ri: hi_rd.to_reg(),
                     imm,
                 });
             } else if let Some(imm) = UImm32Shifted::maybe_from_u64(lo) {
                 // 32-bit shifted immediate
                 insts.push(Inst::Insert64UImm32Shifted {
                     rd,
-                    ri: rd.to_reg(),
+                    ri: hi_rd.to_reg(),
                     imm,
                 });
             } else {
@@ -1172,7 +1177,7 @@ impl MachInst for Inst {
         to_regs: ValueRegs<Writable<Reg>>,
         value: u128,
         ty: Type,
-        _alloc_tmp: F,
+        alloc_tmp: F,
     ) -> SmallVec<[Inst; 4]> {
         let to_reg = to_regs
             .only_reg()
@@ -1204,7 +1209,7 @@ impl MachInst for Inst {
                 ));
                 ret
             }
-            types::I64 | types::R64 => Inst::load_constant64(to_reg, value as u64),
+            types::I64 | types::R64 => Inst::load_constant64(to_reg, value as u64, alloc_tmp),
             types::I8 | types::I16 | types::I32 => Inst::load_constant32(to_reg, value as u32),
             _ => unreachable!(),
         }
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs
index 1f6f5ce9a..3f142aa0a 100644
--- a/cranelift/codegen/src/isa/s390x/lower/isle.rs
+++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs
@@ -126,7 +126,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
     fn defs_init(&mut self, abi: &Sig) -> CallRetList {
         // Allocate writable registers for all retval regs, except for StructRet args.
         let mut defs = smallvec![];
-        for i in 0..self.lower_ctx.sigs()[*abi].num_rets() {
+        for i in 0..self.lower_ctx.sigs().num_rets(*abi) {
             if let &ABIArg::Slots {
                 ref slots, purpose, ..
             } = &self.lower_ctx.sigs().get_ret(*abi, i)
@@ -169,7 +169,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6>
         // Return the index of the first actual return value, excluding
         // any StructReturn that might have been added to Sig.
         let sig = &self.lower_ctx.dfg().signatures[sig_ref];
-        self.lower_ctx.sigs()[*abi].num_rets() - sig.returns.len()
+        self.lower_ctx.sigs().num_rets(*abi) - sig.returns.len()
     }
 
     fn abi_lane_order(&mut self, abi: &Sig) -> LaneOrder {
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index 91bde772c..d2b9d6e18 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -30,8 +30,8 @@ pub(crate) type X64Caller = Caller<X64ABIMachineSpec>;
 pub struct X64ABIMachineSpec;
 
 impl X64ABIMachineSpec {
-    fn gen_probestack_unroll(guard_size: u32, probe_count: u32) -> SmallInstVec<Inst> {
-        let mut insts = SmallVec::with_capacity(probe_count as usize);
+    fn gen_probestack_unroll(insts: &mut SmallInstVec<Inst>, guard_size: u32, probe_count: u32) {
+        insts.reserve(probe_count as usize);
         for i in 0..probe_count {
             let offset = (guard_size * (i + 1)) as i64;
 
@@ -43,9 +43,8 @@ impl X64ABIMachineSpec {
                 I32,
             ));
         }
-        insts
     }
-    fn gen_probestack_loop(frame_size: u32, guard_size: u32) -> SmallInstVec<Inst> {
+    fn gen_probestack_loop(insts: &mut SmallInstVec<Inst>, frame_size: u32, guard_size: u32) {
         // We have to use a caller saved register since clobbering only happens
         // after stack probing.
         //
@@ -57,11 +56,11 @@ impl X64ABIMachineSpec {
             !is_callee_save_systemv(real_reg, false) && !is_callee_save_fastcall(real_reg, false)
         });
 
-        smallvec![Inst::StackProbeLoop {
+        insts.push(Inst::StackProbeLoop {
             tmp: Writable::from_reg(tmp),
             frame_size,
             guard_size,
-        }]
+        });
     }
 }
 
@@ -420,8 +419,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         insts
     }
 
-    fn gen_probestack(frame_size: u32) -> SmallInstVec<Self::I> {
-        let mut insts = SmallVec::new();
+    fn gen_probestack(insts: &mut SmallInstVec<Self::I>, frame_size: u32) {
         insts.push(Inst::imm(
             OperandSize::Size32,
             frame_size as u64,
@@ -438,10 +436,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                 opcode: Opcode::Call,
             }),
         });
-        insts
     }
 
-    fn gen_inline_probestack(frame_size: u32, guard_size: u32) -> SmallInstVec<Self::I> {
+    fn gen_inline_probestack(insts: &mut SmallInstVec<Self::I>, frame_size: u32, guard_size: u32) {
         // Unroll at most n consecutive probes, before falling back to using a loop
         //
         // This was number was picked because the loop version is 38 bytes long. We can fit
@@ -452,9 +449,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         let probe_count = align_to(frame_size, guard_size) / guard_size;
 
         if probe_count <= PROBE_MAX_UNROLL {
-            Self::gen_probestack_unroll(guard_size, probe_count)
+            Self::gen_probestack_unroll(insts, guard_size, probe_count)
         } else {
-            Self::gen_probestack_loop(frame_size, guard_size)
+            Self::gen_probestack_loop(insts, frame_size, guard_size)
         }
     }
 
diff --git a/cranelift/codegen/src/isa/x64/encoding/rex.rs b/cranelift/codegen/src/isa/x64/encoding/rex.rs
index 4ca681a1b..41ae596eb 100644
--- a/cranelift/codegen/src/isa/x64/encoding/rex.rs
+++ b/cranelift/codegen/src/isa/x64/encoding/rex.rs
@@ -13,7 +13,7 @@ use crate::{
     ir::TrapCode,
     isa::x64::inst::{
         args::{Amode, OperandSize},
-        regs, EmitInfo, Inst, LabelUse,
+        regs, Inst, LabelUse,
     },
     machinst::MachBuffer,
 };
@@ -293,7 +293,6 @@ impl Default for LegacyPrefixes {
 /// indicate a 64-bit operation.
 pub(crate) fn emit_std_enc_mem(
     sink: &mut MachBuffer<Inst>,
-    info: &EmitInfo,
     prefixes: LegacyPrefixes,
     opcodes: u32,
     mut num_opcodes: usize,
@@ -315,12 +314,6 @@ pub(crate) fn emit_std_enc_mem(
 
     match *mem_e {
         Amode::ImmReg { simm32, base, .. } => {
-            // If this is an access based off of RSP, it may trap with a stack overflow if it's the
-            // first touch of a new stack page.
-            if base == regs::rsp() && !can_trap && info.flags.enable_probestack() {
-                sink.add_trap(TrapCode::StackOverflow);
-            }
-
             // First, the REX byte.
             let enc_e = int_reg_enc(base);
             rex.emit_two_op(sink, enc_g, enc_e);
@@ -381,12 +374,6 @@ pub(crate) fn emit_std_enc_mem(
             shift,
             ..
         } => {
-            // If this is an access based off of RSP, it may trap with a stack overflow if it's the
-            // first touch of a new stack page.
-            if *reg_base == regs::rsp() && !can_trap && info.flags.enable_probestack() {
-                sink.add_trap(TrapCode::StackOverflow);
-            }
-
             let enc_base = int_reg_enc(*reg_base);
             let enc_index = int_reg_enc(*reg_index);
 
@@ -481,7 +468,6 @@ pub(crate) fn emit_std_enc_enc(
 
 pub(crate) fn emit_std_reg_mem(
     sink: &mut MachBuffer<Inst>,
-    info: &EmitInfo,
     prefixes: LegacyPrefixes,
     opcodes: u32,
     num_opcodes: usize,
@@ -493,7 +479,6 @@ pub(crate) fn emit_std_reg_mem(
     let enc_g = reg_enc(reg_g);
     emit_std_enc_mem(
         sink,
-        info,
         prefixes,
         opcodes,
         num_opcodes,
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 865ed500d..314f82e81 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -1566,22 +1566,10 @@
 ;; Helper for creating an SSE register holding an `i64x2` from two `i64` values.
 (decl make_i64x2_from_lanes (GprMem GprMem) Xmm)
 (rule (make_i64x2_from_lanes lo hi)
-      (let ((dst_xmm WritableXmm (temp_writable_xmm))
-            (dst_reg WritableReg dst_xmm)
-            (_ Unit (emit (MInst.XmmUninitializedValue dst_xmm)))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd)
-                                           dst_reg
-                                           lo
-                                           dst_reg
-                                           0
-                                           (OperandSize.Size64))))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd)
-                                           dst_reg
-                                           hi
-                                           dst_reg
-                                           1
-                                           (OperandSize.Size64)))))
-        dst_xmm))
+      (let ((dst Xmm (xmm_uninit_value))
+            (dst Xmm (x64_pinsrd dst lo 0 (OperandSize.Size64)))
+            (dst Xmm (x64_pinsrd dst hi 1 (OperandSize.Size64))))
+        dst))
 
 ;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary.
 (decl mov_rmi_to_xmm (RegMemImm) XmmMemImm)
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 6b468d2bc..6c22bb045 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -171,7 +171,6 @@ pub(crate) fn emit(
                         let amode = addr.finalize(state, sink);
                         emit_std_reg_mem(
                             sink,
-                            info,
                             LegacyPrefixes::None,
                             0x0FAF,
                             2,
@@ -223,7 +222,6 @@ pub(crate) fn emit(
                         // Here we revert to the "normal" G-E ordering.
                         emit_std_reg_mem(
                             sink,
-                            info,
                             LegacyPrefixes::None,
                             opcode_m,
                             1,
@@ -275,7 +273,6 @@ pub(crate) fn emit(
             let enc_g = int_reg_enc(src2);
             emit_std_enc_mem(
                 sink,
-                info,
                 LegacyPrefixes::None,
                 opcode,
                 1,
@@ -317,17 +314,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr: src } => {
                     let amode = src.finalize(state, sink).with_allocs(allocs);
-                    emit_std_reg_mem(
-                        sink,
-                        info,
-                        prefix,
-                        opcode,
-                        num_opcodes,
-                        dst,
-                        &amode,
-                        rex_flags,
-                        0,
-                    );
+                    emit_std_reg_mem(sink, prefix, opcode, num_opcodes, dst, &amode, rex_flags, 0);
                 }
             }
         }
@@ -414,7 +401,6 @@ pub(crate) fn emit(
                     let amode = src.finalize(state, sink).with_allocs(allocs);
                     emit_std_enc_mem(
                         sink,
-                        info,
                         prefix,
                         opcode,
                         1,
@@ -459,7 +445,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr: src } => {
                     let amode = src.finalize(state, sink).with_allocs(allocs);
-                    emit_std_enc_mem(sink, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags, 0);
+                    emit_std_enc_mem(sink, prefix, 0xF7, 1, subopcode, &amode, rex_flags, 0);
                 }
             }
         }
@@ -779,7 +765,6 @@ pub(crate) fn emit(
 
                     emit_std_reg_mem(
                         sink,
-                        info,
                         LegacyPrefixes::None,
                         opcodes,
                         num_opcodes,
@@ -798,7 +783,6 @@ pub(crate) fn emit(
 
             emit_std_reg_mem(
                 sink,
-                info,
                 LegacyPrefixes::None,
                 0x8B,
                 1,
@@ -815,7 +799,6 @@ pub(crate) fn emit(
 
             emit_std_reg_mem(
                 sink,
-                info,
                 LegacyPrefixes::None,
                 0x8D,
                 1,
@@ -877,7 +860,6 @@ pub(crate) fn emit(
 
                     emit_std_reg_mem(
                         sink,
-                        info,
                         LegacyPrefixes::None,
                         opcodes,
                         num_opcodes,
@@ -913,7 +895,7 @@ pub(crate) fn emit(
             // 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
             // 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
             // 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
-            emit_std_reg_mem(sink, info, prefix, opcode, 1, src, dst, rex, 0);
+            emit_std_reg_mem(sink, prefix, opcode, 1, src, dst, rex, 0);
         }
 
         Inst::ShiftR {
@@ -1023,7 +1005,7 @@ pub(crate) fn emit(
                     }
                     RegMemImm::Mem { addr } => {
                         let addr = &addr.finalize(state, sink).with_allocs(allocs);
-                        emit_std_reg_mem(sink, info, prefix, opcode_bytes, 2, dst, addr, rex, 0);
+                        emit_std_reg_mem(sink, prefix, opcode_bytes, 2, dst, addr, rex, 0);
                     }
                     RegMemImm::Imm { .. } => unreachable!(),
                 }
@@ -1078,7 +1060,7 @@ pub(crate) fn emit(
                         (OperandSize::Size8, false) => 0x84,
                         (_, false) => 0x85,
                     };
-                    emit_std_reg_mem(sink, info, prefix, opcode, 1, reg_g, addr, rex, 0);
+                    emit_std_reg_mem(sink, prefix, opcode, 1, reg_g, addr, rex, 0);
                 }
 
                 RegMemImm::Imm { simm32 } => {
@@ -1167,7 +1149,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink).with_allocs(allocs);
-                    emit_std_reg_mem(sink, info, prefix, opcode, 2, dst, addr, rex_flags, 0);
+                    emit_std_reg_mem(sink, prefix, opcode, 2, dst, addr, rex_flags, 0);
                 }
             }
         }
@@ -1210,10 +1192,6 @@ pub(crate) fn emit(
         Inst::Push64 { src } => {
             let src = src.clone().to_reg_mem_imm().with_allocs(allocs);
 
-            if info.flags.enable_probestack() {
-                sink.add_trap(TrapCode::StackOverflow);
-            }
-
             match src {
                 RegMemImm::Reg { reg } => {
                     let enc_reg = int_reg_enc(reg);
@@ -1228,7 +1206,6 @@ pub(crate) fn emit(
                     let addr = &addr.finalize(state, sink);
                     emit_std_enc_mem(
                         sink,
-                        info,
                         LegacyPrefixes::None,
                         0xFF,
                         1,
@@ -1369,9 +1346,6 @@ pub(crate) fn emit(
             info: call_info,
             ..
         } => {
-            if info.flags.enable_probestack() {
-                sink.add_trap(TrapCode::StackOverflow);
-            }
             if let Some(s) = state.take_stack_map() {
                 sink.add_stack_map(StackMapExtent::UpcomingBytes(5), s);
             }
@@ -1392,9 +1366,6 @@ pub(crate) fn emit(
         } => {
             let dest = dest.with_allocs(allocs);
 
-            if info.flags.enable_probestack() {
-                sink.add_trap(TrapCode::StackOverflow);
-            }
             let start_offset = sink.cur_offset();
             match dest {
                 RegMem::Reg { reg } => {
@@ -1414,7 +1385,6 @@ pub(crate) fn emit(
                     let addr = &addr.finalize(state, sink);
                     emit_std_enc_mem(
                         sink,
-                        info,
                         LegacyPrefixes::None,
                         0xFF,
                         1,
@@ -1516,7 +1486,6 @@ pub(crate) fn emit(
                     let addr = &addr.finalize(state, sink);
                     emit_std_enc_mem(
                         sink,
-                        info,
                         LegacyPrefixes::None,
                         0xFF,
                         1,
@@ -1739,7 +1708,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
-                    emit_std_reg_mem(sink, info, prefix, opcode, num_opcodes, reg_g, addr, rex, 0);
+                    emit_std_reg_mem(sink, prefix, opcode, num_opcodes, reg_g, addr, rex, 0);
                 }
             };
         }
@@ -1765,7 +1734,7 @@ pub(crate) fn emit(
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
                     // N.B.: bytes_at_end == 1, because of the `imm` byte below.
-                    emit_std_reg_mem(sink, info, prefix, opcode, len, dst, addr, rex, 1);
+                    emit_std_reg_mem(sink, prefix, opcode, len, dst, addr, rex, 1);
                 }
             }
             sink.put1(*imm);
@@ -1918,7 +1887,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
-                    emit_std_reg_mem(sink, info, prefix, opcode, length, reg_g, addr, rex, 0);
+                    emit_std_reg_mem(sink, prefix, opcode, length, reg_g, addr, rex, 0);
                 }
             }
         }
@@ -1951,7 +1920,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
-                    emit_std_reg_mem(sink, info, prefix, opcode, length, reg_g, addr, rex, 0);
+                    emit_std_reg_mem(sink, prefix, opcode, length, reg_g, addr, rex, 0);
                 }
             }
         }
@@ -2191,7 +2160,7 @@ pub(crate) fn emit(
                         "No existing way to encode a mem argument in the ModRM r/m field."
                     );
                     // N.B.: bytes_at_end == 1, because of the `imm` byte below.
-                    emit_std_reg_mem(sink, info, prefix, opcode, len, dst, addr, rex, 1);
+                    emit_std_reg_mem(sink, prefix, opcode, len, dst, addr, rex, 1);
                 }
             }
             sink.put1(*imm);
@@ -2217,17 +2186,7 @@ pub(crate) fn emit(
                 _ => unimplemented!("Opcode {:?} not implemented", op),
             };
             let dst = &dst.finalize(state, sink);
-            emit_std_reg_mem(
-                sink,
-                info,
-                prefix,
-                opcode,
-                2,
-                src,
-                dst,
-                RexFlags::clear_w(),
-                0,
-            );
+            emit_std_reg_mem(sink, prefix, opcode, 2, src, dst, RexFlags::clear_w(), 0);
         }
 
         Inst::XmmToGpr {
@@ -2280,7 +2239,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
-                    emit_std_reg_mem(sink, info, prefix, opcode, 2, reg_g, addr, rex, 0);
+                    emit_std_reg_mem(sink, prefix, opcode, 2, reg_g, addr, rex, 0);
                 }
             }
         }
@@ -2303,7 +2262,7 @@ pub(crate) fn emit(
                 }
                 RegMem::Mem { addr } => {
                     let addr = &addr.finalize(state, sink);
-                    emit_std_reg_mem(sink, info, prefix, opcode, len, dst, addr, rex, 0);
+                    emit_std_reg_mem(sink, prefix, opcode, len, dst, addr, rex, 0);
                 }
             }
         }
@@ -2385,6 +2344,7 @@ pub(crate) fn emit(
                 OperandSize::Size64,
                 ShiftKind::ShiftRightLogical,
                 Imm8Gpr::new(Imm8Reg::Imm8 { imm: 1 }).unwrap(),
+                tmp_gpr1,
                 Writable::from_reg(tmp_gpr1),
             );
             inst.emit(&[], sink, info, state);
@@ -2912,7 +2872,7 @@ pub(crate) fn emit(
             };
             let rex = RexFlags::from((OperandSize::from_ty(*ty), replacement));
             let amode = mem.finalize(state, sink);
-            emit_std_reg_mem(sink, info, prefix, opcodes, 2, replacement, &amode, rex, 0);
+            emit_std_reg_mem(sink, prefix, opcodes, 2, replacement, &amode, rex, 0);
         }
 
         Inst::AtomicRmwSeq {
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 4704f809b..f9984c87b 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -431,6 +431,7 @@ impl Inst {
         size: OperandSize,
         kind: ShiftKind,
         num_bits: Imm8Gpr,
+        src: Reg,
         dst: Writable<Reg>,
     ) -> Inst {
         if let Imm8Reg::Imm8 { imm: num_bits } = num_bits.clone().to_imm8_reg() {
@@ -440,7 +441,7 @@ impl Inst {
         Inst::ShiftR {
             size,
             kind,
-            src: Gpr::new(dst.to_reg()).unwrap(),
+            src: Gpr::new(src).unwrap(),
             num_bits,
             dst: WritableGpr::from_writable_reg(dst).unwrap(),
         }
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 53a7de61a..1fa9d652a 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -980,18 +980,17 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
             ));
         } else {
             if size == OperandSize::Size8 {
+                let tmp = self.temp_writable_reg(ty);
                 // The remainder is in AH. Right-shift by 8 bits then move from rax.
                 self.lower_ctx.emit(MInst::shift_r(
                     OperandSize::Size64,
                     ShiftKind::ShiftRightLogical,
                     Imm8Gpr::new(Imm8Reg::Imm8 { imm: 8 }).unwrap(),
-                    dst_quotient,
-                ));
-                self.lower_ctx.emit(MInst::gen_move(
-                    dst.to_writable_reg(),
                     dst_quotient.to_reg(),
-                    ty,
+                    tmp,
                 ));
+                self.lower_ctx
+                    .emit(MInst::gen_move(dst.to_writable_reg(), tmp.to_reg(), ty));
             } else {
                 // The remainder is in rdx.
                 self.lower_ctx.emit(MInst::gen_move(
diff --git a/cranelift/codegen/src/legalizer/heap.rs b/cranelift/codegen/src/legalizer/heap.rs
index 9dba43126..d294772f2 100644
--- a/cranelift/codegen/src/legalizer/heap.rs
+++ b/cranelift/codegen/src/legalizer/heap.rs
@@ -6,268 +6,394 @@
 use crate::cursor::{Cursor, FuncCursor};
 use crate::flowgraph::ControlFlowGraph;
 use crate::ir::condcodes::IntCC;
-use crate::ir::immediates::{Uimm32, Uimm8};
+use crate::ir::immediates::{HeapImmData, Offset32, Uimm32, Uimm8};
 use crate::ir::{self, InstBuilder, RelSourceLoc};
 use crate::isa::TargetIsa;
 use crate::trace;
 
-/// Expand a `heap_addr` instruction according to the definition of the heap.
-pub fn expand_heap_addr(
+/// Expand a `heap_load` instruction according to the definition of the heap.
+pub fn expand_heap_load(
     inst: ir::Inst,
     func: &mut ir::Function,
     cfg: &mut ControlFlowGraph,
     isa: &dyn TargetIsa,
-    heap: ir::Heap,
-    index_operand: ir::Value,
-    offset_immediate: Uimm32,
-    access_size: Uimm8,
+    heap_imm: ir::HeapImm,
+    index: ir::Value,
 ) {
-    trace!(
-        "expanding heap_addr: {:?}: {}",
-        inst,
-        func.dfg.display_inst(inst)
-    );
+    let HeapImmData {
+        flags,
+        heap,
+        offset,
+    } = func.dfg.heap_imms[heap_imm];
 
-    let ir::HeapData {
-        offset_guard_size,
-        style,
-        ..
-    } = &func.heaps[heap];
-    match *style {
-        ir::HeapStyle::Dynamic { bound_gv } => dynamic_addr(
-            isa,
-            inst,
-            heap,
-            index_operand,
-            u32::from(offset_immediate),
-            u8::from(access_size),
-            bound_gv,
-            func,
-        ),
-        ir::HeapStyle::Static { bound } => static_addr(
-            isa,
-            inst,
-            heap,
-            index_operand,
-            u32::from(offset_immediate),
-            u8::from(access_size),
-            bound.into(),
-            (*offset_guard_size).into(),
-            func,
-            cfg,
-        ),
-    }
+    let result_ty = func.dfg.ctrl_typevar(inst);
+    let access_size = result_ty.bytes();
+    let access_size = u8::try_from(access_size).unwrap();
+
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let addr =
+        bounds_check_and_compute_addr(&mut pos, cfg, isa, heap, index, offset.into(), access_size);
+
+    pos.func
+        .dfg
+        .replace(inst)
+        .load(result_ty, flags, addr, Offset32::new(0));
 }
 
-/// Expand a `heap_addr` for a dynamic heap.
-fn dynamic_addr(
-    isa: &dyn TargetIsa,
+/// Expand a `heap_store` instruction according to the definition of the heap.
+pub fn expand_heap_store(
     inst: ir::Inst,
-    heap: ir::Heap,
-    index: ir::Value,
-    offset: u32,
-    access_size: u8,
-    bound_gv: ir::GlobalValue,
     func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+    heap_imm: ir::HeapImm,
+    index: ir::Value,
+    value: ir::Value,
 ) {
-    let index_ty = func.dfg.value_type(index);
-    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
-    let min_size = func.heaps[heap].min_size.into();
+    let HeapImmData {
+        flags,
+        heap,
+        offset,
+    } = func.dfg.heap_imms[heap_imm];
+
+    let store_ty = func.dfg.value_type(value);
+    let access_size = u8::try_from(store_ty.bytes()).unwrap();
+
     let mut pos = FuncCursor::new(func).at_inst(inst);
     pos.use_srcloc(inst);
 
-    let index = cast_index_to_pointer_ty(index, index_ty, addr_ty, &mut pos);
-
-    // Start with the bounds check. Trap if `index + offset + access_size > bound`.
-    let bound = pos.ins().global_value(addr_ty, bound_gv);
-    let (cc, lhs, bound) = if offset == 0 && access_size == 1 {
-        // `index > bound - 1` is the same as `index >= bound`.
-        (IntCC::UnsignedGreaterThanOrEqual, index, bound)
-    } else if offset_plus_size(offset, access_size) <= min_size {
-        // We know that `bound >= min_size`, so here we can compare `offset >
-        // bound - (offset + access_size)` without wrapping.
-        let adj_bound = pos
-            .ins()
-            .iadd_imm(bound, -(offset_plus_size(offset, access_size) as i64));
-        trace!(
-            "  inserting: {}",
-            pos.func.dfg.display_value_inst(adj_bound)
-        );
-        (IntCC::UnsignedGreaterThan, index, adj_bound)
-    } else {
-        // We need an overflow check for the adjusted offset.
-        let access_size_val = pos
-            .ins()
-            .iconst(addr_ty, offset_plus_size(offset, access_size) as i64);
-        let adj_offset =
-            pos.ins()
-                .uadd_overflow_trap(index, access_size_val, ir::TrapCode::HeapOutOfBounds);
-        trace!(
-            "  inserting: {}",
-            pos.func.dfg.display_value_inst(adj_offset)
-        );
-        (IntCC::UnsignedGreaterThan, adj_offset, bound)
-    };
+    let addr =
+        bounds_check_and_compute_addr(&mut pos, cfg, isa, heap, index, offset.into(), access_size);
 
-    let spectre_oob_comparison = if isa.flags().enable_heap_access_spectre_mitigation() {
-        // When we emit a spectre-guarded heap access, we do a `select
-        // is_out_of_bounds, NULL, addr` to compute the address, and so the load
-        // will trap if the address is out of bounds, which means we don't need
-        // to do another explicit bounds check like we do below.
-        Some(SpectreOobComparison {
-            cc,
-            lhs,
-            rhs: bound,
-        })
-    } else {
-        let oob = pos.ins().icmp(cc, lhs, bound);
-        trace!("  inserting: {}", pos.func.dfg.display_value_inst(oob));
+    pos.func
+        .dfg
+        .replace(inst)
+        .store(flags, value, addr, Offset32::new(0));
+}
 
-        let trapnz = pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
-        trace!("  inserting: {}", pos.func.dfg.display_inst(trapnz));
+/// Expand a `heap_addr` instruction according to the definition of the heap.
+pub fn expand_heap_addr(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+    heap: ir::Heap,
+    index: ir::Value,
+    offset: Uimm32,
+    access_size: Uimm8,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
 
-        None
-    };
+    let addr =
+        bounds_check_and_compute_addr(&mut pos, cfg, isa, heap, index, offset.into(), access_size);
 
-    compute_addr(
-        isa,
-        inst,
-        heap,
-        addr_ty,
-        index,
-        offset,
-        pos.func,
-        spectre_oob_comparison,
-    );
+    // Replace the `heap_addr` and its result value with the legalized native
+    // address.
+    let addr_inst = pos.func.dfg.value_def(addr).unwrap_inst();
+    pos.func.dfg.replace_with_aliases(inst, addr_inst);
+    pos.func.layout.remove_inst(inst);
 }
 
-/// Expand a `heap_addr` for a static heap.
-fn static_addr(
+/// Helper used to emit bounds checks (as necessary) and compute the native
+/// address of a heap access.
+///
+/// Returns the `ir::Value` holding the native address of the heap access.
+fn bounds_check_and_compute_addr(
+    pos: &mut FuncCursor,
+    cfg: &mut ControlFlowGraph,
     isa: &dyn TargetIsa,
-    inst: ir::Inst,
     heap: ir::Heap,
+    // Dynamic operand indexing into the heap.
     index: ir::Value,
+    // Static immediate added to the index.
     offset: u32,
+    // Static size of the heap access.
     access_size: u8,
-    bound: u64,
-    guard_size: u64,
-    func: &mut ir::Function,
-    cfg: &mut ControlFlowGraph,
-) {
-    let index_ty = func.dfg.value_type(index);
-    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
-    let mut pos = FuncCursor::new(func).at_inst(inst);
-    pos.use_srcloc(inst);
+) -> ir::Value {
+    let pointer_type = isa.pointer_type();
+    let spectre = isa.flags().enable_heap_access_spectre_mitigation();
+    let offset_and_size = offset_plus_size(offset, access_size);
 
-    // The goal here is to trap if `index + offset + access_size > bound`.
-    //
-    // This first case is a trivial case where we can statically trap.
-    if offset_plus_size(offset, access_size) > bound {
-        // This will simply always trap since `offset >= 0`.
-        let trap = pos.ins().trap(ir::TrapCode::HeapOutOfBounds);
-        trace!("  inserting: {}", pos.func.dfg.display_inst(trap));
-        let iconst = pos.func.dfg.replace(inst).iconst(addr_ty, 0);
-        trace!("  inserting: {}", pos.func.dfg.display_value_inst(iconst));
-
-        // Split the block, as the trap is a terminator instruction.
-        let curr_block = pos.current_block().expect("Cursor is not in a block");
-        let new_block = pos.func.dfg.make_block();
-        pos.insert_block(new_block);
-        cfg.recompute_block(pos.func, curr_block);
-        cfg.recompute_block(pos.func, new_block);
-        return;
-    }
+    let ir::HeapData {
+        base: _,
+        min_size,
+        offset_guard_size: guard_size,
+        style,
+        index_type,
+    } = pos.func.heaps[heap].clone();
 
-    // After the trivial case is done we're now mostly interested in trapping if
-    //
-    //     index + offset + size > bound
-    //
-    // We know `bound - offset - access_size` here is non-negative from the
-    // above comparison, so we can rewrite that as
-    //
-    //     index > bound - offset - size
-    //
-    // Additionally, we add our guard pages (if any) to the right-hand side,
-    // since we can rely on the virtual memory subsystem at runtime to catch
-    // out-of-bound accesses within the range `bound .. bound + guard_size`. So
-    // now we are dealing with
-    //
-    //     index > bound + guard_size - offset - size
+    let index = cast_index_to_pointer_ty(index, index_type, pointer_type, pos);
+
+    // We need to emit code that will trap (or compute an address that will trap
+    // when accessed) if
     //
-    // (Note that `bound + guard_size` cannot overflow for correctly-configured
-    // heaps, as otherwise the heap wouldn't fit in a 64-bit memory space.)
+    //     index + offset + access_size > bound
     //
-    // If we know the right-hand side is greater than or equal to 4GiB - 1, aka
-    // 0xffff_ffff, then with a 32-bit index we're guaranteed:
+    // or if the `index + offset + access_size` addition overflows.
     //
-    //     index <= 0xffff_ffff <= bound + guard_size - offset - access_size
+    // Note that we ultimately want a 64-bit integer (we only target 64-bit
+    // architectures at the moment) and that `offset` is a `u32` and
+    // `access_size` is a `u8`. This means that we can add the latter together
+    // as `u64`s without fear of overflow, and we only have to be concerned with
+    // whether adding in `index` will overflow.
     //
-    // meaning that `index` is always either in bounds or within the guard page
-    // region, neither of which require emitting an explicit bounds check.
-    assert!(
-        bound.checked_add(guard_size).is_some(),
-        "heap's configuration doesn't fit in a 64-bit memory space"
-    );
-    let mut spectre_oob_comparison = None;
-    let index = cast_index_to_pointer_ty(index, index_ty, addr_ty, &mut pos);
-    if index_ty == ir::types::I32
-        && bound + guard_size - offset_plus_size(offset, access_size) >= 0xffff_ffff
-    {
-        // Happy path! No bounds checks necessary!
-    } else {
-        // Since we have to emit explicit bounds checks anyways, ignore the
-        // guard pages and test against the precise limit.
-        let limit = bound - offset_plus_size(offset, access_size);
-        // Here we want to test the condition `index > limit` and if that's true
-        // then this is an out-of-bounds access and needs to trap.
-        let oob = pos
-            .ins()
-            .icmp_imm(IntCC::UnsignedGreaterThan, index, limit as i64);
-        trace!("  inserting: {}", pos.func.dfg.display_value_inst(oob));
-
-        let trapnz = pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
-        trace!("  inserting: {}", pos.func.dfg.display_inst(trapnz));
-
-        if isa.flags().enable_heap_access_spectre_mitigation() {
-            let limit = pos.ins().iconst(addr_ty, limit as i64);
-            trace!("  inserting: {}", pos.func.dfg.display_value_inst(limit));
-            spectre_oob_comparison = Some(SpectreOobComparison {
-                cc: IntCC::UnsignedGreaterThan,
-                lhs: index,
-                rhs: limit,
-            });
+    // Finally, the following right-hand sides of the matches do have a little
+    // bit of duplicated code across them, but I think writing it this way is
+    // worth it for readability and seeing very clearly each of our cases for
+    // different bounds checks and optimizations of those bounds checks. It is
+    // intentionally written in a straightforward case-matching style that will
+    // hopefully make it easy to port to ISLE one day.
+    match style {
+        // ====== Dynamic Memories ======
+        //
+        // 1. First special case for when `offset + access_size == 1`:
+        //
+        //            index + 1 > bound
+        //        ==> index >= bound
+        //
+        //    1.a. When Spectre mitigations are enabled, avoid duplicating
+        //         bounds checks between the mitigations and the regular bounds
+        //         checks.
+        ir::HeapStyle::Dynamic { bound_gv } if offset_and_size == 1 && spectre => {
+            let bound = pos.ins().global_value(pointer_type, bound_gv);
+            compute_addr(
+                isa,
+                pos,
+                heap,
+                pointer_type,
+                index,
+                offset,
+                Some(SpectreOobComparison {
+                    cc: IntCC::UnsignedGreaterThanOrEqual,
+                    lhs: index,
+                    rhs: bound,
+                }),
+            )
+        }
+        //    1.b. Emit explicit `index >= bound` bounds checks.
+        ir::HeapStyle::Dynamic { bound_gv } if offset_and_size == 1 => {
+            let bound = pos.ins().global_value(pointer_type, bound_gv);
+            let oob = pos
+                .ins()
+                .icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound);
+            pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
+            compute_addr(isa, pos, heap, pointer_type, index, offset, None)
         }
-    }
 
-    compute_addr(
-        isa,
-        inst,
-        heap,
-        addr_ty,
-        index,
-        offset,
-        pos.func,
-        spectre_oob_comparison,
-    );
+        // 2. Second special case for when `offset + access_size <= min_size`.
+        //
+        //    We know that `bound >= min_size`, so we can do the following
+        //    comparison, without fear of the right-hand side wrapping around:
+        //
+        //            index + offset + access_size > bound
+        //        ==> index > bound - (offset + access_size)
+        //
+        //    2.a. Dedupe bounds checks with Spectre mitigations.
+        ir::HeapStyle::Dynamic { bound_gv } if offset_and_size <= min_size.into() && spectre => {
+            let bound = pos.ins().global_value(pointer_type, bound_gv);
+            let adjusted_bound = pos.ins().iadd_imm(bound, -(offset_and_size as i64));
+            compute_addr(
+                isa,
+                pos,
+                heap,
+                pointer_type,
+                index,
+                offset,
+                Some(SpectreOobComparison {
+                    cc: IntCC::UnsignedGreaterThan,
+                    lhs: index,
+                    rhs: adjusted_bound,
+                }),
+            )
+        }
+        //    2.b. Emit explicit `index > bound - (offset + access_size)` bounds
+        //         checks.
+        ir::HeapStyle::Dynamic { bound_gv } if offset_and_size <= min_size.into() => {
+            let bound = pos.ins().global_value(pointer_type, bound_gv);
+            let adjusted_bound = pos.ins().iadd_imm(bound, -(offset_and_size as i64));
+            let oob = pos
+                .ins()
+                .icmp(IntCC::UnsignedGreaterThan, index, adjusted_bound);
+            pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
+            compute_addr(isa, pos, heap, pointer_type, index, offset, None)
+        }
+
+        // 3. General case for dynamic memories:
+        //
+        //        index + offset + access_size > bound
+        //
+        //    And we have to handle the overflow case in the left-hand side.
+        //
+        //    3.a. Dedupe bounds checks with Spectre mitigations.
+        ir::HeapStyle::Dynamic { bound_gv } if spectre => {
+            let access_size_val = pos.ins().iconst(pointer_type, offset_and_size as i64);
+            let adjusted_index =
+                pos.ins()
+                    .uadd_overflow_trap(index, access_size_val, ir::TrapCode::HeapOutOfBounds);
+            let bound = pos.ins().global_value(pointer_type, bound_gv);
+            compute_addr(
+                isa,
+                pos,
+                heap,
+                pointer_type,
+                index,
+                offset,
+                Some(SpectreOobComparison {
+                    cc: IntCC::UnsignedGreaterThan,
+                    lhs: adjusted_index,
+                    rhs: bound,
+                }),
+            )
+        }
+        //    3.b. Emit an explicit `index + offset + access_size > bound`
+        //         check.
+        ir::HeapStyle::Dynamic { bound_gv } => {
+            let access_size_val = pos.ins().iconst(pointer_type, offset_and_size as i64);
+            let adjusted_index =
+                pos.ins()
+                    .uadd_overflow_trap(index, access_size_val, ir::TrapCode::HeapOutOfBounds);
+            let bound = pos.ins().global_value(pointer_type, bound_gv);
+            let oob = pos
+                .ins()
+                .icmp(IntCC::UnsignedGreaterThan, adjusted_index, bound);
+            pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
+            compute_addr(isa, pos, heap, pointer_type, index, offset, None)
+        }
+
+        // ====== Static Memories ======
+        //
+        // With static memories we know the size of the heap bound at compile
+        // time.
+        //
+        // 1. First special case: trap immediately if `offset + access_size >
+        //    bound`, since we will end up being out-of-bounds regardless of the
+        //    given `index`.
+        ir::HeapStyle::Static { bound } if offset_and_size > bound.into() => {
+            pos.ins().trap(ir::TrapCode::HeapOutOfBounds);
+
+            // Split the block, as the trap is a terminator instruction.
+            let curr_block = pos.current_block().expect("Cursor is not in a block");
+            let new_block = pos.func.dfg.make_block();
+            pos.insert_block(new_block);
+            cfg.recompute_block(pos.func, curr_block);
+            cfg.recompute_block(pos.func, new_block);
+
+            let null = pos.ins().iconst(pointer_type, 0);
+            return null;
+        }
+
+        // 2. Second special case for when we can completely omit explicit
+        //    bounds checks for 32-bit static memories.
+        //
+        //    First, let's rewrite our comparison to move all of the constants
+        //    to one side:
+        //
+        //            index + offset + access_size > bound
+        //        ==> index > bound - (offset + access_size)
+        //
+        //    We know the subtraction on the right-hand side won't wrap because
+        //    we didn't hit the first special case.
+        //
+        //    Additionally, we add our guard pages (if any) to the right-hand
+        //    side, since we can rely on the virtual memory subsystem at runtime
+        //    to catch out-of-bound accesses within the range `bound .. bound +
+        //    guard_size`. So now we are dealing with
+        //
+        //        index > bound + guard_size - (offset + access_size)
+        //
+        //    Note that `bound + guard_size` cannot overflow for
+        //    correctly-configured heaps, as otherwise the heap wouldn't fit in
+        //    a 64-bit memory space.
+        //
+        //    The complement of our should-this-trap comparison expression is
+        //    the should-this-not-trap comparison expression:
+        //
+        //        index <= bound + guard_size - (offset + access_size)
+        //
+        //    If we know the right-hand side is greater than or equal to
+        //    `u32::MAX`, then
+        //
+        //        index <= u32::MAX <= bound + guard_size - (offset + access_size)
+        //
+        //    This expression is always true when the heap is indexed with
+        //    32-bit integers because `index` cannot be larger than
+        //    `u32::MAX`. This means that `index` is always either in bounds or
+        //    within the guard page region, neither of which require emitting an
+        //    explicit bounds check.
+        ir::HeapStyle::Static { bound }
+            if index_type == ir::types::I32
+                && u64::from(u32::MAX)
+                    <= u64::from(bound) + u64::from(guard_size) - offset_and_size =>
+        {
+            compute_addr(isa, pos, heap, pointer_type, index, offset, None)
+        }
+
+        // 3. General case for static memories.
+        //
+        //    We have to explicitly test whether
+        //
+        //        index > bound - (offset + access_size)
+        //
+        //    and trap if so.
+        //
+        //    Since we have to emit explicit bounds checks, we might as well be
+        //    precise, not rely on the virtual memory subsystem at all, and not
+        //    factor in the guard pages here.
+        //
+        //    3.a. Dedupe the Spectre mitigation and the explicit bounds check.
+        ir::HeapStyle::Static { bound } if spectre => {
+            // NB: this subtraction cannot wrap because we didn't hit the first
+            // special case.
+            let adjusted_bound = u64::from(bound) - offset_and_size;
+            let adjusted_bound = pos.ins().iconst(pointer_type, adjusted_bound as i64);
+            compute_addr(
+                isa,
+                pos,
+                heap,
+                pointer_type,
+                index,
+                offset,
+                Some(SpectreOobComparison {
+                    cc: IntCC::UnsignedGreaterThan,
+                    lhs: index,
+                    rhs: adjusted_bound,
+                }),
+            )
+        }
+        //    3.b. Emit the explicit `index > bound - (offset + access_size)`
+        //         check.
+        ir::HeapStyle::Static { bound } => {
+            // See comment in 3.a. above.
+            let adjusted_bound = u64::from(bound) - offset_and_size;
+            let oob = pos
+                .ins()
+                .icmp_imm(IntCC::UnsignedGreaterThan, index, adjusted_bound as i64);
+            pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
+            compute_addr(isa, pos, heap, pointer_type, index, offset, None)
+        }
+    }
 }
 
 fn cast_index_to_pointer_ty(
     index: ir::Value,
     index_ty: ir::Type,
-    addr_ty: ir::Type,
+    pointer_ty: ir::Type,
     pos: &mut FuncCursor,
 ) -> ir::Value {
-    if index_ty == addr_ty {
+    if index_ty == pointer_ty {
         return index;
     }
     // Note that using 64-bit heaps on a 32-bit host is not currently supported,
     // would require at least a bounds check here to ensure that the truncation
     // from 64-to-32 bits doesn't lose any upper bits. For now though we're
     // mostly interested in the 32-bit-heaps-on-64-bit-hosts cast.
-    assert!(index_ty.bits() < addr_ty.bits());
+    assert!(index_ty.bits() < pointer_ty.bits());
 
     // Convert `index` to `addr_ty`.
-    let extended_index = pos.ins().uextend(addr_ty, index);
+    let extended_index = pos.ins().uextend(pointer_ty, index);
 
     // Add debug value-label alias so that debuginfo can name the extended
     // value as the address
@@ -287,24 +413,22 @@ struct SpectreOobComparison {
     rhs: ir::Value,
 }
 
-/// Emit code for the base address computation of a `heap_addr` instruction.
+/// Emit code for the base address computation of a `heap_addr` instruction,
+/// without any bounds checks (other than optional Spectre mitigations).
 fn compute_addr(
     isa: &dyn TargetIsa,
-    inst: ir::Inst,
+    pos: &mut FuncCursor,
     heap: ir::Heap,
     addr_ty: ir::Type,
     index: ir::Value,
     offset: u32,
-    func: &mut ir::Function,
     // If we are performing Spectre mitigation with conditional selects, the
     // values to compare and the condition code that indicates an out-of bounds
     // condition; on this condition, the conditional move will choose a
     // speculatively safe address (a zero / null pointer) instead.
     spectre_oob_comparison: Option<SpectreOobComparison>,
-) {
-    debug_assert_eq!(func.dfg.value_type(index), addr_ty);
-    let mut pos = FuncCursor::new(func).at_inst(inst);
-    pos.use_srcloc(inst);
+) -> ir::Value {
+    debug_assert_eq!(pos.func.dfg.value_type(index), addr_ty);
 
     // Add the heap base address base
     let base = if isa.flags().enable_pinned_reg() && isa.flags().use_pinned_reg_as_heap_base() {
@@ -339,30 +463,26 @@ fn compute_addr(
         let cmp = pos.ins().icmp(cc, lhs, rhs);
         trace!("  inserting: {}", pos.func.dfg.display_value_inst(cmp));
 
-        let value = pos
-            .func
-            .dfg
-            .replace(inst)
-            .select_spectre_guard(cmp, zero, final_addr);
+        let value = pos.ins().select_spectre_guard(cmp, zero, final_addr);
         trace!("  inserting: {}", pos.func.dfg.display_value_inst(value));
+        value
     } else if offset == 0 {
-        let addr = pos.func.dfg.replace(inst).iadd(base, index);
+        let addr = pos.ins().iadd(base, index);
         trace!("  inserting: {}", pos.func.dfg.display_value_inst(addr));
+        addr
     } else {
         let final_base = pos.ins().iadd(base, index);
         trace!(
             "  inserting: {}",
             pos.func.dfg.display_value_inst(final_base)
         );
-        let addr = pos
-            .func
-            .dfg
-            .replace(inst)
-            .iadd_imm(final_base, offset as i64);
+        let addr = pos.ins().iadd_imm(final_base, offset as i64);
         trace!("  inserting: {}", pos.func.dfg.display_value_inst(addr));
+        addr
     }
 }
 
+#[inline]
 fn offset_plus_size(offset: u32, size: u8) -> u64 {
     // Cannot overflow because we are widening to `u64`.
     offset as u64 + size as u64
diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs
index 0fa815683..775c65be4 100644
--- a/cranelift/codegen/src/legalizer/mod.rs
+++ b/cranelift/codegen/src/legalizer/mod.rs
@@ -26,7 +26,7 @@ mod heap;
 mod table;
 
 use self::globalvalue::expand_global_value;
-use self::heap::expand_heap_addr;
+use self::heap::{expand_heap_addr, expand_heap_load, expand_heap_store};
 use self::table::expand_table_addr;
 
 fn imm_const(pos: &mut FuncCursor, arg: Value, imm: Imm64, is_signed: bool) -> Value {
@@ -78,6 +78,16 @@ pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa:
                     offset,
                     size,
                 } => expand_heap_addr(inst, &mut pos.func, cfg, isa, heap, arg, offset, size),
+                InstructionData::HeapLoad {
+                    opcode: ir::Opcode::HeapLoad,
+                    heap_imm,
+                    arg,
+                } => expand_heap_load(inst, &mut pos.func, cfg, isa, heap_imm, arg),
+                InstructionData::HeapStore {
+                    opcode: ir::Opcode::HeapStore,
+                    heap_imm,
+                    args,
+                } => expand_heap_store(inst, &mut pos.func, cfg, isa, heap_imm, args[0], args[1]),
                 InstructionData::StackLoad {
                     opcode: ir::Opcode::StackLoad,
                     stack_slot,
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index 84b4a7864..b82ce632e 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -500,10 +500,10 @@ pub trait ABIMachineSpec {
     fn gen_epilogue_frame_restore(flags: &settings::Flags) -> SmallInstVec<Self::I>;
 
     /// Generate a probestack call.
-    fn gen_probestack(_frame_size: u32) -> SmallInstVec<Self::I>;
+    fn gen_probestack(insts: &mut SmallInstVec<Self::I>, frame_size: u32);
 
     /// Generate a inline stack probe.
-    fn gen_inline_probestack(_frame_size: u32, _guard_size: u32) -> SmallInstVec<Self::I>;
+    fn gen_inline_probestack(insts: &mut SmallInstVec<Self::I>, frame_size: u32, guard_size: u32);
 
     /// Get all clobbered registers that are callee-saved according to the ABI; the result
     /// contains the registers in a sorted order.
@@ -608,26 +608,42 @@ pub trait ABIMachineSpec {
 pub struct Sig(u32);
 cranelift_entity::entity_impl!(Sig);
 
+impl Sig {
+    fn prev(self) -> Option<Sig> {
+        self.0.checked_sub(1).map(Sig)
+    }
+}
+
 /// ABI information shared between body (callee) and caller.
 #[derive(Clone, Debug)]
 pub struct SigData {
-    /// Argument location starting offset (regs or stack slots). Stack offsets are relative to
+    /// Currently both return values and arguments are stored in a continuous space vector
+    /// in `SigSet::abi_args`.
+    ///
+    /// ```plain
+    ///                  +----------------------------------------------+
+    ///                  | return values                                |
+    ///                  | ...                                          |
+    ///   rets_end   --> +----------------------------------------------+
+    ///                  | arguments                                    |
+    ///                  | ...                                          |
+    ///   args_end   --> +----------------------------------------------+
+    ///
+    /// ```
+    ///
+    /// Note we only store two offsets as rets_end == args_start, and rets_start == prev.args_end.
+    ///
+    /// Argument location ending offset (regs or stack slots). Stack offsets are relative to
     /// SP on entry to function.
     ///
     /// This is a index into the `SigSet::abi_args`.
-    args_start: u32,
-
-    /// Amount of arguments in the stack.
-    args_length: u16,
+    args_end: u32,
 
-    /// Return-value location starting offset. Stack offsets are relative to the return-area
+    /// Return-value location ending offset. Stack offsets are relative to the return-area
     /// pointer.
     ///
     /// This is a index into the `SigSet::abi_args`.
-    rets_start: u32,
-
-    /// Amount of Return value locations in the stack.
-    rets_length: u16,
+    rets_end: u32,
 
     /// Space on stack used to store arguments.
     sized_stack_arg_space: i64,
@@ -643,27 +659,11 @@ pub struct SigData {
 }
 
 impl SigData {
-    /// Get the number of arguments expected.
-    pub fn num_args(&self) -> usize {
-        let num = if self.stack_ret_arg.is_some() {
-            self.args_length - 1
-        } else {
-            self.args_length
-        };
-
-        usize::from(num)
-    }
-
     /// Get total stack space required for arguments.
     pub fn sized_stack_arg_space(&self) -> i64 {
         self.sized_stack_arg_space
     }
 
-    /// Get the number of return values expected.
-    pub fn num_rets(&self) -> usize {
-        usize::from(self.rets_length)
-    }
-
     /// Get total stack space required for return values.
     pub fn sized_stack_ret_space(&self) -> i64 {
         self.sized_stack_ret_space
@@ -806,7 +806,10 @@ impl SigSet {
 
         // Compute args and retvals from signature. Handle retvals first,
         // because we may need to add a return-area arg to the args.
-        let rets_start = u32::try_from(self.abi_args.len()).unwrap();
+
+        // NOTE: We rely on the order of the args (rets -> args) inserted to compute the offsets in
+        // `SigSet::args()` and `SigSet::rets()`. Therefore, we cannot change the two
+        // compute_arg_locs order.
         let (sized_stack_ret_space, _) = M::compute_arg_locs(
             sig.call_conv,
             flags,
@@ -818,7 +821,6 @@ impl SigSet {
         let rets_end = u32::try_from(self.abi_args.len()).unwrap();
 
         let need_stack_return_area = sized_stack_ret_space > 0;
-        let args_start = u32::try_from(self.abi_args.len()).unwrap();
         let (sized_stack_arg_space, stack_ret_arg) = M::compute_arg_locs(
             sig.call_conv,
             flags,
@@ -829,17 +831,12 @@ impl SigSet {
         )?;
         let args_end = u32::try_from(self.abi_args.len()).unwrap();
 
-        let args_length = u16::try_from(args_end - args_start).unwrap();
-        let rets_length = u16::try_from(rets_end - rets_start).unwrap();
-
         trace!(
-            "ABISig: sig {:?} => args start = {} args len = {} rets start = {} rets len = {}
+            "ABISig: sig {:?} => args end = {} rets end = {}
              arg stack = {} ret stack = {} stack_ret_arg = {:?}",
             sig,
-            args_start,
-            args_length,
-            rets_start,
-            rets_length,
+            args_end,
+            rets_end,
             sized_stack_arg_space,
             sized_stack_ret_space,
             need_stack_return_area,
@@ -847,10 +844,8 @@ impl SigSet {
 
         let stack_ret_arg = stack_ret_arg.map(|s| u16::try_from(s).unwrap());
         Ok(SigData {
-            args_start,
-            args_length,
-            rets_start,
-            rets_length,
+            args_end,
+            rets_end,
             sized_stack_arg_space,
             sized_stack_ret_space,
             stack_ret_arg,
@@ -861,8 +856,9 @@ impl SigSet {
     /// Get this signature's ABI arguments.
     pub fn args(&self, sig: Sig) -> &[ABIArg] {
         let sig_data = &self.sigs[sig];
-        let start = usize::try_from(sig_data.args_start).unwrap();
-        let end = usize::try_from(sig_data.args_start + u32::from(sig_data.args_length)).unwrap();
+        // Please see comments in `SigSet::from_func_sig` of how we store the offsets.
+        let start = usize::try_from(sig_data.rets_end).unwrap();
+        let end = usize::try_from(sig_data.args_end).unwrap();
         &self.abi_args[start..end]
     }
 
@@ -885,8 +881,9 @@ impl SigSet {
     /// Get this signature's ABI returns.
     pub fn rets(&self, sig: Sig) -> &[ABIArg] {
         let sig_data = &self.sigs[sig];
-        let start = usize::try_from(sig_data.rets_start).unwrap();
-        let end = usize::try_from(sig_data.rets_start + u32::from(sig_data.rets_length)).unwrap();
+        // Please see comments in `SigSet::from_func_sig` of how we store the offsets.
+        let start = usize::try_from(sig.prev().map_or(0, |prev| self.sigs[prev].args_end)).unwrap();
+        let end = usize::try_from(sig_data.rets_end).unwrap();
         &self.abi_args[start..end]
     }
 
@@ -927,6 +924,21 @@ impl SigSet {
 
         clobbers
     }
+
+    /// Get the number of arguments expected.
+    pub fn num_args(&self, sig: Sig) -> usize {
+        let len = self.args(sig).len();
+        if self.sigs[sig].stack_ret_arg.is_some() {
+            len - 1
+        } else {
+            len
+        }
+    }
+
+    /// Get the number of return values expected.
+    pub fn num_rets(&self, sig: Sig) -> usize {
+        self.rets(sig).len()
+    }
 }
 
 // NB: we do _not_ implement `IndexMut` because these signatures are
@@ -1826,14 +1838,13 @@ impl<M: ABIMachineSpec> Callee<M> {
                 .map_or(false, |min_frame| total_stacksize >= min_frame);
 
             if needs_probestack {
-                insts.extend(
-                    if self.flags.probestack_strategy() == ProbestackStrategy::Inline {
+                match self.flags.probestack_strategy() {
+                    ProbestackStrategy::Inline => {
                         let guard_size = 1 << self.flags.probestack_size_log2();
-                        M::gen_inline_probestack(total_stacksize, guard_size)
-                    } else {
-                        M::gen_probestack(total_stacksize)
-                    },
-                );
+                        M::gen_inline_probestack(&mut insts, total_stacksize, guard_size)
+                    }
+                    ProbestackStrategy::Outline => M::gen_probestack(&mut insts, total_stacksize),
+                }
             }
         }
 
@@ -2097,12 +2108,7 @@ fn adjust_stack_and_nominal_sp<M: ABIMachineSpec>(ctx: &mut Lower<M::I>, off: i3
 impl<M: ABIMachineSpec> Caller<M> {
     /// Get the number of arguments expected.
     pub fn num_args(&self, sigs: &SigSet) -> usize {
-        let len = sigs.args(self.sig).len();
-        if sigs[self.sig].stack_ret_arg.is_some() {
-            len - 1
-        } else {
-            len
-        }
+        sigs.num_args(self.sig)
     }
 
     /// Emit code to pre-adjust the stack, prior to argument copies and call.
@@ -2402,6 +2408,6 @@ mod tests {
     fn sig_data_size() {
         // The size of `SigData` is performance sensitive, so make sure
         // we don't regress it unintentionally.
-        assert_eq!(std::mem::size_of::<SigData>(), 40);
+        assert_eq!(std::mem::size_of::<SigData>(), 32);
     }
 }
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index a7a22ca1a..ab1bc3bb7 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -432,7 +432,7 @@ macro_rules! isle_lower_prelude_methods {
         }
 
         fn abi_num_args(&mut self, abi: &Sig) -> usize {
-            self.lower_ctx.sigs()[*abi].num_args()
+            self.lower_ctx.sigs().num_args(*abi)
         }
 
         fn abi_get_arg(&mut self, abi: &Sig, idx: usize) -> ABIArg {
@@ -440,7 +440,7 @@ macro_rules! isle_lower_prelude_methods {
         }
 
         fn abi_num_rets(&mut self, abi: &Sig) -> usize {
-            self.lower_ctx.sigs()[*abi].num_rets()
+            self.lower_ctx.sigs().num_rets(*abi)
         }
 
         fn abi_get_ret(&mut self, abi: &Sig, idx: usize) -> ABIArg {
@@ -681,7 +681,7 @@ macro_rules! isle_prelude_method_helpers {
         ) -> InstOutput {
             caller.emit_stack_pre_adjust(self.lower_ctx);
 
-            let num_args = self.lower_ctx.sigs()[abi].num_args();
+            let num_args = self.lower_ctx.sigs().num_args(abi);
 
             assert_eq!(
                 inputs.len(&self.lower_ctx.dfg().value_lists) - off,
@@ -710,9 +710,9 @@ macro_rules! isle_prelude_method_helpers {
             let mut retval_insts: crate::machinst::abi::SmallInstVec<_> = smallvec::smallvec![];
             // We take the *last* `num_rets` returns of the sig:
             // this skips a StructReturn, if any, that is present.
-            let sigdata = &self.lower_ctx.sigs()[abi];
-            debug_assert!(num_rets <= sigdata.num_rets());
-            for i in (sigdata.num_rets() - num_rets)..sigdata.num_rets() {
+            let sigdata_num_rets = self.lower_ctx.sigs().num_rets(abi);
+            debug_assert!(num_rets <= sigdata_num_rets);
+            for i in (sigdata_num_rets - num_rets)..sigdata_num_rets {
                 // Borrow `sigdata` again so we don't hold a `self`
                 // borrow across the `&mut self` arg to
                 // `abi_arg_slot_regs()` below.
diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs
index 0a214dfbd..4418f990a 100644
--- a/cranelift/codegen/src/settings.rs
+++ b/cranelift/codegen/src/settings.rs
@@ -545,7 +545,7 @@ enable_llvm_abi_extensions = false
 unwind_info = true
 preserve_frame_pointers = false
 machine_code_cfg_info = false
-enable_probestack = true
+enable_probestack = false
 probestack_func_adjusts_sp = false
 enable_jump_tables = true
 enable_heap_access_spectre_mitigation = true
diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs
index 04b27cc3d..d7df6df08 100644
--- a/cranelift/fuzzgen/src/lib.rs
+++ b/cranelift/fuzzgen/src/lib.rs
@@ -266,10 +266,9 @@ where
             builder.set(flag_name, value.as_str())?;
         }
 
-        // Optionally test inline stackprobes on x86
-        // TODO: inline stack probes are not available on AArch64
+        // Optionally test inline stackprobes on supported platforms
         // TODO: Test outlined stack probes.
-        if cfg!(target_arch = "x86_64") && bool::arbitrary(self.u)? {
+        if supports_inline_probestack() && bool::arbitrary(self.u)? {
             builder.enable("enable_probestack")?;
             builder.set("probestack_strategy", "inline")?;
 
@@ -300,7 +299,11 @@ where
         // into compilation anywhere, we leave it on unconditionally to make sure the generation doesn't panic.
         builder.enable("machine_code_cfg_info")?;
 
-        Ok(Flags::new(builder))
+        return Ok(Flags::new(builder));
+
+        fn supports_inline_probestack() -> bool {
+            cfg!(target_arch = "x86_64") || cfg!(target_arch = "aarch64")
+        }
     }
 
     pub fn generate_test(mut self) -> Result<TestCase> {
diff --git a/cranelift/isle/isle/src/error.rs b/cranelift/isle/isle/src/error.rs
index d919e36d1..999821edd 100644
--- a/cranelift/isle/isle/src/error.rs
+++ b/cranelift/isle/isle/src/error.rs
@@ -24,6 +24,9 @@ impl std::fmt::Debug for Errors {
                 Error::TypeError { msg, .. } => format!("type error: {}", msg),
                 Error::UnreachableError { msg, .. } => format!("unreachable rule: {}", msg),
                 Error::OverlapError { msg, .. } => format!("overlap error: {}", msg),
+                Error::ShadowedError { .. } => {
+                    format!("more general higher-priority rule shadows other rules")
+                }
             };
 
             let labels = match e {
@@ -44,6 +47,16 @@ impl std::fmt::Debug for Errors {
                     );
                     labels
                 }
+
+                Error::ShadowedError { shadowed, mask } => {
+                    let mut labels = vec![Label::primary(mask.from.file, mask)];
+                    labels.extend(
+                        shadowed
+                            .iter()
+                            .map(|span| Label::secondary(span.from.file, span)),
+                    );
+                    labels
+                }
             };
 
             let mut sources = Vec::new();
@@ -114,6 +127,15 @@ pub enum Error {
         /// wildcard case).
         rules: Vec<Span>,
     },
+
+    /// The rules can never match because another rule will always match first.
+    ShadowedError {
+        /// The locations of the unmatchable rules.
+        shadowed: Vec<Span>,
+
+        /// The location of the rule that shadows them.
+        mask: Span,
+    },
 }
 
 impl Errors {
diff --git a/cranelift/isle/isle/src/lib.rs b/cranelift/isle/isle/src/lib.rs
index 1248bcbf4..6d27f2c34 100644
--- a/cranelift/isle/isle/src/lib.rs
+++ b/cranelift/isle/isle/src/lib.rs
@@ -197,6 +197,20 @@ impl<T: Copy + std::fmt::Debug + Eq + Hash> DisjointSets<T> {
     pub fn is_empty(&self) -> bool {
         self.parent.is_empty()
     }
+
+    /// Returns the total number of elements in all sets. This method takes constant time.
+    ///
+    /// ```
+    /// let mut sets = cranelift_isle::DisjointSets::default();
+    /// sets.merge(1, 2);
+    /// assert_eq!(sets.len(), 2);
+    /// sets.merge(3, 4);
+    /// sets.merge(3, 5);
+    /// assert_eq!(sets.len(), 5);
+    /// ```
+    pub fn len(&self) -> usize {
+        self.parent.len()
+    }
 }
 
 pub mod ast;
diff --git a/cranelift/isle/isle/src/overlap.rs b/cranelift/isle/isle/src/overlap.rs
index 4411de8c8..d127fc7f9 100644
--- a/cranelift/isle/isle/src/overlap.rs
+++ b/cranelift/isle/isle/src/overlap.rs
@@ -29,6 +29,9 @@ pub fn check(tyenv: &TypeEnv, termenv: &TermEnv) -> Result<(), error::Errors> {
 struct Errors {
     /// Edges between rules indicating overlap.
     nodes: HashMap<Pos, HashSet<Pos>>,
+    /// For each (mask, shadowed) pair, every rule in `shadowed` is unmatchable because `mask` will
+    /// always match first.
+    shadowed: HashMap<Pos, Vec<Pos>>,
 }
 
 impl Errors {
@@ -66,19 +69,37 @@ impl Errors {
             });
         }
 
+        errors.extend(
+            self.shadowed
+                .into_iter()
+                .map(|(mask, shadowed)| Error::ShadowedError {
+                    shadowed: shadowed.into_iter().map(Span::new_single).collect(),
+                    mask: Span::new_single(mask),
+                }),
+        );
+
         errors.sort_by_key(|err| match err {
-            Error::OverlapError { rules, .. } => rules.first().unwrap().from,
+            Error::ShadowedError { mask, .. } => mask.from,
+            Error::OverlapError { rules, .. } => rules[0].from,
             _ => Pos::default(),
         });
         errors
     }
 
     fn check_pair(&mut self, a: &trie_again::Rule, b: &trie_again::Rule) {
-        if let trie_again::Overlap::Yes { .. } = a.may_overlap(b) {
+        if let trie_again::Overlap::Yes { subset } = a.may_overlap(b) {
             if a.prio == b.prio {
                 // edges are undirected
                 self.nodes.entry(a.pos).or_default().insert(b.pos);
                 self.nodes.entry(b.pos).or_default().insert(a.pos);
+            } else if subset {
+                // One rule's constraints are a subset of the other's, or they're equal.
+                // This is fine as long as the higher-priority rule has more constraints.
+                let (lo, hi) = if a.prio < b.prio { (a, b) } else { (b, a) };
+                if hi.total_constraints() <= lo.total_constraints() {
+                    // Otherwise, the lower-priority rule can never match.
+                    self.shadowed.entry(hi.pos).or_default().push(lo.pos);
+                }
             }
         }
     }
diff --git a/cranelift/isle/isle/src/trie_again.rs b/cranelift/isle/isle/src/trie_again.rs
index ff046ece3..4b02159bc 100644
--- a/cranelift/isle/isle/src/trie_again.rs
+++ b/cranelift/isle/isle/src/trie_again.rs
@@ -151,8 +151,8 @@ pub enum Overlap {
     Yes {
         /// True if every input accepted by one rule is also accepted by the other. This does not
         /// indicate which rule is more general and in fact the rules could match exactly the same
-        /// set of inputs. You can work out which by comparing the number of constraints in both
-        /// rules: The more general rule has fewer constraints.
+        /// set of inputs. You can work out which by comparing `total_constraints()` in both rules:
+        /// The more general rule has fewer constraints.
         subset: bool,
     },
 }
@@ -213,7 +213,10 @@ impl Rule {
         // that they can cause rules to not overlap. However, because we don't have a concrete
         // pattern to compare, the analysis to prove that is complicated. For now, we approximate
         // the result. If either rule has nonlinear constraints, conservatively report that neither
-        // is a subset of the other.
+        // is a subset of the other. Note that this does not disagree with the doc comment for
+        // `Overlap::Yes { subset }` which says to use `total_constraints` to disambiguate, since if
+        // we return `subset: true` here, `equals` is empty for both rules, so `total_constraints()`
+        // equals `constraints.len()`.
         let mut subset = small.equals.is_empty() && big.equals.is_empty();
 
         for (binding, a) in small.constraints.iter() {
@@ -234,6 +237,14 @@ impl Rule {
         Overlap::Yes { subset }
     }
 
+    /// Returns the total number of binding sites which this rule constrains, with either a concrete
+    /// pattern or an equality constraint.
+    pub fn total_constraints(&self) -> usize {
+        // Because of `normalize_equivalence_classes`, these two sets don't overlap, so the size of
+        // the union is the sum of their sizes.
+        self.constraints.len() + self.equals.len()
+    }
+
     /// Returns the constraint that the given binding site must satisfy for this rule to match, if
     /// there is one.
     pub fn get_constraint(&self, source: BindingId) -> Option<Constraint> {
diff --git a/crates/jit-icache-coherence/Cargo.toml b/crates/jit-icache-coherence/Cargo.toml
index a2b8e3d25..c1f6ee1fe 100644
--- a/crates/jit-icache-coherence/Cargo.toml
+++ b/crates/jit-icache-coherence/Cargo.toml
@@ -19,5 +19,5 @@ features = [
     "Win32_System_Diagnostics_Debug",
 ]
 
-[target.'cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))'.dependencies.libc]
+[target.'cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd", target_os = "android"))'.dependencies.libc]
 version = "0.2.42"
diff --git a/crates/jit-icache-coherence/src/libc.rs b/crates/jit-icache-coherence/src/libc.rs
index 6ea9cea08..001e9251f 100644
--- a/crates/jit-icache-coherence/src/libc.rs
+++ b/crates/jit-icache-coherence/src/libc.rs
@@ -1,86 +1,106 @@
-#![allow(unused)]
-
-use libc::{syscall, EINVAL, EPERM};
 use std::ffi::c_void;
-use std::io::{Error, Result};
+use std::io::Result;
 
-const MEMBARRIER_CMD_GLOBAL: libc::c_int = 1;
-const MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 32;
-const MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 64;
+#[cfg(all(
+    target_arch = "aarch64",
+    any(target_os = "linux", target_os = "android")
+))]
+mod details {
+    use super::*;
+    use libc::{syscall, EINVAL, EPERM};
+    use std::io::Error;
 
-/// See docs on [crate::pipeline_flush_mt] for a description of what this function is trying to do.
-#[inline]
-pub(crate) fn pipeline_flush_mt() -> Result<()> {
-    // Ensure that no processor has fetched a stale instruction stream.
-    //
-    // On AArch64 we try to do this by executing a "broadcast" `ISB` which is not something that the
-    // architecture provides us but we can emulate it using the membarrier kernel interface.
-    //
-    // This behaviour was documented in a patch, however it seems that it hasn't been upstreamed yet
-    // Nevertheless it clearly explains the guarantees that the Linux kernel provides us regarding the
-    // membarrier interface, and how to use it for JIT contexts.
-    // https://lkml.kernel.org/lkml/07a8b963002cb955b7516e61bad19514a3acaa82.1623813516.git.luto@kernel.org/
-    //
-    // I couldn't find the follow up for that patch but there doesn't seem to be disagreement about
-    // that specific part in the replies.
-    // TODO: Check if the kernel has updated the membarrier documentation
-    //
-    // See the following issues for more info:
-    //  * https://github.com/bytecodealliance/wasmtime/pull/3426
-    //  * https://github.com/bytecodealliance/wasmtime/pull/4997
-    //
-    // TODO: x86 and s390x have coherent caches so they don't need this, but RISCV does not
-    // guarantee that, so we may need to do something similar for it. However as noted in the above
-    // kernel patch the SYNC_CORE membarrier has different guarantees on each architecture
-    // so we need follow up and check what it provides us.
-    // See: https://github.com/bytecodealliance/wasmtime/issues/5033
-    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
-    match membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE) {
-        Ok(_) => {}
+    const MEMBARRIER_CMD_GLOBAL: libc::c_int = 1;
+    const MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 32;
+    const MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 64;
 
-        // EPERM happens if the calling process hasn't yet called the register membarrier.
-        // We can call the register membarrier now, and then retry the actual membarrier,
+    /// See docs on [crate::pipeline_flush_mt] for a description of what this function is trying to do.
+    #[inline]
+    pub(crate) fn pipeline_flush_mt() -> Result<()> {
+        // Ensure that no processor has fetched a stale instruction stream.
         //
-        // This does have some overhead since on the first time we call this function we
-        // actually execute three membarriers, but this only happens once per process and only
-        // one slow membarrier is actually executed (The last one, which actually generates an IPI).
-        Err(e) if e.raw_os_error().unwrap() == EPERM => {
-            membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)?;
-            membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE)?;
-        }
+        // On AArch64 we try to do this by executing a "broadcast" `ISB` which is not something
+        // that the architecture provides us but we can emulate it using the membarrier kernel
+        // interface.
+        //
+        // This behaviour was documented in a patch, however it seems that it hasn't been
+        // upstreamed yet Nevertheless it clearly explains the guarantees that the Linux kernel
+        // provides us regarding the membarrier interface, and how to use it for JIT contexts.
+        // https://lkml.kernel.org/lkml/07a8b963002cb955b7516e61bad19514a3acaa82.1623813516.git.luto@kernel.org/
+        //
+        // I couldn't find the follow up for that patch but there doesn't seem to be disagreement
+        // about that specific part in the replies.
+        // TODO: Check if the kernel has updated the membarrier documentation
+        //
+        // See the following issues for more info:
+        //  * https://github.com/bytecodealliance/wasmtime/pull/3426
+        //  * https://github.com/bytecodealliance/wasmtime/pull/4997
+        //
+        // TODO: x86 and s390x have coherent caches so they don't need this, but RISCV does not
+        // guarantee that, so we may need to do something similar for it. However as noted in the
+        // above kernel patch the SYNC_CORE membarrier has different guarantees on each
+        // architecture so we need follow up and check what it provides us.
+        // See: https://github.com/bytecodealliance/wasmtime/issues/5033
+        match membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE) {
+            Ok(_) => {}
+
+            // EPERM happens if the calling process hasn't yet called the register membarrier.
+            // We can call the register membarrier now, and then retry the actual membarrier,
+            //
+            // This does have some overhead since on the first time we call this function we
+            // actually execute three membarriers, but this only happens once per process and only
+            // one slow membarrier is actually executed (The last one, which actually generates an
+            // IPI).
+            Err(e) if e.raw_os_error().unwrap() == EPERM => {
+                membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)?;
+                membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE)?;
+            }
 
-        // On kernels older than 4.16 the above syscall does not exist, so we can
-        // fallback to MEMBARRIER_CMD_GLOBAL which is an alias for MEMBARRIER_CMD_SHARED
-        // that has existed since 4.3. GLOBAL is a lot slower, but allows us to have
-        // compatibility with older kernels.
-        Err(e) if e.raw_os_error().unwrap() == EINVAL => {
-            membarrier(MEMBARRIER_CMD_GLOBAL)?;
+            // On kernels older than 4.16 the above syscall does not exist, so we can
+            // fallback to MEMBARRIER_CMD_GLOBAL which is an alias for MEMBARRIER_CMD_SHARED
+            // that has existed since 4.3. GLOBAL is a lot slower, but allows us to have
+            // compatibility with older kernels.
+            Err(e) if e.raw_os_error().unwrap() == EINVAL => {
+                membarrier(MEMBARRIER_CMD_GLOBAL)?;
+            }
+
+            // In any other case we got an actual error, so lets propagate that up
+            e => e?,
         }
 
-        // In any other case we got an actual error, so lets propagate that up
-        e => e?,
+        Ok(())
     }
 
-    Ok(())
+    fn membarrier(barrier: libc::c_int) -> Result<()> {
+        let flags: libc::c_int = 0;
+        let res = unsafe { syscall(libc::SYS_membarrier, barrier, flags) };
+        if res == 0 {
+            Ok(())
+        } else {
+            Err(Error::last_os_error())
+        }
+    }
 }
 
-#[cfg(target_os = "linux")]
-fn membarrier(barrier: libc::c_int) -> Result<()> {
-    let flags: libc::c_int = 0;
-    let res = unsafe { syscall(libc::SYS_membarrier, barrier, flags) };
-    if res == 0 {
+#[cfg(not(all(
+    target_arch = "aarch64",
+    any(target_os = "linux", target_os = "android")
+)))]
+mod details {
+    pub(crate) fn pipeline_flush_mt() -> std::io::Result<()> {
         Ok(())
-    } else {
-        Err(Error::last_os_error())
     }
 }
 
+pub(crate) use details::*;
+
 /// See docs on [crate::clear_cache] for a description of what this function is trying to do.
 #[inline]
 pub(crate) fn clear_cache(_ptr: *const c_void, _len: usize) -> Result<()> {
-    // TODO: On AArch64 we currently rely on the `mprotect` call that switches the memory from W+R to R+X
-    // to do this for us, however that is an implementation detail and should not be relied upon
-    // We should call some implementation of `clear_cache` here
+    // TODO: On AArch64 we currently rely on the `mprotect` call that switches the memory from W+R
+    // to R+X to do this for us, however that is an implementation detail and should not be relied
+    // upon.
+    // We should call some implementation of `clear_cache` here.
     //
     // See: https://github.com/bytecodealliance/wasmtime/issues/3310
 
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index cc345bc7f..e8321baf7 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -7,6 +7,7 @@ use std::fmt;
 #[cfg(feature = "cache")]
 use std::path::Path;
 use std::sync::Arc;
+use target_lexicon::Architecture;
 use wasmparser::WasmFeatures;
 #[cfg(feature = "cache")]
 use wasmtime_cache::CacheConfig;
@@ -191,6 +192,7 @@ impl Config {
             ret.cranelift_debug_verifier(false);
             ret.cranelift_opt_level(OptLevel::Speed);
         }
+
         ret.wasm_reference_types(true);
         ret.wasm_multi_value(true);
         ret.wasm_bulk_memory(true);
@@ -1472,19 +1474,35 @@ impl Config {
         let mut compiler = match self.compiler_config.strategy {
             Strategy::Auto | Strategy::Cranelift => wasmtime_cranelift::builder(),
         };
+
         if let Some(target) = &self.compiler_config.target {
             compiler.target(target.clone())?;
         }
 
+        // If probestack is enabled for a target, Wasmtime will always use the
+        // inline strategy which doesn't require us to define a `__probestack`
+        // function or similar.
+        self.compiler_config
+            .settings
+            .insert("probestack_strategy".into(), "inline".into());
+
+        let host = target_lexicon::Triple::host();
+        let target = self.compiler_config.target.as_ref().unwrap_or(&host);
+
+        // On supported targets, we enable stack probing by default.
+        // This is required on Windows because of the way Windows
+        // commits its stacks, but it's also a good idea on other
+        // platforms to ensure guard pages are hit for large frame
+        // sizes.
+        if probestack_supported(target.architecture) {
+            self.compiler_config
+                .flags
+                .insert("enable_probestack".into());
+        }
+
         if self.native_unwind_info ||
-            // Windows always needs unwind info, since it is part of the ABI.
-            self
-                .compiler_config
-                .target
-                .as_ref()
-                .map_or(cfg!(target_os = "windows"), |target| {
-                    target.operating_system == target_lexicon::OperatingSystem::Windows
-                })
+             // Windows always needs unwind info, since it is part of the ABI.
+             target.operating_system == target_lexicon::OperatingSystem::Windows
         {
             if !self
                 .compiler_config
@@ -1881,3 +1899,10 @@ impl PoolingAllocationConfig {
         self
     }
 }
+
+pub(crate) fn probestack_supported(arch: Architecture) -> bool {
+    matches!(
+        arch,
+        Architecture::X86_64 | Architecture::Aarch64(_) | Architecture::Riscv64(_)
+    )
+}
diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs
index 51cbeedae..9c4a67f25 100644
--- a/crates/wasmtime/src/engine.rs
+++ b/crates/wasmtime/src/engine.rs
@@ -359,6 +359,7 @@ impl Engine {
         flag: &str,
         value: &FlagValue,
     ) -> Result<(), String> {
+        let target = self.target();
         let ok = match flag {
             // These settings must all have be enabled, since their value
             // can affect the way the generated code performs or behaves at
@@ -366,14 +367,14 @@ impl Engine {
             "avoid_div_traps" => *value == FlagValue::Bool(true),
             "libcall_call_conv" => *value == FlagValue::Enum("isa_default".into()),
             "preserve_frame_pointers" => *value == FlagValue::Bool(true),
+            "enable_probestack" => *value == FlagValue::Bool(crate::config::probestack_supported(target.architecture)),
+            "probestack_strategy" => *value == FlagValue::Enum("inline".into()),
 
             // Features wasmtime doesn't use should all be disabled, since
             // otherwise if they are enabled it could change the behavior of
             // generated code.
             "enable_llvm_abi_extensions" => *value == FlagValue::Bool(false),
             "enable_pinned_reg" => *value == FlagValue::Bool(false),
-            "enable_probestack" => *value == FlagValue::Bool(false),
-            "probestack_strategy" => *value == FlagValue::Enum("outline".into()),
             "use_colocated_libcalls" => *value == FlagValue::Bool(false),
             "use_pinned_reg_as_heap_base" => *value == FlagValue::Bool(false),
 
@@ -389,7 +390,7 @@ impl Engine {
 
             // Windows requires unwind info as part of its ABI.
             "unwind_info" => {
-                if self.target().operating_system == target_lexicon::OperatingSystem::Windows {
+                if target.operating_system == target_lexicon::OperatingSystem::Windows {
                     *value == FlagValue::Bool(true)
                 } else {
                     return Ok(())
@@ -599,7 +600,7 @@ impl Engine {
         self.load_code(MmapVec::from_slice(bytes)?, expected)
     }
 
-    /// Like `load_code_bytes`, but crates a mmap from a file on disk.
+    /// Like `load_code_bytes`, but creates a mmap from a file on disk.
     pub(crate) fn load_code_file(
         &self,
         path: &Path,
@@ -613,7 +614,7 @@ impl Engine {
         )
     }
 
-    fn load_code(&self, mmap: MmapVec, expected: ObjectKind) -> Result<Arc<CodeMemory>> {
+    pub(crate) fn load_code(&self, mmap: MmapVec, expected: ObjectKind) -> Result<Arc<CodeMemory>> {
         serialization::check_compatible(self, &mmap, expected)?;
         let mut code = CodeMemory::new(mmap)?;
         code.publish()?;
diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs
index 784700400..97efc56a0 100644
--- a/crates/wasmtime/src/module.rs
+++ b/crates/wasmtime/src/module.rs
@@ -332,8 +332,41 @@ impl Module {
         }
     }
 
-    /// Compiles a binary-encoded WebAssembly module to an artifact usable by
-    /// Wasmtime.
+    /// Creates a new WebAssembly `Module` from the contents of the given `file`
+    /// on disk, but with assumptions that the file is from a trusted source.
+    /// The file should be a binary- or text-format WebAssembly module, or a
+    /// precompiled artifact generated by the same version of Wasmtime.
+    ///
+    /// # Unsafety
+    ///
+    /// All of the reasons that [`deserialize`] is `unsafe` apply to this
+    /// function as well. Arbitrary data loaded from a file may trick Wasmtime
+    /// into arbitrary code execution since the contents of the file are not
+    /// validated to be a valid precompiled module.
+    ///
+    /// [`deserialize`]: Module::deserialize
+    ///
+    /// Additionally though this function is also `unsafe` because the file
+    /// referenced must remain unchanged and a valid precompiled module for the
+    /// entire lifetime of the [`Module`] returned. Any changes to the file on
+    /// disk may change future instantiations of the module to be incorrect.
+    /// This is because the file is mapped into memory and lazily loaded pages
+    /// reflect the current state of the file, not necessarily the origianl
+    /// state of the file.
+    #[cfg(compiler)]
+    #[cfg_attr(nightlydoc, doc(cfg(feature = "cranelift")))] // see build.rs
+    pub unsafe fn from_trusted_file(engine: &Engine, file: impl AsRef<Path>) -> Result<Module> {
+        let mmap = MmapVec::from_file(file.as_ref())?;
+        if &mmap[0..4] == b"\x7fELF" {
+            let code = engine.load_code(mmap, ObjectKind::Module)?;
+            return Module::from_parts(engine, code, None);
+        }
+
+        Module::new(engine, &*mmap)
+    }
+
+    /// Converts an input binary-encoded WebAssembly module to compilation
+    /// artifacts and type information.
     ///
     /// This is where compilation actually happens of WebAssembly modules and
     /// translation/parsing/validation of the binary input occurs. The binary
diff --git a/src/commands/run.rs b/src/commands/run.rs
index 4dc9732d5..9b2ed8268 100644
--- a/src/commands/run.rs
+++ b/src/commands/run.rs
@@ -3,8 +3,6 @@
 use anyhow::{anyhow, bail, Context as _, Result};
 use clap::Parser;
 use once_cell::sync::Lazy;
-use std::fs::File;
-use std::io::Read;
 use std::thread;
 use std::time::Duration;
 use std::{
@@ -29,6 +27,8 @@ fn parse_module(s: &OsStr) -> anyhow::Result<PathBuf> {
         Some("help") | Some("config") | Some("run") | Some("wast") | Some("compile") => {
             bail!("module name cannot be the same as a subcommand")
         }
+        #[cfg(unix)]
+        Some("-") => Ok(PathBuf::from("/dev/stdin")),
         _ => Ok(s.into()),
     }
 }
@@ -423,27 +423,12 @@ impl RunCommand {
     }
 
     fn load_module(&self, engine: &Engine, path: &Path) -> Result<Module> {
-        // Peek at the first few bytes of the file to figure out if this is
-        // something we can pass off to `deserialize_file` which is fastest if
-        // we don't actually read the whole file into memory. Note that this
-        // behavior is disabled by default, though, because it's not safe to
-        // pass arbitrary user input to this command with `--allow-precompiled`
-        let mut file =
-            File::open(path).with_context(|| format!("failed to open: {}", path.display()))?;
-        let mut magic = [0; 4];
-        if let Ok(()) = file.read_exact(&mut magic) {
-            if &magic == b"\x7fELF" {
-                if self.allow_precompiled {
-                    return unsafe { Module::deserialize_file(engine, path) };
-                }
-                bail!(
-                    "cannot load precompiled module `{}` unless --allow-precompiled is passed",
-                    path.display()
-                )
-            }
+        if self.allow_precompiled {
+            unsafe { Module::from_trusted_file(engine, path) }
+        } else {
+            Module::from_file(engine, path)
+                .context("if you're trying to run a precompiled module, pass --allow-precompiled")
         }
-
-        Module::from_file(engine, path)
     }
 }
 


Test patch:
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
index 8e06737f4..b5e926b01 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
@@ -1789,8 +1789,8 @@ fn test_aarch64_binemit() {
             mem: AMode::FPOffset { off: 32768, ty: I8 },
             flags: MemFlags::trusted(),
         },
-        "100090D2B063308B010240F9",
-        "movz x16, #32768 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
+        "100090D2A1EB70F8",
+        "movz x16, #32768 ; ldr x1, [fp, x16, SXTX]",
     ));
     insns.push((
         Inst::ULoad64 {
@@ -1801,8 +1801,8 @@ fn test_aarch64_binemit() {
             },
             flags: MemFlags::trusted(),
         },
-        "F0FF8F92B063308B010240F9",
-        "movn x16, #32767 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
+        "F0FF8F92A1EB70F8",
+        "movn x16, #32767 ; ldr x1, [fp, x16, SXTX]",
     ));
     insns.push((
         Inst::ULoad64 {
@@ -1813,8 +1813,8 @@ fn test_aarch64_binemit() {
             }, // 2^20
             flags: MemFlags::trusted(),
         },
-        "1002A0D2B063308B010240F9",
-        "movz x16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
+        "1002A0D2A1EB70F8",
+        "movz x16, #16, LSL #16 ; ldr x1, [fp, x16, SXTX]",
     ));
     insns.push((
         Inst::ULoad64 {
@@ -1825,8 +1825,8 @@ fn test_aarch64_binemit() {
             }, // 2^20 + 1
             flags: MemFlags::trusted(),
         },
-        "300080521002A072B063308B010240F9",
-        "movz w16, #1 ; movk w16, w16, #16, LSL #16 ; add x16, fp, x16, UXTX ; ldr x1, [x16]",
+        "300080521002A072A1EB70F8",
+        "movz w16, #1 ; movk w16, w16, #16, LSL #16 ; ldr x1, [fp, x16, SXTX]",
     ));
 
     insns.push((
@@ -1867,8 +1867,8 @@ fn test_aarch64_binemit() {
             },
             flags: MemFlags::trusted(),
         },
-        "1002A0D2F060308B010240F9",
-        "movz x16, #16, LSL #16 ; add x16, x7, x16, UXTX ; ldr x1, [x16]",
+        "1002A0D2E1E870F8",
+        "movz x16, #16, LSL #16 ; ldr x1, [x7, x16, SXTX]",
     ));
 
     insns.push((
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 4249846a0..3428b31cf 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -2893,6 +2893,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rdi,
             w_rdi,
         ),
         "D3E7",
@@ -2903,6 +2904,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            r12,
             w_r12,
         ),
         "41D3E4",
@@ -2913,6 +2915,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
+            r8,
             w_r8,
         ),
         "41C1E002",
@@ -2923,6 +2926,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 31 }).unwrap(),
+            r13,
             w_r13,
         ),
         "41C1E51F",
@@ -2933,6 +2937,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            r13,
             w_r13,
         ),
         "49D3E5",
@@ -2943,6 +2948,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rdi,
             w_rdi,
         ),
         "48D3E7",
@@ -2953,6 +2959,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
+            r8,
             w_r8,
         ),
         "49C1E002",
@@ -2963,6 +2970,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 3 }).unwrap(),
+            rbx,
             w_rbx,
         ),
         "48C1E303",
@@ -2973,6 +2981,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftLeft,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 63 }).unwrap(),
+            r13,
             w_r13,
         ),
         "49C1E53F",
@@ -2983,6 +2992,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftRightLogical,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rdi,
             w_rdi,
         ),
         "D3EF",
@@ -2993,6 +3003,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftRightLogical,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
+            r8,
             w_r8,
         ),
         "41C1E802",
@@ -3003,6 +3014,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftRightLogical,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 31 }).unwrap(),
+            r13,
             w_r13,
         ),
         "41C1ED1F",
@@ -3013,6 +3025,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftRightLogical,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rdi,
             w_rdi,
         ),
         "48D3EF",
@@ -3023,6 +3036,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftRightLogical,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
+            r8,
             w_r8,
         ),
         "49C1E802",
@@ -3033,6 +3047,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftRightLogical,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 63 }).unwrap(),
+            r13,
             w_r13,
         ),
         "49C1ED3F",
@@ -3043,6 +3058,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftRightArithmetic,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rdi,
             w_rdi,
         ),
         "D3FF",
@@ -3053,6 +3069,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftRightArithmetic,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
+            r8,
             w_r8,
         ),
         "41C1F802",
@@ -3063,6 +3080,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::ShiftRightArithmetic,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 31 }).unwrap(),
+            r13,
             w_r13,
         ),
         "41C1FD1F",
@@ -3073,6 +3091,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftRightArithmetic,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rdi,
             w_rdi,
         ),
         "48D3FF",
@@ -3083,6 +3102,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftRightArithmetic,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 2 }).unwrap(),
+            r8,
             w_r8,
         ),
         "49C1F802",
@@ -3093,6 +3113,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::ShiftRightArithmetic,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 63 }).unwrap(),
+            r13,
             w_r13,
         ),
         "49C1FD3F",
@@ -3103,6 +3124,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::RotateLeft,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            r8,
             w_r8,
         ),
         "49D3C0",
@@ -3113,6 +3135,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::RotateLeft,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 3 }).unwrap(),
+            r9,
             w_r9,
         ),
         "41C1C103",
@@ -3123,6 +3146,7 @@ fn test_x64_emit() {
             OperandSize::Size32,
             ShiftKind::RotateRight,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rsi,
             w_rsi,
         ),
         "D3CE",
@@ -3133,6 +3157,7 @@ fn test_x64_emit() {
             OperandSize::Size64,
             ShiftKind::RotateRight,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 5 }).unwrap(),
+            r15,
             w_r15,
         ),
         "49C1CF05",
@@ -3143,6 +3168,7 @@ fn test_x64_emit() {
             OperandSize::Size8,
             ShiftKind::RotateRight,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rsi,
             w_rsi,
         ),
         "40D2CE",
@@ -3153,6 +3179,7 @@ fn test_x64_emit() {
             OperandSize::Size8,
             ShiftKind::RotateRight,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rax,
             w_rax,
         ),
         "D2C8",
@@ -3163,6 +3190,7 @@ fn test_x64_emit() {
             OperandSize::Size8,
             ShiftKind::RotateRight,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 5 }).unwrap(),
+            r15,
             w_r15,
         ),
         "41C0CF05",
@@ -3173,6 +3201,7 @@ fn test_x64_emit() {
             OperandSize::Size16,
             ShiftKind::RotateRight,
             Imm8Gpr::new(Imm8Reg::Reg { reg: regs::rcx() }).unwrap(),
+            rsi,
             w_rsi,
         ),
         "66D3CE",
@@ -3183,6 +3212,7 @@ fn test_x64_emit() {
             OperandSize::Size16,
             ShiftKind::RotateRight,
             Imm8Gpr::new(Imm8Reg::Imm8 { imm: 5 }).unwrap(),
+            r15,
             w_r15,
         ),
         "6641C1CF05",
diff --git a/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif b/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
index 4e735fc85..5e48d91e4 100644
--- a/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/heap_addr.clif
@@ -35,18 +35,12 @@ block0(v0: i64, v1: i32):
 
 ; block0:
 ;   mov w6, w1
+;   add x7, x0, x1, UXTW
+;   movz x5, #0
 ;   subs xzr, x6, #65536
-;   b.ls label1 ; b label2
-; block1:
-;   add x8, x0, x1, UXTW
-;   movz x7, #0
-;   subs xzr, x6, #65536
-;   csel x0, x7, x8, hi
+;   csel x0, x5, x7, hi
 ;   csdb
 ;   ret
-; block2:
-;   udf #0xc11f
-
 
 function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
     gv0 = vmctx
@@ -59,16 +53,16 @@ block0(v0: i64, v1: i32):
 }
 
 ; block0:
-;   mov w11, w1
-;   ldr x10, [x0]
+;   mov w10, w1
 ;   movz x9, #24
-;   adds x11, x11, x9
+;   adds x11, x10, x9
 ;   b.lo 8 ; udf
-;   add x12, x0, x1, UXTW
-;   add x12, x12, #16
-;   movz x13, #0
-;   subs xzr, x11, x10
-;   csel x0, x13, x12, hi
+;   ldr x12, [x0]
+;   add x13, x0, x1, UXTW
+;   add x13, x13, #16
+;   movz x10, #0
+;   subs xzr, x11, x12
+;   csel x0, x10, x13, hi
 ;   csdb
 ;   ret
 
@@ -82,19 +76,13 @@ block0(v0: i64, v1: i32):
 }
 
 ; block0:
-;   mov w9, w1
-;   movz w10, #65512
-;   subs xzr, x9, x10
-;   b.ls label1 ; b label2
-; block1:
-;   add x11, x0, x1, UXTW
-;   add x11, x11, #16
-;   movz w10, #65512
-;   movz x12, #0
-;   subs xzr, x9, x10
-;   csel x0, x12, x11, hi
+;   mov w8, w1
+;   add x9, x0, x1, UXTW
+;   add x9, x9, #16
+;   movz w6, #65512
+;   movz x10, #0
+;   subs xzr, x8, x6
+;   csel x0, x10, x9, hi
 ;   csdb
 ;   ret
-; block2:
-;   udf #0xc11f
 
diff --git a/cranelift/filetests/filetests/isa/aarch64/inline-probestack.clif b/cranelift/filetests/filetests/isa/aarch64/inline-probestack.clif
new file mode 100644
index 000000000..38838174b
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/inline-probestack.clif
@@ -0,0 +1,71 @@
+test compile precise-output
+set enable_probestack=true
+set probestack_strategy=inline
+; This is the default and is equivalent to a page size of 4096
+set probestack_size_log2=12
+target aarch64
+
+
+; If the stack size is just one page, we can avoid the stack probe entirely
+function %single_page() -> i64 system_v {
+ss0 = explicit_slot 2048
+
+block0:
+  v1 = stack_addr.i64 ss0
+  return v1
+}
+
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   sub sp, sp, #2048
+; block0:
+;   mov x0, sp
+;   add sp, sp, #2048
+;   ldp fp, lr, [sp], #16
+;   ret
+
+function %unrolled() -> i64 system_v {
+ss0 = explicit_slot 12288
+
+block0:
+  v1 = stack_addr.i64 ss0
+  return v1
+}
+
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   movn x16, #4095 ; str wzr, [sp, x16, SXTX]
+;   movn x16, #8191 ; str wzr, [sp, x16, SXTX]
+;   movn x16, #12287 ; str wzr, [sp, x16, SXTX]
+;   sub sp, sp, #12288
+; block0:
+;   mov x0, sp
+;   add sp, sp, #12288
+;   ldp fp, lr, [sp], #16
+;   ret
+
+function %large() -> i64 system_v {
+ss0 = explicit_slot 100000
+
+block0:
+  v1 = stack_addr.i64 ss0
+  return v1
+}
+
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   movz x16, #0
+;   movz w17, #34464
+;   movk w17, w17, #1, LSL #16
+;   stack_probe_loop x16, x17, #4096
+;   movz w16, #34464
+;   movk w16, w16, #1, LSL #16
+;   sub sp, sp, x16, UXTX
+; block0:
+;   mov x0, sp
+;   movz w16, #34464
+;   movk w16, w16, #1, LSL #16
+;   add sp, sp, x16, UXTX
+;   ldp fp, lr, [sp], #16
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif
index 703d8c4b3..e93dc41c4 100644
--- a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif
@@ -177,7 +177,7 @@ block0(v0: i64):
 
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
-;   movz w16, #6784 ; movk w16, w16, #6, LSL #16 ; add x16, x0, x16, UXTX ; ldr x16, [x16]
+;   movz w16, #6784 ; movk w16, w16, #6, LSL #16 ; ldr x16, [x0, x16, SXTX]
 ;   add x16, x16, #32
 ;   subs xzr, sp, x16, UXTX
 ;   b.hs 8 ; udf
diff --git a/cranelift/filetests/filetests/isa/riscv64/heap-addr.clif b/cranelift/filetests/filetests/isa/riscv64/heap-addr.clif
index 86da6b79a..c8d5d949b 100644
--- a/cranelift/filetests/filetests/isa/riscv64/heap-addr.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/heap-addr.clif
@@ -32,19 +32,13 @@ block0(v0: i64, v1: i32):
 }
 
 ; block0:
-;   uext.w a7,a1
-;   lui a6,16
-;   ule t3,a7,a6##ty=i64
-;   bne t3,zero,taken(label1),not_taken(label2)
-; block1:
-;   add t3,a0,a7
-;   lui a6,16
-;   ugt t4,a7,a6##ty=i64
-;   li t0,0
-;   selectif_spectre_guard a0,t0,t3##test=t4
+;   uext.w a6,a1
+;   add a5,a0,a6
+;   lui a3,16
+;   ugt a6,a6,a3##ty=i64
+;   li a7,0
+;   selectif_spectre_guard a0,a7,a5##test=a6
 ;   ret
-; block2:
-;   udf##trap_code=heap_oob
 
 function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
     gv0 = vmctx
@@ -57,17 +51,17 @@ block0(v0: i64, v1: i32):
 }
 
 ; block0:
-;   uext.w t0,a1
-;   ld t4,0(a0)
+;   uext.w t4,a1
 ;   li a7,24
-;   add t1,t0,a7
-;   ult t2,t1,t0##ty=i64
-;   trap_if t2,heap_oob
-;   add t0,a0,t0
-;   addi t0,t0,16
-;   ugt t4,t1,t4##ty=i64
+;   add t0,t4,a7
+;   ult t1,t0,t4##ty=i64
+;   trap_if t1,heap_oob
+;   ld t1,0(a0)
+;   add t2,a0,t4
+;   addi t2,t2,16
+;   ugt t4,t0,t1##ty=i64
 ;   li t1,0
-;   selectif_spectre_guard a0,t1,t0##test=t4
+;   selectif_spectre_guard a0,t1,t2##test=t4
 ;   ret
 
 function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
@@ -80,20 +74,13 @@ block0(v0: i64, v1: i32):
 }
 
 ; block0:
-;   uext.w t3,a1
-;   lui a7,16
-;   addi a7,a7,4072
-;   ule t0,t3,a7##ty=i64
-;   bne t0,zero,taken(label1),not_taken(label2)
-; block1:
-;   add t0,a0,t3
-;   addi t0,t0,16
-;   lui t4,16
-;   addi t4,t4,4072
-;   ugt t1,t3,t4##ty=i64
-;   li a0,0
-;   selectif_spectre_guard a0,a0,t0##test=t1
+;   uext.w a6,a1
+;   add a7,a0,a6
+;   addi a7,a7,16
+;   lui a4,16
+;   addi a4,a4,4072
+;   ugt t3,a6,a4##ty=i64
+;   li t4,0
+;   selectif_spectre_guard a0,t4,a7##test=t3
 ;   ret
-; block2:
-;   udf##trap_code=heap_oob
 
diff --git a/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif b/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif
index 46f612db4..7706815fa 100644
--- a/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif
@@ -1,5 +1,6 @@
 test compile precise-output
 set unwind_info=false
+set enable_probestack=true
 target riscv64
 
 function %foo() {
diff --git a/cranelift/filetests/filetests/isa/riscv64/stack.clif b/cranelift/filetests/filetests/isa/riscv64/stack.clif
index c5073cf78..d70250ceb 100644
--- a/cranelift/filetests/filetests/isa/riscv64/stack.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/stack.clif
@@ -1,5 +1,6 @@
 test compile precise-output
 set unwind_info=false
+set enable_probestack=true
 target riscv64
 
 function %stack_addr_small() -> i64 {
diff --git a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
index 205564e0f..cafac223a 100644
--- a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
+++ b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
@@ -32,16 +32,11 @@ block0(v0: i64, v1: i32):
 
 ; block0:
 ;   llgfr %r4, %r3
-;   clgfi %r4, 65536
-;   jgnh label1 ; jg label2
-; block1:
 ;   agr %r2, %r4
-;   lghi %r5, 0
+;   lghi %r3, 0
 ;   clgfi %r4, 65536
-;   locgrh %r2, %r5
+;   locgrh %r2, %r3
 ;   br %r14
-; block2:
-;   trap
 
 function %dynamic_heap_check_with_offset(i64 vmctx, i32) -> i64 {
     gv0 = vmctx
@@ -53,19 +48,17 @@ block0(v0: i64, v1: i32):
     return v2
 }
 
-;   stmg %r7, %r15, 56(%r15)
 ; block0:
-;   llgfr %r7, %r3
-;   lg %r4, 0(%r2)
-;   lghi %r5, 24
-;   algfr %r5, %r3
+;   llgfr %r5, %r3
+;   lghi %r4, 24
+;   algfr %r4, %r3
 ;   jle 6 ; trap
-;   agr %r2, %r7
-;   aghi %r2, 16
-;   lghi %r3, 0
-;   clgr %r5, %r4
-;   locgrh %r2, %r3
-;   lmg %r7, %r15, 56(%r15)
+;   lg %r3, 0(%r2)
+;   agrk %r5, %r2, %r5
+;   aghik %r2, %r5, 16
+;   lghi %r5, 0
+;   clgr %r4, %r3
+;   locgrh %r2, %r5
 ;   br %r14
 
 function %static_heap_check_with_offset(i64 vmctx, i32) -> i64 {
@@ -79,15 +72,10 @@ block0(v0: i64, v1: i32):
 
 ; block0:
 ;   llgfr %r5, %r3
+;   agr %r2, %r5
+;   aghi %r2, 16
+;   lghi %r4, 0
 ;   clgfi %r5, 65512
-;   jgnh label1 ; jg label2
-; block1:
-;   agrk %r3, %r2, %r5
-;   aghik %r2, %r3, 16
-;   lghi %r3, 0
-;   clgfi %r5, 65512
-;   locgrh %r2, %r3
+;   locgrh %r2, %r4
 ;   br %r14
-; block2:
-;   trap
 
diff --git a/cranelift/filetests/filetests/isa/x64/heap-no-spectre.clif b/cranelift/filetests/filetests/isa/x64/heap-no-spectre.clif
index ca1595001..a0bc223a9 100644
--- a/cranelift/filetests/filetests/isa/x64/heap-no-spectre.clif
+++ b/cranelift/filetests/filetests/isa/x64/heap-no-spectre.clif
@@ -20,11 +20,11 @@ block0(v0: i32, v1: i64):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %edi, %eax
-;   movq    8(%rsi), %r10
-;   movq    %rax, %r11
-;   addq    %r11, $32768, %r11
+;   movq    %rax, %r10
+;   addq    %r10, $32768, %r10
 ;   jnb ; ud2 heap_oob ;
-;   cmpq    %r10, %r11
+;   movq    8(%rsi), %r11
+;   cmpq    %r11, %r10
 ;   jbe     label1; j label2
 ; block1:
 ;   addq    %rax, 0(%rsi), %rax
diff --git a/cranelift/filetests/filetests/isa/x64/heap.clif b/cranelift/filetests/filetests/isa/x64/heap.clif
index f0e472c48..75e32b073 100644
--- a/cranelift/filetests/filetests/isa/x64/heap.clif
+++ b/cranelift/filetests/filetests/isa/x64/heap.clif
@@ -33,14 +33,14 @@ block0(v0: i32, v1: i64):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %edi, %eax
-;   movq    8(%rsi), %rdi
-;   movq    %rax, %rcx
-;   addq    %rcx, $32768, %rcx
+;   movq    %rax, %rdi
+;   addq    %rdi, $32768, %rdi
 ;   jnb ; ud2 heap_oob ;
+;   movq    8(%rsi), %rcx
 ;   addq    %rax, 0(%rsi), %rax
 ;   addq    %rax, $32768, %rax
 ;   xorq    %rsi, %rsi, %rsi
-;   cmpq    %rdi, %rcx
+;   cmpq    %rcx, %rdi
 ;   cmovnbeq %rsi, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -64,19 +64,14 @@ block0(v0: i64, v1: i32):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %esi, %r9d
-;   cmpq    $4096, %r9
-;   jbe     label1; j label2
-; block1:
 ;   movq    %r9, %rax
 ;   addq    %rax, 0(%rdi), %rax
-;   xorq    %r10, %r10, %r10
+;   xorq    %r8, %r8, %r8
 ;   cmpq    $4096, %r9
-;   cmovnbeq %r10, %rax, %rax
+;   cmovnbeq %r8, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
-; block2:
-;   ud2 heap_oob
 
 ;; When a static memory is the "right" size (4GB memory, 2GB guard regions), the
 ;; Spectre mitigation is not present. Cranelift relies on the memory permissions
@@ -113,17 +108,16 @@ block0(v0: i64, v1: i32):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movq    %rdi, %rax
-;   movl    %esi, %edi
-;   movq    0(%rax), %rsi
-;   movq    %rdi, %rcx
-;   addq    %rcx, $24, %rcx
+;   movl    %esi, %esi
+;   movq    %rsi, %r11
+;   addq    %r11, $24, %r11
 ;   jnb ; ud2 heap_oob ;
-;   addq    %rax, %rdi, %rax
+;   movq    %rdi, %rax
+;   addq    %rax, %rsi, %rax
 ;   addq    %rax, $16, %rax
-;   xorq    %rdi, %rdi, %rdi
-;   cmpq    %rsi, %rcx
-;   cmovnbeq %rdi, %rax, %rax
+;   xorq    %rsi, %rsi, %rsi
+;   cmpq    0(%rdi), %r11
+;   cmovnbeq %rsi, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -141,18 +135,13 @@ block0(v0: i64, v1: i32):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   movl    %esi, %r10d
-;   cmpq    $65512, %r10
-;   jbe     label1; j label2
-; block1:
 ;   movq    %rdi, %rax
 ;   addq    %rax, %r10, %rax
 ;   addq    %rax, $16, %rax
-;   xorq    %r11, %r11, %r11
+;   xorq    %r9, %r9, %r9
 ;   cmpq    $65512, %r10
-;   cmovnbeq %r11, %rax, %rax
+;   cmovnbeq %r9, %rax, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
-; block2:
-;   ud2 heap_oob
 
diff --git a/cranelift/filetests/filetests/legalizer/bounds-checks.clif b/cranelift/filetests/filetests/legalizer/bounds-checks.clif
index 6368ea8b3..8e6f9c8e2 100644
--- a/cranelift/filetests/filetests/legalizer/bounds-checks.clif
+++ b/cranelift/filetests/filetests/legalizer/bounds-checks.clif
@@ -20,13 +20,14 @@ block0(v0: i64, v1: i32):
 
 ; check:  block0(v0: i64, v1: i32):
 ; nextln:     v4 = uextend.i64 v1
-; nextln:     v5 = load.i64 notrap aligned v0+88
-; nextln:     v6 = iconst.i64 4
-; nextln:     v7 = uadd_overflow_trap v4, v6, heap_oob  ; v6 = 4
+; nextln:     v5 = iconst.i64 4
+; nextln:     v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 4
+; nextln:     v7 = load.i64 notrap aligned v0+88
 ; nextln:     v8 = load.i64 notrap aligned v0+80
 ; nextln:     v9 = iadd v8, v4
 ; nextln:     v10 = iconst.i64 0
-; nextln:     v11 = icmp ugt v7, v5
-; nextln:     v2 = select_spectre_guard v11, v10, v9  ; v10 = 0
+; nextln:     v11 = icmp ugt v6, v7
+; nextln:     v12 = select_spectre_guard v11, v10, v9  ; v10 = 0
+; nextln:     v2 -> v12
 ; nextln:     v3 = load.i32 little heap v2
 ; nextln:     return v3
diff --git a/cranelift/filetests/filetests/legalizer/static-heap-with-guard-pages.clif b/cranelift/filetests/filetests/legalizer/static-heap-with-guard-pages.clif
index 7530ef6e8..4ef284b3e 100644
--- a/cranelift/filetests/filetests/legalizer/static-heap-with-guard-pages.clif
+++ b/cranelift/filetests/filetests/legalizer/static-heap-with-guard-pages.clif
@@ -17,5 +17,6 @@ block0(v0: i64, v1: i32):
 ; check:  block0(v0: i64, v1: i32):
 ; nextln:     v3 = uextend.i64 v1
 ; nextln:     v4 = load.i64 notrap aligned v0
-; nextln:     v2 = iadd v4, v3
+; nextln:     v5 = iadd v4, v3
+; nextln:     v2 -> v5
 ; nextln:     return v2
diff --git a/cranelift/filetests/filetests/legalizer/static-heap-without-guard-pages.clif b/cranelift/filetests/filetests/legalizer/static-heap-without-guard-pages.clif
index 729dc69b9..d5fff83f6 100644
--- a/cranelift/filetests/filetests/legalizer/static-heap-without-guard-pages.clif
+++ b/cranelift/filetests/filetests/legalizer/static-heap-without-guard-pages.clif
@@ -3,6 +3,8 @@ set enable_heap_access_spectre_mitigation=true
 target x86_64
 
 ;; The offset guard is not large enough to avoid explicit bounds checks.
+;; Additionally, the explicit bounds check gets deduped with the Spectre
+;; mitigation.
 
 function %test(i64 vmctx, i32) -> i64 {
     gv0 = vmctx
@@ -16,19 +18,11 @@ block0(v0: i64, v1: i32):
 
 ; check:  block0(v0: i64, v1: i32):
 ; nextln:     v3 = uextend.i64 v1
-; nextln:     v10 = iconst.i64 4092
-; nextln:     v4 = icmp ugt v3, v10  ; v10 = 4092
-; nextln:     brz v4, block2
-; nextln:     jump block1
-; nextln: 
-; nextln: block1:
-; nextln:     trap heap_oob
-; nextln: 
-; nextln: block2:
-; nextln:     v5 = iconst.i64 4092
-; nextln:     v6 = load.i64 notrap aligned v0
-; nextln:     v7 = iadd v6, v3
-; nextln:     v8 = iconst.i64 0
-; nextln:     v9 = icmp.i64 ugt v3, v5  ; v5 = 4092
-; nextln:     v2 = select_spectre_guard v9, v8, v7  ; v8 = 0
+; nextln:     v4 = iconst.i64 4092
+; nextln:     v5 = load.i64 notrap aligned v0
+; nextln:     v6 = iadd v5, v3
+; nextln:     v7 = iconst.i64 0
+; nextln:     v8 = icmp ugt v3, v4  ; v4 = 4092
+; nextln:     v9 = select_spectre_guard v8, v7, v6  ; v7 = 0
+; nextln:     v2 -> v9
 ; nextln:     return v2
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_no_spectre.clif
index 5493880b4..a30c50f14 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_yes_spectre.clif
index 045eb6a2e..2a307ef6a 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif
index 56abdc7fd..cdecf0016 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif
index 52081f193..a0d022f7c 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_no_spectre.clif
index 869aff68f..f4ef93d8e 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_yes_spectre.clif
index 6be2b401f..8516f69f0 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif
index 743256fd9..8a942f91a 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif
index db20c1168..3e4d36efd 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_no_spectre.clif
index 24159a168..98a35c1e8 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_yes_spectre.clif
index b8d96a55e..a3e3e5b9c 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif
index b9664bfc2..5c275ea6a 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif
index 42bd0eb49..53cf94139 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i32, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_no_spectre.clif
index 3273ebc54..1dd2ecc6f 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_yes_spectre.clif
index f93488aa7..829f837a3 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_no_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif
index 4890b48b9..383294b79 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=false
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif
index dcb08f9c9..59a53ae26 100644
--- a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif
@@ -1,14 +1,15 @@
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=true
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, i64, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/cranelift/filetests/filetests/runtests/inline-probestack.clif b/cranelift/filetests/filetests/runtests/inline-probestack.clif
index 21426137d..ea5e4241c 100644
--- a/cranelift/filetests/filetests/runtests/inline-probestack.clif
+++ b/cranelift/filetests/filetests/runtests/inline-probestack.clif
@@ -6,9 +6,11 @@ set probestack_strategy=inline
 ; This is the default and is equivalent to a page size of 4096
 set probestack_size_log2=12
 target x86_64
+target aarch64
 ; Test also with 64k pages
 set probestack_size_log2=16
 target x86_64
+target aarch64
 
 ; Create a huge stack slot (1MB), way larger than PAGE_SIZE and touch the end of it.
 ; This guarantees that we bypass the guard page, cause a page fault the OS isn't expecting
diff --git a/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh b/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh
index 4326d71b3..a224d65f8 100755
--- a/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh
+++ b/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh
@@ -39,14 +39,15 @@ function generate_one_test() {
 ;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
 
 test interpret
-;; test run
-;; target x86_64
-;; target s390x
-;; target aarch64
-;; target riscv64
+test run
 
 set enable_heap_access_spectre_mitigation=${enable_spectre}
 
+target x86_64
+target s390x
+target aarch64
+target riscv64
+
 function %do_store(i64 vmctx, ${index_type}, i32) {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs
index 0e002bd26..b8dbfce83 100644
--- a/tests/all/cli_tests.rs
+++ b/tests/all/cli_tests.rs
@@ -1,11 +1,13 @@
-use anyhow::{bail, Result};
-use std::io::Write;
+use anyhow::{bail, Context, Result};
+use std::fs::File;
+use std::io::{Read, Write};
 use std::path::Path;
-use std::process::{Command, Output};
+use std::process::{Command, Output, Stdio};
 use tempfile::{NamedTempFile, TempDir};
 
 // Run the wasmtime CLI with the provided args and return the `Output`.
-fn run_wasmtime_for_output(args: &[&str]) -> Result<Output> {
+// If the `stdin` is `Some`, opens the file and redirects to the child's stdin.
+fn run_wasmtime_for_output(args: &[&str], stdin: Option<&Path>) -> Result<Output> {
     let runner = std::env::vars()
         .filter(|(k, _v)| k.starts_with("CARGO_TARGET") && k.ends_with("RUNNER"))
         .next();
@@ -14,6 +16,11 @@ fn run_wasmtime_for_output(args: &[&str]) -> Result<Output> {
     me.pop(); // chop off `deps`
     me.push("wasmtime");
 
+    let stdin = stdin
+        .map(File::open)
+        .transpose()
+        .context("Cannot open a file to use as stdin")?;
+
     // If we're running tests with a "runner" then we might be doing something
     // like cross-emulation, so spin up the emulator rather than the tests
     // itself, which may not be natively executable.
@@ -28,13 +35,33 @@ fn run_wasmtime_for_output(args: &[&str]) -> Result<Output> {
     } else {
         Command::new(&me)
     };
-    cmd.args(args).output().map_err(Into::into)
+
+    if let Some(mut f) = stdin {
+        let mut buf = Vec::new();
+        f.read_to_end(&mut buf)?;
+
+        let mut child = cmd
+            .stdout(Stdio::piped())
+            .stdin(Stdio::piped())
+            .args(args)
+            .spawn()?;
+
+        let mut stdin = child.stdin.take().unwrap();
+        std::thread::spawn(move || {
+            stdin
+                .write_all(&buf)
+                .expect("failed to write module to child stdin")
+        });
+        child.wait_with_output().map_err(Into::into)
+    } else {
+        cmd.args(args).output().map_err(Into::into)
+    }
 }
 
 // Run the wasmtime CLI with the provided args and, if it succeeds, return
 // the standard output in a `String`.
 fn run_wasmtime(args: &[&str]) -> Result<String> {
-    let output = run_wasmtime_for_output(args)?;
+    let output = run_wasmtime_for_output(args, None)?;
     if !output.status.success() {
         bail!(
             "Failed to execute wasmtime with: {:?}\n{}",
@@ -124,7 +151,8 @@ fn run_wasmtime_simple_wat() -> Result<()> {
 #[test]
 fn run_wasmtime_unreachable_wat() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/unreachable.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
 
     assert_ne!(output.stderr, b"");
     assert_eq!(output.stdout, b"");
@@ -164,13 +192,16 @@ fn hello_wasi_snapshot1() -> Result<()> {
 #[test]
 fn timeout_in_start() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/iloop-start.wat")?;
-    let output = run_wasmtime_for_output(&[
-        "run",
-        wasm.path().to_str().unwrap(),
-        "--wasm-timeout",
-        "1ms",
-        "--disable-cache",
-    ])?;
+    let output = run_wasmtime_for_output(
+        &[
+            "run",
+            wasm.path().to_str().unwrap(),
+            "--wasm-timeout",
+            "1ms",
+            "--disable-cache",
+        ],
+        None,
+    )?;
     assert!(!output.status.success());
     assert_eq!(output.stdout, b"");
     let stderr = String::from_utf8_lossy(&output.stderr);
@@ -185,13 +216,16 @@ fn timeout_in_start() -> Result<()> {
 #[test]
 fn timeout_in_invoke() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/iloop-invoke.wat")?;
-    let output = run_wasmtime_for_output(&[
-        "run",
-        wasm.path().to_str().unwrap(),
-        "--wasm-timeout",
-        "1ms",
-        "--disable-cache",
-    ])?;
+    let output = run_wasmtime_for_output(
+        &[
+            "run",
+            wasm.path().to_str().unwrap(),
+            "--wasm-timeout",
+            "1ms",
+            "--disable-cache",
+        ],
+        None,
+    )?;
     assert!(!output.status.success());
     assert_eq!(output.stdout, b"");
     let stderr = String::from_utf8_lossy(&output.stderr);
@@ -207,7 +241,8 @@ fn timeout_in_invoke() -> Result<()> {
 #[test]
 fn exit2_wasi_snapshot0() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/exit2_wasi_snapshot0.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
     assert_eq!(output.status.code().unwrap(), 2);
     Ok(())
 }
@@ -216,7 +251,8 @@ fn exit2_wasi_snapshot0() -> Result<()> {
 #[test]
 fn exit2_wasi_snapshot1() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/exit2_wasi_snapshot1.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
     assert_eq!(output.status.code().unwrap(), 2);
     Ok(())
 }
@@ -225,7 +261,8 @@ fn exit2_wasi_snapshot1() -> Result<()> {
 #[test]
 fn exit125_wasi_snapshot0() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/exit125_wasi_snapshot0.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
     if cfg!(windows) {
         assert_eq!(output.status.code().unwrap(), 1);
     } else {
@@ -238,7 +275,8 @@ fn exit125_wasi_snapshot0() -> Result<()> {
 #[test]
 fn exit125_wasi_snapshot1() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/exit125_wasi_snapshot1.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
     if cfg!(windows) {
         assert_eq!(output.status.code().unwrap(), 1);
     } else {
@@ -251,7 +289,8 @@ fn exit125_wasi_snapshot1() -> Result<()> {
 #[test]
 fn exit126_wasi_snapshot0() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/exit126_wasi_snapshot0.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
     assert_eq!(output.status.code().unwrap(), 1);
     assert!(output.stdout.is_empty());
     assert!(String::from_utf8_lossy(&output.stderr).contains("invalid exit status"));
@@ -262,7 +301,8 @@ fn exit126_wasi_snapshot0() -> Result<()> {
 #[test]
 fn exit126_wasi_snapshot1() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/exit126_wasi_snapshot1.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
     assert_eq!(output.status.code().unwrap(), 1);
     assert!(output.stdout.is_empty());
     assert!(String::from_utf8_lossy(&output.stderr).contains("invalid exit status"));
@@ -368,7 +408,8 @@ fn greeter_preload_callable_command() -> Result<()> {
 #[test]
 fn exit_with_saved_fprs() -> Result<()> {
     let wasm = build_wasm("tests/all/cli_tests/exit_with_saved_fprs.wat")?;
-    let output = run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"])?;
+    let output =
+        run_wasmtime_for_output(&[wasm.path().to_str().unwrap(), "--disable-cache"], None)?;
     assert_eq!(output.status.code().unwrap(), 0);
     assert!(output.stdout.is_empty());
     Ok(())
@@ -389,3 +430,46 @@ fn run_cwasm() -> Result<()> {
     assert_eq!(stdout, "");
     Ok(())
 }
+
+#[cfg(unix)]
+#[test]
+fn hello_wasi_snapshot0_from_stdin() -> Result<()> {
+    // Run a simple WASI hello world, snapshot0 edition.
+    // The module is piped from standard input.
+    let wasm = build_wasm("tests/all/cli_tests/hello_wasi_snapshot0.wat")?;
+    let stdout = {
+        let path = wasm.path();
+        let args: &[&str] = &["-", "--disable-cache"];
+        let output = run_wasmtime_for_output(args, Some(path))?;
+        if !output.status.success() {
+            bail!(
+                "Failed to execute wasmtime with: {:?}\n{}",
+                args,
+                String::from_utf8_lossy(&output.stderr)
+            );
+        }
+        Ok::<_, anyhow::Error>(String::from_utf8(output.stdout).unwrap())
+    }?;
+    assert_eq!(stdout, "Hello, world!\n");
+    Ok(())
+}
+
+#[cfg(unix)]
+#[test]
+fn run_cwasm_from_stdin() -> Result<()> {
+    let td = TempDir::new()?;
+    let cwasm = td.path().join("foo.cwasm");
+    let stdout = run_wasmtime(&[
+        "compile",
+        "tests/all/cli_tests/simple.wat",
+        "-o",
+        cwasm.to_str().unwrap(),
+    ])?;
+    assert_eq!(stdout, "");
+    let args: &[&str] = &["run", "--allow-precompiled", "-"];
+    let output = run_wasmtime_for_output(args, Some(&cwasm))?;
+    if output.status.success() {
+        bail!("wasmtime should fail loading precompiled modules from piped files, but suceeded");
+    }
+    Ok(())
+}
diff --git a/tests/all/stack_overflow.rs b/tests/all/stack_overflow.rs
index 67ab7e22a..f2e2417d4 100644
--- a/tests/all/stack_overflow.rs
+++ b/tests/all/stack_overflow.rs
@@ -1,8 +1,9 @@
+use anyhow::Result;
 use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
 use wasmtime::*;
 
 #[test]
-fn host_always_has_some_stack() -> anyhow::Result<()> {
+fn host_always_has_some_stack() -> Result<()> {
     static HITS: AtomicUsize = AtomicUsize::new(0);
     // assume hosts always have at least 128k of stack
     const HOST_STACK: usize = 128 * 1024;
@@ -54,3 +55,35 @@ fn host_always_has_some_stack() -> anyhow::Result<()> {
         consume_some_stack(space.as_mut_ptr() as usize, stack.saturating_sub(1024))
     }
 }
+
+#[test]
+fn big_stack_works_ok() -> Result<()> {
+    const N: usize = 10000;
+
+    // Build a module with a function that uses a very large amount of stack space,
+    // modeled here by calling an i64-returning-function many times followed by
+    // adding them all into one i64.
+    //
+    // This should exercise the ability to consume multi-page stacks and
+    // only touch a few internals of it at a time.
+    let mut s = String::new();
+    s.push_str("(module\n");
+    s.push_str("(func (export \"\") (result i64)\n");
+    s.push_str("i64.const 0\n");
+    for _ in 0..N {
+        s.push_str("call $get\n");
+    }
+    for _ in 0..N {
+        s.push_str("i64.add\n");
+    }
+    s.push_str(")\n");
+    s.push_str("(func $get (result i64) i64.const 0)\n");
+    s.push_str(")\n");
+
+    let mut store = Store::<()>::default();
+    let module = Module::new(store.engine(), &s)?;
+    let instance = Instance::new(&mut store, &module, &[])?;
+    let func = instance.get_typed_func::<(), i64>(&mut store, "")?;
+    assert_eq!(func.call(&mut store, ())?, 0);
+    Ok(())
+}
diff --git a/tests/host_segfault.rs b/tests/host_segfault.rs
index 6adac707e..404fb4947 100644
--- a/tests/host_segfault.rs
+++ b/tests/host_segfault.rs
@@ -144,6 +144,16 @@ fn main() {
             },
             true,
         ),
+        (
+            "overrun 8k with misconfigured host",
+            || overrun_with_big_module(8 << 10),
+            true,
+        ),
+        (
+            "overrun 32k with misconfigured host",
+            || overrun_with_big_module(32 << 10),
+            true,
+        ),
         #[cfg(not(any(target_arch = "riscv64")))]
         // Due to `InstanceAllocationStrategy::pooling()` trying to alloc more than 6000G memory space.
         // https://gitlab.com/qemu-project/qemu/-/issues/1214
@@ -178,6 +188,7 @@ fn main() {
         }
         Err(_) => {
             for (name, _test, stack_overflow) in tests {
+                println!("running {name}");
                 run_test(name, *stack_overflow);
             }
         }
@@ -245,7 +256,7 @@ fn is_stack_overflow(status: &ExitStatus, stderr: &str) -> bool {
     use std::os::unix::prelude::*;
 
     // The main thread might overflow or it might be from a fiber stack (SIGSEGV/SIGBUS)
-    stderr.contains("thread 'main' has overflowed its stack")
+    stderr.contains("has overflowed its stack")
         || match status.signal() {
             Some(libc::SIGSEGV) | Some(libc::SIGBUS) => true,
             _ => false,
@@ -267,3 +278,47 @@ fn is_stack_overflow(status: &ExitStatus, _stderr: &str) -> bool {
         _ => false,
     }
 }
+
+fn overrun_with_big_module(approx_stack: usize) {
+    // Each call to `$get` produces ten 8-byte values which need to be saved
+    // onto the stack, so divide `approx_stack` by 80 to get
+    // a rough number of calls to consume `approx_stack` stack.
+    let n = approx_stack / 10 / 8;
+
+    let mut s = String::new();
+    s.push_str("(module\n");
+    s.push_str("(func $big_stack\n");
+    for _ in 0..n {
+        s.push_str("call $get\n");
+    }
+    for _ in 0..n {
+        s.push_str("call $take\n");
+    }
+    s.push_str(")\n");
+    s.push_str("(func $get (result i64 i64 i64 i64 i64 i64 i64 i64 i64 i64) call $big_stack unreachable)\n");
+    s.push_str("(func $take (param i64 i64 i64 i64 i64 i64 i64 i64 i64 i64) unreachable)\n");
+    s.push_str("(func (export \"\") call $big_stack)\n");
+    s.push_str(")\n");
+
+    // Give 100MB of stack to wasm, representing a misconfigured host. Run the
+    // actual module on a 2MB stack in a child thread to guarantee that the
+    // module here will overrun the stack. This should deterministically hit the
+    // guard page.
+    let mut config = Config::default();
+    config.max_wasm_stack(100 << 20).async_stack_size(100 << 20);
+    let engine = Engine::new(&config).unwrap();
+    let module = Module::new(&engine, &s).unwrap();
+    let mut store = Store::new(&engine, ());
+    let i = Instance::new(&mut store, &module, &[]).unwrap();
+    let f = i.get_typed_func::<(), ()>(&mut store, "").unwrap();
+    std::thread::Builder::new()
+        .stack_size(2 << 20)
+        .spawn(move || {
+            println!("{CONFIRM}");
+            f.call(&mut store, ()).unwrap();
+        })
+        .unwrap()
+        .join()
+        .unwrap();
+    unreachable!();
+}