Repository: bytecodealliance/wasmtime
Pull request #8134 (2024-03-14 16:58:15)

cranelift: Choose memory trap code based on memflags
Ideally we'd allow the frontend to specify what trap code a memory access instruction can produce, but we don't have room to store that information.

Instead, I'm adding additional meaning to the `table` and `heap` memflags. Besides being used in alias analysis, these flags now also indicate whether the instruction should produce a trap code of TableOutOfBounds or HeapOutOfBounds, respectively.

In the Cranelift weekly meeting we discussed that it would be nice to assert that either `table` or `heap` is set when the trap-code is requested, but I don't know if all users of the instruction encodings (such as cg-clif or Winch) set those flags, so I've chosen to default to the old behavior if none of the alias-analysis flags are set.

Currently, memory accesses with the `table` flag set always have `notrap` set as well, so the TableOutOfBounds case is never hit, but I wanted to separate this change out from the cranelift-wasm changes which will use it.

abrown: @cfallin, @elliottt: do you want to review this instead? I am not as familiar with these trapping bits as I should be.

Patch:
diff --git a/cranelift/codegen/src/ir/memflags.rs b/cranelift/codegen/src/ir/memflags.rs
index 72d09dae3..8a5626d74 100644
--- a/cranelift/codegen/src/ir/memflags.rs
+++ b/cranelift/codegen/src/ir/memflags.rs
@@ -1,5 +1,6 @@
 //! Memory operation flags.
 
+use super::TrapCode;
 use core::fmt;
 
 #[cfg(feature = "enable-serde")]
@@ -9,6 +10,9 @@ enum FlagBit {
     /// Guaranteed not to trap. This may enable additional
     /// optimizations to be performed.
     Notrap,
+    /// If this memory access traps, use [TrapCode::TableOutOfBounds]
+    /// instead of [TrapCode::HeapOutOfBounds].
+    TableTrap,
     /// Guaranteed to use "natural alignment" for the given type. This
     /// may enable better instruction selection.
     Aligned,
@@ -36,8 +40,17 @@ enum FlagBit {
     Checked,
 }
 
-const NAMES: [&str; 9] = [
-    "notrap", "aligned", "readonly", "little", "big", "heap", "table", "vmctx", "checked",
+const NAMES: [&str; 10] = [
+    "notrap",
+    "tabletrap",
+    "aligned",
+    "readonly",
+    "little",
+    "big",
+    "heap",
+    "table",
+    "vmctx",
+    "checked",
 ];
 
 /// Endianness of a memory access.
@@ -162,6 +175,25 @@ impl MemFlags {
         self.with(FlagBit::Notrap)
     }
 
+    /// Test if the `tabletrap` bit is set.
+    ///
+    /// By default, if this memory access traps, the associated trap
+    /// code will be `HeapOutOfBounds`. With this flag set, the trap
+    /// code will instead be `TableOutOfBounds`.
+    pub const fn tabletrap(self) -> bool {
+        self.read(FlagBit::TableTrap)
+    }
+
+    /// Set the `tabletrap` bit.
+    pub fn set_tabletrap(&mut self) {
+        *self = self.with_tabletrap();
+    }
+
+    /// Set the `tabletrap` bit, returning new flags.
+    pub const fn with_tabletrap(self) -> Self {
+        self.with(FlagBit::TableTrap)
+    }
+
     /// Test if the `aligned` flag is set.
     ///
     /// By default, Cranelift memory instructions work with any unaligned effective address. If the
@@ -296,6 +328,21 @@ impl MemFlags {
     pub const fn with_checked(self) -> Self {
         self.with(FlagBit::Checked)
     }
+
+    /// Get the trap code to report if this memory access traps. If the
+    /// `notrap` flag is set, then return `None`. Otherwise, the choice
+    /// of trap code depends on the `tabletrap` flag. The default trap
+    /// code is `HeapOutOfBounds`, but with `tabletrap` set, the trap
+    /// code is instead `TableOutOfBounds`.
+    pub const fn trap_code(self) -> Option<TrapCode> {
+        if self.notrap() {
+            None
+        } else if self.tabletrap() {
+            Some(TrapCode::TableOutOfBounds)
+        } else {
+            Some(TrapCode::HeapOutOfBounds)
+        }
+    }
 }
 
 impl fmt::Display for MemFlags {
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 880a804a5..b02fe200e 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -4,7 +4,7 @@ use cranelift_control::ControlPlane;
 use regalloc2::Allocation;
 
 use crate::binemit::StackMap;
-use crate::ir::{self, types::*, TrapCode};
+use crate::ir::{self, types::*};
 use crate::isa::aarch64::inst::*;
 use crate::trace;
 
@@ -1005,9 +1005,9 @@ impl MachInstEmit for Inst {
                     _ => unreachable!(),
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 match &mem {
@@ -1140,9 +1140,9 @@ impl MachInstEmit for Inst {
                     _ => unreachable!(),
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual store instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 match &mem {
@@ -1219,9 +1219,9 @@ impl MachInstEmit for Inst {
                 let rt = allocs.next(rt);
                 let rt2 = allocs.next(rt2);
                 let mem = mem.with_allocs(&mut allocs);
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual store instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
                 match &mem {
                     &PairAMode::SignedOffset { reg, simm7 } => {
@@ -1250,9 +1250,9 @@ impl MachInstEmit for Inst {
                 let rt = allocs.next(rt.to_reg());
                 let rt2 = allocs.next(rt2.to_reg());
                 let mem = mem.with_allocs(&mut allocs);
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 match &mem {
@@ -1289,9 +1289,9 @@ impl MachInstEmit for Inst {
                 let rt2 = allocs.next(rt2.to_reg());
                 let mem = mem.with_allocs(&mut allocs);
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 let opc = match self {
@@ -1334,9 +1334,9 @@ impl MachInstEmit for Inst {
                 let rt2 = allocs.next(rt2);
                 let mem = mem.with_allocs(&mut allocs);
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual store instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 let opc = match self {
@@ -1494,8 +1494,8 @@ impl MachInstEmit for Inst {
                 let rt = allocs.next_writable(rt);
                 let rn = allocs.next(rn);
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(enc_acq_rel(ty, op, rs, rt, rn));
@@ -1534,8 +1534,8 @@ impl MachInstEmit for Inst {
                 // again:
                 sink.bind_label(again_label, &mut state.ctrl_plane);
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(enc_ldaxr(ty, x27wr, x25)); // ldaxr x27, [x25]
@@ -1658,8 +1658,8 @@ impl MachInstEmit for Inst {
                     }
                 }
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
                 if op == AtomicRMWLoopOp::Xchg {
                     sink.put4(enc_stlxr(ty, x24wr, x26, x25)); // stlxr w24, x26, [x25]
@@ -1699,8 +1699,8 @@ impl MachInstEmit for Inst {
                     _ => panic!("Unsupported type: {}", ty),
                 };
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(enc_cas(size, rd, rt, rn));
@@ -1733,8 +1733,8 @@ impl MachInstEmit for Inst {
                 // again:
                 sink.bind_label(again_label, &mut state.ctrl_plane);
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
 
                 // ldaxr x27, [x25]
@@ -1760,8 +1760,8 @@ impl MachInstEmit for Inst {
                 ));
                 sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19);
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25]
@@ -1789,8 +1789,8 @@ impl MachInstEmit for Inst {
                 let rn = allocs.next(rn);
                 let rt = allocs.next_writable(rt);
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(enc_ldar(access_ty, rt, rn));
@@ -1804,8 +1804,8 @@ impl MachInstEmit for Inst {
                 let rn = allocs.next(rn);
                 let rt = allocs.next(rt);
 
-                if !flags.notrap() {
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(enc_stlr(access_ty, rt, rn));
@@ -2977,9 +2977,9 @@ impl MachInstEmit for Inst {
                 let rn = allocs.next(rn);
                 let (q, size) = size.enc_size();
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(enc_ldst_vec(q, size, rn, rd));
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index aa86021c4..c4f32c58b 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -630,9 +630,9 @@ impl Inst {
                     _ => return None,
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
                 sink.put2(encode_ci_sp_load(op, rd, imm6));
             }
@@ -701,9 +701,9 @@ impl Inst {
                     encode_cl_type(op, rd, base, imm5)
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
                 sink.put2(encoded);
             }
@@ -732,9 +732,9 @@ impl Inst {
                     _ => return None,
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
                 sink.put2(encode_css_type(op, src, imm6));
             }
@@ -799,9 +799,9 @@ impl Inst {
                     encode_cs_type(op, src, base, imm5)
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
                 sink.put2(encoded);
             }
@@ -1077,9 +1077,9 @@ impl Inst {
                     }
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(encode_i_type(op.op_code(), rd, op.funct3(), addr, imm12));
@@ -1100,9 +1100,9 @@ impl Inst {
                     }
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(encode_s_type(op.op_code(), op.funct3(), addr, src, imm12));
@@ -1492,7 +1492,11 @@ impl Inst {
                 src,
                 amo,
             } => {
-                sink.add_trap(TrapCode::HeapOutOfBounds);
+                // TODO: get flags from original CLIF atomic instruction
+                let flags = MemFlags::new();
+                if let Some(trap_code) = flags.trap_code() {
+                    sink.add_trap(trap_code);
+                }
                 let x = op.op_code()
                     | reg_to_gpr_num(rd.to_reg()) << 7
                     | op.funct3() << 12
@@ -2718,9 +2722,9 @@ impl Inst {
                     }
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(encode_vmem_load(
@@ -2765,9 +2769,9 @@ impl Inst {
                     }
                 };
 
-                if !flags.notrap() {
+                if let Some(trap_code) = flags.trap_code() {
                     // Register the offset at which the actual load instruction starts.
-                    sink.add_trap(TrapCode::HeapOutOfBounds);
+                    sink.add_trap(trap_code);
                 }
 
                 sink.put4(encode_vmem_store(
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 54b3649da..74e9b0e3c 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -2872,8 +2872,8 @@
       (let ((rev Reg (vec_permute_dw_imm $I64X2 reg 1 reg 0)))
         (vec_rot_imm $I16X8 (vec_rot_imm $I32X4 (vec_rot_imm $I64X2 rev 32) 16) 8)))
 
-;; When passing a vector value in register to a function whose ABI uses
-;; a different lane order than the current function, we need to swap lanes.
+;; When passing a vector value to a function whose ABI uses a different
+;; lane order than the current function, we need to swap lanes.
 ;; The first operand is the lane order used by the callee.
 (decl abi_vec_elt_rev (LaneOrder Type Reg) Reg)
 (rule 4 (abi_vec_elt_rev _ (gpr32_ty ty) reg) reg)
@@ -2948,15 +2948,17 @@
 (rule (copy_reg_to_arg_slot uses lo _ (ABIArgSlot.Reg reg ty ext) src)
       (let ((_ Unit (args_builder_push uses (abi_vec_elt_rev lo ty src) reg)))
         (output_none)))
-(rule (copy_reg_to_arg_slot _ _ base (ABIArgSlot.Stack offset ty ext) src)
-      (side_effect (arg_store (abi_ext_ty ext ty) src (memarg_stack_off base offset))))
+(rule (copy_reg_to_arg_slot _ lo base (ABIArgSlot.Stack offset ty ext) src)
+      (side_effect (arg_store (abi_ext_ty ext ty) (abi_vec_elt_rev lo ty src)
+                              (memarg_stack_off base offset))))
 
 ;; Copy one component of an argument/return value from its slot.
 (decl copy_reg_from_arg_slot (CallRetList LaneOrder i64 ABIArgSlot) Reg)
 (rule (copy_reg_from_arg_slot defs lo _ (ABIArgSlot.Reg reg ty ext))
       (abi_vec_elt_rev lo ty (defs_lookup defs reg)))
-(rule (copy_reg_from_arg_slot _ _ base (ABIArgSlot.Stack offset ty ext))
-      (arg_load (abi_ext_ty ext ty) (memarg_stack_off base offset)))
+(rule (copy_reg_from_arg_slot _ lo base (ABIArgSlot.Stack offset ty ext))
+      (abi_vec_elt_rev lo ty (arg_load (abi_ext_ty ext ty)
+                                       (memarg_stack_off base offset))))
 
 ;; Helper to compute the type of an implicitly extended argument/return value.
 (decl abi_ext_ty (ArgumentExtension Type) Type)
diff --git a/cranelift/codegen/src/isa/s390x/inst/args.rs b/cranelift/codegen/src/isa/s390x/inst/args.rs
index 553ca8660..0f1389c80 100644
--- a/cranelift/codegen/src/isa/s390x/inst/args.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/args.rs
@@ -102,10 +102,6 @@ impl MemArg {
         }
     }
 
-    pub(crate) fn can_trap(&self) -> bool {
-        !self.get_flags().notrap()
-    }
-
     /// Edit registers with allocations.
     pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
         match self {
@@ -187,10 +183,6 @@ impl MemArgPair {
         }
     }
 
-    pub(crate) fn can_trap(&self) -> bool {
-        !self.flags.notrap()
-    }
-
     /// Edit registers with allocations.
     pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
         MemArgPair {
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index 6531d7df6..b929957b5 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -184,8 +184,10 @@ pub fn mem_emit(
         inst.emit(&[], sink, emit_info, state);
     }
 
-    if add_trap && mem.can_trap() {
-        sink.add_trap(TrapCode::HeapOutOfBounds);
+    if add_trap {
+        if let Some(trap_code) = mem.get_flags().trap_code() {
+            sink.add_trap(trap_code);
+        }
     }
 
     match &mem {
@@ -245,8 +247,10 @@ pub fn mem_rs_emit(
         inst.emit(&[], sink, emit_info, state);
     }
 
-    if add_trap && mem.can_trap() {
-        sink.add_trap(TrapCode::HeapOutOfBounds);
+    if add_trap {
+        if let Some(trap_code) = mem.get_flags().trap_code() {
+            sink.add_trap(trap_code);
+        }
     }
 
     match &mem {
@@ -294,8 +298,10 @@ pub fn mem_imm8_emit(
         inst.emit(&[], sink, emit_info, state);
     }
 
-    if add_trap && mem.can_trap() {
-        sink.add_trap(TrapCode::HeapOutOfBounds);
+    if add_trap {
+        if let Some(trap_code) = mem.get_flags().trap_code() {
+            sink.add_trap(trap_code);
+        }
     }
 
     match &mem {
@@ -339,8 +345,10 @@ pub fn mem_imm16_emit(
         inst.emit(&[], sink, emit_info, state);
     }
 
-    if add_trap && mem.can_trap() {
-        sink.add_trap(TrapCode::HeapOutOfBounds);
+    if add_trap {
+        if let Some(trap_code) = mem.get_flags().trap_code() {
+            sink.add_trap(trap_code);
+        }
     }
 
     match &mem {
@@ -363,8 +371,10 @@ pub fn mem_mem_emit(
     sink: &mut MachBuffer<Inst>,
     _state: &mut EmitState,
 ) {
-    if add_trap && (dst.can_trap() || src.can_trap()) {
-        sink.add_trap(TrapCode::HeapOutOfBounds);
+    if add_trap {
+        if let Some(trap_code) = dst.flags.trap_code().or(src.flags.trap_code()) {
+            sink.add_trap(trap_code);
+        }
     }
 
     put(
@@ -405,8 +415,10 @@ pub fn mem_vrx_emit(
         inst.emit(&[], sink, emit_info, state);
     }
 
-    if add_trap && mem.can_trap() {
-        sink.add_trap(TrapCode::HeapOutOfBounds);
+    if add_trap {
+        if let Some(trap_code) = mem.get_flags().trap_code() {
+            sink.add_trap(trap_code);
+        }
     }
 
     match &mem {
diff --git a/cranelift/codegen/src/isa/x64/encoding/evex.rs b/cranelift/codegen/src/isa/x64/encoding/evex.rs
index 03a3ac6c7..3805e7215 100644
--- a/cranelift/codegen/src/isa/x64/encoding/evex.rs
+++ b/cranelift/codegen/src/isa/x64/encoding/evex.rs
@@ -15,7 +15,6 @@
 //! Software Development Manual, volume 2A).
 
 use super::rex::{self, LegacyPrefixes, OpcodeMap};
-use crate::ir::TrapCode;
 use crate::isa::x64::args::{Amode, Avx512TupleType};
 use crate::isa::x64::inst::Inst;
 use crate::MachBuffer;
@@ -202,8 +201,8 @@ impl EvexInstruction {
     /// - an optional immediate, if necessary (not currently implemented)
     pub fn encode(&self, sink: &mut MachBuffer<Inst>) {
         if let RegisterOrAmode::Amode(amode) = &self.rm {
-            if amode.can_trap() {
-                sink.add_trap(TrapCode::HeapOutOfBounds);
+            if let Some(trap_code) = amode.get_flags().trap_code() {
+                sink.add_trap(trap_code);
             }
         }
         sink.put4(self.bits);
diff --git a/cranelift/codegen/src/isa/x64/encoding/rex.rs b/cranelift/codegen/src/isa/x64/encoding/rex.rs
index 88b6eca2d..ba7b3b19d 100644
--- a/cranelift/codegen/src/isa/x64/encoding/rex.rs
+++ b/cranelift/codegen/src/isa/x64/encoding/rex.rs
@@ -10,7 +10,6 @@
 
 use crate::machinst::{Reg, RegClass};
 use crate::{
-    ir::TrapCode,
     isa::x64::inst::{
         args::{Amode, OperandSize},
         regs, Inst, LabelUse,
@@ -314,9 +313,8 @@ pub(crate) fn emit_std_enc_mem(
     // 64-bit integer registers, because they are part of an address
     // expression.  But `enc_g` can be derived from a register of any class.
 
-    let can_trap = mem_e.can_trap();
-    if can_trap {
-        sink.add_trap(TrapCode::HeapOutOfBounds);
+    if let Some(trap_code) = mem_e.get_flags().trap_code() {
+        sink.add_trap(trap_code);
     }
 
     prefixes.emit(sink);
diff --git a/cranelift/codegen/src/isa/x64/encoding/vex.rs b/cranelift/codegen/src/isa/x64/encoding/vex.rs
index 7e853c336..1124a6b5b 100644
--- a/cranelift/codegen/src/isa/x64/encoding/vex.rs
+++ b/cranelift/codegen/src/isa/x64/encoding/vex.rs
@@ -4,7 +4,6 @@
 use super::evex::{Register, RegisterOrAmode};
 use super::rex::{LegacyPrefixes, OpcodeMap};
 use super::ByteSink;
-use crate::ir::TrapCode;
 use crate::isa::x64::args::Amode;
 use crate::isa::x64::encoding::rex;
 use crate::isa::x64::inst::Inst;
@@ -251,8 +250,8 @@ impl VexInstruction {
     /// Emit the VEX-encoded instruction to the provided buffer.
     pub fn encode(&self, sink: &mut MachBuffer<Inst>) {
         if let RegisterOrAmode::Amode(amode) = &self.rm {
-            if amode.can_trap() {
-                sink.add_trap(TrapCode::HeapOutOfBounds);
+            if let Some(trap_code) = amode.get_flags().trap_code() {
+                sink.add_trap(trap_code);
             }
         }
 
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 85c70b6b3..45ab8e6e8 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -405,10 +405,6 @@ impl Amode {
         }
     }
 
-    pub(crate) fn can_trap(&self) -> bool {
-        !self.get_flags().notrap()
-    }
-
     pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
         // The order in which we consume allocs here must match the
         // order in which we produce operands in get_operands() above.
diff --git a/cranelift/interpreter/src/interpreter.rs b/cranelift/interpreter/src/interpreter.rs
index 7d0bbe3d0..6737494e6 100644
--- a/cranelift/interpreter/src/interpreter.rs
+++ b/cranelift/interpreter/src/interpreter.rs
@@ -331,7 +331,11 @@ impl<'a> State<'a> for InterpreterState<'a> {
         let src = match addr.region {
             AddressRegion::Stack => {
                 if addr_end > self.stack.len() {
-                    return Err(MemoryError::OutOfBoundsLoad { addr, load_size });
+                    return Err(MemoryError::OutOfBoundsLoad {
+                        addr,
+                        load_size,
+                        mem_flags,
+                    });
                 }
 
                 &self.stack[addr_start..addr_end]
@@ -363,7 +367,11 @@ impl<'a> State<'a> for InterpreterState<'a> {
         let dst = match addr.region {
             AddressRegion::Stack => {
                 if addr_end > self.stack.len() {
-                    return Err(MemoryError::OutOfBoundsStore { addr, store_size });
+                    return Err(MemoryError::OutOfBoundsStore {
+                        addr,
+                        store_size,
+                        mem_flags,
+                    });
                 }
 
                 &mut self.stack[addr_start..addr_end]
diff --git a/cranelift/interpreter/src/state.rs b/cranelift/interpreter/src/state.rs
index 5c5d7a685..9f3bfdddd 100644
--- a/cranelift/interpreter/src/state.rs
+++ b/cranelift/interpreter/src/state.rs
@@ -125,9 +125,17 @@ pub enum MemoryError {
     #[error("Requested an offset of {offset} but max was {max}")]
     InvalidOffset { offset: u64, max: u64 },
     #[error("Load of {load_size} bytes is larger than available size at address {addr:?}")]
-    OutOfBoundsLoad { addr: Address, load_size: usize },
+    OutOfBoundsLoad {
+        addr: Address,
+        load_size: usize,
+        mem_flags: MemFlags,
+    },
     #[error("Store of {store_size} bytes is larger than available size at address {addr:?}")]
-    OutOfBoundsStore { addr: Address, store_size: usize },
+    OutOfBoundsStore {
+        addr: Address,
+        store_size: usize,
+        mem_flags: MemFlags,
+    },
     #[error("Load of {load_size} bytes is misaligned at address {addr:?}")]
     MisalignedLoad { addr: Address, load_size: usize },
     #[error("Store of {store_size} bytes is misaligned at address {addr:?}")]
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index 74cf6b8d1..96d465966 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -160,8 +160,12 @@ where
         MemoryError::InvalidAddressType(_) => TrapCode::HeapOutOfBounds,
         MemoryError::InvalidOffset { .. } => TrapCode::HeapOutOfBounds,
         MemoryError::InvalidEntry { .. } => TrapCode::HeapOutOfBounds,
-        MemoryError::OutOfBoundsStore { .. } => TrapCode::HeapOutOfBounds,
-        MemoryError::OutOfBoundsLoad { .. } => TrapCode::HeapOutOfBounds,
+        MemoryError::OutOfBoundsStore { mem_flags, .. } => mem_flags
+            .trap_code()
+            .expect("store with notrap flag should not trap"),
+        MemoryError::OutOfBoundsLoad { mem_flags, .. } => mem_flags
+            .trap_code()
+            .expect("load with notrap flag should not trap"),
         MemoryError::MisalignedLoad { .. } => TrapCode::HeapMisaligned,
         MemoryError::MisalignedStore { .. } => TrapCode::HeapMisaligned,
     };
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index f4758b7ef..d6ce4a5d9 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -1,16 +1,13 @@
 # Summary
 
 - [Introduction](./introduction.md)
-- [Examples](./examples.md)
-  - [Debugging WebAssembly](./examples-debugging.md)
-    - [Debugging with `gdb` and `lldb`](./examples-debugging-native-debugger.md)
-    - [Debugging with Core Dumps](./examples-debugging-core-dumps.md)
-  - [Profiling WebAssembly](./examples-profiling.md)
-    - [Profiling with Perf](./examples-profiling-perf.md)
-    - [Profiling with VTune](./examples-profiling-vtune.md)
-    - [Profiling with samply](./examples-profiling-samply.md)
-    - [Cross-platform Profiling](./examples-profiling-guest.md)
-  - [Embedding in Rust](./examples-rust-embed.md)
+- [Using the Wasmtime CLI](./cli.md)
+  - [Installation](./cli-install.md)
+  - [CLI Options](./cli-options.md)
+  - [CLI Logging](./cli-logging.md)
+  - [Cache Configuration](./cli-cache.md)
+- [Using the Wasmtime API](./lang.md)
+  - [Rust](./lang-rust.md)
     - [Hello, world!](./examples-rust-hello-world.md)
     - [Calculating the GCD](./examples-rust-gcd.md)
     - [Using Linear Memory](./examples-rust-memory.md)
@@ -19,7 +16,7 @@
     - [Debugging](./examples-rust-debugging.md)
     - [Core Dumps](./examples-rust-core-dumps.md)
     - [Using Multi-Value](./examples-rust-multi-value.md)
-  - [Embedding in C](./examples-c-embed.md)
+  - [C](./lang-c.md)
     - [Hello, World!](./examples-c-hello-world.md)
     - [Calculating the GCD](./examples-c-gcd.md)
     - [Using Linear Memory](./examples-c-memory.md)
@@ -27,21 +24,23 @@
     - [Linking Modules](./examples-c-linking.md)
     - [Debugging](./examples-c-debugging.md)
     - [Using Multi-Value](./examples-c-multi-value.md)
-  - [Checking Guests' Memory Accesses](./wmemcheck.md)
-  - [Building a minimal embedding](./examples-minimal.md)
-- [Using WebAssembly from your language](./lang.md)
-  - [Rust](./lang-rust.md)
-  - [C](./lang-c.md)
   - [Python](./lang-python.md)
   - [.NET](./lang-dotnet.md)
   - [Go](./lang-go.md)
   - [Bash](./lang-bash.md)
   - [Ruby](./lang-ruby.md)
-- [Using the `wasmtime` CLI](./cli.md)
-  - [Installation](./cli-install.md)
-  - [CLI Options](./cli-options.md)
-  - [CLI Logging](./cli-logging.md)
-  - [Cache Configuration](./cli-cache.md)
+  - [Elixir](./lang-elixir.md)
+- [Further Examples](./examples.md)
+  - [Debugging WebAssembly](./examples-debugging.md)
+    - [Debugging with `gdb` and `lldb`](./examples-debugging-native-debugger.md)
+    - [Debugging with Core Dumps](./examples-debugging-core-dumps.md)
+  - [Profiling WebAssembly](./examples-profiling.md)
+    - [Profiling with Perf](./examples-profiling-perf.md)
+    - [Profiling with VTune](./examples-profiling-vtune.md)
+    - [Profiling with samply](./examples-profiling-samply.md)
+    - [Cross-platform Profiling](./examples-profiling-guest.md)
+  - [Checking Guests' Memory Accesses](./wmemcheck.md)
+  - [Building a minimal embedding](./examples-minimal.md)
 - [Stability](stability.md)
   - [Release Process](./stability-release.md)
   - [Tiers of support](./stability-tiers.md)
diff --git a/docs/cli-install.md b/docs/cli-install.md
index b6f98b42b..ffd370f62 100644
--- a/docs/cli-install.md
+++ b/docs/cli-install.md
@@ -2,7 +2,7 @@
 
 Here we'll show you how to install the `wasmtime` command line tool. Note that
 this is distinct from embedding the Wasmtime project into another, for that
-you'll want to consult the [embedding documentation](embed.md).
+you'll want to consult the [embedding documentation](lang.md).
 
 The easiest way to install the `wasmtime` CLI tool is through our installation
 script. Linux and macOS users can execute the following:
diff --git a/docs/examples-c-embed.md b/docs/examples-c-embed.md
index c6dc5b678..57b88955a 100644
--- a/docs/examples-c-embed.md
+++ b/docs/examples-c-embed.md
@@ -1,6 +1,1 @@
-# Embedding in C
-
-This section is intended to showcase the C embedding API for Wasmtime. Full
-reference documentation for the C API [can be found online][online]
-
-[online]: https://bytecodealliance.github.io/wasmtime/c-api/
+# C
diff --git a/docs/examples-profiling.md b/docs/examples-profiling.md
index 348979c03..d1b0ab0c7 100644
--- a/docs/examples-profiling.md
+++ b/docs/examples-profiling.md
@@ -20,14 +20,16 @@ platforms:
 - For Intel's x86 CPUs on Linux or Windows, we support
   [VTune](./examples-profiling-vtune.md).
 
-- For everything else, see the cross-platform profiler below.
+- For Linux and macOS, we support [samply](./examples-profiling-samply.md).
+
+- For everything else, see the [cross-platform
+  profiler](./examples-profiling-guest.md).
 
 The native profilers can measure time spent in WebAssembly guest code as well as
 time spent in the Wasmtime host and potentially even time spent in the kernel.
 This provides a comprehensive view of performance.
 
-If the native profiling tools don't work for you, Wasmtime also provides a
-[cross-platform profiler](./examples-profiling-guest.md). This profiler can only
-measure time spent in WebAssembly guest code, and its timing measurements are
-not as precise as the native profilers. However, it works on every platform that
-Wasmtime supports.
+The cross-platform-profiler can only measure time spent in WebAssembly guest
+code, and its timing measurements are not as precise as the native profilers.
+However, it works on every platform that Wasmtime supports.
+
diff --git a/docs/examples-rust-embed.md b/docs/examples-rust-embed.md
deleted file mode 100644
index 14b4b1ebb..000000000
--- a/docs/examples-rust-embed.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Embedding in Rust
-
-This section is intended to showcase the Rust embedding API for Wasmtime. This
-is done through the [`wasmtime` crate](https://crates.io/crates/wasmtime). In
-addition to browsing the following examples you can also browse the [full API
-documentation](https://docs.rs/wasmtime).
diff --git a/docs/examples.md b/docs/examples.md
index f2cc544ef..1ab9078d1 100644
--- a/docs/examples.md
+++ b/docs/examples.md
@@ -1,4 +1,4 @@
-# Examples
+# Further Examples
 
 The examples contained in this section explain how to use Wasmtime in several
 common scenarios.
diff --git a/docs/lang-c.md b/docs/lang-c.md
index a40cd10d1..5db21c927 100644
--- a/docs/lang-c.md
+++ b/docs/lang-c.md
@@ -1,7 +1,6 @@
-# C
+# Embedding in C
 
-... more coming soon
+This section is intended to showcase the C embedding API for Wasmtime. Full
+reference documentation for the C API [can be found online][online].
 
-In the meantime, have a look at the [C API docs].
-
-[C API docs]: https://bytecodealliance.github.io/wasmtime/c-api/
+[online]: https://bytecodealliance.github.io/wasmtime/c-api/
diff --git a/docs/lang-rust.md b/docs/lang-rust.md
index cc5d57248..9199605a1 100644
--- a/docs/lang-rust.md
+++ b/docs/lang-rust.md
@@ -3,18 +3,16 @@
 This document shows an example of how to embed Wasmtime using the [Rust
 API][apidoc] to execute a simple wasm program. Be sure to also check out the
 [full API documentation][apidoc] for a full listing of what the [`wasmtime`
-crate][wasmtime] has to offer and the [book examples for
-Rust](./examples-rust-embed.md) for more information.
+crate][wasmtime] has to offer.
 
 [apidoc]: https://bytecodealliance.github.io/wasmtime/api/wasmtime/
 [wasmtime]: https://crates.io/crates/wasmtime
 
 ## Creating the WebAssembly to execute
 
-Creation of a WebAssembly file is generally covered by the [Writing
-WebAssembly chapter](./wasm.md), so we'll just assume that you've already got a
-wasm file on hand for the rest of this tutorial. To make things simple we'll
-also just assume you've got a `hello.wat` file which looks like this:
+We'll just assume that you've already got a wasm file on hand for the rest of
+this tutorial. To make things simple we'll also just assume you've got a
+`hello.wat` file which looks like this:
 
 ```wat
 (module
diff --git a/docs/lang.md b/docs/lang.md
index 94e38d1b0..be222bf6e 100644
--- a/docs/lang.md
+++ b/docs/lang.md
@@ -1,4 +1,4 @@
-# Using WebAssembly from your Language
+# Using the Wasmtime API
 
 Wasmtime can be used as a library to embed WebAssembly execution support
 within applications. Wasmtime is written in Rust, but bindings are available


Test patch:
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-abi.clif b/cranelift/filetests/filetests/isa/s390x/vec-abi.clif
index 76130443c..f05272f67 100644
--- a/cranelift/filetests/filetests/isa/s390x/vec-abi.clif
+++ b/cranelift/filetests/filetests/isa/s390x/vec-abi.clif
@@ -1,247 +1,391 @@
 test compile precise-output
 target s390x
 
-function %caller_be_to_be(i64x2, i32x4, i16x8, i8x16) -> i32x4 {
-    fn0 = %callee_be(i64x2, i32x4, i16x8, i8x16) -> i32x4
+function %caller_be_to_be(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4 {
+    fn0 = %callee_be(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4
 
-block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16):
-    v4 = call fn0(v0, v1, v2, v3)
-    return v4
+block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16, v4: i64x2, v5: i32x4, v6: i16x8, v7: i8x16, v8: i64x2, v9: i32x4, v10: i16x8, v11: i8x16):
+    v12 = call fn0(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11)
+    return v12
 }
 
 ; VCode:
 ;   stmg %r14, %r15, 112(%r15)
-;   aghi %r15, -160
-;   virtual_sp_offset_adjust 160
+;   aghi %r15, -224
+;   virtual_sp_offset_adjust 224
 ; block0:
+;   vl %v17, 384(%r15)
+;   vl %v19, 400(%r15)
+;   vl %v21, 416(%r15)
+;   vl %v23, 432(%r15)
+;   vst %v17, 160(%r15)
+;   vst %v19, 176(%r15)
+;   vst %v21, 192(%r15)
+;   vst %v23, 208(%r15)
 ;   bras %r1, 12 ; data %callee_be + 0 ; lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   lmg %r14, %r15, 272(%r15)
+;   lmg %r14, %r15, 336(%r15)
 ;   br %r14
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   stmg %r14, %r15, 0x70(%r15)
-;   aghi %r15, -0xa0
+;   aghi %r15, -0xe0
 ; block1: ; offset 0xa
-;   bras %r1, 0x16
+;   vl %v17, 0x180(%r15)
+;   vl %v19, 0x190(%r15)
+;   vl %v21, 0x1a0(%r15)
+;   vl %v23, 0x1b0(%r15)
+;   vst %v17, 0xa0(%r15)
+;   vst %v19, 0xb0(%r15)
+;   vst %v21, 0xc0(%r15)
+;   vst %v23, 0xd0(%r15)
+;   bras %r1, 0x46
 ;   .byte 0x00, 0x00 ; reloc_external Abs8 %callee_be 0
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   lmg %r14, %r15, 0x110(%r15)
+;   lmg %r14, %r15, 0x150(%r15)
 ;   br %r14
 
-function %caller_be_to_le(i64x2, i32x4, i16x8, i8x16) -> i32x4 {
-    fn0 = %callee_le(i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v
+function %caller_be_to_le(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4 {
+    fn0 = %callee_le(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v
 
-block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16):
-    v4 = call fn0(v0, v1, v2, v3)
-    return v4
+block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16, v4: i64x2, v5: i32x4, v6: i16x8, v7: i8x16, v8: i64x2, v9: i32x4, v10: i16x8, v11: i8x16):
+    v12 = call fn0(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11)
+    return v12
 }
 
 ; VCode:
 ;   stmg %r14, %r15, 112(%r15)
-;   aghi %r15, -224
-;   virtual_sp_offset_adjust 160
-;   std %f8, 160(%r15)
-;   std %f9, 168(%r15)
-;   std %f10, 176(%r15)
-;   std %f11, 184(%r15)
-;   std %f12, 192(%r15)
-;   std %f13, 200(%r15)
-;   std %f14, 208(%r15)
-;   std %f15, 216(%r15)
+;   aghi %r15, -288
+;   virtual_sp_offset_adjust 224
+;   std %f8, 224(%r15)
+;   std %f9, 232(%r15)
+;   std %f10, 240(%r15)
+;   std %f11, 248(%r15)
+;   std %f12, 256(%r15)
+;   std %f13, 264(%r15)
+;   std %f14, 272(%r15)
+;   std %f15, 280(%r15)
 ; block0:
+;   vl %v17, 448(%r15)
+;   vl %v19, 464(%r15)
+;   vl %v21, 480(%r15)
+;   vl %v23, 496(%r15)
 ;   vpdi %v24, %v24, %v24, 4
-;   vpdi %v7, %v25, %v25, 4
-;   verllg %v25, %v7, 32
-;   vpdi %v19, %v26, %v26, 4
-;   verllg %v21, %v19, 32
-;   verllf %v26, %v21, 16
-;   vpdi %v27, %v27, %v27, 4
-;   verllg %v27, %v27, 32
-;   verllf %v29, %v27, 16
-;   verllh %v27, %v29, 8
+;   vpdi %v0, %v25, %v25, 4
+;   verllg %v25, %v0, 32
+;   vpdi %v0, %v26, %v26, 4
+;   verllg %v1, %v0, 32
+;   verllf %v26, %v1, 16
+;   vpdi %v5, %v27, %v27, 4
+;   verllg %v7, %v5, 32
+;   verllf %v18, %v7, 16
+;   verllh %v27, %v18, 8
+;   vpdi %v28, %v28, %v28, 4
+;   vpdi %v29, %v29, %v29, 4
+;   verllg %v29, %v29, 32
+;   vpdi %v30, %v30, %v30, 4
+;   verllg %v30, %v30, 32
+;   verllf %v30, %v30, 16
+;   vpdi %v1, %v31, %v31, 4
+;   verllg %v3, %v1, 32
+;   verllf %v5, %v3, 16
+;   verllh %v31, %v5, 8
+;   vpdi %v17, %v17, %v17, 4
+;   vst %v17, 160(%r15)
+;   vpdi %v20, %v19, %v19, 4
+;   verllg %v22, %v20, 32
+;   vst %v22, 176(%r15)
+;   vpdi %v0, %v21, %v21, 4
+;   verllg %v0, %v0, 32
+;   verllf %v0, %v0, 16
+;   vst %v0, 192(%r15)
+;   vpdi %v0, %v23, %v23, 4
+;   verllg %v2, %v0, 32
+;   verllf %v4, %v2, 16
+;   verllh %v6, %v4, 8
+;   vst %v6, 208(%r15)
 ;   bras %r1, 12 ; data %callee_le + 0 ; lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   vpdi %v5, %v24, %v24, 4
-;   verllg %v24, %v5, 32
-;   ld %f8, 160(%r15)
-;   ld %f9, 168(%r15)
-;   ld %f10, 176(%r15)
-;   ld %f11, 184(%r15)
-;   ld %f12, 192(%r15)
-;   ld %f13, 200(%r15)
-;   ld %f14, 208(%r15)
-;   ld %f15, 216(%r15)
-;   lmg %r14, %r15, 336(%r15)
+;   vpdi %v21, %v24, %v24, 4
+;   verllg %v24, %v21, 32
+;   ld %f8, 224(%r15)
+;   ld %f9, 232(%r15)
+;   ld %f10, 240(%r15)
+;   ld %f11, 248(%r15)
+;   ld %f12, 256(%r15)
+;   ld %f13, 264(%r15)
+;   ld %f14, 272(%r15)
+;   ld %f15, 280(%r15)
+;   lmg %r14, %r15, 400(%r15)
 ;   br %r14
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   stmg %r14, %r15, 0x70(%r15)
-;   aghi %r15, -0xe0
-;   std %f8, 0xa0(%r15)
-;   std %f9, 0xa8(%r15)
-;   std %f10, 0xb0(%r15)
-;   std %f11, 0xb8(%r15)
-;   std %f12, 0xc0(%r15)
-;   std %f13, 0xc8(%r15)
-;   std %f14, 0xd0(%r15)
-;   std %f15, 0xd8(%r15)
+;   aghi %r15, -0x120
+;   std %f8, 0xe0(%r15)
+;   std %f9, 0xe8(%r15)
+;   std %f10, 0xf0(%r15)
+;   std %f11, 0xf8(%r15)
+;   std %f12, 0x100(%r15)
+;   std %f13, 0x108(%r15)
+;   std %f14, 0x110(%r15)
+;   std %f15, 0x118(%r15)
 ; block1: ; offset 0x2a
+;   vl %v17, 0x1c0(%r15)
+;   vl %v19, 0x1d0(%r15)
+;   vl %v21, 0x1e0(%r15)
+;   vl %v23, 0x1f0(%r15)
 ;   vpdi %v24, %v24, %v24, 4
-;   vpdi %v7, %v25, %v25, 4
-;   verllg %v25, %v7, 0x20
-;   vpdi %v19, %v26, %v26, 4
-;   verllg %v21, %v19, 0x20
-;   verllf %v26, %v21, 0x10
-;   vpdi %v27, %v27, %v27, 4
-;   verllg %v27, %v27, 0x20
-;   verllf %v29, %v27, 0x10
-;   verllh %v27, %v29, 8
-;   bras %r1, 0x72
+;   vpdi %v0, %v25, %v25, 4
+;   verllg %v25, %v0, 0x20
+;   vpdi %v0, %v26, %v26, 4
+;   verllg %v1, %v0, 0x20
+;   verllf %v26, %v1, 0x10
+;   vpdi %v5, %v27, %v27, 4
+;   verllg %v7, %v5, 0x20
+;   verllf %v18, %v7, 0x10
+;   verllh %v27, %v18, 8
+;   vpdi %v28, %v28, %v28, 4
+;   vpdi %v29, %v29, %v29, 4
+;   verllg %v29, %v29, 0x20
+;   vpdi %v30, %v30, %v30, 4
+;   verllg %v30, %v30, 0x20
+;   verllf %v30, %v30, 0x10
+;   vpdi %v1, %v31, %v31, 4
+;   verllg %v3, %v1, 0x20
+;   verllf %v5, %v3, 0x10
+;   verllh %v31, %v5, 8
+;   vpdi %v17, %v17, %v17, 4
+;   vst %v17, 0xa0(%r15)
+;   vpdi %v20, %v19, %v19, 4
+;   verllg %v22, %v20, 0x20
+;   vst %v22, 0xb0(%r15)
+;   vpdi %v0, %v21, %v21, 4
+;   verllg %v0, %v0, 0x20
+;   verllf %v0, %v0, 0x10
+;   vst %v0, 0xc0(%r15)
+;   vpdi %v0, %v23, %v23, 4
+;   verllg %v2, %v0, 0x20
+;   verllf %v4, %v2, 0x10
+;   verllh %v6, %v4, 8
+;   vst %v6, 0xd0(%r15)
+;   bras %r1, 0x11a
 ;   .byte 0x00, 0x00 ; reloc_external Abs8 %callee_le 0
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   vpdi %v5, %v24, %v24, 4
-;   verllg %v24, %v5, 0x20
-;   ld %f8, 0xa0(%r15)
-;   ld %f9, 0xa8(%r15)
-;   ld %f10, 0xb0(%r15)
-;   ld %f11, 0xb8(%r15)
-;   ld %f12, 0xc0(%r15)
-;   ld %f13, 0xc8(%r15)
-;   ld %f14, 0xd0(%r15)
-;   ld %f15, 0xd8(%r15)
-;   lmg %r14, %r15, 0x150(%r15)
+;   vpdi %v21, %v24, %v24, 4
+;   verllg %v24, %v21, 0x20
+;   ld %f8, 0xe0(%r15)
+;   ld %f9, 0xe8(%r15)
+;   ld %f10, 0xf0(%r15)
+;   ld %f11, 0xf8(%r15)
+;   ld %f12, 0x100(%r15)
+;   ld %f13, 0x108(%r15)
+;   ld %f14, 0x110(%r15)
+;   ld %f15, 0x118(%r15)
+;   lmg %r14, %r15, 0x190(%r15)
 ;   br %r14
 
-function %caller_le_to_be(i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v {
-    fn0 = %callee_be(i64x2, i32x4, i16x8, i8x16) -> i32x4
+function %caller_le_to_be(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v {
+    fn0 = %callee_be(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4
 
-block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16):
-    v4 = call fn0(v0, v1, v2, v3)
-    return v4
+block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16, v4: i64x2, v5: i32x4, v6: i16x8, v7: i8x16, v8: i64x2, v9: i32x4, v10: i16x8, v11: i8x16):
+    v12 = call fn0(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11)
+    return v12
 }
 
 ; VCode:
 ;   stmg %r14, %r15, 112(%r15)
-;   aghi %r15, -224
-;   virtual_sp_offset_adjust 160
-;   std %f8, 160(%r15)
-;   std %f9, 168(%r15)
-;   std %f10, 176(%r15)
-;   std %f11, 184(%r15)
-;   std %f12, 192(%r15)
-;   std %f13, 200(%r15)
-;   std %f14, 208(%r15)
-;   std %f15, 216(%r15)
+;   aghi %r15, -288
+;   virtual_sp_offset_adjust 224
+;   std %f8, 224(%r15)
+;   std %f9, 232(%r15)
+;   std %f10, 240(%r15)
+;   std %f11, 248(%r15)
+;   std %f12, 256(%r15)
+;   std %f13, 264(%r15)
+;   std %f14, 272(%r15)
+;   std %f15, 280(%r15)
 ; block0:
+;   vl %v17, 448(%r15)
+;   vl %v19, 464(%r15)
+;   vl %v21, 480(%r15)
+;   vl %v23, 496(%r15)
 ;   vpdi %v24, %v24, %v24, 4
-;   vpdi %v7, %v25, %v25, 4
-;   verllg %v25, %v7, 32
-;   vpdi %v19, %v26, %v26, 4
-;   verllg %v21, %v19, 32
-;   verllf %v26, %v21, 16
-;   vpdi %v27, %v27, %v27, 4
-;   verllg %v27, %v27, 32
-;   verllf %v29, %v27, 16
-;   verllh %v27, %v29, 8
+;   vpdi %v0, %v25, %v25, 4
+;   verllg %v25, %v0, 32
+;   vpdi %v0, %v26, %v26, 4
+;   verllg %v1, %v0, 32
+;   verllf %v26, %v1, 16
+;   vpdi %v5, %v27, %v27, 4
+;   verllg %v7, %v5, 32
+;   verllf %v18, %v7, 16
+;   verllh %v27, %v18, 8
+;   vpdi %v28, %v28, %v28, 4
+;   vpdi %v29, %v29, %v29, 4
+;   verllg %v29, %v29, 32
+;   vpdi %v30, %v30, %v30, 4
+;   verllg %v30, %v30, 32
+;   verllf %v30, %v30, 16
+;   vpdi %v1, %v31, %v31, 4
+;   verllg %v3, %v1, 32
+;   verllf %v5, %v3, 16
+;   verllh %v31, %v5, 8
+;   vpdi %v17, %v17, %v17, 4
+;   vst %v17, 160(%r15)
+;   vpdi %v20, %v19, %v19, 4
+;   verllg %v22, %v20, 32
+;   vst %v22, 176(%r15)
+;   vpdi %v0, %v21, %v21, 4
+;   verllg %v0, %v0, 32
+;   verllf %v0, %v0, 16
+;   vst %v0, 192(%r15)
+;   vpdi %v0, %v23, %v23, 4
+;   verllg %v2, %v0, 32
+;   verllf %v4, %v2, 16
+;   verllh %v6, %v4, 8
+;   vst %v6, 208(%r15)
 ;   bras %r1, 12 ; data %callee_be + 0 ; lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   vpdi %v5, %v24, %v24, 4
-;   verllg %v24, %v5, 32
-;   ld %f8, 160(%r15)
-;   ld %f9, 168(%r15)
-;   ld %f10, 176(%r15)
-;   ld %f11, 184(%r15)
-;   ld %f12, 192(%r15)
-;   ld %f13, 200(%r15)
-;   ld %f14, 208(%r15)
-;   ld %f15, 216(%r15)
-;   lmg %r14, %r15, 336(%r15)
+;   vpdi %v21, %v24, %v24, 4
+;   verllg %v24, %v21, 32
+;   ld %f8, 224(%r15)
+;   ld %f9, 232(%r15)
+;   ld %f10, 240(%r15)
+;   ld %f11, 248(%r15)
+;   ld %f12, 256(%r15)
+;   ld %f13, 264(%r15)
+;   ld %f14, 272(%r15)
+;   ld %f15, 280(%r15)
+;   lmg %r14, %r15, 400(%r15)
 ;   br %r14
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   stmg %r14, %r15, 0x70(%r15)
-;   aghi %r15, -0xe0
-;   std %f8, 0xa0(%r15)
-;   std %f9, 0xa8(%r15)
-;   std %f10, 0xb0(%r15)
-;   std %f11, 0xb8(%r15)
-;   std %f12, 0xc0(%r15)
-;   std %f13, 0xc8(%r15)
-;   std %f14, 0xd0(%r15)
-;   std %f15, 0xd8(%r15)
+;   aghi %r15, -0x120
+;   std %f8, 0xe0(%r15)
+;   std %f9, 0xe8(%r15)
+;   std %f10, 0xf0(%r15)
+;   std %f11, 0xf8(%r15)
+;   std %f12, 0x100(%r15)
+;   std %f13, 0x108(%r15)
+;   std %f14, 0x110(%r15)
+;   std %f15, 0x118(%r15)
 ; block1: ; offset 0x2a
+;   vl %v17, 0x1c0(%r15)
+;   vl %v19, 0x1d0(%r15)
+;   vl %v21, 0x1e0(%r15)
+;   vl %v23, 0x1f0(%r15)
 ;   vpdi %v24, %v24, %v24, 4
-;   vpdi %v7, %v25, %v25, 4
-;   verllg %v25, %v7, 0x20
-;   vpdi %v19, %v26, %v26, 4
-;   verllg %v21, %v19, 0x20
-;   verllf %v26, %v21, 0x10
-;   vpdi %v27, %v27, %v27, 4
-;   verllg %v27, %v27, 0x20
-;   verllf %v29, %v27, 0x10
-;   verllh %v27, %v29, 8
-;   bras %r1, 0x72
+;   vpdi %v0, %v25, %v25, 4
+;   verllg %v25, %v0, 0x20
+;   vpdi %v0, %v26, %v26, 4
+;   verllg %v1, %v0, 0x20
+;   verllf %v26, %v1, 0x10
+;   vpdi %v5, %v27, %v27, 4
+;   verllg %v7, %v5, 0x20
+;   verllf %v18, %v7, 0x10
+;   verllh %v27, %v18, 8
+;   vpdi %v28, %v28, %v28, 4
+;   vpdi %v29, %v29, %v29, 4
+;   verllg %v29, %v29, 0x20
+;   vpdi %v30, %v30, %v30, 4
+;   verllg %v30, %v30, 0x20
+;   verllf %v30, %v30, 0x10
+;   vpdi %v1, %v31, %v31, 4
+;   verllg %v3, %v1, 0x20
+;   verllf %v5, %v3, 0x10
+;   verllh %v31, %v5, 8
+;   vpdi %v17, %v17, %v17, 4
+;   vst %v17, 0xa0(%r15)
+;   vpdi %v20, %v19, %v19, 4
+;   verllg %v22, %v20, 0x20
+;   vst %v22, 0xb0(%r15)
+;   vpdi %v0, %v21, %v21, 4
+;   verllg %v0, %v0, 0x20
+;   verllf %v0, %v0, 0x10
+;   vst %v0, 0xc0(%r15)
+;   vpdi %v0, %v23, %v23, 4
+;   verllg %v2, %v0, 0x20
+;   verllf %v4, %v2, 0x10
+;   verllh %v6, %v4, 8
+;   vst %v6, 0xd0(%r15)
+;   bras %r1, 0x11a
 ;   .byte 0x00, 0x00 ; reloc_external Abs8 %callee_be 0
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   vpdi %v5, %v24, %v24, 4
-;   verllg %v24, %v5, 0x20
-;   ld %f8, 0xa0(%r15)
-;   ld %f9, 0xa8(%r15)
-;   ld %f10, 0xb0(%r15)
-;   ld %f11, 0xb8(%r15)
-;   ld %f12, 0xc0(%r15)
-;   ld %f13, 0xc8(%r15)
-;   ld %f14, 0xd0(%r15)
-;   ld %f15, 0xd8(%r15)
-;   lmg %r14, %r15, 0x150(%r15)
+;   vpdi %v21, %v24, %v24, 4
+;   verllg %v24, %v21, 0x20
+;   ld %f8, 0xe0(%r15)
+;   ld %f9, 0xe8(%r15)
+;   ld %f10, 0xf0(%r15)
+;   ld %f11, 0xf8(%r15)
+;   ld %f12, 0x100(%r15)
+;   ld %f13, 0x108(%r15)
+;   ld %f14, 0x110(%r15)
+;   ld %f15, 0x118(%r15)
+;   lmg %r14, %r15, 0x190(%r15)
 ;   br %r14
 
-function %caller_le_to_le(i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v {
-    fn0 = %callee_le(i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v
+function %caller_le_to_le(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v {
+    fn0 = %callee_le(i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16, i64x2, i32x4, i16x8, i8x16) -> i32x4 wasmtime_system_v
 
-block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16):
-    v4 = call fn0(v0, v1, v2, v3)
-    return v4
+block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16, v4: i64x2, v5: i32x4, v6: i16x8, v7: i8x16, v8: i64x2, v9: i32x4, v10: i16x8, v11: i8x16):
+    v12 = call fn0(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11)
+    return v12
 }
 
 ; VCode:
 ;   stmg %r14, %r15, 112(%r15)
-;   aghi %r15, -160
-;   virtual_sp_offset_adjust 160
+;   aghi %r15, -224
+;   virtual_sp_offset_adjust 224
 ; block0:
+;   vl %v17, 384(%r15)
+;   vl %v19, 400(%r15)
+;   vl %v21, 416(%r15)
+;   vl %v23, 432(%r15)
+;   vst %v17, 160(%r15)
+;   vst %v19, 176(%r15)
+;   vst %v21, 192(%r15)
+;   vst %v23, 208(%r15)
 ;   bras %r1, 12 ; data %callee_le + 0 ; lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   lmg %r14, %r15, 272(%r15)
+;   lmg %r14, %r15, 336(%r15)
 ;   br %r14
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   stmg %r14, %r15, 0x70(%r15)
-;   aghi %r15, -0xa0
+;   aghi %r15, -0xe0
 ; block1: ; offset 0xa
-;   bras %r1, 0x16
+;   vl %v17, 0x180(%r15)
+;   vl %v19, 0x190(%r15)
+;   vl %v21, 0x1a0(%r15)
+;   vl %v23, 0x1b0(%r15)
+;   vst %v17, 0xa0(%r15)
+;   vst %v19, 0xb0(%r15)
+;   vst %v21, 0xc0(%r15)
+;   vst %v23, 0xd0(%r15)
+;   bras %r1, 0x46
 ;   .byte 0x00, 0x00 ; reloc_external Abs8 %callee_le 0
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   .byte 0x00, 0x00
 ;   lg %r4, 0(%r1)
 ;   basr %r14, %r4
-;   lmg %r14, %r15, 0x110(%r15)
+;   lmg %r14, %r15, 0x150(%r15)
 ;   br %r14