Repository: bytecodealliance/wasmtime
Pull request #5898 (2023-03-01 05:29:19)

Codegen fix  atomic_rmw_loop missing move result to `dst` register On riscv64.
This will fix https://github.com/bytecodealliance/wasmtime/issues/5884.
I forget move   the result to `dst` register.


Patch:
diff --git a/.gitmodules b/.gitmodules
index ee264b99c..ba7f67934 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -13,3 +13,6 @@
 [submodule "crates/wasi-crypto/spec"]
 	path = crates/wasi-crypto/spec
 	url = https://github.com/WebAssembly/wasi-crypto.git
+[submodule "tests/wasi_testsuite/wasi-threads"]
+	path = tests/wasi_testsuite/wasi-threads
+	url = https://github.com/WebAssembly/wasi-threads
diff --git a/Cargo.lock b/Cargo.lock
index a819a215d..8c6c4ef5d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3674,6 +3674,7 @@ dependencies = [
  "cranelift-filetests",
  "cranelift-fuzzgen",
  "cranelift-interpreter",
+ "cranelift-native",
  "cranelift-reader",
  "cranelift-wasm",
  "libfuzzer-sys",
diff --git a/Cargo.toml b/Cargo.toml
index 407ab2fd0..e92983c12 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -70,6 +70,8 @@ component-macro-test = { path = "crates/misc/component-macro-test" }
 component-test-util = { workspace = true }
 bstr = "0.2.17"
 libc = "0.2.60"
+serde = "1.0"
+serde_json = "1.0"
 
 [target.'cfg(windows)'.dev-dependencies]
 windows-sys = { workspace = true, features = ["Win32_System_Memory"] }
diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index 01ccefb7e..488296999 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -66,7 +66,7 @@ fn gen_formats(formats: &[&InstructionFormat], fmt: &mut Formatter) {
 /// 16 bytes on 64-bit architectures. If more space is needed to represent an instruction, use a
 /// `ValueList` to store the additional information out of line.
 fn gen_instruction_data(formats: &[&InstructionFormat], fmt: &mut Formatter) {
-    fmt.line("#[derive(Copy, Clone, Debug, PartialEq, Hash)]");
+    fmt.line("#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]");
     fmt.line(r#"#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]"#);
     fmt.line("#[allow(missing_docs)]");
     fmtln!(fmt, "pub enum InstructionData {");
diff --git a/cranelift/codegen/src/data_value.rs b/cranelift/codegen/src/data_value.rs
index 2da55fc9c..6cba82c75 100644
--- a/cranelift/codegen/src/data_value.rs
+++ b/cranelift/codegen/src/data_value.rs
@@ -101,12 +101,55 @@ impl DataValue {
         }
     }
 
-    /// Write a [DataValue] to a slice.
+    fn swap_bytes(self) -> Self {
+        match self {
+            DataValue::I8(i) => DataValue::I8(i.swap_bytes()),
+            DataValue::I16(i) => DataValue::I16(i.swap_bytes()),
+            DataValue::I32(i) => DataValue::I32(i.swap_bytes()),
+            DataValue::I64(i) => DataValue::I64(i.swap_bytes()),
+            DataValue::I128(i) => DataValue::I128(i.swap_bytes()),
+            DataValue::U8(i) => DataValue::U8(i.swap_bytes()),
+            DataValue::U16(i) => DataValue::U16(i.swap_bytes()),
+            DataValue::U32(i) => DataValue::U32(i.swap_bytes()),
+            DataValue::U64(i) => DataValue::U64(i.swap_bytes()),
+            DataValue::U128(i) => DataValue::U128(i.swap_bytes()),
+            DataValue::F32(f) => DataValue::F32(Ieee32::with_bits(f.bits().swap_bytes())),
+            DataValue::F64(f) => DataValue::F64(Ieee64::with_bits(f.bits().swap_bytes())),
+            DataValue::V128(mut v) => {
+                v.reverse();
+                DataValue::V128(v)
+            }
+            DataValue::V64(mut v) => {
+                v.reverse();
+                DataValue::V64(v)
+            }
+        }
+    }
+
+    /// Converts `self` to big endian from target's endianness.
+    pub fn to_be(self) -> Self {
+        if cfg!(target_endian = "big") {
+            self
+        } else {
+            self.swap_bytes()
+        }
+    }
+
+    /// Converts `self` to little endian from target's endianness.
+    pub fn to_le(self) -> Self {
+        if cfg!(target_endian = "little") {
+            self
+        } else {
+            self.swap_bytes()
+        }
+    }
+
+    /// Write a [DataValue] to a slice in native-endian byte order.
     ///
     /// # Panics:
     ///
     /// Panics if the slice does not have enough space to accommodate the [DataValue]
-    pub fn write_to_slice(&self, dst: &mut [u8]) {
+    pub fn write_to_slice_ne(&self, dst: &mut [u8]) {
         match self {
             DataValue::I8(i) => dst[..1].copy_from_slice(&i.to_ne_bytes()[..]),
             DataValue::I16(i) => dst[..2].copy_from_slice(&i.to_ne_bytes()[..]),
@@ -121,12 +164,30 @@ impl DataValue {
         };
     }
 
-    /// Read a [DataValue] from a slice using a given [Type].
+    /// Write a [DataValue] to a slice in big-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn write_to_slice_be(&self, dst: &mut [u8]) {
+        self.clone().to_be().write_to_slice_ne(dst);
+    }
+
+    /// Write a [DataValue] to a slice in little-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn write_to_slice_le(&self, dst: &mut [u8]) {
+        self.clone().to_le().write_to_slice_ne(dst);
+    }
+
+    /// Read a [DataValue] from a slice using a given [Type] with native-endian byte order.
     ///
     /// # Panics:
     ///
     /// Panics if the slice does not have enough space to accommodate the [DataValue]
-    pub fn read_from_slice(src: &[u8], ty: Type) -> Self {
+    pub fn read_from_slice_ne(src: &[u8], ty: Type) -> Self {
         match ty {
             types::I8 => DataValue::I8(i8::from_ne_bytes(src[..1].try_into().unwrap())),
             types::I16 => DataValue::I16(i16::from_ne_bytes(src[..2].try_into().unwrap())),
@@ -152,15 +213,33 @@ impl DataValue {
         }
     }
 
-    /// Write a [DataValue] to a memory location.
+    /// Read a [DataValue] from a slice using a given [Type] in big-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn read_from_slice_be(src: &[u8], ty: Type) -> Self {
+        DataValue::read_from_slice_ne(src, ty).to_be()
+    }
+
+    /// Read a [DataValue] from a slice using a given [Type] in little-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn read_from_slice_le(src: &[u8], ty: Type) -> Self {
+        DataValue::read_from_slice_ne(src, ty).to_le()
+    }
+
+    /// Write a [DataValue] to a memory location in native-endian byte order.
     pub unsafe fn write_value_to(&self, p: *mut u128) {
         let size = self.ty().bytes() as usize;
-        self.write_to_slice(std::slice::from_raw_parts_mut(p as *mut u8, size));
+        self.write_to_slice_ne(std::slice::from_raw_parts_mut(p as *mut u8, size));
     }
 
-    /// Read a [DataValue] from a memory location using a given [Type].
+    /// Read a [DataValue] from a memory location using a given [Type] in native-endian byte order.
     pub unsafe fn read_value_from(p: *const u128, ty: Type) -> Self {
-        DataValue::read_from_slice(
+        DataValue::read_from_slice_ne(
             std::slice::from_raw_parts(p as *const u8, ty.bytes() as usize),
             ty,
         )
diff --git a/cranelift/codegen/src/egraph.rs b/cranelift/codegen/src/egraph.rs
index 2cbe412e3..e12b6fed3 100644
--- a/cranelift/codegen/src/egraph.rs
+++ b/cranelift/codegen/src/egraph.rs
@@ -7,13 +7,14 @@ use crate::dominator_tree::DominatorTree;
 use crate::egraph::domtree::DomTreeWithChildren;
 use crate::egraph::elaborate::Elaborator;
 use crate::fx::FxHashSet;
-use crate::inst_predicates::is_pure_for_egraph;
+use crate::inst_predicates::{is_mergeable_for_egraph, is_pure_for_egraph};
 use crate::ir::{
-    DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
+    Block, DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
 };
 use crate::loop_analysis::LoopAnalysis;
 use crate::opts::generated_code::ContextIter;
 use crate::opts::IsleContext;
+use crate::scoped_hash_map::{Entry as ScopedEntry, ScopedHashMap};
 use crate::trace;
 use crate::unionfind::UnionFind;
 use cranelift_entity::packed_option::ReservedValue;
@@ -77,6 +78,7 @@ where
     pub(crate) func: &'opt mut Function,
     pub(crate) value_to_opt_value: &'opt mut SecondaryMap<Value, Value>,
     pub(crate) gvn_map: &'opt mut CtxHashMap<(Type, InstructionData), Value>,
+    pub(crate) effectful_gvn_map: &'opt mut ScopedHashMap<(Type, InstructionData), Value>,
     pub(crate) eclasses: &'opt mut UnionFind<Value>,
     pub(crate) remat_values: &'opt mut FxHashSet<Value>,
     pub(crate) stats: &'opt mut Stats,
@@ -285,10 +287,49 @@ where
     fn optimize_skeleton_inst(&mut self, inst: Inst) -> bool {
         self.stats.skeleton_inst += 1;
 
-        // If a load or store, process it with the alias analysis to see
-        // if we can optimize it (rewrite in terms of an earlier load or
-        // stored value).
-        if let Some(new_result) =
+        // First, can we try to deduplicate? We need to keep some copy
+        // of the instruction around because it's side-effecting, but
+        // we may be able to reuse an earlier instance of it.
+        if is_mergeable_for_egraph(self.func, inst) {
+            let result = self.func.dfg.inst_results(inst)[0];
+            trace!(" -> mergeable side-effecting op {}", inst);
+
+            // Does this instruction already exist? If so, add entries to
+            // the value-map to rewrite uses of its results to the results
+            // of the original (existing) instruction. If not, optimize
+            // the new instruction.
+            //
+            // Note that we use the "effectful GVN map", which is
+            // scoped: because effectful ops are not removed from the
+            // skeleton (`Layout`), we need to be mindful of whether
+            // our current position is dominated by an instance of the
+            // instruction. (See #5796 for details.)
+            let ty = self.func.dfg.ctrl_typevar(inst);
+            match self
+                .effectful_gvn_map
+                .entry((ty, self.func.dfg.insts[inst].clone()))
+            {
+                ScopedEntry::Occupied(o) => {
+                    let orig_result = *o.get();
+                    // Hit in GVN map -- reuse value.
+                    self.value_to_opt_value[result] = orig_result;
+                    self.eclasses.union(orig_result, result);
+                    trace!(" -> merges result {} to {}", result, orig_result);
+                    true
+                }
+                ScopedEntry::Vacant(v) => {
+                    // Otherwise, insert it into the value-map.
+                    self.value_to_opt_value[result] = result;
+                    v.insert(result);
+                    trace!(" -> inserts as new (no GVN)");
+                    false
+                }
+            }
+        }
+        // Otherwise, if a load or store, process it with the alias
+        // analysis to see if we can optimize it (rewrite in terms of
+        // an earlier load or stored value).
+        else if let Some(new_result) =
             self.alias_analysis
                 .process_inst(self.func, self.alias_analysis_state, inst)
         {
@@ -382,83 +423,127 @@ impl<'a> EgraphPass<'a> {
         let mut cursor = FuncCursor::new(self.func);
         let mut value_to_opt_value: SecondaryMap<Value, Value> =
             SecondaryMap::with_default(Value::reserved_value());
+        // Map from instruction to value for hash-consing of pure ops
+        // into the egraph. This can be a standard (non-scoped)
+        // hashmap because pure ops have no location: they are
+        // "outside of" control flow.
+        //
+        // Note also that we keep the controlling typevar (the `Type`
+        // in the tuple below) because it may disambiguate
+        // instructions that are identical except for type.
         let mut gvn_map: CtxHashMap<(Type, InstructionData), Value> =
             CtxHashMap::with_capacity(cursor.func.dfg.num_values());
+        // Map from instruction to value for GVN'ing of effectful but
+        // idempotent ops, which remain in the side-effecting
+        // skeleton. This needs to be scoped because we cannot
+        // deduplicate one instruction to another that is in a
+        // non-dominating block.
+        //
+        // Note that we can use a ScopedHashMap here without the
+        // "context" (as needed by CtxHashMap) because in practice the
+        // ops we want to GVN have all their args inline. Equality on
+        // the InstructionData itself is conservative: two insts whose
+        // struct contents compare shallowly equal are definitely
+        // identical, but identical insts in a deep-equality sense may
+        // not compare shallowly equal, due to list indirection. This
+        // is fine for GVN, because it is still sound to skip any
+        // given GVN opportunity (and keep the original instructions).
+        //
+        // As above, we keep the controlling typevar here as part of
+        // the key: effectful instructions may (as for pure
+        // instructions) be differentiated only on the type.
+        let mut effectful_gvn_map: ScopedHashMap<(Type, InstructionData), Value> =
+            ScopedHashMap::new();
 
         // In domtree preorder, visit blocks. (TODO: factor out an
         // iterator from this and elaborator.)
         let root = self.domtree_children.root();
-        let mut block_stack = vec![root];
-        while let Some(block) = block_stack.pop() {
-            // We popped this block; push children
-            // immediately, then process this block.
-            block_stack.extend(self.domtree_children.children(block));
-
-            trace!("Processing block {}", block);
-            cursor.set_position(CursorPosition::Before(block));
-
-            let mut alias_analysis_state = self.alias_analysis.block_starting_state(block);
-
-            for &param in cursor.func.dfg.block_params(block) {
-                trace!("creating initial singleton eclass for blockparam {}", param);
-                self.eclasses.add(param);
-                value_to_opt_value[param] = param;
-            }
-            while let Some(inst) = cursor.next_inst() {
-                trace!("Processing inst {}", inst);
-
-                // While we're passing over all insts, create initial
-                // singleton eclasses for all result and blockparam
-                // values.  Also do initial analysis of all inst
-                // results.
-                for &result in cursor.func.dfg.inst_results(inst) {
-                    trace!("creating initial singleton eclass for {}", result);
-                    self.eclasses.add(result);
-                }
+        enum StackEntry {
+            Visit(Block),
+            Pop,
+        }
+        let mut block_stack = vec![StackEntry::Visit(root)];
+        while let Some(entry) = block_stack.pop() {
+            match entry {
+                StackEntry::Visit(block) => {
+                    // We popped this block; push children
+                    // immediately, then process this block.
+                    block_stack.push(StackEntry::Pop);
+                    block_stack
+                        .extend(self.domtree_children.children(block).map(StackEntry::Visit));
+                    effectful_gvn_map.increment_depth();
+
+                    trace!("Processing block {}", block);
+                    cursor.set_position(CursorPosition::Before(block));
+
+                    let mut alias_analysis_state = self.alias_analysis.block_starting_state(block);
+
+                    for &param in cursor.func.dfg.block_params(block) {
+                        trace!("creating initial singleton eclass for blockparam {}", param);
+                        self.eclasses.add(param);
+                        value_to_opt_value[param] = param;
+                    }
+                    while let Some(inst) = cursor.next_inst() {
+                        trace!("Processing inst {}", inst);
+
+                        // While we're passing over all insts, create initial
+                        // singleton eclasses for all result and blockparam
+                        // values.  Also do initial analysis of all inst
+                        // results.
+                        for &result in cursor.func.dfg.inst_results(inst) {
+                            trace!("creating initial singleton eclass for {}", result);
+                            self.eclasses.add(result);
+                        }
 
-                // Rewrite args of *all* instructions using the
-                // value-to-opt-value map.
-                cursor.func.dfg.resolve_aliases_in_arguments(inst);
-                cursor.func.dfg.map_inst_values(inst, |_, arg| {
-                    let new_value = value_to_opt_value[arg];
-                    trace!("rewriting arg {} of inst {} to {}", arg, inst, new_value);
-                    debug_assert_ne!(new_value, Value::reserved_value());
-                    new_value
-                });
-
-                // Build a context for optimization, with borrows of
-                // state. We can't invoke a method on `self` because
-                // we've borrowed `self.func` mutably (as
-                // `cursor.func`) so we pull apart the pieces instead
-                // here.
-                let mut ctx = OptimizeCtx {
-                    func: cursor.func,
-                    value_to_opt_value: &mut value_to_opt_value,
-                    gvn_map: &mut gvn_map,
-                    eclasses: &mut self.eclasses,
-                    rewrite_depth: 0,
-                    subsume_values: FxHashSet::default(),
-                    remat_values: &mut self.remat_values,
-                    stats: &mut self.stats,
-                    alias_analysis: self.alias_analysis,
-                    alias_analysis_state: &mut alias_analysis_state,
-                };
-
-                if is_pure_for_egraph(ctx.func, inst) {
-                    // Insert into GVN map and optimize any new nodes
-                    // inserted (recursively performing this work for
-                    // any nodes the optimization rules produce).
-                    let inst = NewOrExistingInst::Existing(inst);
-                    ctx.insert_pure_enode(inst);
-                    // We've now rewritten all uses, or will when we
-                    // see them, and the instruction exists as a pure
-                    // enode in the eclass, so we can remove it.
-                    cursor.remove_inst_and_step_back();
-                } else {
-                    if ctx.optimize_skeleton_inst(inst) {
-                        cursor.remove_inst_and_step_back();
+                        // Rewrite args of *all* instructions using the
+                        // value-to-opt-value map.
+                        cursor.func.dfg.resolve_aliases_in_arguments(inst);
+                        cursor.func.dfg.map_inst_values(inst, |_, arg| {
+                            let new_value = value_to_opt_value[arg];
+                            trace!("rewriting arg {} of inst {} to {}", arg, inst, new_value);
+                            debug_assert_ne!(new_value, Value::reserved_value());
+                            new_value
+                        });
+
+                        // Build a context for optimization, with borrows of
+                        // state. We can't invoke a method on `self` because
+                        // we've borrowed `self.func` mutably (as
+                        // `cursor.func`) so we pull apart the pieces instead
+                        // here.
+                        let mut ctx = OptimizeCtx {
+                            func: cursor.func,
+                            value_to_opt_value: &mut value_to_opt_value,
+                            gvn_map: &mut gvn_map,
+                            effectful_gvn_map: &mut effectful_gvn_map,
+                            eclasses: &mut self.eclasses,
+                            rewrite_depth: 0,
+                            subsume_values: FxHashSet::default(),
+                            remat_values: &mut self.remat_values,
+                            stats: &mut self.stats,
+                            alias_analysis: self.alias_analysis,
+                            alias_analysis_state: &mut alias_analysis_state,
+                        };
+
+                        if is_pure_for_egraph(ctx.func, inst) {
+                            // Insert into GVN map and optimize any new nodes
+                            // inserted (recursively performing this work for
+                            // any nodes the optimization rules produce).
+                            let inst = NewOrExistingInst::Existing(inst);
+                            ctx.insert_pure_enode(inst);
+                            // We've now rewritten all uses, or will when we
+                            // see them, and the instruction exists as a pure
+                            // enode in the eclass, so we can remove it.
+                            cursor.remove_inst_and_step_back();
+                        } else {
+                            if ctx.optimize_skeleton_inst(inst) {
+                                cursor.remove_inst_and_step_back();
+                            }
+                        }
                     }
                 }
+                StackEntry::Pop => {
+                    effectful_gvn_map.decrement_depth();
+                }
             }
         }
     }
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index 9b284e8b8..f82f41daa 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -73,6 +73,25 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
     has_one_result && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
 }
 
+/// Can the given instruction be merged into another copy of itself?
+/// These instructions may have side-effects, but as long as we retain
+/// the first instance of the instruction, the second and further
+/// instances are redundant if they would produce the same trap or
+/// result.
+pub fn is_mergeable_for_egraph(func: &Function, inst: Inst) -> bool {
+    let op = func.dfg.insts[inst].opcode();
+    // We can only merge one-result operators due to the way that GVN
+    // is structured in the egraph implementation.
+    let has_one_result = func.dfg.inst_results(inst).len() == 1;
+    has_one_result
+        // Loads/stores are handled by alias analysis and not
+        // otherwise mergeable.
+        && !op.can_load()
+        && !op.can_store()
+        // Can only have idempotent side-effects.
+        && (!has_side_effect(func, inst) || op.side_effects_idempotent())
+}
+
 /// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,
 /// but not the get_pinned_reg opcode?
 pub fn has_lowering_side_effect(func: &Function, inst: Inst) -> bool {
diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs
index 97e0ac0b5..40507db01 100644
--- a/cranelift/codegen/src/ir/instructions.rs
+++ b/cranelift/codegen/src/ir/instructions.rs
@@ -45,7 +45,7 @@ pub type ValueListPool = entity::ListPool<Value>;
 /// The BlockCall::new function guarantees this layout by requiring a block argument that's written
 /// in as the first element of the EntityList. Any subsequent entries are always assumed to be real
 /// Values.
-#[derive(Debug, Clone, Copy, PartialEq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct BlockCall {
     /// The underlying storage for the BlockCall. The first element of the values EntityList is
@@ -403,7 +403,9 @@ impl InstructionData {
             &InstructionData::Load { flags, .. }
             | &InstructionData::LoadNoOffset { flags, .. }
             | &InstructionData::Store { flags, .. }
-            | &InstructionData::StoreNoOffset { flags, .. } => Some(flags),
+            | &InstructionData::StoreNoOffset { flags, .. }
+            | &InstructionData::AtomicCas { flags, .. }
+            | &InstructionData::AtomicRmw { flags, .. } => Some(flags),
             _ => None,
         }
     }
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index b8bf2ef48..53b0e65ed 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -852,7 +852,7 @@
         (rd WritableReg)
         ;; Offset in range -2^20 .. 2^20.
         (off i32))
-      
+
        ;; Compute the address (using a PC-relative offset) of a 4KB page.
        (Adrp
         (rd WritableReg)
@@ -1401,6 +1401,8 @@
     (Bsl)
     ;; Floating-point fused multiply-add vectors
     (Fmla)
+    ;; Floating-point fused multiply-subtract vectors
+    (Fmls)
 ))
 
 ;; A Vector miscellaneous operation with two registers.
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 90a5dbd93..72b1e1176 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -2906,6 +2906,9 @@ impl MachInstEmit for Inst {
                     VecALUModOp::Fmla => {
                         (0b000_01110_00_1 | (size.enc_float_size() << 1), 0b110011)
                     }
+                    VecALUModOp::Fmls => {
+                        (0b000_01110_10_1 | (size.enc_float_size() << 1), 0b110011)
+                    }
                 };
                 sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
             }
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 182660335..779782907 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -2363,6 +2363,7 @@ impl Inst {
                 let (op, size) = match alu_op {
                     VecALUModOp::Bsl => ("bsl", VectorSize::Size8x16),
                     VecALUModOp::Fmla => ("fmla", size),
+                    VecALUModOp::Fmls => ("fmls", size),
                 };
                 let rd = pretty_print_vreg_vector(rd.to_reg(), size, allocs);
                 let ri = pretty_print_vreg_vector(ri, size, allocs);
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 3eec28c11..bd7e968d7 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -404,7 +404,13 @@
 (rule (lower (has_type ty @ (multi_lane _ _) (fma x y z)))
       (vec_rrr_mod (VecALUModOp.Fmla) z x y (vector_size ty)))
 
-(rule 1 (lower (has_type (ty_scalar_float ty) (fma x y z)))
+(rule 1 (lower (has_type ty @ (multi_lane _ _) (fma (fneg x) y z)))
+      (vec_rrr_mod (VecALUModOp.Fmls) z x y (vector_size ty)))
+
+(rule 2 (lower (has_type ty @ (multi_lane _ _) (fma x (fneg y) z)))
+      (vec_rrr_mod (VecALUModOp.Fmls) z x y (vector_size ty)))
+
+(rule 3 (lower (has_type (ty_scalar_float ty) (fma x y z)))
       (fpu_rrrr (FPUOp3.MAdd) (scalar_size ty) x y z))
 
 ;;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index 007a2c38f..6ae0b541d 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -1375,7 +1375,7 @@ impl MachInstEmit for Inst {
                     | crate::ir::AtomicRmwOp::And
                     | crate::ir::AtomicRmwOp::Or
                     | crate::ir::AtomicRmwOp::Xor => {
-                        AtomicOP::extract(t0, offset, dst.to_reg(), ty)
+                        AtomicOP::extract(dst, offset, dst.to_reg(), ty)
                             .iter()
                             .for_each(|i| i.emit(&[], sink, emit_info, state));
                         Inst::AluRRR {
@@ -1388,7 +1388,7 @@ impl MachInstEmit for Inst {
                                 _ => unreachable!(),
                             },
                             rd: t0,
-                            rs1: t0.to_reg(),
+                            rs1: dst.to_reg(),
                             rs2: x,
                         }
                         .emit(&[], sink, emit_info, state);
@@ -1412,19 +1412,16 @@ impl MachInstEmit for Inst {
                         spilltmp_reg2()
                     }
                     crate::ir::AtomicRmwOp::Nand => {
-                        let x2 = if ty.bits() < 32 {
-                            AtomicOP::extract(t0, offset, dst.to_reg(), ty)
+                        if ty.bits() < 32 {
+                            AtomicOP::extract(dst, offset, dst.to_reg(), ty)
                                 .iter()
                                 .for_each(|i| i.emit(&[], sink, emit_info, state));
-                            t0.to_reg()
-                        } else {
-                            dst.to_reg()
-                        };
+                        }
                         Inst::AluRRR {
                             alu_op: AluOPRRR::And,
                             rd: t0,
                             rs1: x,
-                            rs2: x2,
+                            rs2: dst.to_reg(),
                         }
                         .emit(&[], sink, emit_info, state);
                         Inst::construct_bit_not(t0, t0.to_reg()).emit(&[], sink, emit_info, state);
@@ -1456,12 +1453,13 @@ impl MachInstEmit for Inst {
                     | crate::ir::AtomicRmwOp::Umax
                     | crate::ir::AtomicRmwOp::Smin
                     | crate::ir::AtomicRmwOp::Smax => {
+                        let label_select_dst = sink.get_label();
                         let label_select_done = sink.get_label();
                         if op == crate::ir::AtomicRmwOp::Umin || op == crate::ir::AtomicRmwOp::Umax
                         {
-                            AtomicOP::extract(t0, offset, dst.to_reg(), ty)
+                            AtomicOP::extract(dst, offset, dst.to_reg(), ty)
                         } else {
-                            AtomicOP::extract_sext(t0, offset, dst.to_reg(), ty)
+                            AtomicOP::extract_sext(dst, offset, dst.to_reg(), ty)
                         }
                         .iter()
                         .for_each(|i| i.emit(&[], sink, emit_info, state));
@@ -1473,9 +1471,9 @@ impl MachInstEmit for Inst {
                                 crate::ir::AtomicRmwOp::Smax => IntCC::SignedGreaterThan,
                                 _ => unreachable!(),
                             },
-                            ValueRegs::one(t0.to_reg()),
+                            ValueRegs::one(dst.to_reg()),
                             ValueRegs::one(x),
-                            BranchTarget::Label(label_select_done),
+                            BranchTarget::Label(label_select_dst),
                             BranchTarget::zero(),
                             ty,
                         )
@@ -1483,6 +1481,12 @@ impl MachInstEmit for Inst {
                         .for_each(|i| i.emit(&[], sink, emit_info, state));
                         // here we select x.
                         Inst::gen_move(t0, x, I64).emit(&[], sink, emit_info, state);
+                        Inst::Jal {
+                            dest: BranchTarget::Label(label_select_done),
+                        }
+                        .emit(&[], sink, emit_info, state);
+                        sink.bind_label(label_select_dst);
+                        Inst::gen_move(t0, dst.to_reg(), I64).emit(&[], sink, emit_info, state);
                         sink.bind_label(label_select_done);
                         Inst::Atomic {
                             op: AtomicOP::load_op(ty),
@@ -1504,6 +1508,9 @@ impl MachInstEmit for Inst {
                         spilltmp_reg2()
                     }
                     crate::ir::AtomicRmwOp::Xchg => {
+                        AtomicOP::extract(dst, offset, dst.to_reg(), ty)
+                            .iter()
+                            .for_each(|i| i.emit(&[], sink, emit_info, state));
                         Inst::Atomic {
                             op: AtomicOP::load_op(ty),
                             rd: writable_spilltmp_reg2(),
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 38c1f2de7..ec21968b3 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -522,12 +522,10 @@
             ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be
             ;; correct for half of the lanes; the others must be fixed up with
             ;; the mask below.
-            (unmasked Xmm (x64_psrlw src (mov_rmi_to_xmm masked_amt)))
-            (mask_addr SyntheticAmode (ushr_i8x16_mask masked_amt))
-            (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None))))
+            (unmasked Xmm (x64_psrlw src (mov_rmi_to_xmm masked_amt))))
         (sse_and $I8X16
                  unmasked
-                 (RegMem.Reg mask))))
+                 (ushr_i8x16_mask masked_amt))))
 
 ;; Get the address of the mask to use when fixing up the lanes that weren't
 ;; correctly generated by the 16x8 shift.
@@ -3510,6 +3508,16 @@
 
 ;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; Special case for the `punpckhbw` instruction which interleaves the upper
+;; lanes of the two input registers.
+(rule 4 (lower (shuffle a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))
+      (x64_punpckhbw a b))
+
+;; Special case for the `punpcklbw` instruction which interleaves the lower
+;; lanes of the two input registers.
+(rule 4 (lower (shuffle a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))
+      (x64_punpcklbw a b))
+
 ;; If `lhs` and `rhs` are the same we can use a single PSHUFB to shuffle the XMM
 ;; register. We statically build `constructed_mask` to zero out any unknown lane
 ;; indices (may not be completely necessary: verification could fail incorrect
diff --git a/cranelift/codegen/src/opts/algebraic.isle b/cranelift/codegen/src/opts/algebraic.isle
index ecd71988c..eac165065 100644
--- a/cranelift/codegen/src/opts/algebraic.isle
+++ b/cranelift/codegen/src/opts/algebraic.isle
@@ -527,3 +527,13 @@
 (rule (simplify
        (vselect ty (fcmp _ (FloatCC.GreaterThan) x y) x y))
       (fmax_pseudo ty x y))
+
+;; If both of the multiplied arguments to an `fma` are negated then remove
+;; both of them since they cancel out.
+(rule (simplify (fma ty (fneg ty x) (fneg ty y) z))
+      (fma ty x y z))
+
+;; If both of the multiplied arguments to an `fmul` are negated then remove
+;; both of them since they cancel out.
+(rule (simplify (fmul ty (fneg ty x) (fneg ty y)))
+      (fmul ty x y))
diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs
index 659b94a9b..682eb256f 100644
--- a/cranelift/entity/src/list.rs
+++ b/cranelift/entity/src/list.rs
@@ -62,7 +62,7 @@ use serde::{Deserialize, Serialize};
 ///
 /// The index stored in an `EntityList` points to part 2, the list elements. The value 0 is
 /// reserved for the empty list which isn't allocated in the vector.
-#[derive(Clone, Copy, Debug, PartialEq, Hash)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct EntityList<T: EntityRef + ReservedValue> {
     index: u32,
diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs
index 04d2815ba..26cca8363 100644
--- a/cranelift/fuzzgen/src/lib.rs
+++ b/cranelift/fuzzgen/src/lib.rs
@@ -7,190 +7,27 @@ use cranelift::codegen::data_value::DataValue;
 use cranelift::codegen::ir::{types::*, UserExternalName, UserFuncName};
 use cranelift::codegen::ir::{Function, LibCall};
 use cranelift::codegen::Context;
-use cranelift::prelude::isa;
 use cranelift::prelude::*;
 use cranelift_arbitrary::CraneliftArbitrary;
 use cranelift_native::builder_with_options;
-use std::fmt;
 use target_lexicon::{Architecture, Triple};
 
 mod config;
 mod cranelift_arbitrary;
 mod function_generator;
 mod passes;
+mod print;
 
-/// These libcalls need a interpreter implementation in `cranelift-fuzzgen.rs`
-const ALLOWED_LIBCALLS: &'static [LibCall] = &[
-    LibCall::CeilF32,
-    LibCall::CeilF64,
-    LibCall::FloorF32,
-    LibCall::FloorF64,
-    LibCall::TruncF32,
-    LibCall::TruncF64,
-];
+pub use print::PrintableTestCase;
 
 pub type TestCaseInput = Vec<DataValue>;
 
-/// Print only non default flags.
-fn write_non_default_flags(f: &mut fmt::Formatter<'_>, flags: &settings::Flags) -> fmt::Result {
-    let default_flags = settings::Flags::new(settings::builder());
-    for (default, flag) in default_flags.iter().zip(flags.iter()) {
-        assert_eq!(default.name, flag.name);
-
-        if default.value_string() != flag.value_string() {
-            writeln!(f, "set {}={}", flag.name, flag.value_string())?;
-        }
-    }
-
-    Ok(())
-}
-
-/// A generated function with an ISA that targets one of cranelift's backends.
-pub struct FunctionWithIsa {
-    /// TargetIsa to use when compiling this test case
-    pub isa: isa::OwnedTargetIsa,
-
-    /// Function under test
-    pub func: Function,
-}
-
-impl fmt::Debug for FunctionWithIsa {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, ";; Compile test case\n")?;
-
-        write_non_default_flags(f, self.isa.flags())?;
-
-        writeln!(f, "test compile")?;
-        writeln!(f, "target {}", self.isa.triple().architecture)?;
-        writeln!(f, "{}", self.func)?;
-
-        Ok(())
-    }
-}
-
-impl<'a> Arbitrary<'a> for FunctionWithIsa {
-    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
-        // We filter out targets that aren't supported in the current build
-        // configuration after randomly choosing one, instead of randomly choosing
-        // a supported one, so that the same fuzz input works across different build
-        // configurations.
-        let target = u.choose(isa::ALL_ARCHITECTURES)?;
-        let builder = isa::lookup_by_name(target).map_err(|_| arbitrary::Error::IncorrectFormat)?;
-        let architecture = builder.triple().architecture;
-
-        let mut gen = FuzzGen::new(u);
-        let flags = gen
-            .generate_flags(architecture)
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-        let isa = builder
-            .finish(flags)
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-
-        // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
-        let fname = UserFuncName::user(1, 0);
-
-        // We don't actually generate these functions, we just simulate their signatures and names
-        let func_count = gen.u.int_in_range(gen.config.testcase_funcs.clone())?;
-        let usercalls = (0..func_count)
-            .map(|i| {
-                let name = UserExternalName::new(2, i as u32);
-                let sig = gen.generate_signature(architecture)?;
-                Ok((name, sig))
-            })
-            .collect::<Result<Vec<(UserExternalName, Signature)>>>()
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-
-        let func = gen
-            .generate_func(fname, isa.triple().clone(), usercalls)
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-
-        Ok(FunctionWithIsa { isa, func })
-    }
-}
-
-pub struct TestCase {
-    /// TargetIsa to use when compiling this test case
-    pub isa: isa::OwnedTargetIsa,
-    /// Functions under test
-    /// By convention the first function is the main function.
-    pub functions: Vec<Function>,
-    /// Generate multiple test inputs for each test case.
-    /// This allows us to get more coverage per compilation, which may be somewhat expensive.
-    pub inputs: Vec<TestCaseInput>,
-}
-
-impl fmt::Debug for TestCase {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, ";; Fuzzgen test case\n")?;
-        writeln!(f, "test interpret")?;
-        writeln!(f, "test run")?;
-
-        write_non_default_flags(f, self.isa.flags())?;
-
-        writeln!(f, "target {}\n", self.isa.triple().architecture)?;
-
-        // Print the functions backwards, so that the main function is printed last
-        // and near the test inputs.
-        for func in self.functions.iter().rev() {
-            writeln!(f, "{}\n", func)?;
-        }
-
-        writeln!(f, "; Note: the results in the below test cases are simply a placeholder and probably will be wrong\n")?;
-
-        for input in self.inputs.iter() {
-            // TODO: We don't know the expected outputs, maybe we can run the interpreter
-            // here to figure them out? Should work, however we need to be careful to catch
-            // panics in case its the interpreter that is failing.
-            // For now create a placeholder output consisting of the zero value for the type
-            let returns = &self.main().signature.returns;
-            let placeholder_output = returns
-                .iter()
-                .map(|param| DataValue::read_from_slice(&[0; 16][..], param.value_type))
-                .map(|val| format!("{}", val))
-                .collect::<Vec<_>>()
-                .join(", ");
-
-            // If we have no output, we don't need the == condition
-            let test_condition = match returns.len() {
-                0 => String::new(),
-                1 => format!(" == {}", placeholder_output),
-                _ => format!(" == [{}]", placeholder_output),
-            };
-
-            let args = input
-                .iter()
-                .map(|val| format!("{}", val))
-                .collect::<Vec<_>>()
-                .join(", ");
-
-            writeln!(f, "; run: {}({}){}", self.main().name, args, test_condition)?;
-        }
-
-        Ok(())
-    }
-}
-
-impl<'a> Arbitrary<'a> for TestCase {
-    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
-        FuzzGen::new(u)
-            .generate_host_test()
-            .map_err(|_| arbitrary::Error::IncorrectFormat)
-    }
-}
-
-impl TestCase {
-    /// Returns the main function of this test case.
-    pub fn main(&self) -> &Function {
-        &self.functions[0]
-    }
-}
-
 pub struct FuzzGen<'r, 'data>
 where
     'data: 'r,
 {
-    u: &'r mut Unstructured<'data>,
-    config: Config,
+    pub u: &'r mut Unstructured<'data>,
+    pub config: Config,
 }
 
 impl<'r, 'data> FuzzGen<'r, 'data>
@@ -204,13 +41,13 @@ where
         }
     }
 
-    fn generate_signature(&mut self, architecture: Architecture) -> Result<Signature> {
+    pub fn generate_signature(&mut self, architecture: Architecture) -> Result<Signature> {
         let max_params = self.u.int_in_range(self.config.signature_params.clone())?;
         let max_rets = self.u.int_in_range(self.config.signature_rets.clone())?;
         Ok(self.u.signature(architecture, max_params, max_rets)?)
     }
 
-    fn generate_test_inputs(mut self, signature: &Signature) -> Result<Vec<TestCaseInput>> {
+    pub fn generate_test_inputs(mut self, signature: &Signature) -> Result<Vec<TestCaseInput>> {
         let mut inputs = Vec::new();
 
         // Generate up to "max_test_case_inputs" inputs, we need an upper bound here since
@@ -288,11 +125,12 @@ where
         Ok(ctx.func)
     }
 
-    fn generate_func(
+    pub fn generate_func(
         &mut self,
         name: UserFuncName,
         target_triple: Triple,
         usercalls: Vec<(UserExternalName, Signature)>,
+        libcalls: Vec<LibCall>,
     ) -> Result<Function> {
         let sig = self.generate_signature(target_triple.architecture)?;
 
@@ -303,7 +141,7 @@ where
             name,
             sig,
             usercalls,
-            ALLOWED_LIBCALLS.to_vec(),
+            libcalls,
         )
         .generate()?;
 
@@ -312,7 +150,7 @@ where
 
     /// Generate a random set of cranelift flags.
     /// Only semantics preserving flags are considered
-    fn generate_flags(&mut self, target_arch: Architecture) -> Result<Flags> {
+    pub fn generate_flags(&mut self, target_arch: Architecture) -> Result<Flags> {
         let mut builder = settings::builder();
 
         let opt = self.u.choose(OptLevel::all())?;
@@ -390,48 +228,4 @@ where
 
         Ok(Flags::new(builder))
     }
-
-    pub fn generate_host_test(mut self) -> Result<TestCase> {
-        // TestCase is meant to be consumed by a runner, so we make the assumption here that we're
-        // generating a TargetIsa for the host.
-        let builder =
-            builder_with_options(true).expect("Unable to build a TargetIsa for the current host");
-        let flags = self.generate_flags(builder.triple().architecture)?;
-        let isa = builder.finish(flags)?;
-
-        // When generating functions, we allow each function to call any function that has
-        // already been generated. This guarantees that we never have loops in the call graph.
-        // We generate these backwards, and then reverse them so that the main function is at
-        // the start.
-        let func_count = self.u.int_in_range(self.config.testcase_funcs.clone())?;
-        let mut functions: Vec<Function> = Vec::with_capacity(func_count);
-        for i in (0..func_count).rev() {
-            // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
-            let fname = UserFuncName::user(1, i as u32);
-
-            let usercalls: Vec<(UserExternalName, Signature)> = functions
-                .iter()
-                .map(|f| {
-                    (
-                        f.name.get_user().unwrap().clone(),
-                        f.stencil.signature.clone(),
-                    )
-                })
-                .collect();
-
-            let func = self.generate_func(fname, isa.triple().clone(), usercalls)?;
-            functions.push(func);
-        }
-        // Now reverse the functions so that the main function is at the start.
-        functions.reverse();
-
-        let main = &functions[0];
-        let inputs = self.generate_test_inputs(&main.signature)?;
-
-        Ok(TestCase {
-            isa,
-            functions,
-            inputs,
-        })
-    }
 }
diff --git a/cranelift/fuzzgen/src/print.rs b/cranelift/fuzzgen/src/print.rs
new file mode 100644
index 000000000..e74f0f3a8
--- /dev/null
+++ b/cranelift/fuzzgen/src/print.rs
@@ -0,0 +1,130 @@
+use cranelift::codegen::data_value::DataValue;
+use cranelift::codegen::ir::Function;
+use cranelift::prelude::settings;
+use cranelift::prelude::*;
+use std::fmt;
+
+use crate::TestCaseInput;
+
+#[derive(Debug)]
+enum TestCaseKind {
+    Compile,
+    Run,
+}
+
+/// Provides a way to format a `TestCase` in the .clif format.
+pub struct PrintableTestCase<'a> {
+    kind: TestCaseKind,
+    isa: &'a isa::OwnedTargetIsa,
+    functions: &'a [Function],
+    // Only applicable for run test cases
+    inputs: &'a [TestCaseInput],
+}
+
+impl<'a> PrintableTestCase<'a> {
+    /// Emits a `test compile` test case.
+    pub fn compile(isa: &'a isa::OwnedTargetIsa, functions: &'a [Function]) -> Self {
+        Self {
+            kind: TestCaseKind::Compile,
+            isa,
+            functions,
+            inputs: &[],
+        }
+    }
+
+    /// Emits a `test run` test case. These also include a `test interpret`.
+    ///
+    /// By convention the first function in `functions` will be considered the main function.
+    pub fn run(
+        isa: &'a isa::OwnedTargetIsa,
+        functions: &'a [Function],
+        inputs: &'a [TestCaseInput],
+    ) -> Self {
+        Self {
+            kind: TestCaseKind::Run,
+            isa,
+            functions,
+            inputs,
+        }
+    }
+
+    /// Returns the main function of this test case.
+    pub fn main(&self) -> &Function {
+        &self.functions[0]
+    }
+}
+
+impl<'a> fmt::Debug for PrintableTestCase<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self.kind {
+            TestCaseKind::Compile => {
+                writeln!(f, ";; Compile test case\n")?;
+                writeln!(f, "test compile")?;
+            }
+            TestCaseKind::Run => {
+                writeln!(f, ";; Run test case\n")?;
+                writeln!(f, "test interpret")?;
+                writeln!(f, "test run")?;
+            }
+        };
+
+        write_non_default_flags(f, self.isa.flags())?;
+
+        writeln!(f, "target {}\n", self.isa.triple().architecture)?;
+
+        // Print the functions backwards, so that the main function is printed last
+        // and near the test inputs for run test cases.
+        for func in self.functions.iter().rev() {
+            writeln!(f, "{}\n", func)?;
+        }
+
+        if !self.inputs.is_empty() {
+            writeln!(f, "; Note: the results in the below test cases are simply a placeholder and probably will be wrong\n")?;
+        }
+
+        for input in self.inputs.iter() {
+            // TODO: We don't know the expected outputs, maybe we can run the interpreter
+            // here to figure them out? Should work, however we need to be careful to catch
+            // panics in case its the interpreter that is failing.
+            // For now create a placeholder output consisting of the zero value for the type
+            let returns = &self.main().signature.returns;
+            let placeholder_output = returns
+                .iter()
+                .map(|param| DataValue::read_from_slice_ne(&[0; 16][..], param.value_type))
+                .map(|val| format!("{}", val))
+                .collect::<Vec<_>>()
+                .join(", ");
+
+            // If we have no output, we don't need the == condition
+            let test_condition = match returns.len() {
+                0 => String::new(),
+                1 => format!(" == {}", placeholder_output),
+                _ => format!(" == [{}]", placeholder_output),
+            };
+
+            let args = input
+                .iter()
+                .map(|val| format!("{}", val))
+                .collect::<Vec<_>>()
+                .join(", ");
+
+            writeln!(f, "; run: {}({}){}", self.main().name, args, test_condition)?;
+        }
+
+        Ok(())
+    }
+}
+
+/// Print only non default flags.
+fn write_non_default_flags(f: &mut fmt::Formatter<'_>, flags: &settings::Flags) -> fmt::Result {
+    let default_flags = settings::Flags::new(settings::builder());
+    for (default, flag) in default_flags.iter().zip(flags.iter()) {
+        assert_eq!(default.name, flag.name);
+
+        if default.value_string() != flag.value_string() {
+            writeln!(f, "set {}={}", flag.name, flag.value_string())?;
+        }
+    }
+
+    Ok(())
+}
diff --git a/cranelift/interpreter/src/interpreter.rs b/cranelift/interpreter/src/interpreter.rs
index 290901dc4..aa44f8e3b 100644
--- a/cranelift/interpreter/src/interpreter.rs
+++ b/cranelift/interpreter/src/interpreter.rs
@@ -11,8 +11,8 @@ use crate::step::{step, ControlFlow, StepError};
 use crate::value::{Value, ValueError};
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::{
-    ArgumentPurpose, Block, ExternalName, FuncRef, Function, GlobalValue, GlobalValueData, LibCall,
-    StackSlot, TrapCode, Type, Value as ValueRef,
+    ArgumentPurpose, Block, Endianness, ExternalName, FuncRef, Function, GlobalValue,
+    GlobalValueData, LibCall, MemFlags, StackSlot, TrapCode, Type, Value as ValueRef,
 };
 use log::trace;
 use smallvec::SmallVec;
@@ -192,10 +192,16 @@ pub struct InterpreterState<'a> {
     pub frame_offset: usize,
     pub stack: Vec<u8>,
     pub pinned_reg: DataValue,
+    pub native_endianness: Endianness,
 }
 
 impl Default for InterpreterState<'_> {
     fn default() -> Self {
+        let native_endianness = if cfg!(target_endian = "little") {
+            Endianness::Little
+        } else {
+            Endianness::Big
+        };
         Self {
             functions: FunctionStore::default(),
             libcall_handler: |_, _| Err(TrapCode::UnreachableCodeReached),
@@ -203,6 +209,7 @@ impl Default for InterpreterState<'_> {
             frame_offset: 0,
             stack: Vec::with_capacity(1024),
             pinned_reg: DataValue::U64(0),
+            native_endianness,
         }
     }
 }
@@ -308,7 +315,12 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
         Address::from_parts(size, AddressRegion::Stack, 0, final_offset)
     }
 
-    fn checked_load(&self, addr: Address, ty: Type) -> Result<DataValue, MemoryError> {
+    fn checked_load(
+        &self,
+        addr: Address,
+        ty: Type,
+        mem_flags: MemFlags,
+    ) -> Result<DataValue, MemoryError> {
         let load_size = ty.bytes() as usize;
         let addr_start = addr.offset as usize;
         let addr_end = addr_start + load_size;
@@ -324,10 +336,18 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
             _ => unimplemented!(),
         };
 
-        Ok(DataValue::read_from_slice(src, ty))
+        Ok(match mem_flags.endianness(self.native_endianness) {
+            Endianness::Big => DataValue::read_from_slice_be(src, ty),
+            Endianness::Little => DataValue::read_from_slice_le(src, ty),
+        })
     }
 
-    fn checked_store(&mut self, addr: Address, v: DataValue) -> Result<(), MemoryError> {
+    fn checked_store(
+        &mut self,
+        addr: Address,
+        v: DataValue,
+        mem_flags: MemFlags,
+    ) -> Result<(), MemoryError> {
         let store_size = v.ty().bytes() as usize;
         let addr_start = addr.offset as usize;
         let addr_end = addr_start + store_size;
@@ -343,7 +363,10 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
             _ => unimplemented!(),
         };
 
-        Ok(v.write_to_slice(dst))
+        Ok(match mem_flags.endianness(self.native_endianness) {
+            Endianness::Big => v.write_to_slice_be(dst),
+            Endianness::Little => v.write_to_slice_le(dst),
+        })
     }
 
     fn function_address(
@@ -493,9 +516,10 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
                     global_type,
                 }) => {
                     let mut addr = Address::try_from(current_val)?;
+                    let mem_flags = MemFlags::trusted();
                     // We can forego bounds checking here since its performed in `checked_load`
                     addr.offset += offset as u64;
-                    current_val = self.checked_load(addr, global_type)?;
+                    current_val = self.checked_load(addr, global_type, mem_flags)?;
                 }
 
                 // We are done resolving this, return the current value
diff --git a/cranelift/interpreter/src/state.rs b/cranelift/interpreter/src/state.rs
index b98f0c552..c797bea7e 100644
--- a/cranelift/interpreter/src/state.rs
+++ b/cranelift/interpreter/src/state.rs
@@ -4,7 +4,8 @@ use crate::address::{Address, AddressSize};
 use crate::interpreter::LibCallHandler;
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::{
-    ExternalName, FuncRef, Function, GlobalValue, LibCall, Signature, StackSlot, Type, Value,
+    ExternalName, FuncRef, Function, GlobalValue, LibCall, MemFlags, Signature, StackSlot, Type,
+    Value,
 };
 use cranelift_codegen::isa::CallConv;
 use cranelift_entity::PrimaryMap;
@@ -62,10 +63,20 @@ pub trait State<'a, V> {
     ) -> Result<Address, MemoryError>;
     /// Retrieve a value `V` from memory at the given `address`, checking if it belongs either to the
     /// stack or to one of the heaps; the number of bytes loaded corresponds to the specified [Type].
-    fn checked_load(&self, address: Address, ty: Type) -> Result<V, MemoryError>;
+    fn checked_load(
+        &self,
+        address: Address,
+        ty: Type,
+        mem_flags: MemFlags,
+    ) -> Result<V, MemoryError>;
     /// Store a value `V` into memory at the given `address`, checking if it belongs either to the
     /// stack or to one of the heaps; the number of bytes stored corresponds to the specified [Type].
-    fn checked_store(&mut self, address: Address, v: V) -> Result<(), MemoryError>;
+    fn checked_store(
+        &mut self,
+        address: Address,
+        v: V,
+        mem_flags: MemFlags,
+    ) -> Result<(), MemoryError>;
 
     /// Compute the address of a function given its name.
     fn function_address(
@@ -182,11 +193,21 @@ where
         unimplemented!()
     }
 
-    fn checked_load(&self, _addr: Address, _ty: Type) -> Result<V, MemoryError> {
+    fn checked_load(
+        &self,
+        _addr: Address,
+        _ty: Type,
+        _mem_flags: MemFlags,
+    ) -> Result<V, MemoryError> {
         unimplemented!()
     }
 
-    fn checked_store(&mut self, _addr: Address, _v: V) -> Result<(), MemoryError> {
+    fn checked_store(
+        &mut self,
+        _addr: Address,
+        _v: V,
+        _mem_flags: MemFlags,
+    ) -> Result<(), MemoryError> {
         unimplemented!()
     }
 
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index 0eddf9c35..d0362e93d 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -8,7 +8,7 @@ use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
 use cranelift_codegen::ir::{
     types, AbiParam, AtomicRmwOp, Block, BlockCall, ExternalName, FuncRef, Function,
-    InstructionData, Opcode, TrapCode, Type, Value as ValueRef,
+    InstructionData, MemFlags, Opcode, TrapCode, Type, Value as ValueRef,
 };
 use log::trace;
 use smallvec::{smallvec, SmallVec};
@@ -482,8 +482,10 @@ where
             };
 
             let addr_value = calculate_addr(types::I64, imm(), args()?)?;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             let loaded = assign_or_memtrap(
-                Address::try_from(addr_value).and_then(|addr| state.checked_load(addr, load_ty)),
+                Address::try_from(addr_value)
+                    .and_then(|addr| state.checked_load(addr, load_ty, mem_flags)),
             );
 
             match (loaded, kind) {
@@ -505,33 +507,37 @@ where
             };
 
             let addr_value = calculate_addr(types::I64, imm(), args_range(1..)?)?;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             let reduced = if let Some(c) = kind {
                 arg(0)?.convert(c)?
             } else {
                 arg(0)?
             };
             continue_or_memtrap(
-                Address::try_from(addr_value).and_then(|addr| state.checked_store(addr, reduced)),
+                Address::try_from(addr_value)
+                    .and_then(|addr| state.checked_store(addr, reduced, mem_flags)),
             )
         }
         Opcode::StackLoad => {
             let load_ty = inst_context.controlling_type().unwrap();
             let slot = inst.stack_slot().unwrap();
             let offset = sum(imm(), args()?)? as u64;
+            let mem_flags = MemFlags::trusted();
             assign_or_memtrap({
                 state
                     .stack_address(AddressSize::_64, slot, offset)
-                    .and_then(|addr| state.checked_load(addr, load_ty))
+                    .and_then(|addr| state.checked_load(addr, load_ty, mem_flags))
             })
         }
         Opcode::StackStore => {
             let arg = arg(0)?;
             let slot = inst.stack_slot().unwrap();
             let offset = sum(imm(), args_range(1..)?)? as u64;
+            let mem_flags = MemFlags::trusted();
             continue_or_memtrap({
                 state
                     .stack_address(AddressSize::_64, slot, offset)
-                    .and_then(|addr| state.checked_store(addr, arg))
+                    .and_then(|addr| state.checked_store(addr, arg, mem_flags))
             })
         }
         Opcode::StackAddr => {
@@ -1238,7 +1244,9 @@ where
             let op = inst.atomic_rmw_op().unwrap();
             let val = arg(1)?;
             let addr = arg(0)?.into_int()? as u64;
-            let loaded = Address::try_from(addr).and_then(|addr| state.checked_load(addr, ctrl_ty));
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
+            let loaded = Address::try_from(addr)
+                .and_then(|addr| state.checked_load(addr, ctrl_ty, mem_flags));
             let prev_val = match loaded {
                 Ok(v) => v,
                 Err(e) => return Ok(ControlFlow::Trap(CraneliftTrap::User(memerror_to_trap(e)))),
@@ -1265,13 +1273,15 @@ where
                 )
                 .and_then(|v| Value::convert(v, ValueConversionKind::ToSigned)),
             }?;
-            let stored =
-                Address::try_from(addr).and_then(|addr| state.checked_store(addr, replace));
+            let stored = Address::try_from(addr)
+                .and_then(|addr| state.checked_store(addr, replace, mem_flags));
             assign_or_memtrap(stored.map(|_| prev_val_to_assign))
         }
         Opcode::AtomicCas => {
             let addr = arg(0)?.into_int()? as u64;
-            let loaded = Address::try_from(addr).and_then(|addr| state.checked_load(addr, ctrl_ty));
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
+            let loaded = Address::try_from(addr)
+                .and_then(|addr| state.checked_load(addr, ctrl_ty, mem_flags));
             let loaded_val = match loaded {
                 Ok(v) => v,
                 Err(e) => return Ok(ControlFlow::Trap(CraneliftTrap::User(memerror_to_trap(e)))),
@@ -1280,7 +1290,7 @@ where
             let val_to_assign = if Value::eq(&loaded_val, &expected_val)? {
                 let val_to_store = arg(2)?;
                 Address::try_from(addr)
-                    .and_then(|addr| state.checked_store(addr, val_to_store))
+                    .and_then(|addr| state.checked_store(addr, val_to_store, mem_flags))
                     .map(|_| loaded_val)
             } else {
                 Ok(loaded_val)
@@ -1290,17 +1300,20 @@ where
         Opcode::AtomicLoad => {
             let load_ty = inst_context.controlling_type().unwrap();
             let addr = arg(0)?.into_int()? as u64;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             // We are doing a regular load here, this isn't actually thread safe.
             assign_or_memtrap(
-                Address::try_from(addr).and_then(|addr| state.checked_load(addr, load_ty)),
+                Address::try_from(addr)
+                    .and_then(|addr| state.checked_load(addr, load_ty, mem_flags)),
             )
         }
         Opcode::AtomicStore => {
             let val = arg(0)?;
             let addr = arg(1)?.into_int()? as u64;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             // We are doing a regular store here, this isn't actually thread safe.
             continue_or_memtrap(
-                Address::try_from(addr).and_then(|addr| state.checked_store(addr, val)),
+                Address::try_from(addr).and_then(|addr| state.checked_store(addr, val, mem_flags)),
             )
         }
         Opcode::Fence => {
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index f8443092d..ffffc50a4 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -48,6 +48,7 @@
   - [Tiers of support](./stability-tiers.md)
   - [Platform Support](./stability-platform-support.md)
   - [Wasm Proposals Support](./stability-wasm-proposals-support.md)
+  - [WASI Proposals Support](./stability-wasi-proposals-support.md)
 - [Security](security.md)
   - [Disclosure Policy](./security-disclosure.md)
 - [Contributing](contributing.md)
diff --git a/docs/WASI-api.md b/docs/WASI-api.md
index eb38ff900..ba8a9eff6 100644
--- a/docs/WASI-api.md
+++ b/docs/WASI-api.md
@@ -4,4 +4,4 @@ Development of the WASI API has now moved to the WASI CG Subgroup; see
 the [github repository], including the [API documentation].
 
 [github repository]: https://github.com/WebAssembly/WASI/
-[API documentation]: https://github.com/WebAssembly/WASI/blob/master/phases/snapshot/docs.md
+[API documentation]: https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md
diff --git a/docs/stability-wasi-proposals-support.md b/docs/stability-wasi-proposals-support.md
new file mode 100644
index 000000000..ce8121e24
--- /dev/null
+++ b/docs/stability-wasi-proposals-support.md
@@ -0,0 +1,60 @@
+# WASI Proposals Support
+
+The following table summarizes Wasmtime's support for WASI [proposals]. If a
+proposal is not listed, then it is not supported by Wasmtime.
+
+[proposals]: https://github.com/WebAssembly/WASI/blob/main/Proposals.md
+
+| WASI Proposal                          | Supported in Wasmtime?  | Enabled by default?  | CLI Flag Name [^cli]        |
+|----------------------------------------|-------------------------|----------------------|-----------------------------|
+| [I/O][wasi-io]                         | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Filesystem][wasi-filesystem]          | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Clocks][wasi-clocks]                  | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Random][wasi-random]                  | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Poll][wasi-poll]                      | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Machine Learning (wasi-nn)][wasi-nn]  | **Yes**                 | No                   | `experimental-wasi-nn`      |
+| [Blob Store][wasi-blob-store]          | No                      | No                   | N/A                         |
+| [Crypto][wasi-crypto]                  | **Yes** [^crypto]       | No                   | `experimental-wasi-crypto`  |
+| [Distributed Lock Service][wasi-distributed-lock-service] | No   | No                   | N/A                         |
+| [gRPC][wasi-grpc]                      | No                      | No                   | N/A                         |
+| [HTTP][wasi-http]                      | No                      | No                   | N/A                         |
+| [Key-value Store][wasi-kv-store]       | No                      | No                   | N/A                         |
+| [Message Queue][wasi-message-queue]    | No                      | No                   | N/A                         |
+| [Parallel][wasi-parallel]              | No (see [#4949])        | No                   | N/A                         |
+| [Pub/sub][wasi-pubsub]                 | No                      | No                   | N/A                         |
+| [Runtime Config][wasi-runtime-config]  | No                      | No                   | N/A                         |
+| [Sockets][wasi-sockets]                | No                      | No                   | N/A                         |
+| [SQL][wasi-sql]                        | No                      | No                   | N/A                         |
+| [Threads][wasi-threads]                | **Yes**                 | No                   | `experimental-wasi-threads` |
+
+[^cli]: The CLI flag name refers to to the `--wasi-modules` argument of the
+    `wasmtime` executable; e.g., `--wasi-modules=wasi-crypto`. See `wasmtime run
+    --help` for more information on the flag's default value and configuration.
+[^crypto]: Build Wasmtime with `--features=wasi-crypto` to enable this.
+
+[#4949]: https://github.com/bytecodealliance/wasmtime/pull/4949
+[wasi-blob-store]: https://github.com/WebAssembly/wasi-blob-store
+[wasi-clocks]: https://github.com/WebAssembly/wasi-clocks
+[wasi-classic-command]: https://github.com/WebAssembly/wasi-classic-command
+[wasi-crypto]: https://github.com/WebAssembly/wasi-crypto
+[wasi-data]: https://github.com/singlestore-labs/wasi-data
+[wasi-distributed-lock-service]: https://github.com/WebAssembly/wasi-distributed-lock-service
+[wasi-filesystem]: https://github.com/WebAssembly/wasi-filesystem
+[wasi-grpc]: https://github.com/WebAssembly/wasi-grpc
+[wasi-handle-index]: https://github.com/WebAssembly/wasi-handle-index
+[wasi-http]: https://github.com/WebAssembly/wasi-http
+[wasi-io]: https://github.com/WebAssembly/wasi-io
+[wasi-kv-store]: https://github.com/WebAssembly/wasi-kv-store
+[wasi-message-queue]: https://github.com/WebAssembly/wasi-message-queue
+[wasi-misc]: https://github.com/WebAssembly/wasi-misc
+[wasi-threads]: https://github.com/WebAssembly/wasi-native-threads
+[wasi-nn]: https://github.com/WebAssembly/wasi-nn
+[wasi-random]: https://github.com/WebAssembly/wasi-random
+[wasi-parallel]: https://github.com/WebAssembly/wasi-parallel
+[wasi-poll]: https://github.com/WebAssembly/wasi-poll
+[wasi-proxy-wasm]: https://github.com/proxy-wasm/spec
+[wasi-pubsub]: https://github.com/WebAssembly/wasi-pubsub
+[wasi-runtime-config]: https://github.com/WebAssembly/wasi-runtime-config
+[wasi-sockets]: https://github.com/WebAssembly/wasi-sockets
+[wasi-sql]: https://github.com/WebAssembly/wasi-sql
+[wasi-url]: https://github.com/WebAssembly/wasi-url
diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml
index 11b6ff3d0..f9774cac3 100644
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -16,6 +16,7 @@ cranelift-wasm = { workspace = true }
 cranelift-filetests = { workspace = true }
 cranelift-interpreter = { workspace = true }
 cranelift-fuzzgen = { workspace = true }
+cranelift-native = { workspace = true }
 libfuzzer-sys = { version = "0.4.0", features = ["arbitrary-derive"] }
 target-lexicon = { workspace = true }
 smallvec = { workspace = true }
diff --git a/fuzz/fuzz_targets/cranelift-fuzzgen.rs b/fuzz/fuzz_targets/cranelift-fuzzgen.rs
index 171a8248e..c6a008bd7 100644
--- a/fuzz/fuzz_targets/cranelift-fuzzgen.rs
+++ b/fuzz/fuzz_targets/cranelift-fuzzgen.rs
@@ -1,13 +1,22 @@
 #![no_main]
 
+use cranelift_codegen::ir::Function;
+use cranelift_codegen::ir::Signature;
+use cranelift_codegen::ir::UserExternalName;
+use cranelift_codegen::ir::UserFuncName;
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::arbitrary::Arbitrary;
+use libfuzzer_sys::arbitrary::Unstructured;
 use libfuzzer_sys::fuzz_target;
 use once_cell::sync::Lazy;
 use std::collections::HashMap;
+use std::fmt;
 use std::sync::atomic::AtomicU64;
 use std::sync::atomic::Ordering;
 
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::{LibCall, TrapCode};
+use cranelift_codegen::isa;
 use cranelift_filetests::function_runner::{TestFileCompiler, Trampoline};
 use cranelift_fuzzgen::*;
 use cranelift_interpreter::environment::FuncIndex;
@@ -17,6 +26,7 @@ use cranelift_interpreter::interpreter::{
 };
 use cranelift_interpreter::step::ControlFlow;
 use cranelift_interpreter::step::CraneliftTrap;
+use cranelift_native::builder_with_options;
 use smallvec::smallvec;
 
 const INTERPRETER_FUEL: u64 = 4096;
@@ -26,6 +36,8 @@ struct Statistics {
     /// Inputs that fuzzgen can build a function with
     /// This is also how many compiles we executed
     pub valid_inputs: AtomicU64,
+    /// How many times did we generate an invalid format?
+    pub invalid_inputs: AtomicU64,
 
     /// Total amount of runs that we tried in the interpreter
     /// One fuzzer input can have many runs
@@ -43,11 +55,17 @@ impl Statistics {
     pub fn print(&self, valid_inputs: u64) {
         // We get valid_inputs as a param since we already loaded it previously.
         let total_runs = self.total_runs.load(Ordering::SeqCst);
+        let invalid_inputs = self.invalid_inputs.load(Ordering::SeqCst);
         let run_result_success = self.run_result_success.load(Ordering::SeqCst);
         let run_result_timeout = self.run_result_timeout.load(Ordering::SeqCst);
 
         println!("== FuzzGen Statistics  ====================");
         println!("Valid Inputs: {}", valid_inputs);
+        println!(
+            "Invalid Inputs: {} ({:.1}% of Total Inputs)",
+            invalid_inputs,
+            (invalid_inputs as f64 / (valid_inputs + invalid_inputs) as f64) * 100.0
+        );
         println!("Total Runs: {}", total_runs);
         println!(
             "Successful Runs: {} ({:.1}% of Total Runs)",
@@ -94,6 +112,7 @@ impl Default for Statistics {
 
         Self {
             valid_inputs: AtomicU64::new(0),
+            invalid_inputs: AtomicU64::new(0),
             total_runs: AtomicU64::new(0),
             run_result_success: AtomicU64::new(0),
             run_result_timeout: AtomicU64::new(0),
@@ -120,6 +139,90 @@ impl PartialEq for RunResult {
     }
 }
 
+pub struct TestCase {
+    /// TargetIsa to use when compiling this test case
+    pub isa: isa::OwnedTargetIsa,
+    /// Functions under test
+    /// By convention the first function is the main function.
+    pub functions: Vec<Function>,
+    /// Generate multiple test inputs for each test case.
+    /// This allows us to get more coverage per compilation, which may be somewhat expensive.
+    pub inputs: Vec<TestCaseInput>,
+}
+
+impl fmt::Debug for TestCase {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        PrintableTestCase::run(&self.isa, &self.functions, &self.inputs).fmt(f)
+    }
+}
+
+impl<'a> Arbitrary<'a> for TestCase {
+    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
+        Self::generate(u).map_err(|_| {
+            STATISTICS.invalid_inputs.fetch_add(1, Ordering::SeqCst);
+            arbitrary::Error::IncorrectFormat
+        })
+    }
+}
+
+impl TestCase {
+    pub fn generate(u: &mut Unstructured) -> anyhow::Result<Self> {
+        let mut gen = FuzzGen::new(u);
+
+        // TestCase is meant to be consumed by a runner, so we make the assumption here that we're
+        // generating a TargetIsa for the host.
+        let builder =
+            builder_with_options(true).expect("Unable to build a TargetIsa for the current host");
+        let flags = gen.generate_flags(builder.triple().architecture)?;
+        let isa = builder.finish(flags)?;
+
+        // When generating functions, we allow each function to call any function that has
+        // already been generated. This guarantees that we never have loops in the call graph.
+        // We generate these backwards, and then reverse them so that the main function is at
+        // the start.
+        let func_count = gen.u.int_in_range(gen.config.testcase_funcs.clone())?;
+        let mut functions: Vec<Function> = Vec::with_capacity(func_count);
+        for i in (0..func_count).rev() {
+            // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
+            let fname = UserFuncName::user(1, i as u32);
+
+            let usercalls: Vec<(UserExternalName, Signature)> = functions
+                .iter()
+                .map(|f| {
+                    (
+                        f.name.get_user().unwrap().clone(),
+                        f.stencil.signature.clone(),
+                    )
+                })
+                .collect();
+
+            let func = gen.generate_func(
+                fname,
+                isa.triple().clone(),
+                usercalls,
+                ALLOWED_LIBCALLS.to_vec(),
+            )?;
+            functions.push(func);
+        }
+        // Now reverse the functions so that the main function is at the start.
+        functions.reverse();
+
+        let main = &functions[0];
+        let inputs = gen.generate_test_inputs(&main.signature)?;
+
+        Ok(TestCase {
+            isa,
+            functions,
+            inputs,
+        })
+    }
+
+    /// Returns the main function of this test case.
+    pub fn main(&self) -> &Function {
+        &self.functions[0]
+    }
+}
+
 fn run_in_interpreter(interpreter: &mut Interpreter, args: &[DataValue]) -> RunResult {
     // The entrypoint function is always 0
     let index = FuncIndex::from_u32(0);
@@ -139,6 +242,16 @@ fn run_in_host(trampoline: &Trampoline, args: &[DataValue]) -> RunResult {
     RunResult::Success(res)
 }
 
+/// These libcalls need a interpreter implementation in `build_interpreter`
+const ALLOWED_LIBCALLS: &'static [LibCall] = &[
+    LibCall::CeilF32,
+    LibCall::CeilF64,
+    LibCall::FloorF32,
+    LibCall::FloorF64,
+    LibCall::TruncF32,
+    LibCall::TruncF64,
+];
+
 fn build_interpreter(testcase: &TestCase) -> Interpreter {
     let mut env = FunctionStore::default();
     for func in testcase.functions.iter() {
diff --git a/fuzz/fuzz_targets/cranelift-icache.rs b/fuzz/fuzz_targets/cranelift-icache.rs
index 6023d0041..4c0ca5455 100644
--- a/fuzz/fuzz_targets/cranelift-icache.rs
+++ b/fuzz/fuzz_targets/cranelift-icache.rs
@@ -3,13 +3,103 @@
 use cranelift_codegen::{
     cursor::{Cursor, FuncCursor},
     incremental_cache as icache,
-    ir::{self, immediates::Imm64, ExternalName},
-    Context,
+    ir::{
+        self, immediates::Imm64, ExternalName, Function, LibCall, Signature, UserExternalName,
+        UserFuncName,
+    },
+    isa, Context,
 };
-use libfuzzer_sys::fuzz_target;
+use libfuzzer_sys::{
+    arbitrary::{self, Arbitrary, Unstructured},
+    fuzz_target,
+};
+use std::fmt;
 
 use cranelift_fuzzgen::*;
 
+/// TODO: This *almost* could be replaced with `LibCall::all()`, but
+/// `LibCall::signature` panics for some libcalls, so we need to avoid that.
+const ALLOWED_LIBCALLS: &'static [LibCall] = &[
+    LibCall::CeilF32,
+    LibCall::CeilF64,
+    LibCall::FloorF32,
+    LibCall::FloorF64,
+    LibCall::TruncF32,
+    LibCall::TruncF64,
+    LibCall::NearestF32,
+    LibCall::NearestF64,
+    LibCall::FmaF32,
+    LibCall::FmaF64,
+];
+
+/// A generated function with an ISA that targets one of cranelift's backends.
+pub struct FunctionWithIsa {
+    /// TargetIsa to use when compiling this test case
+    pub isa: isa::OwnedTargetIsa,
+
+    /// Function under test
+    pub func: Function,
+}
+
+impl FunctionWithIsa {
+    pub fn generate(u: &mut Unstructured) -> anyhow::Result<Self> {
+        // We filter out targets that aren't supported in the current build
+        // configuration after randomly choosing one, instead of randomly choosing
+        // a supported one, so that the same fuzz input works across different build
+        // configurations.
+        let target = u.choose(isa::ALL_ARCHITECTURES)?;
+        let builder = isa::lookup_by_name(target).map_err(|_| arbitrary::Error::IncorrectFormat)?;
+        let architecture = builder.triple().architecture;
+
+        let mut gen = FuzzGen::new(u);
+        let flags = gen
+            .generate_flags(architecture)
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+        let isa = builder
+            .finish(flags)
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+
+        // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
+        let fname = UserFuncName::user(1, 0);
+
+        // We don't actually generate these functions, we just simulate their signatures and names
+        let func_count = gen.u.int_in_range(gen.config.testcase_funcs.clone())?;
+        let usercalls = (0..func_count)
+            .map(|i| {
+                let name = UserExternalName::new(2, i as u32);
+                let sig = gen.generate_signature(architecture)?;
+                Ok((name, sig))
+            })
+            .collect::<anyhow::Result<Vec<(UserExternalName, Signature)>>>()
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+
+        let func = gen
+            .generate_func(
+                fname,
+                isa.triple().clone(),
+                usercalls,
+                ALLOWED_LIBCALLS.to_vec(),
+            )
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+
+        Ok(FunctionWithIsa { isa, func })
+    }
+}
+
+impl fmt::Debug for FunctionWithIsa {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // TODO: We could avoid the clone here.
+        let funcs = &[self.func.clone()];
+        PrintableTestCase::compile(&self.isa, funcs).fmt(f)
+    }
+}
+
+impl<'a> Arbitrary<'a> for FunctionWithIsa {
+    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
+        Self::generate(u).map_err(|_| arbitrary::Error::IncorrectFormat)
+    }
+}
+
 fuzz_target!(|func: FunctionWithIsa| {
     let FunctionWithIsa { mut func, isa } = func;
 


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/algebraic.clif b/cranelift/filetests/filetests/egraph/algebraic.clif
index 24460845a..573135d3b 100644
--- a/cranelift/filetests/filetests/egraph/algebraic.clif
+++ b/cranelift/filetests/filetests/egraph/algebraic.clif
@@ -483,3 +483,25 @@ block0(v1: i16):
 
 ; check: v4 = sextend.i64 v1
 ; check: return v4
+
+function %fma_double_fneg(f32, f32, f32) -> f32 {
+block0(v1: f32, v2: f32, v3: f32):
+    v4 = fneg v1
+    v5 = fneg v2
+    v6 = fma v4, v5, v3
+    return v6
+}
+
+; check: v7 = fma v1, v2, v3
+; check: return v7
+
+function %fmul_double_fneg(f32, f32) -> f32 {
+block0(v1: f32, v2: f32):
+    v3 = fneg v1
+    v4 = fneg v2
+    v5 = fmul v3, v4
+    return v5
+}
+
+; check: v6 = fmul v1, v2
+; check: return v6
diff --git a/cranelift/filetests/filetests/egraph/issue-5796.clif b/cranelift/filetests/filetests/egraph/issue-5796.clif
new file mode 100644
index 000000000..66963f5c4
--- /dev/null
+++ b/cranelift/filetests/filetests/egraph/issue-5796.clif
@@ -0,0 +1,76 @@
+test optimize
+set opt_level=speed
+target x86_64
+
+function %f0(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  brif v0, block1, block2
+
+block1:
+  v3 = udiv v1, v2
+  return v3
+
+block2:
+  v4 = udiv v1, v2
+  brif v1, block3, block4
+
+block3:
+  return v4
+
+block4:
+  return v4
+}
+
+; check: block0(v0: i32, v1: i32, v2: i32):
+; check:     brif v0, block1, block2
+
+; check: block1:
+; check:     v3 = udiv.i32 v1, v2
+; check:     return v3
+
+; check: block2:
+; check:     v4 = udiv.i32 v1, v2
+; check:     brif.i32 v1, block3, block4
+
+; check: block3:
+; check:     return v4
+
+; check: block4:
+; check:     return v4
+
+function %f1(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  brif v0, block1, block2
+
+block1:
+  v3 = udiv v1, v2
+  return v3
+
+block2:
+  v4 = udiv v1, v2
+  brif v1, block3, block4
+
+block3:
+  v5 = udiv v1, v2
+  return v5
+
+block4:
+  return v4
+}
+
+; check: block0(v0: i32, v1: i32, v2: i32):
+; check:     brif v0, block1, block2
+
+; check: block1:
+; check:     v3 = udiv.i32 v1, v2
+; check:     return v3
+
+; check: block2:
+; check:     v4 = udiv.i32 v1, v2
+; check:     brif.i32 v1, block3, block4
+
+; check: block3:
+; check:     return v4
+
+; check: block4:
+; check:     return v4
diff --git a/cranelift/filetests/filetests/isa/aarch64/fma.clif b/cranelift/filetests/filetests/isa/aarch64/fma.clif
new file mode 100644
index 000000000..e2f4a172c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/fma.clif
@@ -0,0 +1,159 @@
+test compile precise-output
+target aarch64
+
+function %fma_f32(f32, f32, f32) -> f32 {
+block0(v0: f32, v1: f32, v2: f32):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   fmadd s0, s0, s1, s2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   fmadd s0, s0, s1, s2
+;   ret
+
+function %fma_f64(f64, f64, f64) -> f64 {
+block0(v0: f64, v1: f64, v2: f64):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   fmadd d0, d0, d1, d2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   fmadd d0, d0, d1, d2
+;   ret
+
+function %fma_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.2d, v5.2d, v1.2d
+;   ret
+
+function %fma_neg_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_neg_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v5.2d, v1.2d
+;   ret
+
+function %fma_neg_other_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fneg v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_neg_other_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fneg v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v5.2d, v1.2d
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/aarch64/iconst-imm12_from_negated.isle b/cranelift/filetests/filetests/isa/aarch64/iconst-imm12_from_negated.isle
new file mode 100644
index 000000000..61067e316
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/iconst-imm12_from_negated.isle
@@ -0,0 +1,66 @@
+;; Test our lowerings that try to fit negated iconst values in immediates, like
+;; `x + -1 => sub x 0x1`. Currently, not working as expected for i32 and i16. 
+
+test compile precise-output
+set unwind_info=false
+target aarch64
+
+;; Result uses immediate as expected
+function %c(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 -1
+    v3 = iadd v0, v1
+    return v3
+}
+
+; VCode:
+; block0:
+;   sub x0, x0, #1
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   sub x0, x0, #1
+;   ret
+
+;; 4294967295 is zero-extended i32 -1
+;; Result should use immediate but currently doesn't
+function %b(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 4294967295 
+    v3 = iadd v0, v1
+    return v3
+}
+
+; VCode:
+; block0:
+;   movn w2, #0
+;   add w0, w0, w2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w2, #-1
+;   add w0, w0, w2
+;   ret
+
+;; 65535 is zero-extended i16 -1
+;; Result should use immediate but currently doesn't
+function %a(i16) -> i16 {
+block0(v0: i16):
+    v1 = iconst.i16 65535
+    v3 = iadd v0, v1
+    return v3
+}
+
+; VCode:
+; block0:
+;   movz w2, #65535
+;   add w0, w0, w2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w2, #0xffff
+;   add w0, w0, w2
+;   ret
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle.clif b/cranelift/filetests/filetests/isa/x64/shuffle.clif
new file mode 100644
index 000000000..529b95cc5
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/shuffle.clif
@@ -0,0 +1,54 @@
+test compile precise-output
+set enable_simd
+target x86_64
+
+function %punpcklbw(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, 0x17071606150514041303120211011000
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpcklbw %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpcklbw %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckhbw(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, 0x1f0f1e0e1d0d1c0c1b0b1a0a19091808
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpckhbw %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpckhbw %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
index 3e622bd5f..fdd58032e 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
@@ -1635,8 +1635,7 @@ block0(v0: i8x16, v1: i32):
 ;   vpsrlw  %xmm0, %xmm5, %xmm7
 ;   lea     const(0), %rsi
 ;   shlq    $4, %r10, %r10
-;   movdqu  0(%rsi,%r10,1), %xmm13
-;   vpand   %xmm7, %xmm13, %xmm0
+;   vpand   %xmm7, 0(%rsi,%r10,1), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1652,11 +1651,13 @@ block0(v0: i8x16, v1: i32):
 ;   vpsrlw %xmm5, %xmm0, %xmm7
 ;   leaq 0x15(%rip), %rsi
 ;   shlq $4, %r10
-;   movdqu (%rsi, %r10), %xmm13
-;   vpand %xmm13, %xmm7, %xmm0
+;   vpand (%rsi, %r10), %xmm7, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %bh, %bh
 
 function %i8x16_ushr_imm(i8x16) -> i8x16 {
 block0(v0: i8x16):
@@ -1670,8 +1671,7 @@ block0(v0: i8x16):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   vpsrlw  %xmm0, $1, %xmm2
-;   movdqu  const(0), %xmm4
-;   vpand   %xmm2, %xmm4, %xmm0
+;   vpand   %xmm2, const(0), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1682,14 +1682,15 @@ block0(v0: i8x16):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   vpsrlw $1, %xmm0, %xmm2
-;   movdqu 0xf(%rip), %xmm4
-;   vpand %xmm4, %xmm2, %xmm0
+;   vpand 0xf(%rip), %xmm2, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
 ;   jg 0xa1
 ;   jg 0xa3
 ;   jg 0xa5
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
index 1ecdf31ed..f63cc2231 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
@@ -485,8 +485,7 @@ block0:
 ; block0:
 ;   movdqu  const(1), %xmm0
 ;   psrlw   %xmm0, $1, %xmm0
-;   movdqu  const(0), %xmm3
-;   pand    %xmm0, %xmm3, %xmm0
+;   pand    %xmm0, const(0), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -496,28 +495,21 @@ block0:
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movdqu 0x34(%rip), %xmm0
+;   movdqu 0x24(%rip), %xmm0
 ;   psrlw $1, %xmm0
-;   movdqu 0x17(%rip), %xmm3
-;   pand %xmm3, %xmm0
+;   pand 7(%rip), %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
 ;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   jg 0xb1
-;   jg 0xb3
-;   jg 0xb5
-;   jg 0xb7
-;   jg 0xb9
-;   jg 0xbb
-;   jg 0xbd
-;   jg 0xbf
+;   jg 0xa1
+;   jg 0xa3
+;   jg 0xa5
+;   jg 0xa7
+;   jg 0xa9
+;   jg 0xab
+;   jg 0xad
+;   jg 0xaf
 ;   addb %al, (%rcx)
 ;   addb (%rbx), %al
 ;   addb $5, %al
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif b/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
index 4fb63be0f..9be9323dd 100644
--- a/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif b/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
index ae5871737..f7565d63d 100644
--- a/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 target aarch64
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
index dacbcb708..f0f52c12a 100644
--- a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 target s390x has_mie2
@@ -431,10 +432,10 @@ block0(v0: i32, v1: i64, v2: i16):
     v6 = load.i32 big v3
     return v6
 }
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0x1111) == 0x11115678
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0xffff) == 0xffff5678
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0x1111) == 0x12341111
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0xffff) == 0x1234ffff
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 0, 0x1111) == 0x11115678
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 0, 0xffff) == 0xffff5678
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 2, 0x1111) == 0x12341111
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 2, 0xffff) == 0x1234ffff
 
 
 function %atomic_rmw_xchg_big_i8(i32, i64, i8) -> i32 {
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
index a6d16c7a6..2cb9beba7 100644
--- a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 target s390x has_mie2
diff --git a/cranelift/filetests/filetests/runtests/issue5884.clif b/cranelift/filetests/filetests/runtests/issue5884.clif
new file mode 100644
index 000000000..e3bea5e65
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/issue5884.clif
@@ -0,0 +1,20 @@
+test run
+target aarch64
+target x86_64
+target riscv64
+target s390x
+
+
+function %a(i64, i16) -> i16 {
+    ss0 = explicit_slot 8
+
+block0(v0: i64, v1: i16):
+    v2 = stack_addr.i64 ss0+0
+    store little v0, v2
+
+    v3 = stack_addr.i64 ss0+6
+    v4 = atomic_rmw.i16 little or v3, v1
+    return v4
+}
+
+; run: %a(8608481011852310776, 0) == 30583
\ No newline at end of file
diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
new file mode 100644
index 000000000..b6d4e88dc
--- /dev/null
+++ b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
@@ -0,0 +1,92 @@
+;;! target = "x86_64"
+;;!
+;;! optimize = true
+;;!
+;;! settings = [
+;;!   "enable_heap_access_spectre_mitigation=true",
+;;!   "opt_level=speed_and_size",
+;;!   "use_egraphs=true"
+;;! ]
+;;!
+;;! [globals.vmctx]
+;;! type = "i64"
+;;! vmctx = true
+;;!
+;;! [globals.heap_base]
+;;! type = "i64"
+;;! load = { base = "vmctx", offset = 0 }
+;;!
+;;! [globals.heap_bound]
+;;! type = "i64"
+;;! load = { base = "vmctx", offset = 8 }
+;;!
+;;! [[heaps]]
+;;! base = "heap_base"
+;;! min_size = 0
+;;! offset_guard_size = 0xffffffff
+;;! index_type = "i32"
+;;! style = { kind = "dynamic", bound = "heap_bound" }
+
+(module
+  (memory (export "memory") 0)
+  (func (export "load-without-offset") (param i32) (result i32 i32)
+    local.get 0
+    i32.load
+    local.get 0
+    i32.load
+  )
+  (func (export "load-with-offset") (param i32) (result i32 i32)
+    local.get 0
+    i32.load offset=1234
+    local.get 0
+    i32.load offset=1234
+  )
+)
+
+;; function u0:0(i32, i64 vmctx) -> i32, i32 fast {
+;;     gv0 = vmctx
+;;     gv1 = load.i64 notrap aligned gv0+8
+;;     gv2 = load.i64 notrap aligned gv0
+;;
+;;                                 block0(v0: i32, v1: i64):
+;; @0057                               v4 = uextend.i64 v0
+;; @0057                               v5 = iconst.i64 4
+;; @0057                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 4
+;; @0057                               v7 = load.i64 notrap aligned v1+8
+;; @0057                               v8 = load.i64 notrap aligned v1
+;; @0057                               v11 = icmp ugt v6, v7
+;; @0057                               v10 = iconst.i64 0
+;; @0057                               v9 = iadd v8, v4
+;; @0057                               v12 = select_spectre_guard v11, v10, v9  ; v10 = 0
+;; @0057                               v13 = load.i32 little heap v12
+;;                                     v2 -> v13
+;; @005f                               jump block1
+;;
+;;                                 block1:
+;; @005f                               return v13, v13
+;; }
+;;
+;; function u0:1(i32, i64 vmctx) -> i32, i32 fast {
+;;     gv0 = vmctx
+;;     gv1 = load.i64 notrap aligned gv0+8
+;;     gv2 = load.i64 notrap aligned gv0
+;;
+;;                                 block0(v0: i32, v1: i64):
+;; @0064                               v4 = uextend.i64 v0
+;; @0064                               v5 = iconst.i64 1238
+;; @0064                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 1238
+;; @0064                               v7 = load.i64 notrap aligned v1+8
+;; @0064                               v8 = load.i64 notrap aligned v1
+;; @0064                               v12 = icmp ugt v6, v7
+;; @0064                               v11 = iconst.i64 0
+;; @0064                               v9 = iadd v8, v4
+;;                                     v26 = iconst.i64 1234
+;; @0064                               v10 = iadd v9, v26  ; v26 = 1234
+;; @0064                               v13 = select_spectre_guard v12, v11, v10  ; v11 = 0
+;; @0064                               v14 = load.i32 little heap v13
+;;                                     v2 -> v14
+;; @006e                               jump block1
+;;
+;;                                 block1:
+;; @006e                               return v14, v14
+;; }
diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs
index 7086ebea6..6a1baa47e 100644
--- a/tests/all/cli_tests.rs
+++ b/tests/all/cli_tests.rs
@@ -7,7 +7,7 @@ use tempfile::{NamedTempFile, TempDir};
 
 // Run the wasmtime CLI with the provided args and return the `Output`.
 // If the `stdin` is `Some`, opens the file and redirects to the child's stdin.
-fn run_wasmtime_for_output(args: &[&str], stdin: Option<&Path>) -> Result<Output> {
+pub fn run_wasmtime_for_output(args: &[&str], stdin: Option<&Path>) -> Result<Output> {
     let runner = std::env::vars()
         .filter(|(k, _v)| k.starts_with("CARGO_TARGET") && k.ends_with("RUNNER"))
         .next();
diff --git a/tests/all/main.rs b/tests/all/main.rs
index 9b613ffb1..80455d383 100644
--- a/tests/all/main.rs
+++ b/tests/all/main.rs
@@ -32,6 +32,7 @@ mod table;
 mod threads;
 mod traps;
 mod wait_notify;
+mod wasi_testsuite;
 mod wast;
 
 /// A helper to compile a module in a new store with reference types enabled.
diff --git a/tests/all/wasi_testsuite.rs b/tests/all/wasi_testsuite.rs
new file mode 100644
index 000000000..81d2bf434
--- /dev/null
+++ b/tests/all/wasi_testsuite.rs
@@ -0,0 +1,79 @@
+//! Run the tests in `wasi_testsuite` using Wasmtime's CLI binary and checking
+//! the results with a [wasi-testsuite] spec.
+//!
+//! [wasi-testsuite]: https://github.com/WebAssembly/wasi-testsuite
+
+use crate::cli_tests::run_wasmtime_for_output;
+use anyhow::Result;
+use serde::Deserialize;
+use std::ffi::OsStr;
+use std::fs::{self, DirEntry};
+use std::path::{Path, PathBuf};
+use std::process::Output;
+
+#[test]
+#[cfg_attr(target_os = "windows", ignore)] // TODO: https://github.com/WebAssembly/WASI/issues/524
+fn wasi_threads_testsuite() -> Result<()> {
+    for module in list_modules("tests/wasi_testsuite/wasi-threads/test/testsuite")? {
+        println!("Testing {}", module.display());
+        let result = run(&module)?;
+        let spec = parse_spec(&module.with_extension("json"))?;
+        assert_eq!(spec, result);
+    }
+    Ok(())
+}
+
+fn list_modules(testsuite_dir: &str) -> Result<impl Iterator<Item = PathBuf>> {
+    Ok(fs::read_dir(testsuite_dir)?
+        .filter_map(Result::ok)
+        .filter(is_wasm)
+        .map(|e| e.path()))
+}
+
+fn is_wasm(entry: &DirEntry) -> bool {
+    let path = entry.path();
+    let ext = path.extension().map(OsStr::to_str).flatten();
+    path.is_file() && (ext == Some("wat") || ext == Some("wasm"))
+}
+
+fn run<P: AsRef<Path>>(module: P) -> Result<Output> {
+    run_wasmtime_for_output(
+        &[
+            "run",
+            "--wasi-modules",
+            "experimental-wasi-threads",
+            "--wasm-features",
+            "threads",
+            "--disable-cache",
+            module.as_ref().to_str().unwrap(),
+        ],
+        None,
+    )
+}
+
+fn parse_spec<P: AsRef<Path>>(spec_file: P) -> Result<Spec> {
+    let contents = fs::read_to_string(spec_file)?;
+    let spec = serde_json::from_str(&contents)?;
+    Ok(spec)
+}
+
+#[derive(Debug, Deserialize)]
+struct Spec {
+    exit_code: i32,
+    stdout: Option<String>,
+    stderr: Option<String>,
+}
+
+impl PartialEq<Output> for Spec {
+    fn eq(&self, other: &Output) -> bool {
+        self.exit_code == other.status.code().unwrap()
+            && matches_or_missing(&self.stdout, &other.stdout)
+            && matches_or_missing(&self.stderr, &other.stderr)
+    }
+}
+
+fn matches_or_missing(a: &Option<String>, b: &[u8]) -> bool {
+    a.as_ref()
+        .map(|s| s == &String::from_utf8_lossy(b))
+        .unwrap_or(true)
+}
diff --git a/tests/wasi_testsuite/wasi-threads b/tests/wasi_testsuite/wasi-threads
new file mode 160000
index 000000000..e1893c004
--- /dev/null
+++ b/tests/wasi_testsuite/wasi-threads
@@ -0,0 +1,1 @@
+Subproject commit e1893c00458de3e846f02c9cd2a435992ea8b32c