Repository: bytecodealliance/wasmtime
Pull request #5907 (2023-03-01 23:40:58)

wasi-threads: run test suite
This change enables the running of the wasi-threads [test suite]. It relies on a Wasmtime CLI binary being available and runs all `*.wasm` and `*.wat` files present in the test suite directory. The results of each execution are compared against a JSON spec file with the same base name as the WebAssembly module. The spec file defines the expected exit code, e.g.

This commit does not yet build any `*.c` or `*.s` files from the test suite. That could be done later, perhaps upstream; in the meantime, this work is still valuable as it lays the foundation for running other WASI tests from the in-progress [wasi-testsuite] which share the same JSON spec infrastructure.

[test suite]: https://github.com/WebAssembly/wasi-threads/tree/main/test/testsuite
[wasi-testsuite]: https://github.com/WebAssembly/wasi-testsuite

<!--

Please ensure that the following steps are all taken care of before submitting
the PR.

- [ ] This has been discussed in issue #..., or if not, please tell us why
  here.
- [ ] A short description of what this does, why it is needed; if the
  description becomes long, the matter should probably be discussed in an issue
  first.
- [ ] This PR contains test cases, if meaningful.
- [ ] A reviewer from the core maintainer team has been assigned for this PR.
  If you don't know who could review this, please indicate so. The list of
  suggested reviewers on the right can help you.

Please ensure all communication adheres to the [code of
conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
-->


abrown: I marked this as "draft" for now because I'm not sure where this should live. It might be better up at the top level `tests` directory, which already would have built the Wasmtime CLI binary; in the current state there is no checking that it has been built yet. Additionally, if we end up adding more tests from the WASI testsuite, then we could share some of the infrastructure up there (note that currently `run_wasmtime_for_output` is cloned here from the `tests/all/cli_tests.rs` file).

Patch:
diff --git a/.gitmodules b/.gitmodules
index ee264b99c..ba7f67934 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -13,3 +13,6 @@
 [submodule "crates/wasi-crypto/spec"]
 	path = crates/wasi-crypto/spec
 	url = https://github.com/WebAssembly/wasi-crypto.git
+[submodule "tests/wasi_testsuite/wasi-threads"]
+	path = tests/wasi_testsuite/wasi-threads
+	url = https://github.com/WebAssembly/wasi-threads
diff --git a/Cargo.lock b/Cargo.lock
index a819a215d..8c6c4ef5d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3674,6 +3674,7 @@ dependencies = [
  "cranelift-filetests",
  "cranelift-fuzzgen",
  "cranelift-interpreter",
+ "cranelift-native",
  "cranelift-reader",
  "cranelift-wasm",
  "libfuzzer-sys",
diff --git a/Cargo.toml b/Cargo.toml
index 407ab2fd0..e92983c12 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -70,6 +70,8 @@ component-macro-test = { path = "crates/misc/component-macro-test" }
 component-test-util = { workspace = true }
 bstr = "0.2.17"
 libc = "0.2.60"
+serde = "1.0"
+serde_json = "1.0"
 
 [target.'cfg(windows)'.dev-dependencies]
 windows-sys = { workspace = true, features = ["Win32_System_Memory"] }
diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index 01ccefb7e..488296999 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -66,7 +66,7 @@ fn gen_formats(formats: &[&InstructionFormat], fmt: &mut Formatter) {
 /// 16 bytes on 64-bit architectures. If more space is needed to represent an instruction, use a
 /// `ValueList` to store the additional information out of line.
 fn gen_instruction_data(formats: &[&InstructionFormat], fmt: &mut Formatter) {
-    fmt.line("#[derive(Copy, Clone, Debug, PartialEq, Hash)]");
+    fmt.line("#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]");
     fmt.line(r#"#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]"#);
     fmt.line("#[allow(missing_docs)]");
     fmtln!(fmt, "pub enum InstructionData {");
diff --git a/cranelift/codegen/src/data_value.rs b/cranelift/codegen/src/data_value.rs
index 2da55fc9c..6cba82c75 100644
--- a/cranelift/codegen/src/data_value.rs
+++ b/cranelift/codegen/src/data_value.rs
@@ -101,12 +101,55 @@ impl DataValue {
         }
     }
 
-    /// Write a [DataValue] to a slice.
+    fn swap_bytes(self) -> Self {
+        match self {
+            DataValue::I8(i) => DataValue::I8(i.swap_bytes()),
+            DataValue::I16(i) => DataValue::I16(i.swap_bytes()),
+            DataValue::I32(i) => DataValue::I32(i.swap_bytes()),
+            DataValue::I64(i) => DataValue::I64(i.swap_bytes()),
+            DataValue::I128(i) => DataValue::I128(i.swap_bytes()),
+            DataValue::U8(i) => DataValue::U8(i.swap_bytes()),
+            DataValue::U16(i) => DataValue::U16(i.swap_bytes()),
+            DataValue::U32(i) => DataValue::U32(i.swap_bytes()),
+            DataValue::U64(i) => DataValue::U64(i.swap_bytes()),
+            DataValue::U128(i) => DataValue::U128(i.swap_bytes()),
+            DataValue::F32(f) => DataValue::F32(Ieee32::with_bits(f.bits().swap_bytes())),
+            DataValue::F64(f) => DataValue::F64(Ieee64::with_bits(f.bits().swap_bytes())),
+            DataValue::V128(mut v) => {
+                v.reverse();
+                DataValue::V128(v)
+            }
+            DataValue::V64(mut v) => {
+                v.reverse();
+                DataValue::V64(v)
+            }
+        }
+    }
+
+    /// Converts `self` to big endian from target's endianness.
+    pub fn to_be(self) -> Self {
+        if cfg!(target_endian = "big") {
+            self
+        } else {
+            self.swap_bytes()
+        }
+    }
+
+    /// Converts `self` to little endian from target's endianness.
+    pub fn to_le(self) -> Self {
+        if cfg!(target_endian = "little") {
+            self
+        } else {
+            self.swap_bytes()
+        }
+    }
+
+    /// Write a [DataValue] to a slice in native-endian byte order.
     ///
     /// # Panics:
     ///
     /// Panics if the slice does not have enough space to accommodate the [DataValue]
-    pub fn write_to_slice(&self, dst: &mut [u8]) {
+    pub fn write_to_slice_ne(&self, dst: &mut [u8]) {
         match self {
             DataValue::I8(i) => dst[..1].copy_from_slice(&i.to_ne_bytes()[..]),
             DataValue::I16(i) => dst[..2].copy_from_slice(&i.to_ne_bytes()[..]),
@@ -121,12 +164,30 @@ impl DataValue {
         };
     }
 
-    /// Read a [DataValue] from a slice using a given [Type].
+    /// Write a [DataValue] to a slice in big-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn write_to_slice_be(&self, dst: &mut [u8]) {
+        self.clone().to_be().write_to_slice_ne(dst);
+    }
+
+    /// Write a [DataValue] to a slice in little-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn write_to_slice_le(&self, dst: &mut [u8]) {
+        self.clone().to_le().write_to_slice_ne(dst);
+    }
+
+    /// Read a [DataValue] from a slice using a given [Type] with native-endian byte order.
     ///
     /// # Panics:
     ///
     /// Panics if the slice does not have enough space to accommodate the [DataValue]
-    pub fn read_from_slice(src: &[u8], ty: Type) -> Self {
+    pub fn read_from_slice_ne(src: &[u8], ty: Type) -> Self {
         match ty {
             types::I8 => DataValue::I8(i8::from_ne_bytes(src[..1].try_into().unwrap())),
             types::I16 => DataValue::I16(i16::from_ne_bytes(src[..2].try_into().unwrap())),
@@ -152,15 +213,33 @@ impl DataValue {
         }
     }
 
-    /// Write a [DataValue] to a memory location.
+    /// Read a [DataValue] from a slice using a given [Type] in big-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn read_from_slice_be(src: &[u8], ty: Type) -> Self {
+        DataValue::read_from_slice_ne(src, ty).to_be()
+    }
+
+    /// Read a [DataValue] from a slice using a given [Type] in little-endian byte order.
+    ///
+    /// # Panics:
+    ///
+    /// Panics if the slice does not have enough space to accommodate the [DataValue]
+    pub fn read_from_slice_le(src: &[u8], ty: Type) -> Self {
+        DataValue::read_from_slice_ne(src, ty).to_le()
+    }
+
+    /// Write a [DataValue] to a memory location in native-endian byte order.
     pub unsafe fn write_value_to(&self, p: *mut u128) {
         let size = self.ty().bytes() as usize;
-        self.write_to_slice(std::slice::from_raw_parts_mut(p as *mut u8, size));
+        self.write_to_slice_ne(std::slice::from_raw_parts_mut(p as *mut u8, size));
     }
 
-    /// Read a [DataValue] from a memory location using a given [Type].
+    /// Read a [DataValue] from a memory location using a given [Type] in native-endian byte order.
     pub unsafe fn read_value_from(p: *const u128, ty: Type) -> Self {
-        DataValue::read_from_slice(
+        DataValue::read_from_slice_ne(
             std::slice::from_raw_parts(p as *const u8, ty.bytes() as usize),
             ty,
         )
diff --git a/cranelift/codegen/src/egraph.rs b/cranelift/codegen/src/egraph.rs
index 2cbe412e3..e12b6fed3 100644
--- a/cranelift/codegen/src/egraph.rs
+++ b/cranelift/codegen/src/egraph.rs
@@ -7,13 +7,14 @@ use crate::dominator_tree::DominatorTree;
 use crate::egraph::domtree::DomTreeWithChildren;
 use crate::egraph::elaborate::Elaborator;
 use crate::fx::FxHashSet;
-use crate::inst_predicates::is_pure_for_egraph;
+use crate::inst_predicates::{is_mergeable_for_egraph, is_pure_for_egraph};
 use crate::ir::{
-    DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
+    Block, DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
 };
 use crate::loop_analysis::LoopAnalysis;
 use crate::opts::generated_code::ContextIter;
 use crate::opts::IsleContext;
+use crate::scoped_hash_map::{Entry as ScopedEntry, ScopedHashMap};
 use crate::trace;
 use crate::unionfind::UnionFind;
 use cranelift_entity::packed_option::ReservedValue;
@@ -77,6 +78,7 @@ where
     pub(crate) func: &'opt mut Function,
     pub(crate) value_to_opt_value: &'opt mut SecondaryMap<Value, Value>,
     pub(crate) gvn_map: &'opt mut CtxHashMap<(Type, InstructionData), Value>,
+    pub(crate) effectful_gvn_map: &'opt mut ScopedHashMap<(Type, InstructionData), Value>,
     pub(crate) eclasses: &'opt mut UnionFind<Value>,
     pub(crate) remat_values: &'opt mut FxHashSet<Value>,
     pub(crate) stats: &'opt mut Stats,
@@ -285,10 +287,49 @@ where
     fn optimize_skeleton_inst(&mut self, inst: Inst) -> bool {
         self.stats.skeleton_inst += 1;
 
-        // If a load or store, process it with the alias analysis to see
-        // if we can optimize it (rewrite in terms of an earlier load or
-        // stored value).
-        if let Some(new_result) =
+        // First, can we try to deduplicate? We need to keep some copy
+        // of the instruction around because it's side-effecting, but
+        // we may be able to reuse an earlier instance of it.
+        if is_mergeable_for_egraph(self.func, inst) {
+            let result = self.func.dfg.inst_results(inst)[0];
+            trace!(" -> mergeable side-effecting op {}", inst);
+
+            // Does this instruction already exist? If so, add entries to
+            // the value-map to rewrite uses of its results to the results
+            // of the original (existing) instruction. If not, optimize
+            // the new instruction.
+            //
+            // Note that we use the "effectful GVN map", which is
+            // scoped: because effectful ops are not removed from the
+            // skeleton (`Layout`), we need to be mindful of whether
+            // our current position is dominated by an instance of the
+            // instruction. (See #5796 for details.)
+            let ty = self.func.dfg.ctrl_typevar(inst);
+            match self
+                .effectful_gvn_map
+                .entry((ty, self.func.dfg.insts[inst].clone()))
+            {
+                ScopedEntry::Occupied(o) => {
+                    let orig_result = *o.get();
+                    // Hit in GVN map -- reuse value.
+                    self.value_to_opt_value[result] = orig_result;
+                    self.eclasses.union(orig_result, result);
+                    trace!(" -> merges result {} to {}", result, orig_result);
+                    true
+                }
+                ScopedEntry::Vacant(v) => {
+                    // Otherwise, insert it into the value-map.
+                    self.value_to_opt_value[result] = result;
+                    v.insert(result);
+                    trace!(" -> inserts as new (no GVN)");
+                    false
+                }
+            }
+        }
+        // Otherwise, if a load or store, process it with the alias
+        // analysis to see if we can optimize it (rewrite in terms of
+        // an earlier load or stored value).
+        else if let Some(new_result) =
             self.alias_analysis
                 .process_inst(self.func, self.alias_analysis_state, inst)
         {
@@ -382,83 +423,127 @@ impl<'a> EgraphPass<'a> {
         let mut cursor = FuncCursor::new(self.func);
         let mut value_to_opt_value: SecondaryMap<Value, Value> =
             SecondaryMap::with_default(Value::reserved_value());
+        // Map from instruction to value for hash-consing of pure ops
+        // into the egraph. This can be a standard (non-scoped)
+        // hashmap because pure ops have no location: they are
+        // "outside of" control flow.
+        //
+        // Note also that we keep the controlling typevar (the `Type`
+        // in the tuple below) because it may disambiguate
+        // instructions that are identical except for type.
         let mut gvn_map: CtxHashMap<(Type, InstructionData), Value> =
             CtxHashMap::with_capacity(cursor.func.dfg.num_values());
+        // Map from instruction to value for GVN'ing of effectful but
+        // idempotent ops, which remain in the side-effecting
+        // skeleton. This needs to be scoped because we cannot
+        // deduplicate one instruction to another that is in a
+        // non-dominating block.
+        //
+        // Note that we can use a ScopedHashMap here without the
+        // "context" (as needed by CtxHashMap) because in practice the
+        // ops we want to GVN have all their args inline. Equality on
+        // the InstructionData itself is conservative: two insts whose
+        // struct contents compare shallowly equal are definitely
+        // identical, but identical insts in a deep-equality sense may
+        // not compare shallowly equal, due to list indirection. This
+        // is fine for GVN, because it is still sound to skip any
+        // given GVN opportunity (and keep the original instructions).
+        //
+        // As above, we keep the controlling typevar here as part of
+        // the key: effectful instructions may (as for pure
+        // instructions) be differentiated only on the type.
+        let mut effectful_gvn_map: ScopedHashMap<(Type, InstructionData), Value> =
+            ScopedHashMap::new();
 
         // In domtree preorder, visit blocks. (TODO: factor out an
         // iterator from this and elaborator.)
         let root = self.domtree_children.root();
-        let mut block_stack = vec![root];
-        while let Some(block) = block_stack.pop() {
-            // We popped this block; push children
-            // immediately, then process this block.
-            block_stack.extend(self.domtree_children.children(block));
-
-            trace!("Processing block {}", block);
-            cursor.set_position(CursorPosition::Before(block));
-
-            let mut alias_analysis_state = self.alias_analysis.block_starting_state(block);
-
-            for &param in cursor.func.dfg.block_params(block) {
-                trace!("creating initial singleton eclass for blockparam {}", param);
-                self.eclasses.add(param);
-                value_to_opt_value[param] = param;
-            }
-            while let Some(inst) = cursor.next_inst() {
-                trace!("Processing inst {}", inst);
-
-                // While we're passing over all insts, create initial
-                // singleton eclasses for all result and blockparam
-                // values.  Also do initial analysis of all inst
-                // results.
-                for &result in cursor.func.dfg.inst_results(inst) {
-                    trace!("creating initial singleton eclass for {}", result);
-                    self.eclasses.add(result);
-                }
+        enum StackEntry {
+            Visit(Block),
+            Pop,
+        }
+        let mut block_stack = vec![StackEntry::Visit(root)];
+        while let Some(entry) = block_stack.pop() {
+            match entry {
+                StackEntry::Visit(block) => {
+                    // We popped this block; push children
+                    // immediately, then process this block.
+                    block_stack.push(StackEntry::Pop);
+                    block_stack
+                        .extend(self.domtree_children.children(block).map(StackEntry::Visit));
+                    effectful_gvn_map.increment_depth();
+
+                    trace!("Processing block {}", block);
+                    cursor.set_position(CursorPosition::Before(block));
+
+                    let mut alias_analysis_state = self.alias_analysis.block_starting_state(block);
+
+                    for &param in cursor.func.dfg.block_params(block) {
+                        trace!("creating initial singleton eclass for blockparam {}", param);
+                        self.eclasses.add(param);
+                        value_to_opt_value[param] = param;
+                    }
+                    while let Some(inst) = cursor.next_inst() {
+                        trace!("Processing inst {}", inst);
+
+                        // While we're passing over all insts, create initial
+                        // singleton eclasses for all result and blockparam
+                        // values.  Also do initial analysis of all inst
+                        // results.
+                        for &result in cursor.func.dfg.inst_results(inst) {
+                            trace!("creating initial singleton eclass for {}", result);
+                            self.eclasses.add(result);
+                        }
 
-                // Rewrite args of *all* instructions using the
-                // value-to-opt-value map.
-                cursor.func.dfg.resolve_aliases_in_arguments(inst);
-                cursor.func.dfg.map_inst_values(inst, |_, arg| {
-                    let new_value = value_to_opt_value[arg];
-                    trace!("rewriting arg {} of inst {} to {}", arg, inst, new_value);
-                    debug_assert_ne!(new_value, Value::reserved_value());
-                    new_value
-                });
-
-                // Build a context for optimization, with borrows of
-                // state. We can't invoke a method on `self` because
-                // we've borrowed `self.func` mutably (as
-                // `cursor.func`) so we pull apart the pieces instead
-                // here.
-                let mut ctx = OptimizeCtx {
-                    func: cursor.func,
-                    value_to_opt_value: &mut value_to_opt_value,
-                    gvn_map: &mut gvn_map,
-                    eclasses: &mut self.eclasses,
-                    rewrite_depth: 0,
-                    subsume_values: FxHashSet::default(),
-                    remat_values: &mut self.remat_values,
-                    stats: &mut self.stats,
-                    alias_analysis: self.alias_analysis,
-                    alias_analysis_state: &mut alias_analysis_state,
-                };
-
-                if is_pure_for_egraph(ctx.func, inst) {
-                    // Insert into GVN map and optimize any new nodes
-                    // inserted (recursively performing this work for
-                    // any nodes the optimization rules produce).
-                    let inst = NewOrExistingInst::Existing(inst);
-                    ctx.insert_pure_enode(inst);
-                    // We've now rewritten all uses, or will when we
-                    // see them, and the instruction exists as a pure
-                    // enode in the eclass, so we can remove it.
-                    cursor.remove_inst_and_step_back();
-                } else {
-                    if ctx.optimize_skeleton_inst(inst) {
-                        cursor.remove_inst_and_step_back();
+                        // Rewrite args of *all* instructions using the
+                        // value-to-opt-value map.
+                        cursor.func.dfg.resolve_aliases_in_arguments(inst);
+                        cursor.func.dfg.map_inst_values(inst, |_, arg| {
+                            let new_value = value_to_opt_value[arg];
+                            trace!("rewriting arg {} of inst {} to {}", arg, inst, new_value);
+                            debug_assert_ne!(new_value, Value::reserved_value());
+                            new_value
+                        });
+
+                        // Build a context for optimization, with borrows of
+                        // state. We can't invoke a method on `self` because
+                        // we've borrowed `self.func` mutably (as
+                        // `cursor.func`) so we pull apart the pieces instead
+                        // here.
+                        let mut ctx = OptimizeCtx {
+                            func: cursor.func,
+                            value_to_opt_value: &mut value_to_opt_value,
+                            gvn_map: &mut gvn_map,
+                            effectful_gvn_map: &mut effectful_gvn_map,
+                            eclasses: &mut self.eclasses,
+                            rewrite_depth: 0,
+                            subsume_values: FxHashSet::default(),
+                            remat_values: &mut self.remat_values,
+                            stats: &mut self.stats,
+                            alias_analysis: self.alias_analysis,
+                            alias_analysis_state: &mut alias_analysis_state,
+                        };
+
+                        if is_pure_for_egraph(ctx.func, inst) {
+                            // Insert into GVN map and optimize any new nodes
+                            // inserted (recursively performing this work for
+                            // any nodes the optimization rules produce).
+                            let inst = NewOrExistingInst::Existing(inst);
+                            ctx.insert_pure_enode(inst);
+                            // We've now rewritten all uses, or will when we
+                            // see them, and the instruction exists as a pure
+                            // enode in the eclass, so we can remove it.
+                            cursor.remove_inst_and_step_back();
+                        } else {
+                            if ctx.optimize_skeleton_inst(inst) {
+                                cursor.remove_inst_and_step_back();
+                            }
+                        }
                     }
                 }
+                StackEntry::Pop => {
+                    effectful_gvn_map.decrement_depth();
+                }
             }
         }
     }
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index 9b284e8b8..f82f41daa 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -73,6 +73,25 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
     has_one_result && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
 }
 
+/// Can the given instruction be merged into another copy of itself?
+/// These instructions may have side-effects, but as long as we retain
+/// the first instance of the instruction, the second and further
+/// instances are redundant if they would produce the same trap or
+/// result.
+pub fn is_mergeable_for_egraph(func: &Function, inst: Inst) -> bool {
+    let op = func.dfg.insts[inst].opcode();
+    // We can only merge one-result operators due to the way that GVN
+    // is structured in the egraph implementation.
+    let has_one_result = func.dfg.inst_results(inst).len() == 1;
+    has_one_result
+        // Loads/stores are handled by alias analysis and not
+        // otherwise mergeable.
+        && !op.can_load()
+        && !op.can_store()
+        // Can only have idempotent side-effects.
+        && (!has_side_effect(func, inst) || op.side_effects_idempotent())
+}
+
 /// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,
 /// but not the get_pinned_reg opcode?
 pub fn has_lowering_side_effect(func: &Function, inst: Inst) -> bool {
diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs
index 97e0ac0b5..40507db01 100644
--- a/cranelift/codegen/src/ir/instructions.rs
+++ b/cranelift/codegen/src/ir/instructions.rs
@@ -45,7 +45,7 @@ pub type ValueListPool = entity::ListPool<Value>;
 /// The BlockCall::new function guarantees this layout by requiring a block argument that's written
 /// in as the first element of the EntityList. Any subsequent entries are always assumed to be real
 /// Values.
-#[derive(Debug, Clone, Copy, PartialEq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct BlockCall {
     /// The underlying storage for the BlockCall. The first element of the values EntityList is
@@ -403,7 +403,9 @@ impl InstructionData {
             &InstructionData::Load { flags, .. }
             | &InstructionData::LoadNoOffset { flags, .. }
             | &InstructionData::Store { flags, .. }
-            | &InstructionData::StoreNoOffset { flags, .. } => Some(flags),
+            | &InstructionData::StoreNoOffset { flags, .. }
+            | &InstructionData::AtomicCas { flags, .. }
+            | &InstructionData::AtomicRmw { flags, .. } => Some(flags),
             _ => None,
         }
     }
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index b8bf2ef48..53b0e65ed 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -852,7 +852,7 @@
         (rd WritableReg)
         ;; Offset in range -2^20 .. 2^20.
         (off i32))
-      
+
        ;; Compute the address (using a PC-relative offset) of a 4KB page.
        (Adrp
         (rd WritableReg)
@@ -1401,6 +1401,8 @@
     (Bsl)
     ;; Floating-point fused multiply-add vectors
     (Fmla)
+    ;; Floating-point fused multiply-subtract vectors
+    (Fmls)
 ))
 
 ;; A Vector miscellaneous operation with two registers.
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 90a5dbd93..72b1e1176 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -2906,6 +2906,9 @@ impl MachInstEmit for Inst {
                     VecALUModOp::Fmla => {
                         (0b000_01110_00_1 | (size.enc_float_size() << 1), 0b110011)
                     }
+                    VecALUModOp::Fmls => {
+                        (0b000_01110_10_1 | (size.enc_float_size() << 1), 0b110011)
+                    }
                 };
                 sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
             }
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 182660335..779782907 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -2363,6 +2363,7 @@ impl Inst {
                 let (op, size) = match alu_op {
                     VecALUModOp::Bsl => ("bsl", VectorSize::Size8x16),
                     VecALUModOp::Fmla => ("fmla", size),
+                    VecALUModOp::Fmls => ("fmls", size),
                 };
                 let rd = pretty_print_vreg_vector(rd.to_reg(), size, allocs);
                 let ri = pretty_print_vreg_vector(ri, size, allocs);
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 3eec28c11..bd7e968d7 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -404,7 +404,13 @@
 (rule (lower (has_type ty @ (multi_lane _ _) (fma x y z)))
       (vec_rrr_mod (VecALUModOp.Fmla) z x y (vector_size ty)))
 
-(rule 1 (lower (has_type (ty_scalar_float ty) (fma x y z)))
+(rule 1 (lower (has_type ty @ (multi_lane _ _) (fma (fneg x) y z)))
+      (vec_rrr_mod (VecALUModOp.Fmls) z x y (vector_size ty)))
+
+(rule 2 (lower (has_type ty @ (multi_lane _ _) (fma x (fneg y) z)))
+      (vec_rrr_mod (VecALUModOp.Fmls) z x y (vector_size ty)))
+
+(rule 3 (lower (has_type (ty_scalar_float ty) (fma x y z)))
       (fpu_rrrr (FPUOp3.MAdd) (scalar_size ty) x y z))
 
 ;;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 9f83ccbfb..ec21968b3 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -522,12 +522,10 @@
             ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be
             ;; correct for half of the lanes; the others must be fixed up with
             ;; the mask below.
-            (unmasked Xmm (x64_psrlw src (mov_rmi_to_xmm masked_amt)))
-            (mask_addr SyntheticAmode (ushr_i8x16_mask masked_amt))
-            (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None))))
+            (unmasked Xmm (x64_psrlw src (mov_rmi_to_xmm masked_amt))))
         (sse_and $I8X16
                  unmasked
-                 (RegMem.Reg mask))))
+                 (ushr_i8x16_mask masked_amt))))
 
 ;; Get the address of the mask to use when fixing up the lanes that weren't
 ;; correctly generated by the 16x8 shift.
diff --git a/cranelift/codegen/src/opts/algebraic.isle b/cranelift/codegen/src/opts/algebraic.isle
index ecd71988c..eac165065 100644
--- a/cranelift/codegen/src/opts/algebraic.isle
+++ b/cranelift/codegen/src/opts/algebraic.isle
@@ -527,3 +527,13 @@
 (rule (simplify
        (vselect ty (fcmp _ (FloatCC.GreaterThan) x y) x y))
       (fmax_pseudo ty x y))
+
+;; If both of the multiplied arguments to an `fma` are negated then remove
+;; both of them since they cancel out.
+(rule (simplify (fma ty (fneg ty x) (fneg ty y) z))
+      (fma ty x y z))
+
+;; If both of the multiplied arguments to an `fmul` are negated then remove
+;; both of them since they cancel out.
+(rule (simplify (fmul ty (fneg ty x) (fneg ty y)))
+      (fmul ty x y))
diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs
index 659b94a9b..682eb256f 100644
--- a/cranelift/entity/src/list.rs
+++ b/cranelift/entity/src/list.rs
@@ -62,7 +62,7 @@ use serde::{Deserialize, Serialize};
 ///
 /// The index stored in an `EntityList` points to part 2, the list elements. The value 0 is
 /// reserved for the empty list which isn't allocated in the vector.
-#[derive(Clone, Copy, Debug, PartialEq, Hash)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct EntityList<T: EntityRef + ReservedValue> {
     index: u32,
diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs
index 04d2815ba..26cca8363 100644
--- a/cranelift/fuzzgen/src/lib.rs
+++ b/cranelift/fuzzgen/src/lib.rs
@@ -7,190 +7,27 @@ use cranelift::codegen::data_value::DataValue;
 use cranelift::codegen::ir::{types::*, UserExternalName, UserFuncName};
 use cranelift::codegen::ir::{Function, LibCall};
 use cranelift::codegen::Context;
-use cranelift::prelude::isa;
 use cranelift::prelude::*;
 use cranelift_arbitrary::CraneliftArbitrary;
 use cranelift_native::builder_with_options;
-use std::fmt;
 use target_lexicon::{Architecture, Triple};
 
 mod config;
 mod cranelift_arbitrary;
 mod function_generator;
 mod passes;
+mod print;
 
-/// These libcalls need a interpreter implementation in `cranelift-fuzzgen.rs`
-const ALLOWED_LIBCALLS: &'static [LibCall] = &[
-    LibCall::CeilF32,
-    LibCall::CeilF64,
-    LibCall::FloorF32,
-    LibCall::FloorF64,
-    LibCall::TruncF32,
-    LibCall::TruncF64,
-];
+pub use print::PrintableTestCase;
 
 pub type TestCaseInput = Vec<DataValue>;
 
-/// Print only non default flags.
-fn write_non_default_flags(f: &mut fmt::Formatter<'_>, flags: &settings::Flags) -> fmt::Result {
-    let default_flags = settings::Flags::new(settings::builder());
-    for (default, flag) in default_flags.iter().zip(flags.iter()) {
-        assert_eq!(default.name, flag.name);
-
-        if default.value_string() != flag.value_string() {
-            writeln!(f, "set {}={}", flag.name, flag.value_string())?;
-        }
-    }
-
-    Ok(())
-}
-
-/// A generated function with an ISA that targets one of cranelift's backends.
-pub struct FunctionWithIsa {
-    /// TargetIsa to use when compiling this test case
-    pub isa: isa::OwnedTargetIsa,
-
-    /// Function under test
-    pub func: Function,
-}
-
-impl fmt::Debug for FunctionWithIsa {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, ";; Compile test case\n")?;
-
-        write_non_default_flags(f, self.isa.flags())?;
-
-        writeln!(f, "test compile")?;
-        writeln!(f, "target {}", self.isa.triple().architecture)?;
-        writeln!(f, "{}", self.func)?;
-
-        Ok(())
-    }
-}
-
-impl<'a> Arbitrary<'a> for FunctionWithIsa {
-    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
-        // We filter out targets that aren't supported in the current build
-        // configuration after randomly choosing one, instead of randomly choosing
-        // a supported one, so that the same fuzz input works across different build
-        // configurations.
-        let target = u.choose(isa::ALL_ARCHITECTURES)?;
-        let builder = isa::lookup_by_name(target).map_err(|_| arbitrary::Error::IncorrectFormat)?;
-        let architecture = builder.triple().architecture;
-
-        let mut gen = FuzzGen::new(u);
-        let flags = gen
-            .generate_flags(architecture)
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-        let isa = builder
-            .finish(flags)
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-
-        // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
-        let fname = UserFuncName::user(1, 0);
-
-        // We don't actually generate these functions, we just simulate their signatures and names
-        let func_count = gen.u.int_in_range(gen.config.testcase_funcs.clone())?;
-        let usercalls = (0..func_count)
-            .map(|i| {
-                let name = UserExternalName::new(2, i as u32);
-                let sig = gen.generate_signature(architecture)?;
-                Ok((name, sig))
-            })
-            .collect::<Result<Vec<(UserExternalName, Signature)>>>()
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-
-        let func = gen
-            .generate_func(fname, isa.triple().clone(), usercalls)
-            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
-
-        Ok(FunctionWithIsa { isa, func })
-    }
-}
-
-pub struct TestCase {
-    /// TargetIsa to use when compiling this test case
-    pub isa: isa::OwnedTargetIsa,
-    /// Functions under test
-    /// By convention the first function is the main function.
-    pub functions: Vec<Function>,
-    /// Generate multiple test inputs for each test case.
-    /// This allows us to get more coverage per compilation, which may be somewhat expensive.
-    pub inputs: Vec<TestCaseInput>,
-}
-
-impl fmt::Debug for TestCase {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, ";; Fuzzgen test case\n")?;
-        writeln!(f, "test interpret")?;
-        writeln!(f, "test run")?;
-
-        write_non_default_flags(f, self.isa.flags())?;
-
-        writeln!(f, "target {}\n", self.isa.triple().architecture)?;
-
-        // Print the functions backwards, so that the main function is printed last
-        // and near the test inputs.
-        for func in self.functions.iter().rev() {
-            writeln!(f, "{}\n", func)?;
-        }
-
-        writeln!(f, "; Note: the results in the below test cases are simply a placeholder and probably will be wrong\n")?;
-
-        for input in self.inputs.iter() {
-            // TODO: We don't know the expected outputs, maybe we can run the interpreter
-            // here to figure them out? Should work, however we need to be careful to catch
-            // panics in case its the interpreter that is failing.
-            // For now create a placeholder output consisting of the zero value for the type
-            let returns = &self.main().signature.returns;
-            let placeholder_output = returns
-                .iter()
-                .map(|param| DataValue::read_from_slice(&[0; 16][..], param.value_type))
-                .map(|val| format!("{}", val))
-                .collect::<Vec<_>>()
-                .join(", ");
-
-            // If we have no output, we don't need the == condition
-            let test_condition = match returns.len() {
-                0 => String::new(),
-                1 => format!(" == {}", placeholder_output),
-                _ => format!(" == [{}]", placeholder_output),
-            };
-
-            let args = input
-                .iter()
-                .map(|val| format!("{}", val))
-                .collect::<Vec<_>>()
-                .join(", ");
-
-            writeln!(f, "; run: {}({}){}", self.main().name, args, test_condition)?;
-        }
-
-        Ok(())
-    }
-}
-
-impl<'a> Arbitrary<'a> for TestCase {
-    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
-        FuzzGen::new(u)
-            .generate_host_test()
-            .map_err(|_| arbitrary::Error::IncorrectFormat)
-    }
-}
-
-impl TestCase {
-    /// Returns the main function of this test case.
-    pub fn main(&self) -> &Function {
-        &self.functions[0]
-    }
-}
-
 pub struct FuzzGen<'r, 'data>
 where
     'data: 'r,
 {
-    u: &'r mut Unstructured<'data>,
-    config: Config,
+    pub u: &'r mut Unstructured<'data>,
+    pub config: Config,
 }
 
 impl<'r, 'data> FuzzGen<'r, 'data>
@@ -204,13 +41,13 @@ where
         }
     }
 
-    fn generate_signature(&mut self, architecture: Architecture) -> Result<Signature> {
+    pub fn generate_signature(&mut self, architecture: Architecture) -> Result<Signature> {
         let max_params = self.u.int_in_range(self.config.signature_params.clone())?;
         let max_rets = self.u.int_in_range(self.config.signature_rets.clone())?;
         Ok(self.u.signature(architecture, max_params, max_rets)?)
     }
 
-    fn generate_test_inputs(mut self, signature: &Signature) -> Result<Vec<TestCaseInput>> {
+    pub fn generate_test_inputs(mut self, signature: &Signature) -> Result<Vec<TestCaseInput>> {
         let mut inputs = Vec::new();
 
         // Generate up to "max_test_case_inputs" inputs, we need an upper bound here since
@@ -288,11 +125,12 @@ where
         Ok(ctx.func)
     }
 
-    fn generate_func(
+    pub fn generate_func(
         &mut self,
         name: UserFuncName,
         target_triple: Triple,
         usercalls: Vec<(UserExternalName, Signature)>,
+        libcalls: Vec<LibCall>,
     ) -> Result<Function> {
         let sig = self.generate_signature(target_triple.architecture)?;
 
@@ -303,7 +141,7 @@ where
             name,
             sig,
             usercalls,
-            ALLOWED_LIBCALLS.to_vec(),
+            libcalls,
         )
         .generate()?;
 
@@ -312,7 +150,7 @@ where
 
     /// Generate a random set of cranelift flags.
     /// Only semantics preserving flags are considered
-    fn generate_flags(&mut self, target_arch: Architecture) -> Result<Flags> {
+    pub fn generate_flags(&mut self, target_arch: Architecture) -> Result<Flags> {
         let mut builder = settings::builder();
 
         let opt = self.u.choose(OptLevel::all())?;
@@ -390,48 +228,4 @@ where
 
         Ok(Flags::new(builder))
     }
-
-    pub fn generate_host_test(mut self) -> Result<TestCase> {
-        // TestCase is meant to be consumed by a runner, so we make the assumption here that we're
-        // generating a TargetIsa for the host.
-        let builder =
-            builder_with_options(true).expect("Unable to build a TargetIsa for the current host");
-        let flags = self.generate_flags(builder.triple().architecture)?;
-        let isa = builder.finish(flags)?;
-
-        // When generating functions, we allow each function to call any function that has
-        // already been generated. This guarantees that we never have loops in the call graph.
-        // We generate these backwards, and then reverse them so that the main function is at
-        // the start.
-        let func_count = self.u.int_in_range(self.config.testcase_funcs.clone())?;
-        let mut functions: Vec<Function> = Vec::with_capacity(func_count);
-        for i in (0..func_count).rev() {
-            // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
-            let fname = UserFuncName::user(1, i as u32);
-
-            let usercalls: Vec<(UserExternalName, Signature)> = functions
-                .iter()
-                .map(|f| {
-                    (
-                        f.name.get_user().unwrap().clone(),
-                        f.stencil.signature.clone(),
-                    )
-                })
-                .collect();
-
-            let func = self.generate_func(fname, isa.triple().clone(), usercalls)?;
-            functions.push(func);
-        }
-        // Now reverse the functions so that the main function is at the start.
-        functions.reverse();
-
-        let main = &functions[0];
-        let inputs = self.generate_test_inputs(&main.signature)?;
-
-        Ok(TestCase {
-            isa,
-            functions,
-            inputs,
-        })
-    }
 }
diff --git a/cranelift/fuzzgen/src/print.rs b/cranelift/fuzzgen/src/print.rs
new file mode 100644
index 000000000..e74f0f3a8
--- /dev/null
+++ b/cranelift/fuzzgen/src/print.rs
@@ -0,0 +1,130 @@
+use cranelift::codegen::data_value::DataValue;
+use cranelift::codegen::ir::Function;
+use cranelift::prelude::settings;
+use cranelift::prelude::*;
+use std::fmt;
+
+use crate::TestCaseInput;
+
+#[derive(Debug)]
+enum TestCaseKind {
+    Compile,
+    Run,
+}
+
+/// Provides a way to format a `TestCase` in the .clif format.
+pub struct PrintableTestCase<'a> {
+    kind: TestCaseKind,
+    isa: &'a isa::OwnedTargetIsa,
+    functions: &'a [Function],
+    // Only applicable for run test cases
+    inputs: &'a [TestCaseInput],
+}
+
+impl<'a> PrintableTestCase<'a> {
+    /// Emits a `test compile` test case.
+    pub fn compile(isa: &'a isa::OwnedTargetIsa, functions: &'a [Function]) -> Self {
+        Self {
+            kind: TestCaseKind::Compile,
+            isa,
+            functions,
+            inputs: &[],
+        }
+    }
+
+    /// Emits a `test run` test case. These also include a `test interpret`.
+    ///
+    /// By convention the first function in `functions` will be considered the main function.
+    pub fn run(
+        isa: &'a isa::OwnedTargetIsa,
+        functions: &'a [Function],
+        inputs: &'a [TestCaseInput],
+    ) -> Self {
+        Self {
+            kind: TestCaseKind::Run,
+            isa,
+            functions,
+            inputs,
+        }
+    }
+
+    /// Returns the main function of this test case.
+    pub fn main(&self) -> &Function {
+        &self.functions[0]
+    }
+}
+
+impl<'a> fmt::Debug for PrintableTestCase<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self.kind {
+            TestCaseKind::Compile => {
+                writeln!(f, ";; Compile test case\n")?;
+                writeln!(f, "test compile")?;
+            }
+            TestCaseKind::Run => {
+                writeln!(f, ";; Run test case\n")?;
+                writeln!(f, "test interpret")?;
+                writeln!(f, "test run")?;
+            }
+        };
+
+        write_non_default_flags(f, self.isa.flags())?;
+
+        writeln!(f, "target {}\n", self.isa.triple().architecture)?;
+
+        // Print the functions backwards, so that the main function is printed last
+        // and near the test inputs for run test cases.
+        for func in self.functions.iter().rev() {
+            writeln!(f, "{}\n", func)?;
+        }
+
+        if !self.inputs.is_empty() {
+            writeln!(f, "; Note: the results in the below test cases are simply a placeholder and probably will be wrong\n")?;
+        }
+
+        for input in self.inputs.iter() {
+            // TODO: We don't know the expected outputs, maybe we can run the interpreter
+            // here to figure them out? Should work, however we need to be careful to catch
+            // panics in case its the interpreter that is failing.
+            // For now create a placeholder output consisting of the zero value for the type
+            let returns = &self.main().signature.returns;
+            let placeholder_output = returns
+                .iter()
+                .map(|param| DataValue::read_from_slice_ne(&[0; 16][..], param.value_type))
+                .map(|val| format!("{}", val))
+                .collect::<Vec<_>>()
+                .join(", ");
+
+            // If we have no output, we don't need the == condition
+            let test_condition = match returns.len() {
+                0 => String::new(),
+                1 => format!(" == {}", placeholder_output),
+                _ => format!(" == [{}]", placeholder_output),
+            };
+
+            let args = input
+                .iter()
+                .map(|val| format!("{}", val))
+                .collect::<Vec<_>>()
+                .join(", ");
+
+            writeln!(f, "; run: {}({}){}", self.main().name, args, test_condition)?;
+        }
+
+        Ok(())
+    }
+}
+
+/// Print only non default flags.
+fn write_non_default_flags(f: &mut fmt::Formatter<'_>, flags: &settings::Flags) -> fmt::Result {
+    let default_flags = settings::Flags::new(settings::builder());
+    for (default, flag) in default_flags.iter().zip(flags.iter()) {
+        assert_eq!(default.name, flag.name);
+
+        if default.value_string() != flag.value_string() {
+            writeln!(f, "set {}={}", flag.name, flag.value_string())?;
+        }
+    }
+
+    Ok(())
+}
diff --git a/cranelift/interpreter/src/interpreter.rs b/cranelift/interpreter/src/interpreter.rs
index 290901dc4..aa44f8e3b 100644
--- a/cranelift/interpreter/src/interpreter.rs
+++ b/cranelift/interpreter/src/interpreter.rs
@@ -11,8 +11,8 @@ use crate::step::{step, ControlFlow, StepError};
 use crate::value::{Value, ValueError};
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::{
-    ArgumentPurpose, Block, ExternalName, FuncRef, Function, GlobalValue, GlobalValueData, LibCall,
-    StackSlot, TrapCode, Type, Value as ValueRef,
+    ArgumentPurpose, Block, Endianness, ExternalName, FuncRef, Function, GlobalValue,
+    GlobalValueData, LibCall, MemFlags, StackSlot, TrapCode, Type, Value as ValueRef,
 };
 use log::trace;
 use smallvec::SmallVec;
@@ -192,10 +192,16 @@ pub struct InterpreterState<'a> {
     pub frame_offset: usize,
     pub stack: Vec<u8>,
     pub pinned_reg: DataValue,
+    pub native_endianness: Endianness,
 }
 
 impl Default for InterpreterState<'_> {
     fn default() -> Self {
+        let native_endianness = if cfg!(target_endian = "little") {
+            Endianness::Little
+        } else {
+            Endianness::Big
+        };
         Self {
             functions: FunctionStore::default(),
             libcall_handler: |_, _| Err(TrapCode::UnreachableCodeReached),
@@ -203,6 +209,7 @@ impl Default for InterpreterState<'_> {
             frame_offset: 0,
             stack: Vec::with_capacity(1024),
             pinned_reg: DataValue::U64(0),
+            native_endianness,
         }
     }
 }
@@ -308,7 +315,12 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
         Address::from_parts(size, AddressRegion::Stack, 0, final_offset)
     }
 
-    fn checked_load(&self, addr: Address, ty: Type) -> Result<DataValue, MemoryError> {
+    fn checked_load(
+        &self,
+        addr: Address,
+        ty: Type,
+        mem_flags: MemFlags,
+    ) -> Result<DataValue, MemoryError> {
         let load_size = ty.bytes() as usize;
         let addr_start = addr.offset as usize;
         let addr_end = addr_start + load_size;
@@ -324,10 +336,18 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
             _ => unimplemented!(),
         };
 
-        Ok(DataValue::read_from_slice(src, ty))
+        Ok(match mem_flags.endianness(self.native_endianness) {
+            Endianness::Big => DataValue::read_from_slice_be(src, ty),
+            Endianness::Little => DataValue::read_from_slice_le(src, ty),
+        })
     }
 
-    fn checked_store(&mut self, addr: Address, v: DataValue) -> Result<(), MemoryError> {
+    fn checked_store(
+        &mut self,
+        addr: Address,
+        v: DataValue,
+        mem_flags: MemFlags,
+    ) -> Result<(), MemoryError> {
         let store_size = v.ty().bytes() as usize;
         let addr_start = addr.offset as usize;
         let addr_end = addr_start + store_size;
@@ -343,7 +363,10 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
             _ => unimplemented!(),
         };
 
-        Ok(v.write_to_slice(dst))
+        Ok(match mem_flags.endianness(self.native_endianness) {
+            Endianness::Big => v.write_to_slice_be(dst),
+            Endianness::Little => v.write_to_slice_le(dst),
+        })
     }
 
     fn function_address(
@@ -493,9 +516,10 @@ impl<'a> State<'a, DataValue> for InterpreterState<'a> {
                     global_type,
                 }) => {
                     let mut addr = Address::try_from(current_val)?;
+                    let mem_flags = MemFlags::trusted();
                     // We can forego bounds checking here since its performed in `checked_load`
                     addr.offset += offset as u64;
-                    current_val = self.checked_load(addr, global_type)?;
+                    current_val = self.checked_load(addr, global_type, mem_flags)?;
                 }
 
                 // We are done resolving this, return the current value
diff --git a/cranelift/interpreter/src/state.rs b/cranelift/interpreter/src/state.rs
index b98f0c552..c797bea7e 100644
--- a/cranelift/interpreter/src/state.rs
+++ b/cranelift/interpreter/src/state.rs
@@ -4,7 +4,8 @@ use crate::address::{Address, AddressSize};
 use crate::interpreter::LibCallHandler;
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::{
-    ExternalName, FuncRef, Function, GlobalValue, LibCall, Signature, StackSlot, Type, Value,
+    ExternalName, FuncRef, Function, GlobalValue, LibCall, MemFlags, Signature, StackSlot, Type,
+    Value,
 };
 use cranelift_codegen::isa::CallConv;
 use cranelift_entity::PrimaryMap;
@@ -62,10 +63,20 @@ pub trait State<'a, V> {
     ) -> Result<Address, MemoryError>;
     /// Retrieve a value `V` from memory at the given `address`, checking if it belongs either to the
     /// stack or to one of the heaps; the number of bytes loaded corresponds to the specified [Type].
-    fn checked_load(&self, address: Address, ty: Type) -> Result<V, MemoryError>;
+    fn checked_load(
+        &self,
+        address: Address,
+        ty: Type,
+        mem_flags: MemFlags,
+    ) -> Result<V, MemoryError>;
     /// Store a value `V` into memory at the given `address`, checking if it belongs either to the
     /// stack or to one of the heaps; the number of bytes stored corresponds to the specified [Type].
-    fn checked_store(&mut self, address: Address, v: V) -> Result<(), MemoryError>;
+    fn checked_store(
+        &mut self,
+        address: Address,
+        v: V,
+        mem_flags: MemFlags,
+    ) -> Result<(), MemoryError>;
 
     /// Compute the address of a function given its name.
     fn function_address(
@@ -182,11 +193,21 @@ where
         unimplemented!()
     }
 
-    fn checked_load(&self, _addr: Address, _ty: Type) -> Result<V, MemoryError> {
+    fn checked_load(
+        &self,
+        _addr: Address,
+        _ty: Type,
+        _mem_flags: MemFlags,
+    ) -> Result<V, MemoryError> {
         unimplemented!()
     }
 
-    fn checked_store(&mut self, _addr: Address, _v: V) -> Result<(), MemoryError> {
+    fn checked_store(
+        &mut self,
+        _addr: Address,
+        _v: V,
+        _mem_flags: MemFlags,
+    ) -> Result<(), MemoryError> {
         unimplemented!()
     }
 
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index 0eddf9c35..d0362e93d 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -8,7 +8,7 @@ use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
 use cranelift_codegen::ir::{
     types, AbiParam, AtomicRmwOp, Block, BlockCall, ExternalName, FuncRef, Function,
-    InstructionData, Opcode, TrapCode, Type, Value as ValueRef,
+    InstructionData, MemFlags, Opcode, TrapCode, Type, Value as ValueRef,
 };
 use log::trace;
 use smallvec::{smallvec, SmallVec};
@@ -482,8 +482,10 @@ where
             };
 
             let addr_value = calculate_addr(types::I64, imm(), args()?)?;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             let loaded = assign_or_memtrap(
-                Address::try_from(addr_value).and_then(|addr| state.checked_load(addr, load_ty)),
+                Address::try_from(addr_value)
+                    .and_then(|addr| state.checked_load(addr, load_ty, mem_flags)),
             );
 
             match (loaded, kind) {
@@ -505,33 +507,37 @@ where
             };
 
             let addr_value = calculate_addr(types::I64, imm(), args_range(1..)?)?;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             let reduced = if let Some(c) = kind {
                 arg(0)?.convert(c)?
             } else {
                 arg(0)?
             };
             continue_or_memtrap(
-                Address::try_from(addr_value).and_then(|addr| state.checked_store(addr, reduced)),
+                Address::try_from(addr_value)
+                    .and_then(|addr| state.checked_store(addr, reduced, mem_flags)),
             )
         }
         Opcode::StackLoad => {
             let load_ty = inst_context.controlling_type().unwrap();
             let slot = inst.stack_slot().unwrap();
             let offset = sum(imm(), args()?)? as u64;
+            let mem_flags = MemFlags::trusted();
             assign_or_memtrap({
                 state
                     .stack_address(AddressSize::_64, slot, offset)
-                    .and_then(|addr| state.checked_load(addr, load_ty))
+                    .and_then(|addr| state.checked_load(addr, load_ty, mem_flags))
             })
         }
         Opcode::StackStore => {
             let arg = arg(0)?;
             let slot = inst.stack_slot().unwrap();
             let offset = sum(imm(), args_range(1..)?)? as u64;
+            let mem_flags = MemFlags::trusted();
             continue_or_memtrap({
                 state
                     .stack_address(AddressSize::_64, slot, offset)
-                    .and_then(|addr| state.checked_store(addr, arg))
+                    .and_then(|addr| state.checked_store(addr, arg, mem_flags))
             })
         }
         Opcode::StackAddr => {
@@ -1238,7 +1244,9 @@ where
             let op = inst.atomic_rmw_op().unwrap();
             let val = arg(1)?;
             let addr = arg(0)?.into_int()? as u64;
-            let loaded = Address::try_from(addr).and_then(|addr| state.checked_load(addr, ctrl_ty));
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
+            let loaded = Address::try_from(addr)
+                .and_then(|addr| state.checked_load(addr, ctrl_ty, mem_flags));
             let prev_val = match loaded {
                 Ok(v) => v,
                 Err(e) => return Ok(ControlFlow::Trap(CraneliftTrap::User(memerror_to_trap(e)))),
@@ -1265,13 +1273,15 @@ where
                 )
                 .and_then(|v| Value::convert(v, ValueConversionKind::ToSigned)),
             }?;
-            let stored =
-                Address::try_from(addr).and_then(|addr| state.checked_store(addr, replace));
+            let stored = Address::try_from(addr)
+                .and_then(|addr| state.checked_store(addr, replace, mem_flags));
             assign_or_memtrap(stored.map(|_| prev_val_to_assign))
         }
         Opcode::AtomicCas => {
             let addr = arg(0)?.into_int()? as u64;
-            let loaded = Address::try_from(addr).and_then(|addr| state.checked_load(addr, ctrl_ty));
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
+            let loaded = Address::try_from(addr)
+                .and_then(|addr| state.checked_load(addr, ctrl_ty, mem_flags));
             let loaded_val = match loaded {
                 Ok(v) => v,
                 Err(e) => return Ok(ControlFlow::Trap(CraneliftTrap::User(memerror_to_trap(e)))),
@@ -1280,7 +1290,7 @@ where
             let val_to_assign = if Value::eq(&loaded_val, &expected_val)? {
                 let val_to_store = arg(2)?;
                 Address::try_from(addr)
-                    .and_then(|addr| state.checked_store(addr, val_to_store))
+                    .and_then(|addr| state.checked_store(addr, val_to_store, mem_flags))
                     .map(|_| loaded_val)
             } else {
                 Ok(loaded_val)
@@ -1290,17 +1300,20 @@ where
         Opcode::AtomicLoad => {
             let load_ty = inst_context.controlling_type().unwrap();
             let addr = arg(0)?.into_int()? as u64;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             // We are doing a regular load here, this isn't actually thread safe.
             assign_or_memtrap(
-                Address::try_from(addr).and_then(|addr| state.checked_load(addr, load_ty)),
+                Address::try_from(addr)
+                    .and_then(|addr| state.checked_load(addr, load_ty, mem_flags)),
             )
         }
         Opcode::AtomicStore => {
             let val = arg(0)?;
             let addr = arg(1)?.into_int()? as u64;
+            let mem_flags = inst.memflags().expect("instruction to have memory flags");
             // We are doing a regular store here, this isn't actually thread safe.
             continue_or_memtrap(
-                Address::try_from(addr).and_then(|addr| state.checked_store(addr, val)),
+                Address::try_from(addr).and_then(|addr| state.checked_store(addr, val, mem_flags)),
             )
         }
         Opcode::Fence => {
diff --git a/docs/WASI-api.md b/docs/WASI-api.md
index eb38ff900..ba8a9eff6 100644
--- a/docs/WASI-api.md
+++ b/docs/WASI-api.md
@@ -4,4 +4,4 @@ Development of the WASI API has now moved to the WASI CG Subgroup; see
 the [github repository], including the [API documentation].
 
 [github repository]: https://github.com/WebAssembly/WASI/
-[API documentation]: https://github.com/WebAssembly/WASI/blob/master/phases/snapshot/docs.md
+[API documentation]: https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md
diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml
index 11b6ff3d0..f9774cac3 100644
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -16,6 +16,7 @@ cranelift-wasm = { workspace = true }
 cranelift-filetests = { workspace = true }
 cranelift-interpreter = { workspace = true }
 cranelift-fuzzgen = { workspace = true }
+cranelift-native = { workspace = true }
 libfuzzer-sys = { version = "0.4.0", features = ["arbitrary-derive"] }
 target-lexicon = { workspace = true }
 smallvec = { workspace = true }
diff --git a/fuzz/fuzz_targets/cranelift-fuzzgen.rs b/fuzz/fuzz_targets/cranelift-fuzzgen.rs
index 171a8248e..c6a008bd7 100644
--- a/fuzz/fuzz_targets/cranelift-fuzzgen.rs
+++ b/fuzz/fuzz_targets/cranelift-fuzzgen.rs
@@ -1,13 +1,22 @@
 #![no_main]
 
+use cranelift_codegen::ir::Function;
+use cranelift_codegen::ir::Signature;
+use cranelift_codegen::ir::UserExternalName;
+use cranelift_codegen::ir::UserFuncName;
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::arbitrary::Arbitrary;
+use libfuzzer_sys::arbitrary::Unstructured;
 use libfuzzer_sys::fuzz_target;
 use once_cell::sync::Lazy;
 use std::collections::HashMap;
+use std::fmt;
 use std::sync::atomic::AtomicU64;
 use std::sync::atomic::Ordering;
 
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::{LibCall, TrapCode};
+use cranelift_codegen::isa;
 use cranelift_filetests::function_runner::{TestFileCompiler, Trampoline};
 use cranelift_fuzzgen::*;
 use cranelift_interpreter::environment::FuncIndex;
@@ -17,6 +26,7 @@ use cranelift_interpreter::interpreter::{
 };
 use cranelift_interpreter::step::ControlFlow;
 use cranelift_interpreter::step::CraneliftTrap;
+use cranelift_native::builder_with_options;
 use smallvec::smallvec;
 
 const INTERPRETER_FUEL: u64 = 4096;
@@ -26,6 +36,8 @@ struct Statistics {
     /// Inputs that fuzzgen can build a function with
     /// This is also how many compiles we executed
     pub valid_inputs: AtomicU64,
+    /// How many times did we generate an invalid format?
+    pub invalid_inputs: AtomicU64,
 
     /// Total amount of runs that we tried in the interpreter
     /// One fuzzer input can have many runs
@@ -43,11 +55,17 @@ impl Statistics {
     pub fn print(&self, valid_inputs: u64) {
         // We get valid_inputs as a param since we already loaded it previously.
         let total_runs = self.total_runs.load(Ordering::SeqCst);
+        let invalid_inputs = self.invalid_inputs.load(Ordering::SeqCst);
         let run_result_success = self.run_result_success.load(Ordering::SeqCst);
         let run_result_timeout = self.run_result_timeout.load(Ordering::SeqCst);
 
         println!("== FuzzGen Statistics  ====================");
         println!("Valid Inputs: {}", valid_inputs);
+        println!(
+            "Invalid Inputs: {} ({:.1}% of Total Inputs)",
+            invalid_inputs,
+            (invalid_inputs as f64 / (valid_inputs + invalid_inputs) as f64) * 100.0
+        );
         println!("Total Runs: {}", total_runs);
         println!(
             "Successful Runs: {} ({:.1}% of Total Runs)",
@@ -94,6 +112,7 @@ impl Default for Statistics {
 
         Self {
             valid_inputs: AtomicU64::new(0),
+            invalid_inputs: AtomicU64::new(0),
             total_runs: AtomicU64::new(0),
             run_result_success: AtomicU64::new(0),
             run_result_timeout: AtomicU64::new(0),
@@ -120,6 +139,90 @@ impl PartialEq for RunResult {
     }
 }
 
+pub struct TestCase {
+    /// TargetIsa to use when compiling this test case
+    pub isa: isa::OwnedTargetIsa,
+    /// Functions under test
+    /// By convention the first function is the main function.
+    pub functions: Vec<Function>,
+    /// Generate multiple test inputs for each test case.
+    /// This allows us to get more coverage per compilation, which may be somewhat expensive.
+    pub inputs: Vec<TestCaseInput>,
+}
+
+impl fmt::Debug for TestCase {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        PrintableTestCase::run(&self.isa, &self.functions, &self.inputs).fmt(f)
+    }
+}
+
+impl<'a> Arbitrary<'a> for TestCase {
+    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
+        Self::generate(u).map_err(|_| {
+            STATISTICS.invalid_inputs.fetch_add(1, Ordering::SeqCst);
+            arbitrary::Error::IncorrectFormat
+        })
+    }
+}
+
+impl TestCase {
+    pub fn generate(u: &mut Unstructured) -> anyhow::Result<Self> {
+        let mut gen = FuzzGen::new(u);
+
+        // TestCase is meant to be consumed by a runner, so we make the assumption here that we're
+        // generating a TargetIsa for the host.
+        let builder =
+            builder_with_options(true).expect("Unable to build a TargetIsa for the current host");
+        let flags = gen.generate_flags(builder.triple().architecture)?;
+        let isa = builder.finish(flags)?;
+
+        // When generating functions, we allow each function to call any function that has
+        // already been generated. This guarantees that we never have loops in the call graph.
+        // We generate these backwards, and then reverse them so that the main function is at
+        // the start.
+        let func_count = gen.u.int_in_range(gen.config.testcase_funcs.clone())?;
+        let mut functions: Vec<Function> = Vec::with_capacity(func_count);
+        for i in (0..func_count).rev() {
+            // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
+            let fname = UserFuncName::user(1, i as u32);
+
+            let usercalls: Vec<(UserExternalName, Signature)> = functions
+                .iter()
+                .map(|f| {
+                    (
+                        f.name.get_user().unwrap().clone(),
+                        f.stencil.signature.clone(),
+                    )
+                })
+                .collect();
+
+            let func = gen.generate_func(
+                fname,
+                isa.triple().clone(),
+                usercalls,
+                ALLOWED_LIBCALLS.to_vec(),
+            )?;
+            functions.push(func);
+        }
+        // Now reverse the functions so that the main function is at the start.
+        functions.reverse();
+
+        let main = &functions[0];
+        let inputs = gen.generate_test_inputs(&main.signature)?;
+
+        Ok(TestCase {
+            isa,
+            functions,
+            inputs,
+        })
+    }
+
+    /// Returns the main function of this test case.
+    pub fn main(&self) -> &Function {
+        &self.functions[0]
+    }
+}
+
 fn run_in_interpreter(interpreter: &mut Interpreter, args: &[DataValue]) -> RunResult {
     // The entrypoint function is always 0
     let index = FuncIndex::from_u32(0);
@@ -139,6 +242,16 @@ fn run_in_host(trampoline: &Trampoline, args: &[DataValue]) -> RunResult {
     RunResult::Success(res)
 }
 
+/// These libcalls need a interpreter implementation in `build_interpreter`
+const ALLOWED_LIBCALLS: &'static [LibCall] = &[
+    LibCall::CeilF32,
+    LibCall::CeilF64,
+    LibCall::FloorF32,
+    LibCall::FloorF64,
+    LibCall::TruncF32,
+    LibCall::TruncF64,
+];
+
 fn build_interpreter(testcase: &TestCase) -> Interpreter {
     let mut env = FunctionStore::default();
     for func in testcase.functions.iter() {
diff --git a/fuzz/fuzz_targets/cranelift-icache.rs b/fuzz/fuzz_targets/cranelift-icache.rs
index 6023d0041..4c0ca5455 100644
--- a/fuzz/fuzz_targets/cranelift-icache.rs
+++ b/fuzz/fuzz_targets/cranelift-icache.rs
@@ -3,13 +3,103 @@
 use cranelift_codegen::{
     cursor::{Cursor, FuncCursor},
     incremental_cache as icache,
-    ir::{self, immediates::Imm64, ExternalName},
-    Context,
+    ir::{
+        self, immediates::Imm64, ExternalName, Function, LibCall, Signature, UserExternalName,
+        UserFuncName,
+    },
+    isa, Context,
 };
-use libfuzzer_sys::fuzz_target;
+use libfuzzer_sys::{
+    arbitrary::{self, Arbitrary, Unstructured},
+    fuzz_target,
+};
+use std::fmt;
 
 use cranelift_fuzzgen::*;
 
+/// TODO: This *almost* could be replaced with `LibCall::all()`, but
+/// `LibCall::signature` panics for some libcalls, so we need to avoid that.
+const ALLOWED_LIBCALLS: &'static [LibCall] = &[
+    LibCall::CeilF32,
+    LibCall::CeilF64,
+    LibCall::FloorF32,
+    LibCall::FloorF64,
+    LibCall::TruncF32,
+    LibCall::TruncF64,
+    LibCall::NearestF32,
+    LibCall::NearestF64,
+    LibCall::FmaF32,
+    LibCall::FmaF64,
+];
+
+/// A generated function with an ISA that targets one of cranelift's backends.
+pub struct FunctionWithIsa {
+    /// TargetIsa to use when compiling this test case
+    pub isa: isa::OwnedTargetIsa,
+
+    /// Function under test
+    pub func: Function,
+}
+
+impl FunctionWithIsa {
+    pub fn generate(u: &mut Unstructured) -> anyhow::Result<Self> {
+        // We filter out targets that aren't supported in the current build
+        // configuration after randomly choosing one, instead of randomly choosing
+        // a supported one, so that the same fuzz input works across different build
+        // configurations.
+        let target = u.choose(isa::ALL_ARCHITECTURES)?;
+        let builder = isa::lookup_by_name(target).map_err(|_| arbitrary::Error::IncorrectFormat)?;
+        let architecture = builder.triple().architecture;
+
+        let mut gen = FuzzGen::new(u);
+        let flags = gen
+            .generate_flags(architecture)
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+        let isa = builder
+            .finish(flags)
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+
+        // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
+        let fname = UserFuncName::user(1, 0);
+
+        // We don't actually generate these functions, we just simulate their signatures and names
+        let func_count = gen.u.int_in_range(gen.config.testcase_funcs.clone())?;
+        let usercalls = (0..func_count)
+            .map(|i| {
+                let name = UserExternalName::new(2, i as u32);
+                let sig = gen.generate_signature(architecture)?;
+                Ok((name, sig))
+            })
+            .collect::<anyhow::Result<Vec<(UserExternalName, Signature)>>>()
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+
+        let func = gen
+            .generate_func(
+                fname,
+                isa.triple().clone(),
+                usercalls,
+                ALLOWED_LIBCALLS.to_vec(),
+            )
+            .map_err(|_| arbitrary::Error::IncorrectFormat)?;
+
+        Ok(FunctionWithIsa { isa, func })
+    }
+}
+
+impl fmt::Debug for FunctionWithIsa {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // TODO: We could avoid the clone here.
+        let funcs = &[self.func.clone()];
+        PrintableTestCase::compile(&self.isa, funcs).fmt(f)
+    }
+}
+
+impl<'a> Arbitrary<'a> for FunctionWithIsa {
+    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
+        Self::generate(u).map_err(|_| arbitrary::Error::IncorrectFormat)
+    }
+}
+
 fuzz_target!(|func: FunctionWithIsa| {
     let FunctionWithIsa { mut func, isa } = func;
 


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/algebraic.clif b/cranelift/filetests/filetests/egraph/algebraic.clif
index 24460845a..573135d3b 100644
--- a/cranelift/filetests/filetests/egraph/algebraic.clif
+++ b/cranelift/filetests/filetests/egraph/algebraic.clif
@@ -483,3 +483,25 @@ block0(v1: i16):
 
 ; check: v4 = sextend.i64 v1
 ; check: return v4
+
+function %fma_double_fneg(f32, f32, f32) -> f32 {
+block0(v1: f32, v2: f32, v3: f32):
+    v4 = fneg v1
+    v5 = fneg v2
+    v6 = fma v4, v5, v3
+    return v6
+}
+
+; check: v7 = fma v1, v2, v3
+; check: return v7
+
+function %fmul_double_fneg(f32, f32) -> f32 {
+block0(v1: f32, v2: f32):
+    v3 = fneg v1
+    v4 = fneg v2
+    v5 = fmul v3, v4
+    return v5
+}
+
+; check: v6 = fmul v1, v2
+; check: return v6
diff --git a/cranelift/filetests/filetests/egraph/issue-5796.clif b/cranelift/filetests/filetests/egraph/issue-5796.clif
new file mode 100644
index 000000000..66963f5c4
--- /dev/null
+++ b/cranelift/filetests/filetests/egraph/issue-5796.clif
@@ -0,0 +1,76 @@
+test optimize
+set opt_level=speed
+target x86_64
+
+function %f0(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  brif v0, block1, block2
+
+block1:
+  v3 = udiv v1, v2
+  return v3
+
+block2:
+  v4 = udiv v1, v2
+  brif v1, block3, block4
+
+block3:
+  return v4
+
+block4:
+  return v4
+}
+
+; check: block0(v0: i32, v1: i32, v2: i32):
+; check:     brif v0, block1, block2
+
+; check: block1:
+; check:     v3 = udiv.i32 v1, v2
+; check:     return v3
+
+; check: block2:
+; check:     v4 = udiv.i32 v1, v2
+; check:     brif.i32 v1, block3, block4
+
+; check: block3:
+; check:     return v4
+
+; check: block4:
+; check:     return v4
+
+function %f1(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  brif v0, block1, block2
+
+block1:
+  v3 = udiv v1, v2
+  return v3
+
+block2:
+  v4 = udiv v1, v2
+  brif v1, block3, block4
+
+block3:
+  v5 = udiv v1, v2
+  return v5
+
+block4:
+  return v4
+}
+
+; check: block0(v0: i32, v1: i32, v2: i32):
+; check:     brif v0, block1, block2
+
+; check: block1:
+; check:     v3 = udiv.i32 v1, v2
+; check:     return v3
+
+; check: block2:
+; check:     v4 = udiv.i32 v1, v2
+; check:     brif.i32 v1, block3, block4
+
+; check: block3:
+; check:     return v4
+
+; check: block4:
+; check:     return v4
diff --git a/cranelift/filetests/filetests/isa/aarch64/fma.clif b/cranelift/filetests/filetests/isa/aarch64/fma.clif
new file mode 100644
index 000000000..e2f4a172c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/fma.clif
@@ -0,0 +1,159 @@
+test compile precise-output
+target aarch64
+
+function %fma_f32(f32, f32, f32) -> f32 {
+block0(v0: f32, v1: f32, v2: f32):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   fmadd s0, s0, s1, s2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   fmadd s0, s0, s1, s2
+;   ret
+
+function %fma_f64(f64, f64, f64) -> f64 {
+block0(v0: f64, v1: f64, v2: f64):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   fmadd d0, d0, d1, d2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   fmadd d0, d0, d1, d2
+;   ret
+
+function %fma_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.2d, v5.2d, v1.2d
+;   ret
+
+function %fma_neg_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_neg_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v5.2d, v1.2d
+;   ret
+
+function %fma_neg_other_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fneg v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_neg_other_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fneg v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v5.2d, v1.2d
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
index 3e622bd5f..fdd58032e 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
@@ -1635,8 +1635,7 @@ block0(v0: i8x16, v1: i32):
 ;   vpsrlw  %xmm0, %xmm5, %xmm7
 ;   lea     const(0), %rsi
 ;   shlq    $4, %r10, %r10
-;   movdqu  0(%rsi,%r10,1), %xmm13
-;   vpand   %xmm7, %xmm13, %xmm0
+;   vpand   %xmm7, 0(%rsi,%r10,1), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1652,11 +1651,13 @@ block0(v0: i8x16, v1: i32):
 ;   vpsrlw %xmm5, %xmm0, %xmm7
 ;   leaq 0x15(%rip), %rsi
 ;   shlq $4, %r10
-;   movdqu (%rsi, %r10), %xmm13
-;   vpand %xmm13, %xmm7, %xmm0
+;   vpand (%rsi, %r10), %xmm7, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %bh, %bh
 
 function %i8x16_ushr_imm(i8x16) -> i8x16 {
 block0(v0: i8x16):
@@ -1670,8 +1671,7 @@ block0(v0: i8x16):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   vpsrlw  %xmm0, $1, %xmm2
-;   movdqu  const(0), %xmm4
-;   vpand   %xmm2, %xmm4, %xmm0
+;   vpand   %xmm2, const(0), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1682,14 +1682,15 @@ block0(v0: i8x16):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   vpsrlw $1, %xmm0, %xmm2
-;   movdqu 0xf(%rip), %xmm4
-;   vpand %xmm4, %xmm2, %xmm0
+;   vpand 0xf(%rip), %xmm2, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
 ;   jg 0xa1
 ;   jg 0xa3
 ;   jg 0xa5
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
index 1ecdf31ed..f63cc2231 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
@@ -485,8 +485,7 @@ block0:
 ; block0:
 ;   movdqu  const(1), %xmm0
 ;   psrlw   %xmm0, $1, %xmm0
-;   movdqu  const(0), %xmm3
-;   pand    %xmm0, %xmm3, %xmm0
+;   pand    %xmm0, const(0), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -496,28 +495,21 @@ block0:
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movdqu 0x34(%rip), %xmm0
+;   movdqu 0x24(%rip), %xmm0
 ;   psrlw $1, %xmm0
-;   movdqu 0x17(%rip), %xmm3
-;   pand %xmm3, %xmm0
+;   pand 7(%rip), %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
 ;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   jg 0xb1
-;   jg 0xb3
-;   jg 0xb5
-;   jg 0xb7
-;   jg 0xb9
-;   jg 0xbb
-;   jg 0xbd
-;   jg 0xbf
+;   jg 0xa1
+;   jg 0xa3
+;   jg 0xa5
+;   jg 0xa7
+;   jg 0xa9
+;   jg 0xab
+;   jg 0xad
+;   jg 0xaf
 ;   addb %al, (%rcx)
 ;   addb (%rbx), %al
 ;   addb $5, %al
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif b/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
index 4fb63be0f..9be9323dd 100644
--- a/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-cas-subword-big.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 
diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif b/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
index ae5871737..f7565d63d 100644
--- a/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-cas-subword-little.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 target aarch64
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
index dacbcb708..f0f52c12a 100644
--- a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-big.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 target s390x has_mie2
@@ -431,10 +432,10 @@ block0(v0: i32, v1: i64, v2: i16):
     v6 = load.i32 big v3
     return v6
 }
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0x1111) == 0x11115678
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 0, 0xffff) == 0xffff5678
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0x1111) == 0x12341111
-; run: %atomic_rmw_xchg_little_i16(0x12345678, 2, 0xffff) == 0x1234ffff
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 0, 0x1111) == 0x11115678
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 0, 0xffff) == 0xffff5678
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 2, 0x1111) == 0x12341111
+; run: %atomic_rmw_xchg_big_i16(0x12345678, 2, 0xffff) == 0x1234ffff
 
 
 function %atomic_rmw_xchg_big_i8(i32, i64, i8) -> i32 {
diff --git a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
index a6d16c7a6..2cb9beba7 100644
--- a/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-rmw-subword-little.clif
@@ -1,3 +1,4 @@
+test interpret
 test run
 target s390x
 target s390x has_mie2
diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
new file mode 100644
index 000000000..b6d4e88dc
--- /dev/null
+++ b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
@@ -0,0 +1,92 @@
+;;! target = "x86_64"
+;;!
+;;! optimize = true
+;;!
+;;! settings = [
+;;!   "enable_heap_access_spectre_mitigation=true",
+;;!   "opt_level=speed_and_size",
+;;!   "use_egraphs=true"
+;;! ]
+;;!
+;;! [globals.vmctx]
+;;! type = "i64"
+;;! vmctx = true
+;;!
+;;! [globals.heap_base]
+;;! type = "i64"
+;;! load = { base = "vmctx", offset = 0 }
+;;!
+;;! [globals.heap_bound]
+;;! type = "i64"
+;;! load = { base = "vmctx", offset = 8 }
+;;!
+;;! [[heaps]]
+;;! base = "heap_base"
+;;! min_size = 0
+;;! offset_guard_size = 0xffffffff
+;;! index_type = "i32"
+;;! style = { kind = "dynamic", bound = "heap_bound" }
+
+(module
+  (memory (export "memory") 0)
+  (func (export "load-without-offset") (param i32) (result i32 i32)
+    local.get 0
+    i32.load
+    local.get 0
+    i32.load
+  )
+  (func (export "load-with-offset") (param i32) (result i32 i32)
+    local.get 0
+    i32.load offset=1234
+    local.get 0
+    i32.load offset=1234
+  )
+)
+
+;; function u0:0(i32, i64 vmctx) -> i32, i32 fast {
+;;     gv0 = vmctx
+;;     gv1 = load.i64 notrap aligned gv0+8
+;;     gv2 = load.i64 notrap aligned gv0
+;;
+;;                                 block0(v0: i32, v1: i64):
+;; @0057                               v4 = uextend.i64 v0
+;; @0057                               v5 = iconst.i64 4
+;; @0057                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 4
+;; @0057                               v7 = load.i64 notrap aligned v1+8
+;; @0057                               v8 = load.i64 notrap aligned v1
+;; @0057                               v11 = icmp ugt v6, v7
+;; @0057                               v10 = iconst.i64 0
+;; @0057                               v9 = iadd v8, v4
+;; @0057                               v12 = select_spectre_guard v11, v10, v9  ; v10 = 0
+;; @0057                               v13 = load.i32 little heap v12
+;;                                     v2 -> v13
+;; @005f                               jump block1
+;;
+;;                                 block1:
+;; @005f                               return v13, v13
+;; }
+;;
+;; function u0:1(i32, i64 vmctx) -> i32, i32 fast {
+;;     gv0 = vmctx
+;;     gv1 = load.i64 notrap aligned gv0+8
+;;     gv2 = load.i64 notrap aligned gv0
+;;
+;;                                 block0(v0: i32, v1: i64):
+;; @0064                               v4 = uextend.i64 v0
+;; @0064                               v5 = iconst.i64 1238
+;; @0064                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 1238
+;; @0064                               v7 = load.i64 notrap aligned v1+8
+;; @0064                               v8 = load.i64 notrap aligned v1
+;; @0064                               v12 = icmp ugt v6, v7
+;; @0064                               v11 = iconst.i64 0
+;; @0064                               v9 = iadd v8, v4
+;;                                     v26 = iconst.i64 1234
+;; @0064                               v10 = iadd v9, v26  ; v26 = 1234
+;; @0064                               v13 = select_spectre_guard v12, v11, v10  ; v11 = 0
+;; @0064                               v14 = load.i32 little heap v13
+;;                                     v2 -> v14
+;; @006e                               jump block1
+;;
+;;                                 block1:
+;; @006e                               return v14, v14
+;; }
diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs
index 7086ebea6..6a1baa47e 100644
--- a/tests/all/cli_tests.rs
+++ b/tests/all/cli_tests.rs
@@ -7,7 +7,7 @@ use tempfile::{NamedTempFile, TempDir};
 
 // Run the wasmtime CLI with the provided args and return the `Output`.
 // If the `stdin` is `Some`, opens the file and redirects to the child's stdin.
-fn run_wasmtime_for_output(args: &[&str], stdin: Option<&Path>) -> Result<Output> {
+pub fn run_wasmtime_for_output(args: &[&str], stdin: Option<&Path>) -> Result<Output> {
     let runner = std::env::vars()
         .filter(|(k, _v)| k.starts_with("CARGO_TARGET") && k.ends_with("RUNNER"))
         .next();
diff --git a/tests/all/main.rs b/tests/all/main.rs
index 9b613ffb1..80455d383 100644
--- a/tests/all/main.rs
+++ b/tests/all/main.rs
@@ -32,6 +32,7 @@ mod table;
 mod threads;
 mod traps;
 mod wait_notify;
+mod wasi_testsuite;
 mod wast;
 
 /// A helper to compile a module in a new store with reference types enabled.
diff --git a/tests/all/wasi_testsuite.rs b/tests/all/wasi_testsuite.rs
new file mode 100644
index 000000000..81d2bf434
--- /dev/null
+++ b/tests/all/wasi_testsuite.rs
@@ -0,0 +1,79 @@
+//! Run the tests in `wasi_testsuite` using Wasmtime's CLI binary and checking
+//! the results with a [wasi-testsuite] spec.
+//!
+//! [wasi-testsuite]: https://github.com/WebAssembly/wasi-testsuite
+
+use crate::cli_tests::run_wasmtime_for_output;
+use anyhow::Result;
+use serde::Deserialize;
+use std::ffi::OsStr;
+use std::fs::{self, DirEntry};
+use std::path::{Path, PathBuf};
+use std::process::Output;
+
+#[test]
+#[cfg_attr(target_os = "windows", ignore)] // TODO: https://github.com/WebAssembly/WASI/issues/524
+fn wasi_threads_testsuite() -> Result<()> {
+    for module in list_modules("tests/wasi_testsuite/wasi-threads/test/testsuite")? {
+        println!("Testing {}", module.display());
+        let result = run(&module)?;
+        let spec = parse_spec(&module.with_extension("json"))?;
+        assert_eq!(spec, result);
+    }
+    Ok(())
+}
+
+fn list_modules(testsuite_dir: &str) -> Result<impl Iterator<Item = PathBuf>> {
+    Ok(fs::read_dir(testsuite_dir)?
+        .filter_map(Result::ok)
+        .filter(is_wasm)
+        .map(|e| e.path()))
+}
+
+fn is_wasm(entry: &DirEntry) -> bool {
+    let path = entry.path();
+    let ext = path.extension().map(OsStr::to_str).flatten();
+    path.is_file() && (ext == Some("wat") || ext == Some("wasm"))
+}
+
+fn run<P: AsRef<Path>>(module: P) -> Result<Output> {
+    run_wasmtime_for_output(
+        &[
+            "run",
+            "--wasi-modules",
+            "experimental-wasi-threads",
+            "--wasm-features",
+            "threads",
+            "--disable-cache",
+            module.as_ref().to_str().unwrap(),
+        ],
+        None,
+    )
+}
+
+fn parse_spec<P: AsRef<Path>>(spec_file: P) -> Result<Spec> {
+    let contents = fs::read_to_string(spec_file)?;
+    let spec = serde_json::from_str(&contents)?;
+    Ok(spec)
+}
+
+#[derive(Debug, Deserialize)]
+struct Spec {
+    exit_code: i32,
+    stdout: Option<String>,
+    stderr: Option<String>,
+}
+
+impl PartialEq<Output> for Spec {
+    fn eq(&self, other: &Output) -> bool {
+        self.exit_code == other.status.code().unwrap()
+            && matches_or_missing(&self.stdout, &other.stdout)
+            && matches_or_missing(&self.stderr, &other.stderr)
+    }
+}
+
+fn matches_or_missing(a: &Option<String>, b: &[u8]) -> bool {
+    a.as_ref()
+        .map(|s| s == &String::from_utf8_lossy(b))
+        .unwrap_or(true)
+}
diff --git a/tests/wasi_testsuite/wasi-threads b/tests/wasi_testsuite/wasi-threads
new file mode 160000
index 000000000..e1893c004
--- /dev/null
+++ b/tests/wasi_testsuite/wasi-threads
@@ -0,0 +1,1 @@
+Subproject commit e1893c00458de3e846f02c9cd2a435992ea8b32c