Repository: bytecodealliance/wasmtime
Pull request #5895 (2023-02-28 22:49:44)

aarch64: Add support for the `fmls` instruction
This commit adds lowerings to the AArch64 backend for the `fmls` instruction which is intended to be leveraged in the relaxed-simd proposal for WebAssembly. This should hopefully allow for a teeny-bit-more efficient codegen for this operator instead of using the `fmla` instruction plus a negation instruction.

<!--

Please ensure that the following steps are all taken care of before submitting
the PR.

- [ ] This has been discussed in issue #..., or if not, please tell us why
  here.
- [ ] A short description of what this does, why it is needed; if the
  description becomes long, the matter should probably be discussed in an issue
  first.
- [ ] This PR contains test cases, if meaningful.
- [ ] A reviewer from the core maintainer team has been assigned for this PR.
  If you don't know who could review this, please indicate so. The list of
  suggested reviewers on the right can help you.

Please ensure all communication adheres to the [code of
conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
-->



Patch:
diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index 01ccefb7e..488296999 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -66,7 +66,7 @@ fn gen_formats(formats: &[&InstructionFormat], fmt: &mut Formatter) {
 /// 16 bytes on 64-bit architectures. If more space is needed to represent an instruction, use a
 /// `ValueList` to store the additional information out of line.
 fn gen_instruction_data(formats: &[&InstructionFormat], fmt: &mut Formatter) {
-    fmt.line("#[derive(Copy, Clone, Debug, PartialEq, Hash)]");
+    fmt.line("#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]");
     fmt.line(r#"#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]"#);
     fmt.line("#[allow(missing_docs)]");
     fmtln!(fmt, "pub enum InstructionData {");
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs
index a54a9818c..41a289f1a 100755
--- a/cranelift/codegen/meta/src/shared/instructions.rs
+++ b/cranelift/codegen/meta/src/shared/instructions.rs
@@ -367,6 +367,7 @@ fn define_simd_lane_access(
     );
     let x = &Operand::new("x", I8x16).with_doc("Vector to modify by re-arranging lanes");
     let y = &Operand::new("y", I8x16).with_doc("Mask for re-arranging lanes");
+    let a = &Operand::new("a", I8x16);
 
     ig.push(
         Inst::new(
@@ -388,6 +389,7 @@ fn define_simd_lane_access(
     let x = &Operand::new("x", TxN).with_doc("The vector to modify");
     let y = &Operand::new("y", &TxN.lane_of()).with_doc("New lane value");
     let Idx = &Operand::new("Idx", &imm.uimm8).with_doc("Lane index");
+    let a = &Operand::new("a", TxN);
 
     ig.push(
         Inst::new(
diff --git a/cranelift/codegen/src/egraph.rs b/cranelift/codegen/src/egraph.rs
index 2cbe412e3..e12b6fed3 100644
--- a/cranelift/codegen/src/egraph.rs
+++ b/cranelift/codegen/src/egraph.rs
@@ -7,13 +7,14 @@ use crate::dominator_tree::DominatorTree;
 use crate::egraph::domtree::DomTreeWithChildren;
 use crate::egraph::elaborate::Elaborator;
 use crate::fx::FxHashSet;
-use crate::inst_predicates::is_pure_for_egraph;
+use crate::inst_predicates::{is_mergeable_for_egraph, is_pure_for_egraph};
 use crate::ir::{
-    DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
+    Block, DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
 };
 use crate::loop_analysis::LoopAnalysis;
 use crate::opts::generated_code::ContextIter;
 use crate::opts::IsleContext;
+use crate::scoped_hash_map::{Entry as ScopedEntry, ScopedHashMap};
 use crate::trace;
 use crate::unionfind::UnionFind;
 use cranelift_entity::packed_option::ReservedValue;
@@ -77,6 +78,7 @@ where
     pub(crate) func: &'opt mut Function,
     pub(crate) value_to_opt_value: &'opt mut SecondaryMap<Value, Value>,
     pub(crate) gvn_map: &'opt mut CtxHashMap<(Type, InstructionData), Value>,
+    pub(crate) effectful_gvn_map: &'opt mut ScopedHashMap<(Type, InstructionData), Value>,
     pub(crate) eclasses: &'opt mut UnionFind<Value>,
     pub(crate) remat_values: &'opt mut FxHashSet<Value>,
     pub(crate) stats: &'opt mut Stats,
@@ -285,10 +287,49 @@ where
     fn optimize_skeleton_inst(&mut self, inst: Inst) -> bool {
         self.stats.skeleton_inst += 1;
 
-        // If a load or store, process it with the alias analysis to see
-        // if we can optimize it (rewrite in terms of an earlier load or
-        // stored value).
-        if let Some(new_result) =
+        // First, can we try to deduplicate? We need to keep some copy
+        // of the instruction around because it's side-effecting, but
+        // we may be able to reuse an earlier instance of it.
+        if is_mergeable_for_egraph(self.func, inst) {
+            let result = self.func.dfg.inst_results(inst)[0];
+            trace!(" -> mergeable side-effecting op {}", inst);
+
+            // Does this instruction already exist? If so, add entries to
+            // the value-map to rewrite uses of its results to the results
+            // of the original (existing) instruction. If not, optimize
+            // the new instruction.
+            //
+            // Note that we use the "effectful GVN map", which is
+            // scoped: because effectful ops are not removed from the
+            // skeleton (`Layout`), we need to be mindful of whether
+            // our current position is dominated by an instance of the
+            // instruction. (See #5796 for details.)
+            let ty = self.func.dfg.ctrl_typevar(inst);
+            match self
+                .effectful_gvn_map
+                .entry((ty, self.func.dfg.insts[inst].clone()))
+            {
+                ScopedEntry::Occupied(o) => {
+                    let orig_result = *o.get();
+                    // Hit in GVN map -- reuse value.
+                    self.value_to_opt_value[result] = orig_result;
+                    self.eclasses.union(orig_result, result);
+                    trace!(" -> merges result {} to {}", result, orig_result);
+                    true
+                }
+                ScopedEntry::Vacant(v) => {
+                    // Otherwise, insert it into the value-map.
+                    self.value_to_opt_value[result] = result;
+                    v.insert(result);
+                    trace!(" -> inserts as new (no GVN)");
+                    false
+                }
+            }
+        }
+        // Otherwise, if a load or store, process it with the alias
+        // analysis to see if we can optimize it (rewrite in terms of
+        // an earlier load or stored value).
+        else if let Some(new_result) =
             self.alias_analysis
                 .process_inst(self.func, self.alias_analysis_state, inst)
         {
@@ -382,83 +423,127 @@ impl<'a> EgraphPass<'a> {
         let mut cursor = FuncCursor::new(self.func);
         let mut value_to_opt_value: SecondaryMap<Value, Value> =
             SecondaryMap::with_default(Value::reserved_value());
+        // Map from instruction to value for hash-consing of pure ops
+        // into the egraph. This can be a standard (non-scoped)
+        // hashmap because pure ops have no location: they are
+        // "outside of" control flow.
+        //
+        // Note also that we keep the controlling typevar (the `Type`
+        // in the tuple below) because it may disambiguate
+        // instructions that are identical except for type.
         let mut gvn_map: CtxHashMap<(Type, InstructionData), Value> =
             CtxHashMap::with_capacity(cursor.func.dfg.num_values());
+        // Map from instruction to value for GVN'ing of effectful but
+        // idempotent ops, which remain in the side-effecting
+        // skeleton. This needs to be scoped because we cannot
+        // deduplicate one instruction to another that is in a
+        // non-dominating block.
+        //
+        // Note that we can use a ScopedHashMap here without the
+        // "context" (as needed by CtxHashMap) because in practice the
+        // ops we want to GVN have all their args inline. Equality on
+        // the InstructionData itself is conservative: two insts whose
+        // struct contents compare shallowly equal are definitely
+        // identical, but identical insts in a deep-equality sense may
+        // not compare shallowly equal, due to list indirection. This
+        // is fine for GVN, because it is still sound to skip any
+        // given GVN opportunity (and keep the original instructions).
+        //
+        // As above, we keep the controlling typevar here as part of
+        // the key: effectful instructions may (as for pure
+        // instructions) be differentiated only on the type.
+        let mut effectful_gvn_map: ScopedHashMap<(Type, InstructionData), Value> =
+            ScopedHashMap::new();
 
         // In domtree preorder, visit blocks. (TODO: factor out an
         // iterator from this and elaborator.)
         let root = self.domtree_children.root();
-        let mut block_stack = vec![root];
-        while let Some(block) = block_stack.pop() {
-            // We popped this block; push children
-            // immediately, then process this block.
-            block_stack.extend(self.domtree_children.children(block));
-
-            trace!("Processing block {}", block);
-            cursor.set_position(CursorPosition::Before(block));
-
-            let mut alias_analysis_state = self.alias_analysis.block_starting_state(block);
-
-            for &param in cursor.func.dfg.block_params(block) {
-                trace!("creating initial singleton eclass for blockparam {}", param);
-                self.eclasses.add(param);
-                value_to_opt_value[param] = param;
-            }
-            while let Some(inst) = cursor.next_inst() {
-                trace!("Processing inst {}", inst);
-
-                // While we're passing over all insts, create initial
-                // singleton eclasses for all result and blockparam
-                // values.  Also do initial analysis of all inst
-                // results.
-                for &result in cursor.func.dfg.inst_results(inst) {
-                    trace!("creating initial singleton eclass for {}", result);
-                    self.eclasses.add(result);
-                }
+        enum StackEntry {
+            Visit(Block),
+            Pop,
+        }
+        let mut block_stack = vec![StackEntry::Visit(root)];
+        while let Some(entry) = block_stack.pop() {
+            match entry {
+                StackEntry::Visit(block) => {
+                    // We popped this block; push children
+                    // immediately, then process this block.
+                    block_stack.push(StackEntry::Pop);
+                    block_stack
+                        .extend(self.domtree_children.children(block).map(StackEntry::Visit));
+                    effectful_gvn_map.increment_depth();
+
+                    trace!("Processing block {}", block);
+                    cursor.set_position(CursorPosition::Before(block));
+
+                    let mut alias_analysis_state = self.alias_analysis.block_starting_state(block);
+
+                    for &param in cursor.func.dfg.block_params(block) {
+                        trace!("creating initial singleton eclass for blockparam {}", param);
+                        self.eclasses.add(param);
+                        value_to_opt_value[param] = param;
+                    }
+                    while let Some(inst) = cursor.next_inst() {
+                        trace!("Processing inst {}", inst);
+
+                        // While we're passing over all insts, create initial
+                        // singleton eclasses for all result and blockparam
+                        // values.  Also do initial analysis of all inst
+                        // results.
+                        for &result in cursor.func.dfg.inst_results(inst) {
+                            trace!("creating initial singleton eclass for {}", result);
+                            self.eclasses.add(result);
+                        }
 
-                // Rewrite args of *all* instructions using the
-                // value-to-opt-value map.
-                cursor.func.dfg.resolve_aliases_in_arguments(inst);
-                cursor.func.dfg.map_inst_values(inst, |_, arg| {
-                    let new_value = value_to_opt_value[arg];
-                    trace!("rewriting arg {} of inst {} to {}", arg, inst, new_value);
-                    debug_assert_ne!(new_value, Value::reserved_value());
-                    new_value
-                });
-
-                // Build a context for optimization, with borrows of
-                // state. We can't invoke a method on `self` because
-                // we've borrowed `self.func` mutably (as
-                // `cursor.func`) so we pull apart the pieces instead
-                // here.
-                let mut ctx = OptimizeCtx {
-                    func: cursor.func,
-                    value_to_opt_value: &mut value_to_opt_value,
-                    gvn_map: &mut gvn_map,
-                    eclasses: &mut self.eclasses,
-                    rewrite_depth: 0,
-                    subsume_values: FxHashSet::default(),
-                    remat_values: &mut self.remat_values,
-                    stats: &mut self.stats,
-                    alias_analysis: self.alias_analysis,
-                    alias_analysis_state: &mut alias_analysis_state,
-                };
-
-                if is_pure_for_egraph(ctx.func, inst) {
-                    // Insert into GVN map and optimize any new nodes
-                    // inserted (recursively performing this work for
-                    // any nodes the optimization rules produce).
-                    let inst = NewOrExistingInst::Existing(inst);
-                    ctx.insert_pure_enode(inst);
-                    // We've now rewritten all uses, or will when we
-                    // see them, and the instruction exists as a pure
-                    // enode in the eclass, so we can remove it.
-                    cursor.remove_inst_and_step_back();
-                } else {
-                    if ctx.optimize_skeleton_inst(inst) {
-                        cursor.remove_inst_and_step_back();
+                        // Rewrite args of *all* instructions using the
+                        // value-to-opt-value map.
+                        cursor.func.dfg.resolve_aliases_in_arguments(inst);
+                        cursor.func.dfg.map_inst_values(inst, |_, arg| {
+                            let new_value = value_to_opt_value[arg];
+                            trace!("rewriting arg {} of inst {} to {}", arg, inst, new_value);
+                            debug_assert_ne!(new_value, Value::reserved_value());
+                            new_value
+                        });
+
+                        // Build a context for optimization, with borrows of
+                        // state. We can't invoke a method on `self` because
+                        // we've borrowed `self.func` mutably (as
+                        // `cursor.func`) so we pull apart the pieces instead
+                        // here.
+                        let mut ctx = OptimizeCtx {
+                            func: cursor.func,
+                            value_to_opt_value: &mut value_to_opt_value,
+                            gvn_map: &mut gvn_map,
+                            effectful_gvn_map: &mut effectful_gvn_map,
+                            eclasses: &mut self.eclasses,
+                            rewrite_depth: 0,
+                            subsume_values: FxHashSet::default(),
+                            remat_values: &mut self.remat_values,
+                            stats: &mut self.stats,
+                            alias_analysis: self.alias_analysis,
+                            alias_analysis_state: &mut alias_analysis_state,
+                        };
+
+                        if is_pure_for_egraph(ctx.func, inst) {
+                            // Insert into GVN map and optimize any new nodes
+                            // inserted (recursively performing this work for
+                            // any nodes the optimization rules produce).
+                            let inst = NewOrExistingInst::Existing(inst);
+                            ctx.insert_pure_enode(inst);
+                            // We've now rewritten all uses, or will when we
+                            // see them, and the instruction exists as a pure
+                            // enode in the eclass, so we can remove it.
+                            cursor.remove_inst_and_step_back();
+                        } else {
+                            if ctx.optimize_skeleton_inst(inst) {
+                                cursor.remove_inst_and_step_back();
+                            }
+                        }
                     }
                 }
+                StackEntry::Pop => {
+                    effectful_gvn_map.decrement_depth();
+                }
             }
         }
     }
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index 9b284e8b8..f82f41daa 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -73,6 +73,25 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
     has_one_result && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
 }
 
+/// Can the given instruction be merged into another copy of itself?
+/// These instructions may have side-effects, but as long as we retain
+/// the first instance of the instruction, the second and further
+/// instances are redundant if they would produce the same trap or
+/// result.
+pub fn is_mergeable_for_egraph(func: &Function, inst: Inst) -> bool {
+    let op = func.dfg.insts[inst].opcode();
+    // We can only merge one-result operators due to the way that GVN
+    // is structured in the egraph implementation.
+    let has_one_result = func.dfg.inst_results(inst).len() == 1;
+    has_one_result
+        // Loads/stores are handled by alias analysis and not
+        // otherwise mergeable.
+        && !op.can_load()
+        && !op.can_store()
+        // Can only have idempotent side-effects.
+        && (!has_side_effect(func, inst) || op.side_effects_idempotent())
+}
+
 /// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,
 /// but not the get_pinned_reg opcode?
 pub fn has_lowering_side_effect(func: &Function, inst: Inst) -> bool {
diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs
index 97e0ac0b5..f28cc45d2 100644
--- a/cranelift/codegen/src/ir/instructions.rs
+++ b/cranelift/codegen/src/ir/instructions.rs
@@ -45,7 +45,7 @@ pub type ValueListPool = entity::ListPool<Value>;
 /// The BlockCall::new function guarantees this layout by requiring a block argument that's written
 /// in as the first element of the EntityList. Any subsequent entries are always assumed to be real
 /// Values.
-#[derive(Debug, Clone, Copy, PartialEq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct BlockCall {
     /// The underlying storage for the BlockCall. The first element of the values EntityList is
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index b8bf2ef48..53b0e65ed 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -852,7 +852,7 @@
         (rd WritableReg)
         ;; Offset in range -2^20 .. 2^20.
         (off i32))
-      
+
        ;; Compute the address (using a PC-relative offset) of a 4KB page.
        (Adrp
         (rd WritableReg)
@@ -1401,6 +1401,8 @@
     (Bsl)
     ;; Floating-point fused multiply-add vectors
     (Fmla)
+    ;; Floating-point fused multiply-subtract vectors
+    (Fmls)
 ))
 
 ;; A Vector miscellaneous operation with two registers.
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 90a5dbd93..72b1e1176 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -2906,6 +2906,9 @@ impl MachInstEmit for Inst {
                     VecALUModOp::Fmla => {
                         (0b000_01110_00_1 | (size.enc_float_size() << 1), 0b110011)
                     }
+                    VecALUModOp::Fmls => {
+                        (0b000_01110_10_1 | (size.enc_float_size() << 1), 0b110011)
+                    }
                 };
                 sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
             }
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 182660335..779782907 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -2363,6 +2363,7 @@ impl Inst {
                 let (op, size) = match alu_op {
                     VecALUModOp::Bsl => ("bsl", VectorSize::Size8x16),
                     VecALUModOp::Fmla => ("fmla", size),
+                    VecALUModOp::Fmls => ("fmls", size),
                 };
                 let rd = pretty_print_vreg_vector(rd.to_reg(), size, allocs);
                 let ri = pretty_print_vreg_vector(ri, size, allocs);
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 3eec28c11..bd7e968d7 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -404,7 +404,13 @@
 (rule (lower (has_type ty @ (multi_lane _ _) (fma x y z)))
       (vec_rrr_mod (VecALUModOp.Fmla) z x y (vector_size ty)))
 
-(rule 1 (lower (has_type (ty_scalar_float ty) (fma x y z)))
+(rule 1 (lower (has_type ty @ (multi_lane _ _) (fma (fneg x) y z)))
+      (vec_rrr_mod (VecALUModOp.Fmls) z x y (vector_size ty)))
+
+(rule 2 (lower (has_type ty @ (multi_lane _ _) (fma x (fneg y) z)))
+      (vec_rrr_mod (VecALUModOp.Fmls) z x y (vector_size ty)))
+
+(rule 3 (lower (has_type (ty_scalar_float ty) (fma x y z)))
       (fpu_rrrr (FPUOp3.MAdd) (scalar_size ty) x y z))
 
 ;;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index ca27ac8c5..4cee85ed5 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -1910,21 +1910,20 @@
 (decl lower_cond_br (IntCC ValueRegs VecMachLabel Type) Unit)
 (extern constructor lower_cond_br lower_cond_br)
 
+(decl intcc_to_extend_op (IntCC) ExtendOp)
+(extern constructor intcc_to_extend_op intcc_to_extend_op)
+
 ;; Normalize a value for comparision.
 ;;
 ;; This ensures that types smaller than a register don't accidentally
 ;; pass undefined high bits when being compared as a full register.
-(decl normalize_cmp_value (Type ValueRegs) ValueRegs)
+(decl normalize_cmp_value (Type ValueRegs ExtendOp) ValueRegs)
 
-(rule (normalize_cmp_value $I8 r)
-      (value_reg (alu_rr_imm12 (AluOPRRI.Andi) r (imm12_const 255))))
-(rule (normalize_cmp_value $I16 r)
-      (value_reg (alu_rrr (AluOPRRR.And) r (imm $I16 65535))))
-(rule (normalize_cmp_value $I32 r)
-      (value_reg (alu_rr_imm12 (AluOPRRI.Addiw) r (imm12_const 0))))
+(rule 1 (normalize_cmp_value (fits_in_32 ity) r op)
+      (extend r op ity $I64))
 
-(rule (normalize_cmp_value $I64  r) r)
-(rule (normalize_cmp_value $I128 r) r)
+(rule (normalize_cmp_value $I64  r _) r)
+(rule (normalize_cmp_value $I128 r _) r)
 
 ;; Convert a truthy value, possibly of more than one register (an
 ;; I128), to one register. If narrower than 64 bits, must have already
@@ -1940,7 +1939,7 @@
 ;; Default behavior for branching based on an input value.
 (rule
   (lower_branch (brif v @ (value_type ty) _ _) targets)
-  (lower_cond_br (IntCC.NotEqual) (normalize_cmp_value ty v) targets ty))
+  (lower_cond_br (IntCC.NotEqual) (normalize_cmp_value ty v (ExtendOp.Zero)) targets ty))
 
 ;; Special case for SI128 to reify the comparison value and branch on it.
 (rule 2
@@ -2118,7 +2117,7 @@
 (rule
   0
   (lower_bmask (fits_in_64 _) (fits_in_64 in_ty) val)
-  (let ((input Reg (normalize_cmp_value in_ty val))
+  (let ((input Reg (normalize_cmp_value in_ty val (ExtendOp.Zero)))
         (zero Reg (zero_reg))
         (ones Reg (load_imm12 -1)))
   (value_reg (gen_select_reg (IntCC.Equal) zero input zero ones))))
diff --git a/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs
index d62d8ba4a..1803ecfec 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/unwind/systemv.rs
@@ -143,7 +143,7 @@ mod tests {
 
         assert_eq!(
             format!("{:?}", fde),
-            "FrameDescriptionEntry { address: Constant(4321), length: 16, lsda: None, instructions: [] }"
+            "FrameDescriptionEntry { address: Constant(4321), length: 20, lsda: None, instructions: [] }"
         );
     }
 
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 9d1f43af5..b08d6558f 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -626,12 +626,12 @@
 ;;;;;  Rules for `select`;;;;;;;;;
 (rule
   (lower (has_type ty (select c @ (value_type cty) x y)))
-  (gen_select ty (truthy_to_reg cty (normalize_cmp_value cty c)) x y))
+  (gen_select ty (truthy_to_reg cty (normalize_cmp_value cty c (ExtendOp.Zero))) x y))
 
 (rule 1
   (lower (has_type (fits_in_64 ty) (select (icmp cc a b @ (value_type in_ty)) x y)))
-  (let ((a Reg (normalize_cmp_value in_ty a))
-        (b Reg (normalize_cmp_value in_ty b)))
+  (let ((a Reg (normalize_cmp_value in_ty a (intcc_to_extend_op cc)))
+        (b Reg (normalize_cmp_value in_ty b (intcc_to_extend_op cc))))
     (gen_select_reg cc a b x y)))
 
 ;;;;;  Rules for `bitselect`;;;;;;;;;
@@ -851,7 +851,7 @@
 
 (rule -1
   (lower (has_type ty (select_spectre_guard c @ (value_type cty) x y)))
-  (gen_select ty (truthy_to_reg cty (normalize_cmp_value cty c)) x y))
+  (gen_select ty (truthy_to_reg cty (normalize_cmp_value cty c (ExtendOp.Zero))) x y))
 
 ;;;;;  Rules for `bmask`;;;;;;;;;
 (rule
diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
index f1b3fed92..1c088b4be 100644
--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
@@ -3,7 +3,7 @@
 // Pull in the ISLE generated code.
 #[allow(unused)]
 pub mod generated_code;
-use generated_code::{Context, MInst};
+use generated_code::{Context, ExtendOp, MInst};
 
 // Types that the generated ISLE code uses via `use super::*`.
 use super::{writable_zero_reg, zero_reg};
@@ -60,7 +60,22 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Riscv64Backend> {
             _ => unreachable!(),
         }
     }
-
+    fn intcc_to_extend_op(&mut self, cc: &IntCC) -> ExtendOp {
+        use IntCC::*;
+        match *cc {
+            Equal
+            | NotEqual
+            | UnsignedLessThan
+            | UnsignedGreaterThanOrEqual
+            | UnsignedGreaterThan
+            | UnsignedLessThanOrEqual => ExtendOp::Zero,
+
+            SignedLessThan
+            | SignedGreaterThanOrEqual
+            | SignedGreaterThan
+            | SignedLessThanOrEqual => ExtendOp::Signed,
+        }
+    }
     fn lower_cond_br(
         &mut self,
         cc: &IntCC,
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index d25cc7efc..993b4984d 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -860,7 +860,10 @@
             Ucomisd
             Unpcklps
             Xorps
-            Xorpd))
+            Xorpd
+            Phaddw
+            Phaddd
+          ))
 
 (type CmpOpcode extern
       (enum Cmp
@@ -1356,6 +1359,8 @@
             Vcvtps2pd
             Vcvttpd2dq
             Vcvttps2dq
+            Vphaddw
+            Vphaddd
           ))
 
 (type Avx512Opcode extern
@@ -2482,6 +2487,22 @@
       (if-let $true (has_avx))
       (xmm_rmir_vex (AvxOpcode.Vpaddsw) src1 src2))
 
+;; Helper for creating `phaddw` instructions.
+(decl x64_phaddw (Xmm XmmMem) Xmm)
+(rule 0 (x64_phaddw src1 src2)
+      (xmm_rm_r (SseOpcode.Phaddw) src1 src2))
+(rule 1 (x64_phaddw src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vphaddw) src1 src2))
+
+;; Helper for creating `phaddd` instructions.
+(decl x64_phaddd (Xmm XmmMem) Xmm)
+(rule 0 (x64_phaddd src1 src2)
+      (xmm_rm_r (SseOpcode.Phaddd) src1 src2))
+(rule 1 (x64_phaddd src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vphaddd) src1 src2))
+
 ;; Helper for creating `paddusb` instructions.
 (decl x64_paddusb (Xmm XmmMem) Xmm)
 (rule 0 (x64_paddusb src1 src2)
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 772727133..ca6e40ce5 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -1115,6 +1115,8 @@ pub enum SseOpcode {
     Unpcklps,
     Xorps,
     Xorpd,
+    Phaddw,
+    Phaddd,
 }
 
 impl SseOpcode {
@@ -1261,7 +1263,9 @@ impl SseOpcode {
             | SseOpcode::Pabsd
             | SseOpcode::Palignr
             | SseOpcode::Pmulhrsw
-            | SseOpcode::Pshufb => SSSE3,
+            | SseOpcode::Pshufb
+            | SseOpcode::Phaddw
+            | SseOpcode::Phaddd => SSSE3,
 
             SseOpcode::Blendvpd
             | SseOpcode::Blendvps
@@ -1495,6 +1499,8 @@ impl fmt::Debug for SseOpcode {
             SseOpcode::Unpcklps => "unpcklps",
             SseOpcode::Xorps => "xorps",
             SseOpcode::Xorpd => "xorpd",
+            SseOpcode::Phaddw => "phaddw",
+            SseOpcode::Phaddd => "phaddd",
         };
         write!(fmt, "{}", name)
     }
@@ -1661,7 +1667,9 @@ impl AvxOpcode {
             | AvxOpcode::Vcvtpd2ps
             | AvxOpcode::Vcvtps2pd
             | AvxOpcode::Vcvttpd2dq
-            | AvxOpcode::Vcvttps2dq => {
+            | AvxOpcode::Vcvttps2dq
+            | AvxOpcode::Vphaddw
+            | AvxOpcode::Vphaddd => {
                 smallvec![InstructionSet::AVX]
             }
         }
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index e43746c42..b92e1ecd9 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1954,6 +1954,8 @@ pub(crate) fn emit(
                 SseOpcode::Unpcklps => (LegacyPrefixes::None, 0x0F14, 2),
                 SseOpcode::Xorps => (LegacyPrefixes::None, 0x0F57, 2),
                 SseOpcode::Xorpd => (LegacyPrefixes::_66, 0x0F57, 2),
+                SseOpcode::Phaddw => (LegacyPrefixes::_66, 0x0F3801, 3),
+                SseOpcode::Phaddd => (LegacyPrefixes::_66, 0x0F3802, 3),
                 _ => unimplemented!("Opcode {:?} not implemented", op),
             };
 
@@ -2167,6 +2169,8 @@ pub(crate) fn emit(
                 AvxOpcode::Vminsd => (LP::_F2, OM::_0F, 0x5D),
                 AvxOpcode::Vmaxss => (LP::_F3, OM::_0F, 0x5F),
                 AvxOpcode::Vmaxsd => (LP::_F2, OM::_0F, 0x5F),
+                AvxOpcode::Vphaddw => (LP::_66, OM::_0F38, 0x01),
+                AvxOpcode::Vphaddd => (LP::_66, OM::_0F38, 0x02),
                 _ => panic!("unexpected rmir vex opcode {op:?}"),
             };
             VexInstruction::new()
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 40b40f9b9..ec21968b3 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -522,12 +522,10 @@
             ;; Shift `src` using 16x8. Unfortunately, a 16x8 shift will only be
             ;; correct for half of the lanes; the others must be fixed up with
             ;; the mask below.
-            (unmasked Xmm (x64_psrlw src (mov_rmi_to_xmm masked_amt)))
-            (mask_addr SyntheticAmode (ushr_i8x16_mask masked_amt))
-            (mask Reg (x64_load $I8X16 mask_addr (ExtKind.None))))
+            (unmasked Xmm (x64_psrlw src (mov_rmi_to_xmm masked_amt))))
         (sse_and $I8X16
                  unmasked
-                 (RegMem.Reg mask))))
+                 (ushr_i8x16_mask masked_amt))))
 
 ;; Get the address of the mask to use when fixing up the lanes that weren't
 ;; correctly generated by the 16x8 shift.
@@ -3173,8 +3171,14 @@
 
 ;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(rule (lower (has_type $I16X8 (iadd_pairwise x y)))
+      (x64_phaddw x y))
+
+(rule (lower (has_type $I32X4 (iadd_pairwise x y)))
+      (x64_phaddd x y))
+
 ;; special case for the `i16x8.extadd_pairwise_i8x16_s` wasm instruction
-(rule (lower
+(rule 1 (lower
         (has_type $I16X8 (iadd_pairwise
                            (swiden_low val @ (value_type $I8X16))
                            (swiden_high val))))
@@ -3182,7 +3186,7 @@
         (x64_pmaddubsw mul_const val)))
 
 ;; special case for the `i32x4.extadd_pairwise_i16x8_s` wasm instruction
-(rule (lower
+(rule 1 (lower
         (has_type $I32X4 (iadd_pairwise
                            (swiden_low val @ (value_type $I16X8))
                            (swiden_high val))))
@@ -3190,7 +3194,7 @@
         (x64_pmaddwd val mul_const)))
 
 ;; special case for the `i16x8.extadd_pairwise_i8x16_u` wasm instruction
-(rule (lower
+(rule 1 (lower
         (has_type $I16X8 (iadd_pairwise
                            (uwiden_low val @ (value_type $I8X16))
                            (uwiden_high val))))
@@ -3198,7 +3202,7 @@
         (x64_pmaddubsw val mul_const)))
 
 ;; special case for the `i32x4.extadd_pairwise_i16x8_u` wasm instruction
-(rule (lower
+(rule 1 (lower
         (has_type $I32X4 (iadd_pairwise
                            (uwiden_low val @ (value_type $I16X8))
                            (uwiden_high val))))
@@ -3212,7 +3216,7 @@
         (x64_paddd dst addd_const)))
 
 ;; special case for the `i32x4.dot_i16x8_s` wasm instruction
-(rule (lower
+(rule 1 (lower
         (has_type $I32X4 (iadd_pairwise
                            (imul (swiden_low x) (swiden_low y))
                            (imul (swiden_high x) (swiden_high y)))))
@@ -3504,6 +3508,16 @@
 
 ;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; Special case for the `punpckhbw` instruction which interleaves the upper
+;; lanes of the two input registers.
+(rule 4 (lower (shuffle a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))
+      (x64_punpckhbw a b))
+
+;; Special case for the `punpcklbw` instruction which interleaves the lower
+;; lanes of the two input registers.
+(rule 4 (lower (shuffle a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))
+      (x64_punpcklbw a b))
+
 ;; If `lhs` and `rhs` are the same we can use a single PSHUFB to shuffle the XMM
 ;; register. We statically build `constructed_mask` to zero out any unknown lane
 ;; indices (may not be completely necessary: verification could fail incorrect
diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs
index 659b94a9b..682eb256f 100644
--- a/cranelift/entity/src/list.rs
+++ b/cranelift/entity/src/list.rs
@@ -62,7 +62,7 @@ use serde::{Deserialize, Serialize};
 ///
 /// The index stored in an `EntityList` points to part 2, the list elements. The value 0 is
 /// reserved for the empty list which isn't allocated in the vector.
-#[derive(Clone, Copy, Debug, PartialEq, Hash)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct EntityList<T: EntityRef + ReservedValue> {
     index: u32,
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index d4da0a2d2..0eddf9c35 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -1027,7 +1027,7 @@ where
                     new[i] = x[s[i] as usize];
                 } // else leave as 0
             }
-            assign(Value::vector(new, ctrl_ty)?)
+            assign(Value::vector(new, types::I8X16)?)
         }
         Opcode::Splat => {
             let mut new_vector = SimdVec::new();
diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs
index 94301f932..b3786a107 100644
--- a/cranelift/wasm/src/code_translator.rs
+++ b/cranelift/wasm/src/code_translator.rs
@@ -1678,7 +1678,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
         }
         Operator::I8x16Swizzle => {
             let (a, b) = pop2_with_bitcast(state, I8X16, builder);
-            state.push1(builder.ins().swizzle(I8X16, a, b))
+            state.push1(builder.ins().swizzle(a, b))
         }
         Operator::I8x16Add | Operator::I16x8Add | Operator::I32x4Add | Operator::I64x2Add => {
             let (a, b) = pop2_with_bitcast(state, type_of(op), builder);
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index f8443092d..ffffc50a4 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -48,6 +48,7 @@
   - [Tiers of support](./stability-tiers.md)
   - [Platform Support](./stability-platform-support.md)
   - [Wasm Proposals Support](./stability-wasm-proposals-support.md)
+  - [WASI Proposals Support](./stability-wasi-proposals-support.md)
 - [Security](security.md)
   - [Disclosure Policy](./security-disclosure.md)
 - [Contributing](contributing.md)
diff --git a/docs/stability-wasi-proposals-support.md b/docs/stability-wasi-proposals-support.md
new file mode 100644
index 000000000..ce8121e24
--- /dev/null
+++ b/docs/stability-wasi-proposals-support.md
@@ -0,0 +1,60 @@
+# WASI Proposals Support
+
+The following table summarizes Wasmtime's support for WASI [proposals]. If a
+proposal is not listed, then it is not supported by Wasmtime.
+
+[proposals]: https://github.com/WebAssembly/WASI/blob/main/Proposals.md
+
+| WASI Proposal                          | Supported in Wasmtime?  | Enabled by default?  | CLI Flag Name [^cli]        |
+|----------------------------------------|-------------------------|----------------------|-----------------------------|
+| [I/O][wasi-io]                         | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Filesystem][wasi-filesystem]          | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Clocks][wasi-clocks]                  | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Random][wasi-random]                  | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Poll][wasi-poll]                      | **Yes**                 | **Yes**              | `wasi-common`               |
+| [Machine Learning (wasi-nn)][wasi-nn]  | **Yes**                 | No                   | `experimental-wasi-nn`      |
+| [Blob Store][wasi-blob-store]          | No                      | No                   | N/A                         |
+| [Crypto][wasi-crypto]                  | **Yes** [^crypto]       | No                   | `experimental-wasi-crypto`  |
+| [Distributed Lock Service][wasi-distributed-lock-service] | No   | No                   | N/A                         |
+| [gRPC][wasi-grpc]                      | No                      | No                   | N/A                         |
+| [HTTP][wasi-http]                      | No                      | No                   | N/A                         |
+| [Key-value Store][wasi-kv-store]       | No                      | No                   | N/A                         |
+| [Message Queue][wasi-message-queue]    | No                      | No                   | N/A                         |
+| [Parallel][wasi-parallel]              | No (see [#4949])        | No                   | N/A                         |
+| [Pub/sub][wasi-pubsub]                 | No                      | No                   | N/A                         |
+| [Runtime Config][wasi-runtime-config]  | No                      | No                   | N/A                         |
+| [Sockets][wasi-sockets]                | No                      | No                   | N/A                         |
+| [SQL][wasi-sql]                        | No                      | No                   | N/A                         |
+| [Threads][wasi-threads]                | **Yes**                 | No                   | `experimental-wasi-threads` |
+
+[^cli]: The CLI flag name refers to to the `--wasi-modules` argument of the
+    `wasmtime` executable; e.g., `--wasi-modules=wasi-crypto`. See `wasmtime run
+    --help` for more information on the flag's default value and configuration.
+[^crypto]: Build Wasmtime with `--features=wasi-crypto` to enable this.
+
+[#4949]: https://github.com/bytecodealliance/wasmtime/pull/4949
+[wasi-blob-store]: https://github.com/WebAssembly/wasi-blob-store
+[wasi-clocks]: https://github.com/WebAssembly/wasi-clocks
+[wasi-classic-command]: https://github.com/WebAssembly/wasi-classic-command
+[wasi-crypto]: https://github.com/WebAssembly/wasi-crypto
+[wasi-data]: https://github.com/singlestore-labs/wasi-data
+[wasi-distributed-lock-service]: https://github.com/WebAssembly/wasi-distributed-lock-service
+[wasi-filesystem]: https://github.com/WebAssembly/wasi-filesystem
+[wasi-grpc]: https://github.com/WebAssembly/wasi-grpc
+[wasi-handle-index]: https://github.com/WebAssembly/wasi-handle-index
+[wasi-http]: https://github.com/WebAssembly/wasi-http
+[wasi-io]: https://github.com/WebAssembly/wasi-io
+[wasi-kv-store]: https://github.com/WebAssembly/wasi-kv-store
+[wasi-message-queue]: https://github.com/WebAssembly/wasi-message-queue
+[wasi-misc]: https://github.com/WebAssembly/wasi-misc
+[wasi-threads]: https://github.com/WebAssembly/wasi-native-threads
+[wasi-nn]: https://github.com/WebAssembly/wasi-nn
+[wasi-random]: https://github.com/WebAssembly/wasi-random
+[wasi-parallel]: https://github.com/WebAssembly/wasi-parallel
+[wasi-poll]: https://github.com/WebAssembly/wasi-poll
+[wasi-proxy-wasm]: https://github.com/proxy-wasm/spec
+[wasi-pubsub]: https://github.com/WebAssembly/wasi-pubsub
+[wasi-runtime-config]: https://github.com/WebAssembly/wasi-runtime-config
+[wasi-sockets]: https://github.com/WebAssembly/wasi-sockets
+[wasi-sql]: https://github.com/WebAssembly/wasi-sql
+[wasi-url]: https://github.com/WebAssembly/wasi-url


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/issue-5796.clif b/cranelift/filetests/filetests/egraph/issue-5796.clif
new file mode 100644
index 000000000..66963f5c4
--- /dev/null
+++ b/cranelift/filetests/filetests/egraph/issue-5796.clif
@@ -0,0 +1,76 @@
+test optimize
+set opt_level=speed
+target x86_64
+
+function %f0(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  brif v0, block1, block2
+
+block1:
+  v3 = udiv v1, v2
+  return v3
+
+block2:
+  v4 = udiv v1, v2
+  brif v1, block3, block4
+
+block3:
+  return v4
+
+block4:
+  return v4
+}
+
+; check: block0(v0: i32, v1: i32, v2: i32):
+; check:     brif v0, block1, block2
+
+; check: block1:
+; check:     v3 = udiv.i32 v1, v2
+; check:     return v3
+
+; check: block2:
+; check:     v4 = udiv.i32 v1, v2
+; check:     brif.i32 v1, block3, block4
+
+; check: block3:
+; check:     return v4
+
+; check: block4:
+; check:     return v4
+
+function %f1(i32, i32, i32) -> i32 {
+block0(v0: i32, v1: i32, v2: i32):
+  brif v0, block1, block2
+
+block1:
+  v3 = udiv v1, v2
+  return v3
+
+block2:
+  v4 = udiv v1, v2
+  brif v1, block3, block4
+
+block3:
+  v5 = udiv v1, v2
+  return v5
+
+block4:
+  return v4
+}
+
+; check: block0(v0: i32, v1: i32, v2: i32):
+; check:     brif v0, block1, block2
+
+; check: block1:
+; check:     v3 = udiv.i32 v1, v2
+; check:     return v3
+
+; check: block2:
+; check:     v4 = udiv.i32 v1, v2
+; check:     brif.i32 v1, block3, block4
+
+; check: block3:
+; check:     return v4
+
+; check: block4:
+; check:     return v4
diff --git a/cranelift/filetests/filetests/isa/aarch64/fma.clif b/cranelift/filetests/filetests/isa/aarch64/fma.clif
new file mode 100644
index 000000000..e2f4a172c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/fma.clif
@@ -0,0 +1,159 @@
+test compile precise-output
+target aarch64
+
+function %fma_f32(f32, f32, f32) -> f32 {
+block0(v0: f32, v1: f32, v2: f32):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   fmadd s0, s0, s1, s2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   fmadd s0, s0, s1, s2
+;   ret
+
+function %fma_f64(f64, f64, f64) -> f64 {
+block0(v0: f64, v1: f64, v2: f64):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   fmadd d0, d0, d1, d2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   fmadd d0, d0, d1, d2
+;   ret
+
+function %fma_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmla v0.2d, v5.2d, v1.2d
+;   ret
+
+function %fma_neg_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_neg_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v5.2d, v1.2d
+;   ret
+
+function %fma_neg_other_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fneg v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v0.4s, v5.4s, v1.4s
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.4s, v5.4s, v1.4s
+;   ret
+
+function %fma_neg_other_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fneg v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+; block0:
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v0.2d, v5.2d, v1.2d
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov v5.16b, v0.16b
+;   mov v0.16b, v2.16b
+;   fmls v0.2d, v5.2d, v1.2d
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/aarch64/iconst-imm12_from_negated.isle b/cranelift/filetests/filetests/isa/aarch64/iconst-imm12_from_negated.isle
new file mode 100644
index 000000000..61067e316
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/iconst-imm12_from_negated.isle
@@ -0,0 +1,66 @@
+;; Test our lowerings that try to fit negated iconst values in immediates, like
+;; `x + -1 => sub x 0x1`. Currently, not working as expected for i32 and i16. 
+
+test compile precise-output
+set unwind_info=false
+target aarch64
+
+;; Result uses immediate as expected
+function %c(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 -1
+    v3 = iadd v0, v1
+    return v3
+}
+
+; VCode:
+; block0:
+;   sub x0, x0, #1
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   sub x0, x0, #1
+;   ret
+
+;; 4294967295 is zero-extended i32 -1
+;; Result should use immediate but currently doesn't
+function %b(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 4294967295 
+    v3 = iadd v0, v1
+    return v3
+}
+
+; VCode:
+; block0:
+;   movn w2, #0
+;   add w0, w0, w2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w2, #-1
+;   add w0, w0, w2
+;   ret
+
+;; 65535 is zero-extended i16 -1
+;; Result should use immediate but currently doesn't
+function %a(i16) -> i16 {
+block0(v0: i16):
+    v1 = iconst.i16 65535
+    v3 = iadd v0, v1
+    return v3
+}
+
+; VCode:
+; block0:
+;   movz w2, #65535
+;   add w0, w0, w2
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w2, #0xffff
+;   add w0, w0, w2
+;   ret
diff --git a/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif b/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif
index c83e56142..af539f84b 100644
--- a/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif
@@ -69,7 +69,7 @@ function %swizzle() -> i8x16 {
 block0:
     v0 = vconst.i8x16 [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]
     v1 = vconst.i8x16 [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]
-    v2 = swizzle.i8x16 v0, v1
+    v2 = swizzle v0, v1
     return v2
 }
 
diff --git a/cranelift/filetests/filetests/isa/riscv64/cold-blocks.clif b/cranelift/filetests/filetests/isa/riscv64/cold-blocks.clif
index 0b7a8cde0..bfaab0334 100644
--- a/cranelift/filetests/filetests/isa/riscv64/cold-blocks.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/cold-blocks.clif
@@ -16,8 +16,9 @@ block2:
 
 ; VCode:
 ; block0:
-;   sext.w a2,a0
-;   bne a2,zero,taken(label1),not_taken(label2)
+;   slli a2,a0,32
+;   srli a4,a2,32
+;   bne a4,zero,taken(label1),not_taken(label2)
 ; block1:
 ;   j label3
 ; block2:
@@ -28,11 +29,12 @@ block2:
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   sext.w a2, a0
-;   bnez a2, 8
-; block1: ; offset 0x8
+;   slli a2, a0, 0x20
+;   srli a4, a2, 0x20
+;   bnez a4, 8
+; block1: ; offset 0xc
 ;   addi a0, zero, 0x61
-; block2: ; offset 0xc
+; block2: ; offset 0x10
 ;   ret
 
 function %cold_annotation(i32) -> i32 {
@@ -49,8 +51,9 @@ block2 cold:
 
 ; VCode:
 ; block0:
-;   sext.w a2,a0
-;   bne a2,zero,taken(label1),not_taken(label2)
+;   slli a2,a0,32
+;   srli a4,a2,32
+;   bne a4,zero,taken(label1),not_taken(label2)
 ; block1:
 ;   j label3
 ; block3:
@@ -61,11 +64,12 @@ block2 cold:
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   sext.w a2, a0
-;   beqz a2, 8
-; block1: ; offset 0x8
+;   slli a2, a0, 0x20
+;   srli a4, a2, 0x20
+;   beqz a4, 8
+; block1: ; offset 0xc
 ;   ret
-; block2: ; offset 0xc
+; block2: ; offset 0x10
 ;   addi a0, zero, 0x61
 ;   j -8
 
diff --git a/cranelift/filetests/filetests/isa/riscv64/condbr.clif b/cranelift/filetests/filetests/isa/riscv64/condbr.clif
index 3a0c18efe..617afdfe0 100644
--- a/cranelift/filetests/filetests/isa/riscv64/condbr.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/condbr.clif
@@ -675,10 +675,9 @@ block1:
 
 ; VCode:
 ; block0:
-;   lui a1,16
-;   addi a1,a1,4095
-;   and a3,a0,a1
-;   bne a3,zero,taken(label1),not_taken(label2)
+;   slli t2,a0,48
+;   srli a1,t2,48
+;   bne a1,zero,taken(label1),not_taken(label2)
 ; block1:
 ;   j label3
 ; block2:
@@ -688,10 +687,9 @@ block1:
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   lui a1, 0x10
-;   addi a1, a1, -1
-;   and a3, a0, a1
-; block1: ; offset 0xc
+;   slli t2, a0, 0x30
+;   srli a1, t2, 0x30
+; block1: ; offset 0x8
 ;   ret
 
 function %i32_brif(i32){
@@ -705,8 +703,9 @@ block1:
 
 ; VCode:
 ; block0:
-;   sext.w t2,a0
-;   bne t2,zero,taken(label1),not_taken(label2)
+;   slli t2,a0,32
+;   srli a1,t2,32
+;   bne a1,zero,taken(label1),not_taken(label2)
 ; block1:
 ;   j label3
 ; block2:
@@ -716,8 +715,9 @@ block1:
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   sext.w t2, a0
-; block1: ; offset 0x4
+;   slli t2, a0, 0x20
+;   srli a1, t2, 0x20
+; block1: ; offset 0x8
 ;   ret
 
 function %i64_brif(i64){
diff --git a/cranelift/filetests/filetests/isa/riscv64/condops.clif b/cranelift/filetests/filetests/isa/riscv64/condops.clif
index 35bd747ef..c2c1334ef 100644
--- a/cranelift/filetests/filetests/isa/riscv64/condops.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/condops.clif
@@ -108,18 +108,22 @@ block0(v0: i32, v1: i8, v2: i8):
 
 ; VCode:
 ; block0:
-;   sext.w a3,a0
-;   li a4,42
-;   sext.w a5,a4
-;   select_reg a0,a1,a2##condition=(a3 eq a5)
+;   slli a3,a0,32
+;   srli a3,a3,32
+;   li a5,42
+;   slli a7,a5,32
+;   srli t4,a7,32
+;   select_reg a0,a1,a2##condition=(a3 eq t4)
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   sext.w a3, a0
-;   addi a4, zero, 0x2a
-;   sext.w a5, a4
-;   beq a3, a5, 0xc
+;   slli a3, a0, 0x20
+;   srli a3, a3, 0x20
+;   addi a5, zero, 0x2a
+;   slli a7, a5, 0x20
+;   srli t4, a7, 0x20
+;   beq a3, t4, 0xc
 ;   ori a0, a2, 0
 ;   j 8
 ;   ori a0, a1, 0
diff --git a/cranelift/filetests/filetests/isa/riscv64/i128-bmask.clif b/cranelift/filetests/filetests/isa/riscv64/i128-bmask.clif
index 967338b90..e802e74d8 100644
--- a/cranelift/filetests/filetests/isa/riscv64/i128-bmask.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/i128-bmask.clif
@@ -150,17 +150,20 @@ block0(v0: i32):
 
 ; VCode:
 ; block0:
-;   sext.w t2,a0
-;   li a1,-1
-;   select_reg a1,zero,a1##condition=(zero eq t2)
+;   slli t2,a0,32
+;   srli a1,t2,32
+;   li a3,-1
+;   select_reg a1,zero,a3##condition=(zero eq a1)
 ;   mv a0,a1
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   sext.w t2, a0
-;   addi a1, zero, -1
-;   beq zero, t2, 8
+;   slli t2, a0, 0x20
+;   srli a1, t2, 0x20
+;   addi a3, zero, -1
+;   beq zero, a1, 0xc
+;   ori a1, a3, 0
 ;   j 8
 ;   ori a1, zero, 0
 ;   ori a0, a1, 0
@@ -174,22 +177,20 @@ block0(v0: i16):
 
 ; VCode:
 ; block0:
-;   lui a1,16
-;   addi a1,a1,4095
-;   and a3,a0,a1
-;   li a5,-1
-;   select_reg a1,zero,a5##condition=(zero eq a3)
+;   slli t2,a0,48
+;   srli a1,t2,48
+;   li a3,-1
+;   select_reg a1,zero,a3##condition=(zero eq a1)
 ;   mv a0,a1
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   lui a1, 0x10
-;   addi a1, a1, -1
-;   and a3, a0, a1
-;   addi a5, zero, -1
-;   beq zero, a3, 0xc
-;   ori a1, a5, 0
+;   slli t2, a0, 0x30
+;   srli a1, t2, 0x30
+;   addi a3, zero, -1
+;   beq zero, a1, 0xc
+;   ori a1, a3, 0
 ;   j 8
 ;   ori a1, zero, 0
 ;   ori a0, a1, 0
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif b/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif
index 61caaaa1f..968886d5a 100644
--- a/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif
+++ b/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif
@@ -3,7 +3,7 @@ target s390x
 
 function %swizzle(i8x16, i8x16) -> i8x16 wasmtime_system_v {
 block0(v0: i8x16, v1: i8x16):
-    v2 = swizzle.i8x16 v0, v1
+    v2 = swizzle v0, v1
     return v2
 }
 
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-permute.clif b/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
index 2fa220a68..3e8295884 100644
--- a/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
+++ b/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
@@ -3,7 +3,7 @@ target s390x
 
 function %swizzle(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
-    v2 = swizzle.i8x16 v0, v1
+    v2 = swizzle v0, v1
     return v2
 }
 
diff --git a/cranelift/filetests/filetests/isa/x64/iadd-pairwise-avx.clif b/cranelift/filetests/filetests/isa/x64/iadd-pairwise-avx.clif
new file mode 100644
index 000000000..4ad4003e8
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/iadd-pairwise-avx.clif
@@ -0,0 +1,54 @@
+test compile precise-output
+set enable_simd
+target x86_64 has_avx
+
+function %iadd_pairwise_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+  v2 = iadd_pairwise v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vphaddw %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vphaddw %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %iadd_pairwise_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+  v2 = iadd_pairwise v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vphaddd %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vphaddd %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/iadd-pairwise.clif b/cranelift/filetests/filetests/isa/x64/iadd-pairwise.clif
new file mode 100644
index 000000000..2e1d1c85d
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/iadd-pairwise.clif
@@ -0,0 +1,54 @@
+test compile precise-output
+set enable_simd
+target x86_64
+
+function %iadd_pairwise_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+  v2 = iadd_pairwise v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   phaddw  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   phaddw %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %iadd_pairwise_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+  v2 = iadd_pairwise v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   phaddd  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   phaddd %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle.clif b/cranelift/filetests/filetests/isa/x64/shuffle.clif
new file mode 100644
index 000000000..529b95cc5
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/shuffle.clif
@@ -0,0 +1,54 @@
+test compile precise-output
+set enable_simd
+target x86_64
+
+function %punpcklbw(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, 0x17071606150514041303120211011000
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpcklbw %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpcklbw %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckhbw(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, 0x1f0f1e0e1d0d1c0c1b0b1a0a19091808
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpckhbw %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpckhbw %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
index 3e622bd5f..fdd58032e 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
@@ -1635,8 +1635,7 @@ block0(v0: i8x16, v1: i32):
 ;   vpsrlw  %xmm0, %xmm5, %xmm7
 ;   lea     const(0), %rsi
 ;   shlq    $4, %r10, %r10
-;   movdqu  0(%rsi,%r10,1), %xmm13
-;   vpand   %xmm7, %xmm13, %xmm0
+;   vpand   %xmm7, 0(%rsi,%r10,1), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1652,11 +1651,13 @@ block0(v0: i8x16, v1: i32):
 ;   vpsrlw %xmm5, %xmm0, %xmm7
 ;   leaq 0x15(%rip), %rsi
 ;   shlq $4, %r10
-;   movdqu (%rsi, %r10), %xmm13
-;   vpand %xmm13, %xmm7, %xmm0
+;   vpand (%rsi, %r10), %xmm7, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %bh, %bh
 
 function %i8x16_ushr_imm(i8x16) -> i8x16 {
 block0(v0: i8x16):
@@ -1670,8 +1671,7 @@ block0(v0: i8x16):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   vpsrlw  %xmm0, $1, %xmm2
-;   movdqu  const(0), %xmm4
-;   vpand   %xmm2, %xmm4, %xmm0
+;   vpand   %xmm2, const(0), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1682,14 +1682,15 @@ block0(v0: i8x16):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   vpsrlw $1, %xmm0, %xmm2
-;   movdqu 0xf(%rip), %xmm4
-;   vpand %xmm4, %xmm2, %xmm0
+;   vpand 0xf(%rip), %xmm2, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
 ;   jg 0xa1
 ;   jg 0xa3
 ;   jg 0xa5
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
index 1ecdf31ed..f63cc2231 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
@@ -485,8 +485,7 @@ block0:
 ; block0:
 ;   movdqu  const(1), %xmm0
 ;   psrlw   %xmm0, $1, %xmm0
-;   movdqu  const(0), %xmm3
-;   pand    %xmm0, %xmm3, %xmm0
+;   pand    %xmm0, const(0), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -496,28 +495,21 @@ block0:
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movdqu 0x34(%rip), %xmm0
+;   movdqu 0x24(%rip), %xmm0
 ;   psrlw $1, %xmm0
-;   movdqu 0x17(%rip), %xmm3
-;   pand %xmm3, %xmm0
+;   pand 7(%rip), %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
 ;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   jg 0xb1
-;   jg 0xb3
-;   jg 0xb5
-;   jg 0xb7
-;   jg 0xb9
-;   jg 0xbb
-;   jg 0xbd
-;   jg 0xbf
+;   jg 0xa1
+;   jg 0xa3
+;   jg 0xa5
+;   jg 0xa7
+;   jg 0xa9
+;   jg 0xab
+;   jg 0xad
+;   jg 0xaf
 ;   addb %al, (%rcx)
 ;   addb (%rbx), %al
 ;   addb $5, %al
diff --git a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif
index b6f75d779..f58cad93a 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif
@@ -114,7 +114,7 @@ function %swizzle() -> i8x16 {
 block0:
     v0 = vconst.i8x16 [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]
     v1 = vconst.i8x16 [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]
-    v2 = swizzle.i8x16 v0, v1
+    v2 = swizzle v0, v1
     return v2
 }
 
diff --git a/cranelift/filetests/filetests/runtests/issue5839.clif b/cranelift/filetests/filetests/runtests/issue5839.clif
new file mode 100644
index 000000000..4e2a2c391
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/issue5839.clif
@@ -0,0 +1,20 @@
+test interpret
+test run
+set opt_level=speed
+target aarch64
+target s390x
+target x86_64
+target riscv64
+
+function %a(i8, i8) -> i32 {
+block0(v0: i8, v1: i8):
+    v2 = icmp sle v0, v1
+    v3 = uextend.i32 v2
+    v4 = iconst.i32 0
+    v5 = iconst.i32 1
+    v6 = icmp.i32 eq v3, v4  ; v4 = 0
+    v7 = select v6, v5, v3  ; v5 = 1
+    return v7
+}
+
+; run: %a(20, -11) == 1
\ No newline at end of file
diff --git a/cranelift/filetests/filetests/runtests/simd-lane-access.clif b/cranelift/filetests/filetests/runtests/simd-lane-access.clif
index 54de6a450..04cdf0bdf 100644
--- a/cranelift/filetests/filetests/runtests/simd-lane-access.clif
+++ b/cranelift/filetests/filetests/runtests/simd-lane-access.clif
@@ -52,7 +52,7 @@ block0:
 
 function %swizzle(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
-    v2 = swizzle.i8x16 v0, v1
+    v2 = swizzle v0, v1
     return v2
 }
 ; reverse the lanes, with over-large index 42 using lane 0
diff --git a/cranelift/filetests/filetests/runtests/simd-swizzle.clif b/cranelift/filetests/filetests/runtests/simd-swizzle.clif
index 2c53cfcee..d74c8c65c 100644
--- a/cranelift/filetests/filetests/runtests/simd-swizzle.clif
+++ b/cranelift/filetests/filetests/runtests/simd-swizzle.clif
@@ -7,7 +7,7 @@ target x86_64 has_sse3 has_ssse3 has_sse41
 
 function %swizzle_i8x16(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
-    v2 = swizzle.i8x16 v0, v1
+    v2 = swizzle v0, v1
     return v2
 }
 ; run: %swizzle_i8x16([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [0 9 15 1 6 13 7 11 10 8 100 12 4 2 3 5]) == [1 10 16 2 7 14 8 12 11 9 0 13 5 3 4 6]
diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
new file mode 100644
index 000000000..b6d4e88dc
--- /dev/null
+++ b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
@@ -0,0 +1,92 @@
+;;! target = "x86_64"
+;;!
+;;! optimize = true
+;;!
+;;! settings = [
+;;!   "enable_heap_access_spectre_mitigation=true",
+;;!   "opt_level=speed_and_size",
+;;!   "use_egraphs=true"
+;;! ]
+;;!
+;;! [globals.vmctx]
+;;! type = "i64"
+;;! vmctx = true
+;;!
+;;! [globals.heap_base]
+;;! type = "i64"
+;;! load = { base = "vmctx", offset = 0 }
+;;!
+;;! [globals.heap_bound]
+;;! type = "i64"
+;;! load = { base = "vmctx", offset = 8 }
+;;!
+;;! [[heaps]]
+;;! base = "heap_base"
+;;! min_size = 0
+;;! offset_guard_size = 0xffffffff
+;;! index_type = "i32"
+;;! style = { kind = "dynamic", bound = "heap_bound" }
+
+(module
+  (memory (export "memory") 0)
+  (func (export "load-without-offset") (param i32) (result i32 i32)
+    local.get 0
+    i32.load
+    local.get 0
+    i32.load
+  )
+  (func (export "load-with-offset") (param i32) (result i32 i32)
+    local.get 0
+    i32.load offset=1234
+    local.get 0
+    i32.load offset=1234
+  )
+)
+
+;; function u0:0(i32, i64 vmctx) -> i32, i32 fast {
+;;     gv0 = vmctx
+;;     gv1 = load.i64 notrap aligned gv0+8
+;;     gv2 = load.i64 notrap aligned gv0
+;;
+;;                                 block0(v0: i32, v1: i64):
+;; @0057                               v4 = uextend.i64 v0
+;; @0057                               v5 = iconst.i64 4
+;; @0057                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 4
+;; @0057                               v7 = load.i64 notrap aligned v1+8
+;; @0057                               v8 = load.i64 notrap aligned v1
+;; @0057                               v11 = icmp ugt v6, v7
+;; @0057                               v10 = iconst.i64 0
+;; @0057                               v9 = iadd v8, v4
+;; @0057                               v12 = select_spectre_guard v11, v10, v9  ; v10 = 0
+;; @0057                               v13 = load.i32 little heap v12
+;;                                     v2 -> v13
+;; @005f                               jump block1
+;;
+;;                                 block1:
+;; @005f                               return v13, v13
+;; }
+;;
+;; function u0:1(i32, i64 vmctx) -> i32, i32 fast {
+;;     gv0 = vmctx
+;;     gv1 = load.i64 notrap aligned gv0+8
+;;     gv2 = load.i64 notrap aligned gv0
+;;
+;;                                 block0(v0: i32, v1: i64):
+;; @0064                               v4 = uextend.i64 v0
+;; @0064                               v5 = iconst.i64 1238
+;; @0064                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 1238
+;; @0064                               v7 = load.i64 notrap aligned v1+8
+;; @0064                               v8 = load.i64 notrap aligned v1
+;; @0064                               v12 = icmp ugt v6, v7
+;; @0064                               v11 = iconst.i64 0
+;; @0064                               v9 = iadd v8, v4
+;;                                     v26 = iconst.i64 1234
+;; @0064                               v10 = iadd v9, v26  ; v26 = 1234
+;; @0064                               v13 = select_spectre_guard v12, v11, v10  ; v11 = 0
+;; @0064                               v14 = load.i32 little heap v13
+;;                                     v2 -> v14
+;; @006e                               jump block1
+;;
+;;                                 block1:
+;; @006e                               return v14, v14
+;; }