Repository: bytecodealliance/wasmtime
Pull request #5847 (2023-02-21 16:12:13)

riscv64: Add bitmanip extension flags
This PR clarifies our bit manipulation set of ISA flags.

It seems that our current "B" extension is the full set of instructions defined in the Bit-Manipulation spec ([pdf](https://github.com/riscv/riscv-bitmanip/releases/download/1.0.0/bitmanip-1.0.0-38-g865e7a7.pdf)). However we should be querying each extension individually since some processors may not implement all of them.

Additionally it may cause confusion with the currently reserved B extension defined in Chapter 19 of the ISA ([pdf](https://github.com/riscv/riscv-isa-manual/releases/download/draft-20230131-c0b298a/riscv-spec.pdf)). These are unrelated and [there is no connection between "B" and the various extensions defined in the Bit-Manipulation spec](https://github.com/riscv/riscv-isa-manual/issues/907).

As an example, the [SiFive U74](https://www.sifive.com/cores/u74) implements only a subset of these (Zbb, Zba) so it wouldn't qualify for most of these optimizations previously, despite most of them only needing Zbb.


Patch:
diff --git a/cranelift/codegen/meta/src/isa/riscv64.rs b/cranelift/codegen/meta/src/isa/riscv64.rs
index 3b1cc6254..00b3bd6c6 100644
--- a/cranelift/codegen/meta/src/isa/riscv64.rs
+++ b/cranelift/codegen/meta/src/isa/riscv64.rs
@@ -11,13 +11,50 @@ fn define_settings(_shared: &SettingGroup) -> SettingGroup {
     let _has_f = setting.add_bool("has_f", "has extension F?", "", false);
     let _has_d = setting.add_bool("has_d", "has extension D?", "", false);
     let _has_v = setting.add_bool("has_v", "has extension V?", "", false);
-    let _has_b = setting.add_bool("has_b", "has extension B?", "", false);
     let _has_c = setting.add_bool("has_c", "has extension C?", "", false);
-    let _has_zbkb = setting.add_bool("has_zbkb", "has extension zbkb?", "", false);
-    let _has_zbb = setting.add_bool("has_zbb", "has extension zbb?", "", false);
+    let _has_zbkb = setting.add_bool(
+        "has_zbkb",
+        "has extension zbkb?",
+        "Zbkb: Bit-manipulation for Cryptography",
+        false,
+    );
+    let _has_zba = setting.add_bool(
+        "has_zba",
+        "has extension zba?",
+        "Zba: Address Generation",
+        false,
+    );
+    let _has_zbb = setting.add_bool(
+        "has_zbb",
+        "has extension zbb?",
+        "Zbb: Basic bit-manipulation",
+        false,
+    );
+    let _has_zbc = setting.add_bool(
+        "has_zbc",
+        "has extension zbc?",
+        "Zbc: Carry-less multiplication",
+        false,
+    );
+    let _has_zbx = setting.add_bool(
+        "has_zbs",
+        "has extension zbs?",
+        "Zbs: Single-bit instructions",
+        false,
+    );
 
-    let _has_zicsr = setting.add_bool("has_zicsr", "has extension zicsr?", "", false);
-    let _has_zifencei = setting.add_bool("has_zifencei", "has extension zifencei?", "", false);
+    let _has_zicsr = setting.add_bool(
+        "has_zicsr",
+        "has extension zicsr?",
+        "Zicsr: Control and Status Register (CSR) Instructions",
+        false,
+    );
+    let _has_zifencei = setting.add_bool(
+        "has_zifencei",
+        "has extension zifencei?",
+        "Zifencei: Instruction-Fetch Fence",
+        false,
+    );
 
     setting.build()
 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 10bcbd193..46fbd8f4c 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -213,6 +213,7 @@
     (BrTable
       (index Reg)
       (tmp1 WritableReg)
+      (tmp2 WritableReg)
       (targets VecBranchTarget))
 
     ;; atomic compare and set operation
@@ -530,39 +531,44 @@
   (RemU)
 
   ;; RV64M Standard Extension (in addition to RV32M)
-
   (Mulw)
   (Divw)
   (Divuw)
   (Remw)
   (Remuw)
 
-  ;; bitmapip
+  ;; Zba: Address Generation Instructions
   (Adduw)
+  (Sh1add)
+  (Sh1adduw)
+  (Sh2add)
+  (Sh2adduw)
+  (Sh3add)
+  (Sh3adduw)
+
+  ;; Zbb: Bit Manipulation Instructions
   (Andn)
-  (Bclr)
-  (Bext)
-  (Binv)
-  (Bset)
-  (Clmul)
-  (Clmulh)
-  (Clmulr)
+  (Orn)
+  (Xnor)
   (Max)
   (Maxu)
   (Min)
   (Minu)
-  (Orn)
   (Rol)
   (Rolw)
   (Ror)
   (Rorw)
-  (Sh1add)
-  (Sh1adduw)
-  (Sh2add)
-  (Sh2adduw)
-  (Sh3add)
-  (Sh3adduw)
-  (Xnor)
+
+  ;; Zbs: Single-bit instructions
+  (Bclr)
+  (Bext)
+  (Binv)
+  (Bset)
+
+  ;; Zbc: Carry-less multiplication
+  (Clmul)
+  (Clmulh)
+  (Clmulr)
 ))
 
 
@@ -600,6 +606,7 @@
 
 
 (type AluOPRRI (enum
+  ;; Base ISA
   (Addi)
   (Slti)
   (SltiU)
@@ -613,25 +620,31 @@
   (Slliw)
   (SrliW)
   (Sraiw)
-  (Bclri)
-  (Bexti)
-  (Binvi)
-  (Bseti)
-  (Rori)
-  (Roriw)
+
+  ;; Zba: Address Generation Instructions
   (SlliUw)
+
+  ;; Zbb: Bit Manipulation Instructions
   (Clz)
   (Clzw)
-  (Cpop)
-  (Cpopw)
   (Ctz)
   (Ctzw)
-  (Rev8)
+  (Cpop)
+  (Cpopw)
   (Sextb)
   (Sexth)
   (Zexth)
-  (Orcb)
+  (Rori)
+  (Roriw)
+  (Rev8)
   (Brev8)
+  (Orcb)
+
+  ;; Zbs: Single-bit instructions
+  (Bclri)
+  (Bexti)
+  (Binvi)
+  (Bseti)
 ))
 
 
@@ -694,6 +707,23 @@
 (type AMO (primitive AMO))
 (type VecMachLabel extern (enum))
 
+;; ISA Extension helpers
+
+(decl pure has_zbkb () bool)
+(extern constructor has_zbkb has_zbkb)
+
+(decl pure has_zba () bool)
+(extern constructor has_zba has_zba)
+
+(decl pure has_zbb () bool)
+(extern constructor has_zbb has_zbb)
+
+(decl pure has_zbc () bool)
+(extern constructor has_zbc has_zbc)
+
+(decl pure has_zbs () bool)
+(extern constructor has_zbs has_zbs)
+
 ;; Helper for creating the zero register.
 (decl zero_reg () Reg)
 (extern constructor zero_reg zero_reg)
@@ -907,22 +937,24 @@
 (decl lower_ctz (Type Reg) Reg)
 (rule
   (lower_ctz ty x)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (gen_cltz $false x ty))
 
 (rule 2
   (lower_ctz $I64 x)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rr_funct12 (AluOPRRI.Ctz) x))
 
 (rule 2
   (lower_ctz $I32 x)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rr_funct12 (AluOPRRI.Ctzw) x))
+
 ;;;; for I8 and I16
 (rule 1
   (lower_ctz ty x)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
+  (if-let $true (has_zbs))
   (let
     ((tmp Reg (alu_rr_imm12 (AluOPRRI.Bseti) x (imm12_const (ty_bits ty)))))
     (alu_rr_funct12 (AluOPRRI.Ctzw) x)))
@@ -953,21 +985,21 @@
 (decl lower_clz (Type Reg) Reg)
 (rule
   (lower_clz ty rs)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (gen_cltz $true rs ty))
 (rule 2
   (lower_clz $I64 r)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rr_funct12 (AluOPRRI.Clz) r))
 (rule 2
   (lower_clz $I32 r)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rr_funct12 (AluOPRRI.Clzw) r))
 
 ;;; for I8 and I16
 (rule 1
   (lower_clz ty r)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (let
     ( ;; narrow int make all upper bits are zeros.
       (tmp Reg (ext_int_if_need $false r ty ))
@@ -1077,30 +1109,26 @@
     (alu_rr_imm12 (AluOPRRI.Srli) tmp (imm12_const (ty_bits ty)))))
 
 
-;;; has extension B??
-(decl pure has_b () bool)
-(extern constructor has_b has_b)
-
 (decl lower_rotl (Type Reg Reg) Reg)
 
 (rule 1
   (lower_rotl $I64 rs amount)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rrr (AluOPRRR.Rol) rs amount))
 
 (rule
   (lower_rotl $I64 rs amount)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (lower_rotl_shift $I64 rs amount))
 
 (rule 1
   (lower_rotl $I32 rs amount)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rrr (AluOPRRR.Rolw) rs amount))
 
 (rule
   (lower_rotl $I32 rs amount)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (lower_rotl_shift $I32 rs amount))
 
 (rule -1
@@ -1135,21 +1163,21 @@
 
 (rule 1
   (lower_rotr $I64 rs amount)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rrr (AluOPRRR.Ror) rs amount))
 (rule
   (lower_rotr $I64 rs amount)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (lower_rotr_shift $I64 rs amount))
 
 (rule 1
   (lower_rotr $I32 rs amount)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rrr (AluOPRRR.Rorw) rs amount))
 
 (rule
   (lower_rotr $I32 rs amount)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (lower_rotr_shift $I32 rs amount))
 
 (rule -1
@@ -1207,10 +1235,10 @@
 
 (decl lower_popcnt (Reg Type) Reg)
 (rule 1 (lower_popcnt rs ty )
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rr_funct12 (AluOPRRI.Cpop) (ext_int_if_need $false rs ty)))
 (rule (lower_popcnt rs ty)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (gen_popcnt rs ty))
 
 (decl lower_popcnt_i128 (ValueRegs) ValueRegs)
@@ -1961,12 +1989,12 @@
 (decl gen_rev8 (Reg) Reg)
 (rule 1
   (gen_rev8 rs)
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (alu_rr_funct12 (AluOPRRI.Rev8) rs))
 
 (rule
   (gen_rev8 rs)
-  (if-let $false (has_b))
+  (if-let $false (has_zbb))
   (let
     ((rd WritableReg (temp_writable_reg $I64))
       (tmp WritableReg (temp_writable_reg $I64))
@@ -1974,11 +2002,6 @@
       (_ Unit (emit (MInst.Rev8 rs step tmp rd))))
     (writable_reg_to_reg rd)))
 
-(decl pure has_zbkb () bool)
-(extern constructor has_zbkb has_zbkb)
-
-(decl pure has_zbb () bool)
-(extern constructor has_zbb has_zbb)
 
 (decl gen_brev8 (Reg Type) Reg)
 (rule 1
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index 0bee43571..b85ee23b3 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -984,11 +984,13 @@ impl MachInstEmit for Inst {
             &Inst::BrTable {
                 index,
                 tmp1,
+                tmp2,
                 ref targets,
             } => {
                 let index = allocs.next(index);
                 let tmp1 = allocs.next_writable(tmp1);
-                let tmp2 = writable_spilltmp_reg();
+                let tmp2 = allocs.next_writable(tmp2);
+                let ext_index = writable_spilltmp_reg();
 
                 // The default target is passed in as the 0th element of `targets`
                 // separate it here for clarity.
@@ -1001,15 +1003,18 @@ impl MachInstEmit for Inst {
                 // that offset. Each jump table entry is a regular auipc+jalr which we emit sequentially.
                 //
                 // Build the following sequence:
+                //
+                // extend_index:
+                //     zext.w  ext_index, index
                 // bounds_check:
                 //     li      tmp, n_labels
-                //     bltu    index, tmp, compute_target
+                //     bltu    ext_index, tmp, compute_target
                 // jump_to_default_block:
                 //     auipc   pc, 0
                 //     jalr    zero, pc, default_block
                 // compute_target:
                 //     auipc   pc, 0
-                //     slli    tmp, index, 3
+                //     slli    tmp, ext_index, 3
                 //     add     pc, pc, tmp
                 //     jalr    zero, pc, 0x10
                 // jump_table:
@@ -1017,6 +1022,19 @@ impl MachInstEmit for Inst {
                 //     auipc   pc, 0
                 //     jalr    zero, pc, block_target
 
+                // Extend the index to 64 bits.
+                //
+                // This prevents us branching on the top 32 bits of the index, which
+                // are undefined.
+                Inst::Extend {
+                    rd: ext_index,
+                    rn: index,
+                    signed: false,
+                    from_bits: 32,
+                    to_bits: 64,
+                }
+                .emit(&[], sink, emit_info, state);
+
                 // Bounds check.
                 //
                 // Check if the index passed in is larger than the number of jumptable
@@ -1030,7 +1048,7 @@ impl MachInstEmit for Inst {
                     not_taken: BranchTarget::zero(),
                     kind: IntegerCompare {
                         kind: IntCC::UnsignedLessThan,
-                        rs1: index,
+                        rs1: ext_index.to_reg(),
                         rs2: tmp2.to_reg(),
                     },
                 }
@@ -1059,7 +1077,7 @@ impl MachInstEmit for Inst {
                 Inst::AluRRImm12 {
                     alu_op: AluOPRRI::Slli,
                     rd: tmp2,
-                    rs: index,
+                    rs: ext_index.to_reg(),
                     imm12: Imm12::from_bits(3),
                 }
                 .emit(&[], sink, emit_info, state);
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
index 350b69b8a..776af8cf8 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -338,9 +338,12 @@ fn riscv64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
     match inst {
         &Inst::Nop0 => {}
         &Inst::Nop4 => {}
-        &Inst::BrTable { index, tmp1, .. } => {
+        &Inst::BrTable {
+            index, tmp1, tmp2, ..
+        } => {
             collector.reg_use(index);
             collector.reg_early_def(tmp1);
+            collector.reg_early_def(tmp2);
         }
         &Inst::Auipc { rd, .. } => collector.reg_def(rd),
         &Inst::Lui { rd, .. } => collector.reg_def(rd),
@@ -1171,15 +1174,17 @@ impl Inst {
             &Inst::BrTable {
                 index,
                 tmp1,
+                tmp2,
                 ref targets,
             } => {
                 let targets: Vec<_> = targets.iter().map(|x| x.as_label().unwrap()).collect();
                 format!(
-                    "{} {},{}##tmp1={}",
+                    "{} {},{}##tmp1={},tmp2={}",
                     "br_table",
                     format_reg(index, allocs),
                     format_labels(&targets[..]),
                     format_reg(tmp1.to_reg(), allocs),
+                    format_reg(tmp2.to_reg(), allocs),
                 )
             }
             &Inst::Auipc { rd, imm } => {
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index fdaa7102c..9813c184f 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -209,19 +209,19 @@
 ;; forms early on.
 
 (rule 3 (lower (has_type (fits_in_64 ty) (band x (bnot y))))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (gen_andn x y))
 (rule 4 (lower (has_type (fits_in_64 ty) (band (bnot y) x)))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (gen_andn x y))
 (rule 5 (lower (has_type $I128 (band x (bnot y))))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (let
     ((low Reg (gen_andn (value_regs_get x 0) (value_regs_get y 0)))
       (high Reg (gen_andn (value_regs_get x 1) (value_regs_get y 1))))
     (value_regs low high)))
 (rule 6 (lower (has_type $I128 (band (bnot y) x)))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (let
     ((low Reg (gen_andn (value_regs_get x 0) (value_regs_get y 0)))
       (high Reg (gen_andn (value_regs_get x 1) (value_regs_get y 1))))
@@ -250,20 +250,20 @@
 ;; forms early on.
 
 (rule 3 (lower (has_type (fits_in_64 ty) (bor x (bnot y))))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (gen_orn x y))
 (rule 4 (lower (has_type (fits_in_64 ty) (bor (bnot y) x)))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (gen_orn x y))
 
 (rule 5 (lower (has_type $I128 (bor x (bnot y))))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (let
     ((low Reg (gen_orn (value_regs_get x 0) (value_regs_get y 0)))
       (high Reg (gen_orn (value_regs_get x 1) (value_regs_get y 1))))
     (value_regs low high)))
 (rule 6 (lower (has_type $I128 (bor (bnot y) x)))
-  (if-let $true (has_b))
+  (if-let $true (has_zbb))
   (let
     ((low Reg (gen_orn (value_regs_get x 0) (value_regs_get y 0)))
       (high Reg (gen_orn (value_regs_get x 1) (value_regs_get y 1))))
diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
index 343992cd8..368e5959a 100644
--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
@@ -279,16 +279,26 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Riscv64Backend> {
         ValueRegs::two(shamt, len_sub_shamt)
     }
 
-    fn has_b(&mut self) -> bool {
-        self.backend.isa_flags.has_b()
-    }
     fn has_zbkb(&mut self) -> bool {
         self.backend.isa_flags.has_zbkb()
     }
+
+    fn has_zba(&mut self) -> bool {
+        self.backend.isa_flags.has_zba()
+    }
+
     fn has_zbb(&mut self) -> bool {
         self.backend.isa_flags.has_zbb()
     }
 
+    fn has_zbc(&mut self) -> bool {
+        self.backend.isa_flags.has_zbc()
+    }
+
+    fn has_zbs(&mut self) -> bool {
+        self.backend.isa_flags.has_zbs()
+    }
+
     fn inst_output_get(&mut self, x: InstOutput, index: u8) -> ValueRegs {
         x[index as usize]
     }
@@ -378,6 +388,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Riscv64Backend> {
 
     fn lower_br_table(&mut self, index: Reg, targets: &VecMachLabel) -> Unit {
         let tmp1 = self.temp_writable_reg(I64);
+        let tmp2 = self.temp_writable_reg(I64);
         let targets: Vec<BranchTarget> = targets
             .into_iter()
             .copied()
@@ -386,6 +397,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Riscv64Backend> {
         self.emit(&MInst::BrTable {
             index,
             tmp1,
+            tmp2,
             targets,
         });
     }
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 3da3aa66c..954d0c364 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -162,7 +162,7 @@
        ;; Arithmetic SIMD shifts.
        (XmmRmiReg (opcode SseOpcode)
                   (src1 Xmm)
-                  (src2 XmmMemImm)
+                  (src2 XmmMemAlignedImm)
                   (dst WritableXmm))
 
        ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
@@ -193,7 +193,7 @@
        ;; XMM conditional move; overwrites the destination register.
        (XmmCmove (ty Type)
                  (cc CC)
-                 (consequent XmmMem)
+                 (consequent XmmMemAligned)
                  (alternative Xmm)
                  (dst WritableXmm))
 
@@ -218,9 +218,15 @@
        ;; (32 64) (reg addr) reg
        (XmmRmR (op SseOpcode)
                (src1 Xmm)
-               (src2 XmmMem)
+               (src2 XmmMemAligned)
                (dst WritableXmm))
 
+       ;; Same as `XmmRmR` except the memory operand can be unaligned
+       (XmmRmRUnaligned (op SseOpcode)
+                        (src1 Xmm)
+                        (src2 XmmMem)
+                        (dst WritableXmm))
+
        ;; XMM (scalar or vector) production of a constant value by operating
        ;; on a register with itself.
        ;;
@@ -235,7 +241,7 @@
        (XmmRmRBlend
          (op SseOpcode)
          (src1 Xmm)
-         (src2 XmmMem)
+         (src2 XmmMemAligned)
          (mask Xmm)
          (dst WritableXmm))
 
@@ -300,16 +306,22 @@
        ;; not have to be a previously valid value. This is characteristic of mov
        ;; instructions.
        (XmmUnaryRmR (op SseOpcode)
-                    (src XmmMem)
+                    (src XmmMemAligned)
                     (dst WritableXmm))
 
+       ;; Same as `XmmUnaryRmR` but used for opcodes where the memory address
+       ;; can be unaligned.
+       (XmmUnaryRmRUnaligned (op SseOpcode)
+                             (src XmmMem)
+                             (dst WritableXmm))
+
        ;; XMM (scalar or vector) unary op with immediate: roundss, roundsd, etc.
        ;;
        ;; This differs from XMM_RM_R_IMM in that the dst register of
        ;; XmmUnaryRmRImm is not used in the computation of the instruction dst
        ;; value and so does not have to be a previously valid value.
        (XmmUnaryRmRImm (op SseOpcode)
-                       (src XmmMem)
+                       (src XmmMemAligned)
                        (imm u8)
                        (dst WritableXmm))
 
@@ -331,6 +343,12 @@
                  (dst WritableGpr)
                  (dst_size OperandSize))
 
+       ;; XMM (scalar) unary op (from xmm to integer reg): pextr{w,b,d,q}
+       (XmmToGprImm (op SseOpcode)
+                    (src Xmm)
+                    (dst WritableGpr)
+                    (imm u8))
+
        ;; XMM (scalar) unary op (from integer to float reg): movd, movq,
        ;; cvtsi2s{s,d}
        (GprToXmm (op SseOpcode)
@@ -374,7 +392,7 @@
 
        ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
        (XmmCmpRmR (op SseOpcode)
-                  (src XmmMem)
+                  (src XmmMemAligned)
                   (dst Xmm))
 
        ;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm
@@ -749,6 +767,7 @@
             Pextrb
             Pextrw
             Pextrd
+            Pextrq
             Pinsrb
             Pinsrw
             Pinsrd
@@ -1180,6 +1199,18 @@
             Vfmadd213sd
             Vfmadd213ps
             Vfmadd213pd
+            Vfmadd132ss
+            Vfmadd132sd
+            Vfmadd132ps
+            Vfmadd132pd
+            Vfnmadd213ss
+            Vfnmadd213sd
+            Vfnmadd213ps
+            Vfnmadd213pd
+            Vfnmadd132ss
+            Vfnmadd132sd
+            Vfnmadd132ps
+            Vfnmadd132pd
             Vcmpps
             Vcmppd
             Vpsrlw
@@ -1327,7 +1358,9 @@
 (type WritableXmm (primitive WritableXmm))
 (type OptionWritableXmm (primitive OptionWritableXmm))
 (type XmmMem extern (enum))
+(type XmmMemAligned extern (enum))
 (type XmmMemImm extern (enum))
+(type XmmMemAlignedImm extern (enum))
 
 ;; Convert an `Imm8Reg` into an `Imm8Gpr`.
 (decl imm8_reg_to_imm8_gpr (Imm8Reg) Imm8Gpr)
@@ -1377,6 +1410,25 @@
 (decl xmm_mem_to_xmm_mem_imm (XmmMem) XmmMemImm)
 (extern constructor xmm_mem_to_xmm_mem_imm xmm_mem_to_xmm_mem_imm)
 
+;; Convert an `XmmMem` into an `XmmMemAligned`.
+;;
+;; Note that this is an infallible conversion, not a fallible one. If the
+;; original `XmmMem` source is a register, then it's passed through directly.
+;; If it's `Mem` and refers to aligned memory, it's also passed through
+;; directly. Otherwise, though, it's a memory source which is not aligned to
+;; 16 bytes so a load is performed and the temporary register which is the
+;; result of the load is passed through. The end-result is that the return value
+;; here is guaranteed to be a register or an aligned memory location.
+(decl xmm_mem_to_xmm_mem_aligned (XmmMem) XmmMemAligned)
+(extern constructor xmm_mem_to_xmm_mem_aligned xmm_mem_to_xmm_mem_aligned)
+
+;; Convert an `XmmMemImm` into an `XmmMemImmAligned`.
+;;
+;; Note that this is the same as `xmm_mem_to_xmm_mem_aligned` except it handles
+;; an immediate case as well.
+(decl xmm_mem_imm_to_xmm_mem_aligned_imm (XmmMemImm) XmmMemAlignedImm)
+(extern constructor xmm_mem_imm_to_xmm_mem_aligned_imm xmm_mem_imm_to_xmm_mem_aligned_imm)
+
 ;; Allocate a new temporary GPR register.
 (decl temp_writable_gpr () WritableGpr)
 (extern constructor temp_writable_gpr temp_writable_gpr)
@@ -1583,8 +1635,8 @@
 (decl use_popcnt (bool) Type)
 (extern extractor infallible use_popcnt use_popcnt)
 
-(decl use_fma (bool) Type)
-(extern extractor infallible use_fma use_fma)
+(decl pure use_fma () bool)
+(extern constructor use_fma use_fma)
 
 (decl use_sse41 (bool) Type)
 (extern extractor infallible use_sse41 use_sse41)
@@ -1794,23 +1846,19 @@
         dst))
 
 (rule 2 (x64_load $F32 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movss)
-                      addr))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movss) addr))
 
 (rule 2 (x64_load $F64 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movsd)
-                      addr))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movsd) addr))
 
 (rule 2 (x64_load $F32X4 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movups)
-                      addr))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movups) addr))
 
 (rule 2 (x64_load $F64X2 addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movupd)
-                      addr))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movupd) addr))
 
 (rule 0 (x64_load (multi_lane _bits _lanes) addr _ext_kind)
-      (xmm_unary_rm_r (SseOpcode.Movdqu) addr))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movdqu) addr))
 
 (decl x64_mov (Amode) Reg)
 (rule (x64_mov addr)
@@ -1832,19 +1880,19 @@
 
 (decl x64_movss_load (XmmMem) Xmm)
 (rule (x64_movss_load from)
-      (xmm_unary_rm_r (SseOpcode.Movss) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movss) from))
 
 (decl x64_movsd_load (XmmMem) Xmm)
 (rule (x64_movsd_load from)
-      (xmm_unary_rm_r (SseOpcode.Movsd) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movsd) from))
 
 (decl x64_movups (XmmMem) Xmm)
 (rule (x64_movups from)
-      (xmm_unary_rm_r (SseOpcode.Movups) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movups) from))
 
 (decl x64_movupd (XmmMem) Xmm)
 (rule (x64_movupd from)
-      (xmm_unary_rm_r (SseOpcode.Movupd) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movupd) from))
 
 (decl x64_movd (Xmm) Gpr)
 (rule (x64_movd from)
@@ -1852,7 +1900,7 @@
 
 (decl x64_movdqu (XmmMem) Xmm)
 (rule (x64_movdqu from)
-      (xmm_unary_rm_r (SseOpcode.Movdqu) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Movdqu) from))
 
 (decl x64_movapd (XmmMem) Xmm)
 (rule (x64_movapd src)
@@ -1860,27 +1908,27 @@
 
 (decl x64_pmovsxbw (XmmMem) Xmm)
 (rule (x64_pmovsxbw from)
-      (xmm_unary_rm_r (SseOpcode.Pmovsxbw) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Pmovsxbw) from))
 
 (decl x64_pmovzxbw (XmmMem) Xmm)
 (rule (x64_pmovzxbw from)
-      (xmm_unary_rm_r (SseOpcode.Pmovzxbw) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Pmovzxbw) from))
 
 (decl x64_pmovsxwd (XmmMem) Xmm)
 (rule (x64_pmovsxwd from)
-      (xmm_unary_rm_r (SseOpcode.Pmovsxwd) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Pmovsxwd) from))
 
 (decl x64_pmovzxwd (XmmMem) Xmm)
 (rule (x64_pmovzxwd from)
-      (xmm_unary_rm_r (SseOpcode.Pmovzxwd) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Pmovzxwd) from))
 
 (decl x64_pmovsxdq (XmmMem) Xmm)
 (rule (x64_pmovsxdq from)
-      (xmm_unary_rm_r (SseOpcode.Pmovsxdq) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Pmovsxdq) from))
 
 (decl x64_pmovzxdq (XmmMem) Xmm)
 (rule (x64_pmovzxdq from)
-      (xmm_unary_rm_r (SseOpcode.Pmovzxdq) from))
+      (xmm_unary_rm_r_unaligned (SseOpcode.Pmovzxdq) from))
 
 (decl x64_movrm (Type SyntheticAmode Gpr) SideEffectNoResult)
 (rule (x64_movrm ty addr data)
@@ -2171,7 +2219,7 @@
       (cmp_rmi_r size (CmpOpcode.Cmp) (RegMemImm.Imm src1) src2))
 
 ;; Helper for creating `MInst.XmmCmpRmR` instructions.
-(decl xmm_cmp_rm_r (SseOpcode XmmMem Xmm) ProducesFlags)
+(decl xmm_cmp_rm_r (SseOpcode XmmMemAligned Xmm) ProducesFlags)
 (rule (xmm_cmp_rm_r opcode src1 src2)
       (ProducesFlags.ProducesFlagsSideEffect
        (MInst.XmmCmpRmR opcode src1 src2)))
@@ -2206,7 +2254,7 @@
          (MInst.Cmove size cc consequent alternative dst)
          dst)))
 
-(decl cmove_xmm (Type CC XmmMem Xmm) ConsumesFlags)
+(decl cmove_xmm (Type CC XmmMemAligned Xmm) ConsumesFlags)
 (rule (cmove_xmm ty cc consequent alternative)
       (let ((dst WritableXmm (temp_writable_xmm)))
         (ConsumesFlags.ConsumesFlagsReturnsReg
@@ -2259,7 +2307,7 @@
          cmove2
          dst)))
 
-(decl cmove_or_xmm (Type CC CC XmmMem Xmm) ConsumesFlags)
+(decl cmove_or_xmm (Type CC CC XmmMemAligned Xmm) ConsumesFlags)
 (rule (cmove_or_xmm ty cc1 cc2 consequent alternative)
       (let ((dst WritableXmm (temp_writable_xmm))
             (tmp WritableXmm (temp_writable_xmm))
@@ -2317,12 +2365,19 @@
          dst)))
 
 ;; Helper for creating `MInst.XmmRmR` instructions.
-(decl xmm_rm_r (SseOpcode Xmm XmmMem) Xmm)
+(decl xmm_rm_r (SseOpcode Xmm XmmMemAligned) Xmm)
 (rule (xmm_rm_r op src1 src2)
       (let ((dst WritableXmm (temp_writable_xmm))
             (_ Unit (emit (MInst.XmmRmR op src1 src2 dst))))
         dst))
 
+;; Helper for creating `MInst.XmmRmRUnaligned` instructions.
+(decl xmm_rm_r_unaligned (SseOpcode Xmm XmmMem) Xmm)
+(rule (xmm_rm_r_unaligned op src1 src2)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmRmRUnaligned op src1 src2 dst))))
+        dst))
+
 ;; Helper for creating `paddb` instructions.
 (decl x64_paddb (Xmm XmmMem) Xmm)
 (rule 0 (x64_paddb src1 src2)
@@ -2646,12 +2701,12 @@
 ;; Helper for creating `addss` instructions.
 (decl x64_addss (Xmm XmmMem) Xmm)
 (rule (x64_addss src1 src2)
-      (xmm_rm_r (SseOpcode.Addss) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Addss) src1 src2))
 
 ;; Helper for creating `addsd` instructions.
 (decl x64_addsd (Xmm XmmMem) Xmm)
 (rule (x64_addsd src1 src2)
-      (xmm_rm_r (SseOpcode.Addsd) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Addsd) src1 src2))
 
 ;; Helper for creating `addps` instructions.
 (decl x64_addps (Xmm XmmMem) Xmm)
@@ -2672,12 +2727,12 @@
 ;; Helper for creating `subss` instructions.
 (decl x64_subss (Xmm XmmMem) Xmm)
 (rule (x64_subss src1 src2)
-      (xmm_rm_r (SseOpcode.Subss) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Subss) src1 src2))
 
 ;; Helper for creating `subsd` instructions.
 (decl x64_subsd (Xmm XmmMem) Xmm)
 (rule (x64_subsd src1 src2)
-      (xmm_rm_r (SseOpcode.Subsd) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Subsd) src1 src2))
 
 ;; Helper for creating `subps` instructions.
 (decl x64_subps (Xmm XmmMem) Xmm)
@@ -2698,12 +2753,12 @@
 ;; Helper for creating `mulss` instructions.
 (decl x64_mulss (Xmm XmmMem) Xmm)
 (rule (x64_mulss src1 src2)
-      (xmm_rm_r (SseOpcode.Mulss) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Mulss) src1 src2))
 
 ;; Helper for creating `mulsd` instructions.
 (decl x64_mulsd (Xmm XmmMem) Xmm)
 (rule (x64_mulsd src1 src2)
-      (xmm_rm_r (SseOpcode.Mulsd) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Mulsd) src1 src2))
 
 ;; Helper for creating `mulps` instructions.
 (decl x64_mulps (Xmm XmmMem) Xmm)
@@ -2724,12 +2779,12 @@
 ;; Helper for creating `divss` instructions.
 (decl x64_divss (Xmm XmmMem) Xmm)
 (rule (x64_divss src1 src2)
-      (xmm_rm_r (SseOpcode.Divss) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Divss) src1 src2))
 
 ;; Helper for creating `divsd` instructions.
 (decl x64_divsd (Xmm XmmMem) Xmm)
 (rule (x64_divsd src1 src2)
-      (xmm_rm_r (SseOpcode.Divsd) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Divsd) src1 src2))
 
 ;; Helper for creating `divps` instructions.
 (decl x64_divps (Xmm XmmMem) Xmm)
@@ -2748,7 +2803,7 @@
       (xmm_rmir_vex (AvxOpcode.Vdivpd) src1 src2))
 
 ;; Helper for creating `XmmRmRBlend` instructions
-(decl xmm_rm_r_blend (SseOpcode Xmm XmmMem Xmm) Xmm)
+(decl xmm_rm_r_blend (SseOpcode Xmm XmmMemAligned Xmm) Xmm)
 (rule (xmm_rm_r_blend op src1 src2 mask)
       (let ((dst WritableXmm (temp_writable_xmm))
             (_ Unit (emit (MInst.XmmRmRBlend op src1 src2 mask dst))))
@@ -2794,7 +2849,7 @@
 ;; Helper for creating `movsd` instructions.
 (decl x64_movsd_regmove (Xmm XmmMem) Xmm)
 (rule (x64_movsd_regmove src1 src2)
-      (xmm_rm_r (SseOpcode.Movsd) src1 src2))
+      (xmm_rm_r_unaligned (SseOpcode.Movsd) src1 src2))
 
 ;; Helper for creating `movlhps` instructions.
 (decl x64_movlhps (Xmm XmmMem) Xmm)
@@ -3056,7 +3111,7 @@
         dst))
 
 ;; Helper for constructing `XmmUnaryRmRImm` instructions.
-(decl xmm_unary_rm_r_imm (SseOpcode XmmMem u8) Xmm)
+(decl xmm_unary_rm_r_imm (SseOpcode XmmMemAligned u8) Xmm)
 (rule (xmm_unary_rm_r_imm op src1 imm)
       (let ((dst WritableXmm (temp_writable_xmm))
             (_ Unit (emit (MInst.XmmUnaryRmRImm op src1 imm dst))))
@@ -3110,16 +3165,9 @@
       (xmm_rmr_imm_vex (AvxOpcode.Vinsertps) src1 src2 lane))
 
 ;; Helper for creating `pshufd` instructions.
-(decl x64_pshufd (XmmMem u8 OperandSize) Xmm)
-(rule (x64_pshufd src imm size)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd)
-                                           dst
-                                           src
-                                           dst
-                                           imm
-                                           size))))
-        dst))
+(decl x64_pshufd (XmmMem u8) Xmm)
+(rule (x64_pshufd src imm)
+      (xmm_unary_rm_r_imm (SseOpcode.Pshufd) src imm))
 
 ;; Helper for creating `pshufb` instructions.
 (decl x64_pshufb (Xmm XmmMem) Xmm)
@@ -3142,12 +3190,19 @@
       (xmm_rmr_imm_vex (AvxOpcode.Vshufps) src1 src2 byte))
 
 ;; Helper for creating `MInst.XmmUnaryRmR` instructions.
-(decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm)
+(decl xmm_unary_rm_r (SseOpcode XmmMemAligned) Xmm)
 (rule (xmm_unary_rm_r op src)
       (let ((dst WritableXmm (temp_writable_xmm))
             (_ Unit (emit (MInst.XmmUnaryRmR op src dst))))
         dst))
 
+;; Helper for creating `MInst.XmmUnaryRmRUnaligned` instructions.
+(decl xmm_unary_rm_r_unaligned (SseOpcode XmmMem) Xmm)
+(rule (xmm_unary_rm_r_unaligned op src)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmUnaryRmRUnaligned op src dst))))
+        dst))
+
 ;; Helper for creating `pabsb` instructions.
 (decl x64_pabsb (XmmMem) Xmm)
 (rule (x64_pabsb src)
@@ -3240,7 +3295,7 @@
       (mul_hi ty $false src1 src2))
 
 ;; Helper for creating `MInst.XmmRmiXmm` instructions.
-(decl xmm_rmi_xmm (SseOpcode Xmm XmmMemImm) Xmm)
+(decl xmm_rmi_xmm (SseOpcode Xmm XmmMemAlignedImm) Xmm)
 (rule (xmm_rmi_xmm op src1 src2)
       (let ((dst WritableXmm (temp_writable_xmm))
             (_ Unit (emit (MInst.XmmRmiReg op
@@ -3314,40 +3369,24 @@
       (xmm_rmir_vex (AvxOpcode.Vpsrad) src1 src2))
 
 ;; Helper for creating `pextrb` instructions.
-(decl x64_pextrb (Type Xmm u8) Gpr)
-(rule (x64_pextrb ty src lane)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrb)
-                                           dst
-                                           src
-                                           dst
-                                           lane
-                                           (operand_size_of_type_32_64 (lane_type ty))))))
-        dst))
+(decl x64_pextrb (Xmm u8) Gpr)
+(rule (x64_pextrb src lane)
+      (xmm_to_gpr_imm (SseOpcode.Pextrb) src lane))
 
 ;; Helper for creating `pextrw` instructions.
-(decl x64_pextrw (Type Xmm u8) Gpr)
-(rule (x64_pextrw ty src lane)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrw)
-                                           dst
-                                           src
-                                           dst
-                                           lane
-                                           (operand_size_of_type_32_64 (lane_type ty))))))
-        dst))
+(decl x64_pextrw (Xmm u8) Gpr)
+(rule (x64_pextrw src lane)
+      (xmm_to_gpr_imm (SseOpcode.Pextrw) src lane))
 
 ;; Helper for creating `pextrd` instructions.
-(decl x64_pextrd (Type Xmm u8) Gpr)
-(rule (x64_pextrd ty src lane)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd)
-                                           dst
-                                           src
-                                           dst
-                                           lane
-                                           (operand_size_of_type_32_64 (lane_type ty))))))
-        dst))
+(decl x64_pextrd (Xmm u8) Gpr)
+(rule (x64_pextrd src lane)
+      (xmm_to_gpr_imm (SseOpcode.Pextrd) src lane))
+
+;; Helper for creating `pextrq` instructions.
+(decl x64_pextrq (Xmm u8) Gpr)
+(rule (x64_pextrq src lane)
+      (xmm_to_gpr_imm (SseOpcode.Pextrq) src lane))
 
 ;; Helper for creating `MInst.XmmToGpr` instructions.
 (decl xmm_to_gpr (SseOpcode Xmm OperandSize) Gpr)
@@ -3356,6 +3395,13 @@
             (_ Unit (emit (MInst.XmmToGpr op src dst size))))
         dst))
 
+;; Helper for creating `MInst.XmmToGpr` instructions.
+(decl xmm_to_gpr_imm (SseOpcode Xmm u8) Gpr)
+(rule (xmm_to_gpr_imm op src imm)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (_ Unit (emit (MInst.XmmToGprImm op src dst imm))))
+        dst))
+
 ;; Helper for creating `pmovmskb` instructions.
 (decl x64_pmovmskb (OperandSize Xmm) Gpr)
 (rule (x64_pmovmskb size src)
@@ -3491,22 +3537,17 @@
         dst))
 
 ;; Helper for creating `minss` instructions.
-(decl x64_minss (Xmm Xmm) Xmm)
+(decl x64_minss (Xmm XmmMem) Xmm)
 (rule (x64_minss x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Minss) x y dst))))
-        dst))
+      (xmm_rm_r_unaligned (SseOpcode.Minss) x y))
 
 ;; Helper for creating `minsd` instructions.
-(decl x64_minsd (Xmm Xmm) Xmm)
+(decl x64_minsd (Xmm XmmMem) Xmm)
 (rule (x64_minsd x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Minsd) x y dst))))
-        dst))
-
+      (xmm_rm_r_unaligned (SseOpcode.Minsd) x y))
 
 ;; Helper for creating `minps` instructions.
-(decl x64_minps (Xmm Xmm) Xmm)
+(decl x64_minps (Xmm XmmMem) Xmm)
 (rule 0 (x64_minps x y)
       (xmm_rm_r (SseOpcode.Minps) x y))
 (rule 1 (x64_minps x y)
@@ -3514,7 +3555,7 @@
       (xmm_rmir_vex (AvxOpcode.Vminps) x y))
 
 ;; Helper for creating `minpd` instructions.
-(decl x64_minpd (Xmm Xmm) Xmm)
+(decl x64_minpd (Xmm XmmMem) Xmm)
 (rule 0 (x64_minpd x y)
       (xmm_rm_r (SseOpcode.Minpd) x y))
 (rule 1 (x64_minpd x y)
@@ -3522,17 +3563,17 @@
       (xmm_rmir_vex (AvxOpcode.Vminpd) x y))
 
 ;; Helper for creating `maxss` instructions.
-(decl x64_maxss (Xmm Xmm) Xmm)
+(decl x64_maxss (Xmm XmmMem) Xmm)
 (rule (x64_maxss x y)
-      (xmm_rm_r (SseOpcode.Maxss) x y))
+      (xmm_rm_r_unaligned (SseOpcode.Maxss) x y))
 
 ;; Helper for creating `maxsd` instructions.
-(decl x64_maxsd (Xmm Xmm) Xmm)
+(decl x64_maxsd (Xmm XmmMem) Xmm)
 (rule (x64_maxsd x y)
-      (xmm_rm_r (SseOpcode.Maxsd) x y))
+      (xmm_rm_r_unaligned (SseOpcode.Maxsd) x y))
 
 ;; Helper for creating `maxps` instructions.
-(decl x64_maxps (Xmm Xmm) Xmm)
+(decl x64_maxps (Xmm XmmMem) Xmm)
 (rule 0 (x64_maxps x y)
       (xmm_rm_r (SseOpcode.Maxps) x y))
 (rule 1 (x64_maxps x y)
@@ -3540,7 +3581,7 @@
       (xmm_rmir_vex (AvxOpcode.Vmaxps) x y))
 
 ;; Helper for creating `maxpd` instructions.
-(decl x64_maxpd (Xmm Xmm) Xmm)
+(decl x64_maxpd (Xmm XmmMem) Xmm)
 (rule 0 (x64_maxpd x y)
       (xmm_rm_r (SseOpcode.Maxpd) x y))
 (rule 1 (x64_maxpd x y)
@@ -3569,49 +3610,48 @@
             (_ Unit (emit (MInst.XmmRmRVex3 op src1 src2 src3 dst))))
         dst))
 
-;; Helper for creating `vfmadd213ss` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213ss (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213ss x y z)
-      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213ss) x y z))
-
-;; Helper for creating `vfmadd213sd` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213sd (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213sd x y z)
-      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213sd) x y z))
-
-;; Helper for creating `vfmadd213ps` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213ps (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213ps x y z)
-      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213ps) x y z))
-
-;; Helper for creating `vfmadd213pd` instructions.
-; TODO: This should have the (Xmm Xmm XmmMem) signature
-; but we don't support VEX memory encodings yet
-(decl x64_vfmadd213pd (Xmm Xmm Xmm) Xmm)
-(rule (x64_vfmadd213pd x y z)
-      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213pd) x y z))
-
+;; Helper for creating `vfmadd213*` instructions
+(decl x64_vfmadd213 (Type Xmm Xmm XmmMem) Xmm)
+(rule (x64_vfmadd213 $F32 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd213ss) a b c))
+(rule (x64_vfmadd213 $F64 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd213sd) a b c))
+(rule (x64_vfmadd213 $F32X4 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd213ps) a b c))
+(rule (x64_vfmadd213 $F64X2 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd213pd) a b c))
+
+;; Helper for creating `vfmadd132*` instructions
+(decl x64_vfmadd132 (Type Xmm Xmm XmmMem) Xmm)
+(rule (x64_vfmadd132 $F32 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd132ss) a b c))
+(rule (x64_vfmadd132 $F64 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd132sd) a b c))
+(rule (x64_vfmadd132 $F32X4 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd132ps) a b c))
+(rule (x64_vfmadd132 $F64X2 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfmadd132pd) a b c))
+
+;; Helper for creating `vfnmadd213*` instructions
+(decl x64_vfnmadd213 (Type Xmm Xmm XmmMem) Xmm)
+(rule (x64_vfnmadd213 $F32 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd213ss) a b c))
+(rule (x64_vfnmadd213 $F64 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd213sd) a b c))
+(rule (x64_vfnmadd213 $F32X4 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd213ps) a b c))
+(rule (x64_vfnmadd213 $F64X2 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd213pd) a b c))
+
+;; Helper for creating `vfnmadd132*` instructions
+(decl x64_vfnmadd132 (Type Xmm Xmm XmmMem) Xmm)
+(rule (x64_vfnmadd132 $F32 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd132ss) a b c))
+(rule (x64_vfnmadd132 $F64 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd132sd) a b c))
+(rule (x64_vfnmadd132 $F32X4 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd132ps) a b c))
+(rule (x64_vfnmadd132 $F64X2 a b c) (xmm_rmr_vex3 (AvxOpcode.Vfnmadd132pd) a b c))
 
 ;; Helper for creating `sqrtss` instructions.
-(decl x64_sqrtss (Xmm) Xmm)
-(rule (x64_sqrtss x) (xmm_unary_rm_r (SseOpcode.Sqrtss) x))
+(decl x64_sqrtss (XmmMem) Xmm)
+(rule (x64_sqrtss x) (xmm_unary_rm_r_unaligned (SseOpcode.Sqrtss) x))
 
 ;; Helper for creating `sqrtsd` instructions.
-(decl x64_sqrtsd (Xmm) Xmm)
-(rule (x64_sqrtsd x) (xmm_unary_rm_r (SseOpcode.Sqrtsd) x))
+(decl x64_sqrtsd (XmmMem) Xmm)
+(rule (x64_sqrtsd x) (xmm_unary_rm_r_unaligned (SseOpcode.Sqrtsd) x))
 
 ;; Helper for creating `sqrtps` instructions.
-(decl x64_sqrtps (Xmm) Xmm)
+(decl x64_sqrtps (XmmMem) Xmm)
 (rule (x64_sqrtps x) (xmm_unary_rm_r (SseOpcode.Sqrtps) x))
 
 ;; Helper for creating `sqrtpd` instructions.
-(decl x64_sqrtpd (Xmm) Xmm)
+(decl x64_sqrtpd (XmmMem) Xmm)
 (rule (x64_sqrtpd x) (xmm_unary_rm_r (SseOpcode.Sqrtpd) x))
 
 ;; Helper for creating `cvtss2sd` instructions.
@@ -3623,19 +3663,19 @@
 (rule (x64_cvtsd2ss x) (xmm_unary_rm_r (SseOpcode.Cvtsd2ss) x))
 
 ;; Helper for creating `cvtdq2ps` instructions.
-(decl x64_cvtdq2ps (Xmm) Xmm)
+(decl x64_cvtdq2ps (XmmMem) Xmm)
 (rule (x64_cvtdq2ps x) (xmm_unary_rm_r (SseOpcode.Cvtdq2ps) x))
 
 ;; Helper for creating `cvtps2pd` instructions.
-(decl x64_cvtps2pd (Xmm) Xmm)
+(decl x64_cvtps2pd (XmmMem) Xmm)
 (rule (x64_cvtps2pd x) (xmm_unary_rm_r (SseOpcode.Cvtps2pd) x))
 
 ;; Helper for creating `cvtpd2ps` instructions.
-(decl x64_cvtpd2ps (Xmm) Xmm)
+(decl x64_cvtpd2ps (XmmMem) Xmm)
 (rule (x64_cvtpd2ps x) (xmm_unary_rm_r (SseOpcode.Cvtpd2ps) x))
 
 ;; Helper for creating `cvtdq2pd` instructions.
-(decl x64_cvtdq2pd (Xmm) Xmm)
+(decl x64_cvtdq2pd (XmmMem) Xmm)
 (rule (x64_cvtdq2pd x) (xmm_unary_rm_r (SseOpcode.Cvtdq2pd) x))
 
 ;; Helper for creating `cvtsi2ss` instructions.
@@ -3649,8 +3689,8 @@
       (gpr_to_xmm (SseOpcode.Cvtsi2sd) x (raw_operand_size_of_type ty)))
 
 ;; Helper for creating `cvttps2dq` instructions.
-(decl x64_cvttps2dq (Type XmmMem) Xmm)
-(rule (x64_cvttps2dq ty x)
+(decl x64_cvttps2dq (XmmMem) Xmm)
+(rule (x64_cvttps2dq x)
       (xmm_unary_rm_r (SseOpcode.Cvttps2dq) x))
 
 ;; Helper for creating `cvttpd2dq` instructions.
@@ -4271,6 +4311,7 @@
 (convert Xmm InstOutput output_xmm)
 (convert Value Xmm put_in_xmm)
 (convert Value XmmMem put_in_xmm_mem)
+(convert Value XmmMemAligned put_in_xmm_mem_aligned)
 (convert Value XmmMemImm put_in_xmm_mem_imm)
 (convert Xmm Reg xmm_to_reg)
 (convert Xmm RegMem xmm_to_reg_mem)
@@ -4281,6 +4322,7 @@
 (convert RegMemImm XmmMemImm mov_rmi_to_xmm)
 (convert Xmm XmmMem xmm_to_xmm_mem)
 (convert Xmm XmmMemImm xmm_to_xmm_mem_imm)
+(convert Xmm XmmMemAligned xmm_to_xmm_mem_aligned)
 (convert XmmMem XmmMemImm xmm_mem_to_xmm_mem_imm)
 (convert XmmMem RegMem xmm_mem_to_reg_mem)
 (convert WritableXmm Xmm writable_xmm_to_xmm)
@@ -4289,6 +4331,15 @@
 (convert WritableXmm XmmMem writable_xmm_to_xmm_mem)
 (convert WritableXmm ValueRegs writable_xmm_to_value_regs)
 
+;; Note that these conversions will introduce a `movupd` instruction if
+;; the memory location is not aligned to a 16-byte boundary. This is primarily
+;; used to convert `XmmMem` inputs, which themselves were typically created
+;; via the `put_in_xmm_mem` constructor, into operands of SSE instructions.
+;; Most pre-AVX instructions working with 16-bytes of data (e.g. full xmm
+;; registers) require 16-byte alignment.
+(convert XmmMem XmmMemAligned xmm_mem_to_xmm_mem_aligned)
+(convert XmmMemImm XmmMemAlignedImm xmm_mem_imm_to_xmm_mem_aligned_imm)
+
 (convert Gpr Imm8Gpr gpr_to_imm8_gpr)
 (convert Imm8Reg Imm8Gpr imm8_reg_to_imm8_gpr)
 
@@ -4297,10 +4348,16 @@
 (convert SyntheticAmode GprMem synthetic_amode_to_gpr_mem)
 (convert Amode XmmMem amode_to_xmm_mem)
 (convert SyntheticAmode XmmMem synthetic_amode_to_xmm_mem)
+(convert Amode XmmMemAligned amode_to_xmm_mem_aligned)
+(convert SyntheticAmode XmmMemAligned synthetic_amode_to_xmm_mem_aligned)
 
 (convert IntCC CC intcc_to_cc)
 (convert AtomicRmwOp MachAtomicRmwOp atomic_rmw_op_to_mach_atomic_rmw_op)
 
+(convert SinkableLoad RegMem sink_load)
+(convert SinkableLoad GprMemImm sink_load_to_gpr_mem_imm)
+(convert SinkableLoad XmmMem sink_load_to_xmm_mem)
+
 (decl reg_to_xmm_mem (Reg) XmmMem)
 (rule (reg_to_xmm_mem r)
       (xmm_to_xmm_mem (xmm_new r)))
@@ -4342,6 +4399,15 @@
 (decl const_to_synthetic_amode (VCodeConstant) SyntheticAmode)
 (extern constructor const_to_synthetic_amode const_to_synthetic_amode)
 
+(decl xmm_to_xmm_mem_aligned (Xmm) XmmMemAligned)
+(rule (xmm_to_xmm_mem_aligned reg) (xmm_mem_to_xmm_mem_aligned reg))
+(decl amode_to_xmm_mem_aligned (Amode) XmmMemAligned)
+(rule (amode_to_xmm_mem_aligned mode) (amode_to_xmm_mem mode))
+(decl synthetic_amode_to_xmm_mem_aligned (SyntheticAmode) XmmMemAligned)
+(rule (synthetic_amode_to_xmm_mem_aligned mode) (synthetic_amode_to_xmm_mem mode))
+(decl put_in_xmm_mem_aligned (Value) XmmMemAligned)
+(rule (put_in_xmm_mem_aligned val) (put_in_xmm_mem val))
+
 ;; Helper for creating `MovFromPReg` instructions.
 (decl mov_from_preg (PReg) Reg)
 (rule (mov_from_preg preg)
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 57f644c8e..b85fdfc20 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -31,8 +31,8 @@ macro_rules! newtype_of_reg {
         $newtype_reg:ident,
         $newtype_writable_reg:ident,
         $newtype_option_writable_reg:ident,
-        $newtype_reg_mem:ident,
-        $newtype_reg_mem_imm:ident,
+        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
+        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
         $newtype_imm8_reg:ident,
         |$check_reg:ident| $check:expr
     ) => {
@@ -102,108 +102,130 @@ macro_rules! newtype_of_reg {
             }
         }
 
-        /// A newtype wrapper around `RegMem` for general-purpose registers.
-        #[derive(Clone, Debug)]
-        pub struct $newtype_reg_mem(RegMem);
+        $(
+            /// A newtype wrapper around `RegMem` for general-purpose registers.
+            #[derive(Clone, Debug)]
+            pub struct $newtype_reg_mem(RegMem);
 
-        impl From<$newtype_reg_mem> for RegMem {
-            fn from(rm: $newtype_reg_mem) -> Self {
-                rm.0
+            impl From<$newtype_reg_mem> for RegMem {
+                fn from(rm: $newtype_reg_mem) -> Self {
+                    rm.0
+                }
             }
-        }
 
-        impl From<$newtype_reg> for $newtype_reg_mem {
-            fn from(r: $newtype_reg) -> Self {
-                $newtype_reg_mem(RegMem::reg(r.into()))
+            impl From<$newtype_reg> for $newtype_reg_mem {
+                fn from(r: $newtype_reg) -> Self {
+                    $newtype_reg_mem(RegMem::reg(r.into()))
+                }
             }
-        }
 
-        impl $newtype_reg_mem {
-            /// Construct a `RegMem` newtype from the given `RegMem`, or return
-            /// `None` if the `RegMem` is not a valid instance of this `RegMem`
-            /// newtype.
-            pub fn new(rm: RegMem) -> Option<Self> {
-                match rm {
-                    RegMem::Mem { addr: _ } => Some(Self(rm)),
-                    RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)),
-                    RegMem::Reg { reg: _ } => None,
+            impl $newtype_reg_mem {
+                /// Construct a `RegMem` newtype from the given `RegMem`, or return
+                /// `None` if the `RegMem` is not a valid instance of this `RegMem`
+                /// newtype.
+                pub fn new(rm: RegMem) -> Option<Self> {
+                    match rm {
+                        RegMem::Mem { addr } => {
+                            let mut _allow = true;
+                            $(
+                                if $aligned {
+                                    _allow = addr.aligned();
+                                }
+                            )?
+                            if _allow {
+                                Some(Self(RegMem::Mem { addr }))
+                            } else {
+                                None
+                            }
+                        }
+                        RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)),
+                        RegMem::Reg { reg: _ } => None,
+                    }
                 }
-            }
 
-            /// Convert this newtype into its underlying `RegMem`.
-            pub fn to_reg_mem(self) -> RegMem {
-                self.0
-            }
+                /// Convert this newtype into its underlying `RegMem`.
+                pub fn to_reg_mem(self) -> RegMem {
+                    self.0
+                }
 
-            #[allow(dead_code)] // Used by some newtypes and not others.
-            pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-                &self,
-                collector: &mut OperandCollector<'_, F>,
-            ) {
-                self.0.get_operands(collector);
+                #[allow(dead_code)] // Used by some newtypes and not others.
+                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
+                    &self,
+                    collector: &mut OperandCollector<'_, F>,
+                ) {
+                    self.0.get_operands(collector);
+                }
             }
-        }
-        impl PrettyPrint for $newtype_reg_mem {
-            fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
-                self.0.pretty_print(size, allocs)
+            impl PrettyPrint for $newtype_reg_mem {
+                fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
+                    self.0.pretty_print(size, allocs)
+                }
             }
-        }
+        )*
 
-        /// A newtype wrapper around `RegMemImm`.
-        #[derive(Clone, Debug)]
-        pub struct $newtype_reg_mem_imm(RegMemImm);
+        $(
+            /// A newtype wrapper around `RegMemImm`.
+            #[derive(Clone, Debug)]
+            pub struct $newtype_reg_mem_imm(RegMemImm);
 
-        impl From<$newtype_reg_mem_imm> for RegMemImm {
-            fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
-                rmi.0
-            }
-        }
-
-        impl From<$newtype_reg> for $newtype_reg_mem_imm {
-            fn from(r: $newtype_reg) -> Self {
-                $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
+            impl From<$newtype_reg_mem_imm> for RegMemImm {
+                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
+                    rmi.0
+                }
             }
-        }
 
-        impl From<$newtype_reg_mem> for $newtype_reg_mem_imm {
-            fn from(r: $newtype_reg_mem) -> Self {
-                $newtype_reg_mem_imm(r.0.into())
+            impl From<$newtype_reg> for $newtype_reg_mem_imm {
+                fn from(r: $newtype_reg) -> Self {
+                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
+                }
             }
-        }
 
-        impl $newtype_reg_mem_imm {
-            /// Construct this newtype from the given `RegMemImm`, or return
-            /// `None` if the `RegMemImm` is not a valid instance of this
-            /// newtype.
-            pub fn new(rmi: RegMemImm) -> Option<Self> {
-                match rmi {
-                    RegMemImm::Imm { .. } => Some(Self(rmi)),
-                    RegMemImm::Mem { addr: _ } => Some(Self(rmi)),
-                    RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)),
-                    RegMemImm::Reg { reg: _ } => None,
+            impl $newtype_reg_mem_imm {
+                /// Construct this newtype from the given `RegMemImm`, or return
+                /// `None` if the `RegMemImm` is not a valid instance of this
+                /// newtype.
+                pub fn new(rmi: RegMemImm) -> Option<Self> {
+                    match rmi {
+                        RegMemImm::Imm { .. } => Some(Self(rmi)),
+                        RegMemImm::Mem { addr } => {
+                            let mut _allow = true;
+                            $(
+                                if $aligned_imm {
+                                    _allow = addr.aligned();
+                                }
+                            )?
+                            if _allow {
+                                Some(Self(RegMemImm::Mem { addr }))
+                            } else {
+                                None
+                            }
+                        }
+                        RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)),
+                        RegMemImm::Reg { reg: _ } => None,
+                    }
                 }
-            }
 
-            /// Convert this newtype into its underlying `RegMemImm`.
-            #[allow(dead_code)] // Used by some newtypes and not others.
-            pub fn to_reg_mem_imm(self) -> RegMemImm {
-                self.0
-            }
+                /// Convert this newtype into its underlying `RegMemImm`.
+                #[allow(dead_code)] // Used by some newtypes and not others.
+                pub fn to_reg_mem_imm(self) -> RegMemImm {
+                    self.0
+                }
 
-            #[allow(dead_code)] // Used by some newtypes and not others.
-            pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-                &self,
-                collector: &mut OperandCollector<'_, F>,
-            ) {
-                self.0.get_operands(collector);
+                #[allow(dead_code)] // Used by some newtypes and not others.
+                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
+                    &self,
+                    collector: &mut OperandCollector<'_, F>,
+                ) {
+                    self.0.get_operands(collector);
+                }
             }
-        }
 
-        impl PrettyPrint for $newtype_reg_mem_imm {
-            fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
-                self.0.pretty_print(size, allocs)
+            impl PrettyPrint for $newtype_reg_mem_imm {
+                fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
+                    self.0.pretty_print(size, allocs)
+                }
             }
-        }
+        )*
 
         /// A newtype wrapper around `Imm8Reg`.
         #[derive(Clone, Debug)]
@@ -242,8 +264,8 @@ newtype_of_reg!(
     Gpr,
     WritableGpr,
     OptionWritableGpr,
-    GprMem,
-    GprMemImm,
+    reg_mem: (GprMem),
+    reg_mem_imm: (GprMemImm),
     Imm8Gpr,
     |reg| reg.class() == RegClass::Int
 );
@@ -253,8 +275,8 @@ newtype_of_reg!(
     Xmm,
     WritableXmm,
     OptionWritableXmm,
-    XmmMem,
-    XmmMemImm,
+    reg_mem: (XmmMem, XmmMemAligned aligned:true),
+    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
     Imm8Xmm,
     |reg| reg.class() == RegClass::Float
 );
@@ -420,6 +442,10 @@ impl Amode {
         }
         ret
     }
+
+    pub(crate) fn aligned(&self) -> bool {
+        self.get_flags().aligned()
+    }
 }
 
 impl PrettyPrint for Amode {
@@ -531,6 +557,13 @@ impl SyntheticAmode {
             }
         }
     }
+
+    pub(crate) fn aligned(&self) -> bool {
+        match self {
+            SyntheticAmode::Real(addr) => addr.aligned(),
+            SyntheticAmode::NominalSPOffset { .. } | SyntheticAmode::ConstantOffset { .. } => true,
+        }
+    }
 }
 
 impl Into<SyntheticAmode> for Amode {
@@ -999,6 +1032,7 @@ pub enum SseOpcode {
     Pextrb,
     Pextrw,
     Pextrd,
+    Pextrq,
     Pinsrb,
     Pinsrw,
     Pinsrd,
@@ -1237,6 +1271,7 @@ impl SseOpcode {
             | SseOpcode::Pcmpeqq
             | SseOpcode::Pextrb
             | SseOpcode::Pextrd
+            | SseOpcode::Pextrq
             | SseOpcode::Pinsrb
             | SseOpcode::Pinsrd
             | SseOpcode::Pmaxsb
@@ -1278,22 +1313,6 @@ impl SseOpcode {
             _ => 8,
         }
     }
-
-    /// Does an XmmRmmRImm with this opcode use src1? FIXME: split
-    /// into separate instructions.
-    pub(crate) fn uses_src1(&self) -> bool {
-        match self {
-            SseOpcode::Pextrb => false,
-            SseOpcode::Pextrw => false,
-            SseOpcode::Pextrd => false,
-            SseOpcode::Pshufd => false,
-            SseOpcode::Roundss => false,
-            SseOpcode::Roundsd => false,
-            SseOpcode::Roundps => false,
-            SseOpcode::Roundpd => false,
-            _ => true,
-        }
-    }
 }
 
 impl fmt::Debug for SseOpcode {
@@ -1393,6 +1412,7 @@ impl fmt::Debug for SseOpcode {
             SseOpcode::Pextrb => "pextrb",
             SseOpcode::Pextrw => "pextrw",
             SseOpcode::Pextrd => "pextrd",
+            SseOpcode::Pextrq => "pextrq",
             SseOpcode::Pinsrb => "pinsrb",
             SseOpcode::Pinsrw => "pinsrw",
             SseOpcode::Pinsrd => "pinsrd",
@@ -1495,7 +1515,19 @@ impl AvxOpcode {
             AvxOpcode::Vfmadd213ss
             | AvxOpcode::Vfmadd213sd
             | AvxOpcode::Vfmadd213ps
-            | AvxOpcode::Vfmadd213pd => smallvec![InstructionSet::FMA],
+            | AvxOpcode::Vfmadd213pd
+            | AvxOpcode::Vfmadd132ss
+            | AvxOpcode::Vfmadd132sd
+            | AvxOpcode::Vfmadd132ps
+            | AvxOpcode::Vfmadd132pd
+            | AvxOpcode::Vfnmadd213ss
+            | AvxOpcode::Vfnmadd213sd
+            | AvxOpcode::Vfnmadd213ps
+            | AvxOpcode::Vfnmadd213pd
+            | AvxOpcode::Vfnmadd132ss
+            | AvxOpcode::Vfnmadd132sd
+            | AvxOpcode::Vfnmadd132ps
+            | AvxOpcode::Vfnmadd132pd => smallvec![InstructionSet::FMA],
             AvxOpcode::Vminps
             | AvxOpcode::Vminpd
             | AvxOpcode::Vmaxps
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index ebbbf16b2..44de9450f 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1731,7 +1731,21 @@ pub(crate) fn emit(
             sink.bind_label(else_label);
         }
 
-        Inst::XmmUnaryRmR {
+        Inst::XmmUnaryRmR { op, src, dst } => {
+            emit(
+                &Inst::XmmUnaryRmRUnaligned {
+                    op: *op,
+                    src: XmmMem::new(src.clone().into()).unwrap(),
+                    dst: *dst,
+                },
+                allocs,
+                sink,
+                info,
+                state,
+            );
+        }
+
+        Inst::XmmUnaryRmRUnaligned {
             op,
             src: src_e,
             dst: reg_g,
@@ -1792,8 +1806,6 @@ pub(crate) fn emit(
         }
 
         Inst::XmmUnaryRmRImm { op, src, dst, imm } => {
-            debug_assert!(!op.uses_src1());
-
             let dst = allocs.next(dst.to_reg().to_reg());
             let src = src.clone().to_reg_mem().with_allocs(allocs);
             let rex = RexFlags::clear_w();
@@ -1803,6 +1815,7 @@ pub(crate) fn emit(
                 SseOpcode::Roundss => (LegacyPrefixes::_66, 0x0F3A0A, 3),
                 SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3),
                 SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3),
+                SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2),
                 _ => unimplemented!("Opcode {:?} not implemented", op),
             };
             match src {
@@ -1843,6 +1856,24 @@ pub(crate) fn emit(
         }
 
         Inst::XmmRmR {
+            op,
+            src1,
+            src2,
+            dst,
+        } => emit(
+            &Inst::XmmRmRUnaligned {
+                op: *op,
+                dst: *dst,
+                src1: *src1,
+                src2: XmmMem::new(src2.clone().to_reg_mem()).unwrap(),
+            },
+            allocs,
+            sink,
+            info,
+            state,
+        ),
+
+        Inst::XmmRmRUnaligned {
             op,
             src1,
             src2: src_e,
@@ -2250,32 +2281,46 @@ pub(crate) fn emit(
             let dst = allocs.next(dst.to_reg().to_reg());
             debug_assert_eq!(src1, dst);
             let src2 = allocs.next(src2.to_reg());
-            let src3 = src3.clone().to_reg_mem().with_allocs(allocs);
+            let src3 = match src3.clone().to_reg_mem().with_allocs(allocs) {
+                RegMem::Reg { reg } => {
+                    RegisterOrAmode::Register(reg.to_real_reg().unwrap().hw_enc().into())
+                }
+                RegMem::Mem { addr } => RegisterOrAmode::Amode(addr.finalize(state, sink)),
+            };
 
             let (w, map, opcode) = match op {
+                AvxOpcode::Vfmadd132ss => (false, OpcodeMap::_0F38, 0x99),
                 AvxOpcode::Vfmadd213ss => (false, OpcodeMap::_0F38, 0xA9),
+                AvxOpcode::Vfnmadd132ss => (false, OpcodeMap::_0F38, 0x9D),
+                AvxOpcode::Vfnmadd213ss => (false, OpcodeMap::_0F38, 0xAD),
+                AvxOpcode::Vfmadd132sd => (true, OpcodeMap::_0F38, 0x99),
                 AvxOpcode::Vfmadd213sd => (true, OpcodeMap::_0F38, 0xA9),
+                AvxOpcode::Vfnmadd132sd => (true, OpcodeMap::_0F38, 0x9D),
+                AvxOpcode::Vfnmadd213sd => (true, OpcodeMap::_0F38, 0xAD),
+                AvxOpcode::Vfmadd132ps => (false, OpcodeMap::_0F38, 0x98),
                 AvxOpcode::Vfmadd213ps => (false, OpcodeMap::_0F38, 0xA8),
+                AvxOpcode::Vfnmadd132ps => (false, OpcodeMap::_0F38, 0x9C),
+                AvxOpcode::Vfnmadd213ps => (false, OpcodeMap::_0F38, 0xAC),
+                AvxOpcode::Vfmadd132pd => (true, OpcodeMap::_0F38, 0x98),
                 AvxOpcode::Vfmadd213pd => (true, OpcodeMap::_0F38, 0xA8),
+                AvxOpcode::Vfnmadd132pd => (true, OpcodeMap::_0F38, 0x9C),
+                AvxOpcode::Vfnmadd213pd => (true, OpcodeMap::_0F38, 0xAC),
                 AvxOpcode::Vblendvps => (false, OpcodeMap::_0F3A, 0x4A),
                 AvxOpcode::Vblendvpd => (false, OpcodeMap::_0F3A, 0x4B),
                 AvxOpcode::Vpblendvb => (false, OpcodeMap::_0F3A, 0x4C),
                 _ => unreachable!(),
             };
 
-            match src3 {
-                RegMem::Reg { reg: src } => VexInstruction::new()
-                    .length(VexVectorLength::V128)
-                    .prefix(LegacyPrefixes::_66)
-                    .map(map)
-                    .w(w)
-                    .opcode(opcode)
-                    .reg(dst.to_real_reg().unwrap().hw_enc())
-                    .rm(src.to_real_reg().unwrap().hw_enc())
-                    .vvvv(src2.to_real_reg().unwrap().hw_enc())
-                    .encode(sink),
-                _ => todo!(),
-            };
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .prefix(LegacyPrefixes::_66)
+                .map(map)
+                .w(w)
+                .opcode(opcode)
+                .reg(dst.to_real_reg().unwrap().hw_enc())
+                .rm(src3)
+                .vvvv(src2.to_real_reg().unwrap().hw_enc())
+                .encode(sink);
         }
 
         Inst::XmmRmRBlendVex {
@@ -2458,17 +2503,10 @@ pub(crate) fn emit(
             imm,
             size,
         } => {
-            let (src2, dst) = if !op.uses_src1() {
-                let dst = allocs.next(dst.to_reg());
-                let src2 = src2.with_allocs(allocs);
-                (src2, dst)
-            } else {
-                let src1 = allocs.next(*src1);
-                let dst = allocs.next(dst.to_reg());
-                let src2 = src2.with_allocs(allocs);
-                debug_assert_eq!(src1, dst);
-                (src2, dst)
-            };
+            let src1 = allocs.next(*src1);
+            let dst = allocs.next(dst.to_reg());
+            let src2 = src2.with_allocs(allocs);
+            debug_assert_eq!(src1, dst);
 
             let (prefix, opcode, len) = match op {
                 SseOpcode::Cmpps => (LegacyPrefixes::None, 0x0FC2, 2),
@@ -2480,10 +2518,6 @@ pub(crate) fn emit(
                 SseOpcode::Pinsrb => (LegacyPrefixes::_66, 0x0F3A20, 3),
                 SseOpcode::Pinsrw => (LegacyPrefixes::_66, 0x0FC4, 2),
                 SseOpcode::Pinsrd => (LegacyPrefixes::_66, 0x0F3A22, 3),
-                SseOpcode::Pextrb => (LegacyPrefixes::_66, 0x0F3A14, 3),
-                SseOpcode::Pextrw => (LegacyPrefixes::_66, 0x0FC5, 2),
-                SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3),
-                SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2),
                 SseOpcode::Shufps => (LegacyPrefixes::None, 0x0FC6, 2),
                 _ => unimplemented!("Opcode {:?} not implemented", op),
             };
@@ -2566,6 +2600,26 @@ pub(crate) fn emit(
             emit_std_reg_reg(sink, prefix, opcode, 2, src, dst, rex);
         }
 
+        Inst::XmmToGprImm { op, src, dst, imm } => {
+            use OperandSize as OS;
+
+            let src = allocs.next(src.to_reg());
+            let dst = allocs.next(dst.to_reg().to_reg());
+
+            let (prefix, opcode, opcode_bytes, dst_size, dst_first) = match op {
+                SseOpcode::Pextrb => (LegacyPrefixes::_66, 0x0F3A14, 3, OS::Size32, false),
+                SseOpcode::Pextrw => (LegacyPrefixes::_66, 0x0FC5, 2, OS::Size32, true),
+                SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3, OS::Size32, false),
+                SseOpcode::Pextrq => (LegacyPrefixes::_66, 0x0F3A16, 3, OS::Size64, false),
+                _ => panic!("unexpected opcode {:?}", op),
+            };
+            let rex = RexFlags::from(dst_size);
+            let (src, dst) = if dst_first { (dst, src) } else { (src, dst) };
+
+            emit_std_reg_reg(sink, prefix, opcode, opcode_bytes, src, dst, rex);
+            sink.put1(*imm);
+        }
+
         Inst::GprToXmm {
             op,
             src: src_e,
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 217b4b4db..1a5ca8354 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -133,10 +133,13 @@ impl Inst {
             | Inst::XmmMovRM { op, .. }
             | Inst::XmmRmiReg { opcode: op, .. }
             | Inst::XmmRmR { op, .. }
+            | Inst::XmmRmRUnaligned { op, .. }
             | Inst::XmmRmRBlend { op, .. }
             | Inst::XmmRmRImm { op, .. }
             | Inst::XmmToGpr { op, .. }
+            | Inst::XmmToGprImm { op, .. }
             | Inst::XmmUnaryRmRImm { op, .. }
+            | Inst::XmmUnaryRmRUnaligned { op, .. }
             | Inst::XmmUnaryRmR { op, .. }
             | Inst::XmmConstOp { op, .. } => smallvec![op.available_from()],
 
@@ -292,7 +295,7 @@ impl Inst {
         debug_assert!(dst.to_reg().class() == RegClass::Float);
         Inst::XmmUnaryRmR {
             op,
-            src: XmmMem::new(src).unwrap(),
+            src: XmmMemAligned::new(src).unwrap(),
             dst: WritableXmm::from_writable_reg(dst).unwrap(),
         }
     }
@@ -303,7 +306,7 @@ impl Inst {
         Inst::XmmRmR {
             op,
             src1: Xmm::new(dst.to_reg()).unwrap(),
-            src2: XmmMem::new(src).unwrap(),
+            src2: XmmMemAligned::new(src).unwrap(),
             dst: WritableXmm::from_writable_reg(dst).unwrap(),
         }
     }
@@ -368,7 +371,7 @@ impl Inst {
     pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst {
         src.assert_regclass_is(RegClass::Float);
         debug_assert!(dst.class() == RegClass::Float);
-        let src = XmmMem::new(src).unwrap();
+        let src = XmmMemAligned::new(src).unwrap();
         let dst = Xmm::new(dst).unwrap();
         Inst::XmmCmpRmR { op, src, dst }
     }
@@ -893,6 +896,12 @@ impl PrettyPrint for Inst {
                 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
             }
 
+            Inst::XmmUnaryRmRUnaligned { op, src, dst, .. } => {
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size(), allocs);
+                let src = src.pretty_print(op.src_size(), allocs);
+                format!("{} {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
             Inst::XmmUnaryRmRImm {
                 op, src, dst, imm, ..
             } => {
@@ -926,6 +935,19 @@ impl PrettyPrint for Inst {
                 format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
             }
 
+            Inst::XmmRmRUnaligned {
+                op,
+                src1,
+                src2,
+                dst,
+                ..
+            } => {
+                let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
+                let src2 = src2.pretty_print(8, allocs);
+                format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
+            }
+
             Inst::XmmConstOp { op, dst } => {
                 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
                 format!("{} {dst}, {dst}, {dst}", ljustify(op.to_string()))
@@ -1111,15 +1133,11 @@ impl PrettyPrint for Inst {
                 size,
                 ..
             } => {
-                let src1 = if op.uses_src1() {
-                    pretty_print_reg(*src1, 8, allocs) + ", "
-                } else {
-                    "".into()
-                };
+                let src1 = pretty_print_reg(*src1, 8, allocs);
                 let dst = pretty_print_reg(dst.to_reg(), 8, allocs);
                 let src2 = src2.pretty_print(8, allocs);
                 format!(
-                    "{} ${}, {}{}, {}",
+                    "{} ${imm}, {src1}, {src2}, {dst}",
                     ljustify(format!(
                         "{}{}",
                         op.to_string(),
@@ -1129,10 +1147,6 @@ impl PrettyPrint for Inst {
                             ""
                         }
                     )),
-                    imm,
-                    src1,
-                    src2,
-                    dst,
                 )
             }
 
@@ -1153,6 +1167,12 @@ impl PrettyPrint for Inst {
                 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
             }
 
+            Inst::XmmToGprImm { op, src, dst, imm } => {
+                let src = pretty_print_reg(src.to_reg(), 8, allocs);
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
+                format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
             Inst::GprToXmm {
                 op,
                 src,
@@ -1863,9 +1883,11 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_def(dst.to_writable_reg());
             src.get_operands(collector);
         }
-        Inst::XmmUnaryRmR { src, dst, .. }
-        | Inst::XmmUnaryRmREvex { src, dst, .. }
-        | Inst::XmmUnaryRmRImm { src, dst, .. } => {
+        Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmRImm { src, dst, .. } => {
+            collector.reg_def(dst.to_writable_reg());
+            src.get_operands(collector);
+        }
+        Inst::XmmUnaryRmREvex { src, dst, .. } | Inst::XmmUnaryRmRUnaligned { src, dst, .. } => {
             collector.reg_def(dst.to_writable_reg());
             src.get_operands(collector);
         }
@@ -1876,6 +1898,13 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_reuse_def(dst.to_writable_reg(), 0);
             src2.get_operands(collector);
         }
+        Inst::XmmRmRUnaligned {
+            src1, src2, dst, ..
+        } => {
+            collector.reg_use(src1.to_reg());
+            collector.reg_reuse_def(dst.to_writable_reg(), 0);
+            src2.get_operands(collector);
+        }
         Inst::XmmRmRBlend {
             src1,
             src2,
@@ -1915,23 +1944,12 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             src2.get_operands(collector);
         }
         Inst::XmmRmRVex3 {
-            op,
             src1,
             src2,
             src3,
             dst,
             ..
         } => {
-            // Vfmadd uses and defs the dst reg, that is not the case with all
-            // AVX's ops, if you're adding a new op, make sure to correctly define
-            // register uses.
-            assert!(
-                *op == AvxOpcode::Vfmadd213ss
-                    || *op == AvxOpcode::Vfmadd213sd
-                    || *op == AvxOpcode::Vfmadd213ps
-                    || *op == AvxOpcode::Vfmadd213pd
-            );
-
             collector.reg_use(src1.to_reg());
             collector.reg_reuse_def(dst.to_writable_reg(), 0);
             collector.reg_use(src2.to_reg());
@@ -1976,23 +1994,11 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             src1.get_operands(collector);
         }
         Inst::XmmRmRImm {
-            op,
-            src1,
-            src2,
-            dst,
-            ..
+            src1, src2, dst, ..
         } => {
-            if !op.uses_src1() {
-                // FIXME: split this instruction into two, so we don't
-                // need this awkward src1-is-only-sometimes-an-arg
-                // behavior.
-                collector.reg_def(*dst);
-                src2.get_operands(collector);
-            } else {
-                collector.reg_use(*src1);
-                collector.reg_reuse_def(*dst, 0);
-                src2.get_operands(collector);
-            }
+            collector.reg_use(*src1);
+            collector.reg_reuse_def(*dst, 0);
+            src2.get_operands(collector);
         }
         Inst::XmmConstOp { dst, .. } => {
             collector.reg_def(dst.to_writable_reg());
@@ -2035,7 +2041,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_use(src.to_reg());
             collector.reg_fixed_nonallocatable(*dst);
         }
-        Inst::XmmToGpr { src, dst, .. } => {
+        Inst::XmmToGpr { src, dst, .. } | Inst::XmmToGprImm { src, dst, .. } => {
             collector.reg_use(src.to_reg());
             collector.reg_def(dst.to_writable_reg());
         }
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index d8a89c2af..9945f0869 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -46,29 +46,15 @@
                        (iadd x y)))
       (x64_add ty x y))
 
-;; Add a register and an immediate.
+;; The above case handles when the rhs is an immediate or a sinkable load, but
+;; additionally add lhs meets these criteria.
 
 (rule -4 (lower (has_type (fits_in_64 ty)
-                       (iadd x (simm32_from_value y))))
-      (x64_add ty x y))
-
-(rule -3 (lower (has_type (fits_in_64 ty)
                        (iadd (simm32_from_value x) y)))
       (x64_add ty y x))
-
-;; Add a register and memory.
-
-(rule -2 (lower (has_type (fits_in_64 ty)
-                       (iadd x (sinkable_load y))))
-      (x64_add ty
-           x
-           (sink_load_to_gpr_mem_imm y)))
-
-(rule -1 (lower (has_type (fits_in_64 ty)
+(rule -3 (lower (has_type (fits_in_64 ty)
                        (iadd (sinkable_load x) y)))
-      (x64_add ty
-           y
-           (sink_load_to_gpr_mem_imm x)))
+      (x64_add ty y x))
 
 ;; SSE.
 
@@ -144,17 +130,6 @@
                        (isub x y)))
       (x64_sub ty x y))
 
-;; Sub a register and an immediate.
-(rule -2 (lower (has_type (fits_in_64 ty)
-                       (isub x (simm32_from_value y))))
-      (x64_sub ty x y))
-
-;; Sub a register and memory.
-(rule -1 (lower (has_type (fits_in_64 ty)
-                       (isub x (sinkable_load y))))
-      (x64_sub ty x
-           (sink_load_to_gpr_mem_imm y)))
-
 ;; SSE.
 
 (rule (lower (has_type (multi_lane 8 16)
@@ -216,26 +191,14 @@
       (if (ty_int_ref_scalar_64 ty))
       (x64_and ty x y))
 
-;; And with a memory operand.
+;; The above case automatically handles when the rhs is an immediate or a
+;; sinkable load, but additionally handle the lhs here.
 
-(rule 1 (lower (has_type ty (band x (sinkable_load y))))
+(rule 1 (lower (has_type ty (band (sinkable_load x) y)))
       (if (ty_int_ref_scalar_64 ty))
-      (x64_and ty x
-               (sink_load_to_gpr_mem_imm y)))
-
-(rule 2 (lower (has_type ty (band (sinkable_load x) y)))
-      (if (ty_int_ref_scalar_64 ty))
-      (x64_and ty
-               y
-               (sink_load_to_gpr_mem_imm x)))
-
-;; And with an immediate.
-
-(rule 3 (lower (has_type ty (band x (simm32_from_value y))))
-      (if (ty_int_ref_scalar_64 ty))
-      (x64_and ty x y))
+      (x64_and ty y x))
 
-(rule 4 (lower (has_type ty (band (simm32_from_value x) y)))
+(rule 2 (lower (has_type ty (band (simm32_from_value x) y)))
       (if (ty_int_ref_scalar_64 ty))
       (x64_and ty y x))
 
@@ -308,25 +271,14 @@
       (if (ty_int_ref_scalar_64 ty))
       (x64_or ty x y))
 
-;; Or with a memory operand.
-
-(rule 1 (lower (has_type ty (bor x (sinkable_load y))))
-      (if (ty_int_ref_scalar_64 ty))
-      (x64_or ty x
-          (sink_load_to_gpr_mem_imm y)))
-
-(rule 2 (lower (has_type ty (bor (sinkable_load x) y)))
-      (if (ty_int_ref_scalar_64 ty))
-      (x64_or ty y
-          (sink_load_to_gpr_mem_imm x)))
-
-;; Or with an immediate.
+;; Handle immediates/sinkable loads on the lhs in addition to the automatic
+;; handling of the rhs above
 
-(rule 3 (lower (has_type ty (bor x (simm32_from_value y))))
+(rule 1 (lower (has_type ty (bor (sinkable_load x) y)))
       (if (ty_int_ref_scalar_64 ty))
-      (x64_or ty x y))
+      (x64_or ty y x))
 
-(rule 4 (lower (has_type ty (bor (simm32_from_value x) y)))
+(rule 2 (lower (has_type ty (bor (simm32_from_value x) y)))
       (if (ty_int_ref_scalar_64 ty))
       (x64_or ty y x))
 
@@ -371,23 +323,12 @@
       (if (ty_int_ref_scalar_64 ty))
       (x64_xor ty x y))
 
-;; Xor with a memory operand.
-
-(rule 1 (lower (has_type ty (bxor x (sinkable_load y))))
-      (if (ty_int_ref_scalar_64 ty))
-      (x64_xor ty x
-           (sink_load_to_gpr_mem_imm y)))
-
-(rule 2 (lower (has_type ty (bxor (sinkable_load x) y)))
-      (if (ty_int_ref_scalar_64 ty))
-      (x64_xor ty y
-           (sink_load_to_gpr_mem_imm x)))
-
-;; Xor with an immediate.
+;; Handle xor with lhs immediates/sinkable loads in addition to the automatic
+;; handling of the rhs above.
 
-(rule 3 (lower (has_type ty (bxor x (simm32_from_value y))))
+(rule 1 (lower (has_type ty (bxor (sinkable_load x) y)))
       (if (ty_int_ref_scalar_64 ty))
-      (x64_xor ty x y))
+      (x64_xor ty y x))
 
 (rule 4 (lower (has_type ty (bxor (simm32_from_value x) y)))
       (if (ty_int_ref_scalar_64 ty))
@@ -743,8 +684,8 @@
 ;; (TODO: when EVEX support is available, add an alternate lowering here).
 (rule (lower (has_type $I64X2 (sshr src amt)))
       (let ((src_ Xmm (put_in_xmm src))
-            (lo Gpr (x64_pextrd $I64 src_ 0))
-            (hi Gpr (x64_pextrd $I64 src_ 1))
+            (lo Gpr (x64_pextrq src_ 0))
+            (hi Gpr (x64_pextrq src_ 1))
             (amt_ Imm8Gpr (put_masked_in_imm8_gpr amt $I64))
             (shifted_lo Gpr (x64_sar $I64 lo amt_))
             (shifted_hi Gpr (x64_sar $I64 hi amt_)))
@@ -841,28 +782,15 @@
 (rule -5 (lower (has_type (fits_in_64 ty) (imul x y)))
       (x64_mul ty x y))
 
-;; Multiply a register and an immediate.
-
-(rule -3 (lower (has_type (fits_in_64 ty)
-                       (imul x (simm32_from_value y))))
-      (x64_mul ty x y))
+;; Handle multiplication where the lhs is an immediate or sinkable load in
+;; addition to the automatic rhs handling above.
 
 (rule -4 (lower (has_type (fits_in_64 ty)
                        (imul (simm32_from_value x) y)))
       (x64_mul ty y x))
-
-;; Multiply a register and a memory load.
-
-(rule -2 (lower (has_type (fits_in_64 ty)
-                       (imul x (sinkable_load y))))
-      (x64_mul ty
-           x
-           (sink_load_to_gpr_mem_imm y)))
-
-(rule -1 (lower (has_type (fits_in_64 ty)
+(rule -3 (lower (has_type (fits_in_64 ty)
                        (imul (sinkable_load x) y)))
-      (x64_mul ty y
-           (sink_load_to_gpr_mem_imm x)))
+      (x64_mul ty y x))
 
 ;; `i128`.
 
@@ -993,12 +921,8 @@
                                                x))
                              (swiden_high (and (value_type (multi_lane 32 4))
                                                y)))))
-      (let ((x2 Xmm (x64_pshufd x
-                            0xFA
-                            (OperandSize.Size32)))
-            (y2 Xmm (x64_pshufd y
-                            0xFA
-                            (OperandSize.Size32))))
+      (let ((x2 Xmm (x64_pshufd x 0xFA))
+            (y2 Xmm (x64_pshufd y 0xFA)))
         (x64_pmuldq x2 y2)))
 
 ;; Special case for `i16x8.extmul_low_i8x16_s`.
@@ -1029,12 +953,8 @@
                                               x))
                              (swiden_low (and (value_type (multi_lane 32 4))
                                               y)))))
-      (let ((x2 Xmm (x64_pshufd x
-                            0x50
-                            (OperandSize.Size32)))
-            (y2 Xmm (x64_pshufd y
-                            0x50
-                            (OperandSize.Size32))))
+      (let ((x2 Xmm (x64_pshufd x 0x50))
+            (y2 Xmm (x64_pshufd y 0x50)))
         (x64_pmuldq x2 y2)))
 
 ;; Special case for `i16x8.extmul_high_i8x16_u`.
@@ -1069,12 +989,8 @@
                                                x))
                              (uwiden_high (and (value_type (multi_lane 32 4))
                                                y)))))
-      (let ((x2 Xmm (x64_pshufd x
-                            0xFA
-                            (OperandSize.Size32)))
-            (y2 Xmm (x64_pshufd y
-                            0xFA
-                            (OperandSize.Size32))))
+      (let ((x2 Xmm (x64_pshufd x 0xFA))
+            (y2 Xmm (x64_pshufd y 0xFA)))
         (x64_pmuludq x2 y2)))
 
 ;; Special case for `i16x8.extmul_low_i8x16_u`.
@@ -1105,12 +1021,8 @@
                                               x))
                              (uwiden_low (and (value_type (multi_lane 32 4))
                                               y)))))
-      (let ((x2 Xmm (x64_pshufd x
-                            0x50
-                            (OperandSize.Size32)))
-            (y2 Xmm (x64_pshufd y
-                            0x50
-                            (OperandSize.Size32))))
+      (let ((x2 Xmm (x64_pshufd x 0x50))
+            (y2 Xmm (x64_pshufd y 0x50)))
         (x64_pmuludq x2 y2)))
 
 ;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1459,32 +1371,19 @@
         (x64_add_with_flags_paired ty a b)
         (trap_if (CC.B) tc)))
 
-;; Add a register and an immediate.
+;; Handle lhs immediates/sinkable loads in addition to the automatic rhs
+;; handling of above.
 
 (rule 1 (lower (has_type (fits_in_64 ty)
-                         (uadd_overflow_trap a (simm32_from_value b) tc)))
-      (with_flags
-        (x64_add_with_flags_paired ty a b)
-        (trap_if (CC.B) tc)))
-
-(rule 2 (lower (has_type (fits_in_64 ty)
                          (uadd_overflow_trap (simm32_from_value a) b tc)))
       (with_flags
         (x64_add_with_flags_paired ty b a)
         (trap_if (CC.B) tc)))
 
-;; Add a register and memory.
-
-(rule 3 (lower (has_type (fits_in_64 ty)
-                         (uadd_overflow_trap a (sinkable_load b) tc)))
-      (with_flags
-        (x64_add_with_flags_paired ty a (sink_load_to_gpr_mem_imm b))
-        (trap_if (CC.B) tc)))
-
-(rule 4 (lower (has_type (fits_in_64 ty)
+(rule 2 (lower (has_type (fits_in_64 ty)
                          (uadd_overflow_trap (sinkable_load a) b tc)))
       (with_flags
-        (x64_add_with_flags_paired ty b (sink_load_to_gpr_mem_imm a))
+        (x64_add_with_flags_paired ty b a)
         (trap_if (CC.B) tc)))
 
 ;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2256,10 +2155,6 @@
 
 ;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; N.B.: there are no load-op merging rules here. We can't guarantee
-;; the RHS (if a load) is 128-bit aligned, so we must avoid merging a
-;; load. Likewise for other ops below.
-
 (rule (lower (has_type $F32 (fadd x y)))
       (x64_addss x y))
 (rule (lower (has_type $F64 (fadd x y)))
@@ -2269,6 +2164,17 @@
 (rule (lower (has_type $F64X2 (fadd x y)))
       (x64_addpd x y))
 
+;; The above rules automatically sink loads for rhs operands, so additionally
+;; add rules for sinking loads with lhs operands.
+(rule 1 (lower (has_type $F32 (fadd (sinkable_load x) y)))
+      (x64_addss y x))
+(rule 1 (lower (has_type $F64 (fadd (sinkable_load x) y)))
+      (x64_addsd y x))
+(rule 1 (lower (has_type $F32X4 (fadd (sinkable_load x) y)))
+      (x64_addps y x))
+(rule 1 (lower (has_type $F64X2 (fadd (sinkable_load x) y)))
+      (x64_addpd y x))
+
 ;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (has_type $F32 (fsub x y)))
@@ -2291,6 +2197,17 @@
 (rule (lower (has_type $F64X2 (fmul x y)))
       (x64_mulpd x y))
 
+;; The above rules automatically sink loads for rhs operands, so additionally
+;; add rules for sinking loads with lhs operands.
+(rule 1 (lower (has_type $F32 (fmul (sinkable_load x) y)))
+      (x64_mulss y x))
+(rule 1 (lower (has_type $F64 (fmul (sinkable_load x) y)))
+      (x64_mulsd y x))
+(rule 1 (lower (has_type $F32X4 (fmul (sinkable_load x) y)))
+      (x64_mulps y x))
+(rule 1 (lower (has_type $F64X2 (fmul (sinkable_load x) y)))
+      (x64_mulpd y x))
+
 ;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (has_type $F32 (fdiv x y)))
@@ -2521,18 +2438,83 @@
 
 ;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; Base case for fma is to call out to one of two libcalls. For vectors they
+;; need to be decomposed, handle each element individually, and then recomposed.
+
 (rule (lower (has_type $F32 (fma x y z)))
       (libcall_3 (LibCall.FmaF32) x y z))
 (rule (lower (has_type $F64 (fma x y z)))
       (libcall_3 (LibCall.FmaF64) x y z))
-(rule 1 (lower (has_type (and (use_fma $true) $F32) (fma x y z)))
-      (x64_vfmadd213ss x y z))
-(rule 1 (lower (has_type (and (use_fma $true) $F64) (fma x y z)))
-      (x64_vfmadd213sd x y z))
-(rule (lower (has_type (and (use_fma $true) $F32X4) (fma x y z)))
-      (x64_vfmadd213ps x y z))
-(rule (lower (has_type (and (use_fma $true) $F64X2) (fma x y z)))
-      (x64_vfmadd213pd x y z))
+
+(rule (lower (has_type $F32X4 (fma x y z)))
+      (let (
+          (x Xmm (put_in_xmm x))
+          (y Xmm (put_in_xmm y))
+          (z Xmm (put_in_xmm z))
+          (x0 Xmm (libcall_3 (LibCall.FmaF32) x y z))
+          (x1 Xmm (libcall_3 (LibCall.FmaF32)
+            (x64_pshufd x 1)
+            (x64_pshufd y 1)
+            (x64_pshufd z 1)))
+          (x2 Xmm (libcall_3 (LibCall.FmaF32)
+            (x64_pshufd x 2)
+            (x64_pshufd y 2)
+            (x64_pshufd z 2)))
+          (x3 Xmm (libcall_3 (LibCall.FmaF32)
+            (x64_pshufd x 3)
+            (x64_pshufd y 3)
+            (x64_pshufd z 3)))
+
+          (tmp Xmm (vec_insert_lane $F32X4 x0 x1 1))
+          (tmp Xmm (vec_insert_lane $F32X4 tmp x2 2))
+          (tmp Xmm (vec_insert_lane $F32X4 tmp x3 3))
+        )
+        tmp))
+(rule (lower (has_type $F64X2 (fma x y z)))
+      (let (
+          (x Xmm (put_in_xmm x))
+          (y Xmm (put_in_xmm y))
+          (z Xmm (put_in_xmm z))
+          (x0 Xmm (libcall_3 (LibCall.FmaF64) x y z))
+          (x1 Xmm (libcall_3 (LibCall.FmaF64)
+            (x64_pshufd x 0xee)
+            (x64_pshufd y 0xee)
+            (x64_pshufd z 0xee)))
+        )
+        (vec_insert_lane $F64X2 x0 x1 1)))
+
+
+;; Special case for when the `fma` feature is active and a native instruction
+;; can be used.
+(rule 1 (lower (has_type ty (fma x y z)))
+      (if-let $true (use_fma))
+      (fmadd ty x y z))
+
+(decl fmadd (Type Value Value Value) Xmm)
+(decl fnmadd (Type Value Value Value) Xmm)
+
+;; Base case. Note that this will automatically sink a load with `z`, the value
+;; to add.
+(rule (fmadd ty x y z) (x64_vfmadd213 ty x y z))
+
+;; Allow sinking loads with one of the two values being multiplied in addition
+;; to the value being added. Note that both x and y can be sunk here due to
+;; multiplication being commutative.
+(rule 1 (fmadd ty (sinkable_load x) y z) (x64_vfmadd132 ty y z x))
+(rule 2 (fmadd ty x (sinkable_load y) z) (x64_vfmadd132 ty x z y))
+
+;; If one of the values being multiplied is negated then use a `vfnmadd*`
+;; instruction instead
+(rule 3 (fmadd ty (fneg x) y z) (fnmadd ty x y z))
+(rule 4 (fmadd ty x (fneg y) z) (fnmadd ty x y z))
+
+(rule (fnmadd ty x y z) (x64_vfnmadd213 ty x y z))
+(rule 1 (fnmadd ty (sinkable_load x) y z) (x64_vfnmadd132 ty y z x))
+(rule 2 (fnmadd ty x (sinkable_load y) z) (x64_vfnmadd132 ty x z y))
+
+;; Like `fmadd` if one argument is negated switch which one is being codegen'd
+(rule 3 (fnmadd ty (fneg x) y z) (fmadd ty x y z))
+(rule 4 (fnmadd ty x (fneg y) z) (fmadd ty x y z))
 
 ;; Rules for `load*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -3084,7 +3066,7 @@
             (tmp Xmm (x64_pxor tmp dst))
 
             ;; Convert the packed float to packed doubleword.
-            (dst Xmm (x64_cvttps2dq $F32X4 dst))
+            (dst Xmm (x64_cvttps2dq dst))
 
             ;; Set top bit only if < 0
             (tmp Xmm (x64_pand dst tmp))
@@ -3165,7 +3147,7 @@
             ;; Overflow lanes greater than the maximum allowed signed value will
             ;; set to 0x80000000. Negative and NaN lanes will be 0x0
             (tmp1 Xmm dst)
-            (dst Xmm (x64_cvttps2dq $F32X4 dst))
+            (dst Xmm (x64_cvttps2dq dst))
 
             ;; Set lanes to src - max_signed_int
             (tmp1 Xmm (x64_subps tmp1 tmp2))
@@ -3175,7 +3157,7 @@
             (tmp2 Xmm (x64_cmpps tmp2 tmp1 (FcmpImm.LessThanOrEqual)))
 
             ;; Convert those set of lanes that have the max_signed_int factored out.
-            (tmp1 Xmm (x64_cvttps2dq $F32X4 tmp1))
+            (tmp1 Xmm (x64_cvttps2dq tmp1))
 
             ;; Prepare converted lanes by zeroing negative lanes and prepping lanes
             ;; that have positive overflow (based on the mask) by setting these lanes
@@ -3246,7 +3228,7 @@
         (x64_pmovsxwd (x64_palignr x x 8 (OperandSize.Size32)))))
 
 (rule (lower (has_type $I64X2 (swiden_high val @ (value_type $I32X4))))
-      (x64_pmovsxdq (x64_pshufd val 0xEE (OperandSize.Size32))))
+      (x64_pmovsxdq (x64_pshufd val 0xEE)))
 
 ;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -3270,7 +3252,7 @@
         (x64_pmovzxwd (x64_palignr x x 8 (OperandSize.Size32)))))
 
 (rule (lower (has_type $I64X2 (uwiden_high val @ (value_type $I32X4))))
-      (x64_pmovzxdq (x64_pshufd val 0xEE (OperandSize.Size32))))
+      (x64_pmovzxdq (x64_pshufd val 0xEE)))
 
 ;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -3566,25 +3548,25 @@
 ;; Cases 2-4 for an F32X4
 (rule 1 (lower (has_type $F32 (extractlane val @ (value_type (ty_vec128 ty))
                                          (u8_from_uimm8 lane))))
-      (x64_pshufd val lane (OperandSize.Size32)))
+      (x64_pshufd val lane))
 
 ;; This is the only remaining case for F64X2
 (rule 1 (lower (has_type $F64 (extractlane val @ (value_type (ty_vec128 ty))
                                          (u8_from_uimm8 1))))
       ;; 0xee == 0b11_10_11_10
-      (x64_pshufd val 0xee (OperandSize.Size32)))
+      (x64_pshufd val 0xee))
 
 (rule 0 (lower (extractlane val @ (value_type ty @ (multi_lane 8 16)) (u8_from_uimm8 lane)))
-      (x64_pextrb ty val lane))
+      (x64_pextrb val lane))
 
 (rule 0 (lower (extractlane val @ (value_type ty @ (multi_lane 16 8)) (u8_from_uimm8 lane)))
-      (x64_pextrw ty val lane))
+      (x64_pextrw val lane))
 
 (rule 0 (lower (extractlane val @ (value_type ty @ (multi_lane 32 4)) (u8_from_uimm8 lane)))
-      (x64_pextrd ty val lane))
+      (x64_pextrd val lane))
 
 (rule 0 (lower (extractlane val @ (value_type ty @ (multi_lane 64 2)) (u8_from_uimm8 lane)))
-      (x64_pextrd ty val lane))
+      (x64_pextrq val lane))
 
 ;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -3603,9 +3585,9 @@
 ;; Case 3: when presented with `load + scalar_to_vector`, coalesce into a single
 ;; MOVSS/MOVSD instruction.
 (rule 2 (lower (scalar_to_vector (and (sinkable_load src) (value_type (ty_32 _)))))
-      (x64_movss_load (sink_load_to_xmm_mem src)))
+      (x64_movss_load src))
 (rule 3 (lower (scalar_to_vector (and (sinkable_load src) (value_type (ty_64 _)))))
-      (x64_movsd_load (sink_load_to_xmm_mem src)))
+      (x64_movsd_load src))
 
 ;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -3622,7 +3604,7 @@
             (vec Xmm (vec_insert_lane $I16X8 (xmm_uninit_value) src 0))
             (vec Xmm (vec_insert_lane $I16X8 vec src 1)))
         ;; Shuffle the lowest two lanes to all other lanes.
-        (x64_pshufd vec 0 (OperandSize.Size32))))
+        (x64_pshufd vec 0)))
 
 (rule 1 (lower (has_type (multi_lane 32 4) (splat src @ (value_type (ty_scalar_float _)))))
       (lower_splat_32x4 $F32X4 src))
@@ -3635,7 +3617,7 @@
       (let ((src RegMem src)
             (vec Xmm (vec_insert_lane ty (xmm_uninit_value) src 0)))
         ;; Shuffle the lowest lane to all other lanes.
-        (x64_pshufd vec 0 (OperandSize.Size32))))
+        (x64_pshufd vec 0)))
 
 (rule 1 (lower (has_type (multi_lane 64 2) (splat src @ (value_type (ty_scalar_float _)))))
       (lower_splat_64x2 $F64X2 src))
diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs
index c294fad0b..868d0860e 100644
--- a/cranelift/codegen/src/isa/x64/lower.rs
+++ b/cranelift/codegen/src/isa/x64/lower.rs
@@ -92,12 +92,6 @@ fn is_mergeable_load(ctx: &mut Lower<Inst>, src_insn: IRInst) -> Option<(InsnInp
         return None;
     }
 
-    // SIMD instructions can only be load-coalesced when the loaded value comes
-    // from an aligned address.
-    if load_ty.is_vector() && !insn_data.memflags().map_or(false, |f| f.aligned()) {
-        return None;
-    }
-
     // Just testing the opcode is enough, because the width will always match if
     // the type does (and the type should match if the CLIF is properly
     // constructed).
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 0e050cece..0267c3d32 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -100,23 +100,9 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
             if let Some(imm) = to_simm32(c as i64) {
                 return imm.to_reg_mem_imm();
             }
-
-            // A load from the constant pool is better than a
-            // rematerialization into a register, because it reduces
-            // register pressure.
-            let vcode_constant = self.emit_u64_le_const(c);
-            return RegMemImm::mem(SyntheticAmode::ConstantOffset(vcode_constant));
         }
 
-        if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {
-            if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) {
-                self.lower_ctx.sink_inst(src_insn);
-                let amode = lower_to_amode(self.lower_ctx, addr_input, offset);
-                return RegMemImm::mem(amode);
-            }
-        }
-
-        RegMemImm::reg(self.put_in_reg(val))
+        self.put_in_reg_mem(val).into()
     }
 
     fn put_in_xmm_mem_imm(&mut self, val: Value) -> XmmMemImm {
@@ -150,7 +136,7 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
                 .unwrap();
         }
 
-        XmmMem::new(RegMem::reg(self.put_in_reg(val))).unwrap()
+        XmmMem::new(self.put_in_reg_mem(val)).unwrap()
     }
 
     fn put_in_reg_mem(&mut self, val: Value) -> RegMem {
@@ -164,12 +150,8 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
             return RegMem::mem(SyntheticAmode::ConstantOffset(vcode_constant));
         }
 
-        if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {
-            if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) {
-                self.lower_ctx.sink_inst(src_insn);
-                let amode = lower_to_amode(self.lower_ctx, addr_input, offset);
-                return RegMem::mem(amode);
-            }
+        if let Some(load) = self.sinkable_load(val) {
+            return self.sink_load(&load);
         }
 
         RegMem::reg(self.put_in_reg(val))
@@ -231,7 +213,7 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
     }
 
     #[inline]
-    fn use_fma(&mut self, _: Type) -> bool {
+    fn use_fma(&mut self) -> bool {
         self.backend.x64_flags.use_fma()
     }
 
@@ -446,7 +428,7 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
 
     #[inline]
     fn xmm_mem_to_xmm_mem_imm(&mut self, r: &XmmMem) -> XmmMemImm {
-        r.clone().into()
+        XmmMemImm::new(r.clone().to_reg_mem().into()).unwrap()
     }
 
     #[inline]
@@ -997,10 +979,40 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
             }
         }
     }
+
+    fn xmm_mem_to_xmm_mem_aligned(&mut self, arg: &XmmMem) -> XmmMemAligned {
+        match XmmMemAligned::new(arg.clone().into()) {
+            Some(aligned) => aligned,
+            None => match arg.clone().into() {
+                RegMem::Mem { addr } => self.load_xmm_unaligned(addr).into(),
+                _ => unreachable!(),
+            },
+        }
+    }
+
+    fn xmm_mem_imm_to_xmm_mem_aligned_imm(&mut self, arg: &XmmMemImm) -> XmmMemAlignedImm {
+        match XmmMemAlignedImm::new(arg.clone().into()) {
+            Some(aligned) => aligned,
+            None => match arg.clone().into() {
+                RegMemImm::Mem { addr } => self.load_xmm_unaligned(addr).into(),
+                _ => unreachable!(),
+            },
+        }
+    }
 }
 
 impl IsleContext<'_, '_, MInst, X64Backend> {
     isle_prelude_method_helpers!(X64Caller);
+
+    fn load_xmm_unaligned(&mut self, addr: SyntheticAmode) -> Xmm {
+        let tmp = self.lower_ctx.alloc_tmp(types::F32X4).only_reg().unwrap();
+        self.lower_ctx.emit(MInst::XmmUnaryRmRUnaligned {
+            op: SseOpcode::Movdqu,
+            src: XmmMem::new(RegMem::mem(addr)).unwrap(),
+            dst: Writable::from_reg(Xmm::new(tmp.to_reg()).unwrap()),
+        });
+        Xmm::new(tmp.to_reg()).unwrap()
+    }
 }
 
 // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we
diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rs
index 653b2dc3a..3becbeb1e 100644
--- a/cranelift/native/src/lib.rs
+++ b/cranelift/native/src/lib.rs
@@ -213,8 +213,9 @@ pub fn builder_with_options(infer_native_flags: bool) -> Result<isa::Builder, &'
             isa_builder.enable("has_v").unwrap();
         }
 
-        // TODO: ZiFencei does not have a bit associated with it
-        // TODO: Zbkb does not have a bit associated with it
+        // In general extensions that are longer than one letter
+        // won't have a bit associated with them. The Linux kernel
+        // is currently working on a new way to query the extensions.
     }
 
     // squelch warnings about unused mut/variables on some platforms.


Test patch:
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 1c19fd182..1ccaf6c7d 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -34,7 +34,7 @@ impl Inst {
         debug_assert!(dst.to_reg().class() == RegClass::Float);
         Inst::XmmUnaryRmRImm {
             op,
-            src: XmmMem::new(src).unwrap(),
+            src: XmmMemAligned::new(src).unwrap(),
             imm,
             dst: WritableXmm::from_writable_reg(dst).unwrap(),
         }
@@ -56,7 +56,7 @@ impl Inst {
         Inst::XmmRmiReg {
             opcode,
             src1: Xmm::new(dst.to_reg()).unwrap(),
-            src2: XmmMemImm::new(src).unwrap(),
+            src2: XmmMemAlignedImm::new(src).unwrap(),
             dst: WritableXmm::from_writable_reg(dst).unwrap(),
         }
     }
@@ -96,7 +96,7 @@ impl Inst {
         debug_assert!(dst.to_reg().class() == RegClass::Float);
         Inst::XmmUnaryRmR {
             op,
-            src: XmmMem::new(src).unwrap(),
+            src: XmmMemAligned::new(src).unwrap(),
             dst: WritableXmm::from_writable_reg(dst).unwrap(),
         }
     }
@@ -136,7 +136,7 @@ impl Inst {
         Inst::XmmRmRBlend {
             op,
             src1: Xmm::new(dst.to_reg()).unwrap(),
-            src2: XmmMem::new(src2).unwrap(),
+            src2: XmmMemAligned::new(src2).unwrap(),
             mask: Xmm::new(regs::xmm0()).unwrap(),
             dst: WritableXmm::from_writable_reg(dst).unwrap(),
         }
diff --git a/cranelift/filetests/filetests/isa/riscv64/bitops-optimized.clif b/cranelift/filetests/filetests/isa/riscv64/bitops-optimized.clif
index 75c3863fe..901161bb5 100644
--- a/cranelift/filetests/filetests/isa/riscv64/bitops-optimized.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/bitops-optimized.clif
@@ -1,6 +1,6 @@
 test compile precise-output
 set opt_level=speed
-target riscv64 has_b
+target riscv64 has_zbb
 
 function %band_not_i32(i32, i32) -> i32 {
 block0(v0: i32, v1: i32):
diff --git a/cranelift/filetests/filetests/isa/riscv64/br_table.clif b/cranelift/filetests/filetests/isa/riscv64/br_table.clif
new file mode 100644
index 000000000..aeacae35c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/br_table.clif
@@ -0,0 +1,95 @@
+test compile precise-output
+set unwind_info=false
+target riscv64
+
+function %br_table(i32) -> i32 {
+block0(v0: i32):
+  br_table v0, block4, [block1, block2, block2, block3]
+
+block1:
+  v1 = iconst.i32 1
+  jump block5(v1)
+
+block2:
+  v2 = iconst.i32 2
+  jump block5(v2)
+
+block3:
+  v3 = iconst.i32 3
+  jump block5(v3)
+
+block4:
+  v4 = iconst.i32 4
+  jump block5(v4)
+
+block5(v5: i32):
+  v6 = iadd.i32 v0, v5
+  return v6
+}
+
+; VCode:
+; block0:
+;   br_table a0,[MachLabel(1),MachLabel(3),MachLabel(5),MachLabel(6),MachLabel(8)]##tmp1=t3,tmp2=t4
+; block1:
+;   li a1,4
+;   j label2
+; block2:
+;   j label10
+; block3:
+;   li a1,1
+;   j label4
+; block4:
+;   j label10
+; block5:
+;   j label7
+; block6:
+;   j label7
+; block7:
+;   li a1,2
+;   j label10
+; block8:
+;   li a1,3
+;   j label9
+; block9:
+;   j label10
+; block10:
+;   addw a0,a0,a1
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   slli t6, a0, 0x20
+;   srli t6, t6, 0x20
+;   addi t4, zero, 4
+;   bltu t6, t4, 0xc
+;   auipc t4, 0
+;   jalr zero, t4, 0x38
+;   auipc t3, 0
+;   slli t4, t6, 3
+;   add t3, t3, t4
+;   jalr zero, t3, 0x10
+;   auipc t4, 0
+;   jalr zero, t4, 0x28
+;   auipc t4, 0
+;   jalr zero, t4, 0x28
+;   auipc t4, 0
+;   jalr zero, t4, 0x20
+;   auipc t4, 0
+;   jalr zero, t4, 0x20
+; block1: ; offset 0x48
+;   addi a1, zero, 4
+; block2: ; offset 0x4c
+;   j 0x18
+; block3: ; offset 0x50
+;   addi a1, zero, 1
+; block4: ; offset 0x54
+;   j 0x10
+; block5: ; offset 0x58
+;   addi a1, zero, 2
+;   j 8
+; block6: ; offset 0x60
+;   addi a1, zero, 3
+; block7: ; offset 0x64
+;   addw a0, a0, a1
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/x64/extractlane.clif b/cranelift/filetests/filetests/isa/x64/extractlane.clif
index abe9882d3..1cbdfbf7d 100644
--- a/cranelift/filetests/filetests/isa/x64/extractlane.clif
+++ b/cranelift/filetests/filetests/isa/x64/extractlane.clif
@@ -86,7 +86,7 @@ block0(v0: i64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pextrd.w $1, %xmm0, %rax
+;   pextrq  $1, %xmm0, %rax
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
diff --git a/cranelift/filetests/filetests/isa/x64/fastcall.clif b/cranelift/filetests/filetests/isa/x64/fastcall.clif
index cdb178f13..11eab5008 100644
--- a/cranelift/filetests/filetests/isa/x64/fastcall.clif
+++ b/cranelift/filetests/filetests/isa/x64/fastcall.clif
@@ -333,44 +333,42 @@ block0(v0: i64):
 ;   unwind PushFrameRegs { offset_upward_to_caller_sp: 16 }
 ;   movq    %rsp, %rbp
 ;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 160 }
-;   subq    %rsp, $256, %rsp
-;   movdqu  %xmm6, 96(%rsp)
+;   subq    %rsp, $224, %rsp
+;   movdqu  %xmm6, 64(%rsp)
 ;   unwind SaveReg { clobber_offset: 0, reg: p6f }
-;   movdqu  %xmm7, 112(%rsp)
+;   movdqu  %xmm7, 80(%rsp)
 ;   unwind SaveReg { clobber_offset: 16, reg: p7f }
-;   movdqu  %xmm8, 128(%rsp)
+;   movdqu  %xmm8, 96(%rsp)
 ;   unwind SaveReg { clobber_offset: 32, reg: p8f }
-;   movdqu  %xmm9, 144(%rsp)
+;   movdqu  %xmm9, 112(%rsp)
 ;   unwind SaveReg { clobber_offset: 48, reg: p9f }
-;   movdqu  %xmm10, 160(%rsp)
+;   movdqu  %xmm10, 128(%rsp)
 ;   unwind SaveReg { clobber_offset: 64, reg: p10f }
-;   movdqu  %xmm11, 176(%rsp)
+;   movdqu  %xmm11, 144(%rsp)
 ;   unwind SaveReg { clobber_offset: 80, reg: p11f }
-;   movdqu  %xmm12, 192(%rsp)
+;   movdqu  %xmm12, 160(%rsp)
 ;   unwind SaveReg { clobber_offset: 96, reg: p12f }
-;   movdqu  %xmm13, 208(%rsp)
+;   movdqu  %xmm13, 176(%rsp)
 ;   unwind SaveReg { clobber_offset: 112, reg: p13f }
-;   movdqu  %xmm14, 224(%rsp)
+;   movdqu  %xmm14, 192(%rsp)
 ;   unwind SaveReg { clobber_offset: 128, reg: p14f }
-;   movdqu  %xmm15, 240(%rsp)
+;   movdqu  %xmm15, 208(%rsp)
 ;   unwind SaveReg { clobber_offset: 144, reg: p15f }
 ; block0:
 ;   movsd   0(%rcx), %xmm0
 ;   movsd   8(%rcx), %xmm10
-;   movdqu  %xmm10, rsp(80 + virtual offset)
-;   movsd   16(%rcx), %xmm2
-;   movdqu  %xmm2, rsp(0 + virtual offset)
+;   movdqu  %xmm10, rsp(48 + virtual offset)
+;   movsd   16(%rcx), %xmm5
 ;   movsd   24(%rcx), %xmm14
-;   movdqu  %xmm14, rsp(64 + virtual offset)
+;   movdqu  %xmm14, rsp(32 + virtual offset)
 ;   movsd   32(%rcx), %xmm13
 ;   movsd   40(%rcx), %xmm15
-;   movdqu  %xmm15, rsp(48 + virtual offset)
+;   movdqu  %xmm15, rsp(16 + virtual offset)
 ;   movsd   48(%rcx), %xmm7
-;   movsd   56(%rcx), %xmm5
-;   movdqu  %xmm5, rsp(32 + virtual offset)
+;   movsd   56(%rcx), %xmm8
+;   movdqu  %xmm8, rsp(0 + virtual offset)
 ;   movsd   64(%rcx), %xmm12
-;   movsd   72(%rcx), %xmm4
-;   movdqu  %xmm4, rsp(16 + virtual offset)
+;   movsd   72(%rcx), %xmm2
 ;   movsd   80(%rcx), %xmm9
 ;   movsd   88(%rcx), %xmm4
 ;   movsd   96(%rcx), %xmm3
@@ -380,24 +378,21 @@ block0(v0: i64):
 ;   movsd   128(%rcx), %xmm6
 ;   movsd   136(%rcx), %xmm14
 ;   movsd   144(%rcx), %xmm1
-;   movsd   152(%rcx), %xmm15
-;   movdqu  rsp(80 + virtual offset), %xmm2
-;   addsd   %xmm0, %xmm2, %xmm0
-;   movdqu  rsp(0 + virtual offset), %xmm2
-;   movdqu  rsp(64 + virtual offset), %xmm5
-;   addsd   %xmm2, %xmm5, %xmm2
-;   movdqu  rsp(48 + virtual offset), %xmm5
-;   addsd   %xmm13, %xmm5, %xmm13
-;   movdqu  rsp(32 + virtual offset), %xmm5
-;   addsd   %xmm7, %xmm5, %xmm7
-;   movdqu  rsp(16 + virtual offset), %xmm5
-;   addsd   %xmm12, %xmm5, %xmm12
+;   movdqu  rsp(48 + virtual offset), %xmm15
+;   addsd   %xmm0, %xmm15, %xmm0
+;   movdqu  rsp(32 + virtual offset), %xmm15
+;   addsd   %xmm5, %xmm15, %xmm5
+;   movdqu  rsp(16 + virtual offset), %xmm15
+;   addsd   %xmm13, %xmm15, %xmm13
+;   movdqu  rsp(0 + virtual offset), %xmm15
+;   addsd   %xmm7, %xmm15, %xmm7
+;   addsd   %xmm12, %xmm2, %xmm12
 ;   addsd   %xmm9, %xmm4, %xmm9
 ;   addsd   %xmm3, %xmm8, %xmm3
 ;   addsd   %xmm11, %xmm10, %xmm11
 ;   addsd   %xmm6, %xmm14, %xmm6
-;   addsd   %xmm1, %xmm15, %xmm1
-;   addsd   %xmm0, %xmm2, %xmm0
+;   addsd   %xmm1, 152(%rcx), %xmm1
+;   addsd   %xmm0, %xmm5, %xmm0
 ;   addsd   %xmm13, %xmm7, %xmm13
 ;   addsd   %xmm12, %xmm9, %xmm12
 ;   addsd   %xmm3, %xmm11, %xmm3
@@ -406,17 +401,17 @@ block0(v0: i64):
 ;   addsd   %xmm12, %xmm3, %xmm12
 ;   addsd   %xmm0, %xmm12, %xmm0
 ;   addsd   %xmm0, %xmm6, %xmm0
-;   movdqu  96(%rsp), %xmm6
-;   movdqu  112(%rsp), %xmm7
-;   movdqu  128(%rsp), %xmm8
-;   movdqu  144(%rsp), %xmm9
-;   movdqu  160(%rsp), %xmm10
-;   movdqu  176(%rsp), %xmm11
-;   movdqu  192(%rsp), %xmm12
-;   movdqu  208(%rsp), %xmm13
-;   movdqu  224(%rsp), %xmm14
-;   movdqu  240(%rsp), %xmm15
-;   addq    %rsp, $256, %rsp
+;   movdqu  64(%rsp), %xmm6
+;   movdqu  80(%rsp), %xmm7
+;   movdqu  96(%rsp), %xmm8
+;   movdqu  112(%rsp), %xmm9
+;   movdqu  128(%rsp), %xmm10
+;   movdqu  144(%rsp), %xmm11
+;   movdqu  160(%rsp), %xmm12
+;   movdqu  176(%rsp), %xmm13
+;   movdqu  192(%rsp), %xmm14
+;   movdqu  208(%rsp), %xmm15
+;   addq    %rsp, $224, %rsp
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -425,34 +420,32 @@ block0(v0: i64):
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
-;   subq $0x100, %rsp
-;   movdqu %xmm6, 0x60(%rsp)
-;   movdqu %xmm7, 0x70(%rsp)
-;   movdqu %xmm8, 0x80(%rsp)
-;   movdqu %xmm9, 0x90(%rsp)
-;   movdqu %xmm10, 0xa0(%rsp)
-;   movdqu %xmm11, 0xb0(%rsp)
-;   movdqu %xmm12, 0xc0(%rsp)
-;   movdqu %xmm13, 0xd0(%rsp)
-;   movdqu %xmm14, 0xe0(%rsp)
-;   movdqu %xmm15, 0xf0(%rsp)
-; block1: ; offset 0x67
+;   subq $0xe0, %rsp
+;   movdqu %xmm6, 0x40(%rsp)
+;   movdqu %xmm7, 0x50(%rsp)
+;   movdqu %xmm8, 0x60(%rsp)
+;   movdqu %xmm9, 0x70(%rsp)
+;   movdqu %xmm10, 0x80(%rsp)
+;   movdqu %xmm11, 0x90(%rsp)
+;   movdqu %xmm12, 0xa0(%rsp)
+;   movdqu %xmm13, 0xb0(%rsp)
+;   movdqu %xmm14, 0xc0(%rsp)
+;   movdqu %xmm15, 0xd0(%rsp)
+; block1: ; offset 0x61
 ;   movsd (%rcx), %xmm0 ; trap: heap_oob
 ;   movsd 8(%rcx), %xmm10 ; trap: heap_oob
-;   movdqu %xmm10, 0x50(%rsp)
-;   movsd 0x10(%rcx), %xmm2 ; trap: heap_oob
-;   movdqu %xmm2, (%rsp)
+;   movdqu %xmm10, 0x30(%rsp)
+;   movsd 0x10(%rcx), %xmm5 ; trap: heap_oob
 ;   movsd 0x18(%rcx), %xmm14 ; trap: heap_oob
-;   movdqu %xmm14, 0x40(%rsp)
+;   movdqu %xmm14, 0x20(%rsp)
 ;   movsd 0x20(%rcx), %xmm13 ; trap: heap_oob
 ;   movsd 0x28(%rcx), %xmm15 ; trap: heap_oob
-;   movdqu %xmm15, 0x30(%rsp)
+;   movdqu %xmm15, 0x10(%rsp)
 ;   movsd 0x30(%rcx), %xmm7 ; trap: heap_oob
-;   movsd 0x38(%rcx), %xmm5 ; trap: heap_oob
-;   movdqu %xmm5, 0x20(%rsp)
+;   movsd 0x38(%rcx), %xmm8 ; trap: heap_oob
+;   movdqu %xmm8, (%rsp)
 ;   movsd 0x40(%rcx), %xmm12 ; trap: heap_oob
-;   movsd 0x48(%rcx), %xmm4 ; trap: heap_oob
-;   movdqu %xmm4, 0x10(%rsp)
+;   movsd 0x48(%rcx), %xmm2 ; trap: heap_oob
 ;   movsd 0x50(%rcx), %xmm9 ; trap: heap_oob
 ;   movsd 0x58(%rcx), %xmm4 ; trap: heap_oob
 ;   movsd 0x60(%rcx), %xmm3 ; trap: heap_oob
@@ -462,24 +455,21 @@ block0(v0: i64):
 ;   movsd 0x80(%rcx), %xmm6 ; trap: heap_oob
 ;   movsd 0x88(%rcx), %xmm14 ; trap: heap_oob
 ;   movsd 0x90(%rcx), %xmm1 ; trap: heap_oob
-;   movsd 0x98(%rcx), %xmm15 ; trap: heap_oob
-;   movdqu 0x50(%rsp), %xmm2
-;   addsd %xmm2, %xmm0
-;   movdqu (%rsp), %xmm2
-;   movdqu 0x40(%rsp), %xmm5
-;   addsd %xmm5, %xmm2
-;   movdqu 0x30(%rsp), %xmm5
-;   addsd %xmm5, %xmm13
-;   movdqu 0x20(%rsp), %xmm5
-;   addsd %xmm5, %xmm7
-;   movdqu 0x10(%rsp), %xmm5
-;   addsd %xmm5, %xmm12
+;   movdqu 0x30(%rsp), %xmm15
+;   addsd %xmm15, %xmm0
+;   movdqu 0x20(%rsp), %xmm15
+;   addsd %xmm15, %xmm5
+;   movdqu 0x10(%rsp), %xmm15
+;   addsd %xmm15, %xmm13
+;   movdqu (%rsp), %xmm15
+;   addsd %xmm15, %xmm7
+;   addsd %xmm2, %xmm12
 ;   addsd %xmm4, %xmm9
 ;   addsd %xmm8, %xmm3
 ;   addsd %xmm10, %xmm11
 ;   addsd %xmm14, %xmm6
-;   addsd %xmm15, %xmm1
-;   addsd %xmm2, %xmm0
+;   addsd 0x98(%rcx), %xmm1 ; trap: heap_oob
+;   addsd %xmm5, %xmm0
 ;   addsd %xmm7, %xmm13
 ;   addsd %xmm9, %xmm12
 ;   addsd %xmm11, %xmm3
@@ -488,17 +478,17 @@ block0(v0: i64):
 ;   addsd %xmm3, %xmm12
 ;   addsd %xmm12, %xmm0
 ;   addsd %xmm6, %xmm0
-;   movdqu 0x60(%rsp), %xmm6
-;   movdqu 0x70(%rsp), %xmm7
-;   movdqu 0x80(%rsp), %xmm8
-;   movdqu 0x90(%rsp), %xmm9
-;   movdqu 0xa0(%rsp), %xmm10
-;   movdqu 0xb0(%rsp), %xmm11
-;   movdqu 0xc0(%rsp), %xmm12
-;   movdqu 0xd0(%rsp), %xmm13
-;   movdqu 0xe0(%rsp), %xmm14
-;   movdqu 0xf0(%rsp), %xmm15
-;   addq $0x100, %rsp
+;   movdqu 0x40(%rsp), %xmm6
+;   movdqu 0x50(%rsp), %xmm7
+;   movdqu 0x60(%rsp), %xmm8
+;   movdqu 0x70(%rsp), %xmm9
+;   movdqu 0x80(%rsp), %xmm10
+;   movdqu 0x90(%rsp), %xmm11
+;   movdqu 0xa0(%rsp), %xmm12
+;   movdqu 0xb0(%rsp), %xmm13
+;   movdqu 0xc0(%rsp), %xmm14
+;   movdqu 0xd0(%rsp), %xmm15
+;   addq $0xe0, %rsp
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/fma-call.clif b/cranelift/filetests/filetests/isa/x64/fma-call.clif
index 25b62371b..153fc48c6 100644
--- a/cranelift/filetests/filetests/isa/x64/fma-call.clif
+++ b/cranelift/filetests/filetests/isa/x64/fma-call.clif
@@ -55,3 +55,183 @@ block0(v0: f64, v1: f64, v2: f64):
 ;   popq %rbp
 ;   retq
 
+function %fma_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+;   subq    %rsp, $96, %rsp
+; block0:
+;   movdqu  %xmm0, rsp(0 + virtual offset)
+;   movdqu  %xmm1, rsp(16 + virtual offset)
+;   movdqu  %xmm2, rsp(32 + virtual offset)
+;   load_ext_name %FmaF32+0, %r8
+;   movdqu  rsp(0 + virtual offset), %xmm0
+;   movdqu  rsp(16 + virtual offset), %xmm1
+;   movdqu  rsp(32 + virtual offset), %xmm2
+;   call    *%r8
+;   movdqu  %xmm0, rsp(48 + virtual offset)
+;   movdqu  rsp(0 + virtual offset), %xmm4
+;   pshufd  $1, %xmm4, %xmm0
+;   movdqu  rsp(16 + virtual offset), %xmm2
+;   pshufd  $1, %xmm2, %xmm1
+;   movdqu  rsp(32 + virtual offset), %xmm3
+;   pshufd  $1, %xmm3, %xmm2
+;   load_ext_name %FmaF32+0, %r9
+;   call    *%r9
+;   movdqu  %xmm0, rsp(64 + virtual offset)
+;   movdqu  rsp(0 + virtual offset), %xmm14
+;   pshufd  $2, %xmm14, %xmm0
+;   movdqu  rsp(16 + virtual offset), %xmm13
+;   pshufd  $2, %xmm13, %xmm1
+;   movdqu  rsp(32 + virtual offset), %xmm15
+;   pshufd  $2, %xmm15, %xmm2
+;   load_ext_name %FmaF32+0, %r10
+;   call    *%r10
+;   movdqu  %xmm0, rsp(80 + virtual offset)
+;   movdqu  rsp(0 + virtual offset), %xmm14
+;   pshufd  $3, %xmm14, %xmm0
+;   movdqu  rsp(16 + virtual offset), %xmm1
+;   pshufd  $3, %xmm1, %xmm1
+;   movdqu  rsp(32 + virtual offset), %xmm2
+;   pshufd  $3, %xmm2, %xmm2
+;   load_ext_name %FmaF32+0, %r11
+;   call    *%r11
+;   movdqa  %xmm0, %xmm13
+;   movdqu  rsp(64 + virtual offset), %xmm4
+;   movdqu  rsp(48 + virtual offset), %xmm0
+;   insertps $16, %xmm0, %xmm4, %xmm0
+;   movdqu  rsp(80 + virtual offset), %xmm10
+;   insertps $32, %xmm0, %xmm10, %xmm0
+;   movdqa  %xmm13, %xmm1
+;   insertps $48, %xmm0, %xmm1, %xmm0
+;   addq    %rsp, $96, %rsp
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+;   subq $0x60, %rsp
+; block1: ; offset 0x8
+;   movdqu %xmm0, (%rsp)
+;   movdqu %xmm1, 0x10(%rsp)
+;   movdqu %xmm2, 0x20(%rsp)
+;   movabsq $0, %r8 ; reloc_external Abs8 %FmaF32 0
+;   movdqu (%rsp), %xmm0
+;   movdqu 0x10(%rsp), %xmm1
+;   movdqu 0x20(%rsp), %xmm2
+;   callq *%r8
+;   movdqu %xmm0, 0x30(%rsp)
+;   movdqu (%rsp), %xmm4
+;   pshufd $1, %xmm4, %xmm0
+;   movdqu 0x10(%rsp), %xmm2
+;   pshufd $1, %xmm2, %xmm1
+;   movdqu 0x20(%rsp), %xmm3
+;   pshufd $1, %xmm3, %xmm2
+;   movabsq $0, %r9 ; reloc_external Abs8 %FmaF32 0
+;   callq *%r9
+;   movdqu %xmm0, 0x40(%rsp)
+;   movdqu (%rsp), %xmm14
+;   pshufd $2, %xmm14, %xmm0
+;   movdqu 0x10(%rsp), %xmm13
+;   pshufd $2, %xmm13, %xmm1
+;   movdqu 0x20(%rsp), %xmm15
+;   pshufd $2, %xmm15, %xmm2
+;   movabsq $0, %r10 ; reloc_external Abs8 %FmaF32 0
+;   callq *%r10
+;   movdqu %xmm0, 0x50(%rsp)
+;   movdqu (%rsp), %xmm14
+;   pshufd $3, %xmm14, %xmm0
+;   movdqu 0x10(%rsp), %xmm1
+;   pshufd $3, %xmm1, %xmm1
+;   movdqu 0x20(%rsp), %xmm2
+;   pshufd $3, %xmm2, %xmm2
+;   movabsq $0, %r11 ; reloc_external Abs8 %FmaF32 0
+;   callq *%r11
+;   movdqa %xmm0, %xmm13
+;   movdqu 0x40(%rsp), %xmm4
+;   movdqu 0x30(%rsp), %xmm0
+;   insertps $0x10, %xmm4, %xmm0
+;   movdqu 0x50(%rsp), %xmm10
+;   insertps $0x20, %xmm10, %xmm0
+;   movdqa %xmm13, %xmm1
+;   insertps $0x30, %xmm1, %xmm0
+;   addq $0x60, %rsp
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %fma_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+;   subq    %rsp, $64, %rsp
+; block0:
+;   movdqu  %xmm0, rsp(0 + virtual offset)
+;   movdqu  %xmm1, rsp(16 + virtual offset)
+;   movdqu  %xmm2, rsp(32 + virtual offset)
+;   load_ext_name %FmaF64+0, %r8
+;   movdqu  rsp(0 + virtual offset), %xmm0
+;   movdqu  rsp(16 + virtual offset), %xmm1
+;   movdqu  rsp(32 + virtual offset), %xmm2
+;   call    *%r8
+;   movdqu  %xmm0, rsp(48 + virtual offset)
+;   movdqu  rsp(0 + virtual offset), %xmm0
+;   pshufd  $238, %xmm0, %xmm0
+;   movdqu  rsp(16 + virtual offset), %xmm1
+;   pshufd  $238, %xmm1, %xmm1
+;   movdqu  rsp(32 + virtual offset), %xmm2
+;   pshufd  $238, %xmm2, %xmm2
+;   load_ext_name %FmaF64+0, %r9
+;   call    *%r9
+;   movdqa  %xmm0, %xmm14
+;   movdqu  rsp(48 + virtual offset), %xmm0
+;   movlhps %xmm0, %xmm14, %xmm0
+;   addq    %rsp, $64, %rsp
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+;   subq $0x40, %rsp
+; block1: ; offset 0x8
+;   movdqu %xmm0, (%rsp)
+;   movdqu %xmm1, 0x10(%rsp)
+;   movdqu %xmm2, 0x20(%rsp)
+;   movabsq $0, %r8 ; reloc_external Abs8 %FmaF64 0
+;   movdqu (%rsp), %xmm0
+;   movdqu 0x10(%rsp), %xmm1
+;   movdqu 0x20(%rsp), %xmm2
+;   callq *%r8
+;   movdqu %xmm0, 0x30(%rsp)
+;   movdqu (%rsp), %xmm0
+;   pshufd $0xee, %xmm0, %xmm0
+;   movdqu 0x10(%rsp), %xmm1
+;   pshufd $0xee, %xmm1, %xmm1
+;   movdqu 0x20(%rsp), %xmm2
+;   pshufd $0xee, %xmm2, %xmm2
+;   movabsq $0, %r9 ; reloc_external Abs8 %FmaF64 0
+;   callq *%r9
+;   movdqa %xmm0, %xmm14
+;   movdqu 0x30(%rsp), %xmm0
+;   movlhps %xmm14, %xmm0
+;   addq $0x40, %rsp
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/fma-inst.clif b/cranelift/filetests/filetests/isa/x64/fma-inst.clif
index 16f6ca226..e8697e66c 100644
--- a/cranelift/filetests/filetests/isa/x64/fma-inst.clif
+++ b/cranelift/filetests/filetests/isa/x64/fma-inst.clif
@@ -1,7 +1,7 @@
 test compile precise-output
 target x86_64 has_avx=true has_fma=true
 
-function %fma_f32(f32, f32, f32) -> f32 {
+function %vfmadd213ss(f32, f32, f32) -> f32 {
 block0(v0: f32, v1: f32, v2: f32):
     v3 = fma v0, v1, v2
     return v3
@@ -26,8 +26,59 @@ block0(v0: f32, v1: f32, v2: f32):
 ;   popq %rbp
 ;   retq
 
-function %fma_f64(f64, f64, f64) -> f64 {
-block0(v0: f64, v1: f64, v2: f64):
+function %vfmadd213sd(f64, f64, i64) -> f64 {
+block0(v0: f64, v1: f64, v2: i64):
+    v3 = load.f64 v2
+    v4 = fma v0, v1, v3
+    return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfmadd213sd %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfmadd213sd (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfmadd213ps(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+    v3 = fma v0, v1, v2
+    return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfmadd213ps %xmm0, %xmm1, %xmm2, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfmadd213ps %xmm2, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfmadd213pd(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
     v3 = fma v0, v1, v2
     return v3
 }
@@ -36,7 +87,325 @@ block0(v0: f64, v1: f64, v2: f64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vfmadd213sd %xmm0, %xmm1, %xmm2, %xmm0
+;   vfmadd213pd %xmm0, %xmm1, %xmm2, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfmadd213pd %xmm2, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfmadd132ss(f32, i64, f32) -> f32 {
+block0(v0: f32, v1: i64, v2: f32):
+    v3 = load.f32 v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfmadd132ss %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfmadd132ss (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfmadd132sd(i64, f64, f64) -> f64 {
+block0(v0: i64, v1: f64, v2: f64):
+    v3 = load.f64 v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfmadd132sd %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfmadd132sd (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfmadd132ps(f32x4, i64, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: i64, v2: f32x4):
+    v3 = load.f32x4 v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfmadd132ps %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfmadd132ps (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfmadd132pd(i64, f64x2, f64x2) -> f64x2 {
+block0(v0: i64, v1: f64x2, v2: f64x2):
+    v3 = load.f64x2 v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfmadd132pd %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfmadd132pd (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd213ss(f32, f32, f32) -> f32 {
+block0(v0: f32, v1: f32, v2: f32):
+    v3 = fneg v0
+    v4 = fma v3, v1, v2
+    return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd213ss %xmm0, %xmm1, %xmm2, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfnmadd213ss %xmm2, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd213sd(f64, f64, f64) -> f64 {
+block0(v0: f64, v1: f64, v2: f64):
+    v3 = fneg v1
+    v4 = fma v0, v3, v2
+    return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd213sd %xmm0, %xmm1, %xmm2, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfnmadd213sd %xmm2, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd213ps(f32x4, f32x4, i64) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: i64):
+    v3 = fneg v0
+    v4 = load.f32x4 v2
+    v5 = fma v3, v1, v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd213ps %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfnmadd213ps (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd213pd(f64x2, f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: i64):
+    v3 = fneg v1
+    v4 = load.f64x2 v2
+    v5 = fma v0, v3, v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd213pd %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfnmadd213pd (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd132ss(f32, i64, f32) -> f32 {
+block0(v0: f32, v1: i64, v2: f32):
+    v3 = fneg v0
+    v4 = load.f32 v1
+    v5 = fma v3, v4, v2
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd132ss %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfnmadd132ss (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd132sd(i64, f64, f64) -> f64 {
+block0(v0: i64, v1: f64, v2: f64):
+    v3 = fneg v1
+    v4 = load.f64 v0
+    v5 = fma v4, v3, v2
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd132sd %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfnmadd132sd (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd132ps(i64, f32x4, f32x4) -> f32x4 {
+block0(v0: i64, v1: f32x4, v2: f32x4):
+    v3 = load.f32x4 v0
+    v4 = fneg v3
+    v5 = fma v4, v1, v2
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd132ps %xmm0, %xmm1, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vfnmadd132ps (%rdi), %xmm1, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vfnmadd132pd(f64x2, i64, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: i64, v2: f64x2):
+    v3 = load.f64x2 v1
+    v4 = fneg v3
+    v5 = fma v0, v4, v2
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vfnmadd132pd %xmm0, %xmm1, 0(%rdi), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -46,7 +415,7 @@ block0(v0: f64, v1: f64, v2: f64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   vfmadd213sd %xmm2, %xmm1, %xmm0
+;   vfnmadd132pd (%rdi), %xmm1, %xmm0 ; trap: heap_oob
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
index 88225f5a8..479844fe6 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
@@ -13,8 +13,7 @@ block0(v0: f32x4, v1: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movups  0(%rdi), %xmm4
-;   vorps   %xmm0, %xmm4, %xmm0
+;   vorps   %xmm0, 0(%rdi), %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -24,8 +23,7 @@ block0(v0: f32x4, v1: i64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movups (%rdi), %xmm4
-;   vorps %xmm4, %xmm0, %xmm0
+;   vorps (%rdi), %xmm0, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -42,12 +40,11 @@ block0(v0: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movss   0(%rdi), %xmm7
-;   movl    $-2147483648, %ecx
-;   movd    %ecx, %xmm5
-;   vandnps %xmm5, const(0), %xmm8
-;   vandps  %xmm5, %xmm7, %xmm9
-;   vorps   %xmm8, %xmm9, %xmm0
+;   movl    $-2147483648, %eax
+;   movd    %eax, %xmm4
+;   vandnps %xmm4, const(0), %xmm6
+;   vandps  %xmm4, 0(%rdi), %xmm8
+;   vorps   %xmm6, %xmm8, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -57,12 +54,11 @@ block0(v0: i64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movss (%rdi), %xmm7
-;   movl $0x80000000, %ecx
-;   movd %ecx, %xmm5
-;   vandnps 0x17(%rip), %xmm5, %xmm8
-;   vandps %xmm7, %xmm5, %xmm9
-;   vorps %xmm9, %xmm8, %xmm0
+;   movl $0x80000000, %eax
+;   movd %eax, %xmm4
+;   vandnps 0x1b(%rip), %xmm4, %xmm6
+;   vandps (%rdi), %xmm4, %xmm8
+;   vorps %xmm8, %xmm6, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -78,6 +74,8 @@ block0(v0: i64):
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
 
 function %bor_f32x4(f32x4, f32x4) -> f32x4 {
 block0(v0: f32x4, v1: f32x4):
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
index 24950d773..1ecdf31ed 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
@@ -753,8 +753,8 @@ block0(v0: i64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pextrd.w $0, %xmm0, %rdx
-;   pextrd.w $1, %xmm0, %r9
+;   pextrq  $0, %xmm0, %rdx
+;   pextrq  $1, %xmm0, %r9
 ;   sarq    $36, %rdx, %rdx
 ;   sarq    $36, %r9, %r9
 ;   uninit  %xmm0
@@ -789,8 +789,8 @@ block0(v0: i64x2, v1: i32):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pextrd.w $0, %xmm0, %r8
-;   pextrd.w $1, %xmm0, %r10
+;   pextrq  $0, %xmm0, %r8
+;   pextrq  $1, %xmm0, %r10
 ;   movq    %rdi, %rcx
 ;   sarq    %cl, %r8, %r8
 ;   sarq    %cl, %r10, %r10
diff --git a/cranelift/filetests/filetests/isa/x64/simd-load-extend.clif b/cranelift/filetests/filetests/isa/x64/simd-load-extend.clif
new file mode 100644
index 000000000..3bca887ff
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/simd-load-extend.clif
@@ -0,0 +1,154 @@
+test compile precise-output
+set enable_simd
+target x86_64
+
+function %uload8x8(i64) -> i16x8 {
+block0(v0: i64):
+  v1 = uload8x8 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pmovzxbw 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pmovzxbw (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %sload8x8(i64) -> i16x8 {
+block0(v0: i64):
+  v1 = sload8x8 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pmovsxbw 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pmovsxbw (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %uload16x4(i64) -> i32x4 {
+block0(v0: i64):
+  v1 = uload16x4 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pmovzxwd 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pmovzxwd (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %sload16x4(i64) -> i32x4 {
+block0(v0: i64):
+  v1 = sload16x4 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pmovsxwd 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pmovsxwd (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %uload32x2(i64) -> i64x2 {
+block0(v0: i64):
+  v1 = uload32x2 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pmovzxdq 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pmovzxdq (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %sload32x2(i64) -> i64x2 {
+block0(v0: i64):
+  v1 = sload32x2 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pmovsxdq 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pmovsxdq (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/runtests/br_table.clif b/cranelift/filetests/filetests/runtests/br_table.clif
index 7151adcaa..f5ccc43cd 100644
--- a/cranelift/filetests/filetests/runtests/br_table.clif
+++ b/cranelift/filetests/filetests/runtests/br_table.clif
@@ -100,3 +100,32 @@ block1(v5: i32):
 ; run: %br_table_i32_inline_varied(4) == 4
 ; run: %br_table_i32_inline_varied(297) == 4
 ; run: %br_table_i32_inline_varied(65535) == 4
+
+
+
+; This is a regression test for #5831.
+; The riscv64 backend was failing to clear the upper half of the
+; index register on a br_table, which caused it to jump to the wrong
+; block.
+function %br_table_upper_reg() -> i32 {
+block0:
+    v0 = iconst.i32 -555163938
+    v1 = iconst.i8 -34
+    jump block1(v0, v1)
+
+block1(v2: i32, v3: i8):
+    v4 = ishl.i32 v2, v2
+    v5 = rotr v4, v3
+    br_table v5, block2, [block2, block2, block3]
+
+block2 cold:
+    v100 = iconst.i32 100
+    return v100
+
+block3 cold:
+    v200 = iconst.i32 200
+    return v200
+}
+
+
+; run: %br_table_upper_reg() == 200
diff --git a/cranelift/filetests/filetests/runtests/simd-fma.clif b/cranelift/filetests/filetests/runtests/simd-fma.clif
index 4ff5e5104..37bb30a88 100644
--- a/cranelift/filetests/filetests/runtests/simd-fma.clif
+++ b/cranelift/filetests/filetests/runtests/simd-fma.clif
@@ -1,6 +1,7 @@
 test interpret
 test run
 target x86_64 has_avx has_fma
+target x86_64 has_avx=false has_fma=false
 target aarch64
 
 function %fma_f32x4(f32x4, f32x4, f32x4) -> f32x4 {