Repository: bytecodealliance/wasmtime
Pull request #5997 (2023-03-12 02:10:08)

aarch64: Use `VCodeConstant` for f64/v128 constants
As was [pointed out recently](https://github.com/bytecodealliance/wasmtime/pull/5976#discussion_r1131926356) one minor improvement to the AArch64 backend is to use the same infrastructure that x64 uses for loading constants which is to pool them at the end of the function instead of having them inline in the body. This commit implements this strategy to ensure that fallback `shuffle` instructions, for example, don't generate a branch instruction to skip over immediates.

Additionally in these commits is a refactoring of float/splat lowering in the aarch64 backend. Previously it was all living in Rust but to get to know it I translated it all to ISLE. I felt that the logic was a bit more understandable after the transition and more easily shows off what all the special cases are (the problem is more well-suited for ISLE I think than Rust, nothing against the prior code of course which was perfectly readable).


Patch:
diff --git a/cranelift/codegen/meta/src/isa/riscv64.rs b/cranelift/codegen/meta/src/isa/riscv64.rs
index 00b3bd6c6..3e0d88302 100644
--- a/cranelift/codegen/meta/src/isa/riscv64.rs
+++ b/cranelift/codegen/meta/src/isa/riscv64.rs
@@ -36,7 +36,7 @@ fn define_settings(_shared: &SettingGroup) -> SettingGroup {
         "Zbc: Carry-less multiplication",
         false,
     );
-    let _has_zbx = setting.add_bool(
+    let _has_zbs = setting.add_bool(
         "has_zbs",
         "has extension zbs?",
         "Zbs: Single-bit instructions",
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs
index f36790f26..3ab1e0b1a 100755
--- a/cranelift/codegen/meta/src/shared/instructions.rs
+++ b/cranelift/codegen/meta/src/shared/instructions.rs
@@ -1422,7 +1422,7 @@ pub(crate) fn define(
         Shuffle two vectors using the given immediate bytes. For each of the 16 bytes of the
         immediate, a value i of 0-15 selects the i-th element of the first vector and a value i of
         16-31 selects the (i-16)th element of the second vector. Immediate values outside of the
-        0-31 range place a 0 in the resulting vector lane.
+        0-31 range are not valid.
         "#,
             &formats.shuffle,
         )
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index f035789c1..8d182a92d 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -466,14 +466,6 @@
         (mem PairAMode)
         (flags MemFlags))
 
-       (LoadFpuConst64
-        (rd WritableReg)
-        (const_data u64))
-
-       (LoadFpuConst128
-        (rd WritableReg)
-        (const_data u128))
-
        ;; Conversion: FP -> integer.
        (FpuToInt
         (op FpuToIntOp)
@@ -1135,6 +1127,11 @@
          (off i64)
          (ty Type))
 
+        ;; A reference to a constant which is placed outside of the function's
+        ;; body, typically at the end.
+        (Const
+          (addr VCodeConstant))
+
         ;; Offset from the "nominal stack pointer", which is where the real SP is
         ;; just after stack and spill slots are allocated in the function prologue.
         ;; At emission time, this is converted to `SPOffset` with a fixup added to
@@ -1194,6 +1191,16 @@
 (rule (lane_size (dynamic_lane 32 _)) (ScalarSize.Size32))
 (rule (lane_size (dynamic_lane 64 _)) (ScalarSize.Size64))
 
+;; Helper for extracting the size of a lane from the input `VectorSize`
+(decl pure vector_lane_size (VectorSize) ScalarSize)
+(rule (vector_lane_size (VectorSize.Size8x16)) (ScalarSize.Size8))
+(rule (vector_lane_size (VectorSize.Size8x8))  (ScalarSize.Size8))
+(rule (vector_lane_size (VectorSize.Size16x8)) (ScalarSize.Size16))
+(rule (vector_lane_size (VectorSize.Size16x4)) (ScalarSize.Size16))
+(rule (vector_lane_size (VectorSize.Size32x4)) (ScalarSize.Size32))
+(rule (vector_lane_size (VectorSize.Size32x2)) (ScalarSize.Size32))
+(rule (vector_lane_size (VectorSize.Size64x2)) (ScalarSize.Size64))
+
 (type Cond extern
   (enum
     (Eq)
@@ -1908,6 +1915,13 @@
             (_ Unit (emit (MInst.VecDupFromFpu dst src size lane))))
         dst))
 
+;; Helper for emitting `MInst.VecDupImm` instructions.
+(decl vec_dup_imm (ASIMDMovModImm bool VectorSize) Reg)
+(rule (vec_dup_imm imm invert size)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.VecDupImm dst imm invert size))))
+        dst))
+
 ;; Helper for emitting `MInst.AluRRImm12` instructions.
 (decl alu_rr_imm12 (ALUOp Type Reg Imm12) Reg)
 (rule (alu_rr_imm12 op ty src imm)
@@ -2158,6 +2172,13 @@
             (_ Unit (emit (MInst.MovToFpu dst x size))))
         dst))
 
+;; Helper for emitting `MInst.FpuMoveFPImm` instructions.
+(decl fpu_move_fp_imm (ASIMDFPModImm ScalarSize) Reg)
+(rule (fpu_move_fp_imm imm size)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.FpuMoveFPImm dst imm size))))
+        dst))
+
 ;; Helper for emitting `MInst.MovToVec` instructions.
 (decl mov_to_vec (Reg Reg u8 VectorSize) Reg)
 (rule (mov_to_vec src1 src2 lane size)
@@ -2986,24 +3007,122 @@
            (amode ty addr offset)))
 
 ;; Lower a constant f32.
-(decl constant_f32 (u64) Reg)
-;; TODO: Port lower_constant_f32() to ISLE.
-(extern constructor constant_f32 constant_f32)
+;;
+;; Note that we must make sure that all bits outside the lowest 32 are set to 0
+;; because this function is also used to load wider constants (that have zeros
+;; in their most significant bits).
+(decl constant_f32 (u32) Reg)
+(rule 2 (constant_f32 0)
+        (vec_dup_imm (asimd_mov_mod_imm_zero (ScalarSize.Size32))
+                     $false
+                     (VectorSize.Size32x2)))
+(rule 1 (constant_f32 n)
+        (if-let imm (asimd_fp_mod_imm_from_u64 n (ScalarSize.Size32)))
+        (fpu_move_fp_imm imm (ScalarSize.Size32)))
+(rule (constant_f32 n)
+      (mov_to_fpu (imm $I32 (ImmExtend.Zero) n) (ScalarSize.Size32)))
 
 ;; Lower a constant f64.
+;;
+;; Note that we must make sure that all bits outside the lowest 64 are set to 0
+;; because this function is also used to load wider constants (that have zeros
+;; in their most significant bits).
+;; TODO: Treat as half of a 128 bit vector and consider replicated patterns.
+;; Scalar MOVI might also be an option.
 (decl constant_f64 (u64) Reg)
-;; TODO: Port lower_constant_f64() to ISLE.
-(extern constructor constant_f64 constant_f64)
+(rule 4 (constant_f64 0)
+        (vec_dup_imm (asimd_mov_mod_imm_zero (ScalarSize.Size32))
+                     $false
+                     (VectorSize.Size32x2)))
+(rule 3 (constant_f64 n)
+        (if-let imm (asimd_fp_mod_imm_from_u64 n (ScalarSize.Size64)))
+        (fpu_move_fp_imm imm (ScalarSize.Size64)))
+(rule 2 (constant_f64 (u64_as_u32 n))
+        (constant_f32 n))
+(rule 1 (constant_f64 (u64_low32_bits_unset n))
+        (mov_to_fpu (imm $I64 (ImmExtend.Zero) n) (ScalarSize.Size64)))
+(rule (constant_f64 n)
+      (fpu_load64 (AMode.Const (emit_u64_le_const n)) (mem_flags_trusted)))
+
+;; Tests whether the low 32 bits in the input are all zero.
+(decl u64_low32_bits_unset (u64) u64)
+(extern extractor u64_low32_bits_unset u64_low32_bits_unset)
 
 ;; Lower a constant f128.
 (decl constant_f128 (u128) Reg)
-;; TODO: Port lower_constant_f128() to ISLE.
-(extern constructor constant_f128 constant_f128)
+(rule 3 (constant_f128 0)
+        (vec_dup_imm (asimd_mov_mod_imm_zero (ScalarSize.Size8))
+                     $false
+                     (VectorSize.Size8x16)))
+
+;; If the upper 64-bits are all zero then defer to `constant_f64`.
+(rule 2 (constant_f128 (u128_as_u64 n)) (constant_f64 n))
+
+;; If the low half of the u128 equals the high half then delegate to the splat
+;; logic as a splat of a 64-bit value.
+(rule 1 (constant_f128 (u128_replicated_u64 n))
+        (splat_const n (VectorSize.Size64x2)))
+
+;; Base case is to load the constant from memory.
+(rule (constant_f128 n)
+      (fpu_load128 (AMode.Const (emit_u128_le_const n)) (mem_flags_trusted)))
 
 ;; Lower a vector splat with a constant parameter.
+;;
+;; The 64-bit input here only uses the low bits for the lane size in
+;; `VectorSize` and all other bits are ignored.
 (decl splat_const (u64 VectorSize) Reg)
-;; TODO: Port lower_splat_const() to ISLE.
-(extern constructor splat_const splat_const)
+
+;; If the splat'd constant can itself be reduced in size then attempt to do so
+;; as it will make it easier to create the immediates in the instructions below.
+(rule 5 (splat_const (u64_replicated_u32 n) (VectorSize.Size64x2))
+        (splat_const n (VectorSize.Size32x4)))
+(rule 5 (splat_const (u32_replicated_u16 n) (VectorSize.Size32x4))
+        (splat_const n (VectorSize.Size16x8)))
+(rule 5 (splat_const (u32_replicated_u16 n) (VectorSize.Size32x2))
+        (splat_const n (VectorSize.Size16x4)))
+(rule 5 (splat_const (u16_replicated_u8 n) (VectorSize.Size16x8))
+        (splat_const n (VectorSize.Size8x16)))
+(rule 5 (splat_const (u16_replicated_u8 n) (VectorSize.Size16x4))
+        (splat_const n (VectorSize.Size8x8)))
+
+;; Special cases for `vec_dup_imm` instructions where the input is either
+;; negated or not.
+(rule 4 (splat_const n size)
+        (if-let imm (asimd_mov_mod_imm_from_u64 n (vector_lane_size size)))
+        (vec_dup_imm imm $false size))
+(rule 3 (splat_const n size)
+        (if-let imm (asimd_mov_mod_imm_from_u64 (u64_not n) (vector_lane_size size)))
+        (vec_dup_imm imm $true size))
+
+;; Special case a 32-bit splat where an immediate can be created by
+;; concatenating the 32-bit constant into a 64-bit value
+(rule 2 (splat_const n (VectorSize.Size32x4))
+        (if-let imm (asimd_mov_mod_imm_from_u64 (u64_or n (u64_shl n 32)) (ScalarSize.Size64)))
+        (vec_dup_imm imm $false (VectorSize.Size64x2)))
+(rule 2 (splat_const n (VectorSize.Size32x2))
+        (if-let imm (asimd_mov_mod_imm_from_u64 (u64_or n (u64_shl n 32)) (ScalarSize.Size64)))
+        (fpu_extend (vec_dup_imm imm $false (VectorSize.Size64x2)) (ScalarSize.Size64)))
+
+(rule 1 (splat_const n size)
+        (if-let imm (asimd_fp_mod_imm_from_u64 n (vector_lane_size size)))
+        (vec_dup_fp_imm imm size))
+
+;; The base case for splat is to use `vec_dup` with the immediate loaded into a
+;; register.
+(rule (splat_const n size)
+      (vec_dup (imm $I64 (ImmExtend.Zero) n) size))
+
+;; Each of these extractors tests whether the upper half of the input equals the
+;; lower half of the input
+(decl u128_replicated_u64 (u64) u128)
+(extern extractor u128_replicated_u64 u128_replicated_u64)
+(decl u64_replicated_u32 (u64) u64)
+(extern extractor u64_replicated_u32 u64_replicated_u32)
+(decl u32_replicated_u16 (u64) u64)
+(extern extractor u32_replicated_u16 u32_replicated_u16)
+(decl u16_replicated_u8 (u64) u64)
+(extern extractor u16_replicated_u8 u16_replicated_u8)
 
 ;; Lower a FloatCC to a Cond.
 (decl fp_cond_code (FloatCC) Cond)
@@ -3814,3 +3933,36 @@
 ;; Helper for emitting the `trn2` instruction
 (decl vec_trn2 (Reg Reg VectorSize) Reg)
 (rule (vec_trn2 rn rm size) (vec_rrr (VecALUOp.Trn2) rn rm size))
+
+;; Helper for creating a zero value `ASIMDMovModImm` immediate.
+(decl asimd_mov_mod_imm_zero (ScalarSize) ASIMDMovModImm)
+(extern constructor asimd_mov_mod_imm_zero asimd_mov_mod_imm_zero)
+
+;; Helper for fallibly creating an `ASIMDMovModImm` immediate from its parts.
+(decl pure partial asimd_mov_mod_imm_from_u64 (u64 ScalarSize) ASIMDMovModImm)
+(extern constructor asimd_mov_mod_imm_from_u64 asimd_mov_mod_imm_from_u64)
+
+;; Helper for fallibly creating an `ASIMDFPModImm` immediate from its parts.
+(decl pure partial asimd_fp_mod_imm_from_u64 (u64 ScalarSize) ASIMDFPModImm)
+(extern constructor asimd_fp_mod_imm_from_u64 asimd_fp_mod_imm_from_u64)
+
+;; Helper for creating a `VecDupFPImm` instruction
+(decl vec_dup_fp_imm (ASIMDFPModImm VectorSize) Reg)
+(rule (vec_dup_fp_imm imm size)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.VecDupFPImm dst imm size))))
+       dst))
+
+;; Helper for creating a `FpuLoad64` instruction
+(decl fpu_load64 (AMode MemFlags) Reg)
+(rule (fpu_load64 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.FpuLoad64 dst amode flags))))
+       dst))
+
+;; Helper for creating a `FpuLoad128` instruction
+(decl fpu_load128 (AMode MemFlags) Reg)
+(rule (fpu_load128 amode flags)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.FpuLoad128 dst amode flags))))
+       dst))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs
index 69eb7e525..1c29591ba 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/args.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs
@@ -124,6 +124,9 @@ pub enum MemLabel {
     /// offset from this instruction. This form must be used at emission time;
     /// see `memlabel_finalize()` for how other forms are lowered to this one.
     PCRel(i32),
+    /// An address that refers to a label within a `MachBuffer`, for example a
+    /// constant that lives in the pool at the end of the function.
+    Mach(MachLabel),
 }
 
 impl AMode {
@@ -194,6 +197,7 @@ impl AMode {
             | &AMode::FPOffset { .. }
             | &AMode::SPOffset { .. }
             | &AMode::NominalSPOffset { .. }
+            | &AMode::Const { .. }
             | AMode::Label { .. } => self.clone(),
         }
     }
@@ -382,7 +386,8 @@ impl PrettyPrint for ExtendOp {
 impl PrettyPrint for MemLabel {
     fn pretty_print(&self, _: u8, _: &mut AllocationConsumer<'_>) -> String {
         match self {
-            &MemLabel::PCRel(off) => format!("pc+{}", off),
+            MemLabel::PCRel(off) => format!("pc+{}", off),
+            MemLabel::Mach(off) => format!("label({})", off.get()),
         }
     }
 }
@@ -465,6 +470,8 @@ impl PrettyPrint for AMode {
                 let simm9 = simm9.pretty_print(8, allocs);
                 format!("[sp], {}", simm9)
             }
+            AMode::Const { addr } => format!("[const({})]", addr.as_u32()),
+
             // Eliminated by `mem_finalize()`.
             &AMode::SPOffset { .. }
             | &AMode::FPOffset { .. }
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 2332ff19b..4f0288dc2 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -2,7 +2,7 @@
 
 use regalloc2::Allocation;
 
-use crate::binemit::{CodeOffset, Reloc, StackMap};
+use crate::binemit::{Reloc, StackMap};
 use crate::ir::{types::*, RelSourceLoc};
 use crate::ir::{LibCall, MemFlags, TrapCode};
 use crate::isa::aarch64::inst::*;
@@ -10,20 +10,12 @@ use crate::machinst::{ty_bits, Reg, RegClass, Writable};
 use crate::trace;
 use core::convert::TryFrom;
 
-/// Memory label/reference finalization: convert a MemLabel to a PC-relative
-/// offset, possibly emitting relocation(s) as necessary.
-pub fn memlabel_finalize(_insn_off: CodeOffset, label: &MemLabel) -> i32 {
-    match label {
-        &MemLabel::PCRel(rel) => rel,
-    }
-}
-
 /// Memory addressing mode finalization: convert "special" modes (e.g.,
 /// generic arbitrary stack offset) into real addressing modes, possibly by
 /// emitting some helper instructions that come immediately before the use
 /// of this amode.
 pub fn mem_finalize(
-    insn_off: CodeOffset,
+    sink: Option<&mut MachBuffer<Inst>>,
     mem: &AMode,
     state: &EmitState,
 ) -> (SmallVec<[Inst; 4]>, AMode) {
@@ -74,14 +66,14 @@ pub fn mem_finalize(
             }
         }
 
-        &AMode::Label { ref label } => {
-            let off = memlabel_finalize(insn_off, label);
-            (
-                smallvec![],
-                AMode::Label {
-                    label: MemLabel::PCRel(off),
-                },
-            )
+        AMode::Const { addr } => {
+            let sink = match sink {
+                Some(sink) => sink,
+                None => return (smallvec![], mem.clone()),
+            };
+            let label = sink.get_label_for_constant(*addr);
+            let label = MemLabel::Mach(label);
+            (smallvec![], AMode::Label { label })
         }
 
         _ => (smallvec![], mem.clone()),
@@ -959,7 +951,7 @@ impl MachInstEmit for Inst {
             | &Inst::FpuLoad128 { rd, ref mem, flags } => {
                 let rd = allocs.next_writable(rd);
                 let mem = mem.with_allocs(&mut allocs);
-                let (mem_insts, mem) = mem_finalize(sink.cur_offset(), &mem, state);
+                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, state);
 
                 for inst in mem_insts.into_iter() {
                     inst.emit(&[], sink, emit_info, state);
@@ -1039,7 +1031,19 @@ impl MachInstEmit for Inst {
                     &AMode::Label { ref label } => {
                         let offset = match label {
                             // cast i32 to u32 (two's-complement)
-                            &MemLabel::PCRel(off) => off as u32,
+                            MemLabel::PCRel(off) => *off as u32,
+                            // Emit a relocation into the `MachBuffer`
+                            // for the label that's being loaded from and
+                            // encode an address of 0 in its place which will
+                            // get filled in by relocation resolution later on.
+                            MemLabel::Mach(label) => {
+                                sink.use_label_at_offset(
+                                    sink.cur_offset(),
+                                    *label,
+                                    LabelUse::Ldr19,
+                                );
+                                0
+                            }
                         } / 4;
                         assert!(offset < (1 << 19));
                         match self {
@@ -1076,6 +1080,7 @@ impl MachInstEmit for Inst {
                     &AMode::SPOffset { .. }
                     | &AMode::FPOffset { .. }
                     | &AMode::NominalSPOffset { .. }
+                    | &AMode::Const { .. }
                     | &AMode::RegOffset { .. } => {
                         panic!("Should not see {:?} here!", mem)
                     }
@@ -1091,7 +1096,7 @@ impl MachInstEmit for Inst {
             | &Inst::FpuStore128 { rd, ref mem, flags } => {
                 let rd = allocs.next(rd);
                 let mem = mem.with_allocs(&mut allocs);
-                let (mem_insts, mem) = mem_finalize(sink.cur_offset(), &mem, state);
+                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, state);
 
                 for inst in mem_insts.into_iter() {
                     inst.emit(&[], sink, emit_info, state);
@@ -1172,6 +1177,7 @@ impl MachInstEmit for Inst {
                     &AMode::SPOffset { .. }
                     | &AMode::FPOffset { .. }
                     | &AMode::NominalSPOffset { .. }
+                    | &AMode::Const { .. }
                     | &AMode::RegOffset { .. } => {
                         panic!("Should not see {:?} here!", mem)
                     }
@@ -2319,41 +2325,6 @@ impl MachInstEmit for Inst {
                 };
                 sink.put4(enc_inttofpu(top16, rd, rn));
             }
-            &Inst::LoadFpuConst64 { rd, const_data } => {
-                let rd = allocs.next_writable(rd);
-                let inst = Inst::FpuLoad64 {
-                    rd,
-                    mem: AMode::Label {
-                        label: MemLabel::PCRel(8),
-                    },
-                    flags: MemFlags::trusted(),
-                };
-                inst.emit(&[], sink, emit_info, state);
-                let inst = Inst::Jump {
-                    dest: BranchTarget::ResolvedOffset(12),
-                };
-                inst.emit(&[], sink, emit_info, state);
-                sink.put8(const_data);
-            }
-            &Inst::LoadFpuConst128 { rd, const_data } => {
-                let rd = allocs.next_writable(rd);
-                let inst = Inst::FpuLoad128 {
-                    rd,
-                    mem: AMode::Label {
-                        label: MemLabel::PCRel(8),
-                    },
-                    flags: MemFlags::trusted(),
-                };
-                inst.emit(&[], sink, emit_info, state);
-                let inst = Inst::Jump {
-                    dest: BranchTarget::ResolvedOffset(20),
-                };
-                inst.emit(&[], sink, emit_info, state);
-
-                for i in const_data.to_le_bytes().iter() {
-                    sink.put1(*i);
-                }
-            }
             &Inst::FpuCSel32 { rd, rn, rm, cond } => {
                 let rd = allocs.next_writable(rd);
                 let rn = allocs.next(rn);
@@ -3350,7 +3321,7 @@ impl MachInstEmit for Inst {
             &Inst::LoadAddr { rd, ref mem } => {
                 let rd = allocs.next_writable(rd);
                 let mem = mem.with_allocs(&mut allocs);
-                let (mem_insts, mem) = mem_finalize(sink.cur_offset(), &mem, state);
+                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, state);
                 for inst in mem_insts.into_iter() {
                     inst.emit(&[], sink, emit_info, state);
                 }
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 6e8e602de..22487b264 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -10,7 +10,6 @@ use crate::{settings, CodegenError, CodegenResult};
 use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
 
 use alloc::vec::Vec;
-use core::convert::TryFrom;
 use regalloc2::{PRegSet, VReg};
 use smallvec::{smallvec, SmallVec};
 use std::string::{String, ToString};
@@ -250,215 +249,6 @@ impl Inst {
         }
     }
 
-    /// Create instructions that load a 32-bit floating-point constant.
-    pub fn load_fp_constant32<F: FnMut(Type) -> Writable<Reg>>(
-        rd: Writable<Reg>,
-        const_data: u32,
-        mut alloc_tmp: F,
-    ) -> SmallVec<[Inst; 4]> {
-        // Note that we must make sure that all bits outside the lowest 32 are set to 0
-        // because this function is also used to load wider constants (that have zeros
-        // in their most significant bits).
-        if const_data == 0 {
-            smallvec![Inst::VecDupImm {
-                rd,
-                imm: ASIMDMovModImm::zero(ScalarSize::Size32),
-                invert: false,
-                size: VectorSize::Size32x2,
-            }]
-        } else if let Some(imm) =
-            ASIMDFPModImm::maybe_from_u64(const_data.into(), ScalarSize::Size32)
-        {
-            smallvec![Inst::FpuMoveFPImm {
-                rd,
-                imm,
-                size: ScalarSize::Size32,
-            }]
-        } else {
-            let tmp = alloc_tmp(I32);
-            let mut insts = Inst::load_constant(tmp, const_data as u64, &mut alloc_tmp);
-
-            insts.push(Inst::MovToFpu {
-                rd,
-                rn: tmp.to_reg(),
-                size: ScalarSize::Size32,
-            });
-
-            insts
-        }
-    }
-
-    /// Create instructions that load a 64-bit floating-point constant.
-    pub fn load_fp_constant64<F: FnMut(Type) -> Writable<Reg>>(
-        rd: Writable<Reg>,
-        const_data: u64,
-        mut alloc_tmp: F,
-    ) -> SmallVec<[Inst; 4]> {
-        // Note that we must make sure that all bits outside the lowest 64 are set to 0
-        // because this function is also used to load wider constants (that have zeros
-        // in their most significant bits).
-        // TODO: Treat as half of a 128 bit vector and consider replicated patterns.
-        // Scalar MOVI might also be an option.
-        if const_data == 0 {
-            smallvec![Inst::VecDupImm {
-                rd,
-                imm: ASIMDMovModImm::zero(ScalarSize::Size32),
-                invert: false,
-                size: VectorSize::Size32x2,
-            }]
-        } else if let Some(imm) = ASIMDFPModImm::maybe_from_u64(const_data, ScalarSize::Size64) {
-            smallvec![Inst::FpuMoveFPImm {
-                rd,
-                imm,
-                size: ScalarSize::Size64,
-            }]
-        } else if let Ok(const_data) = u32::try_from(const_data) {
-            Inst::load_fp_constant32(rd, const_data, alloc_tmp)
-        } else if const_data & (u32::MAX as u64) == 0 {
-            let tmp = alloc_tmp(I64);
-            let mut insts = Inst::load_constant(tmp, const_data, &mut alloc_tmp);
-
-            insts.push(Inst::MovToFpu {
-                rd,
-                rn: tmp.to_reg(),
-                size: ScalarSize::Size64,
-            });
-
-            insts
-        } else {
-            smallvec![Inst::LoadFpuConst64 { rd, const_data }]
-        }
-    }
-
-    /// Create instructions that load a 128-bit vector constant.
-    pub fn load_fp_constant128<F: FnMut(Type) -> Writable<Reg>>(
-        rd: Writable<Reg>,
-        const_data: u128,
-        alloc_tmp: F,
-    ) -> SmallVec<[Inst; 5]> {
-        if let Ok(const_data) = u64::try_from(const_data) {
-            SmallVec::from(&Inst::load_fp_constant64(rd, const_data, alloc_tmp)[..])
-        } else if let Some((pattern, size)) =
-            Inst::get_replicated_vector_pattern(const_data, ScalarSize::Size64)
-        {
-            Inst::load_replicated_vector_pattern(
-                rd,
-                pattern,
-                VectorSize::from_lane_size(size, true),
-                alloc_tmp,
-            )
-        } else {
-            smallvec![Inst::LoadFpuConst128 { rd, const_data }]
-        }
-    }
-
-    /// Determine whether a 128-bit constant represents a vector consisting of elements with
-    /// the same value.
-    pub fn get_replicated_vector_pattern(
-        value: u128,
-        size: ScalarSize,
-    ) -> Option<(u64, ScalarSize)> {
-        let (mask, shift, next_size) = match size {
-            ScalarSize::Size8 => (u8::MAX as u128, 8, ScalarSize::Size128),
-            ScalarSize::Size16 => (u16::MAX as u128, 16, ScalarSize::Size8),
-            ScalarSize::Size32 => (u32::MAX as u128, 32, ScalarSize::Size16),
-            ScalarSize::Size64 => (u64::MAX as u128, 64, ScalarSize::Size32),
-            _ => return None,
-        };
-        let mut r = None;
-        let v = value & mask;
-
-        if (value >> shift) & mask == v {
-            r = Inst::get_replicated_vector_pattern(v, next_size);
-
-            if r.is_none() {
-                r = Some((v as u64, size));
-            }
-        }
-
-        r
-    }
-
-    /// Create instructions that load a vector constant consisting of elements with
-    /// the same value.
-    pub fn load_replicated_vector_pattern<F: FnMut(Type) -> Writable<Reg>>(
-        rd: Writable<Reg>,
-        pattern: u64,
-        size: VectorSize,
-        mut alloc_tmp: F,
-    ) -> SmallVec<[Inst; 5]> {
-        let lane_size = size.lane_size();
-        let widen_32_bit_pattern = |pattern, lane_size| {
-            if lane_size == ScalarSize::Size32 {
-                let pattern = pattern as u32 as u64;
-
-                ASIMDMovModImm::maybe_from_u64(pattern | (pattern << 32), ScalarSize::Size64)
-            } else {
-                None
-            }
-        };
-
-        if let Some(imm) = ASIMDMovModImm::maybe_from_u64(pattern, lane_size) {
-            smallvec![Inst::VecDupImm {
-                rd,
-                imm,
-                invert: false,
-                size
-            }]
-        } else if let Some(imm) = ASIMDMovModImm::maybe_from_u64(!pattern, lane_size) {
-            debug_assert_ne!(lane_size, ScalarSize::Size8);
-            debug_assert_ne!(lane_size, ScalarSize::Size64);
-
-            smallvec![Inst::VecDupImm {
-                rd,
-                imm,
-                invert: true,
-                size
-            }]
-        } else if let Some(imm) = widen_32_bit_pattern(pattern, lane_size) {
-            let mut insts = smallvec![];
-
-            // TODO: Implement support for 64-bit scalar MOVI; we zero-extend the
-            // lower 64 bits instead.
-            if !size.is_128bits() {
-                let tmp = alloc_tmp(types::I64X2);
-                insts.push(Inst::VecDupImm {
-                    rd: tmp,
-                    imm,
-                    invert: false,
-                    size: VectorSize::Size64x2,
-                });
-                insts.push(Inst::FpuExtend {
-                    rd,
-                    rn: tmp.to_reg(),
-                    size: ScalarSize::Size64,
-                });
-            } else {
-                insts.push(Inst::VecDupImm {
-                    rd,
-                    imm,
-                    invert: false,
-                    size: VectorSize::Size64x2,
-                });
-            }
-
-            insts
-        } else if let Some(imm) = ASIMDFPModImm::maybe_from_u64(pattern, lane_size) {
-            smallvec![Inst::VecDupFPImm { rd, imm, size }]
-        } else {
-            let tmp = alloc_tmp(I64);
-            let mut insts = SmallVec::from(&Inst::load_constant(tmp, pattern, &mut alloc_tmp)[..]);
-
-            insts.push(Inst::VecDup {
-                rd,
-                rn: tmp.to_reg(),
-                size,
-            });
-
-            insts
-        }
-    }
-
     /// Generic constructor for a load (zero-extending where appropriate).
     pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst {
         match ty {
@@ -585,6 +375,7 @@ fn memarg_operands<F: Fn(VReg) -> VReg>(memarg: &AMode, collector: &mut OperandC
         &AMode::RegOffset { rn, .. } => {
             collector.reg_use(rn);
         }
+        &AMode::Const { .. } => {}
     }
 }
 
@@ -928,9 +719,6 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
             collector.reg_use(rt2);
             pairmemarg_operands(mem, collector);
         }
-        &Inst::LoadFpuConst64 { rd, .. } | &Inst::LoadFpuConst128 { rd, .. } => {
-            collector.reg_def(rd);
-        }
         &Inst::FpuToInt { rd, rn, .. } => {
             collector.reg_def(rd);
             collector.reg_use(rn);
@@ -1318,7 +1106,7 @@ impl MachInst for Inst {
 // Pretty-printing of instructions.
 
 fn mem_finalize_for_show(mem: &AMode, state: &EmitState) -> (String, AMode) {
-    let (mem_insts, mem) = mem_finalize(0, mem, state);
+    let (mem_insts, mem) = mem_finalize(None, mem, state);
     let mut mem_str = mem_insts
         .into_iter()
         .map(|inst| {
@@ -2007,18 +1795,6 @@ impl Inst {
 
                 format!("stp {}, {}, {}", rt, rt2, mem)
             }
-            &Inst::LoadFpuConst64 { rd, const_data } => {
-                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64, allocs);
-                format!(
-                    "ldr {}, pc+8 ; b 12 ; data.f64 {}",
-                    rd,
-                    f64::from_bits(const_data)
-                )
-            }
-            &Inst::LoadFpuConst128 { rd, const_data } => {
-                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size128, allocs);
-                format!("ldr {}, pc+8 ; b 20 ; data.f128 0x{:032x}", rd, const_data)
-            }
             &Inst::FpuToInt { op, rd, rn } => {
                 let (op, sizesrc, sizedest) = match op {
                     FpuToIntOp::F32ToI32 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size32),
@@ -2820,7 +2596,7 @@ impl Inst {
                 // of the existing legalization framework).
                 let rd = allocs.next_writable(rd);
                 let mem = mem.with_allocs(allocs);
-                let (mem_insts, mem) = mem_finalize(0, &mem, state);
+                let (mem_insts, mem) = mem_finalize(None, &mem, state);
                 let mut ret = String::new();
                 for inst in mem_insts.into_iter() {
                     ret.push_str(
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index f3a3db703..2b0d678f1 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -26,7 +26,7 @@
 
 ;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(rule (lower (f32const (u64_from_ieee32 n)))
+(rule (lower (f32const (u32_from_ieee32 n)))
       (constant_f32 n))
 
 ;;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1954,7 +1954,7 @@
 (rule -2 (lower (has_type ty (splat x @ (value_type (ty_scalar_float _)))))
       (vec_dup_from_fpu x (vector_size ty) 0))
 
-(rule (lower (has_type ty (splat (f32const (u64_from_ieee32 n)))))
+(rule (lower (has_type ty (splat (f32const (u32_from_ieee32 n)))))
       (splat_const n (vector_size ty)))
 
 (rule (lower (has_type ty (splat (f64const (u64_from_ieee64 n)))))
diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs
index d219451cf..fcc38d0d5 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower.rs
@@ -570,67 +570,6 @@ pub(crate) fn lower_constant_u64(ctx: &mut Lower<Inst>, rd: Writable<Reg>, value
     }
 }
 
-pub(crate) fn lower_constant_f32(ctx: &mut Lower<Inst>, rd: Writable<Reg>, value: f32) {
-    let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
-
-    for inst in Inst::load_fp_constant32(rd, value.to_bits(), alloc_tmp) {
-        ctx.emit(inst);
-    }
-}
-
-pub(crate) fn lower_constant_f64(ctx: &mut Lower<Inst>, rd: Writable<Reg>, value: f64) {
-    let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
-
-    for inst in Inst::load_fp_constant64(rd, value.to_bits(), alloc_tmp) {
-        ctx.emit(inst);
-    }
-}
-
-pub(crate) fn lower_constant_f128(ctx: &mut Lower<Inst>, rd: Writable<Reg>, value: u128) {
-    if value == 0 {
-        // Fast-track a common case.  The general case, viz, calling `Inst::load_fp_constant128`,
-        // is potentially expensive.
-        ctx.emit(Inst::VecDupImm {
-            rd,
-            imm: ASIMDMovModImm::zero(ScalarSize::Size8),
-            invert: false,
-            size: VectorSize::Size8x16,
-        });
-    } else {
-        let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
-        for inst in Inst::load_fp_constant128(rd, value, alloc_tmp) {
-            ctx.emit(inst);
-        }
-    }
-}
-
-pub(crate) fn lower_splat_const(
-    ctx: &mut Lower<Inst>,
-    rd: Writable<Reg>,
-    value: u64,
-    size: VectorSize,
-) {
-    let (value, narrow_size) = match size.lane_size() {
-        ScalarSize::Size8 => (value as u8 as u64, ScalarSize::Size128),
-        ScalarSize::Size16 => (value as u16 as u64, ScalarSize::Size8),
-        ScalarSize::Size32 => (value as u32 as u64, ScalarSize::Size16),
-        ScalarSize::Size64 => (value, ScalarSize::Size32),
-        _ => unreachable!(),
-    };
-    let (value, size) = match Inst::get_replicated_vector_pattern(value as u128, narrow_size) {
-        Some((value, lane_size)) => (
-            value,
-            VectorSize::from_lane_size(lane_size, size.is_128bits()),
-        ),
-        None => (value, size),
-    };
-    let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
-
-    for inst in Inst::load_replicated_vector_pattern(rd, value, size, alloc_tmp) {
-        ctx.emit(inst);
-    }
-}
-
 pub(crate) fn lower_condcode(cc: IntCC) -> Cond {
     match cc {
         IntCC::Equal => Cond::Eq,
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index 707243d2a..d4545565d 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -7,17 +7,16 @@ use smallvec::SmallVec;
 
 // Types that the generated ISLE code uses via `use super::*`.
 use super::{
-    fp_reg, lower_condcode, lower_constant_f128, lower_constant_f32, lower_constant_f64,
-    lower_fp_condcode, stack_reg, writable_link_reg, writable_zero_reg, zero_reg, AMode,
-    ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp,
-    FPUOpRI, FPUOpRIMod, FloatCC, Imm12, ImmLogic, ImmShift, Inst as MInst, IntCC, JTSequenceInfo,
-    MachLabel, MemLabel, MoveWideConst, MoveWideOp, NarrowValueMode, Opcode, OperandSize,
-    PairAMode, Reg, SImm9, ScalarSize, ShiftOpAndAmt, UImm12Scaled, UImm5, VecMisc2, VectorSize,
-    NZCV,
+    fp_reg, lower_condcode, lower_fp_condcode, stack_reg, writable_link_reg, writable_zero_reg,
+    zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo, CallInfo, Cond,
+    CondBrKind, ExtendOp, FPUOpRI, FPUOpRIMod, FloatCC, Imm12, ImmLogic, ImmShift, Inst as MInst,
+    IntCC, JTSequenceInfo, MachLabel, MemLabel, MoveWideConst, MoveWideOp, NarrowValueMode, Opcode,
+    OperandSize, PairAMode, Reg, SImm9, ScalarSize, ShiftOpAndAmt, UImm12Scaled, UImm5, VecMisc2,
+    VectorSize, NZCV,
 };
 use crate::ir::condcodes;
 use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm};
-use crate::isa::aarch64::lower::{lower_address, lower_pair_address, lower_splat_const};
+use crate::isa::aarch64::lower::{lower_address, lower_pair_address};
 use crate::isa::aarch64::AArch64Backend;
 use crate::machinst::valueregs;
 use crate::machinst::{isle::*, InputSourceInst};
@@ -524,38 +523,6 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
         lower_pair_address(self.lower_ctx, addr, offset as i32)
     }
 
-    fn constant_f32(&mut self, value: u64) -> Reg {
-        let rd = self.temp_writable_reg(I8X16);
-
-        lower_constant_f32(self.lower_ctx, rd, f32::from_bits(value as u32));
-
-        rd.to_reg()
-    }
-
-    fn constant_f64(&mut self, value: u64) -> Reg {
-        let rd = self.temp_writable_reg(I8X16);
-
-        lower_constant_f64(self.lower_ctx, rd, f64::from_bits(value));
-
-        rd.to_reg()
-    }
-
-    fn constant_f128(&mut self, value: u128) -> Reg {
-        let rd = self.temp_writable_reg(I8X16);
-
-        lower_constant_f128(self.lower_ctx, rd, value);
-
-        rd.to_reg()
-    }
-
-    fn splat_const(&mut self, value: u64, size: &VectorSize) -> Reg {
-        let rd = self.temp_writable_reg(I8X16);
-
-        lower_splat_const(self.lower_ctx, rd, value, *size);
-
-        rd.to_reg()
-    }
-
     fn fp_cond_code(&mut self, cc: &condcodes::FloatCC) -> Cond {
         lower_fp_condcode(*cc)
     }
@@ -612,8 +579,6 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
     }
 
     fn min_fp_value(&mut self, signed: bool, in_bits: u8, out_bits: u8) -> Reg {
-        let tmp = self.lower_ctx.alloc_tmp(I8X16).only_reg().unwrap();
-
         if in_bits == 32 {
             // From float32.
             let min = match (signed, out_bits) {
@@ -630,7 +595,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
                 ),
             };
 
-            lower_constant_f32(self.lower_ctx, tmp, min);
+            generated_code::constructor_constant_f32(self, min.to_bits())
         } else if in_bits == 64 {
             // From float64.
             let min = match (signed, out_bits) {
@@ -647,7 +612,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
                 ),
             };
 
-            lower_constant_f64(self.lower_ctx, tmp, min);
+            generated_code::constructor_constant_f64(self, min.to_bits())
         } else {
             unimplemented!(
                 "unexpected input size for min_fp_value: {} (signed: {}, output size: {})",
@@ -656,13 +621,9 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
                 out_bits
             );
         }
-
-        tmp.to_reg()
     }
 
     fn max_fp_value(&mut self, signed: bool, in_bits: u8, out_bits: u8) -> Reg {
-        let tmp = self.lower_ctx.alloc_tmp(I8X16).only_reg().unwrap();
-
         if in_bits == 32 {
             // From float32.
             let max = match (signed, out_bits) {
@@ -682,7 +643,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
                 ),
             };
 
-            lower_constant_f32(self.lower_ctx, tmp, max);
+            generated_code::constructor_constant_f32(self, max.to_bits())
         } else if in_bits == 64 {
             // From float64.
             let max = match (signed, out_bits) {
@@ -702,7 +663,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
                 ),
             };
 
-            lower_constant_f64(self.lower_ctx, tmp, max);
+            generated_code::constructor_constant_f64(self, max.to_bits())
         } else {
             unimplemented!(
                 "unexpected input size for max_fp_value: {} (signed: {}, output size: {})",
@@ -711,8 +672,6 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
                 out_bits
             );
         }
-
-        tmp.to_reg()
     }
 
     fn fpu_op_ri_ushr(&mut self, ty_bits: u8, shift: u8) -> FPUOpRI {
@@ -785,4 +744,66 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
             None
         }
     }
+
+    fn asimd_mov_mod_imm_zero(&mut self, size: &ScalarSize) -> ASIMDMovModImm {
+        ASIMDMovModImm::zero(*size)
+    }
+
+    fn asimd_mov_mod_imm_from_u64(
+        &mut self,
+        val: u64,
+        size: &ScalarSize,
+    ) -> Option<ASIMDMovModImm> {
+        ASIMDMovModImm::maybe_from_u64(val, *size)
+    }
+
+    fn asimd_fp_mod_imm_from_u64(&mut self, val: u64, size: &ScalarSize) -> Option<ASIMDFPModImm> {
+        ASIMDFPModImm::maybe_from_u64(val, *size)
+    }
+
+    fn u64_low32_bits_unset(&mut self, val: u64) -> Option<u64> {
+        if val & 0xffffffff == 0 {
+            Some(val)
+        } else {
+            None
+        }
+    }
+
+    fn u128_replicated_u64(&mut self, val: u128) -> Option<u64> {
+        let low64 = val as u64 as u128;
+        if (low64 | (low64 << 64)) == val {
+            Some(low64 as u64)
+        } else {
+            None
+        }
+    }
+
+    fn u64_replicated_u32(&mut self, val: u64) -> Option<u64> {
+        let low32 = val as u32 as u64;
+        if (low32 | (low32 << 32)) == val {
+            Some(low32)
+        } else {
+            None
+        }
+    }
+
+    fn u32_replicated_u16(&mut self, val: u64) -> Option<u64> {
+        let val = val as u32;
+        let low16 = val as u16 as u32;
+        if (low16 | (low16 << 16)) == val {
+            Some(low16.into())
+        } else {
+            None
+        }
+    }
+
+    fn u16_replicated_u8(&mut self, val: u64) -> Option<u64> {
+        let val = val as u16;
+        let low8 = val as u8 as u16;
+        if (low8 | (low8 << 8)) == val {
+            Some(low8.into())
+        } else {
+            None
+        }
+    }
 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 66a0d942a..5eae84301 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -702,6 +702,20 @@
 (type AMO (primitive AMO))
 (type VecMachLabel extern (enum))
 
+;; Converters
+
+(convert u8 i32 u8_as_i32)
+(decl u8_as_i32 (u8) i32)
+(extern constructor u8_as_i32 u8_as_i32)
+
+(convert u8 u64 u8_as_u64)
+
+(decl convert_valueregs_reg (ValueRegs) Reg)
+(rule (convert_valueregs_reg x)
+  (value_regs_get x 0))
+(convert ValueRegs Reg convert_valueregs_reg)
+
+
 ;; ISA Extension helpers
 
 (decl pure has_zbkb () bool)
@@ -719,14 +733,6 @@
 (decl pure has_zbs () bool)
 (extern constructor has_zbs has_zbs)
 
-;; Helper for creating the zero register.
-(decl zero_reg () Reg)
-(extern constructor zero_reg zero_reg)
-
-(decl value_regs_zero () ValueRegs)
-(rule (value_regs_zero)
-  (value_regs (imm $I64 0) (imm $I64 0)))
-
 (decl gen_float_round (FloatRoundOP Reg Type) Reg)
 (rule
   (gen_float_round op rs ty)
@@ -759,18 +765,50 @@
 (decl imm (Type u64) Reg)
 (extern constructor imm imm)
 
+;; Imm12 Rules
+
+(decl pure imm12_zero () Imm12)
+(rule
+  (imm12_zero)
+  (imm12_const 0))
+
+(decl pure imm12_const (i32) Imm12)
+(extern constructor imm12_const imm12_const)
+
+(decl load_imm12 (i32) Reg)
+(rule
+  (load_imm12 x)
+  (alu_rr_imm12 (AluOPRRI.Addi) (zero_reg) (imm12_const x)))
+  
 ;; for load immediate
 (decl imm_from_bits (u64) Imm12)
 (extern constructor imm_from_bits imm_from_bits)
 
 (decl imm_from_neg_bits (i64) Imm12)
 (extern constructor imm_from_neg_bits imm_from_neg_bits)
-;;
+
+(decl imm12_const_add (i32 i32) Imm12)
+(extern constructor imm12_const_add imm12_const_add)
+
+(decl imm12_and (Imm12 i32) Imm12)
+(extern constructor imm12_and imm12_and)
+
+;; Helper for get negative of Imm12
+(decl neg_imm12 (Imm12) Imm12)
+(extern constructor neg_imm12 neg_imm12)
+
+;; Imm12 Extractors
+
+;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
+(decl imm12_from_value (Imm12) Value)
+(extractor
+  (imm12_from_value n)
+  (def_inst (iconst (u64_from_imm64 (imm12_from_u64 n)))))
+
 (decl imm12_from_u64 (Imm12) u64)
 (extern extractor imm12_from_u64 imm12_from_u64)
 
-(decl writable_zero_reg () WritableReg)
-(extern constructor writable_zero_reg writable_zero_reg)
+;; Float Helpers
 
 (decl gen_default_frm () OptionFloatRoundingMode)
 (extern constructor gen_default_frm gen_default_frm)
@@ -853,16 +891,6 @@
             (_ Unit (emit (MInst.AluRRImm12 op dst src (imm12_zero)))))
         dst))
 
-;; Helper for get negative of Imm12
-(decl neg_imm12 (Imm12) Imm12)
-(extern constructor neg_imm12 neg_imm12)
-
-
-;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
-(decl imm12_from_value (Imm12) Value)
-(extractor
-  (imm12_from_value n)
-  (def_inst (iconst (u64_from_imm64 (imm12_from_u64 n)))))
 
 (decl select_addi (Type) AluOPRRI)
 (rule 1 (select_addi (fits_in_32 ty)) (AluOPRRI.Addiw))
@@ -919,11 +947,6 @@
     (gen_brev8 tmp $I64)))
 
 
-(decl imm12_zero () Imm12)
-(rule
-  (imm12_zero)
-  (imm12_const 0))
-
 (decl lower_ctz (Type Reg) Reg)
 (rule
   (lower_ctz ty x)
@@ -967,11 +990,6 @@
       (result Reg (alu_add low high)))
     (value_regs result (load_u64_constant 0))))
 
-(convert u8 i32 u8_as_i32)
-(decl u8_as_i32 (u8) i32)
-(extern constructor u8_as_i32 u8_as_i32)
-
-(convert u8 u64 u8_as_u64)
 (decl lower_clz (Type Reg) Reg)
 (rule
   (lower_clz ty rs)
@@ -999,14 +1017,6 @@
       (result Reg (alu_rr_imm12 (AluOPRRI.Addi) count (imm12_const_add (ty_bits ty) -64))))
     result))
 
-;; paramter is "intcc compare_a compare_b rs1 rs2".
-(decl gen_select_reg (IntCC Reg Reg Reg Reg) Reg)
-(extern constructor gen_select_reg gen_select_reg)
-
-;; load a constant into reg.
-(decl load_u64_constant (u64) Reg)
-(extern constructor load_u64_constant load_u64_constant)
-
 (decl lower_clz_i128 (ValueRegs) ValueRegs)
 (rule
   (lower_clz_i128 x)
@@ -1443,10 +1453,6 @@
       (gen_select_reg (IntCC.UnsignedGreaterThanOrEqual) shamt_128 const64 high low)
       (gen_select_reg (IntCC.UnsignedGreaterThanOrEqual) shamt_128 const64 high_replacement high))))
 
-(decl load_imm12 (i32) Reg)
-(rule
-  (load_imm12 x)
-  (alu_rr_imm12 (AluOPRRI.Addi) (zero_reg) (imm12_const x)))
 
 (decl lower_cls_i128 (ValueRegs) ValueRegs)
 (rule
@@ -1465,16 +1471,6 @@
       (result Reg (alu_rr_imm12 (AluOPRRI.Addi) count (imm12_const -1))))
     (value_regs result (load_u64_constant 0))))
 
-(decl imm12_const (i32) Imm12)
-(extern constructor imm12_const imm12_const)
-
-;;;;
-(decl imm12_const_add (i32 i32) Imm12)
-(extern constructor imm12_const_add imm12_const_add)
-
-(decl imm12_and (Imm12 i32) Imm12)
-(extern constructor imm12_and imm12_and)
-
 
 (decl gen_amode (Reg Offset32 Type) AMode)
 (extern constructor gen_amode gen_amode)
@@ -1711,6 +1707,14 @@
       (reuslt VecWritableReg (vec_writable_clone dst))
       (_ Unit (emit (MInst.Select dst ty c x y))))
     (vec_writable_to_regs reuslt)))
+  
+;; Parameters are "intcc compare_a compare_b rs1 rs2".
+(decl gen_select_reg (IntCC Reg Reg Reg Reg) Reg)
+(extern constructor gen_select_reg gen_select_reg)
+
+;; load a constant into reg.
+(decl load_u64_constant (u64) Reg)
+(extern constructor load_u64_constant load_u64_constant)
 
 ;;; clone WritableReg
 ;;; if not rust compiler will complain about use moved value.
@@ -1829,12 +1833,6 @@
     (move_x_to_f tmp2 ty)))
 
 
-(decl convert_valueregs_reg (ValueRegs) Reg)
-(rule
-  (convert_valueregs_reg x)
-  (value_regs_get x 0))
-(convert ValueRegs Reg convert_valueregs_reg)
-
 ;;; lower icmp
 (decl lower_icmp (IntCC ValueRegs ValueRegs Type) Reg)
 (rule 1 (lower_icmp cc x y ty)
@@ -2172,6 +2170,17 @@
 (decl sp_reg () PReg)
 (extern constructor sp_reg sp_reg)
 
+;; Helper for creating the zero register.
+(decl zero_reg () Reg)
+(extern constructor zero_reg zero_reg)
+
+(decl value_regs_zero () ValueRegs)
+(rule (value_regs_zero)
+  (value_regs (imm $I64 0) (imm $I64 0)))
+
+(decl writable_zero_reg () WritableReg)
+(extern constructor writable_zero_reg writable_zero_reg)
+
 
 ;;;; Helpers for floating point comparisons ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 163d0cb63..ec74f4555 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -11,7 +11,7 @@
 
 ;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(rule (lower (f32const (u64_from_ieee32 n)))
+(rule (lower (f32const (u32_from_ieee32 n)))
   (imm $F32 n))
 
 ;;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -186,14 +186,14 @@
     (alu_rrr (AluOPRRR.RemU) x y)))
 
 ;;;; Rules for `and` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule -1 (lower (has_type (fits_in_64 ty) (band x y)))
+(rule -1 (lower (has_type (fits_in_64 (ty_int ty)) (band x y)))
   (alu_rrr (AluOPRRR.And) x y))
 
 ;; Special cases for when one operand is an immediate that fits in 12 bits.
-(rule 2 (lower (has_type (fits_in_64 ty) (band x (imm12_from_value y))))
+(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (band x (imm12_from_value y))))
   (alu_rr_imm12 (AluOPRRI.Andi) x y))
 
-(rule 1 (lower (has_type (fits_in_64 ty) (band (imm12_from_value x) y)))
+(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (band (imm12_from_value x) y)))
   (alu_rr_imm12 (AluOPRRI.Andi) y x))
 
 (rule (lower (has_type $I128 (band x y)))
@@ -201,6 +201,7 @@
 
 (rule (lower (has_type $F32 (band x y)))
   (lower_float_binary (AluOPRRR.And) x y $F32))
+
 (rule (lower (has_type $F64 (band x y)))
   (lower_float_binary (AluOPRRR.And) x y $F64))
 
@@ -208,18 +209,21 @@
 ;; by Cranelift's `band_not` instruction that is legalized into the simpler
 ;; forms early on.
 
-(rule 3 (lower (has_type (fits_in_64 ty) (band x (bnot y))))
+(rule 3 (lower (has_type (fits_in_64 (ty_int ty)) (band x (bnot y))))
   (if-let $true (has_zbb))
   (gen_andn x y))
-(rule 4 (lower (has_type (fits_in_64 ty) (band (bnot y) x)))
+
+(rule 4 (lower (has_type (fits_in_64 (ty_int ty)) (band (bnot y) x)))
   (if-let $true (has_zbb))
   (gen_andn x y))
+
 (rule 5 (lower (has_type $I128 (band x (bnot y))))
   (if-let $true (has_zbb))
   (let
     ((low Reg (gen_andn (value_regs_get x 0) (value_regs_get y 0)))
       (high Reg (gen_andn (value_regs_get x 1) (value_regs_get y 1))))
     (value_regs low high)))
+
 (rule 6 (lower (has_type $I128 (band (bnot y) x)))
   (if-let $true (has_zbb))
   (let
@@ -229,19 +233,22 @@
 
 
 ;;;; Rules for `or` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule -1 (lower (has_type (fits_in_64 ty) (bor x y)))
+(rule -1 (lower (has_type (fits_in_64 (ty_int ty)) (bor x y)))
   (alu_rrr (AluOPRRR.Or) x y))
 
 ;; Special cases for when one operand is an immediate that fits in 12 bits.
-(rule 2 (lower (has_type (fits_in_64 ty) (bor x (imm12_from_value y))))
+(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor x (imm12_from_value y))))
   (alu_rr_imm12 (AluOPRRI.Ori) x y))
 
-(rule 1 (lower (has_type (fits_in_64 ty) (bor (imm12_from_value x) y)))
+(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor (imm12_from_value x) y)))
   (alu_rr_imm12 (AluOPRRI.Ori) y x))
+
 (rule (lower (has_type $I128 (bor x y)))
   (lower_b128_binary (AluOPRRR.Or) x y))
+
 (rule (lower (has_type $F32 (bor x y)))
   (lower_float_binary (AluOPRRR.Or) x y $F32))
+
 (rule (lower (has_type $F64 (bor x y)))
   (lower_float_binary (AluOPRRR.Or) x y $F64))
 
@@ -249,10 +256,11 @@
 ;; by Cranelift's `bor_not` instruction that is legalized into the simpler
 ;; forms early on.
 
-(rule 3 (lower (has_type (fits_in_64 ty) (bor x (bnot y))))
+(rule 3 (lower (has_type (fits_in_64 (ty_int ty)) (bor x (bnot y))))
   (if-let $true (has_zbb))
   (gen_orn x y))
-(rule 4 (lower (has_type (fits_in_64 ty) (bor (bnot y) x)))
+
+(rule 4 (lower (has_type (fits_in_64 (ty_int ty)) (bor (bnot y) x)))
   (if-let $true (has_zbb))
   (gen_orn x y))
 
@@ -262,6 +270,7 @@
     ((low Reg (gen_orn (value_regs_get x 0) (value_regs_get y 0)))
       (high Reg (gen_orn (value_regs_get x 1) (value_regs_get y 1))))
     (value_regs low high)))
+
 (rule 6 (lower (has_type $I128 (bor (bnot y) x)))
   (if-let $true (has_zbb))
   (let
@@ -271,40 +280,43 @@
 
 
 ;;;; Rules for `xor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule -1 (lower (has_type (fits_in_64 ty) (bxor x y)))
+(rule -1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor x y)))
   (alu_rrr (AluOPRRR.Xor) x y))
 
 ;; Special cases for when one operand is an immediate that fits in 12 bits.
-(rule 2 (lower (has_type (fits_in_64 ty) (bxor x (imm12_from_value y))))
+(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor x (imm12_from_value y))))
   (alu_rr_imm12 (AluOPRRI.Xori) x y))
 
-(rule 1 (lower (has_type (fits_in_64 ty) (bxor (imm12_from_value x) y)))
+(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor (imm12_from_value x) y)))
   (alu_rr_imm12 (AluOPRRI.Xori) y x))
+
 (rule (lower (has_type $I128 (bxor x y)))
   (lower_b128_binary (AluOPRRR.Xor) x y))
+
 (rule (lower (has_type $F32 (bxor x y)))
   (lower_float_binary (AluOPRRR.Xor) x y $F32))
+
 (rule (lower (has_type $F64 (bxor x y)))
   (lower_float_binary (AluOPRRR.Xor) x y $F64))
 
 
 ;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule -1 (lower (has_type fits_in_64 (bnot x)))
+(rule -1 (lower (has_type (fits_in_64 (ty_int ty)) (bnot x)))
   (alu_rr_imm12 (AluOPRRI.Xori) x (imm_from_neg_bits -1)))
 
 (rule (lower (has_type $I128 (bnot x)))
   (bnot_128 x))
+
 (rule
   (lower (has_type $F32 (bnot x)))
-  (lower_float_bnot x $F32)
-)
+  (lower_float_bnot x $F32))
+
 (rule
   (lower (has_type $F64 (bnot x)))
-  (lower_float_bnot x $F64)
-)
+  (lower_float_bnot x $F64))
 
 ;;;; Rules for `bit_reverse` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule (lower (has_type ty (bitrev x)))
+(rule (lower (has_type (fits_in_64 (ty_int ty)) (bitrev x)))
   (lower_bit_reverse x ty))
 
 (rule 1 (lower (has_type $I128 (bitrev x)))
@@ -324,8 +336,16 @@
 ;;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 (rule (lower (has_type ty (clz x)))
   (lower_clz ty x))
+
 (rule 1 (lower (has_type $I128 (clz x)))
   (lower_clz_i128 x))
+  
+;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(rule (lower (has_type (fits_in_64 ty) (cls x)))
+  (lower_cls x ty))
+
+(rule 1 (lower (has_type $I128 (cls x)))
+  (lower_cls_i128 x))
 
 ;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 (rule (lower (has_type out_ty (uextend val @ (value_type in_ty))))
@@ -435,13 +455,6 @@
   (lower_i128_rotr x y))
 
 
-;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule (lower (has_type (fits_in_64 ty) (cls x)))
-  (lower_cls x ty))
-(rule 1 (lower (has_type $I128 (cls x)))
-  (lower_cls_i128 x))
-
-
 ;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 (rule
   (lower (has_type ty (fabs x)))
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index ffc5f7350..4330b33e0 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -896,7 +896,7 @@
     (CallInd
       (link WritableReg)
       (info BoxCallIndInfo))
-    
+
     ;; A pseudo-instruction that captures register arguments in vregs.
     (Args
       (args VecArgPair))
@@ -1555,8 +1555,8 @@
 (decl u8_as_u16 (u8) u16)
 (extern constructor u8_as_u16 u8_as_u16)
 
-(decl u64_as_u32 (u64) u32)
-(extern constructor u64_as_u32 u64_as_u32)
+(decl u64_truncate_to_u32 (u64) u32)
+(extern constructor u64_truncate_to_u32 u64_truncate_to_u32)
 
 (decl u64_as_i16 (u64) i16)
 (extern constructor u64_as_i16 u64_as_i16)
@@ -3000,7 +3000,7 @@
 ;; 32-bit result type, any value
 (rule 5 (imm (gpr32_ty ty) n)
       (let ((dst WritableReg (temp_writable_reg ty))
-            (_ Unit (emit (MInst.Mov32Imm dst (u64_as_u32 n)))))
+            (_ Unit (emit (MInst.Mov32Imm dst (u64_truncate_to_u32 n)))))
         dst))
 
 ;; 64-bit result type, value fits in i16
@@ -3051,7 +3051,7 @@
 ;; TODO: use LZER to load 0.0
 (rule 8 (imm $F32 n)
       (let ((dst WritableReg (temp_writable_reg $F32))
-            (_ Unit (emit (MInst.LoadFpuConst32 dst (u64_as_u32 n)))))
+            (_ Unit (emit (MInst.LoadFpuConst32 dst (u64_truncate_to_u32 n)))))
         dst))
 
 ;; 64-bit floating-point type, any value.  Loaded from literal pool.
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index 02563f4d5..965074fa6 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -18,7 +18,7 @@
 
 ;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(rule (lower (f32const (u64_from_ieee32 x)))
+(rule (lower (f32const (u32_from_ieee32 x)))
       (imm $F32 x))
 
 
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs
index a6dfe21e5..7baf0f5cf 100644
--- a/cranelift/codegen/src/isa/s390x/lower/isle.rs
+++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs
@@ -436,7 +436,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> {
     }
 
     #[inline]
-    fn u64_as_u32(&mut self, n: u64) -> u32 {
+    fn u64_truncate_to_u32(&mut self, n: u64) -> u32 {
         n as u32
     }
 
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 99cfeda06..bcc63bfdb 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -290,8 +290,18 @@
        ;; XMM (scalar or vector) unary op (from xmm to reg/mem) using the
        ;; VEX prefix
        (XmmMovRMVex (op AvxOpcode)
-                    (src Reg)
+                    (src Xmm)
                     (dst SyntheticAmode))
+       (XmmMovRMImmVex (op AvxOpcode)
+                       (src Xmm)
+                       (dst SyntheticAmode)
+                       (imm u8))
+
+       ;; XMM (scalar) unary op (from xmm to integer reg): vpextr{w,b,d,q}
+       (XmmToGprImmVex (op AvxOpcode)
+                       (src Xmm)
+                       (dst WritableGpr)
+                       (imm u8))
 
        ;; XMM (scalar or vector) binary op that relies on the EVEX
        ;; prefix. Takes two inputs.
@@ -343,8 +353,12 @@
        ;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd,
        ;; movq
        (XmmMovRM (op SseOpcode)
-                 (src Reg)
+                 (src Xmm)
                  (dst SyntheticAmode))
+       (XmmMovRMImm (op SseOpcode)
+                    (src Xmm)
+                    (dst SyntheticAmode)
+                    (imm u8))
 
        ;; XMM (scalar) unary op (from xmm to integer reg): movd, movq,
        ;; cvtts{s,d}2si
@@ -1364,6 +1378,10 @@
             Vmovups
             Vmovupd
             Vmovdqu
+            Vpextrb
+            Vpextrw
+            Vpextrd
+            Vpextrq
           ))
 
 (type Avx512Opcode extern
@@ -2043,10 +2061,18 @@
 (rule (xmm_movrm op addr data)
       (SideEffectNoResult.Inst (MInst.XmmMovRM op data addr)))
 
+(decl xmm_movrm_imm (SseOpcode SyntheticAmode Xmm u8) SideEffectNoResult)
+(rule (xmm_movrm_imm op addr data imm)
+      (SideEffectNoResult.Inst (MInst.XmmMovRMImm op data addr imm)))
+
 (decl xmm_movrm_vex (AvxOpcode SyntheticAmode Xmm) SideEffectNoResult)
 (rule (xmm_movrm_vex op addr data)
       (SideEffectNoResult.Inst (MInst.XmmMovRMVex op data addr)))
 
+(decl xmm_movrm_imm_vex (AvxOpcode SyntheticAmode Xmm u8) SideEffectNoResult)
+(rule (xmm_movrm_imm_vex op addr data imm)
+      (SideEffectNoResult.Inst (MInst.XmmMovRMImmVex op data addr imm)))
+
 ;; Load a constant into an XMM register.
 (decl x64_xmm_load_const (Type VCodeConstant) Xmm)
 (rule (x64_xmm_load_const ty const)
@@ -3603,21 +3629,61 @@
 (decl x64_pextrb (Xmm u8) Gpr)
 (rule (x64_pextrb src lane)
       (xmm_to_gpr_imm (SseOpcode.Pextrb) src lane))
+(rule 1 (x64_pextrb src lane)
+        (if-let $true (has_avx))
+        (xmm_to_gpr_imm_vex (AvxOpcode.Vpextrb) src lane))
+
+(decl x64_pextrb_store (SyntheticAmode Xmm u8) SideEffectNoResult)
+(rule (x64_pextrb_store addr src lane)
+      (xmm_movrm_imm (SseOpcode.Pextrb) addr src lane))
+(rule 1 (x64_pextrb_store addr src lane)
+        (if-let $true (has_avx))
+        (xmm_movrm_imm_vex (AvxOpcode.Vpextrb) addr src lane))
 
 ;; Helper for creating `pextrw` instructions.
 (decl x64_pextrw (Xmm u8) Gpr)
 (rule (x64_pextrw src lane)
       (xmm_to_gpr_imm (SseOpcode.Pextrw) src lane))
+(rule 1 (x64_pextrw src lane)
+        (if-let $true (has_avx))
+        (xmm_to_gpr_imm_vex (AvxOpcode.Vpextrw) src lane))
+
+(decl x64_pextrw_store (SyntheticAmode Xmm u8) SideEffectNoResult)
+(rule (x64_pextrw_store addr src lane)
+      (xmm_movrm_imm (SseOpcode.Pextrw) addr src lane))
+(rule 1 (x64_pextrw_store addr src lane)
+        (if-let $true (has_avx))
+        (xmm_movrm_imm_vex (AvxOpcode.Vpextrw) addr src lane))
 
 ;; Helper for creating `pextrd` instructions.
 (decl x64_pextrd (Xmm u8) Gpr)
 (rule (x64_pextrd src lane)
       (xmm_to_gpr_imm (SseOpcode.Pextrd) src lane))
+(rule 1 (x64_pextrd src lane)
+        (if-let $true (has_avx))
+        (xmm_to_gpr_imm_vex (AvxOpcode.Vpextrd) src lane))
+
+(decl x64_pextrd_store (SyntheticAmode Xmm u8) SideEffectNoResult)
+(rule (x64_pextrd_store addr src lane)
+      (xmm_movrm_imm (SseOpcode.Pextrd) addr src lane))
+(rule 1 (x64_pextrd_store addr src lane)
+        (if-let $true (has_avx))
+        (xmm_movrm_imm_vex (AvxOpcode.Vpextrd) addr src lane))
 
 ;; Helper for creating `pextrq` instructions.
 (decl x64_pextrq (Xmm u8) Gpr)
 (rule (x64_pextrq src lane)
       (xmm_to_gpr_imm (SseOpcode.Pextrq) src lane))
+(rule 1 (x64_pextrq src lane)
+        (if-let $true (has_avx))
+        (xmm_to_gpr_imm_vex (AvxOpcode.Vpextrq) src lane))
+
+(decl x64_pextrq_store (SyntheticAmode Xmm u8) SideEffectNoResult)
+(rule (x64_pextrq_store addr src lane)
+      (xmm_movrm_imm (SseOpcode.Pextrq) addr src lane))
+(rule 1 (x64_pextrq_store addr src lane)
+        (if-let $true (has_avx))
+        (xmm_movrm_imm_vex (AvxOpcode.Vpextrq) addr src lane))
 
 ;; Helper for creating `MInst.XmmToGpr` instructions.
 (decl xmm_to_gpr (SseOpcode Xmm OperandSize) Gpr)
@@ -3626,13 +3692,20 @@
             (_ Unit (emit (MInst.XmmToGpr op src dst size))))
         dst))
 
-;; Helper for creating `MInst.XmmToGpr` instructions.
+;; Helper for creating `MInst.XmmToGprImm` instructions.
 (decl xmm_to_gpr_imm (SseOpcode Xmm u8) Gpr)
 (rule (xmm_to_gpr_imm op src imm)
       (let ((dst WritableGpr (temp_writable_gpr))
             (_ Unit (emit (MInst.XmmToGprImm op src dst imm))))
         dst))
 
+;; Helper for creating `MInst.XmmToGprImmVex` instructions.
+(decl xmm_to_gpr_imm_vex (AvxOpcode Xmm u8) Gpr)
+(rule (xmm_to_gpr_imm_vex op src imm)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (_ Unit (emit (MInst.XmmToGprImmVex op src dst imm))))
+        dst))
+
 ;; Helper for creating `pmovmskb` instructions.
 (decl x64_pmovmskb (OperandSize Xmm) Gpr)
 (rule (x64_pmovmskb size src)
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 6ae024c01..069c38f31 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -1699,7 +1699,11 @@ impl AvxOpcode {
             | AvxOpcode::Vmovsd
             | AvxOpcode::Vmovups
             | AvxOpcode::Vmovupd
-            | AvxOpcode::Vmovdqu => {
+            | AvxOpcode::Vmovdqu
+            | AvxOpcode::Vpextrb
+            | AvxOpcode::Vpextrw
+            | AvxOpcode::Vpextrd
+            | AvxOpcode::Vpextrq => {
                 smallvec![InstructionSet::AVX]
             }
         }
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 9fb2c1599..a6da7d867 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -2430,7 +2430,7 @@ pub(crate) fn emit(
         }
 
         Inst::XmmMovRMVex { op, src, dst } => {
-            let src = allocs.next(*src);
+            let src = allocs.next(src.to_reg());
             let dst = dst.with_allocs(allocs).finalize(state, sink);
 
             let (prefix, map, opcode) = match op {
@@ -2451,6 +2451,52 @@ pub(crate) fn emit(
                 .encode(sink);
         }
 
+        Inst::XmmMovRMImmVex { op, src, dst, imm } => {
+            let src = allocs.next(src.to_reg());
+            let dst = dst.with_allocs(allocs).finalize(state, sink);
+
+            let (w, prefix, map, opcode) = match op {
+                AvxOpcode::Vpextrb => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x14),
+                AvxOpcode::Vpextrw => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x15),
+                AvxOpcode::Vpextrd => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
+                AvxOpcode::Vpextrq => (true, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
+                _ => unimplemented!("Opcode {:?} not implemented", op),
+            };
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .w(w)
+                .prefix(prefix)
+                .map(map)
+                .opcode(opcode)
+                .rm(dst)
+                .reg(src.to_real_reg().unwrap().hw_enc())
+                .imm(*imm)
+                .encode(sink);
+        }
+
+        Inst::XmmToGprImmVex { op, src, dst, imm } => {
+            let src = allocs.next(src.to_reg());
+            let dst = allocs.next(dst.to_reg().to_reg());
+
+            let (w, prefix, map, opcode) = match op {
+                AvxOpcode::Vpextrb => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x14),
+                AvxOpcode::Vpextrw => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x15),
+                AvxOpcode::Vpextrd => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
+                AvxOpcode::Vpextrq => (true, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
+                _ => unimplemented!("Opcode {:?} not implemented", op),
+            };
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .w(w)
+                .prefix(prefix)
+                .map(map)
+                .opcode(opcode)
+                .rm(dst.to_real_reg().unwrap().hw_enc())
+                .reg(src.to_real_reg().unwrap().hw_enc())
+                .imm(*imm)
+                .encode(sink);
+        }
+
         Inst::XmmRmREvex {
             op,
             src1,
@@ -2649,7 +2695,7 @@ pub(crate) fn emit(
         }
 
         Inst::XmmMovRM { op, src, dst } => {
-            let src = allocs.next(*src);
+            let src = allocs.next(src.to_reg());
             let dst = dst.with_allocs(allocs);
 
             let (prefix, opcode) = match op {
@@ -2666,6 +2712,27 @@ pub(crate) fn emit(
             emit_std_reg_mem(sink, prefix, opcode, 2, src, dst, RexFlags::clear_w(), 0);
         }
 
+        Inst::XmmMovRMImm { op, src, dst, imm } => {
+            let src = allocs.next(src.to_reg());
+            let dst = dst.with_allocs(allocs);
+
+            let (w, prefix, opcode) = match op {
+                SseOpcode::Pextrb => (false, LegacyPrefixes::_66, 0x0F3A14),
+                SseOpcode::Pextrw => (false, LegacyPrefixes::_66, 0x0F3A15),
+                SseOpcode::Pextrd => (false, LegacyPrefixes::_66, 0x0F3A16),
+                SseOpcode::Pextrq => (true, LegacyPrefixes::_66, 0x0F3A16),
+                _ => unimplemented!("Opcode {:?} not implemented", op),
+            };
+            let rex = if w {
+                RexFlags::set_w()
+            } else {
+                RexFlags::clear_w()
+            };
+            let dst = &dst.finalize(state, sink);
+            emit_std_reg_mem(sink, prefix, opcode, 3, src, dst, rex, 1);
+            sink.put1(*imm);
+        }
+
         Inst::XmmToGpr {
             op,
             src,
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 396f9fd4a..95e39293d 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -30,7 +30,7 @@ use args::*;
 // `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here.
 pub use super::lower::isle::generated_code::MInst as Inst;
 
-// Out-of-line data for calls, to keep the size of `Inst` downn.
+/// Out-of-line data for calls, to keep the size of `Inst` down.
 #[derive(Clone, Debug)]
 pub struct CallInfo {
     /// Register uses of this call.
@@ -131,6 +131,7 @@ impl Inst {
             // These use dynamic SSE opcodes.
             Inst::GprToXmm { op, .. }
             | Inst::XmmMovRM { op, .. }
+            | Inst::XmmMovRMImm { op, .. }
             | Inst::XmmRmiReg { opcode: op, .. }
             | Inst::XmmRmR { op, .. }
             | Inst::XmmRmRUnaligned { op, .. }
@@ -153,7 +154,9 @@ impl Inst {
             | Inst::XmmVexPinsr { op, .. }
             | Inst::XmmUnaryRmRVex { op, .. }
             | Inst::XmmUnaryRmRImmVex { op, .. }
-            | Inst::XmmMovRMVex { op, .. } => op.available_from(),
+            | Inst::XmmMovRMVex { op, .. }
+            | Inst::XmmMovRMImmVex { op, .. }
+            | Inst::XmmToGprImmVex { op, .. } => op.available_from(),
         }
     }
 }
@@ -331,7 +334,7 @@ impl Inst {
         debug_assert!(src.class() == RegClass::Float);
         Inst::XmmMovRM {
             op,
-            src,
+            src: Xmm::new(src).unwrap(),
             dst: dst.into(),
         }
     }
@@ -933,17 +936,33 @@ impl PrettyPrint for Inst {
             }
 
             Inst::XmmMovRM { op, src, dst, .. } => {
-                let src = pretty_print_reg(*src, 8, allocs);
+                let src = pretty_print_reg(src.to_reg(), 8, allocs);
                 let dst = dst.pretty_print(8, allocs);
                 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
             }
 
             Inst::XmmMovRMVex { op, src, dst, .. } => {
-                let src = pretty_print_reg(*src, 8, allocs);
+                let src = pretty_print_reg(src.to_reg(), 8, allocs);
                 let dst = dst.pretty_print(8, allocs);
                 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
             }
 
+            Inst::XmmMovRMImm {
+                op, src, dst, imm, ..
+            } => {
+                let src = pretty_print_reg(src.to_reg(), 8, allocs);
+                let dst = dst.pretty_print(8, allocs);
+                format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
+            Inst::XmmMovRMImmVex {
+                op, src, dst, imm, ..
+            } => {
+                let src = pretty_print_reg(src.to_reg(), 8, allocs);
+                let dst = dst.pretty_print(8, allocs);
+                format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
             Inst::XmmRmR {
                 op,
                 src1,
@@ -1023,7 +1042,7 @@ impl PrettyPrint for Inst {
                 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
                 let src2 = src2.pretty_print(8, allocs);
 
-                format!("{} ${imm} {src1}, {src2}, {dst}", ljustify(op.to_string()))
+                format!("{} ${imm}, {src1}, {src2}, {dst}", ljustify(op.to_string()))
             }
 
             Inst::XmmVexPinsr {
@@ -1038,7 +1057,7 @@ impl PrettyPrint for Inst {
                 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
                 let src2 = src2.pretty_print(8, allocs);
 
-                format!("{} ${imm} {src1}, {src2}, {dst}", ljustify(op.to_string()))
+                format!("{} ${imm}, {src1}, {src2}, {dst}", ljustify(op.to_string()))
             }
 
             Inst::XmmRmRVex3 {
@@ -1190,6 +1209,12 @@ impl PrettyPrint for Inst {
                 format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
             }
 
+            Inst::XmmToGprImmVex { op, src, dst, imm } => {
+                let src = pretty_print_reg(src.to_reg(), 8, allocs);
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
+                format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
             Inst::GprToXmm {
                 op,
                 src,
@@ -2033,8 +2058,11 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_reuse_def(dst.to_writable_reg(), 0); // Reuse RHS.
             src2.get_operands(collector);
         }
-        Inst::XmmMovRM { src, dst, .. } | Inst::XmmMovRMVex { src, dst, .. } => {
-            collector.reg_use(*src);
+        Inst::XmmMovRM { src, dst, .. }
+        | Inst::XmmMovRMVex { src, dst, .. }
+        | Inst::XmmMovRMImm { src, dst, .. }
+        | Inst::XmmMovRMImmVex { src, dst, .. } => {
+            collector.reg_use(src.to_reg());
             dst.get_operands(collector);
         }
         Inst::XmmCmpRmR { src, dst, .. } => {
@@ -2058,7 +2086,9 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_use(src.to_reg());
             collector.reg_fixed_nonallocatable(*dst);
         }
-        Inst::XmmToGpr { src, dst, .. } | Inst::XmmToGprImm { src, dst, .. } => {
+        Inst::XmmToGpr { src, dst, .. }
+        | Inst::XmmToGprImm { src, dst, .. }
+        | Inst::XmmToGprImmVex { src, dst, .. } => {
             collector.reg_use(src.to_reg());
             collector.reg_def(dst.to_writable_reg());
         }
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 56d2dc0f2..5b4773f6c 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -24,7 +24,7 @@
 
 ;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(rule (lower (f32const (u64_from_ieee32 x)))
+(rule (lower (f32const (u32_from_ieee32 x)))
       (imm $F32 x))
 
 ;;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2659,17 +2659,41 @@
 ;; f32 or f64 despite the source perhaps being an integer vector since the
 ;; result of the instruction is the same.
 (rule 2 (lower (store flags
-                    (has_type (ty_32 _) (extractlane value (u8_from_uimm8 0)))
+                    (has_type $F32 (extractlane value (u8_from_uimm8 0)))
                     address
                     offset))
       (side_effect
        (x64_movss_store (to_amode flags address offset) value)))
-(rule 3 (lower (store flags
-                    (has_type (ty_64 _) (extractlane value (u8_from_uimm8 0)))
+(rule 2 (lower (store flags
+                    (has_type $F64 (extractlane value (u8_from_uimm8 0)))
                     address
                     offset))
       (side_effect
        (x64_movsd_store (to_amode flags address offset) value)))
+(rule 2 (lower (store flags
+                    (has_type $I8 (extractlane value (u8_from_uimm8 n)))
+                    address
+                    offset))
+      (side_effect
+       (x64_pextrb_store (to_amode flags address offset) value n)))
+(rule 2 (lower (store flags
+                    (has_type $I16 (extractlane value (u8_from_uimm8 n)))
+                    address
+                    offset))
+      (side_effect
+       (x64_pextrw_store (to_amode flags address offset) value n)))
+(rule 2 (lower (store flags
+                    (has_type $I32 (extractlane value (u8_from_uimm8 n)))
+                    address
+                    offset))
+      (side_effect
+       (x64_pextrd_store (to_amode flags address offset) value n)))
+(rule 2 (lower (store flags
+                    (has_type $I64 (extractlane value (u8_from_uimm8 n)))
+                    address
+                    offset))
+      (side_effect
+       (x64_pextrq_store (to_amode flags address offset) value n)))
 
 ;; Rules for `load*` + ALU op + `store*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs
index 79f79c7e2..dd84f4395 100644
--- a/cranelift/codegen/src/isa/x64/mod.rs
+++ b/cranelift/codegen/src/isa/x64/mod.rs
@@ -1,6 +1,6 @@
 //! X86_64-bit Instruction Set Architecture.
 
-pub use self::inst::{args, EmitInfo, EmitState, Inst};
+pub use self::inst::{args, CallInfo, EmitInfo, EmitState, Inst};
 
 use super::{OwnedTargetIsa, TargetIsa};
 use crate::dominator_tree::DominatorTree;
diff --git a/cranelift/codegen/src/isle_prelude.rs b/cranelift/codegen/src/isle_prelude.rs
index aa3714f14..9d78044b3 100644
--- a/cranelift/codegen/src/isle_prelude.rs
+++ b/cranelift/codegen/src/isle_prelude.rs
@@ -80,6 +80,11 @@ macro_rules! isle_common_prelude_methods {
             x ^ y
         }
 
+        #[inline]
+        fn u64_shl(&mut self, x: u64, y: u64) -> u64 {
+            x << y
+        }
+
         #[inline]
         fn imm64_shl(&mut self, ty: Type, x: Imm64, y: Imm64) -> Imm64 {
             // Mask off any excess shift bits.
@@ -502,8 +507,8 @@ macro_rules! isle_common_prelude_methods {
             }
         }
 
-        fn u64_from_ieee32(&mut self, val: Ieee32) -> u64 {
-            val.bits().into()
+        fn u32_from_ieee32(&mut self, val: Ieee32) -> u32 {
+            val.bits()
         }
 
         fn u64_from_ieee64(&mut self, val: Ieee64) -> u64 {
@@ -748,5 +753,13 @@ macro_rules! isle_common_prelude_methods {
         fn pack_block_array_2(&mut self, a: BlockCall, b: BlockCall) -> BlockArray2 {
             [a, b]
         }
+
+        fn u128_as_u64(&mut self, val: u128) -> Option<u64> {
+            u64::try_from(val).ok()
+        }
+
+        fn u64_as_u32(&mut self, val: u64) -> Option<u32> {
+            u32::try_from(val).ok()
+        }
     };
 }
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index fb22e29c0..4815cf635 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -43,14 +43,6 @@
 (decl value_type (Type) Value)
 (extern extractor infallible value_type value_type)
 
-(decl pure u32_add (u32 u32) u32)
-(extern constructor u32_add u32_add)
-
-;; Pure/fallible constructor that tries to add two `u32`s, interpreted
-;; as signed values, and fails to match on overflow.
-(decl pure partial s32_add_fallible (u32 u32) u32)
-(extern constructor s32_add_fallible s32_add_fallible)
-
 ;; Extractor that matches a `u32` only if non-negative.
 (decl u32_nonnegative (u32) u32)
 (extern extractor u32_nonnegative u32_nonnegative)
@@ -83,9 +75,6 @@
 (decl uimm8 (u8) Imm64)
 (extern extractor uimm8 uimm8)
 
-(decl pure u8_and (u8 u8) u8)
-(extern constructor u8_and u8_and)
-
 ;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (decl pure u8_as_u32 (u8) u32)
@@ -99,12 +88,30 @@
 
 (decl pure u32_as_u64 (u32) u64)
 (extern constructor u32_as_u64 u32_as_u64)
+(convert u32 u64 u32_as_u64)
 
 (decl pure i64_as_u64 (i64) u64)
 (extern constructor i64_as_u64 i64_as_u64)
 
+(decl u128_as_u64 (u64) u128)
+(extern extractor u128_as_u64 u128_as_u64)
+
+(decl u64_as_u32 (u32) u64)
+(extern extractor u64_as_u32 u64_as_u32)
+
 ;;;; Primitive Arithmetic ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(decl pure u8_and (u8 u8) u8)
+(extern constructor u8_and u8_and)
+
+(decl pure u32_add (u32 u32) u32)
+(extern constructor u32_add u32_add)
+
+;; Pure/fallible constructor that tries to add two `u32`s, interpreted
+;; as signed values, and fails to match on overflow.
+(decl pure partial s32_add_fallible (u32 u32) u32)
+(extern constructor s32_add_fallible s32_add_fallible)
+
 (decl pure u64_add (u64 u64) u64)
 (extern constructor u64_add u64_add)
 
@@ -129,6 +136,9 @@
 (decl pure u64_xor (u64 u64) u64)
 (extern constructor u64_xor u64_xor)
 
+(decl pure u64_shl (u64 u64) u64)
+(extern constructor u64_shl u64_shl)
+
 (decl pure imm64_shl (Type Imm64 Imm64) Imm64)
 (extern constructor imm64_shl imm64_shl)
 
@@ -388,8 +398,8 @@
 (extern constructor imm64_masked imm64_masked)
 
 ;; Extract a `u64` from an `Ieee32`.
-(decl u64_from_ieee32 (u64) Ieee32)
-(extern extractor infallible u64_from_ieee32 u64_from_ieee32)
+(decl u32_from_ieee32 (u32) Ieee32)
+(extern extractor infallible u32_from_ieee32 u32_from_ieee32)
 
 ;; Extract a `u64` from an `Ieee64`.
 (decl u64_from_ieee64 (u64) Ieee64)
diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs
index 4c46ac225..cd8d76771 100644
--- a/cranelift/codegen/src/verifier/mod.rs
+++ b/cranelift/codegen/src/verifier/mod.rs
@@ -1685,6 +1685,28 @@ impl<'a> Verifier<'a> {
                     Ok(())
                 }
             }
+            ir::InstructionData::Shuffle {
+                opcode: ir::instructions::Opcode::Shuffle,
+                imm,
+                ..
+            } => {
+                let imm = self.func.dfg.immediates.get(imm).unwrap().as_slice();
+                if imm.len() != 16 {
+                    errors.fatal((
+                        inst,
+                        self.context(inst),
+                        format!("the shuffle immediate wasn't 16-bytes long"),
+                    ))
+                } else if let Some(i) = imm.iter().find(|i| **i >= 32) {
+                    errors.fatal((
+                        inst,
+                        self.context(inst),
+                        format!("shuffle immediate index {i} is larger than the maximum 31"),
+                    ))
+                } else {
+                    Ok(())
+                }
+            }
             _ => Ok(()),
         }
     }
diff --git a/cranelift/entity/src/set.rs b/cranelift/entity/src/set.rs
index 4b64d79f4..a5c8b7b29 100644
--- a/cranelift/entity/src/set.rs
+++ b/cranelift/entity/src/set.rs
@@ -5,6 +5,9 @@ use crate::EntityRef;
 use alloc::vec::Vec;
 use core::marker::PhantomData;
 
+// How many bits are used to represent a single element in `EntitySet`.
+const BITS: usize = core::mem::size_of::<usize>() * 8;
+
 /// A set of `K` for densely indexed entity references.
 ///
 /// The `EntitySet` data structure uses the dense index space to implement a set with a bitvector.
@@ -14,7 +17,7 @@ pub struct EntitySet<K>
 where
     K: EntityRef,
 {
-    elems: Vec<u8>,
+    elems: Vec<usize>,
     len: usize,
     unused: PhantomData<K>,
 }
@@ -42,7 +45,7 @@ where
     /// Creates a new empty set with the specified capacity.
     pub fn with_capacity(capacity: usize) -> Self {
         Self {
-            elems: Vec::with_capacity((capacity + 7) / 8),
+            elems: Vec::with_capacity((capacity + (BITS - 1)) / BITS),
             ..Self::new()
         }
     }
@@ -51,7 +54,7 @@ where
     pub fn contains(&self, k: K) -> bool {
         let index = k.index();
         if index < self.len {
-            (self.elems[index / 8] & (1 << (index % 8))) != 0
+            (self.elems[index / BITS] & (1 << (index % BITS))) != 0
         } else {
             false
         }
@@ -71,11 +74,11 @@ where
     /// `clear`ed or created with `new`.
     pub fn cardinality(&self) -> usize {
         let mut n: usize = 0;
-        for byte_ix in 0..self.len / 8 {
-            n += self.elems[byte_ix].count_ones() as usize;
+        for idx in 0..self.len / BITS {
+            n += self.elems[idx].count_ones() as usize;
         }
-        for bit_ix in (self.len / 8) * 8..self.len {
-            if (self.elems[bit_ix / 8] & (1 << (bit_ix % 8))) != 0 {
+        for bit_ix in (self.len / BITS) * BITS..self.len {
+            if (self.elems[bit_ix / BITS] & (1 << (bit_ix % BITS))) != 0 {
                 n += 1;
             }
         }
@@ -95,7 +98,7 @@ where
 
     /// Resize the set to have `n` entries by adding default entries as needed.
     pub fn resize(&mut self, n: usize) {
-        self.elems.resize((n + 7) / 8, 0);
+        self.elems.resize((n + (BITS - 1)) / BITS, 0);
         self.len = n
     }
 
@@ -106,7 +109,7 @@ where
             self.resize(index + 1)
         }
         let result = !self.contains(k);
-        self.elems[index / 8] |= 1 << (index % 8);
+        self.elems[index / BITS] |= 1 << (index % BITS);
         result
     }
 
@@ -118,7 +121,7 @@ where
 
         // Clear the last known entity in the list.
         let last_index = self.len - 1;
-        self.elems[last_index / 8] &= !(1 << (last_index % 8));
+        self.elems[last_index / BITS] &= !(1 << (last_index % BITS));
 
         // Set the length to the next last stored entity or zero if we pop'ed
         // the last entity.
@@ -127,12 +130,14 @@ where
             .iter()
             .enumerate()
             .rev()
-            .find(|(_, &byte)| byte != 0)
-            // Map `i` from byte index to bit level index.
-            // `(i + 1) * 8` = Last bit in byte.
-            // `last - byte.leading_zeros()` = last set bit in byte.
+            .find(|(_, &elem)| elem != 0)
+            // Map `i` from `elem` index to bit level index.
+            // `(i + 1) * BITS` = Last bit in `elem`.
+            // `last - elem.leading_zeros()` = last set bit in `elem`.
             // `as usize` won't ever truncate as the potential range is `0..=8`.
-            .map_or(0, |(i, byte)| ((i + 1) * 8) - byte.leading_zeros() as usize);
+            .map_or(0, |(i, elem)| {
+                ((i + 1) * BITS) - elem.leading_zeros() as usize
+            });
 
         Some(K::new(last_index))
     }
diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs
index 527677466..ff7ae0b88 100644
--- a/cranelift/fuzzgen/src/function_generator.rs
+++ b/cranelift/fuzzgen/src/function_generator.rs
@@ -339,7 +339,10 @@ fn insert_shuffle(
     let rhs = builder.use_var(fgen.get_variable_of_type(ctrl_type)?);
 
     let mask = {
-        let lanes = fgen.u.arbitrary::<[u8; 16]>()?;
+        let mut lanes = [0u8; 16];
+        for lane in lanes.iter_mut() {
+            *lane = fgen.u.int_in_range(0..=31)?;
+        }
         let lanes = ConstantData::from(lanes.as_ref());
         builder.func.dfg.immediates.push(lanes)
     };
diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs
index d9c9d8c95..aed73950e 100644
--- a/crates/cranelift/src/lib.rs
+++ b/crates/cranelift/src/lib.rs
@@ -135,7 +135,7 @@ fn indirect_signature(isa: &dyn TargetIsa, wasm: &WasmFuncType) -> ir::Signature
     return sig;
 }
 
-/// Returns the cranelift fucntion signature of the function specified.
+/// Returns the cranelift function signature of the function specified.
 ///
 /// Note that this will determine the calling convention for the function, and
 /// namely includes an optimization where functions never exported from a module


Test patch:
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
index e06221836..c0ba8ce1b 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
@@ -6891,24 +6891,6 @@ fn test_aarch64_binemit() {
         "stp q18, q22, [sp], #304",
     ));
 
-    insns.push((
-        Inst::LoadFpuConst64 {
-            rd: writable_vreg(16),
-            const_data: 1.0_f64.to_bits(),
-        },
-        "5000005C03000014000000000000F03F",
-        "ldr d16, pc+8 ; b 12 ; data.f64 1",
-    ));
-
-    insns.push((
-        Inst::LoadFpuConst128 {
-            rd: writable_vreg(5),
-            const_data: 0x0f0e0d0c0b0a09080706050403020100,
-        },
-        "4500009C05000014000102030405060708090A0B0C0D0E0F",
-        "ldr q5, pc+8 ; b 20 ; data.f128 0x0f0e0d0c0b0a09080706050403020100",
-    ));
-
     insns.push((
         Inst::FpuCSel32 {
             rd: writable_vreg(1),
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 717e0fc39..d94828c55 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -4871,7 +4871,7 @@ fn test_x64_emit() {
             imm: 2,
         },
         "C4430920EF02",
-        "vpinsrb $2 %xmm14, %r15, %xmm13",
+        "vpinsrb $2, %xmm14, %r15, %xmm13",
     ));
 
     // ========================================================
diff --git a/cranelift/filetests/filetests/isa/aarch64/constants.clif b/cranelift/filetests/filetests/isa/aarch64/constants.clif
index 53795f2ec..01d6f5c17 100644
--- a/cranelift/filetests/filetests/isa/aarch64/constants.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/constants.clif
@@ -356,14 +356,14 @@ block0:
 
 ; VCode:
 ; block0:
-;   movz x1, #16457, LSL #48
-;   fmov d0, x1
+;   movz x0, #16457, LSL #48
+;   fmov d0, x0
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   mov x1, #0x4049000000000000
-;   fmov d0, x1
+;   mov x0, #0x4049000000000000
+;   fmov d0, x0
 ;   ret
 
 function %f() -> f32 {
@@ -374,14 +374,14 @@ block0:
 
 ; VCode:
 ; block0:
-;   movz x1, #16968, LSL #16
-;   fmov s0, w1
+;   movz w0, #16968, LSL #16
+;   fmov s0, w0
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   mov x1, #0x42480000
-;   fmov s0, w1
+;   mov w0, #0x42480000
+;   fmov s0, w0
 ;   ret
 
 function %f() -> f64 {
diff --git a/cranelift/filetests/filetests/isa/aarch64/fcvt-small.clif b/cranelift/filetests/filetests/isa/aarch64/fcvt-small.clif
index 2bd77df45..1074c034a 100644
--- a/cranelift/filetests/filetests/isa/aarch64/fcvt-small.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/fcvt-small.clif
@@ -87,9 +87,9 @@ block0(v0: f32):
 ;   fmov s4, #-1
 ;   fcmp s0, s4
 ;   b.gt 8 ; udf
-;   movz x9, #17280, LSL #16
-;   fmov s17, w9
-;   fcmp s0, s17
+;   movz w8, #17280, LSL #16
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, s0
 ;   ret
@@ -103,9 +103,9 @@ block0(v0: f32):
 ;   fcmp s0, s4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x43800000
-;   fmov s17, w9
-;   fcmp s0, s17
+;   mov w8, #0x43800000
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, s0
@@ -124,9 +124,9 @@ block0(v0: f64):
 ;   fmov d4, #-1
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #16496, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #16496, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, d0
 ;   ret
@@ -140,9 +140,9 @@ block0(v0: f64):
 ;   fcmp d0, d4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x4070000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
+;   mov x8, #0x4070000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, d0
@@ -161,9 +161,9 @@ block0(v0: f32):
 ;   fmov s4, #-1
 ;   fcmp s0, s4
 ;   b.gt 8 ; udf
-;   movz x9, #18304, LSL #16
-;   fmov s17, w9
-;   fcmp s0, s17
+;   movz w8, #18304, LSL #16
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, s0
 ;   ret
@@ -177,9 +177,9 @@ block0(v0: f32):
 ;   fcmp s0, s4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x47800000
-;   fmov s17, w9
-;   fcmp s0, s17
+;   mov w8, #0x47800000
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, s0
@@ -198,9 +198,9 @@ block0(v0: f64):
 ;   fmov d4, #-1
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #16624, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #16624, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, d0
 ;   ret
@@ -214,9 +214,9 @@ block0(v0: f64):
 ;   fcmp d0, d4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x40f0000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
+;   mov x8, #0x40f0000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, d0
diff --git a/cranelift/filetests/filetests/isa/aarch64/fcvt.clif b/cranelift/filetests/filetests/isa/aarch64/fcvt.clif
index 06ba98d8b..c17f495cc 100644
--- a/cranelift/filetests/filetests/isa/aarch64/fcvt.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/fcvt.clif
@@ -241,9 +241,9 @@ block0(v0: f32):
 ;   fmov s4, #-1
 ;   fcmp s0, s4
 ;   b.gt 8 ; udf
-;   movz x9, #20352, LSL #16
-;   fmov s17, w9
-;   fcmp s0, s17
+;   movz w8, #20352, LSL #16
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, s0
 ;   ret
@@ -257,9 +257,9 @@ block0(v0: f32):
 ;   fcmp s0, s4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x4f800000
-;   fmov s17, w9
-;   fcmp s0, s17
+;   mov w8, #0x4f800000
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, s0
@@ -278,9 +278,9 @@ block0(v0: f32):
 ;   fmov s4, #-1
 ;   fcmp s0, s4
 ;   b.gt 8 ; udf
-;   movz x9, #24448, LSL #16
-;   fmov s17, w9
-;   fcmp s0, s17
+;   movz w8, #24448, LSL #16
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt 8 ; udf
 ;   fcvtzu x0, s0
 ;   ret
@@ -294,9 +294,9 @@ block0(v0: f32):
 ;   fcmp s0, s4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x5f800000
-;   fmov s17, w9
-;   fcmp s0, s17
+;   mov w8, #0x5f800000
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu x0, s0
@@ -315,9 +315,9 @@ block0(v0: f64):
 ;   fmov d4, #-1
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #16880, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #16880, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, d0
 ;   ret
@@ -331,9 +331,9 @@ block0(v0: f64):
 ;   fcmp d0, d4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x41f0000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
+;   mov x8, #0x41f0000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, d0
@@ -352,9 +352,9 @@ block0(v0: f64):
 ;   fmov d4, #-1
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #17392, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #17392, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzu x0, d0
 ;   ret
@@ -368,9 +368,9 @@ block0(v0: f64):
 ;   fcmp d0, d4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x43f0000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
+;   mov x8, #0x43f0000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu x0, d0
@@ -450,13 +450,13 @@ block0(v0: f32):
 ; block0:
 ;   fcmp s0, s0
 ;   b.vc 8 ; udf
-;   movz x5, #52992, LSL #16
-;   fmov s5, w5
-;   fcmp s0, s5
+;   movz w4, #52992, LSL #16
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge 8 ; udf
-;   movz x11, #20224, LSL #16
-;   fmov s19, w11
-;   fcmp s0, s19
+;   movz w10, #20224, LSL #16
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt 8 ; udf
 ;   fcvtzs w0, s0
 ;   ret
@@ -466,14 +466,14 @@ block0(v0: f32):
 ;   fcmp s0, s0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   mov x5, #0xcf000000
-;   fmov s5, w5
-;   fcmp s0, s5
+;   mov w4, #-0x31000000
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge #0x20
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x11, #0x4f000000
-;   fmov s19, w11
-;   fcmp s0, s19
+;   mov w10, #0x4f000000
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt #0x34
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs w0, s0
@@ -489,13 +489,13 @@ block0(v0: f32):
 ; block0:
 ;   fcmp s0, s0
 ;   b.vc 8 ; udf
-;   movz x5, #57088, LSL #16
-;   fmov s5, w5
-;   fcmp s0, s5
+;   movz w4, #57088, LSL #16
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge 8 ; udf
-;   movz x11, #24320, LSL #16
-;   fmov s19, w11
-;   fcmp s0, s19
+;   movz w10, #24320, LSL #16
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt 8 ; udf
 ;   fcvtzs x0, s0
 ;   ret
@@ -505,14 +505,14 @@ block0(v0: f32):
 ;   fcmp s0, s0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   mov x5, #0xdf000000
-;   fmov s5, w5
-;   fcmp s0, s5
+;   mov w4, #-0x21000000
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge #0x20
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x11, #0x5f000000
-;   fmov s19, w11
-;   fcmp s0, s19
+;   mov w10, #0x5f000000
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt #0x34
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs x0, s0
@@ -528,12 +528,12 @@ block0(v0: f64):
 ; block0:
 ;   fcmp d0, d0
 ;   b.vc 8 ; udf
-;   ldr d4, pc+8 ; b 12 ; data.f64 -2147483649
+;   ldr d4, [const(0)]
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #16864, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #16864, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzs w0, d0
 ;   ret
@@ -543,20 +543,19 @@ block0(v0: f64):
 ;   fcmp d0, d0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   ldr d4, #0x14
-;   b #0x1c
-;   .byte 0x00, 0x00, 0x20, 0x00
-;   .byte 0x00, 0x00, 0xe0, 0xc1
+;   ldr d4, #0x38
 ;   fcmp d0, d4
-;   b.gt #0x28
+;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x41e0000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
-;   b.lt #0x3c
+;   mov x8, #0x41e0000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
+;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs w0, d0
 ;   ret
+;   .byte 0x00, 0x00, 0x20, 0x00
+;   .byte 0x00, 0x00, 0xe0, 0xc1
 
 function %f24(f64) -> i64 {
 block0(v0: f64):
@@ -568,13 +567,13 @@ block0(v0: f64):
 ; block0:
 ;   fcmp d0, d0
 ;   b.vc 8 ; udf
-;   movz x5, #50144, LSL #48
-;   fmov d5, x5
-;   fcmp d0, d5
+;   movz x4, #50144, LSL #48
+;   fmov d6, x4
+;   fcmp d0, d6
 ;   b.ge 8 ; udf
-;   movz x11, #17376, LSL #48
-;   fmov d19, x11
-;   fcmp d0, d19
+;   movz x10, #17376, LSL #48
+;   fmov d20, x10
+;   fcmp d0, d20
 ;   b.lt 8 ; udf
 ;   fcvtzs x0, d0
 ;   ret
@@ -584,14 +583,14 @@ block0(v0: f64):
 ;   fcmp d0, d0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   mov x5, #-0x3c20000000000000
-;   fmov d5, x5
-;   fcmp d0, d5
+;   mov x4, #-0x3c20000000000000
+;   fmov d6, x4
+;   fcmp d0, d6
 ;   b.ge #0x20
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x11, #0x43e0000000000000
-;   fmov d19, x11
-;   fcmp d0, d19
+;   mov x10, #0x43e0000000000000
+;   fmov d20, x10
+;   fcmp d0, d20
 ;   b.lt #0x34
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs x0, d0
diff --git a/cranelift/filetests/filetests/isa/aarch64/floating-point.clif b/cranelift/filetests/filetests/isa/aarch64/floating-point.clif
index 3ec617954..186b06ad9 100644
--- a/cranelift/filetests/filetests/isa/aarch64/floating-point.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/floating-point.clif
@@ -531,9 +531,9 @@ block0(v0: f32):
 ;   fmov s4, #-1
 ;   fcmp s0, s4
 ;   b.gt 8 ; udf
-;   movz x9, #20352, LSL #16
-;   fmov s17, w9
-;   fcmp s0, s17
+;   movz w8, #20352, LSL #16
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, s0
 ;   ret
@@ -547,9 +547,9 @@ block0(v0: f32):
 ;   fcmp s0, s4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x4f800000
-;   fmov s17, w9
-;   fcmp s0, s17
+;   mov w8, #0x4f800000
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, s0
@@ -565,13 +565,13 @@ block0(v0: f32):
 ; block0:
 ;   fcmp s0, s0
 ;   b.vc 8 ; udf
-;   movz x5, #52992, LSL #16
-;   fmov s5, w5
-;   fcmp s0, s5
+;   movz w4, #52992, LSL #16
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge 8 ; udf
-;   movz x11, #20224, LSL #16
-;   fmov s19, w11
-;   fcmp s0, s19
+;   movz w10, #20224, LSL #16
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt 8 ; udf
 ;   fcvtzs w0, s0
 ;   ret
@@ -581,14 +581,14 @@ block0(v0: f32):
 ;   fcmp s0, s0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   mov x5, #0xcf000000
-;   fmov s5, w5
-;   fcmp s0, s5
+;   mov w4, #-0x31000000
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge #0x20
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x11, #0x4f000000
-;   fmov s19, w11
-;   fcmp s0, s19
+;   mov w10, #0x4f000000
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt #0x34
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs w0, s0
@@ -607,9 +607,9 @@ block0(v0: f32):
 ;   fmov s4, #-1
 ;   fcmp s0, s4
 ;   b.gt 8 ; udf
-;   movz x9, #24448, LSL #16
-;   fmov s17, w9
-;   fcmp s0, s17
+;   movz w8, #24448, LSL #16
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt 8 ; udf
 ;   fcvtzu x0, s0
 ;   ret
@@ -623,9 +623,9 @@ block0(v0: f32):
 ;   fcmp s0, s4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x5f800000
-;   fmov s17, w9
-;   fcmp s0, s17
+;   mov w8, #0x5f800000
+;   fmov s18, w8
+;   fcmp s0, s18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu x0, s0
@@ -641,13 +641,13 @@ block0(v0: f32):
 ; block0:
 ;   fcmp s0, s0
 ;   b.vc 8 ; udf
-;   movz x5, #57088, LSL #16
-;   fmov s5, w5
-;   fcmp s0, s5
+;   movz w4, #57088, LSL #16
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge 8 ; udf
-;   movz x11, #24320, LSL #16
-;   fmov s19, w11
-;   fcmp s0, s19
+;   movz w10, #24320, LSL #16
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt 8 ; udf
 ;   fcvtzs x0, s0
 ;   ret
@@ -657,14 +657,14 @@ block0(v0: f32):
 ;   fcmp s0, s0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   mov x5, #0xdf000000
-;   fmov s5, w5
-;   fcmp s0, s5
+;   mov w4, #-0x21000000
+;   fmov s6, w4
+;   fcmp s0, s6
 ;   b.ge #0x20
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x11, #0x5f000000
-;   fmov s19, w11
-;   fcmp s0, s19
+;   mov w10, #0x5f000000
+;   fmov s20, w10
+;   fcmp s0, s20
 ;   b.lt #0x34
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs x0, s0
@@ -683,9 +683,9 @@ block0(v0: f64):
 ;   fmov d4, #-1
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #16880, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #16880, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzu w0, d0
 ;   ret
@@ -699,9 +699,9 @@ block0(v0: f64):
 ;   fcmp d0, d4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x41f0000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
+;   mov x8, #0x41f0000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu w0, d0
@@ -717,12 +717,12 @@ block0(v0: f64):
 ; block0:
 ;   fcmp d0, d0
 ;   b.vc 8 ; udf
-;   ldr d4, pc+8 ; b 12 ; data.f64 -2147483649
+;   ldr d4, [const(0)]
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #16864, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #16864, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzs w0, d0
 ;   ret
@@ -732,20 +732,19 @@ block0(v0: f64):
 ;   fcmp d0, d0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   ldr d4, #0x14
-;   b #0x1c
-;   .byte 0x00, 0x00, 0x20, 0x00
-;   .byte 0x00, 0x00, 0xe0, 0xc1
+;   ldr d4, #0x38
 ;   fcmp d0, d4
-;   b.gt #0x28
+;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x41e0000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
-;   b.lt #0x3c
+;   mov x8, #0x41e0000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
+;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs w0, d0
 ;   ret
+;   .byte 0x00, 0x00, 0x20, 0x00
+;   .byte 0x00, 0x00, 0xe0, 0xc1
 
 function %f39(f64) -> i64 {
 block0(v0: f64):
@@ -760,9 +759,9 @@ block0(v0: f64):
 ;   fmov d4, #-1
 ;   fcmp d0, d4
 ;   b.gt 8 ; udf
-;   movz x9, #17392, LSL #48
-;   fmov d17, x9
-;   fcmp d0, d17
+;   movz x8, #17392, LSL #48
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt 8 ; udf
 ;   fcvtzu x0, d0
 ;   ret
@@ -776,9 +775,9 @@ block0(v0: f64):
 ;   fcmp d0, d4
 ;   b.gt #0x1c
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x9, #0x43f0000000000000
-;   fmov d17, x9
-;   fcmp d0, d17
+;   mov x8, #0x43f0000000000000
+;   fmov d18, x8
+;   fcmp d0, d18
 ;   b.lt #0x30
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzu x0, d0
@@ -794,13 +793,13 @@ block0(v0: f64):
 ; block0:
 ;   fcmp d0, d0
 ;   b.vc 8 ; udf
-;   movz x5, #50144, LSL #48
-;   fmov d5, x5
-;   fcmp d0, d5
+;   movz x4, #50144, LSL #48
+;   fmov d6, x4
+;   fcmp d0, d6
 ;   b.ge 8 ; udf
-;   movz x11, #17376, LSL #48
-;   fmov d19, x11
-;   fcmp d0, d19
+;   movz x10, #17376, LSL #48
+;   fmov d20, x10
+;   fcmp d0, d20
 ;   b.lt 8 ; udf
 ;   fcvtzs x0, d0
 ;   ret
@@ -810,14 +809,14 @@ block0(v0: f64):
 ;   fcmp d0, d0
 ;   b.vc #0xc
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: bad_toint
-;   mov x5, #-0x3c20000000000000
-;   fmov d5, x5
-;   fcmp d0, d5
+;   mov x4, #-0x3c20000000000000
+;   fmov d6, x4
+;   fcmp d0, d6
 ;   b.ge #0x20
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
-;   mov x11, #0x43e0000000000000
-;   fmov d19, x11
-;   fcmp d0, d19
+;   mov x10, #0x43e0000000000000
+;   fmov d20, x10
+;   fcmp d0, d20
 ;   b.lt #0x34
 ;   .byte 0x1f, 0xc1, 0x00, 0x00 ; trap: int_ovf
 ;   fcvtzs x0, d0
diff --git a/cranelift/filetests/filetests/isa/aarch64/shuffle.clif b/cranelift/filetests/filetests/isa/aarch64/shuffle.clif
index c52959b88..c9020f0cc 100644
--- a/cranelift/filetests/filetests/isa/aarch64/shuffle.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/shuffle.clif
@@ -12,7 +12,7 @@ block0(v0: i8x16, v1: i8x16):
 ; block0:
 ;   mov v30.16b, v0.16b
 ;   mov v31.16b, v1.16b
-;   ldr q3, pc+8 ; b 20 ; data.f128 0x05110f0204180d170b0c06041a1f0003
+;   ldr q3, [const(0)]
 ;   tbl v0.16b, { v30.16b, v31.16b }, v3.16b
 ;   ret
 ; 
@@ -20,14 +20,16 @@ block0(v0: i8x16, v1: i8x16):
 ; block0: ; offset 0x0
 ;   mov v30.16b, v0.16b
 ;   mov v31.16b, v1.16b
-;   ldr q3, #0x10
-;   b #0x20
+;   ldr q3, #0x20
+;   tbl v0.16b, {v30.16b, v31.16b}, v3.16b
+;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
 ;   adc w3, w0, wzr
 ;   add w4, w16, w12, lsl #1
 ;   orr z23.b, p3/m, z23.b, z8.b
 ;   mov z2.b, p1/z, #0x78
-;   tbl v0.16b, {v30.16b, v31.16b}, v3.16b
-;   ret
 
 function %aarch64_uzp1_i8x16(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
@@ -541,7 +543,7 @@ block0(v0: i8x16, v1: i8x16):
 ; block0:
 ;   mov v30.16b, v0.16b
 ;   mov v31.16b, v1.16b
-;   ldr q3, pc+8 ; b 20 ; data.f128 0x1f1e1d1c1b1a19181716151413121110
+;   ldr q3, [const(0)]
 ;   tbl v0.16b, { v30.16b, v31.16b }, v3.16b
 ;   ret
 ; 
@@ -549,14 +551,16 @@ block0(v0: i8x16, v1: i8x16):
 ; block0: ; offset 0x0
 ;   mov v30.16b, v0.16b
 ;   mov v31.16b, v1.16b
-;   ldr q3, #0x10
-;   b #0x20
+;   ldr q3, #0x20
+;   tbl v0.16b, {v30.16b, v31.16b}, v3.16b
+;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
 ;   sbfiz w16, w8, #0xe, #5
-;   b #0xfffffffffc585464
+;   b #0xfffffffffc585474
 ;   madd w24, w8, w26, w6
 ;   fmadd s28, s8, s30, s7
-;   tbl v0.16b, {v30.16b, v31.16b}, v3.16b
-;   ret
 
 function %aarch64_dup_i8x16(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
diff --git a/cranelift/filetests/filetests/isa/aarch64/simd-arithmetic.clif b/cranelift/filetests/filetests/isa/aarch64/simd-arithmetic.clif
index 3487163f4..5a05e8925 100644
--- a/cranelift/filetests/filetests/isa/aarch64/simd-arithmetic.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/simd-arithmetic.clif
@@ -106,10 +106,10 @@ block0(v0: i64x2, v1: i64x2):
 
 ; VCode:
 ; block0:
-;   movz x4, #1
-;   dup v4.2d, x4
+;   movz x3, #1
+;   dup v5.2d, x3
 ;   orr v7.16b, v0.16b, v1.16b
-;   and v17.16b, v7.16b, v4.16b
+;   and v17.16b, v7.16b, v5.16b
 ;   ushr v19.2d, v0.2d, #1
 ;   ushr v21.2d, v1.2d, #1
 ;   add v23.2d, v19.2d, v21.2d
@@ -118,10 +118,10 @@ block0(v0: i64x2, v1: i64x2):
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   mov x4, #1
-;   dup v4.2d, x4
+;   mov x3, #1
+;   dup v5.2d, x3
 ;   orr v7.16b, v0.16b, v1.16b
-;   and v17.16b, v7.16b, v4.16b
+;   and v17.16b, v7.16b, v5.16b
 ;   ushr v19.2d, v0.2d, #1
 ;   ushr v21.2d, v1.2d, #1
 ;   add v23.2d, v19.2d, v21.2d
diff --git a/cranelift/filetests/filetests/isa/aarch64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/aarch64/simd-bitwise-compile.clif
index b4449a967..9b7b85112 100644
--- a/cranelift/filetests/filetests/isa/aarch64/simd-bitwise-compile.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/simd-bitwise-compile.clif
@@ -228,7 +228,7 @@ block0(v0: i32):
 
 ; VCode:
 ; block0:
-;   ldr q5, pc+8 ; b 20 ; data.f128 0x0f0e0d0c0b0a09080706050403020100
+;   ldr q5, [const(0)]
 ;   and w3, w0, #7
 ;   dup v6.16b, w3
 ;   sshl v0.16b, v5.16b, v6.16b
@@ -236,16 +236,18 @@ block0(v0: i32):
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   ldr q5, #8
-;   b #0x18
-;   .byte 0x00, 0x01, 0x02, 0x03
-;   .byte 0x04, 0x05, 0x06, 0x07
-;   add w8, w8, w10, lsl #2
-;   .byte 0x0c, 0x0d, 0x0e, 0x0f
+;   ldr q5, #0x20
 ;   and w3, w0, #7
 ;   dup v6.16b, w3
 ;   sshl v0.16b, v5.16b, v6.16b
 ;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x01, 0x02, 0x03
+;   .byte 0x04, 0x05, 0x06, 0x07
+;   add w8, w8, w10, lsl #2
+;   .byte 0x0c, 0x0d, 0x0e, 0x0f
 
 function %ushr_i8x16_imm() -> i8x16 {
 block0:
@@ -257,7 +259,7 @@ block0:
 
 ; VCode:
 ; block0:
-;   ldr q5, pc+8 ; b 20 ; data.f128 0x0f0e0d0c0b0a09080706050403020100
+;   ldr q5, [const(0)]
 ;   movz w1, #1
 ;   and w3, w1, #7
 ;   sub x5, xzr, x3
@@ -267,18 +269,18 @@ block0:
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   ldr q5, #8
-;   b #0x18
-;   .byte 0x00, 0x01, 0x02, 0x03
-;   .byte 0x04, 0x05, 0x06, 0x07
-;   add w8, w8, w10, lsl #2
-;   .byte 0x0c, 0x0d, 0x0e, 0x0f
+;   ldr q5, #0x20
 ;   mov w1, #1
 ;   and w3, w1, #7
 ;   neg x5, x3
 ;   dup v7.16b, w5
 ;   ushl v0.16b, v5.16b, v7.16b
 ;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x01, 0x02, 0x03
+;   .byte 0x04, 0x05, 0x06, 0x07
+;   add w8, w8, w10, lsl #2
+;   .byte 0x0c, 0x0d, 0x0e, 0x0f
 
 function %sshr_i8x16(i32) -> i8x16 {
 block0(v0: i32):
@@ -289,7 +291,7 @@ block0(v0: i32):
 
 ; VCode:
 ; block0:
-;   ldr q6, pc+8 ; b 20 ; data.f128 0x0f0e0d0c0b0a09080706050403020100
+;   ldr q6, [const(0)]
 ;   and w3, w0, #7
 ;   sub x5, xzr, x3
 ;   dup v7.16b, w5
@@ -298,17 +300,18 @@ block0(v0: i32):
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   ldr q6, #8
-;   b #0x18
-;   .byte 0x00, 0x01, 0x02, 0x03
-;   .byte 0x04, 0x05, 0x06, 0x07
-;   add w8, w8, w10, lsl #2
-;   .byte 0x0c, 0x0d, 0x0e, 0x0f
+;   ldr q6, #0x20
 ;   and w3, w0, #7
 ;   neg x5, x3
 ;   dup v7.16b, w5
 ;   sshl v0.16b, v6.16b, v7.16b
 ;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x01, 0x02, 0x03
+;   .byte 0x04, 0x05, 0x06, 0x07
+;   add w8, w8, w10, lsl #2
+;   .byte 0x0c, 0x0d, 0x0e, 0x0f
 
 function %sshr_i8x16_imm(i8x16, i32) -> i8x16 {
 block0(v0: i8x16, v1: i32):
diff --git a/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif b/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif
index af539f84b..3b19f5d0e 100644
--- a/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/simd-lane-access-compile.clif
@@ -15,25 +15,26 @@ block0:
 ; VCode:
 ; block0:
 ;   movi v30.16b, #0
-;   movz x4, #1
-;   fmov s31, w4
-;   ldr q3, pc+8 ; b 20 ; data.f128 0x11000000000000000000000000000000
+;   movz w3, #1
+;   fmov s31, w3
+;   ldr q3, [const(0)]
 ;   tbl v0.16b, { v30.16b, v31.16b }, v3.16b
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   movi v30.16b, #0
-;   mov x4, #1
-;   fmov s31, w4
-;   ldr q3, #0x14
-;   b #0x24
+;   mov w3, #1
+;   fmov s31, w3
+;   ldr q3, #0x20
+;   tbl v0.16b, {v30.16b, v31.16b}, v3.16b
+;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
 ;   .byte 0x00, 0x00, 0x00, 0x00
 ;   .byte 0x00, 0x00, 0x00, 0x00
 ;   .byte 0x00, 0x00, 0x00, 0x00
 ;   add w0, w0, #0
-;   tbl v0.16b, {v30.16b, v31.16b}, v3.16b
-;   ret
 
 function %shuffle_same_ssa_value() -> i8x16 {
 block0:
@@ -44,26 +45,27 @@ block0:
 
 ; VCode:
 ; block0:
-;   movz x3, #1
-;   fmov s31, w3
-;   ldr q2, pc+8 ; b 20 ; data.f128 0x13000000000000000000000000000000
+;   movz w2, #1
+;   fmov s31, w2
+;   ldr q2, [const(0)]
 ;   mov v30.16b, v31.16b
 ;   tbl v0.16b, { v30.16b, v31.16b }, v2.16b
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   mov x3, #1
-;   fmov s31, w3
-;   ldr q2, #0x10
-;   b #0x20
+;   mov w2, #1
+;   fmov s31, w2
+;   ldr q2, #0x20
+;   mov v30.16b, v31.16b
+;   tbl v0.16b, {v30.16b, v31.16b}, v2.16b
+;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
 ;   .byte 0x00, 0x00, 0x00, 0x00
 ;   .byte 0x00, 0x00, 0x00, 0x00
 ;   .byte 0x00, 0x00, 0x00, 0x00
 ;   sbfx w0, w0, #0, #1
-;   mov v30.16b, v31.16b
-;   tbl v0.16b, {v30.16b, v31.16b}, v2.16b
-;   ret
 
 function %swizzle() -> i8x16 {
 block0:
@@ -75,27 +77,25 @@ block0:
 
 ; VCode:
 ; block0:
-;   ldr q2, pc+8 ; b 20 ; data.f128 0x0f0e0d0c0b0a09080706050403020100
-;   ldr q3, pc+8 ; b 20 ; data.f128 0x0f0e0d0c0b0a09080706050403020100
+;   ldr q2, [const(1)]
+;   ldr q3, [const(0)]
 ;   tbl v0.16b, { v2.16b }, v3.16b
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   ldr q2, #8
-;   b #0x18
+;   ldr q2, #0x20
+;   ldr q3, #0x10
+;   tbl v0.16b, {v2.16b}, v3.16b
+;   ret
 ;   .byte 0x00, 0x01, 0x02, 0x03
 ;   .byte 0x04, 0x05, 0x06, 0x07
 ;   add w8, w8, w10, lsl #2
 ;   .byte 0x0c, 0x0d, 0x0e, 0x0f
-;   ldr q3, #0x20
-;   b #0x30
 ;   .byte 0x00, 0x01, 0x02, 0x03
 ;   .byte 0x04, 0x05, 0x06, 0x07
 ;   add w8, w8, w10, lsl #2
 ;   .byte 0x0c, 0x0d, 0x0e, 0x0f
-;   tbl v0.16b, {v2.16b}, v3.16b
-;   ret
 
 function %splat_i8(i8) -> i8x16 {
 block0(v0: i8):
diff --git a/cranelift/filetests/filetests/isa/aarch64/simd.clif b/cranelift/filetests/filetests/isa/aarch64/simd.clif
index 4933878ac..47e1f23f8 100644
--- a/cranelift/filetests/filetests/isa/aarch64/simd.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/simd.clif
@@ -33,14 +33,14 @@ block0:
 
 ; VCode:
 ; block0:
-;   movz x1, #42679
-;   dup v0.8h, w1
+;   movz x0, #42679
+;   dup v0.8h, w0
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   mov x1, #0xa6b7
-;   dup v0.8h, w1
+;   mov x0, #0xa6b7
+;   dup v0.8h, w0
 ;   ret
 
 function %f4(i32, i8x16, i8x16) -> i8x16 {
@@ -156,14 +156,14 @@ block0:
 
 ; VCode:
 ; block0:
-;   movi v1.2d, #18374687579166474495
-;   fmov d0, d1
+;   movi v0.2d, #18374687579166474495
+;   fmov d0, d0
 ;   ret
 ; 
 ; Disassembled:
 ; block0: ; offset 0x0
-;   movi v1.2d, #0xff0000ffff0000ff
-;   fmov d0, d1
+;   movi v0.2d, #0xff0000ffff0000ff
+;   fmov d0, d0
 ;   ret
 
 function %f10() -> i32x4 {
diff --git a/cranelift/filetests/filetests/isa/aarch64/vhigh_bits.clif b/cranelift/filetests/filetests/isa/aarch64/vhigh_bits.clif
index e0034049a..6a36c9019 100644
--- a/cranelift/filetests/filetests/isa/aarch64/vhigh_bits.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/vhigh_bits.clif
@@ -10,12 +10,12 @@ block0(v0: i8x16):
 ; VCode:
 ; block0:
 ;   sshr v2.16b, v0.16b, #7
-;   movz x5, #513
-;   movk x5, x5, #2052, LSL #16
-;   movk x5, x5, #8208, LSL #32
-;   movk x5, x5, #32832, LSL #48
-;   dup v16.2d, x5
-;   and v22.16b, v2.16b, v16.16b
+;   movz x7, #513
+;   movk x7, x7, #2052, LSL #16
+;   movk x7, x7, #8208, LSL #32
+;   movk x7, x7, #32832, LSL #48
+;   dup v20.2d, x7
+;   and v22.16b, v2.16b, v20.16b
 ;   ext v24.16b, v22.16b, v22.16b, #8
 ;   zip1 v26.16b, v22.16b, v24.16b
 ;   addv h28, v26.8h
@@ -25,12 +25,12 @@ block0(v0: i8x16):
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   sshr v2.16b, v0.16b, #7
-;   mov x5, #0x201
-;   movk x5, #0x804, lsl #16
-;   movk x5, #0x2010, lsl #32
-;   movk x5, #0x8040, lsl #48
-;   dup v16.2d, x5
-;   and v22.16b, v2.16b, v16.16b
+;   mov x7, #0x201
+;   movk x7, #0x804, lsl #16
+;   movk x7, #0x2010, lsl #32
+;   movk x7, #0x8040, lsl #48
+;   dup v20.2d, x7
+;   and v22.16b, v2.16b, v20.16b
 ;   ext v24.16b, v22.16b, v22.16b, #8
 ;   zip1 v26.16b, v22.16b, v24.16b
 ;   addv h28, v26.8h
@@ -46,12 +46,12 @@ block0(v0: i8x16):
 ; VCode:
 ; block0:
 ;   sshr v2.16b, v0.16b, #7
-;   movz x5, #513
-;   movk x5, x5, #2052, LSL #16
-;   movk x5, x5, #8208, LSL #32
-;   movk x5, x5, #32832, LSL #48
-;   dup v16.2d, x5
-;   and v22.16b, v2.16b, v16.16b
+;   movz x7, #513
+;   movk x7, x7, #2052, LSL #16
+;   movk x7, x7, #8208, LSL #32
+;   movk x7, x7, #32832, LSL #48
+;   dup v20.2d, x7
+;   and v22.16b, v2.16b, v20.16b
 ;   ext v24.16b, v22.16b, v22.16b, #8
 ;   zip1 v26.16b, v22.16b, v24.16b
 ;   addv h28, v26.8h
@@ -61,12 +61,12 @@ block0(v0: i8x16):
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   sshr v2.16b, v0.16b, #7
-;   mov x5, #0x201
-;   movk x5, #0x804, lsl #16
-;   movk x5, #0x2010, lsl #32
-;   movk x5, #0x8040, lsl #48
-;   dup v16.2d, x5
-;   and v22.16b, v2.16b, v16.16b
+;   mov x7, #0x201
+;   movk x7, #0x804, lsl #16
+;   movk x7, #0x2010, lsl #32
+;   movk x7, #0x8040, lsl #48
+;   dup v20.2d, x7
+;   and v22.16b, v2.16b, v20.16b
 ;   ext v24.16b, v22.16b, v22.16b, #8
 ;   zip1 v26.16b, v22.16b, v24.16b
 ;   addv h28, v26.8h
@@ -82,7 +82,7 @@ block0(v0: i16x8):
 ; VCode:
 ; block0:
 ;   sshr v2.8h, v0.8h, #15
-;   ldr q4, pc+8 ; b 20 ; data.f128 0x00800040002000100008000400020001
+;   ldr q4, [const(0)]
 ;   and v6.16b, v2.16b, v4.16b
 ;   addv h16, v6.8h
 ;   umov w0, v16.h[0]
@@ -91,16 +91,17 @@ block0(v0: i16x8):
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   sshr v2.8h, v0.8h, #0xf
-;   ldr q4, #0xc
-;   b #0x1c
-;   .byte 0x01, 0x00, 0x02, 0x00
-;   .byte 0x04, 0x00, 0x08, 0x00
-;   .byte 0x10, 0x00, 0x20, 0x00
-;   .byte 0x40, 0x00, 0x80, 0x00
+;   ldr q4, #0x20
 ;   and v6.16b, v2.16b, v4.16b
 ;   addv h16, v6.8h
 ;   umov w0, v16.h[0]
 ;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x01, 0x00, 0x02, 0x00
+;   .byte 0x04, 0x00, 0x08, 0x00
+;   .byte 0x10, 0x00, 0x20, 0x00
+;   .byte 0x40, 0x00, 0x80, 0x00
 
 function %f4(i32x4) -> i8 {
 block0(v0: i32x4):
@@ -111,7 +112,7 @@ block0(v0: i32x4):
 ; VCode:
 ; block0:
 ;   sshr v2.4s, v0.4s, #31
-;   ldr q4, pc+8 ; b 20 ; data.f128 0x00000008000000040000000200000001
+;   ldr q4, [const(0)]
 ;   and v6.16b, v2.16b, v4.16b
 ;   addv s16, v6.4s
 ;   mov w0, v16.s[0]
@@ -120,16 +121,17 @@ block0(v0: i32x4):
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   sshr v2.4s, v0.4s, #0x1f
-;   ldr q4, #0xc
-;   b #0x1c
-;   .byte 0x01, 0x00, 0x00, 0x00
-;   .byte 0x02, 0x00, 0x00, 0x00
-;   .byte 0x04, 0x00, 0x00, 0x00
-;   .byte 0x08, 0x00, 0x00, 0x00
+;   ldr q4, #0x20
 ;   and v6.16b, v2.16b, v4.16b
 ;   addv s16, v6.4s
 ;   mov w0, v16.s[0]
 ;   ret
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   .byte 0x01, 0x00, 0x00, 0x00
+;   .byte 0x02, 0x00, 0x00, 0x00
+;   .byte 0x04, 0x00, 0x00, 0x00
+;   .byte 0x08, 0x00, 0x00, 0x00
 
 function %f5(i64x2) -> i8 {
 block0(v0: i64x2):
diff --git a/cranelift/filetests/filetests/isa/riscv64/bitops-float.clif b/cranelift/filetests/filetests/isa/riscv64/bitops-float.clif
new file mode 100644
index 000000000..ffd36afd0
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/bitops-float.clif
@@ -0,0 +1,79 @@
+test compile precise-output
+set unwind_info=false
+target riscv64 has_zbb
+
+;; This is a regression test for a bug in the RISC-V backend where
+;; When enabling `Zbb` the backend would try to use one of the
+;; integer instructions (`orn`) to implement a float operation
+;; causing a regalloc panic.
+
+function %or_not_optimization_float() -> i32 system_v {
+block0:
+    v0 = iconst.i32 0
+    v1 = f32const 0.0
+    v2 = bnot v1 
+    v3 = bor v2, v2
+    br_table v0, block1(v3), [block1(v1)]
+
+block1(v4: f32):
+    return v0
+}
+
+; VCode:
+; block0:
+;   li a1,0
+;   fmv.w.x ft9,a1
+;   li t1,0
+;   fmv.w.x fa6,t1
+;   fmv.x.w a1,fa6
+;   not a3,a1
+;   fmv.w.x ft1,a3
+;   fmv.x.w t1,ft1
+;   fmv.x.w a0,ft1
+;   or a2,t1,a0
+;   fmv.w.x fa2,a2
+;   li t2,0
+;   br_table t2,[MachLabel(1),MachLabel(2)]##tmp1=a1,tmp2=a2
+; block1:
+;   j label3
+; block2:
+;   fmv.d fa2,ft9
+;   j label3
+; block3:
+;   li a0,0
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mv a1, zero
+;   fmv.w.x ft9, a1
+;   mv t1, zero
+;   fmv.w.x fa6, t1
+;   fmv.x.w a1, fa6
+;   not a3, a1
+;   fmv.w.x ft1, a3
+;   fmv.x.w t1, ft1
+;   fmv.x.w a0, ft1
+;   or a2, t1, a0
+;   fmv.w.x fa2, a2
+;   mv t2, zero
+;   slli t6, t2, 0x20
+;   srli t6, t6, 0x20
+;   addi a2, zero, 1
+;   bltu t6, a2, 0xc
+;   auipc a2, 0
+;   jalr zero, a2, 0x28
+;   auipc a1, 0
+;   slli a2, t6, 3
+;   add a1, a1, a2
+;   jalr zero, a1, 0x10
+;   auipc a2, 0
+;   jalr zero, a2, 0xc
+; block1: ; offset 0x60
+;   j 8
+; block2: ; offset 0x64
+;   fmv.d fa2, ft9
+; block3: ; offset 0x68
+;   mv a0, zero
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif b/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif
index 968886d5a..f657ef47e 100644
--- a/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif
+++ b/cranelift/filetests/filetests/isa/s390x/vec-permute-le-lane.clif
@@ -70,37 +70,6 @@ block0(v0: i8x16, v1: i8x16):
 ;   vperm %v24, %v24, %v25, %v3
 ;   br %r14
 
-function %shuffle_2(i8x16, i8x16) -> i8x16 wasmtime_system_v {
-block0(v0: i8x16, v1: i8x16):
-    v2 = shuffle v0, v1, [0 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47]
-    return v2
-}
-
-; VCode:
-; block0:
-;   vgbm %v3, 1
-;   bras %r1, 20 ; data.u128 0x8080808080808080808080808080800f ; vl %v5, 0(%r1)
-;   vperm %v7, %v24, %v25, %v5
-;   vn %v24, %v3, %v7
-;   br %r14
-; 
-; Disassembled:
-; block0: ; offset 0x0
-;   vgbm %v3, 1
-;   bras %r1, 0x1a
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x0f
-;   vl %v5, 0(%r1)
-;   vperm %v7, %v24, %v25, %v5
-;   vn %v24, %v3, %v7
-;   br %r14
-
 function %shuffle_vmrhg_xy(i8x16, i8x16) -> i8x16 wasmtime_system_v {
 block0(v0: i8x16, v1: i8x16):
     v2 = shuffle v0, v1, [24 25 26 27 28 29 30 31 8 9 10 11 12 13 14 15]
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-permute.clif b/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
index 3e8295884..e8ac72764 100644
--- a/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
+++ b/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
@@ -68,37 +68,6 @@ block0(v0: i8x16, v1: i8x16):
 ;   vperm %v24, %v24, %v25, %v3
 ;   br %r14
 
-function %shuffle_2(i8x16, i8x16) -> i8x16 {
-block0(v0: i8x16, v1: i8x16):
-    v2 = shuffle v0, v1, [0 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47]
-    return v2
-}
-
-; VCode:
-; block0:
-;   vgbm %v3, 32768
-;   bras %r1, 20 ; data.u128 0x00808080808080808080808080808080 ; vl %v5, 0(%r1)
-;   vperm %v7, %v24, %v25, %v5
-;   vn %v24, %v3, %v7
-;   br %r14
-; 
-; Disassembled:
-; block0: ; offset 0x0
-;   vgbm %v3, 0x8000
-;   bras %r1, 0x1a
-;   .byte 0x00, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   .byte 0x80, 0x80
-;   vl %v5, 0(%r1)
-;   vperm %v7, %v24, %v25, %v5
-;   vn %v24, %v3, %v7
-;   br %r14
-
 function %shuffle_vmrhg_xy(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
     v2 = shuffle v0, v1, [0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23]
diff --git a/cranelift/filetests/filetests/isa/x64/extractlane-avx.clif b/cranelift/filetests/filetests/isa/x64/extractlane-avx.clif
new file mode 100644
index 000000000..cfefa99e8
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/extractlane-avx.clif
@@ -0,0 +1,309 @@
+test compile precise-output
+target x86_64 has_avx
+
+function %f1(i8x16) -> i8 {
+block0(v0: i8x16):
+  v1 = extractlane v0, 1
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrb $1, %xmm0, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrb $1, %xmm0, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f2(i16x8) -> i16 {
+block0(v0: i16x8):
+  v1 = extractlane v0, 1
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrw $1, %xmm0, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrw $1, %xmm0, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f3(i32x4) -> i32 {
+block0(v0: i32x4):
+  v1 = extractlane v0, 1
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrd $1, %xmm0, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrd $1, %xmm0, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f4(i64x2) -> i64 {
+block0(v0: i64x2):
+  v1 = extractlane v0, 1
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrq $1, %xmm0, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrq $1, %xmm0, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f5(f32x4) -> f32 {
+block0(v0: f32x4):
+  v1 = extractlane v0, 1
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpshufd $1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpshufd $1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f6(f64x2) -> f64 {
+block0(v0: f64x2):
+  v1 = extractlane v0, 1
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpshufd $238, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpshufd $0xee, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extract_i8x16_lane0_to_memory(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrb $0, %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrb $0, %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extract_i16x8_lane0_to_memory(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrw $0, %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrw $0, %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extract_i32x4_lane0_to_memory(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrd $0, %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrd $0, %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extract_f32x4_lane0_to_memory(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovss  %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovss %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extract_i64x2_lane0_to_memory(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpextrq $0, %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpextrq $0, %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extract_f64x2_lane0_to_memory(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovsd  %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovsd %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/extractlane.clif b/cranelift/filetests/filetests/isa/x64/extractlane.clif
index bf6ce4cc9..b4ff83b12 100644
--- a/cranelift/filetests/filetests/isa/x64/extractlane.clif
+++ b/cranelift/filetests/filetests/isa/x64/extractlane.clif
@@ -151,6 +151,58 @@ block0(v0: f64x2):
 ;   popq %rbp
 ;   retq
 
+function %extract_i8x16_lane0_to_memory(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pextrb  $0, %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pextrb $0, %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extract_i16x8_lane0_to_memory(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+  v2 = extractlane v0, 0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pextrw  $0, %xmm0, 0(%rdi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pextrw $0, %xmm0, (%rdi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
 function %extract_i32x4_lane0_to_memory(i32x4, i64) {
 block0(v0: i32x4, v1: i64):
   v2 = extractlane v0, 0
@@ -162,7 +214,7 @@ block0(v0: i32x4, v1: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movss   %xmm0, 0(%rdi)
+;   pextrd  $0, %xmm0, 0(%rdi)
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -172,7 +224,7 @@ block0(v0: i32x4, v1: i64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movss %xmm0, (%rdi) ; trap: heap_oob
+;   pextrd $0, %xmm0, (%rdi) ; trap: heap_oob
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -214,7 +266,7 @@ block0(v0: i64x2, v1: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movsd   %xmm0, 0(%rdi)
+;   pextrq  $0, %xmm0, 0(%rdi)
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -224,7 +276,7 @@ block0(v0: i64x2, v1: i64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movsd %xmm0, (%rdi) ; trap: heap_oob
+;   pextrq $0, %xmm0, (%rdi) ; trap: heap_oob
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/float-avx.clif b/cranelift/filetests/filetests/isa/x64/float-avx.clif
index 4e29340a3..178a0ee05 100644
--- a/cranelift/filetests/filetests/isa/x64/float-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/float-avx.clif
@@ -526,7 +526,7 @@ block0(v0: f32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vcmpps  $0 %xmm0, %xmm0, %xmm2
+;   vcmpps  $0, %xmm0, %xmm0, %xmm2
 ;   vandps  %xmm0, %xmm2, %xmm4
 ;   vpxor   %xmm2, %xmm4, %xmm6
 ;   vcvttps2dq %xmm4, %xmm8
@@ -565,7 +565,7 @@ block0(v0: f64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vcmppd  $0 %xmm0, %xmm0, %xmm2
+;   vcmppd  $0, %xmm0, %xmm0, %xmm2
 ;   vandps  %xmm2, const(0), %xmm4
 ;   vminpd  %xmm0, %xmm4, %xmm6
 ;   vcvttpd2dq %xmm6, %xmm0
diff --git a/cranelift/filetests/filetests/isa/x64/insertlane-avx.clif b/cranelift/filetests/filetests/isa/x64/insertlane-avx.clif
new file mode 100644
index 000000000..784c790be
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/insertlane-avx.clif
@@ -0,0 +1,190 @@
+test compile precise-output
+set enable_simd
+target x86_64 has_avx
+
+function %insertlane_f64x2_zero(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+  v2 = insertlane v0, v1, 0
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovsd  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovsd %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_f64x2_one(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+  v2 = insertlane v0, v1, 1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovlhps %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovlhps %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_f64x2_zero_with_load(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+  v2 = load.f64 v1
+  v3 = insertlane v0, v2, 0
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovsd  0(%rdi), %xmm3
+;   vmovsd  %xmm0, %xmm3, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovsd (%rdi), %xmm3 ; trap: heap_oob
+;   vmovsd %xmm3, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i8x16_one_load(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+  v2 = load.i8 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movzbq  0(%rdi), %rdx
+;   vpinsrb $1, %xmm0, %rdx, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movzbq (%rdi), %rdx ; trap: heap_oob
+;   vpinsrb $1, %edx, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i16x8_one_load(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+  v2 = load.i16 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movzwq  0(%rdi), %rdx
+;   vpinsrw $1, %xmm0, %rdx, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movzwq (%rdi), %rdx ; trap: heap_oob
+;   vpinsrw $1, %edx, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i32x4_one_load(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+  v2 = load.i32 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpinsrd $1, %xmm0, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpinsrd $1, (%rdi), %xmm0, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i64x2_one_load(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+  v2 = load.i64 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpinsrq $1, %xmm0, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpinsrq $1, (%rdi), %xmm0, %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/insertlane.clif b/cranelift/filetests/filetests/isa/x64/insertlane.clif
index 7f2569d02..f20210c33 100644
--- a/cranelift/filetests/filetests/isa/x64/insertlane.clif
+++ b/cranelift/filetests/filetests/isa/x64/insertlane.clif
@@ -1,6 +1,6 @@
 test compile precise-output
 set enable_simd
-target x86_64 has_avx
+target x86_64
 
 function %insertlane_f64x2_zero(f64x2, f64) -> f64x2 {
 block0(v0: f64x2, v1: f64):
@@ -12,7 +12,7 @@ block0(v0: f64x2, v1: f64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vmovsd  %xmm0, %xmm1, %xmm0
+;   movsd   %xmm0, %xmm1, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -22,7 +22,7 @@ block0(v0: f64x2, v1: f64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   vmovsd %xmm1, %xmm0, %xmm0
+;   movsd %xmm1, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -37,7 +37,7 @@ block0(v0: f64x2, v1: f64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vmovlhps %xmm0, %xmm1, %xmm0
+;   movlhps %xmm0, %xmm1, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -47,7 +47,7 @@ block0(v0: f64x2, v1: f64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   vmovlhps %xmm1, %xmm0, %xmm0
+;   movlhps %xmm1, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -63,8 +63,8 @@ block0(v0: f64x2, v1: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vmovsd  0(%rdi), %xmm3
-;   vmovsd  %xmm0, %xmm3, %xmm0
+;   movsd   0(%rdi), %xmm3
+;   movsd   %xmm0, %xmm3, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -74,8 +74,116 @@ block0(v0: f64x2, v1: i64):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   vmovsd (%rdi), %xmm3 ; trap: heap_oob
-;   vmovsd %xmm3, %xmm0, %xmm0
+;   movsd (%rdi), %xmm3 ; trap: heap_oob
+;   movsd %xmm3, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i8x16_one_load(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+  v2 = load.i8 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movzbq  0(%rdi), %rdx
+;   pinsrb  $1, %xmm0, %rdx, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movzbq (%rdi), %rdx ; trap: heap_oob
+;   pinsrb $1, %edx, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i16x8_one_load(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+  v2 = load.i16 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movzwq  0(%rdi), %rdx
+;   pinsrw  $1, %xmm0, %rdx, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movzwq (%rdi), %rdx ; trap: heap_oob
+;   pinsrw $1, %edx, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i32x4_one_load(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+  v2 = load.i32 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pinsrd  $1, %xmm0, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pinsrd $1, (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_i64x2_one_load(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+  v2 = load.i64 v1
+  v3 = insertlane v0, v2, 1
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pinsrd.w $1, %xmm0, 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pinsrq $1, (%rdi), %xmm0 ; trap: heap_oob
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif b/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif
index 6f6301049..f0b4eaff2 100644
--- a/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif
+++ b/cranelift/filetests/filetests/isa/x64/shuffle-avx512.clif
@@ -42,55 +42,6 @@ block0(v0: i8x16, v1: i8x16):
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
 
-function %shuffle_out_of_bounds(i8x16, i8x16) -> i8x16 {
-block0(v0: i8x16, v1: i8x16):
-    ;; pick zero for the first lane, the rest use first lane of v0
-    ;; This should introduce two constants, one for the permutation and one to
-    ;; mask the non-zero values for lanes 1-15
-    v2 = shuffle v0, v1, 0x80000000000000000000000000000000
-    return v2
-}
-
-; VCode:
-;   pushq   %rbp
-;   movq    %rsp, %rbp
-; block0:
-;   movdqa  %xmm0, %xmm6
-;   movdqu  const(0), %xmm0
-;   movdqa  %xmm6, %xmm7
-;   vpermi2b %xmm1, %xmm7, %xmm0, %xmm0
-;   andps   %xmm0, const(1), %xmm0
-;   movq    %rbp, %rsp
-;   popq    %rbp
-;   ret
-; 
-; Disassembled:
-; block0: ; offset 0x0
-;   pushq %rbp
-;   movq %rsp, %rbp
-; block1: ; offset 0x4
-;   movdqa %xmm0, %xmm6
-;   movdqu 0x20(%rip), %xmm0
-;   movdqa %xmm6, %xmm7
-;   vpermi2b %xmm1, %xmm7, %xmm0
-;   andps 0x1f(%rip), %xmm0
-;   movq %rbp, %rsp
-;   popq %rbp
-;   retq
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, (%rax)
-;   addb %al, -1(%rax)
-
 function %f3(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
     v2 = shuffle v0, v1, [3 0 31 26 4 6 12 11 23 13 24 4 2 15 17 5]
diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
index 87c1b7710..ac0421604 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
@@ -1204,7 +1204,7 @@ block0(v0: i8x16):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vpalignr $8 %xmm0, %xmm0, %xmm2
+;   vpalignr $8, %xmm0, %xmm0, %xmm2
 ;   vpmovzxbw %xmm2, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -1316,7 +1316,7 @@ block0(v0: i8):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   uninit  %xmm2
-;   vpinsrb $0 %xmm2, %rdi, %xmm4
+;   vpinsrb $0, %xmm2, %rdi, %xmm4
 ;   uninit  %xmm6
 ;   vpxor   %xmm6, %xmm6, %xmm8
 ;   vpshufb %xmm4, %xmm8, %xmm0
@@ -1354,7 +1354,7 @@ block0(v0: f64x2):
 ;   vminpd  %xmm6, const(0), %xmm8
 ;   vroundpd $3, %xmm8, %xmm10
 ;   vaddpd  %xmm10, const(1), %xmm12
-;   vshufps $136 %xmm12, %xmm4, %xmm0
+;   vshufps $136, %xmm12, %xmm4, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
index 7da79a0cc..023d52d81 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
@@ -529,7 +529,7 @@ block0(v0: f32x4, v1: f32):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vinsertps $16 %xmm0, %xmm1, %xmm0
+;   vinsertps $16, %xmm0, %xmm1, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -579,7 +579,7 @@ block0(v0: i8x16, v1: i8):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vpinsrb $1 %xmm0, %rdi, %xmm0
+;   vpinsrb $1, %xmm0, %rdi, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -604,7 +604,7 @@ block0(v0: i16x8, v1: i16):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vpinsrw $1 %xmm0, %rdi, %xmm0
+;   vpinsrw $1, %xmm0, %rdi, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -629,7 +629,7 @@ block0(v0: i32x4, v1: i32):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vpinsrd $1 %xmm0, %rdi, %xmm0
+;   vpinsrd $1, %xmm0, %rdi, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -654,7 +654,7 @@ block0(v0: i64x2, v1: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   vpinsrq $1 %xmm0, %rdi, %xmm0
+;   vpinsrq $1, %xmm0, %rdi, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
diff --git a/cranelift/filetests/filetests/isa/x64/simd-cmp-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-cmp-avx.clif
index b46b4e722..9c69ac019 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-cmp-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-cmp-avx.clif
@@ -215,7 +215,7 @@ block0(v0: f32x4, v1: f32x4):
 ;   vminps  %xmm0, %xmm1, %xmm3
 ;   vminps  %xmm1, %xmm0, %xmm5
 ;   vorps   %xmm3, %xmm5, %xmm7
-;   vcmpps  $3 %xmm7, %xmm5, %xmm9
+;   vcmpps  $3, %xmm7, %xmm5, %xmm9
 ;   vorps   %xmm7, %xmm9, %xmm11
 ;   vpsrld  %xmm9, $10, %xmm13
 ;   vandnps %xmm13, %xmm11, %xmm0
@@ -252,7 +252,7 @@ block0(v0: f64x2, v1: f64x2):
 ;   vminpd  %xmm0, %xmm1, %xmm3
 ;   vminpd  %xmm1, %xmm0, %xmm5
 ;   vorpd   %xmm3, %xmm5, %xmm7
-;   vcmppd  $3 %xmm3, %xmm5, %xmm9
+;   vcmppd  $3, %xmm3, %xmm5, %xmm9
 ;   vorpd   %xmm7, %xmm9, %xmm11
 ;   vpsrlq  %xmm9, $13, %xmm13
 ;   vandnpd %xmm13, %xmm11, %xmm0
@@ -291,7 +291,7 @@ block0(v0: f32x4, v1: f32x4):
 ;   vxorps  %xmm3, %xmm5, %xmm7
 ;   vorps   %xmm3, %xmm7, %xmm9
 ;   vsubps  %xmm9, %xmm7, %xmm11
-;   vcmpps  $3 %xmm9, %xmm9, %xmm13
+;   vcmpps  $3, %xmm9, %xmm9, %xmm13
 ;   vpsrld  %xmm13, $10, %xmm15
 ;   vandnps %xmm15, %xmm11, %xmm0
 ;   movq    %rbp, %rsp
@@ -330,7 +330,7 @@ block0(v0: f64x2, v1: f64x2):
 ;   vxorpd  %xmm3, %xmm5, %xmm7
 ;   vorpd   %xmm3, %xmm7, %xmm9
 ;   vsubpd  %xmm9, %xmm7, %xmm11
-;   vcmppd  $3 %xmm9, %xmm9, %xmm13
+;   vcmppd  $3, %xmm9, %xmm9, %xmm13
 ;   vpsrlq  %xmm13, $13, %xmm15
 ;   vandnpd %xmm15, %xmm11, %xmm0
 ;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/runtests/simd-extractlane.clif b/cranelift/filetests/filetests/runtests/simd-extractlane.clif
index 471130f8c..2216d68c8 100644
--- a/cranelift/filetests/filetests/runtests/simd-extractlane.clif
+++ b/cranelift/filetests/filetests/runtests/simd-extractlane.clif
@@ -4,6 +4,7 @@ target aarch64
 target s390x
 set enable_simd
 target x86_64 has_sse3 has_ssse3 has_sse41
+target x86_64 has_sse3 has_ssse3 has_sse41 has_avx
 
 function %extractlane_4(i8x16) -> i8 {
 block0(v0: i8x16):
@@ -33,3 +34,69 @@ block0(v0: i64x2):
     return v1
 }
 ; run: %extractlane_1([0 4294967297]) == 4294967297
+
+function %extractlane_i8x16_through_stack(i8x16) -> i8 {
+    ss0 = explicit_slot 8
+block0(v0: i8x16):
+    v2 = stack_addr.i64 ss0
+    v3 = extractlane v0, 1
+    store v3, v2
+    v4 = load.i8 v2
+    return v4
+}
+; run: %extractlane_i8x16_through_stack([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16]) == 2
+
+function %extractlane_i16x8_through_stack(i16x8) -> i16 {
+    ss0 = explicit_slot 8
+block0(v0: i16x8):
+    v2 = stack_addr.i64 ss0
+    v3 = extractlane v0, 2
+    store v3, v2
+    v4 = load.i16 v2
+    return v4
+}
+; run: %extractlane_i16x8_through_stack([1 2 3 4 5 6 7 8]) == 3
+
+function %extractlane_i32x4_through_stack(i32x4) -> i32 {
+    ss0 = explicit_slot 8
+block0(v0: i32x4):
+    v2 = stack_addr.i64 ss0
+    v3 = extractlane v0, 3
+    store v3, v2
+    v4 = load.i32 v2
+    return v4
+}
+; run: %extractlane_i32x4_through_stack([1 2 3 4]) == 4
+
+function %extractlane_i64x2_through_stack(i64x2) -> i64 {
+    ss0 = explicit_slot 8
+block0(v0: i64x2):
+    v2 = stack_addr.i64 ss0
+    v3 = extractlane v0, 0
+    store v3, v2
+    v4 = load.i64 v2
+    return v4
+}
+; run: %extractlane_i64x2_through_stack([1 2]) == 1
+
+function %extractlane_f32x4_through_stack(f32x4) -> f32 {
+    ss0 = explicit_slot 8
+block0(v0: f32x4):
+    v2 = stack_addr.i64 ss0
+    v3 = extractlane v0, 3
+    store v3, v2
+    v4 = load.f32 v2
+    return v4
+}
+; run: %extractlane_f32x4_through_stack([0x1.0 0x2.0 0x3.0 0x4.0]) == 0x4.0
+
+function %extractlane_f64x2_through_stack(f64x2) -> f64 {
+    ss0 = explicit_slot 8
+block0(v0: f64x2):
+    v2 = stack_addr.i64 ss0
+    v3 = extractlane v0, 0
+    store v3, v2
+    v4 = load.f64 v2
+    return v4
+}
+; run: %extractlane_f64x2_through_stack([0x1.0 0x2.0]) == 0x1.0
diff --git a/cranelift/filetests/filetests/runtests/simd-insertlane.clif b/cranelift/filetests/filetests/runtests/simd-insertlane.clif
index 6e0182b1c..e9a944ab4 100644
--- a/cranelift/filetests/filetests/runtests/simd-insertlane.clif
+++ b/cranelift/filetests/filetests/runtests/simd-insertlane.clif
@@ -47,3 +47,91 @@ block0(v0: f64x2, v1: f64):
     return v2
 }
 ; run: %insertlane_1_in_f64x2([0x1.0 0x2.0], 0x3.0) == [0x1.0 0x3.0]
+
+function %insertlane_i8x16_through_stack(i8x16, i8) -> i8x16 {
+    ss0 = explicit_slot 8
+block0(v0: i8x16, v1: i8):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.i8 v2
+    v4 = insertlane v0, v3, 1
+    return v4
+}
+; run: %insertlane_i8x16_through_stack([1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], 2) == [1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1]
+
+function %insertlane_i16x8_through_stack(i16x8, i16) -> i16x8 {
+    ss0 = explicit_slot 8
+block0(v0: i16x8, v1: i16):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.i16 v2
+    v4 = insertlane v0, v3, 2
+    return v4
+}
+; run: %insertlane_i16x8_through_stack([1 1 1 1 1 1 1 1], 2) == [1 1 2 1 1 1 1 1]
+
+function %insertlane_i32x4_through_stack(i32x4, i32) -> i32x4 {
+    ss0 = explicit_slot 8
+block0(v0: i32x4, v1: i32):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.i32 v2
+    v4 = insertlane v0, v3, 3
+    return v4
+}
+; run: %insertlane_i32x4_through_stack([1 1 1 1], 2) == [1 1 1 2]
+
+function %insertlane_i64x2_through_stack(i64x2, i64) -> i64x2 {
+    ss0 = explicit_slot 8
+block0(v0: i64x2, v1: i64):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.i64 v2
+    v4 = insertlane v0, v3, 0
+    return v4
+}
+; run: %insertlane_i64x2_through_stack([1 1], 2) == [2 1]
+
+function %insertlane_f32x4_through_stack(f32x4, f32) -> f32x4 {
+    ss0 = explicit_slot 8
+block0(v0: f32x4, v1: f32):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.f32 v2
+    v4 = insertlane v0, v3, 3
+    return v4
+}
+; run: %insertlane_f32x4_through_stack([0x1.0 0x1.0 0x1.0 0x1.0], 0x2.0) == [0x1.0 0x1.0 0x1.0 0x2.0]
+
+function %insertlane_f32x4_through_stack2(f32x4, f32) -> f32x4 {
+    ss0 = explicit_slot 8
+block0(v0: f32x4, v1: f32):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.f32 v2
+    v4 = insertlane v0, v3, 0
+    return v4
+}
+; run: %insertlane_f32x4_through_stack2([0x1.0 0x1.0 0x1.0 0x1.0], 0x2.0) == [0x2.0 0x1.0 0x1.0 0x1.0]
+
+function %insertlane_f64x2_through_stack(f64x2, f64) -> f64x2 {
+    ss0 = explicit_slot 8
+block0(v0: f64x2, v1: f64):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.f64 v2
+    v4 = insertlane v0, v3, 0
+    return v4
+}
+; run: %insertlane_f64x2_through_stack([0x1.0 0x1.0], 0x2.0) == [0x2.0 0x1.0]
+
+function %insertlane_f64x2_through_stack2(f64x2, f64) -> f64x2 {
+    ss0 = explicit_slot 8
+block0(v0: f64x2, v1: f64):
+    v2 = stack_addr.i64 ss0
+    store v1, v2
+    v3 = load.f64 v2
+    v4 = insertlane v0, v3, 1
+    return v4
+}
+; run: %insertlane_f64x2_through_stack2([0x1.0 0x1.0], 0x2.0) == [0x1.0 0x2.0]
diff --git a/cranelift/filetests/filetests/runtests/simd-shuffle.clif b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
index 6056fb3dd..f236a0205 100644
--- a/cranelift/filetests/filetests/runtests/simd-shuffle.clif
+++ b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
@@ -14,13 +14,6 @@ block0(v0: i8x16, v1: i8x16):
 }
 ; run: %shuffle_i8x16([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [4 1 32 27 5 7 13 12 24 14 25 5 3 16 18 6]
 
-function %shuffle_zeros(i8x16, i8x16) -> i8x16 {
-block0(v0: i8x16, v1: i8x16):
-    v2 = shuffle v0, v1, [3 0 32 255 4 6 12 11 23 13 24 4 2 97 17 5]
-    return v2
-}
-; run: %shuffle_zeros([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [4 1 0 0 5 7 13 12 24 14 25 5 3 0 18 6]
-
 function %shuffle1(i8x16) -> i8x16 {
 block0(v0: i8x16):
     v1 = shuffle v0, v0, [8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23]
diff --git a/cranelift/filetests/filetests/verifier/type_check.clif b/cranelift/filetests/filetests/verifier/type_check.clif
index 0c6710b61..ea5ed07ed 100644
--- a/cranelift/filetests/filetests/verifier/type_check.clif
+++ b/cranelift/filetests/filetests/verifier/type_check.clif
@@ -123,3 +123,9 @@ block0:
     v1 = ireduce.i64 v0 ; error: input i32 must be larger than output i64
     return
 }
+
+function %bad_shuffle(i8x16, i8x16) {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 32] ; error: shuffle immediate index 32 is larger than the maximum 31
+    return
+}
diff --git a/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat b/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat
index 8d7eca8c0..8d80f59da 100644
--- a/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat
+++ b/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat
@@ -44,7 +44,7 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   vcmpps  $0 %xmm0, %xmm0, %xmm3
+;;   vcmpps  $0, %xmm0, %xmm0, %xmm3
 ;;   vandps  %xmm0, %xmm3, %xmm5
 ;;   vpxor   %xmm3, %xmm5, %xmm7
 ;;   vcvttps2dq %xmm5, %xmm9
@@ -71,7 +71,7 @@
 ;;   vcvtdq2ps %xmm11, %xmm13
 ;;   vcvttps2dq %xmm7, %xmm15
 ;;   vsubps  %xmm7, %xmm13, %xmm1
-;;   vcmpps  $2 %xmm13, %xmm1, %xmm3
+;;   vcmpps  $2, %xmm13, %xmm1, %xmm3
 ;;   vcvttps2dq %xmm1, %xmm5
 ;;   vpxor   %xmm5, %xmm3, %xmm7
 ;;   uninit  %xmm9
@@ -90,7 +90,7 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   vcmppd  $0 %xmm0, %xmm0, %xmm3
+;;   vcmppd  $0, %xmm0, %xmm0, %xmm3
 ;;   vandps  %xmm3, const(0), %xmm5
 ;;   vminpd  %xmm0, %xmm5, %xmm7
 ;;   vcvttpd2dq %xmm7, %xmm0
@@ -112,7 +112,7 @@
 ;;   vminpd  %xmm7, const(0), %xmm9
 ;;   vroundpd $3, %xmm9, %xmm11
 ;;   vaddpd  %xmm11, const(1), %xmm13
-;;   vshufps $136 %xmm13, %xmm5, %xmm0
+;;   vshufps $136, %xmm13, %xmm5, %xmm0
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -128,9 +128,9 @@
 ;;   vpmovsxbw %xmm0, %xmm10
 ;;   vpmovsxbw %xmm1, %xmm12
 ;;   vpmullw %xmm10, %xmm12, %xmm14
-;;   vpalignr $8 %xmm0, %xmm0, %xmm8
+;;   vpalignr $8, %xmm0, %xmm0, %xmm8
 ;;   vpmovsxbw %xmm8, %xmm10
-;;   vpalignr $8 %xmm1, %xmm1, %xmm12
+;;   vpalignr $8, %xmm1, %xmm1, %xmm12
 ;;   vpmovsxbw %xmm12, %xmm15
 ;;   vpmullw %xmm10, %xmm15, %xmm0
 ;;   vphaddw %xmm14, %xmm0, %xmm0
@@ -149,9 +149,9 @@
 ;;   vpmovsxbw %xmm0, %xmm13
 ;;   vpmovsxbw %xmm1, %xmm15
 ;;   vpmullw %xmm13, %xmm15, %xmm3
-;;   vpalignr $8 %xmm0, %xmm0, %xmm11
+;;   vpalignr $8, %xmm0, %xmm0, %xmm11
 ;;   vpmovsxbw %xmm11, %xmm13
-;;   vpalignr $8 %xmm1, %xmm1, %xmm15
+;;   vpalignr $8, %xmm1, %xmm1, %xmm15
 ;;   vpmovsxbw %xmm15, %xmm1
 ;;   vpmullw %xmm13, %xmm1, %xmm4
 ;;   vphaddw %xmm3, %xmm4, %xmm15