Repository: bytecodealliance/wasmtime
Pull request #5931 (2023-03-04 22:33:17)

x64: Add more support for more AVX instructions
One aspect of AVX that I have just recently become aware of is that there's apparently a performance penalty associated with mixing AVX and SSE intructions. One reason for this is that AVX instructions which operate over 128-bit values always zero the higher-than-128-bits of each register operate on. SSE instructions, however, don't do this. This means that false dependencies can be created between instructions because SSE instructions look like they're intentionally preserving higher bits where AVX instructions intentionally zero them. According to [this stackoverflow question](https://stackoverflow.com/questions/41303780/why-is-this-sse-code-6-times-slower-without-vzeroupper-on-skylake/41349852#41349852) the processor also tracks whether an instruction has been executed and there's a "scary red line" for mixing AVX/SSE.

On the local meshoptimizer benchmark this PR doesn't actually have any effect on the generate code's performance, or not one that I can measure. In that sense this is more of a hygiene thing than anything else.

Specifically the changes here were to refactor many ISLE helpers that were generating instructions with `SseOpcode.XXX` manually to instead using the instruction helpers which will use the AVX variant if enabled. Additionally more AVX instructions were added for moving data to/from memory and such.

I don't think this 100% handles all the SSE instructions cranelift can generate when AVX is enabled, but it at least raises the bar further and removes a bunch of cases of SSE-generated instructions when AVX is enabled.

alexcrichton: Oh good points! Shame on me for not actually reading all the way through on these bits... 

So hypothetically if the host uses ymm registers in its own code, that might cause stalls but given that the stall requires hopping between the guest and the host it probably isn't really going to affect much?

Otherwise though locally I can't measure a difference before/after this PR, so the main motivation at this point is to copy what v8 does.

Patch:
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 260796cc5..99cfeda06 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -227,13 +227,6 @@
                         (src2 XmmMem)
                         (dst WritableXmm))
 
-       ;; XMM (scalar or vector) production of a constant value by operating
-       ;; on a register with itself.
-       ;;
-       ;; Used to produce all zeros with xor or all one with a comparison.
-       (XmmConstOp (op SseOpcode)
-                   (dst WritableXmm))
-
        ;; XMM (scalar or vector) blend op. The mask is used to blend between
        ;; src1 and src2. This differs from a use of `XmmRmR` as the mask is
        ;; implicitly in register xmm0; this special case exists to allow us to
@@ -294,6 +287,12 @@
                           (dst WritableXmm)
                           (imm u8))
 
+       ;; XMM (scalar or vector) unary op (from xmm to reg/mem) using the
+       ;; VEX prefix
+       (XmmMovRMVex (op AvxOpcode)
+                    (src Reg)
+                    (dst SyntheticAmode))
+
        ;; XMM (scalar or vector) binary op that relies on the EVEX
        ;; prefix. Takes two inputs.
        (XmmRmREvex (op Avx512Opcode)
@@ -863,6 +862,12 @@
             Xorpd
             Phaddw
             Phaddd
+            Punpckhdq
+            Punpckldq
+            Punpckhqdq
+            Punpcklqdq
+            Pshuflw
+            Pshufhw
           ))
 
 (type CmpOpcode extern
@@ -1347,6 +1352,18 @@
             Vcvttps2dq
             Vphaddw
             Vphaddd
+            Vpunpckhdq
+            Vpunpckldq
+            Vpunpckhqdq
+            Vpunpcklqdq
+            Vpshuflw
+            Vpshufhw
+            Vpshufd
+            Vmovss
+            Vmovsd
+            Vmovups
+            Vmovupd
+            Vmovdqu
           ))
 
 (type Avx512Opcode extern
@@ -1714,21 +1731,27 @@
 (decl sinkable_load (SinkableLoad) Value)
 (extern extractor sinkable_load sinkable_load)
 
-;; Sink a `SinkableLoad` into a `RegMemImm.Mem`.
+;; Sink a `SinkableLoad` into a `SyntheticAmode`.
 ;;
 ;; This is a side-effectful operation that notifies the context that the
 ;; instruction that produced the `SinkableImm` has been sunk into another
 ;; instruction, and no longer needs to be lowered.
-(decl sink_load (SinkableLoad) RegMem)
+(decl sink_load (SinkableLoad) SyntheticAmode)
 (extern constructor sink_load sink_load)
 
 (decl sink_load_to_gpr_mem_imm (SinkableLoad) GprMemImm)
 (rule (sink_load_to_gpr_mem_imm load)
-      (gpr_mem_imm_new (sink_load load)))
+      (gpr_mem_imm_new load))
 
 (decl sink_load_to_xmm_mem (SinkableLoad) XmmMem)
 (rule (sink_load_to_xmm_mem load)
-      (reg_mem_to_xmm_mem (sink_load load)))
+      (reg_mem_to_xmm_mem load))
+
+(decl sink_load_to_reg_mem (SinkableLoad) RegMem)
+(rule (sink_load_to_reg_mem load) (RegMem.Mem load))
+
+(decl sink_load_to_reg_mem_imm (SinkableLoad) RegMemImm)
+(rule (sink_load_to_reg_mem_imm load) (RegMemImm.Mem load))
 
 ;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1787,28 +1810,13 @@
 (rule (vec_int_type (multi_lane 32 4)) $I32X4)
 (rule (vec_int_type (multi_lane 64 2)) $I64X2)
 
-;; Determine the appropriate operation for xor-ing vectors of the specified type
-(decl sse_xor_op (Type) SseOpcode)
-(rule 1 (sse_xor_op $F32X4) (SseOpcode.Xorps))
-(rule 1 (sse_xor_op $F64X2) (SseOpcode.Xorpd))
-(rule 1 (sse_xor_op $F32) (SseOpcode.Xorps))
-(rule 1 (sse_xor_op $F64) (SseOpcode.Xorpd))
-
-;; Priority 0 because multi_lane overlaps with the previous two explicit type
-;; patterns.
-(rule 0 (sse_xor_op (multi_lane _bits _lanes)) (SseOpcode.Pxor))
-
-(decl avx_xor_op (Type) AvxOpcode)
-(rule 1 (avx_xor_op $F32X4) (AvxOpcode.Vxorps))
-(rule 1 (avx_xor_op $F64X2) (AvxOpcode.Vxorpd))
-(rule 0 (avx_xor_op (multi_lane _bits _lanes)) (AvxOpcode.Vpxor))
-
 ;; Performs an xor operation of the two operands specified.
-(decl sse_xor (Type Xmm XmmMem) Xmm)
-(rule 0 (sse_xor ty x y) (xmm_rm_r (sse_xor_op ty) x y))
-(rule 1 (sse_xor ty @ (multi_lane _ _) x y)
-        (if-let $true (has_avx))
-        (xmm_rmir_vex (avx_xor_op ty) x y))
+(decl x64_xor_vector (Type Xmm XmmMem) Xmm)
+(rule 1 (x64_xor_vector $F32 x y) (x64_xorps x y))
+(rule 1 (x64_xor_vector $F64 x y) (x64_xorpd x y))
+(rule 1 (x64_xor_vector $F32X4 x y) (x64_xorps x y))
+(rule 1 (x64_xor_vector $F64X2 x y) (x64_xorpd x y))
+(rule 0 (x64_xor_vector (multi_lane _ _) x y) (x64_pxor x y))
 
 ;; Generates a register value which has an all-ones pattern.
 ;;
@@ -1821,9 +1829,8 @@
 ;; we're guaranteeed that everything is equal to itself.
 (decl vector_all_ones () Xmm)
 (rule (vector_all_ones)
-      (let ((r WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmConstOp (SseOpcode.Pcmpeqd) r))))
-        r))
+      (let ((tmp Xmm (xmm_uninit_value)))
+        (x64_pcmpeqd tmp tmp)))
 
 ;; Helper for creating XmmUninitializedValue instructions.
 (decl xmm_uninit_value () Xmm)
@@ -1879,19 +1886,19 @@
         dst))
 
 (rule 2 (x64_load $F32 addr _ext_kind)
-      (xmm_unary_rm_r_unaligned (SseOpcode.Movss) addr))
+      (x64_movss_load addr))
 
 (rule 2 (x64_load $F64 addr _ext_kind)
-      (xmm_unary_rm_r_unaligned (SseOpcode.Movsd) addr))
+      (x64_movsd_load addr))
 
 (rule 2 (x64_load $F32X4 addr _ext_kind)
-      (xmm_unary_rm_r_unaligned (SseOpcode.Movups) addr))
+      (x64_movups_load addr))
 
 (rule 2 (x64_load $F64X2 addr _ext_kind)
-      (xmm_unary_rm_r_unaligned (SseOpcode.Movupd) addr))
+      (x64_movupd_load addr))
 
 (rule 0 (x64_load (multi_lane _bits _lanes) addr _ext_kind)
-      (xmm_unary_rm_r_unaligned (SseOpcode.Movdqu) addr))
+      (x64_movdqu_load addr))
 
 (decl x64_mov (Amode) Reg)
 (rule (x64_mov addr)
@@ -1911,29 +1918,79 @@
             (_ Unit (emit (MInst.MovsxRmR mode src dst))))
         dst))
 
-(decl x64_movss_load (XmmMem) Xmm)
+(decl x64_movss_load (SyntheticAmode) Xmm)
 (rule (x64_movss_load from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Movss) from))
+(rule 1 (x64_movss_load from)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vmovss) from))
 
-(decl x64_movsd_load (XmmMem) Xmm)
+(decl x64_movss_store (SyntheticAmode Xmm) SideEffectNoResult)
+(rule (x64_movss_store addr data)
+      (xmm_movrm (SseOpcode.Movss) addr data))
+(rule 1 (x64_movss_store addr data)
+        (if-let $true (has_avx))
+        (xmm_movrm_vex (AvxOpcode.Vmovss) addr data))
+
+(decl x64_movsd_load (SyntheticAmode) Xmm)
 (rule (x64_movsd_load from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Movsd) from))
+(rule 1 (x64_movsd_load from)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vmovsd) from))
+
+(decl x64_movsd_store (SyntheticAmode Xmm) SideEffectNoResult)
+(rule (x64_movsd_store addr data)
+      (xmm_movrm (SseOpcode.Movsd) addr data))
+(rule 1 (x64_movsd_store addr data)
+        (if-let $true (has_avx))
+        (xmm_movrm_vex (AvxOpcode.Vmovsd) addr data))
 
-(decl x64_movups (XmmMem) Xmm)
-(rule (x64_movups from)
+(decl x64_movups_load (SyntheticAmode) Xmm)
+(rule (x64_movups_load from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Movups) from))
+(rule 1 (x64_movups_load from)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vmovups) from))
+
+(decl x64_movups_store (SyntheticAmode Xmm) SideEffectNoResult)
+(rule (x64_movups_store addr data)
+      (xmm_movrm (SseOpcode.Movups) addr data))
+(rule 1 (x64_movups_store addr data)
+        (if-let $true (has_avx))
+        (xmm_movrm_vex (AvxOpcode.Vmovups) addr data))
 
-(decl x64_movupd (XmmMem) Xmm)
-(rule (x64_movupd from)
+(decl x64_movupd_load (SyntheticAmode) Xmm)
+(rule (x64_movupd_load from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Movupd) from))
+(rule 1 (x64_movupd_load from)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vmovupd) from))
+
+(decl x64_movupd_store (SyntheticAmode Xmm) SideEffectNoResult)
+(rule (x64_movupd_store addr data)
+      (xmm_movrm (SseOpcode.Movupd) addr data))
+(rule 1 (x64_movupd_store addr data)
+        (if-let $true (has_avx))
+        (xmm_movrm_vex (AvxOpcode.Vmovupd) addr data))
 
 (decl x64_movd (Xmm) Gpr)
 (rule (x64_movd from)
       (xmm_to_gpr (SseOpcode.Movd) from (OperandSize.Size32)))
 
-(decl x64_movdqu (XmmMem) Xmm)
-(rule (x64_movdqu from)
+(decl x64_movdqu_load (XmmMem) Xmm)
+(rule (x64_movdqu_load from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Movdqu) from))
+(rule 1 (x64_movdqu_load from)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vmovdqu) from))
+
+(decl x64_movdqu_store (SyntheticAmode Xmm) SideEffectNoResult)
+(rule (x64_movdqu_store addr data)
+      (xmm_movrm (SseOpcode.Movdqu) addr data))
+(rule 1 (x64_movdqu_store addr data)
+        (if-let $true (has_avx))
+        (xmm_movrm_vex (AvxOpcode.Vmovdqu) addr data))
 
 (decl x64_pmovsxbw (XmmMem) Xmm)
 (rule (x64_pmovsxbw from)
@@ -1982,10 +2039,14 @@
       (let ((size OperandSize (raw_operand_size_of_type ty)))
         (SideEffectNoResult.Inst (MInst.MovRM size data addr))))
 
-(decl x64_xmm_movrm (SseOpcode SyntheticAmode Xmm) SideEffectNoResult)
-(rule (x64_xmm_movrm op addr data)
+(decl xmm_movrm (SseOpcode SyntheticAmode Xmm) SideEffectNoResult)
+(rule (xmm_movrm op addr data)
       (SideEffectNoResult.Inst (MInst.XmmMovRM op data addr)))
 
+(decl xmm_movrm_vex (AvxOpcode SyntheticAmode Xmm) SideEffectNoResult)
+(rule (xmm_movrm_vex op addr data)
+      (SideEffectNoResult.Inst (MInst.XmmMovRMVex op data addr)))
+
 ;; Load a constant into an XMM register.
 (decl x64_xmm_load_const (Type VCodeConstant) Xmm)
 (rule (x64_xmm_load_const ty const)
@@ -2180,26 +2241,19 @@
       (xmm_to_reg (xmm_zero ty)))
 
 ;; Special case for `f32` zero immediates
-(rule 2 (imm ty @ $F32 (u64_zero))
-      (let ((wr WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmConstOp (SseOpcode.Xorps) wr))))
-        (xmm_to_reg wr)))
+(rule 2 (imm ty @ $F32 (u64_zero)) (xmm_zero ty))
 
 ;; TODO: use cmpeqps for all 1s
 
 ;; Special case for `f64` zero immediates to use `xorpd`.
-(rule 2 (imm ty @ $F64 (u64_zero))
-      (let ((wr WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmConstOp (SseOpcode.Xorpd) wr))))
-        (xmm_to_reg wr)))
+(rule 2 (imm ty @ $F64 (u64_zero)) (xmm_zero ty))
 
 ;; TODO: use cmpeqpd for all 1s
 
 (decl xmm_zero (Type) Xmm)
 (rule (xmm_zero ty)
-      (let ((wr WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmConstOp (sse_xor_op ty) wr))))
-        wr))
+      (let ((tmp Xmm (xmm_uninit_value)))
+        (x64_xor_vector ty tmp tmp)))
 
 ;; Helper for creating `MInst.ShiftR` instructions.
 (decl shift_r (Type ShiftKind Gpr Imm8Gpr) Gpr)
@@ -2729,6 +2783,38 @@
       (if-let $true (has_avx))
       (xmm_rmir_vex (AvxOpcode.Vpunpcklwd) src1 src2))
 
+;; Helper for creating `punpckldq` instructions.
+(decl x64_punpckldq (Xmm XmmMem) Xmm)
+(rule 0 (x64_punpckldq src1 src2)
+      (xmm_rm_r (SseOpcode.Punpckldq) src1 src2))
+(rule 1 (x64_punpckldq src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vpunpckldq) src1 src2))
+
+;; Helper for creating `punpckhdq` instructions.
+(decl x64_punpckhdq (Xmm XmmMem) Xmm)
+(rule 0 (x64_punpckhdq src1 src2)
+      (xmm_rm_r (SseOpcode.Punpckhdq) src1 src2))
+(rule 1 (x64_punpckhdq src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vpunpckhdq) src1 src2))
+
+;; Helper for creating `punpcklqdq` instructions.
+(decl x64_punpcklqdq (Xmm XmmMem) Xmm)
+(rule 0 (x64_punpcklqdq src1 src2)
+      (xmm_rm_r (SseOpcode.Punpcklqdq) src1 src2))
+(rule 1 (x64_punpcklqdq src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vpunpcklqdq) src1 src2))
+
+;; Helper for creating `punpckhqdq` instructions.
+(decl x64_punpckhqdq (Xmm XmmMem) Xmm)
+(rule 0 (x64_punpckhqdq src1 src2)
+      (xmm_rm_r (SseOpcode.Punpckhqdq) src1 src2))
+(rule 1 (x64_punpckhqdq src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vpunpckhqdq) src1 src2))
+
 ;; Helper for creating `unpcklps` instructions.
 (decl x64_unpcklps (Xmm XmmMem) Xmm)
 (rule 0 (x64_unpcklps src1 src2)
@@ -2947,10 +3033,20 @@
       (if-let $true (has_avx))
       (xmm_rmr_blend_vex (AvxOpcode.Vpblendvb) src1 src2 mask))
 
-;; Helper for creating `movsd` instructions.
-(decl x64_movsd_regmove (Xmm XmmMem) Xmm)
+;; Helper for creating a `movsd` instruction which creates a new vector
+;; register where the upper 64-bits are from the first operand and the low
+;; 64-bits are from the second operand.
+;;
+;; Note that the second argument here is specifically `Xmm` instead of `XmmMem`
+;; because there is no encoding of a 3-operand form of `movsd` and otherwise
+;; when used as a load instruction it wipes out the entire destination register
+;; which defeats the purpose of this being a 2-operand instruction.
+(decl x64_movsd_regmove (Xmm Xmm) Xmm)
 (rule (x64_movsd_regmove src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Movsd) src1 src2))
+(rule 1 (x64_movsd_regmove src1 src2)
+        (if-let $true (has_avx))
+        (xmm_rmir_vex (AvxOpcode.Vmovsd) src1 src2))
 
 ;; Helper for creating `movlhps` instructions.
 (decl x64_movlhps (Xmm XmmMem) Xmm)
@@ -3275,6 +3371,9 @@
 (decl x64_pshufd (XmmMem u8) Xmm)
 (rule (x64_pshufd src imm)
       (xmm_unary_rm_r_imm (SseOpcode.Pshufd) src imm))
+(rule 1 (x64_pshufd src imm)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_imm_vex (AvxOpcode.Vpshufd) src imm))
 
 ;; Helper for creating `pshufb` instructions.
 (decl x64_pshufb (Xmm XmmMem) Xmm)
@@ -3284,6 +3383,22 @@
       (if-let $true (has_avx))
       (xmm_rmir_vex (AvxOpcode.Vpshufb) src1 src2))
 
+;; Helper for creating `pshuflw` instructions.
+(decl x64_pshuflw (XmmMem u8) Xmm)
+(rule (x64_pshuflw src imm)
+      (xmm_unary_rm_r_imm (SseOpcode.Pshuflw) src imm))
+(rule 1 (x64_pshuflw src imm)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_imm_vex (AvxOpcode.Vpshuflw) src imm))
+
+;; Helper for creating `pshufhw` instructions.
+(decl x64_pshufhw (XmmMem u8) Xmm)
+(rule (x64_pshufhw src imm)
+      (xmm_unary_rm_r_imm (SseOpcode.Pshufhw) src imm))
+(rule 1 (x64_pshufhw src imm)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_imm_vex (AvxOpcode.Vpshufhw) src imm))
+
 ;; Helper for creating `shufps` instructions.
 (decl x64_shufps (Xmm XmmMem u8) Xmm)
 (rule 0 (x64_shufps src1 src2 byte)
@@ -4502,9 +4617,11 @@
 (convert IntCC CC intcc_to_cc)
 (convert AtomicRmwOp MachAtomicRmwOp atomic_rmw_op_to_mach_atomic_rmw_op)
 
-(convert SinkableLoad RegMem sink_load)
+(convert SinkableLoad RegMem sink_load_to_reg_mem)
+(convert SinkableLoad RegMemImm sink_load_to_reg_mem_imm)
 (convert SinkableLoad GprMemImm sink_load_to_gpr_mem_imm)
 (convert SinkableLoad XmmMem sink_load_to_xmm_mem)
+(convert SinkableLoad SyntheticAmode sink_load)
 
 (decl reg_to_xmm_mem (Reg) XmmMem)
 (rule (reg_to_xmm_mem r)
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index ca6e40ce5..6ae024c01 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -1117,6 +1117,12 @@ pub enum SseOpcode {
     Xorpd,
     Phaddw,
     Phaddd,
+    Punpckhdq,
+    Punpckldq,
+    Punpckhqdq,
+    Punpcklqdq,
+    Pshuflw,
+    Pshufhw,
 }
 
 impl SseOpcode {
@@ -1256,7 +1262,13 @@ impl SseOpcode {
             | SseOpcode::Subpd
             | SseOpcode::Subsd
             | SseOpcode::Ucomisd
-            | SseOpcode::Xorpd => SSE2,
+            | SseOpcode::Xorpd
+            | SseOpcode::Punpckldq
+            | SseOpcode::Punpckhdq
+            | SseOpcode::Punpcklqdq
+            | SseOpcode::Punpckhqdq
+            | SseOpcode::Pshuflw
+            | SseOpcode::Pshufhw => SSE2,
 
             SseOpcode::Pabsb
             | SseOpcode::Pabsw
@@ -1501,6 +1513,12 @@ impl fmt::Debug for SseOpcode {
             SseOpcode::Xorpd => "xorpd",
             SseOpcode::Phaddw => "phaddw",
             SseOpcode::Phaddd => "phaddd",
+            SseOpcode::Punpckldq => "punpckldq",
+            SseOpcode::Punpckhdq => "punpckhdq",
+            SseOpcode::Punpcklqdq => "punpcklqdq",
+            SseOpcode::Punpckhqdq => "punpckhqdq",
+            SseOpcode::Pshuflw => "pshuflw",
+            SseOpcode::Pshufhw => "pshufhw",
         };
         write!(fmt, "{}", name)
     }
@@ -1669,7 +1687,19 @@ impl AvxOpcode {
             | AvxOpcode::Vcvttpd2dq
             | AvxOpcode::Vcvttps2dq
             | AvxOpcode::Vphaddw
-            | AvxOpcode::Vphaddd => {
+            | AvxOpcode::Vphaddd
+            | AvxOpcode::Vpunpckldq
+            | AvxOpcode::Vpunpckhdq
+            | AvxOpcode::Vpunpcklqdq
+            | AvxOpcode::Vpunpckhqdq
+            | AvxOpcode::Vpshuflw
+            | AvxOpcode::Vpshufhw
+            | AvxOpcode::Vpshufd
+            | AvxOpcode::Vmovss
+            | AvxOpcode::Vmovsd
+            | AvxOpcode::Vmovups
+            | AvxOpcode::Vmovupd
+            | AvxOpcode::Vmovdqu => {
                 smallvec![InstructionSet::AVX]
             }
         }
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index b92e1ecd9..9fb2c1599 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1789,6 +1789,8 @@ pub(crate) fn emit(
                 SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3),
                 SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3),
                 SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2),
+                SseOpcode::Pshuflw => (LegacyPrefixes::_F2, 0x0F70, 2),
+                SseOpcode::Pshufhw => (LegacyPrefixes::_F3, 0x0F70, 2),
                 _ => unimplemented!("Opcode {:?} not implemented", op),
             };
             match src {
@@ -1946,6 +1948,10 @@ pub(crate) fn emit(
                 SseOpcode::Punpckhwd => (LegacyPrefixes::_66, 0x0F69, 2),
                 SseOpcode::Punpcklbw => (LegacyPrefixes::_66, 0x0F60, 2),
                 SseOpcode::Punpcklwd => (LegacyPrefixes::_66, 0x0F61, 2),
+                SseOpcode::Punpckldq => (LegacyPrefixes::_66, 0x0F62, 2),
+                SseOpcode::Punpcklqdq => (LegacyPrefixes::_66, 0x0F6C, 2),
+                SseOpcode::Punpckhdq => (LegacyPrefixes::_66, 0x0F6A, 2),
+                SseOpcode::Punpckhqdq => (LegacyPrefixes::_66, 0x0F6D, 2),
                 SseOpcode::Pxor => (LegacyPrefixes::_66, 0x0FEF, 2),
                 SseOpcode::Subps => (LegacyPrefixes::None, 0x0F5C, 2),
                 SseOpcode::Subpd => (LegacyPrefixes::_66, 0x0F5C, 2),
@@ -1970,22 +1976,6 @@ pub(crate) fn emit(
             }
         }
 
-        Inst::XmmConstOp { op, dst } => {
-            let dst = allocs.next(dst.to_reg().to_reg());
-            emit(
-                &Inst::XmmRmR {
-                    op: *op,
-                    dst: Writable::from_reg(Xmm::new(dst).unwrap()),
-                    src1: Xmm::new(dst).unwrap(),
-                    src2: Xmm::new(dst).unwrap().into(),
-                },
-                allocs,
-                sink,
-                info,
-                state,
-            );
-        }
-
         Inst::XmmRmRBlend {
             op,
             src1,
@@ -2171,6 +2161,11 @@ pub(crate) fn emit(
                 AvxOpcode::Vmaxsd => (LP::_F2, OM::_0F, 0x5F),
                 AvxOpcode::Vphaddw => (LP::_66, OM::_0F38, 0x01),
                 AvxOpcode::Vphaddd => (LP::_66, OM::_0F38, 0x02),
+                AvxOpcode::Vpunpckldq => (LP::_66, OM::_0F, 0x62),
+                AvxOpcode::Vpunpckhdq => (LP::_66, OM::_0F, 0x6A),
+                AvxOpcode::Vpunpcklqdq => (LP::_66, OM::_0F, 0x6C),
+                AvxOpcode::Vpunpckhqdq => (LP::_66, OM::_0F, 0x6D),
+                AvxOpcode::Vmovsd => (LP::_F2, OM::_0F, 0x10),
                 _ => panic!("unexpected rmir vex opcode {op:?}"),
             };
             VexInstruction::new()
@@ -2375,6 +2370,23 @@ pub(crate) fn emit(
                 AvxOpcode::Vcvtps2pd => (LegacyPrefixes::None, OpcodeMap::_0F, 0x5A),
                 AvxOpcode::Vcvttpd2dq => (LegacyPrefixes::_66, OpcodeMap::_0F, 0xE6),
                 AvxOpcode::Vcvttps2dq => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x5B),
+                AvxOpcode::Vmovdqu => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x6F),
+                AvxOpcode::Vmovups => (LegacyPrefixes::None, OpcodeMap::_0F, 0x10),
+                AvxOpcode::Vmovupd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x10),
+
+                // Note that for `vmov{s,d}` the `inst.isle` rules should
+                // statically ensure that only `Amode` operands are used here.
+                // Otherwise the other encodings of `vmovss` are more like
+                // 2-operand instructions which this unary encoding does not
+                // have.
+                AvxOpcode::Vmovss => match &src {
+                    RegisterOrAmode::Amode(_) => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x10),
+                    _ => unreachable!(),
+                },
+                AvxOpcode::Vmovsd => match &src {
+                    RegisterOrAmode::Amode(_) => (LegacyPrefixes::_F2, OpcodeMap::_0F, 0x10),
+                    _ => unreachable!(),
+                },
                 _ => panic!("unexpected rmr_imm_vex opcode {op:?}"),
             };
 
@@ -2400,6 +2412,9 @@ pub(crate) fn emit(
             let (prefix, map, opcode) = match op {
                 AvxOpcode::Vroundps => (LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x08),
                 AvxOpcode::Vroundpd => (LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x09),
+                AvxOpcode::Vpshuflw => (LegacyPrefixes::_F2, OpcodeMap::_0F, 0x70),
+                AvxOpcode::Vpshufhw => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x70),
+                AvxOpcode::Vpshufd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x70),
                 _ => panic!("unexpected rmr_imm_vex opcode {op:?}"),
             };
 
@@ -2414,6 +2429,28 @@ pub(crate) fn emit(
                 .encode(sink);
         }
 
+        Inst::XmmMovRMVex { op, src, dst } => {
+            let src = allocs.next(*src);
+            let dst = dst.with_allocs(allocs).finalize(state, sink);
+
+            let (prefix, map, opcode) = match op {
+                AvxOpcode::Vmovdqu => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x7F),
+                AvxOpcode::Vmovss => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x11),
+                AvxOpcode::Vmovsd => (LegacyPrefixes::_F2, OpcodeMap::_0F, 0x11),
+                AvxOpcode::Vmovups => (LegacyPrefixes::None, OpcodeMap::_0F, 0x11),
+                AvxOpcode::Vmovupd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x11),
+                _ => unimplemented!("Opcode {:?} not implemented", op),
+            };
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .prefix(prefix)
+                .map(map)
+                .opcode(opcode)
+                .rm(dst)
+                .reg(src.to_real_reg().unwrap().hw_enc())
+                .encode(sink);
+        }
+
         Inst::XmmRmREvex {
             op,
             src1,
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index b0002c59f..396f9fd4a 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -140,8 +140,7 @@ impl Inst {
             | Inst::XmmToGprImm { op, .. }
             | Inst::XmmUnaryRmRImm { op, .. }
             | Inst::XmmUnaryRmRUnaligned { op, .. }
-            | Inst::XmmUnaryRmR { op, .. }
-            | Inst::XmmConstOp { op, .. } => smallvec![op.available_from()],
+            | Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()],
 
             Inst::XmmUnaryRmREvex { op, .. }
             | Inst::XmmRmREvex { op, .. }
@@ -153,7 +152,8 @@ impl Inst {
             | Inst::XmmRmRBlendVex { op, .. }
             | Inst::XmmVexPinsr { op, .. }
             | Inst::XmmUnaryRmRVex { op, .. }
-            | Inst::XmmUnaryRmRImmVex { op, .. } => op.available_from(),
+            | Inst::XmmUnaryRmRImmVex { op, .. }
+            | Inst::XmmMovRMVex { op, .. } => op.available_from(),
         }
     }
 }
@@ -938,6 +938,12 @@ impl PrettyPrint for Inst {
                 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
             }
 
+            Inst::XmmMovRMVex { op, src, dst, .. } => {
+                let src = pretty_print_reg(*src, 8, allocs);
+                let dst = dst.pretty_print(8, allocs);
+                format!("{} {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
             Inst::XmmRmR {
                 op,
                 src1,
@@ -964,11 +970,6 @@ impl PrettyPrint for Inst {
                 format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
             }
 
-            Inst::XmmConstOp { op, dst } => {
-                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
-                format!("{} {dst}, {dst}, {dst}", ljustify(op.to_string()))
-            }
-
             Inst::XmmRmRBlend {
                 op,
                 src1,
@@ -2019,9 +2020,6 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_reuse_def(*dst, 0);
             src2.get_operands(collector);
         }
-        Inst::XmmConstOp { dst, .. } => {
-            collector.reg_def(dst.to_writable_reg());
-        }
         Inst::XmmUninitializedValue { dst } => collector.reg_def(dst.to_writable_reg()),
         Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
             collector.reg_use(rhs.to_reg());
@@ -2035,7 +2033,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_reuse_def(dst.to_writable_reg(), 0); // Reuse RHS.
             src2.get_operands(collector);
         }
-        Inst::XmmMovRM { src, dst, .. } => {
+        Inst::XmmMovRM { src, dst, .. } | Inst::XmmMovRMVex { src, dst, .. } => {
             collector.reg_use(*src);
             dst.get_operands(collector);
         }
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index ac27aae4d..0f13b9500 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -337,12 +337,12 @@
 ;; f32 and f64
 
 (rule 5 (lower (has_type (ty_scalar_float ty) (bxor x y)))
-      (sse_xor ty x y))
+      (x64_xor_vector ty x y))
 
 ;; SSE.
 
 (rule 6 (lower (has_type ty @ (multi_lane _bits _lanes) (bxor x y)))
-      (sse_xor ty x y))
+      (x64_xor_vector ty x y))
 
 ;; `{i,b}128`.
 
@@ -1171,12 +1171,12 @@
 ;; f32 and f64
 
 (rule -3 (lower (has_type (ty_scalar_float ty) (bnot x)))
-      (sse_xor ty x (vector_all_ones)))
+      (x64_xor_vector ty x (vector_all_ones)))
 
 ;; Special case for vector-types where bit-negation is an xor against an
 ;; all-one value
 (rule -1 (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x)))
-      (sse_xor ty x (vector_all_ones)))
+      (x64_xor_vector ty x (vector_all_ones)))
 
 ;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1267,20 +1267,10 @@
 ;; Here the `movsd` instruction is used specifically to specialize moving
 ;; into the fist lane where unlike above cases we're not using the lane
 ;; immediate as an immediate to the instruction itself.
-;;
-;; Note, though, the `movsd` has different behavior with respect to the second
-;; lane of the f64x2 depending on whether the RegMem operand is a register or
-;; memory. When loading from a register `movsd` preserves the upper bits, but
-;; when loading from memory it zeros the upper bits. We specifically want to
-;; preserve the upper bits so if a `RegMem.Mem` is passed in we need to emit
-;; two `movsd` instructions. The first `movsd` (used as `xmm_unary_rm_r`) will
-;; load from memory into a temp register and then the second `movsd` (modeled
-;; internally as `xmm_rm_r` will merge the temp register into our `vec`
-;; register.
-(rule 1 (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0)
+(rule (vec_insert_lane $F64X2 vec (RegMem.Reg val) 0)
       (x64_movsd_regmove vec val))
-(rule (vec_insert_lane $F64X2 vec mem 0)
-      (x64_movsd_regmove vec (x64_movsd_load mem)))
+(rule (vec_insert_lane $F64X2 vec (RegMem.Mem val) 0)
+      (x64_movsd_regmove vec (x64_movsd_load val)))
 
 ;; f64x2.replace_lane 1
 ;;
@@ -1288,7 +1278,7 @@
 ;; into the second lane where unlike above cases we're not using the lane
 ;; immediate as an immediate to the instruction itself.
 (rule (vec_insert_lane $F64X2 vec val 1)
-      (x64_movlhps vec (reg_mem_to_xmm_mem val)))
+      (x64_movlhps vec val))
 
 ;;;; Rules for `smin`, `smax`, `umin`, `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -2557,11 +2547,11 @@
 (rule (lower (has_type $F64 (load flags address offset)))
       (x64_movsd_load (to_amode flags address offset)))
 (rule (lower (has_type $F32X4 (load flags address offset)))
-      (x64_movups (to_amode flags address offset)))
+      (x64_movups_load (to_amode flags address offset)))
 (rule (lower (has_type $F64X2 (load flags address offset)))
-      (x64_movupd (to_amode flags address offset)))
+      (x64_movupd_load (to_amode flags address offset)))
 (rule -2 (lower (has_type (ty_vec128 ty) (load flags address offset)))
-      (x64_movdqu (to_amode flags address offset)))
+      (x64_movdqu_load (to_amode flags address offset)))
 
 ;; We can load an I128 by doing two 64-bit loads.
 (rule -3 (lower (has_type $I128
@@ -2614,7 +2604,7 @@
                     address
                     offset))
       (side_effect
-       (x64_xmm_movrm (SseOpcode.Movss) (to_amode flags address offset) value)))
+       (x64_movss_store (to_amode flags address offset) value)))
 
 ;; F64 stores of values in XMM registers.
 (rule 1 (lower (store flags
@@ -2622,7 +2612,7 @@
                     address
                     offset))
       (side_effect
-       (x64_xmm_movrm (SseOpcode.Movsd) (to_amode flags address offset) value)))
+       (x64_movsd_store (to_amode flags address offset) value)))
 
 ;; Stores of F32X4 vectors.
 (rule 1 (lower (store flags
@@ -2630,7 +2620,7 @@
                     address
                     offset))
       (side_effect
-       (x64_xmm_movrm (SseOpcode.Movups) (to_amode flags address offset) value)))
+       (x64_movups_store (to_amode flags address offset) value)))
 
 ;; Stores of F64X2 vectors.
 (rule 1 (lower (store flags
@@ -2638,7 +2628,7 @@
                     address
                     offset))
       (side_effect
-       (x64_xmm_movrm (SseOpcode.Movupd) (to_amode flags address offset) value)))
+       (x64_movupd_store (to_amode flags address offset) value)))
 
 ;; Stores of all other 128-bit vector types with integer lanes.
 (rule -1 (lower (store flags
@@ -2646,7 +2636,7 @@
                     address
                     offset))
       (side_effect
-       (x64_xmm_movrm (SseOpcode.Movdqu) (to_amode flags address offset) value)))
+       (x64_movdqu_store (to_amode flags address offset) value)))
 
 ;; Stores of I128 values: store the two 64-bit halves separately.
 (rule 0 (lower (store flags
@@ -2675,7 +2665,7 @@
                               src2))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_add_mem ty (to_amode flags addr offset) src2))))
 
@@ -2689,7 +2679,7 @@
                                (load flags addr offset))))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_add_mem ty (to_amode flags addr offset) src2))))
 
@@ -2703,7 +2693,7 @@
                               src2))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_sub_mem ty (to_amode flags addr offset) src2))))
 
@@ -2717,7 +2707,7 @@
                               src2))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_and_mem ty (to_amode flags addr offset) src2))))
 
@@ -2731,7 +2721,7 @@
                                (load flags addr offset))))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_and_mem ty (to_amode flags addr offset) src2))))
 
@@ -2745,7 +2735,7 @@
                               src2))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_or_mem ty (to_amode flags addr offset) src2))))
 
@@ -2759,7 +2749,7 @@
                                (load flags addr offset))))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_or_mem ty (to_amode flags addr offset) src2))))
 
@@ -2773,7 +2763,7 @@
                               src2))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_xor_mem ty (to_amode flags addr offset) src2))))
 
@@ -2787,7 +2777,7 @@
                                (load flags addr offset))))
               addr
               offset))
-      (let ((_ RegMemImm (sink_load sink)))
+      (let ((_ RegMemImm sink))
         (side_effect
          (x64_xor_mem ty (to_amode flags addr offset) src2))))
 
@@ -3529,16 +3519,98 @@
 
 ;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; Special case for the `punpckhbw` instruction which interleaves the upper
-;; lanes of the two input registers.
-(rule 4 (lower (shuffle a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))
+;; Special case the `pshuf{l,h}w` instruction which shuffles four 16-bit
+;; integers within one value, preserving the other four 16-bit integers in that
+;; value (either the high or low half). The complicated logic is in the
+;; extractors here implemented in Rust and note that there's two cases for each
+;; instruction here to match when either the first or second shuffle operand is
+;; used.
+(rule 12 (lower (shuffle x y (pshuflw_lhs_imm imm)))
+      (x64_pshuflw x imm))
+(rule 11 (lower (shuffle x y (pshuflw_rhs_imm imm)))
+      (x64_pshuflw y imm))
+(rule 10 (lower (shuffle x y (pshufhw_lhs_imm imm)))
+      (x64_pshufhw x imm))
+(rule 9 (lower (shuffle x y (pshufhw_rhs_imm imm)))
+      (x64_pshufhw y imm))
+
+(decl pshuflw_lhs_imm (u8) Immediate)
+(extern extractor pshuflw_lhs_imm pshuflw_lhs_imm)
+(decl pshuflw_rhs_imm (u8) Immediate)
+(extern extractor pshuflw_rhs_imm pshuflw_rhs_imm)
+(decl pshufhw_lhs_imm (u8) Immediate)
+(extern extractor pshufhw_lhs_imm pshufhw_lhs_imm)
+(decl pshufhw_rhs_imm (u8) Immediate)
+(extern extractor pshufhw_rhs_imm pshufhw_rhs_imm)
+
+;; Special case for the `pshufd` instruction which will permute 32-bit values
+;; within a single register. This is only applicable if the `imm` specified
+;; selects 32-bit values from either `x` or `y`, but not both. This means
+;; there's one rule for selecting from `x` and another rule for selecting from
+;; `y`.
+(rule 8 (lower (shuffle x y (pshufd_lhs_imm imm)))
+      (x64_pshufd x imm))
+(rule 7 (lower (shuffle x y (pshufd_rhs_imm imm)))
+      (x64_pshufd y imm))
+
+(decl pshufd_lhs_imm (u8) Immediate)
+(extern extractor pshufd_lhs_imm pshufd_lhs_imm)
+(decl pshufd_rhs_imm (u8) Immediate)
+(extern extractor pshufd_rhs_imm pshufd_rhs_imm)
+
+;; Special case for i8-level interleaving of upper/low bytes.
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))
       (x64_punpckhbw a b))
-
-;; Special case for the `punpcklbw` instruction which interleaves the lower
-;; lanes of the two input registers.
-(rule 4 (lower (shuffle a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))
       (x64_punpcklbw a b))
 
+;; Special case for i16-level interleaving of upper/low bytes.
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f1e_0f0e_1d1c_0d0c_1b1a_0b0a_1918_0908)))
+      (x64_punpckhwd a b))
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x1716_0706_1514_0504_1312_0302_1110_0100)))
+      (x64_punpcklwd a b))
+
+;; Special case for i32-level interleaving of upper/low bytes.
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c_0f0e0d0c_1b1a1918_0b0a0908)))
+      (x64_punpckhdq a b))
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x17161514_07060504_13121110_03020100)))
+      (x64_punpckldq a b))
+
+;; Special case for i64-level interleaving of upper/low bytes.
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c1b1a1918_0f0e0d0c0b0a0908)))
+      (x64_punpckhqdq a b))
+(rule 6 (lower (shuffle a b (u128_from_immediate 0x1716151413121110_0706050403020100)))
+      (x64_punpcklqdq a b))
+
+;; If the vector shift mask is all 0s then that means the first byte of the
+;; first operand is broadcast to all bytes. Falling through would load an
+;; all-zeros constant from a rip-relative location but it should be slightly
+;; more efficient to execute the `pshufb` here-and-now with an xor'd-to-be-zero
+;; register.
+(rule 6 (lower (shuffle a _ (u128_from_immediate 0)))
+      (x64_pshufb a (xmm_zero $I8X16)))
+
+;; Special case for the `shufps` instruction which will select two 32-bit values
+;; from the first operand and two 32-bit values from the second operand. Note
+;; that there is a second case here as well for when the operands can be
+;; swapped.
+;;
+;; Note that the priority of this instruction is currently lower than the above
+;; special cases since `shufps` handles many of them and for now it's
+;; hypothesized that the dedicated instructions are better than `shufps`.
+;; Someone with more knowledge about x86 timings should perhaps reorder the
+;; rules here eventually though.
+(rule 5 (lower (shuffle x y (shufps_imm imm)))
+      (x64_shufps x y imm))
+(rule 4 (lower (shuffle x y (shufps_rev_imm imm)))
+      (x64_shufps y x imm))
+
+(decl shufps_imm(u8) Immediate)
+(extern extractor shufps_imm shufps_imm)
+(decl shufps_rev_imm(u8) Immediate)
+(extern extractor shufps_rev_imm shufps_rev_imm)
+
+
 ;; If `lhs` and `rhs` are the same we can use a single PSHUFB to shuffle the XMM
 ;; register. We statically build `constructed_mask` to zero out any unknown lane
 ;; indices (may not be completely necessary: verification could fail incorrect
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 61be54a00..815e40e35 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -151,7 +151,9 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
         }
 
         if let Some(load) = self.sinkable_load(val) {
-            return self.sink_load(&load);
+            return RegMem::Mem {
+                addr: self.sink_load(&load),
+            };
         }
 
         RegMem::reg(self.put_in_reg(val))
@@ -277,12 +279,10 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
         None
     }
 
-    fn sink_load(&mut self, load: &SinkableLoad) -> RegMem {
+    fn sink_load(&mut self, load: &SinkableLoad) -> SyntheticAmode {
         self.lower_ctx.sink_inst(load.inst);
         let addr = lower_to_amode(self.lower_ctx, load.addr_input, load.offset);
-        RegMem::Mem {
-            addr: SyntheticAmode::Real(addr),
-        }
+        SyntheticAmode::Real(addr)
     }
 
     #[inline]
@@ -999,6 +999,124 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
             },
         }
     }
+
+    fn pshufd_lhs_imm(&mut self, imm: Immediate) -> Option<u8> {
+        let (a, b, c, d) = self.shuffle32_from_imm(imm)?;
+        if a < 4 && b < 4 && c < 4 && d < 4 {
+            Some(a | (b << 2) | (c << 4) | (d << 6))
+        } else {
+            None
+        }
+    }
+
+    fn pshufd_rhs_imm(&mut self, imm: Immediate) -> Option<u8> {
+        let (a, b, c, d) = self.shuffle32_from_imm(imm)?;
+        // When selecting from the right-hand-side, subtract these all by 4
+        // which will bail out if anything is less than 4. Afterwards the check
+        // is the same as `pshufd_lhs_imm` above.
+        let a = a.checked_sub(4)?;
+        let b = b.checked_sub(4)?;
+        let c = c.checked_sub(4)?;
+        let d = d.checked_sub(4)?;
+        if a < 4 && b < 4 && c < 4 && d < 4 {
+            Some(a | (b << 2) | (c << 4) | (d << 6))
+        } else {
+            None
+        }
+    }
+
+    fn shufps_imm(&mut self, imm: Immediate) -> Option<u8> {
+        // The `shufps` instruction selects the first two elements from the
+        // first vector and the second two elements from the second vector, so
+        // offset the third/fourth selectors by 4 and then make sure everything
+        // fits in 32-bits.
+        let (a, b, c, d) = self.shuffle32_from_imm(imm)?;
+        let c = c.checked_sub(4)?;
+        let d = d.checked_sub(4)?;
+        if a < 4 && b < 4 && c < 4 && d < 4 {
+            Some(a | (b << 2) | (c << 4) | (d << 6))
+        } else {
+            None
+        }
+    }
+
+    fn shufps_rev_imm(&mut self, imm: Immediate) -> Option<u8> {
+        // This is almost the same as `shufps_imm` except the elements that are
+        // subtracted are reversed. This handles the case that `shufps`
+        // instruction can be emitted if the order of the operands are swapped.
+        let (a, b, c, d) = self.shuffle32_from_imm(imm)?;
+        let a = a.checked_sub(4)?;
+        let b = b.checked_sub(4)?;
+        if a < 4 && b < 4 && c < 4 && d < 4 {
+            Some(a | (b << 2) | (c << 4) | (d << 6))
+        } else {
+            None
+        }
+    }
+
+    fn pshuflw_lhs_imm(&mut self, imm: Immediate) -> Option<u8> {
+        // Similar to `shufps` except this operates over 16-bit values so four
+        // of them must be fixed and the other four must be in-range to encode
+        // in the immediate.
+        let (a, b, c, d, e, f, g, h) = self.shuffle16_from_imm(imm)?;
+        if a < 4 && b < 4 && c < 4 && d < 4 && [e, f, g, h] == [4, 5, 6, 7] {
+            Some(a | (b << 2) | (c << 4) | (d << 6))
+        } else {
+            None
+        }
+    }
+
+    fn pshuflw_rhs_imm(&mut self, imm: Immediate) -> Option<u8> {
+        let (a, b, c, d, e, f, g, h) = self.shuffle16_from_imm(imm)?;
+        let a = a.checked_sub(8)?;
+        let b = b.checked_sub(8)?;
+        let c = c.checked_sub(8)?;
+        let d = d.checked_sub(8)?;
+        let e = e.checked_sub(8)?;
+        let f = f.checked_sub(8)?;
+        let g = g.checked_sub(8)?;
+        let h = h.checked_sub(8)?;
+        if a < 4 && b < 4 && c < 4 && d < 4 && [e, f, g, h] == [4, 5, 6, 7] {
+            Some(a | (b << 2) | (c << 4) | (d << 6))
+        } else {
+            None
+        }
+    }
+
+    fn pshufhw_lhs_imm(&mut self, imm: Immediate) -> Option<u8> {
+        // Similar to `pshuflw` except that the first four operands must be
+        // fixed and the second four are offset by an extra 4 and tested to
+        // make sure they're all in the range [4, 8).
+        let (a, b, c, d, e, f, g, h) = self.shuffle16_from_imm(imm)?;
+        let e = e.checked_sub(4)?;
+        let f = f.checked_sub(4)?;
+        let g = g.checked_sub(4)?;
+        let h = h.checked_sub(4)?;
+        if e < 4 && f < 4 && g < 4 && h < 4 && [a, b, c, d] == [0, 1, 2, 3] {
+            Some(e | (f << 2) | (g << 4) | (h << 6))
+        } else {
+            None
+        }
+    }
+
+    fn pshufhw_rhs_imm(&mut self, imm: Immediate) -> Option<u8> {
+        // Note that everything here is offset by at least 8 and the upper
+        // bits are offset by 12 to test they're in the range of [12, 16).
+        let (a, b, c, d, e, f, g, h) = self.shuffle16_from_imm(imm)?;
+        let a = a.checked_sub(8)?;
+        let b = b.checked_sub(8)?;
+        let c = c.checked_sub(8)?;
+        let d = d.checked_sub(8)?;
+        let e = e.checked_sub(12)?;
+        let f = f.checked_sub(12)?;
+        let g = g.checked_sub(12)?;
+        let h = h.checked_sub(12)?;
+        if e < 4 && f < 4 && g < 4 && h < 4 && [a, b, c, d] == [0, 1, 2, 3] {
+            Some(e | (f << 2) | (g << 4) | (h << 6))
+        } else {
+            None
+        }
+    }
 }
 
 impl IsleContext<'_, '_, MInst, X64Backend> {
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index 70bcb7d12..3eb0db7ea 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -585,9 +585,86 @@ macro_rules! isle_lower_prelude_methods {
                 .collect();
             self.lower_ctx.gen_return(rets);
         }
+
+        /// Attempts to interpret the shuffle immediate `imm` as a shuffle of
+        /// 32-bit lanes, returning four integers, each of which is less than 8,
+        /// which represents a permutation of 32-bit lanes as specified by
+        /// `imm`.
+        ///
+        /// For example the shuffle immediate
+        ///
+        /// `0 1 2 3 8 9 10 11 16 17 18 19 24 25 26 27`
+        ///
+        /// would return `Some((0, 2, 4, 6))`.
+        fn shuffle32_from_imm(&mut self, imm: Immediate) -> Option<(u8, u8, u8, u8)> {
+            use crate::machinst::isle::shuffle_imm_as_le_lane_idx;
+
+            let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
+            Some((
+                shuffle_imm_as_le_lane_idx(4, &bytes[0..4])?,
+                shuffle_imm_as_le_lane_idx(4, &bytes[4..8])?,
+                shuffle_imm_as_le_lane_idx(4, &bytes[8..12])?,
+                shuffle_imm_as_le_lane_idx(4, &bytes[12..16])?,
+            ))
+        }
+
+        /// Same as `shuffle32_from_imm`, but for 16-bit lane shuffles.
+        fn shuffle16_from_imm(
+            &mut self,
+            imm: Immediate,
+        ) -> Option<(u8, u8, u8, u8, u8, u8, u8, u8)> {
+            use crate::machinst::isle::shuffle_imm_as_le_lane_idx;
+            let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
+            Some((
+                shuffle_imm_as_le_lane_idx(2, &bytes[0..2])?,
+                shuffle_imm_as_le_lane_idx(2, &bytes[2..4])?,
+                shuffle_imm_as_le_lane_idx(2, &bytes[4..6])?,
+                shuffle_imm_as_le_lane_idx(2, &bytes[6..8])?,
+                shuffle_imm_as_le_lane_idx(2, &bytes[8..10])?,
+                shuffle_imm_as_le_lane_idx(2, &bytes[10..12])?,
+                shuffle_imm_as_le_lane_idx(2, &bytes[12..14])?,
+                shuffle_imm_as_le_lane_idx(2, &bytes[14..16])?,
+            ))
+        }
     };
 }
 
+/// Returns the `size`-byte lane referred to by the shuffle immediate specified
+/// in `bytes`.
+///
+/// This helper is used by `shuffleNN_from_imm` above and is used to interpret a
+/// byte-based shuffle as a higher-level shuffle of bigger lanes. This will see
+/// if the `bytes` specified, which must have `size` length, specifies a lane in
+/// vectors aligned to a `size`-byte boundary.
+///
+/// Returns `None` if `bytes` doesn't specify a `size`-byte lane aligned
+/// appropriately, or returns `Some(n)` where `n` is the index of the lane being
+/// shuffled.
+pub fn shuffle_imm_as_le_lane_idx(size: u8, bytes: &[u8]) -> Option<u8> {
+    assert_eq!(bytes.len(), usize::from(size));
+
+    // The first index in `bytes` must be aligned to a `size` boundary for the
+    // bytes to be a valid specifier for a lane of `size` bytes.
+    if bytes[0] % size != 0 {
+        return None;
+    }
+
+    // Afterwards the bytes must all be one larger than the prior to specify a
+    // contiguous sequence of bytes that's being shuffled. Basically `bytes`
+    // must refer to the entire `size`-byte lane, in little-endian order.
+    for i in 0..size - 1 {
+        let idx = usize::from(i);
+        if bytes[idx] + 1 != bytes[idx + 1] {
+            return None;
+        }
+    }
+
+    // All of the `bytes` are in-order, meaning that this is a valid shuffle
+    // immediate to specify a lane of `size` bytes. The index, when viewed as
+    // `size`-byte immediates, will be the first byte divided by the byte size.
+    Some(bytes[0] / size)
+}
+
 /// Helpers specifically for machines that use ABICaller.
 #[macro_export]
 #[doc(hidden)]
diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isle
index 51e15cb2a..a7e59d590 100644
--- a/cranelift/codegen/src/prelude_lower.isle
+++ b/cranelift/codegen/src/prelude_lower.isle
@@ -592,6 +592,16 @@
 (decl u64_from_constant (u64) Constant)
 (extern extractor u64_from_constant u64_from_constant)
 
+;; Extracts lane indices, represented as u8's, if the immediate for a
+;; `shuffle` instruction represents shuffling N-bit values. The u8 values
+;; returned will be in the range of 0 to (256/N)-1, inclusive, and index the
+;; N-bit chunks of two concatenated 128-bit vectors starting from the
+;; least-significant bits.
+(decl shuffle32_from_imm (u8 u8 u8 u8) Immediate)
+(extern extractor shuffle32_from_imm shuffle32_from_imm)
+(decl shuffle16_from_imm (u8 u8 u8 u8 u8 u8 u8 u8) Immediate)
+(extern extractor shuffle16_from_imm shuffle16_from_imm)
+
 ;;;; Helpers for generating returns ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Extractor to check for the special case that a `WritableValueRegs`


Test patch:
diff --git a/cranelift/filetests/filetests/isa/x64/fabs.clif b/cranelift/filetests/filetests/isa/x64/fabs.clif
index 574d34cc1..c915d9c59 100644
--- a/cranelift/filetests/filetests/isa/x64/fabs.clif
+++ b/cranelift/filetests/filetests/isa/x64/fabs.clif
@@ -69,9 +69,10 @@ block0(v0: f32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pcmpeqd %xmm3, %xmm3, %xmm3
-;   psrld   %xmm3, $1, %xmm3
-;   andps   %xmm0, %xmm3, %xmm0
+;   uninit  %xmm4
+;   pcmpeqd %xmm4, %xmm4, %xmm4
+;   psrld   %xmm4, $1, %xmm4
+;   andps   %xmm0, %xmm4, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -81,9 +82,9 @@ block0(v0: f32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pcmpeqd %xmm3, %xmm3
-;   psrld $1, %xmm3
-;   andps %xmm3, %xmm0
+;   pcmpeqd %xmm4, %xmm4
+;   psrld $1, %xmm4
+;   andps %xmm4, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -98,9 +99,10 @@ block0(v0: f64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pcmpeqd %xmm3, %xmm3, %xmm3
-;   psrlq   %xmm3, $1, %xmm3
-;   andpd   %xmm0, %xmm3, %xmm0
+;   uninit  %xmm4
+;   pcmpeqd %xmm4, %xmm4, %xmm4
+;   psrlq   %xmm4, $1, %xmm4
+;   andpd   %xmm0, %xmm4, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -110,9 +112,9 @@ block0(v0: f64x2):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pcmpeqd %xmm3, %xmm3
-;   psrlq $1, %xmm3
-;   andpd %xmm3, %xmm0
+;   pcmpeqd %xmm4, %xmm4
+;   psrlq $1, %xmm4
+;   andpd %xmm4, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/fcvt.clif b/cranelift/filetests/filetests/isa/x64/fcvt.clif
index aee96700a..1a4ab0cc1 100644
--- a/cranelift/filetests/filetests/isa/x64/fcvt.clif
+++ b/cranelift/filetests/filetests/isa/x64/fcvt.clif
@@ -1032,20 +1032,22 @@ block0(v0: f32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   xorps   %xmm5, %xmm5, %xmm5
-;   movdqa  %xmm0, %xmm9
-;   maxps   %xmm9, %xmm5, %xmm9
-;   pcmpeqd %xmm5, %xmm5, %xmm5
-;   psrld   %xmm5, $1, %xmm5
-;   cvtdq2ps %xmm5, %xmm13
-;   cvttps2dq %xmm9, %xmm12
-;   subps   %xmm9, %xmm13, %xmm9
-;   cmpps   $2, %xmm13, %xmm9, %xmm13
-;   cvttps2dq %xmm9, %xmm0
-;   pxor    %xmm0, %xmm13, %xmm0
-;   pxor    %xmm6, %xmm6, %xmm6
-;   pmaxsd  %xmm0, %xmm6, %xmm0
-;   paddd   %xmm0, %xmm12, %xmm0
+;   uninit  %xmm6
+;   xorps   %xmm6, %xmm6, %xmm6
+;   movdqa  %xmm0, %xmm11
+;   maxps   %xmm11, %xmm6, %xmm11
+;   pcmpeqd %xmm6, %xmm6, %xmm6
+;   psrld   %xmm6, $1, %xmm6
+;   cvtdq2ps %xmm6, %xmm15
+;   cvttps2dq %xmm11, %xmm14
+;   subps   %xmm11, %xmm15, %xmm11
+;   cmpps   $2, %xmm15, %xmm11, %xmm15
+;   cvttps2dq %xmm11, %xmm0
+;   pxor    %xmm0, %xmm15, %xmm0
+;   uninit  %xmm9
+;   pxor    %xmm9, %xmm9, %xmm9
+;   pmaxsd  %xmm0, %xmm9, %xmm0
+;   paddd   %xmm0, %xmm14, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1055,20 +1057,20 @@ block0(v0: f32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   xorps %xmm5, %xmm5
-;   movdqa %xmm0, %xmm9
-;   maxps %xmm5, %xmm9
-;   pcmpeqd %xmm5, %xmm5
-;   psrld $1, %xmm5
-;   cvtdq2ps %xmm5, %xmm13
-;   cvttps2dq %xmm9, %xmm12
-;   subps %xmm13, %xmm9
-;   cmpleps %xmm9, %xmm13
-;   cvttps2dq %xmm9, %xmm0
-;   pxor %xmm13, %xmm0
-;   pxor %xmm6, %xmm6
-;   pmaxsd %xmm6, %xmm0
-;   paddd %xmm12, %xmm0
+;   xorps %xmm6, %xmm6
+;   movdqa %xmm0, %xmm11
+;   maxps %xmm6, %xmm11
+;   pcmpeqd %xmm6, %xmm6
+;   psrld $1, %xmm6
+;   cvtdq2ps %xmm6, %xmm15
+;   cvttps2dq %xmm11, %xmm14
+;   subps %xmm15, %xmm11
+;   cmpleps %xmm11, %xmm15
+;   cvttps2dq %xmm11, %xmm0
+;   pxor %xmm15, %xmm0
+;   pxor %xmm9, %xmm9
+;   pmaxsd %xmm9, %xmm0
+;   paddd %xmm14, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/float-avx.clif b/cranelift/filetests/filetests/isa/x64/float-avx.clif
index 6776bb529..4e29340a3 100644
--- a/cranelift/filetests/filetests/isa/x64/float-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/float-avx.clif
@@ -589,3 +589,59 @@ block0(v0: f64x2):
 ;   addb %al, (%rax)
 ;   sarb $0xff, %bh
 
+function %load_and_store_f32(i64, i64) {
+block0(v0: i64, v1: i64):
+  v2 = load.f32 v0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovss  0(%rdi), %xmm3
+;   vmovss  %xmm3, 0(%rsi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovss (%rdi), %xmm3 ; trap: heap_oob
+;   vmovss %xmm3, (%rsi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %load_and_store_f64(i64, i64) {
+block0(v0: i64, v1: i64):
+  v2 = load.f64 v0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovsd  0(%rdi), %xmm3
+;   vmovsd  %xmm3, 0(%rsi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovsd (%rdi), %xmm3 ; trap: heap_oob
+;   vmovsd %xmm3, (%rsi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/fneg.clif b/cranelift/filetests/filetests/isa/x64/fneg.clif
index d990c31c1..4d0b2535e 100644
--- a/cranelift/filetests/filetests/isa/x64/fneg.clif
+++ b/cranelift/filetests/filetests/isa/x64/fneg.clif
@@ -69,9 +69,10 @@ block0(v0: f32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pcmpeqd %xmm3, %xmm3, %xmm3
-;   pslld   %xmm3, $31, %xmm3
-;   xorps   %xmm0, %xmm3, %xmm0
+;   uninit  %xmm4
+;   pcmpeqd %xmm4, %xmm4, %xmm4
+;   pslld   %xmm4, $31, %xmm4
+;   xorps   %xmm0, %xmm4, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -81,9 +82,9 @@ block0(v0: f32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pcmpeqd %xmm3, %xmm3
-;   pslld $0x1f, %xmm3
-;   xorps %xmm3, %xmm0
+;   pcmpeqd %xmm4, %xmm4
+;   pslld $0x1f, %xmm4
+;   xorps %xmm4, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -98,9 +99,10 @@ block0(v0: f64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pcmpeqd %xmm3, %xmm3, %xmm3
-;   psllq   %xmm3, $63, %xmm3
-;   xorpd   %xmm0, %xmm3, %xmm0
+;   uninit  %xmm4
+;   pcmpeqd %xmm4, %xmm4, %xmm4
+;   psllq   %xmm4, $63, %xmm4
+;   xorpd   %xmm0, %xmm4, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -110,9 +112,9 @@ block0(v0: f64x2):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pcmpeqd %xmm3, %xmm3
-;   psllq $0x3f, %xmm3
-;   xorpd %xmm3, %xmm0
+;   pcmpeqd %xmm4, %xmm4
+;   psllq $0x3f, %xmm4
+;   xorpd %xmm4, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/insertlane.clif b/cranelift/filetests/filetests/isa/x64/insertlane.clif
new file mode 100644
index 000000000..7f2569d02
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/insertlane.clif
@@ -0,0 +1,82 @@
+test compile precise-output
+set enable_simd
+target x86_64 has_avx
+
+function %insertlane_f64x2_zero(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+  v2 = insertlane v0, v1, 0
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovsd  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovsd %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_f64x2_one(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+  v2 = insertlane v0, v1, 1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovlhps %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovlhps %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %insertlane_f64x2_zero_with_load(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+  v2 = load.f64 v1
+  v3 = insertlane v0, v2, 0
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovsd  0(%rdi), %xmm3
+;   vmovsd  %xmm0, %xmm3, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovsd (%rdi), %xmm3 ; trap: heap_oob
+;   vmovsd %xmm3, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle-avx.clif b/cranelift/filetests/filetests/isa/x64/shuffle-avx.clif
new file mode 100644
index 000000000..30cf9721e
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/shuffle-avx.clif
@@ -0,0 +1,116 @@
+test compile precise-output
+set enable_simd
+target x86_64 has_avx
+
+function %punpckldq(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 16 17 18 19 4 5 6 7 20 21 22 23]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpunpckldq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpunpckldq %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckhdq(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 24 25 26 27 12 13 14 15 28 29 30 31]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpunpckhdq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpunpckhdq %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpcklqdq(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23]
+    v5 = bitcast.i64x2 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpunpcklqdq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpunpcklqdq %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckhqdq(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31]
+    v5 = bitcast.i64x2 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpunpckhqdq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpunpckhqdq %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle.clif b/cranelift/filetests/filetests/isa/x64/shuffle.clif
index 529b95cc5..ce1d1b484 100644
--- a/cranelift/filetests/filetests/isa/x64/shuffle.clif
+++ b/cranelift/filetests/filetests/isa/x64/shuffle.clif
@@ -52,3 +52,595 @@ block0(v0: i8x16, v1: i8x16):
 ;   popq %rbp
 ;   retq
 
+function %punpcklwd(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 16 17 2 3 18 19 4 5 20 21 6 7 22 23]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpcklwd %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpcklwd %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckhwd(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 24 25 10 11 26 27 12 13 28 29 14 15 30 31]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpckhwd %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpckhwd %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshufd_0022(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 0 1 2 3 8 9 10 11 8 9 10 11]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshufd  $160, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshufd $0xa0, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshufd_3120(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [12 13 14 15 4 5 6 7 8 9 10 11 0 1 2 3]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshufd  $39, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshufd $0x27, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshufd_7546(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [28 29 30 31 20 21 22 23 16 17 18 19 24 25 26 27]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshufd  $135, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshufd $0x87, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %not_single_pshufd(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   shufps  $78, %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   shufps $0x4e, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckldq(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 16 17 18 19 4 5 6 7 20 21 22 23]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpckldq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpckldq %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckhdq(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 24 25 26 27 12 13 14 15 28 29 30 31]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpckhdq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpckhdq %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpcklqdq(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23]
+    v5 = bitcast.i64x2 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpcklqdq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpcklqdq %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %punpckhqdq(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31]
+    v5 = bitcast.i64x2 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   punpckhqdq %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   punpckhqdq %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %shufps_3277(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [12 13 14 15 8 9 10 11 28 29 30 31 28 29 30 31]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   shufps  $251, %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   shufps $0xfb, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %shufps_6500(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [24 25 26 27 20 21 22 23 0 1 2 3 0 1 2 3]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movdqa  %xmm0, %xmm4
+;   movdqa  %xmm1, %xmm0
+;   shufps  $6, %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movdqa %xmm0, %xmm4
+;   movdqa %xmm1, %xmm0
+;   shufps $6, %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshuflw_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [6 7 4 5 2 3 0 1 8 9 10 11 12 13 14 15]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshuflw $27, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshuflw $0x1b, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshuflw_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [6 7 4 5 6 7 4 5 8 9 10 11 12 13 14 15]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshuflw $187, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshuflw $0xbb, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshuflw_rhs_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [22 23 20 21 18 19 16 17 24 25 26 27 28 29 30 31]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshuflw $27, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshuflw $0x1b, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshuflw_rhs_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [22 23 18 19 22 23 18 19 24 25 26 27 28 29 30 31]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshuflw $119, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshuflw $0x77, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshufhw_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 14 15 12 13 10 11 8 9]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshufhw $27, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshufhw $0x1b, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshufhw_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 14 15 10 11 14 15 10 11]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshufhw $119, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshufhw $0x77, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshufhw_rhs_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [16 17 18 19 20 21 22 23 30 31 28 29 26 27 24 25]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshufhw $27, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshufhw $0x1b, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %pshufhw_rhs_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [16 17 18 19 20 21 22 23 30 31 26 27 30 31 26 27]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pshufhw $119, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pshufhw $0x77, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %shuffle_all_zeros(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   uninit  %xmm4
+;   pxor    %xmm4, %xmm4, %xmm4
+;   pshufb  %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pxor %xmm4, %xmm4
+;   pshufb %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
index fdd58032e..87c1b7710 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
@@ -610,8 +610,8 @@ block0(v0: i32x4, v1: i32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pshufd  $250, %xmm0, %xmm3
-;   pshufd  $250, %xmm1, %xmm5
+;   vpshufd $250, %xmm0, %xmm3
+;   vpshufd $250, %xmm1, %xmm5
 ;   vpmuldq %xmm3, %xmm5, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -622,8 +622,8 @@ block0(v0: i32x4, v1: i32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pshufd $0xfa, %xmm0, %xmm3
-;   pshufd $0xfa, %xmm1, %xmm5
+;   vpshufd $0xfa, %xmm0, %xmm3
+;   vpshufd $0xfa, %xmm1, %xmm5
 ;   vpmuldq %xmm5, %xmm3, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
@@ -641,8 +641,8 @@ block0(v0: i32x4, v1: i32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pshufd  $80, %xmm0, %xmm3
-;   pshufd  $80, %xmm1, %xmm5
+;   vpshufd $80, %xmm0, %xmm3
+;   vpshufd $80, %xmm1, %xmm5
 ;   vpmuludq %xmm3, %xmm5, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -653,8 +653,8 @@ block0(v0: i32x4, v1: i32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pshufd $0x50, %xmm0, %xmm3
-;   pshufd $0x50, %xmm1, %xmm5
+;   vpshufd $0x50, %xmm0, %xmm3
+;   vpshufd $0x50, %xmm1, %xmm5
 ;   vpmuludq %xmm5, %xmm3, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
@@ -1233,7 +1233,7 @@ block0(v0: i8x16):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movdqu  const(0), %xmm2
+;   vmovdqu const(0), %xmm2
 ;   vpmaddubsw %xmm2, %xmm0, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -1244,7 +1244,7 @@ block0(v0: i8x16):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movdqu 0x14(%rip), %xmm2
+;   vmovdqu 0x14(%rip), %xmm2
 ;   vpmaddubsw %xmm0, %xmm2, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
@@ -1317,8 +1317,9 @@ block0(v0: i8):
 ; block0:
 ;   uninit  %xmm2
 ;   vpinsrb $0 %xmm2, %rdi, %xmm4
-;   pxor    %xmm6, %xmm6, %xmm6
-;   vpshufb %xmm4, %xmm6, %xmm0
+;   uninit  %xmm6
+;   vpxor   %xmm6, %xmm6, %xmm8
+;   vpshufb %xmm4, %xmm8, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1329,8 +1330,8 @@ block0(v0: i8):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   vpinsrb $0, %edi, %xmm2, %xmm4
-;   pxor %xmm6, %xmm6
-;   vpshufb %xmm6, %xmm4, %xmm0
+;   vpxor %xmm6, %xmm6, %xmm8
+;   vpshufb %xmm8, %xmm4, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -1347,12 +1348,13 @@ block0(v0: f64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   xorpd   %xmm2, %xmm2, %xmm2
-;   vmaxpd  %xmm0, %xmm2, %xmm4
-;   vminpd  %xmm4, const(0), %xmm6
-;   vroundpd $3, %xmm6, %xmm8
-;   vaddpd  %xmm8, const(1), %xmm10
-;   vshufps $136 %xmm10, %xmm2, %xmm0
+;   uninit  %xmm2
+;   vxorpd  %xmm2, %xmm2, %xmm4
+;   vmaxpd  %xmm0, %xmm4, %xmm6
+;   vminpd  %xmm6, const(0), %xmm8
+;   vroundpd $3, %xmm8, %xmm10
+;   vaddpd  %xmm10, const(1), %xmm12
+;   vshufps $136 %xmm12, %xmm4, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1362,12 +1364,12 @@ block0(v0: f64x2):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   xorpd %xmm2, %xmm2
-;   vmaxpd %xmm2, %xmm0, %xmm4
-;   vminpd 0x1c(%rip), %xmm4, %xmm6
-;   vroundpd $3, %xmm6, %xmm8
-;   vaddpd 0x1e(%rip), %xmm8, %xmm10
-;   vshufps $0x88, %xmm2, %xmm10, %xmm0
+;   vxorpd %xmm2, %xmm2, %xmm4
+;   vmaxpd %xmm4, %xmm0, %xmm6
+;   vminpd 0x1c(%rip), %xmm6, %xmm8
+;   vroundpd $3, %xmm8, %xmm10
+;   vaddpd 0x1e(%rip), %xmm10, %xmm12
+;   vshufps $0x88, %xmm4, %xmm12, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -1392,7 +1394,7 @@ block0(v0: i8x16, v1: i32):
 ;   vpsllw  %xmm0, %xmm5, %xmm7
 ;   lea     const(0), %rsi
 ;   shlq    $4, %r10, %r10
-;   movdqu  0(%rsi,%r10,1), %xmm13
+;   vmovdqu 0(%rsi,%r10,1), %xmm13
 ;   vpand   %xmm7, %xmm13, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -1409,7 +1411,7 @@ block0(v0: i8x16, v1: i32):
 ;   vpsllw %xmm5, %xmm0, %xmm7
 ;   leaq 0x15(%rip), %rsi
 ;   shlq $4, %r10
-;   movdqu (%rsi, %r10), %xmm13
+;   vmovdqu (%rsi, %r10), %xmm13
 ;   vpand %xmm13, %xmm7, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
@@ -1427,7 +1429,7 @@ block0(v0: i8x16):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   vpsllw  %xmm0, $1, %xmm2
-;   movdqu  const(0), %xmm4
+;   vmovdqu const(0), %xmm4
 ;   vpand   %xmm2, %xmm4, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -1439,7 +1441,7 @@ block0(v0: i8x16):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   vpsllw $1, %xmm0, %xmm2
-;   movdqu 0xf(%rip), %xmm4
+;   vmovdqu 0xf(%rip), %xmm4
 ;   vpand %xmm4, %xmm2, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
index 4cf435295..7da79a0cc 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
@@ -187,9 +187,10 @@ block0(v0: f32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pcmpeqd %xmm2, %xmm2, %xmm2
-;   vpsrld  %xmm2, $1, %xmm4
-;   vandps  %xmm0, %xmm4, %xmm0
+;   uninit  %xmm2
+;   vpcmpeqd %xmm2, %xmm2, %xmm4
+;   vpsrld  %xmm4, $1, %xmm6
+;   vandps  %xmm0, %xmm6, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -199,9 +200,9 @@ block0(v0: f32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pcmpeqd %xmm2, %xmm2
-;   vpsrld $1, %xmm2, %xmm4
-;   vandps %xmm4, %xmm0, %xmm0
+;   vpcmpeqd %xmm2, %xmm2, %xmm4
+;   vpsrld $1, %xmm4, %xmm6
+;   vandps %xmm6, %xmm0, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/simd-comparison-legalize.clif b/cranelift/filetests/filetests/isa/x64/simd-comparison-legalize.clif
index 80e7fa248..9dd8be2b1 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-comparison-legalize.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-comparison-legalize.clif
@@ -13,8 +13,9 @@ block0(v0: i32x4, v1: i32x4):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   pcmpeqd %xmm0, %xmm1, %xmm0
-;   pcmpeqd %xmm5, %xmm5, %xmm5
-;   pxor    %xmm0, %xmm5, %xmm0
+;   uninit  %xmm6
+;   pcmpeqd %xmm6, %xmm6, %xmm6
+;   pxor    %xmm0, %xmm6, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -25,8 +26,8 @@ block0(v0: i32x4, v1: i32x4):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   pcmpeqd %xmm1, %xmm0
-;   pcmpeqd %xmm5, %xmm5
-;   pxor %xmm5, %xmm0
+;   pcmpeqd %xmm6, %xmm6
+;   pxor %xmm6, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -43,8 +44,9 @@ block0(v0: i32x4, v1: i32x4):
 ; block0:
 ;   pmaxud  %xmm0, %xmm1, %xmm0
 ;   pcmpeqd %xmm0, %xmm1, %xmm0
-;   pcmpeqd %xmm7, %xmm7, %xmm7
-;   pxor    %xmm0, %xmm7, %xmm0
+;   uninit  %xmm8
+;   pcmpeqd %xmm8, %xmm8, %xmm8
+;   pxor    %xmm0, %xmm8, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -56,8 +58,8 @@ block0(v0: i32x4, v1: i32x4):
 ; block1: ; offset 0x4
 ;   pmaxud %xmm1, %xmm0
 ;   pcmpeqd %xmm1, %xmm0
-;   pcmpeqd %xmm7, %xmm7
-;   pxor %xmm7, %xmm0
+;   pcmpeqd %xmm8, %xmm8
+;   pxor %xmm8, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif
index f414054ed..f2ce65fec 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-lane-access-compile.clif
@@ -172,8 +172,9 @@ block0(v0: i8):
 ; block0:
 ;   uninit  %xmm0
 ;   pinsrb  $0, %xmm0, %rdi, %xmm0
-;   pxor    %xmm6, %xmm6, %xmm6
-;   pshufb  %xmm0, %xmm6, %xmm0
+;   uninit  %xmm7
+;   pxor    %xmm7, %xmm7, %xmm7
+;   pshufb  %xmm0, %xmm7, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -184,8 +185,8 @@ block0(v0: i8):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   pinsrb $0, %edi, %xmm0
-;   pxor %xmm6, %xmm6
-;   pshufb %xmm6, %xmm0
+;   pxor %xmm7, %xmm7
+;   pshufb %xmm7, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/isa/x64/simd-load-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-load-avx.clif
index df4f25a99..6f9856e66 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-load-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-load-avx.clif
@@ -152,3 +152,87 @@ block0(v0: i64):
 ;   popq %rbp
 ;   retq
 
+function %load_store_i8x16(i64, i64) {
+block0(v0: i64, v1: i64):
+  v2 = load.i8x16 v0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovdqu 0(%rdi), %xmm3
+;   vmovdqu %xmm3, 0(%rsi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovdqu (%rdi), %xmm3 ; trap: heap_oob
+;   vmovdqu %xmm3, (%rsi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %load_store_f32x4(i64, i64) {
+block0(v0: i64, v1: i64):
+  v2 = load.f32x4 v0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovups 0(%rdi), %xmm3
+;   vmovups %xmm3, 0(%rsi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovups (%rdi), %xmm3 ; trap: heap_oob
+;   vmovups %xmm3, (%rsi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %load_store_f64x2(i64, i64) {
+block0(v0: i64, v1: i64):
+  v2 = load.f64x2 v0
+  store v2, v1
+  return
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmovupd 0(%rdi), %xmm3
+;   vmovupd %xmm3, 0(%rsi)
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmovupd (%rdi), %xmm3 ; trap: heap_oob
+;   vmovupd %xmm3, (%rsi) ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif
index a4098176d..60ad9a419 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-logical-compile.clif
@@ -12,8 +12,9 @@ block0(v0: i32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pcmpeqd %xmm2, %xmm2, %xmm2
-;   pxor    %xmm0, %xmm2, %xmm0
+;   uninit  %xmm3
+;   pcmpeqd %xmm3, %xmm3, %xmm3
+;   pxor    %xmm0, %xmm3, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -23,8 +24,8 @@ block0(v0: i32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pcmpeqd %xmm2, %xmm2
-;   pxor %xmm2, %xmm0
+;   pcmpeqd %xmm3, %xmm3
+;   pxor %xmm3, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -66,10 +67,11 @@ block0(v0: i64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   pxor    %xmm2, %xmm2, %xmm2
-;   movdqa  %xmm0, %xmm4
-;   pcmpeqq %xmm4, %xmm2, %xmm4
-;   ptest   %xmm4, %xmm4
+;   uninit  %xmm3
+;   pxor    %xmm3, %xmm3, %xmm3
+;   movdqa  %xmm0, %xmm6
+;   pcmpeqq %xmm6, %xmm3, %xmm6
+;   ptest   %xmm6, %xmm6
 ;   setz    %al
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
@@ -80,10 +82,10 @@ block0(v0: i64x2):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   pxor %xmm2, %xmm2
-;   movdqa %xmm0, %xmm4
-;   pcmpeqq %xmm2, %xmm4
-;   ptest %xmm4, %xmm4
+;   pxor %xmm3, %xmm3
+;   movdqa %xmm0, %xmm6
+;   pcmpeqq %xmm3, %xmm6
+;   ptest %xmm6, %xmm6
 ;   sete %al
 ;   movq %rbp, %rsp
 ;   popq %rbp
diff --git a/cranelift/filetests/filetests/isa/x64/uunarrow.clif b/cranelift/filetests/filetests/isa/x64/uunarrow.clif
index 643b32f8e..7bae44a65 100644
--- a/cranelift/filetests/filetests/isa/x64/uunarrow.clif
+++ b/cranelift/filetests/filetests/isa/x64/uunarrow.clif
@@ -13,13 +13,14 @@ block0(v0: f64x2):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   xorpd   %xmm2, %xmm2, %xmm2
-;   movdqa  %xmm0, %xmm5
-;   maxpd   %xmm5, %xmm2, %xmm5
-;   minpd   %xmm5, const(0), %xmm5
-;   roundpd $3, %xmm5, %xmm0
+;   uninit  %xmm3
+;   xorpd   %xmm3, %xmm3, %xmm3
+;   movdqa  %xmm0, %xmm7
+;   maxpd   %xmm7, %xmm3, %xmm7
+;   minpd   %xmm7, const(0), %xmm7
+;   roundpd $3, %xmm7, %xmm0
 ;   addpd   %xmm0, const(1), %xmm0
-;   shufps  $136, %xmm0, %xmm2, %xmm0
+;   shufps  $136, %xmm0, %xmm3, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -29,13 +30,13 @@ block0(v0: f64x2):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   xorpd %xmm2, %xmm2
-;   movdqa %xmm0, %xmm5
-;   maxpd %xmm2, %xmm5
-;   minpd 0x18(%rip), %xmm5
-;   roundpd $3, %xmm5, %xmm0
+;   xorpd %xmm3, %xmm3
+;   movdqa %xmm0, %xmm7
+;   maxpd %xmm3, %xmm7
+;   minpd 0x18(%rip), %xmm7
+;   roundpd $3, %xmm7, %xmm0
 ;   addpd 0x1a(%rip), %xmm0
-;   shufps $0x88, %xmm2, %xmm0
+;   shufps $0x88, %xmm3, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
diff --git a/cranelift/filetests/filetests/runtests/simd-insertlane.clif b/cranelift/filetests/filetests/runtests/simd-insertlane.clif
index e0965d832..6e0182b1c 100644
--- a/cranelift/filetests/filetests/runtests/simd-insertlane.clif
+++ b/cranelift/filetests/filetests/runtests/simd-insertlane.clif
@@ -4,6 +4,7 @@ target aarch64
 target s390x
 set enable_simd
 target x86_64 has_sse3 has_ssse3 has_sse41
+target x86_64 has_sse3 has_ssse3 has_sse41 has_avx
 
 function %insertlane_15(i8x16, i8) -> i8x16 {
 block0(v0: i8x16, v1: i8):
@@ -32,3 +33,17 @@ block0(v0: i64x2, v1: i64):
     return v2
 }
 ; run: %insertlane_0([1 1], 5000000000) == [5000000000 1]
+
+function %insertlane_0_in_f64x2(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+    v2 = insertlane v0, v1, 0
+    return v2
+}
+; run: %insertlane_0_in_f64x2([0x1.0 0x2.0], 0x3.0) == [0x3.0 0x2.0]
+
+function %insertlane_1_in_f64x2(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+    v2 = insertlane v0, v1, 1
+    return v2
+}
+; run: %insertlane_1_in_f64x2([0x1.0 0x2.0], 0x3.0) == [0x1.0 0x3.0]
diff --git a/cranelift/filetests/filetests/runtests/simd-shuffle.clif b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
index 621eebda6..60b515628 100644
--- a/cranelift/filetests/filetests/runtests/simd-shuffle.clif
+++ b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
@@ -1,9 +1,10 @@
-test interpret
+;; test interpret ;; FIXME(#5915)
 test run
 target aarch64
 target s390x
 set enable_simd
 target x86_64 has_sse3 has_ssse3 has_sse41
+target x86_64 has_sse3 has_ssse3 has_sse41 has_avx
 target x86_64 has_sse3 has_ssse3 has_sse41 has_avx512vl has_avx512vbmi
 
 function %shuffle_i8x16(i8x16, i8x16) -> i8x16 {
@@ -26,3 +27,234 @@ block0(v0: i8x16):
     return v1
 }
 ; run: %shuffle1([0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]) == [8 9 10 11 12 13 14 15 0 1 2 3 4 5 6 7]
+
+function %punpcklbw(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [0 16 1 17 2 18 3 19 4 20 5 21 6 22 7 23]
+    return v2
+}
+; run: %punpcklbw([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [1 17 2 18 3 19 4 20 5 21 6 22 7 23 8 24]
+
+function %punpckhbw(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [8 24 9 25 10 26 11 27 12 28 13 29 14 30 15 31]
+    return v2
+}
+; run: %punpckhbw([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [9 25 10 26 11 27 12 28 13 29 14 30 15 31 16 32]
+
+function %punpcklwd(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 16 17 2 3 18 19 4 5 20 21 6 7 22 23]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %punpcklwd([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [1 9 2 10 3 11 4 12]
+
+function %punpckhwd(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 24 25 10 11 26 27 12 13 28 29 14 15 30 31]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %punpckhwd([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [5 13 6 14 7 15 8 16]
+
+function %pshufd_0022(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 0 1 2 3 8 9 10 11 8 9 10 11]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %pshufd_0022([1 2 3 4], [5 6 7 8]) == [1 1 3 3]
+
+function %pshufd_3120(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [12 13 14 15 4 5 6 7 8 9 10 11 0 1 2 3]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %pshufd_0022([1 2 3 4], [5 6 7 8]) == [4 2 3 1]
+
+function %pshufd_7546(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [28 29 30 31 20 21 22 23 16 17 18 19 24 25 26 27]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %pshufd_0022([1 2 3 4], [5 6 7 8]) == [8 6 5 7]
+
+function %not_pshufd(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %pshufd_0022([1 2 3 4], [5 6 7 8]) == [3 4 5 6]
+
+function %punpckldq(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 16 17 18 19 4 5 6 7 20 21 22 23]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %punpckldq([1 2 3 4], [5 6 7 8]) == [1 5 2 6]
+
+function %punpckhdq(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 24 25 26 27 12 13 14 15 28 29 30 31]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %punpckldq([1 2 3 4], [5 6 7 8]) == [3 7 4 8]
+
+function %punpcklqdq(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23]
+    v5 = bitcast.i64x2 little v4
+    return v5
+}
+; run: %punpcklqdq([1 2], [5 6]) == [1 5]
+
+function %punpckhqdq(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31]
+    v5 = bitcast.i64x2 little v4
+    return v5
+}
+; run: %punpckhqdq([1 2], [5 6]) == [2 6]
+
+function %shufps_0145(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %shufps_0145([1 2 3 4], [5 6 7 8]) == [1 2 5 6]
+
+function %shufps_3277(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [12 13 14 15 8 9 10 11 28 29 30 31 28 29 30 31]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %shufps_0145([1 2 3 4], [5 6 7 8]) == [4 3 8 8]
+
+function %shufps_6500(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [24 25 26 27 20 21 22 23 0 1 2 3 0 1 2 3]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %shufps_0145([1 2 3 4], [5 6 7 8]) == [7 6 1 1]
+
+function %pshuflw_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [6 7 4 5 2 3 0 1 8 9 10 11 12 13 14 15]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshuflw_3210([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [4 3 2 1 5 6 7 8]
+
+function %pshuflw_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [6 7 4 5 6 7 4 5 8 9 10 11 12 13 14 15]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshuflw_3131([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [4 3 4 3 5 6 7 8]
+
+function %pshuflw_rhs_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [22 23 20 21 18 19 16 17 24 25 26 27 28 29 30 31]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshuflw_rhs_3210([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [12 11 10 9 13 14 15 16]
+
+function %pshuflw_rhs_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [22 23 18 19 22 23 18 19 24 25 26 27 28 29 30 31]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshuflw_rhs_3131([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [12 10 12 10 13 14 15 16]
+
+function %pshufhw_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 14 15 12 13 10 11 8 9]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshufhw_3210([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [1 2 3 4 8 7 6 5]
+
+function %pshufhw_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [0 1 2 3 4 5 6 7 14 15 10 11 14 15 10 11]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshufhw_3131([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [1 2 3 4 8 6 8 6]
+
+function %pshufhw_rhs_3210(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [16 17 18 19 20 21 22 23 30 31 28 29 26 27 24 25]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshufhw_rhs_3210([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [9 10 11 12 16 15 14 13]
+
+function %pshufhw_rhs_3131(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [16 17 18 19 20 21 22 23 30 31 26 27 30 31 26 27]
+    v5 = bitcast.i16x8 little v4
+    return v5
+}
+; run: %pshufhw_rhs_3131([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [9 10 11 12 16 14 16 14]
+
+function %shuffle_all_zeros(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+    return v2
+}
+; run: %shuffle_all_zeros([5 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1]) == [5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5]
diff --git a/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat b/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat
index f3ae7c735..8d7eca8c0 100644
--- a/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat
+++ b/cranelift/filetests/filetests/wasm/x64-relaxed-simd-deterministic.wat
@@ -63,19 +63,21 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   xorps   %xmm3, %xmm3, %xmm3
-;;   vmaxps  %xmm0, %xmm3, %xmm5
-;;   vpcmpeqd %xmm3, %xmm3, %xmm7
-;;   vpsrld  %xmm7, $1, %xmm9
-;;   vcvtdq2ps %xmm9, %xmm11
-;;   vcvttps2dq %xmm5, %xmm13
-;;   vsubps  %xmm5, %xmm11, %xmm15
-;;   vcmpps  $2 %xmm11, %xmm15, %xmm1
-;;   vcvttps2dq %xmm15, %xmm3
-;;   vpxor   %xmm3, %xmm1, %xmm5
-;;   pxor    %xmm7, %xmm7, %xmm7
-;;   vpmaxsd %xmm5, %xmm7, %xmm9
-;;   vpaddd  %xmm9, %xmm13, %xmm0
+;;   uninit  %xmm3
+;;   vxorps  %xmm3, %xmm3, %xmm5
+;;   vmaxps  %xmm0, %xmm5, %xmm7
+;;   vpcmpeqd %xmm5, %xmm5, %xmm9
+;;   vpsrld  %xmm9, $1, %xmm11
+;;   vcvtdq2ps %xmm11, %xmm13
+;;   vcvttps2dq %xmm7, %xmm15
+;;   vsubps  %xmm7, %xmm13, %xmm1
+;;   vcmpps  $2 %xmm13, %xmm1, %xmm3
+;;   vcvttps2dq %xmm1, %xmm5
+;;   vpxor   %xmm5, %xmm3, %xmm7
+;;   uninit  %xmm9
+;;   vpxor   %xmm9, %xmm9, %xmm11
+;;   vpmaxsd %xmm7, %xmm11, %xmm13
+;;   vpaddd  %xmm13, %xmm15, %xmm0
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -104,12 +106,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   xorpd   %xmm3, %xmm3, %xmm3
-;;   vmaxpd  %xmm0, %xmm3, %xmm5
-;;   vminpd  %xmm5, const(0), %xmm7
-;;   vroundpd $3, %xmm7, %xmm9
-;;   vaddpd  %xmm9, const(1), %xmm11
-;;   vshufps $136 %xmm11, %xmm3, %xmm0
+;;   uninit  %xmm3
+;;   vxorpd  %xmm3, %xmm3, %xmm5
+;;   vmaxpd  %xmm0, %xmm5, %xmm7
+;;   vminpd  %xmm7, const(0), %xmm9
+;;   vroundpd $3, %xmm9, %xmm11
+;;   vaddpd  %xmm11, const(1), %xmm13
+;;   vshufps $136 %xmm13, %xmm5, %xmm0
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/wasm/x64-relaxed-simd.wat b/cranelift/filetests/filetests/wasm/x64-relaxed-simd.wat
index 43586fcb2..cef3910e4 100644
--- a/cranelift/filetests/filetests/wasm/x64-relaxed-simd.wat
+++ b/cranelift/filetests/filetests/wasm/x64-relaxed-simd.wat
@@ -55,20 +55,22 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   xorps   %xmm6, %xmm6, %xmm6
-;;   movdqa  %xmm0, %xmm10
-;;   maxps   %xmm10, %xmm6, %xmm10
-;;   pcmpeqd %xmm6, %xmm6, %xmm6
-;;   psrld   %xmm6, $1, %xmm6
-;;   cvtdq2ps %xmm6, %xmm14
-;;   cvttps2dq %xmm10, %xmm13
-;;   subps   %xmm10, %xmm14, %xmm10
-;;   cmpps   $2, %xmm14, %xmm10, %xmm14
-;;   cvttps2dq %xmm10, %xmm0
-;;   pxor    %xmm0, %xmm14, %xmm0
-;;   pxor    %xmm7, %xmm7, %xmm7
-;;   pmaxsd  %xmm0, %xmm7, %xmm0
-;;   paddd   %xmm0, %xmm13, %xmm0
+;;   uninit  %xmm7
+;;   xorps   %xmm7, %xmm7, %xmm7
+;;   movdqa  %xmm0, %xmm12
+;;   maxps   %xmm12, %xmm7, %xmm12
+;;   pcmpeqd %xmm7, %xmm7, %xmm7
+;;   psrld   %xmm7, $1, %xmm7
+;;   cvtdq2ps %xmm7, %xmm1
+;;   cvttps2dq %xmm12, %xmm15
+;;   subps   %xmm12, %xmm1, %xmm12
+;;   cmpps   $2, %xmm1, %xmm12, %xmm1
+;;   cvttps2dq %xmm12, %xmm0
+;;   pxor    %xmm0, %xmm1, %xmm0
+;;   uninit  %xmm10
+;;   pxor    %xmm10, %xmm10, %xmm10
+;;   pmaxsd  %xmm0, %xmm10, %xmm0
+;;   paddd   %xmm0, %xmm15, %xmm0
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -94,13 +96,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   xorpd   %xmm3, %xmm3, %xmm3
-;;   movdqa  %xmm0, %xmm6
-;;   maxpd   %xmm6, %xmm3, %xmm6
-;;   minpd   %xmm6, const(0), %xmm6
-;;   roundpd $3, %xmm6, %xmm0
+;;   uninit  %xmm4
+;;   xorpd   %xmm4, %xmm4, %xmm4
+;;   movdqa  %xmm0, %xmm8
+;;   maxpd   %xmm8, %xmm4, %xmm8
+;;   minpd   %xmm8, const(0), %xmm8
+;;   roundpd $3, %xmm8, %xmm0
 ;;   addpd   %xmm0, const(1), %xmm0
-;;   shufps  $136, %xmm0, %xmm3, %xmm0
+;;   shufps  $136, %xmm0, %xmm4, %xmm0
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp