Repository: bytecodealliance/wasmtime
Pull request #5870 (2023-02-23 22:03:34)

Remove module-level code generation tests
We have good coverage for these test cases in the per-isa precise-output tests, and after #5780 we can see both the VCode and disassembled output of those tests. These tests have historically been difficult to keep up to date, see the comment about dumping bytes out to a binary file and running objdump on each test, so removing them will help simplify the process of updating test expectations.
<!--

Please ensure that the following steps are all taken care of before submitting
the PR.

- [ ] This has been discussed in issue #..., or if not, please tell us why
  here.
- [ ] A short description of what this does, why it is needed; if the
  description becomes long, the matter should probably be discussed in an issue
  first.
- [ ] This PR contains test cases, if meaningful.
- [ ] A reviewer from the core maintainer team has been assigned for this PR.
  If you don't know who could review this, please indicate so. The list of
  suggested reviewers on the right can help you.

Please ensure all communication adheres to the [code of
conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
-->



Patch:
diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs
index 7c2428102..afd32dd40 100644
--- a/cranelift/codegen/src/isa/aarch64/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/mod.rs
@@ -239,223 +239,3 @@ pub fn isa_builder(triple: Triple) -> IsaBuilder {
         },
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::cursor::{Cursor, FuncCursor};
-    use crate::dominator_tree::DominatorTree;
-    use crate::flowgraph::ControlFlowGraph;
-    use crate::ir::types::*;
-    use crate::ir::{AbiParam, Function, InstBuilder, JumpTableData, Signature, UserFuncName};
-    use crate::isa::CallConv;
-    use crate::settings;
-    use crate::settings::Configurable;
-    use core::str::FromStr;
-    use target_lexicon::Triple;
-
-    #[test]
-    fn test_compile_function() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-
-        let mut pos = FuncCursor::new(&mut func);
-        pos.insert_block(bb0);
-        let v0 = pos.ins().iconst(I32, 0x1234);
-        let v1 = pos.ins().iadd(arg0, v0);
-        pos.ins().return_(&[v1]);
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = aarch64_settings::Flags::new(&shared_flags, aarch64_settings::builder());
-        let backend = AArch64Backend::new_with_flags(
-            Triple::from_str("aarch64").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let buffer = backend
-            .compile_function(&mut func, &domtree, false)
-            .unwrap()
-            .buffer;
-        let code = buffer.data();
-
-        // To update this comment, write the golden bytes to a file, and run the following command
-        // on it to update:
-        // > aarch64-linux-gnu-objdump -b binary -D <file> -m aarch64
-        //
-        // 0:   52824682        mov     w2, #0x1234                     // #4660
-        // 4:   0b020000        add     w0, w0, w2
-        // 8:   d65f03c0        ret
-
-        let golden = vec![130, 70, 130, 82, 0, 0, 2, 11, 192, 3, 95, 214];
-
-        assert_eq!(code, &golden[..]);
-    }
-
-    #[test]
-    fn test_branch_lowering() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-        let bb1 = func.dfg.make_block();
-        let bb2 = func.dfg.make_block();
-        let bb3 = func.dfg.make_block();
-
-        let mut pos = FuncCursor::new(&mut func);
-        pos.insert_block(bb0);
-        let v0 = pos.ins().iconst(I32, 0x1234);
-        let v1 = pos.ins().iadd(arg0, v0);
-        pos.ins().brif(v1, bb1, &[], bb2, &[]);
-        pos.insert_block(bb1);
-        pos.ins().brif(v1, bb2, &[], bb3, &[]);
-        pos.insert_block(bb2);
-        let v2 = pos.ins().iadd(v1, v0);
-        pos.ins().brif(v2, bb2, &[], bb1, &[]);
-        pos.insert_block(bb3);
-        let v3 = pos.ins().isub(v1, v0);
-        pos.ins().return_(&[v3]);
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = aarch64_settings::Flags::new(&shared_flags, aarch64_settings::builder());
-        let backend = AArch64Backend::new_with_flags(
-            Triple::from_str("aarch64").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let result = backend
-            .compile_function(&mut func, &domtree, /* want_disasm = */ false)
-            .unwrap();
-        let code = result.buffer.data();
-
-        // To update this comment, write the golden bytes to a file, and run the following command
-        // on it to update:
-        // > aarch64-linux-gnu-objdump -b binary -D <file> -m aarch64
-        //
-        //   0:   52824689        mov     w9, #0x1234                     // #4660
-        //   4:   0b09000b        add     w11, w0, w9
-        //   8:   2a0b03ea        mov     w10, w11
-        //   c:   b40000ca        cbz     x10, 0x24
-        //  10:   2a0b03ed        mov     w13, w11
-        //  14:   b500008d        cbnz    x13, 0x24
-        //  18:   5282468e        mov     w14, #0x1234                    // #4660
-        //  1c:   4b0e0160        sub     w0, w11, w14
-        //  20:   d65f03c0        ret
-        //  24:   5282468f        mov     w15, #0x1234                    // #4660
-        //  28:   0b0f0161        add     w1, w11, w15
-        //  2c:   2a0103e0        mov     w0, w1
-        //  30:   b5ffffa0        cbnz    x0, 0x24
-        //  34:   17fffff7        b       0x10
-
-        let golden = vec![
-            137, 70, 130, 82, 11, 0, 9, 11, 234, 3, 11, 42, 202, 0, 0, 180, 237, 3, 11, 42, 141, 0,
-            0, 181, 142, 70, 130, 82, 96, 1, 14, 75, 192, 3, 95, 214, 143, 70, 130, 82, 97, 1, 15,
-            11, 224, 3, 1, 42, 160, 255, 255, 181, 247, 255, 255, 23,
-        ];
-
-        assert_eq!(code, &golden[..]);
-    }
-
-    #[test]
-    fn test_br_table() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-        let bb1 = func.dfg.make_block();
-        let bb2 = func.dfg.make_block();
-        let bb3 = func.dfg.make_block();
-
-        let mut pos = FuncCursor::new(&mut func);
-
-        pos.insert_block(bb0);
-        let jt_data = JumpTableData::new(
-            pos.func.dfg.block_call(bb3, &[]),
-            &[
-                pos.func.dfg.block_call(bb1, &[]),
-                pos.func.dfg.block_call(bb2, &[]),
-            ],
-        );
-        let jt = pos.func.create_jump_table(jt_data);
-        pos.ins().br_table(arg0, jt);
-
-        pos.insert_block(bb1);
-        let v1 = pos.ins().iconst(I32, 1);
-        pos.ins().return_(&[v1]);
-
-        pos.insert_block(bb2);
-        let v2 = pos.ins().iconst(I32, 2);
-        pos.ins().return_(&[v2]);
-
-        pos.insert_block(bb3);
-        let v3 = pos.ins().iconst(I32, 3);
-        pos.ins().return_(&[v3]);
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        shared_flags_builder.set("enable_verifier", "true").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = aarch64_settings::Flags::new(&shared_flags, aarch64_settings::builder());
-        let backend = AArch64Backend::new_with_flags(
-            Triple::from_str("aarch64").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let result = backend
-            .compile_function(&mut func, &domtree, /* want_disasm = */ false)
-            .unwrap();
-        let code = result.buffer.data();
-
-        // To update this comment, write the golden bytes to a file, and run the following command
-        // on it to update:
-        // > aarch64-linux-gnu-objdump -b binary -D <file> -m aarch64
-        //
-        //   0:   7100081f        cmp     w0, #0x2
-        //   4:   540001a2        b.cs    0x38  // b.hs, b.nlast
-        //   8:   9a8023e8        csel    x8, xzr, x0, cs  // cs = hs, nlast
-        //   c:   d503229f        csdb
-        //  10:   10000087        adr     x7, 0x20
-        //  14:   b8a858e8        ldrsw   x8, [x7, w8, uxtw #2]
-        //  18:   8b0800e7        add     x7, x7, x8
-        //  1c:   d61f00e0        br      x7
-        //  20:   00000010        udf     #16
-        //  24:   00000008        udf     #8
-        //  28:   52800040        mov     w0, #0x2                        // #2
-        //  2c:   d65f03c0        ret
-        //  30:   52800020        mov     w0, #0x1                        // #1
-        //  34:   d65f03c0        ret
-        //  38:   52800060        mov     w0, #0x3                        // #3
-        //  3c:   d65f03c0        ret
-
-        let golden = vec![
-            31, 8, 0, 113, 162, 1, 0, 84, 232, 35, 128, 154, 159, 34, 3, 213, 135, 0, 0, 16, 232,
-            88, 168, 184, 231, 0, 8, 139, 224, 0, 31, 214, 16, 0, 0, 0, 8, 0, 0, 0, 64, 0, 128, 82,
-            192, 3, 95, 214, 32, 0, 128, 82, 192, 3, 95, 214, 96, 0, 128, 82, 192, 3, 95, 214,
-        ];
-
-        assert_eq!(code, &golden[..]);
-    }
-}
diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs
index 1a068a14d..53d903481 100644
--- a/cranelift/codegen/src/isa/riscv64/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/mod.rs
@@ -214,64 +214,3 @@ pub fn isa_builder(triple: Triple) -> IsaBuilder {
         },
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::cursor::{Cursor, FuncCursor};
-    use crate::dominator_tree::DominatorTree;
-    use crate::flowgraph::ControlFlowGraph;
-    use crate::ir::types::*;
-    use crate::ir::{AbiParam, Function, InstBuilder, Signature, UserFuncName};
-    use crate::isa::CallConv;
-    use crate::settings;
-    use crate::settings::Configurable;
-    use core::str::FromStr;
-    use target_lexicon::Triple;
-
-    #[test]
-    fn test_compile_function() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-
-        let mut pos = FuncCursor::new(&mut func);
-        pos.insert_block(bb0);
-        let v0 = pos.ins().iconst(I32, 0x1234);
-        let v1 = pos.ins().iadd(arg0, v0);
-        pos.ins().return_(&[v1]);
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = riscv_settings::Flags::new(&shared_flags, riscv_settings::builder());
-        let backend = Riscv64Backend::new_with_flags(
-            Triple::from_str("riscv64").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let buffer = backend.compile_function(&mut func, &domtree, true).unwrap();
-        let code = buffer.buffer.data();
-
-        // To update this comment, write the golden bytes to a file, and run the following command
-        // on it to update:
-        // > riscv64-linux-gnu-objdump -b binary -D <file> -m riscv
-        //
-        // 0:   000015b7                lui     a1,0x1
-        // 4:   23458593                addi    a1,a1,564 # 0x1234
-        // 8:   00b5053b                .4byte  0xb5053b
-        // c:   00008067                ret
-
-        let golden = vec![
-            183, 21, 0, 0, 147, 133, 69, 35, 59, 5, 181, 0, 103, 128, 0, 0,
-        ];
-        assert_eq!(code, &golden[..]);
-    }
-}
diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs
index 893e30ec1..6a6dad94c 100644
--- a/cranelift/codegen/src/isa/s390x/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/mod.rs
@@ -211,130 +211,3 @@ pub fn isa_builder(triple: Triple) -> IsaBuilder {
         },
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::cursor::{Cursor, FuncCursor};
-    use crate::dominator_tree::DominatorTree;
-    use crate::flowgraph::ControlFlowGraph;
-    use crate::ir::types::*;
-    use crate::ir::UserFuncName;
-    use crate::ir::{AbiParam, Function, InstBuilder, Signature};
-    use crate::isa::CallConv;
-    use crate::settings;
-    use crate::settings::Configurable;
-    use core::str::FromStr;
-    use target_lexicon::Triple;
-
-    #[test]
-    fn test_compile_function() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-
-        let mut pos = FuncCursor::new(&mut func);
-        pos.insert_block(bb0);
-        let v0 = pos.ins().iconst(I32, 0x1234);
-        let v1 = pos.ins().iadd(arg0, v0);
-        pos.ins().return_(&[v1]);
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = s390x_settings::Flags::new(&shared_flags, s390x_settings::builder());
-        let backend = S390xBackend::new_with_flags(
-            Triple::from_str("s390x").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let result = backend
-            .compile_function(&mut func, &domtree, /* want_disasm = */ false)
-            .unwrap();
-        let code = result.buffer.data();
-
-        // ahi %r2, 0x1234
-        // br %r14
-        let golden = vec![0xa7, 0x2a, 0x12, 0x34, 0x07, 0xfe];
-
-        assert_eq!(code, &golden[..]);
-    }
-
-    #[test]
-    fn test_branch_lowering() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-        let bb1 = func.dfg.make_block();
-        let bb2 = func.dfg.make_block();
-        let bb3 = func.dfg.make_block();
-
-        let mut pos = FuncCursor::new(&mut func);
-        pos.insert_block(bb0);
-        let v0 = pos.ins().iconst(I32, 0x1234);
-        let v1 = pos.ins().iadd(arg0, v0);
-        pos.ins().brif(v1, bb1, &[], bb2, &[]);
-        pos.insert_block(bb1);
-        pos.ins().brif(v1, bb2, &[], bb3, &[]);
-        pos.insert_block(bb2);
-        let v2 = pos.ins().iadd(v1, v0);
-        pos.ins().brif(v2, bb2, &[], bb1, &[]);
-        pos.insert_block(bb3);
-        let v3 = pos.ins().isub(v1, v0);
-        pos.ins().return_(&[v3]);
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = s390x_settings::Flags::new(&shared_flags, s390x_settings::builder());
-        let backend = S390xBackend::new_with_flags(
-            Triple::from_str("s390x").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let result = backend
-            .compile_function(&mut func, &domtree, /* want_disasm = */ false)
-            .unwrap();
-        let code = result.buffer.data();
-
-        // FIXME: the branching logic should be optimized more
-
-        // To update this comment, write the golden bytes to a file, and run the following command
-        // on it to update:
-        // > s390x-linux-gnu-objdump -b binary -D <file> -m s390
-        //
-        //  0:   a7 2a 12 34             ahi     %r2,4660
-        //  4:   a7 2e 00 00             chi     %r2,0
-        //  8:   c0 94 00 00 00 0b       jgnlh   0x1e
-        //  e:   a7 2e 00 00             chi     %r2,0
-        // 12:   c0 64 00 00 00 06       jglh    0x1e
-        // 18:   a7 2a ed cc             ahi     %r2,-4660
-        // 1c:   07 fe                   br      %r14
-        // 1e:   ec 32 12 34 00 d8       ahik    %r3,%r2,4660
-        // 24:   a7 3e 00 00             chi     %r3,0
-        // 28:   c0 64 ff ff ff fb       jglh    0x1e
-        // 2e:   c0 f4 ff ff ff f0       jg      0xe
-
-        let golden = vec![
-            167, 42, 18, 52, 167, 46, 0, 0, 192, 148, 0, 0, 0, 11, 167, 46, 0, 0, 192, 100, 0, 0,
-            0, 6, 167, 42, 237, 204, 7, 254, 236, 50, 18, 52, 0, 216, 167, 62, 0, 0, 192, 100, 255,
-            255, 255, 251, 192, 244, 255, 255, 255, 240,
-        ];
-
-        assert_eq!(code, &golden[..]);
-    }
-}
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 954d0c364..68f3c566a 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -283,6 +283,17 @@
                        (mask Xmm)
                        (dst WritableXmm))
 
+       ;; XMM unary op using a VEX encoding (aka AVX).
+       (XmmUnaryRmRVex (op AvxOpcode)
+                       (src XmmMem)
+                       (dst WritableXmm))
+
+       ;; XMM unary op using a VEX encoding (aka AVX) with an immediate.
+       (XmmUnaryRmRImmVex (op AvxOpcode)
+                          (src XmmMem)
+                          (dst WritableXmm)
+                          (imm u8))
+
        ;; XMM (scalar or vector) binary op that relies on the EVEX
        ;; prefix. Takes two inputs.
        (XmmRmREvex (op Avx512Opcode)
@@ -1314,6 +1325,37 @@
             Vpsllq
             Vpsraw
             Vpsrad
+            Vpmovsxbw
+            Vpmovzxbw
+            Vpmovsxwd
+            Vpmovzxwd
+            Vpmovsxdq
+            Vpmovzxdq
+            Vaddss
+            Vaddsd
+            Vmulss
+            Vmulsd
+            Vsubss
+            Vsubsd
+            Vdivss
+            Vdivsd
+            Vpabsb
+            Vpabsw
+            Vpabsd
+            Vminss
+            Vminsd
+            Vmaxss
+            Vmaxsd
+            Vsqrtps
+            Vsqrtpd
+            Vroundps
+            Vroundpd
+            Vcvtdq2pd
+            Vcvtdq2ps
+            Vcvtpd2ps
+            Vcvtps2pd
+            Vcvttpd2dq
+            Vcvttps2dq
           ))
 
 (type Avx512Opcode extern
@@ -1902,33 +1944,47 @@
 (rule (x64_movdqu from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Movdqu) from))
 
-(decl x64_movapd (XmmMem) Xmm)
-(rule (x64_movapd src)
-      (xmm_unary_rm_r (SseOpcode.Movapd) src))
-
 (decl x64_pmovsxbw (XmmMem) Xmm)
 (rule (x64_pmovsxbw from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Pmovsxbw) from))
+(rule 1 (x64_pmovsxbw from)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpmovsxbw) from))
 
 (decl x64_pmovzxbw (XmmMem) Xmm)
 (rule (x64_pmovzxbw from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Pmovzxbw) from))
+(rule 1 (x64_pmovzxbw from)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpmovzxbw) from))
 
 (decl x64_pmovsxwd (XmmMem) Xmm)
 (rule (x64_pmovsxwd from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Pmovsxwd) from))
+(rule 1 (x64_pmovsxwd from)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpmovsxwd) from))
 
 (decl x64_pmovzxwd (XmmMem) Xmm)
 (rule (x64_pmovzxwd from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Pmovzxwd) from))
+(rule 1 (x64_pmovzxwd from)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpmovzxwd) from))
 
 (decl x64_pmovsxdq (XmmMem) Xmm)
 (rule (x64_pmovsxdq from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Pmovsxdq) from))
+(rule 1 (x64_pmovsxdq from)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpmovsxdq) from))
 
 (decl x64_pmovzxdq (XmmMem) Xmm)
 (rule (x64_pmovzxdq from)
       (xmm_unary_rm_r_unaligned (SseOpcode.Pmovzxdq) from))
+(rule 1 (x64_pmovzxdq from)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpmovzxdq) from))
 
 (decl x64_movrm (Type SyntheticAmode Gpr) SideEffectNoResult)
 (rule (x64_movrm ty addr data)
@@ -2702,11 +2758,17 @@
 (decl x64_addss (Xmm XmmMem) Xmm)
 (rule (x64_addss src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Addss) src1 src2))
+(rule 1 (x64_addss src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vaddss) src1 src2))
 
 ;; Helper for creating `addsd` instructions.
 (decl x64_addsd (Xmm XmmMem) Xmm)
 (rule (x64_addsd src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Addsd) src1 src2))
+(rule 1 (x64_addsd src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vaddsd) src1 src2))
 
 ;; Helper for creating `addps` instructions.
 (decl x64_addps (Xmm XmmMem) Xmm)
@@ -2728,11 +2790,17 @@
 (decl x64_subss (Xmm XmmMem) Xmm)
 (rule (x64_subss src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Subss) src1 src2))
+(rule 1 (x64_subss src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vsubss) src1 src2))
 
 ;; Helper for creating `subsd` instructions.
 (decl x64_subsd (Xmm XmmMem) Xmm)
 (rule (x64_subsd src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Subsd) src1 src2))
+(rule 1 (x64_subsd src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vsubsd) src1 src2))
 
 ;; Helper for creating `subps` instructions.
 (decl x64_subps (Xmm XmmMem) Xmm)
@@ -2754,11 +2822,17 @@
 (decl x64_mulss (Xmm XmmMem) Xmm)
 (rule (x64_mulss src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Mulss) src1 src2))
+(rule 1 (x64_mulss src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vmulss) src1 src2))
 
 ;; Helper for creating `mulsd` instructions.
 (decl x64_mulsd (Xmm XmmMem) Xmm)
 (rule (x64_mulsd src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Mulsd) src1 src2))
+(rule 1 (x64_mulsd src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vmulsd) src1 src2))
 
 ;; Helper for creating `mulps` instructions.
 (decl x64_mulps (Xmm XmmMem) Xmm)
@@ -2780,11 +2854,17 @@
 (decl x64_divss (Xmm XmmMem) Xmm)
 (rule (x64_divss src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Divss) src1 src2))
+(rule 1 (x64_divss src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vdivss) src1 src2))
 
 ;; Helper for creating `divsd` instructions.
 (decl x64_divsd (Xmm XmmMem) Xmm)
 (rule (x64_divsd src1 src2)
       (xmm_rm_r_unaligned (SseOpcode.Divsd) src1 src2))
+(rule 1 (x64_divsd src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vdivsd) src1 src2))
 
 ;; Helper for creating `divps` instructions.
 (decl x64_divps (Xmm XmmMem) Xmm)
@@ -2816,6 +2896,20 @@
             (_ Unit (emit (MInst.XmmRmRBlendVex op src1 src2 mask dst))))
         dst))
 
+;; Helper for creating `XmmUnaryRmRVex` instructions
+(decl xmm_unary_rm_r_vex (AvxOpcode XmmMem) Xmm)
+(rule (xmm_unary_rm_r_vex op src)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmUnaryRmRVex op src dst))))
+        dst))
+
+;; Helper for creating `XmmUnaryRmRImmVex` instructions
+(decl xmm_unary_rm_r_imm_vex (AvxOpcode XmmMem u8) Xmm)
+(rule (xmm_unary_rm_r_imm_vex op src imm)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmUnaryRmRImmVex op src dst imm))))
+        dst))
+
 ;; Helper for creating `blendvp{d,s}` and `pblendvb` instructions.
 (decl x64_blend (Type Xmm XmmMem Xmm) Xmm)
 (rule 1 (x64_blend $F32X4 mask src1 src2) (x64_blendvps src2 src1 mask))
@@ -3131,11 +3225,17 @@
 (decl x64_roundps (XmmMem RoundImm) Xmm)
 (rule (x64_roundps src1 round)
       (xmm_unary_rm_r_imm (SseOpcode.Roundps) src1 (encode_round_imm round)))
+(rule 1 (x64_roundps src1 round)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_imm_vex (AvxOpcode.Vroundps) src1 (encode_round_imm round)))
 
 ;; Helper for creating `roundpd` instructions.
 (decl x64_roundpd (XmmMem RoundImm) Xmm)
 (rule (x64_roundpd src1 round)
       (xmm_unary_rm_r_imm (SseOpcode.Roundpd) src1 (encode_round_imm round)))
+(rule 1 (x64_roundpd src1 round)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_imm_vex (AvxOpcode.Vroundpd) src1 (encode_round_imm round)))
 
 ;; Helper for creating `pmaddwd` instructions.
 (decl x64_pmaddwd (Xmm XmmMem) Xmm)
@@ -3207,16 +3307,25 @@
 (decl x64_pabsb (XmmMem) Xmm)
 (rule (x64_pabsb src)
       (xmm_unary_rm_r (SseOpcode.Pabsb) src))
+(rule 1 (x64_pabsb src)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpabsb) src))
 
 ;; Helper for creating `pabsw` instructions.
 (decl x64_pabsw (XmmMem) Xmm)
 (rule (x64_pabsw src)
       (xmm_unary_rm_r (SseOpcode.Pabsw) src))
+(rule 1 (x64_pabsw src)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpabsw) src))
 
 ;; Helper for creating `pabsd` instructions.
 (decl x64_pabsd (XmmMem) Xmm)
 (rule (x64_pabsd src)
       (xmm_unary_rm_r (SseOpcode.Pabsd) src))
+(rule 1 (x64_pabsd src)
+      (if-let $true (has_avx))
+      (xmm_unary_rm_r_vex (AvxOpcode.Vpabsd) src))
 
 ;; Helper for creating `MInst.XmmUnaryRmREvex` instructions.
 (decl xmm_unary_rm_r_evex (Avx512Opcode XmmMem) Xmm)
@@ -3540,11 +3649,17 @@
 (decl x64_minss (Xmm XmmMem) Xmm)
 (rule (x64_minss x y)
       (xmm_rm_r_unaligned (SseOpcode.Minss) x y))
+(rule 1 (x64_minss x y)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vminss) x y))
 
 ;; Helper for creating `minsd` instructions.
 (decl x64_minsd (Xmm XmmMem) Xmm)
 (rule (x64_minsd x y)
       (xmm_rm_r_unaligned (SseOpcode.Minsd) x y))
+(rule 1 (x64_minsd x y)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vminsd) x y))
 
 ;; Helper for creating `minps` instructions.
 (decl x64_minps (Xmm XmmMem) Xmm)
@@ -3566,11 +3681,17 @@
 (decl x64_maxss (Xmm XmmMem) Xmm)
 (rule (x64_maxss x y)
       (xmm_rm_r_unaligned (SseOpcode.Maxss) x y))
+(rule 1 (x64_maxss x y)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vmaxss) x y))
 
 ;; Helper for creating `maxsd` instructions.
 (decl x64_maxsd (Xmm XmmMem) Xmm)
 (rule (x64_maxsd x y)
       (xmm_rm_r_unaligned (SseOpcode.Maxsd) x y))
+(rule 1 (x64_maxsd x y)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vmaxsd) x y))
 
 ;; Helper for creating `maxps` instructions.
 (decl x64_maxps (Xmm XmmMem) Xmm)
@@ -3649,10 +3770,16 @@
 ;; Helper for creating `sqrtps` instructions.
 (decl x64_sqrtps (XmmMem) Xmm)
 (rule (x64_sqrtps x) (xmm_unary_rm_r (SseOpcode.Sqrtps) x))
+(rule 1 (x64_sqrtps x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vsqrtps) x))
 
 ;; Helper for creating `sqrtpd` instructions.
 (decl x64_sqrtpd (XmmMem) Xmm)
 (rule (x64_sqrtpd x) (xmm_unary_rm_r (SseOpcode.Sqrtpd) x))
+(rule 1 (x64_sqrtpd x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vsqrtpd) x))
 
 ;; Helper for creating `cvtss2sd` instructions.
 (decl x64_cvtss2sd (Xmm) Xmm)
@@ -3665,18 +3792,30 @@
 ;; Helper for creating `cvtdq2ps` instructions.
 (decl x64_cvtdq2ps (XmmMem) Xmm)
 (rule (x64_cvtdq2ps x) (xmm_unary_rm_r (SseOpcode.Cvtdq2ps) x))
+(rule 1 (x64_cvtdq2ps x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vcvtdq2ps) x))
 
 ;; Helper for creating `cvtps2pd` instructions.
 (decl x64_cvtps2pd (XmmMem) Xmm)
 (rule (x64_cvtps2pd x) (xmm_unary_rm_r (SseOpcode.Cvtps2pd) x))
+(rule 1 (x64_cvtps2pd x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vcvtps2pd) x))
 
 ;; Helper for creating `cvtpd2ps` instructions.
 (decl x64_cvtpd2ps (XmmMem) Xmm)
 (rule (x64_cvtpd2ps x) (xmm_unary_rm_r (SseOpcode.Cvtpd2ps) x))
+(rule 1 (x64_cvtpd2ps x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vcvtpd2ps) x))
 
 ;; Helper for creating `cvtdq2pd` instructions.
 (decl x64_cvtdq2pd (XmmMem) Xmm)
 (rule (x64_cvtdq2pd x) (xmm_unary_rm_r (SseOpcode.Cvtdq2pd) x))
+(rule 1 (x64_cvtdq2pd x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vcvtdq2pd) x))
 
 ;; Helper for creating `cvtsi2ss` instructions.
 (decl x64_cvtsi2ss (Type GprMem) Xmm)
@@ -3692,11 +3831,17 @@
 (decl x64_cvttps2dq (XmmMem) Xmm)
 (rule (x64_cvttps2dq x)
       (xmm_unary_rm_r (SseOpcode.Cvttps2dq) x))
+(rule 1 (x64_cvttps2dq x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vcvttps2dq) x))
 
 ;; Helper for creating `cvttpd2dq` instructions.
 (decl x64_cvttpd2dq (XmmMem) Xmm)
 (rule (x64_cvttpd2dq x)
       (xmm_unary_rm_r (SseOpcode.Cvttpd2dq) x))
+(rule 1 (x64_cvttpd2dq x)
+        (if-let $true (has_avx))
+        (xmm_unary_rm_r_vex (AvxOpcode.Vcvttpd2dq) x))
 
 (decl cvt_u64_to_float_seq (Type Gpr) Xmm)
 (rule (cvt_u64_to_float_seq ty src)
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index b85fdfc20..772727133 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -1630,7 +1630,38 @@ impl AvxOpcode {
             | AvxOpcode::Vpslld
             | AvxOpcode::Vpsllq
             | AvxOpcode::Vpsraw
-            | AvxOpcode::Vpsrad => {
+            | AvxOpcode::Vpsrad
+            | AvxOpcode::Vpmovsxbw
+            | AvxOpcode::Vpmovzxbw
+            | AvxOpcode::Vpmovsxwd
+            | AvxOpcode::Vpmovzxwd
+            | AvxOpcode::Vpmovsxdq
+            | AvxOpcode::Vpmovzxdq
+            | AvxOpcode::Vaddss
+            | AvxOpcode::Vaddsd
+            | AvxOpcode::Vmulss
+            | AvxOpcode::Vmulsd
+            | AvxOpcode::Vsubss
+            | AvxOpcode::Vsubsd
+            | AvxOpcode::Vdivss
+            | AvxOpcode::Vdivsd
+            | AvxOpcode::Vpabsb
+            | AvxOpcode::Vpabsw
+            | AvxOpcode::Vpabsd
+            | AvxOpcode::Vminss
+            | AvxOpcode::Vminsd
+            | AvxOpcode::Vmaxss
+            | AvxOpcode::Vmaxsd
+            | AvxOpcode::Vsqrtps
+            | AvxOpcode::Vsqrtpd
+            | AvxOpcode::Vroundpd
+            | AvxOpcode::Vroundps
+            | AvxOpcode::Vcvtdq2pd
+            | AvxOpcode::Vcvtdq2ps
+            | AvxOpcode::Vcvtpd2ps
+            | AvxOpcode::Vcvtps2pd
+            | AvxOpcode::Vcvttpd2dq
+            | AvxOpcode::Vcvttps2dq => {
                 smallvec![InstructionSet::AVX]
             }
         }
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 44de9450f..3674c6c29 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -2182,6 +2182,18 @@ pub(crate) fn emit(
                 AvxOpcode::Vpsllq => (LP::_66, OM::_0F, 0xF3),
                 AvxOpcode::Vpsraw => (LP::_66, OM::_0F, 0xE1),
                 AvxOpcode::Vpsrad => (LP::_66, OM::_0F, 0xE2),
+                AvxOpcode::Vaddss => (LP::_F3, OM::_0F, 0x58),
+                AvxOpcode::Vaddsd => (LP::_F2, OM::_0F, 0x58),
+                AvxOpcode::Vmulss => (LP::_F3, OM::_0F, 0x59),
+                AvxOpcode::Vmulsd => (LP::_F2, OM::_0F, 0x59),
+                AvxOpcode::Vsubss => (LP::_F3, OM::_0F, 0x5C),
+                AvxOpcode::Vsubsd => (LP::_F2, OM::_0F, 0x5C),
+                AvxOpcode::Vdivss => (LP::_F3, OM::_0F, 0x5E),
+                AvxOpcode::Vdivsd => (LP::_F2, OM::_0F, 0x5E),
+                AvxOpcode::Vminss => (LP::_F3, OM::_0F, 0x5D),
+                AvxOpcode::Vminsd => (LP::_F2, OM::_0F, 0x5D),
+                AvxOpcode::Vmaxss => (LP::_F3, OM::_0F, 0x5F),
+                AvxOpcode::Vmaxsd => (LP::_F2, OM::_0F, 0x5F),
                 _ => panic!("unexpected rmir vex opcode {op:?}"),
             };
             VexInstruction::new()
@@ -2359,6 +2371,72 @@ pub(crate) fn emit(
                 .encode(sink);
         }
 
+        Inst::XmmUnaryRmRVex { op, src, dst } => {
+            let dst = allocs.next(dst.to_reg().to_reg());
+            let src = match src.clone().to_reg_mem().with_allocs(allocs) {
+                RegMem::Reg { reg } => {
+                    RegisterOrAmode::Register(reg.to_real_reg().unwrap().hw_enc().into())
+                }
+                RegMem::Mem { addr } => RegisterOrAmode::Amode(addr.finalize(state, sink)),
+            };
+
+            let (prefix, map, opcode) = match op {
+                AvxOpcode::Vpmovsxbw => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x20),
+                AvxOpcode::Vpmovzxbw => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x30),
+                AvxOpcode::Vpmovsxwd => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x23),
+                AvxOpcode::Vpmovzxwd => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x33),
+                AvxOpcode::Vpmovsxdq => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x25),
+                AvxOpcode::Vpmovzxdq => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x35),
+                AvxOpcode::Vpabsb => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x1C),
+                AvxOpcode::Vpabsw => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x1D),
+                AvxOpcode::Vpabsd => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x1E),
+                AvxOpcode::Vsqrtps => (LegacyPrefixes::None, OpcodeMap::_0F, 0x51),
+                AvxOpcode::Vsqrtpd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x51),
+                AvxOpcode::Vcvtdq2pd => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0xE6),
+                AvxOpcode::Vcvtdq2ps => (LegacyPrefixes::None, OpcodeMap::_0F, 0x5B),
+                AvxOpcode::Vcvtpd2ps => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x5A),
+                AvxOpcode::Vcvtps2pd => (LegacyPrefixes::None, OpcodeMap::_0F, 0x5A),
+                AvxOpcode::Vcvttpd2dq => (LegacyPrefixes::_66, OpcodeMap::_0F, 0xE6),
+                AvxOpcode::Vcvttps2dq => (LegacyPrefixes::_F3, OpcodeMap::_0F, 0x5B),
+                _ => panic!("unexpected rmr_imm_vex opcode {op:?}"),
+            };
+
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .prefix(prefix)
+                .map(map)
+                .opcode(opcode)
+                .reg(dst.to_real_reg().unwrap().hw_enc())
+                .rm(src)
+                .encode(sink);
+        }
+
+        Inst::XmmUnaryRmRImmVex { op, src, dst, imm } => {
+            let dst = allocs.next(dst.to_reg().to_reg());
+            let src = match src.clone().to_reg_mem().with_allocs(allocs) {
+                RegMem::Reg { reg } => {
+                    RegisterOrAmode::Register(reg.to_real_reg().unwrap().hw_enc().into())
+                }
+                RegMem::Mem { addr } => RegisterOrAmode::Amode(addr.finalize(state, sink)),
+            };
+
+            let (prefix, map, opcode) = match op {
+                AvxOpcode::Vroundps => (LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x08),
+                AvxOpcode::Vroundpd => (LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x09),
+                _ => panic!("unexpected rmr_imm_vex opcode {op:?}"),
+            };
+
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .prefix(prefix)
+                .map(map)
+                .opcode(opcode)
+                .reg(dst.to_real_reg().unwrap().hw_enc())
+                .rm(src)
+                .imm(*imm)
+                .encode(sink);
+        }
+
         Inst::XmmRmREvex {
             op,
             src1,
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 1a5ca8354..4e15b2aea 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -151,7 +151,9 @@ impl Inst {
             | Inst::XmmRmRVex3 { op, .. }
             | Inst::XmmRmRImmVex { op, .. }
             | Inst::XmmRmRBlendVex { op, .. }
-            | Inst::XmmVexPinsr { op, .. } => op.available_from(),
+            | Inst::XmmVexPinsr { op, .. }
+            | Inst::XmmUnaryRmRVex { op, .. }
+            | Inst::XmmUnaryRmRImmVex { op, .. } => op.available_from(),
         }
     }
 }
@@ -910,6 +912,20 @@ impl PrettyPrint for Inst {
                 format!("{} ${}, {}, {}", ljustify(op.to_string()), imm, src, dst)
             }
 
+            Inst::XmmUnaryRmRVex { op, src, dst, .. } => {
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
+                let src = src.pretty_print(8, allocs);
+                format!("{} {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
+            Inst::XmmUnaryRmRImmVex {
+                op, src, dst, imm, ..
+            } => {
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
+                let src = src.pretty_print(8, allocs);
+                format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
+            }
+
             Inst::XmmUnaryRmREvex { op, src, dst, .. } => {
                 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
                 let src = src.pretty_print(8, allocs);
@@ -1887,7 +1903,10 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_def(dst.to_writable_reg());
             src.get_operands(collector);
         }
-        Inst::XmmUnaryRmREvex { src, dst, .. } | Inst::XmmUnaryRmRUnaligned { src, dst, .. } => {
+        Inst::XmmUnaryRmREvex { src, dst, .. }
+        | Inst::XmmUnaryRmRUnaligned { src, dst, .. }
+        | Inst::XmmUnaryRmRVex { src, dst, .. }
+        | Inst::XmmUnaryRmRImmVex { src, dst, .. } => {
             collector.reg_def(dst.to_writable_reg());
             src.get_operands(collector);
         }
diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs
index 67c3475f7..7c085ad58 100644
--- a/cranelift/codegen/src/isa/x64/mod.rs
+++ b/cranelift/codegen/src/isa/x64/mod.rs
@@ -233,161 +233,8 @@ fn isa_constructor(
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::cursor::{Cursor, FuncCursor};
-    use crate::dominator_tree::DominatorTree;
-    use crate::flowgraph::ControlFlowGraph;
-    use crate::ir::{types::*, RelSourceLoc, SourceLoc, UserFuncName, ValueLabel, ValueLabelStart};
-    use crate::ir::{AbiParam, Function, InstBuilder, JumpTableData, Signature};
-    use crate::isa::CallConv;
     use crate::settings;
     use crate::settings::Configurable;
-    use core::str::FromStr;
-    use cranelift_entity::EntityRef;
-    use target_lexicon::Triple;
-
-    /// We have to test cold blocks by observing final machine code,
-    /// rather than VCode, because the VCode orders blocks in lowering
-    /// order, not emission order. (The exact difference between the
-    /// two is that cold blocks are sunk in the latter.) We might as
-    /// well do the test here, where we have a backend to use.
-    #[test]
-    fn test_cold_blocks() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-        // Add debug info: this tests the debug machinery wrt cold
-        // blocks as well.
-        func.dfg.collect_debug_info();
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-        let bb1 = func.dfg.make_block();
-        let bb2 = func.dfg.make_block();
-        let bb3 = func.dfg.make_block();
-        let bb1_param = func.dfg.append_block_param(bb1, I32);
-        let bb3_param = func.dfg.append_block_param(bb3, I32);
-
-        let mut pos = FuncCursor::new(&mut func);
-
-        pos.insert_block(bb0);
-        pos.set_srcloc(SourceLoc::new(1));
-        let v0 = pos.ins().iconst(I32, 0x1234);
-        pos.set_srcloc(SourceLoc::new(2));
-        let v1 = pos.ins().iadd(arg0, v0);
-        pos.ins().brif(v1, bb1, &[v1], bb2, &[]);
-
-        pos.insert_block(bb1);
-        pos.set_srcloc(SourceLoc::new(3));
-        let v2 = pos.ins().isub(v1, v0);
-        pos.set_srcloc(SourceLoc::new(4));
-        let v3 = pos.ins().iadd(v2, bb1_param);
-        pos.ins().brif(v1, bb2, &[], bb3, &[v3]);
-
-        pos.func.layout.set_cold(bb2);
-        pos.insert_block(bb2);
-        pos.set_srcloc(SourceLoc::new(5));
-        let v4 = pos.ins().iadd(v1, v0);
-        pos.ins().brif(v4, bb2, &[], bb1, &[v4]);
-
-        pos.insert_block(bb3);
-        pos.set_srcloc(SourceLoc::new(6));
-        pos.ins().return_(&[bb3_param]);
-
-        // Create some debug info. Make one label that follows all the
-        // values around. Note that this is usually done via an API on
-        // the FunctionBuilder, but that's in cranelift_frontend
-        // (i.e., a higher level of the crate DAG) so we have to build
-        // it manually here.
-        pos.func.dfg.values_labels.as_mut().unwrap().insert(
-            v0,
-            crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart {
-                from: RelSourceLoc::new(1),
-                label: ValueLabel::new(1),
-            }]),
-        );
-        pos.func.dfg.values_labels.as_mut().unwrap().insert(
-            v1,
-            crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart {
-                from: RelSourceLoc::new(2),
-                label: ValueLabel::new(1),
-            }]),
-        );
-        pos.func.dfg.values_labels.as_mut().unwrap().insert(
-            v2,
-            crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart {
-                from: RelSourceLoc::new(3),
-                label: ValueLabel::new(1),
-            }]),
-        );
-        pos.func.dfg.values_labels.as_mut().unwrap().insert(
-            v3,
-            crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart {
-                from: RelSourceLoc::new(4),
-                label: ValueLabel::new(1),
-            }]),
-        );
-        pos.func.dfg.values_labels.as_mut().unwrap().insert(
-            v4,
-            crate::ir::ValueLabelAssignments::Starts(vec![ValueLabelStart {
-                from: RelSourceLoc::new(5),
-                label: ValueLabel::new(1),
-            }]),
-        );
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        shared_flags_builder.set("enable_verifier", "true").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = x64_settings::Flags::new(&shared_flags, x64_settings::builder());
-        let backend = X64Backend::new_with_flags(
-            Triple::from_str("x86_64").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let result = backend
-            .compile_function(&mut func, &domtree, /* want_disasm = */ false)
-            .unwrap();
-        let code = result.buffer.data();
-
-        // To update this comment, write the golden bytes to a file, and run the following
-        // command on it:
-        // > objdump -b binary -D <file> -m i386:x86-64 -M intel
-        //
-        //  0:   55                      push   rbp
-        //  1:   48 89 e5                mov    rbp,rsp
-        //  4:   48 89 fe                mov    rsi,rdi
-        //  7:   81 c6 34 12 00 00       add    esi,0x1234
-        //  d:   85 f6                   test   esi,esi
-        //  f:   0f 84 21 00 00 00       je     0x36
-        // 15:   49 89 f0                mov    r8,rsi
-        // 18:   48 89 f0                mov    rax,rsi
-        // 1b:   81 e8 34 12 00 00       sub    eax,0x1234
-        // 21:   44 01 c0                add    eax,r8d
-        // 24:   85 f6                   test   esi,esi
-        // 26:   0f 85 0a 00 00 00       jne    0x36
-        // 2c:   48 89 ec                mov    rsp,rbp
-        // 2f:   5d                      pop    rbp
-        // 30:   c3                      ret
-        // 31:   e9 e2 ff ff ff          jmp    0x18
-        // 36:   49 89 f0                mov    r8,rsi
-        // 39:   41 81 c0 34 12 00 00    add    r8d,0x1234
-        // 40:   45 85 c0                test   r8d,r8d
-        // 43:   0f 84 cf ff ff ff       je     0x18
-        // 49:   e9 e8 ff ff ff          jmp    0x36
-
-        let golden = vec![
-            85, 72, 137, 229, 72, 137, 254, 129, 198, 52, 18, 0, 0, 133, 246, 15, 132, 33, 0, 0, 0,
-            73, 137, 240, 72, 137, 240, 129, 232, 52, 18, 0, 0, 68, 1, 192, 133, 246, 15, 133, 10,
-            0, 0, 0, 72, 137, 236, 93, 195, 233, 226, 255, 255, 255, 73, 137, 240, 65, 129, 192,
-            52, 18, 0, 0, 69, 133, 192, 15, 132, 207, 255, 255, 255, 233, 232, 255, 255, 255,
-        ];
-
-        assert_eq!(code, &golden[..]);
-    }
 
     // Check that feature tests for SIMD work correctly.
     #[test]
@@ -405,105 +252,4 @@ mod test {
             Err(CodegenError::Unsupported(_)),
         ));
     }
-
-    // Check that br_table lowers properly. We can't test this with an
-    // ordinary compile-test because the br_table pseudoinstruction
-    // expands during emission.
-    #[test]
-    fn br_table() {
-        let name = UserFuncName::testcase("test0");
-        let mut sig = Signature::new(CallConv::SystemV);
-        sig.params.push(AbiParam::new(I32));
-        sig.returns.push(AbiParam::new(I32));
-        let mut func = Function::with_name_signature(name, sig);
-
-        let bb0 = func.dfg.make_block();
-        let arg0 = func.dfg.append_block_param(bb0, I32);
-        let bb1 = func.dfg.make_block();
-        let bb2 = func.dfg.make_block();
-        let bb3 = func.dfg.make_block();
-
-        let mut pos = FuncCursor::new(&mut func);
-
-        pos.insert_block(bb0);
-        let jt_data = JumpTableData::new(
-            pos.func.dfg.block_call(bb3, &[]),
-            &[
-                pos.func.dfg.block_call(bb1, &[]),
-                pos.func.dfg.block_call(bb2, &[]),
-            ],
-        );
-        let jt = pos.func.create_jump_table(jt_data);
-        pos.ins().br_table(arg0, jt);
-
-        pos.insert_block(bb1);
-        let v1 = pos.ins().iconst(I32, 1);
-        pos.ins().return_(&[v1]);
-
-        pos.insert_block(bb2);
-        let v2 = pos.ins().iconst(I32, 2);
-        pos.ins().return_(&[v2]);
-
-        pos.insert_block(bb3);
-        let v3 = pos.ins().iconst(I32, 3);
-        pos.ins().return_(&[v3]);
-
-        let mut shared_flags_builder = settings::builder();
-        shared_flags_builder.set("opt_level", "none").unwrap();
-        shared_flags_builder.set("enable_verifier", "true").unwrap();
-        let shared_flags = settings::Flags::new(shared_flags_builder);
-        let isa_flags = x64_settings::Flags::new(&shared_flags, x64_settings::builder());
-        let backend = X64Backend::new_with_flags(
-            Triple::from_str("x86_64").unwrap(),
-            shared_flags,
-            isa_flags,
-        );
-        let cfg = ControlFlowGraph::with_function(&func);
-        let domtree = DominatorTree::with_function(&func, &cfg);
-        let result = backend
-            .compile_function(&mut func, &domtree, /* want_disasm = */ false)
-            .unwrap();
-        let code = result.buffer.data();
-
-        // To update this comment, write the golden bytes to a file, and run the following
-        // command on it:
-        // > objdump -b binary -D <file> -m i386:x86-64 -M intel
-        //
-        //  0:   55                      push   rbp
-        //  1:   48 89 e5                mov    rbp,rsp
-        //  4:   83 ff 02                cmp    edi,0x2
-        //  7:   0f 83 3b 00 00 00       jae    0x48
-        //  d:   44 8b d7                mov    r10d,edi
-        // 10:   41 b9 00 00 00 00       mov    r9d,0x0
-        // 16:   4d 0f 43 d1             cmovae r10,r9
-        // 1a:   4c 8d 0d 0b 00 00 00    lea    r9,[rip+0xb]        # 0x2c
-        // 21:   4f 63 54 91 00          movsxd r10,DWORD PTR [r9+r10*4+0x0]
-        // 26:   4d 01 d1                add    r9,r10
-        // 29:   41 ff e1                jmp    r9
-        // 2c:   12 00                   adc    al,BYTE PTR [rax]
-        // 2e:   00 00                   add    BYTE PTR [rax],al
-        // 30:   08 00                   or     BYTE PTR [rax],al
-        // 32:   00 00                   add    BYTE PTR [rax],al
-        // 34:   b8 02 00 00 00          mov    eax,0x2
-        // 39:   48 89 ec                mov    rsp,rbp
-        // 3c:   5d                      pop    rbp
-        // 3d:   c3                      ret
-        // 3e:   b8 01 00 00 00          mov    eax,0x1
-        // 43:   48 89 ec                mov    rsp,rbp
-        // 46:   5d                      pop    rbp
-        // 47:   c3                      ret
-        // 48:   b8 03 00 00 00          mov    eax,0x3
-        // 4d:   48 89 ec                mov    rsp,rbp
-        // 50:   5d                      pop    rbp
-        // 51:   c3                      ret
-
-        let golden = vec![
-            85, 72, 137, 229, 131, 255, 2, 15, 131, 59, 0, 0, 0, 68, 139, 215, 65, 185, 0, 0, 0, 0,
-            77, 15, 67, 209, 76, 141, 13, 11, 0, 0, 0, 79, 99, 84, 145, 0, 77, 1, 209, 65, 255,
-            225, 18, 0, 0, 0, 8, 0, 0, 0, 184, 2, 0, 0, 0, 72, 137, 236, 93, 195, 184, 1, 0, 0, 0,
-            72, 137, 236, 93, 195, 184, 3, 0, 0, 0, 72, 137, 236, 93, 195,
-        ];
-
-        assert_eq!(code, &golden[..]);
-    }
 }
diff --git a/crates/runtime/src/parking_spot.rs b/crates/runtime/src/parking_spot.rs
index dee1019b2..33f9bce0f 100644
--- a/crates/runtime/src/parking_spot.rs
+++ b/crates/runtime/src/parking_spot.rs
@@ -104,17 +104,53 @@ impl ParkingSpot {
             let spot = inner.get_mut(&key).expect("failed to get spot");
 
             if timed_out {
-                if let Some(timeout) = timeout {
-                    if Instant::now() < timeout {
-                        // Did not sleep long enough, try again.
-                        continue;
-                    }
+                // If waiting on the cvar timed out then due to how system cvars
+                // are implemented we may need to continue to sleep longer. If
+                // the deadline has not been reached then turn the crank again
+                // and go back to sleep.
+                if Instant::now() < timeout.unwrap() {
+                    continue;
+                }
+
+                // Opportunistically consume `to_unpark` signals even on
+                // timeout. From the perspective of `unpark` this "agent" raced
+                // between its own timeout and receiving the unpark signal, but
+                // from unpark's perspective it's definitely going to wake up N
+                // agents as returned from the `unpark` return value.
+                //
+                // Note that this may actually prevent other threads from
+                // getting unparked. For example:
+                //
+                // * Thread A parks with a timeout
+                // * Thread B parks with no timeout
+                // * Thread C decides to unpark 1 thread
+                // * Thread A's cvar wakes up due to a timeout, blocks on the
+                //   lock
+                // * Thread C finishes unpark and signals the cvar once
+                // * Thread B wakes up
+                // * Thread A and B contend for the lock and A wins
+                // * A consumes the "to_unpark" value
+                // * B goes back to sleep since `to_unpark == 0`, thinking that
+                //   a spurious wakeup happened.
+                //
+                // It's believed that this is ok, however, since from C's
+                // perspective one agent was still woken up and is allowed to
+                // continue, notably A in this case. C doesn't know that A raced
+                // with B and "stole" its wakeup signal.
+                if spot.to_unpark > 0 {
+                    spot.to_unpark -= 1;
                 }
             } else {
                 if spot.to_unpark == 0 {
+                    // If no timeout happen but nothing has unparked this spot (as
+                    // signaled through `to_unpark`) then this is indicative of a
+                    // spurious wakeup. In this situation turn the crank again and
+                    // go back to sleep as this interface doesn't allow for spurious
+                    // wakeups.
                     continue;
                 }
-
+                // No timeout happened, and some other thread registered to
+                // unpark this thread, so consume one unpark notification.
                 spot.to_unpark -= 1;
             }
 
@@ -176,55 +212,53 @@ impl ParkingSpot {
 #[cfg(test)]
 mod tests {
     use super::ParkingSpot;
-    use once_cell::sync::Lazy;
     use std::ptr::addr_of;
     use std::sync::atomic::{AtomicU64, Ordering};
     use std::thread;
-
-    static PARKING_SPOT: Lazy<ParkingSpot> = Lazy::new(ParkingSpot::default);
-
-    static ATOMIC: AtomicU64 = AtomicU64::new(0);
+    use std::time::{Duration, Instant};
 
     #[test]
     fn atomic_wait_notify() {
-        let thread1 = thread::spawn(|| {
-            let atomic_key = addr_of!(ATOMIC) as u64;
-            ATOMIC.store(1, Ordering::SeqCst);
-            PARKING_SPOT.unpark(atomic_key, u32::MAX);
-            PARKING_SPOT.park(atomic_key, || ATOMIC.load(Ordering::SeqCst) == 1, None);
-        });
+        let parking_spot = &ParkingSpot::default();
+        let atomic = &AtomicU64::new(0);
+
+        thread::scope(|s| {
+            let atomic_key = addr_of!(atomic) as u64;
+            let thread1 = s.spawn(move || {
+                atomic.store(1, Ordering::SeqCst);
+                parking_spot.unpark(atomic_key, u32::MAX);
+                parking_spot.park(atomic_key, || atomic.load(Ordering::SeqCst) == 1, None);
+            });
+
+            let thread2 = s.spawn(move || {
+                while atomic.load(Ordering::SeqCst) != 1 {
+                    parking_spot.park(atomic_key, || atomic.load(Ordering::SeqCst) != 1, None);
+                }
+                atomic.store(2, Ordering::SeqCst);
+                parking_spot.unpark(atomic_key, u32::MAX);
+                parking_spot.park(atomic_key, || atomic.load(Ordering::SeqCst) == 2, None);
+            });
+
+            let thread3 = s.spawn(move || {
+                while atomic.load(Ordering::SeqCst) != 2 {
+                    parking_spot.park(atomic_key, || atomic.load(Ordering::SeqCst) != 2, None);
+                }
+                atomic.store(3, Ordering::SeqCst);
+                parking_spot.unpark(atomic_key, u32::MAX);
 
-        let thread2 = thread::spawn(|| {
-            let atomic_key = addr_of!(ATOMIC) as u64;
-            while ATOMIC.load(Ordering::SeqCst) != 1 {
-                PARKING_SPOT.park(atomic_key, || ATOMIC.load(Ordering::SeqCst) != 1, None);
-            }
-            ATOMIC.store(2, Ordering::SeqCst);
-            PARKING_SPOT.unpark(atomic_key, u32::MAX);
-            PARKING_SPOT.park(atomic_key, || ATOMIC.load(Ordering::SeqCst) == 2, None);
-        });
+                parking_spot.park(atomic_key, || atomic.load(Ordering::SeqCst) == 3, None);
+            });
 
-        let thread3 = thread::spawn(|| {
-            let atomic_key = addr_of!(ATOMIC) as u64;
-            while ATOMIC.load(Ordering::SeqCst) != 2 {
-                PARKING_SPOT.park(atomic_key, || ATOMIC.load(Ordering::SeqCst) != 2, None);
+            while atomic.load(Ordering::SeqCst) != 3 {
+                parking_spot.park(atomic_key, || atomic.load(Ordering::SeqCst) != 3, None);
             }
-            ATOMIC.store(3, Ordering::SeqCst);
-            PARKING_SPOT.unpark(atomic_key, u32::MAX);
+            atomic.store(4, Ordering::SeqCst);
+            parking_spot.unpark(atomic_key, u32::MAX);
 
-            PARKING_SPOT.park(atomic_key, || ATOMIC.load(Ordering::SeqCst) == 3, None);
+            thread1.join().unwrap();
+            thread2.join().unwrap();
+            thread3.join().unwrap();
         });
-
-        let atomic_key = addr_of!(ATOMIC) as u64;
-        while ATOMIC.load(Ordering::SeqCst) != 3 {
-            PARKING_SPOT.park(atomic_key, || ATOMIC.load(Ordering::SeqCst) != 3, None);
-        }
-        ATOMIC.store(4, Ordering::SeqCst);
-        PARKING_SPOT.unpark(atomic_key, u32::MAX);
-
-        thread1.join().unwrap();
-        thread2.join().unwrap();
-        thread3.join().unwrap();
     }
 
     mod parking_lot {
@@ -302,47 +336,53 @@ mod tests {
             num_threads: u32,
             num_single_unparks: u32,
         ) {
-            let mut tests = Vec::with_capacity(num_latches);
-
-            for _ in 0..num_latches {
-                let test = Arc::new(SingleLatchTest::new(num_threads));
-                let mut threads = Vec::with_capacity(num_threads as _);
-                for _ in 0..num_threads {
-                    let test = test.clone();
-                    threads.push(thread::spawn(move || test.run()));
+            let spot = ParkingSpot::default();
+
+            thread::scope(|s| {
+                let mut tests = Vec::with_capacity(num_latches);
+
+                for _ in 0..num_latches {
+                    let test = Arc::new(SingleLatchTest::new(num_threads, &spot));
+                    let mut threads = Vec::with_capacity(num_threads as _);
+                    for _ in 0..num_threads {
+                        let test = test.clone();
+                        threads.push(s.spawn(move || test.run()));
+                    }
+                    tests.push((test, threads));
                 }
-                tests.push((test, threads));
-            }
 
-            for unpark_index in 0..num_single_unparks {
-                thread::sleep(delay);
-                for (test, _) in &tests {
-                    test.unpark_one(unpark_index);
+                for unpark_index in 0..num_single_unparks {
+                    thread::sleep(delay);
+                    for (test, _) in &tests {
+                        test.unpark_one(unpark_index);
+                    }
                 }
-            }
 
-            for (test, threads) in tests {
-                test.finish(num_single_unparks);
-                for thread in threads {
-                    thread.join().expect("Test thread panic");
+                for (test, threads) in tests {
+                    test.finish(num_single_unparks);
+                    for thread in threads {
+                        thread.join().expect("Test thread panic");
+                    }
                 }
-            }
+            });
         }
 
-        struct SingleLatchTest {
+        struct SingleLatchTest<'a> {
             semaphore: AtomicIsize,
             num_awake: AtomicU32,
             /// Total number of threads participating in this test.
             num_threads: u32,
+            spot: &'a ParkingSpot,
         }
 
-        impl SingleLatchTest {
-            pub fn new(num_threads: u32) -> Self {
+        impl<'a> SingleLatchTest<'a> {
+            pub fn new(num_threads: u32, spot: &'a ParkingSpot) -> Self {
                 Self {
                     // This implements a fair (FIFO) semaphore, and it starts out unavailable.
                     semaphore: AtomicIsize::new(0),
                     num_awake: AtomicU32::new(0),
                     num_threads,
+                    spot,
                 }
             }
 
@@ -373,14 +413,14 @@ mod tests {
                 // still be threads that has not yet parked.
                 while num_threads_left > 0 {
                     let mut num_waiting_on_address = 0;
-                    PARKING_SPOT.with_lot(self.semaphore_addr(), |thread_data| {
+                    self.spot.with_lot(self.semaphore_addr(), |thread_data| {
                         num_waiting_on_address = thread_data.num_parked;
                     });
                     assert!(num_waiting_on_address <= num_threads_left);
 
                     let num_awake_before_unpark = self.num_awake.load(Ordering::SeqCst);
 
-                    let num_unparked = PARKING_SPOT.unpark(self.semaphore_addr(), u32::MAX);
+                    let num_unparked = self.spot.unpark(self.semaphore_addr(), u32::MAX);
                     assert!(num_unparked >= num_waiting_on_address);
                     assert!(num_unparked <= num_threads_left);
 
@@ -398,7 +438,7 @@ mod tests {
 
                 // Make sure no thread is parked on our semaphore address
                 let mut num_waiting_on_address = 0;
-                PARKING_SPOT.with_lot(self.semaphore_addr(), |thread_data| {
+                self.spot.with_lot(self.semaphore_addr(), |thread_data| {
                     num_waiting_on_address = thread_data.num_parked;
                 });
                 assert_eq!(num_waiting_on_address, 0);
@@ -414,7 +454,7 @@ mod tests {
 
                 // We need to wait.
                 let validate = || true;
-                PARKING_SPOT.park(self.semaphore_addr(), validate, None);
+                self.spot.park(self.semaphore_addr(), validate, None);
             }
 
             pub fn up(&self) {
@@ -426,7 +466,7 @@ mod tests {
                     // the thread we want to pass ownership to has decremented the semaphore counter,
                     // but not yet parked.
                     loop {
-                        match PARKING_SPOT.unpark(self.semaphore_addr(), 1) {
+                        match self.spot.unpark(self.semaphore_addr(), 1) {
                             1 => break,
                             0 => (),
                             i => panic!("Should not wake up {i} threads"),
@@ -440,4 +480,42 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn wait_with_timeout() {
+        let parking_spot = &ParkingSpot::default();
+        let atomic = &AtomicU64::new(0);
+
+        thread::scope(|s| {
+            let atomic_key = addr_of!(atomic) as u64;
+
+            const N: u64 = 5;
+            const M: u64 = 1000;
+
+            let thread = s.spawn(move || {
+                while atomic.load(Ordering::SeqCst) != N * M {
+                    let timeout = Instant::now() + Duration::from_millis(1);
+                    parking_spot.park(
+                        atomic_key,
+                        || atomic.load(Ordering::SeqCst) != N * M,
+                        Some(timeout),
+                    );
+                }
+            });
+
+            let mut threads = vec![thread];
+            for _ in 0..N {
+                threads.push(s.spawn(move || {
+                    for _ in 0..M {
+                        atomic.fetch_add(1, Ordering::SeqCst);
+                        parking_spot.unpark(atomic_key, 1);
+                    }
+                }));
+            }
+
+            for thread in threads {
+                thread.join().unwrap();
+            }
+        });
+    }
 }


Test patch:
diff --git a/cranelift/filetests/filetests/isa/aarch64/cold-blocks.clif b/cranelift/filetests/filetests/isa/aarch64/cold-blocks.clif
new file mode 100644
index 000000000..329dd5fb0
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/cold-blocks.clif
@@ -0,0 +1,71 @@
+test compile precise-output
+
+target aarch64
+
+function %no_cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+; block0:
+;   mov w5, w0
+;   cbnz x5, label1 ; b label2
+; block1:
+;   b label3
+; block2:
+;   movz w0, #97
+;   b label3
+; block3:
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w5, w0
+;   cbnz x5, #0xc
+; block1: ; offset 0x8
+;   mov w0, #0x61
+; block2: ; offset 0xc
+;   ret
+
+function %cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2 cold:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+; block0:
+;   mov w5, w0
+;   cbnz x5, label1 ; b label2
+; block1:
+;   b label3
+; block3:
+;   ret
+; block2:
+;   movz w0, #97
+;   b label3
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w5, w0
+;   cbz x5, #0xc
+; block1: ; offset 0x8
+;   ret
+; block2: ; offset 0xc
+;   mov w0, #0x61
+;   b #8
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/cold-blocks.clif b/cranelift/filetests/filetests/isa/riscv64/cold-blocks.clif
new file mode 100644
index 000000000..0b7a8cde0
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/cold-blocks.clif
@@ -0,0 +1,71 @@
+test compile precise-output
+
+target riscv64
+
+function %no_cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+; block0:
+;   sext.w a2,a0
+;   bne a2,zero,taken(label1),not_taken(label2)
+; block1:
+;   j label3
+; block2:
+;   li a0,97
+;   j label3
+; block3:
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   sext.w a2, a0
+;   bnez a2, 8
+; block1: ; offset 0x8
+;   addi a0, zero, 0x61
+; block2: ; offset 0xc
+;   ret
+
+function %cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2 cold:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+; block0:
+;   sext.w a2,a0
+;   bne a2,zero,taken(label1),not_taken(label2)
+; block1:
+;   j label3
+; block3:
+;   ret
+; block2:
+;   li a0,97
+;   j label3
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   sext.w a2, a0
+;   beqz a2, 8
+; block1: ; offset 0x8
+;   ret
+; block2: ; offset 0xc
+;   addi a0, zero, 0x61
+;   j -8
+
diff --git a/cranelift/filetests/filetests/isa/s390x/cold-blocks.clif b/cranelift/filetests/filetests/isa/s390x/cold-blocks.clif
new file mode 100644
index 000000000..9bb8e8ecd
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/cold-blocks.clif
@@ -0,0 +1,71 @@
+test compile precise-output
+
+target s390x
+
+function %no_cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+; block0:
+;   chi %r2, 0
+;   jglh label1 ; jg label2
+; block1:
+;   jg label3
+; block2:
+;   lhi %r2, 97
+;   jg label3
+; block3:
+;   br %r14
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   chi %r2, 0
+;   jglh 0xe
+; block1: ; offset 0xa
+;   lhi %r2, 0x61
+; block2: ; offset 0xe
+;   br %r14
+
+function %cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2 cold:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+; block0:
+;   chi %r2, 0
+;   jglh label1 ; jg label2
+; block1:
+;   jg label3
+; block3:
+;   br %r14
+; block2:
+;   lhi %r2, 97
+;   jg label3
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   chi %r2, 0
+;   jgnlh 0xc
+; block1: ; offset 0xa
+;   br %r14
+; block2: ; offset 0xc
+;   lhi %r2, 0x61
+;   jg 0xa
+
diff --git a/cranelift/filetests/filetests/isa/x64/cold-blocks.clif b/cranelift/filetests/filetests/isa/x64/cold-blocks.clif
new file mode 100644
index 000000000..76a5f5374
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/cold-blocks.clif
@@ -0,0 +1,96 @@
+test compile precise-output
+
+target x86_64
+
+function %no_cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   testl   %edi, %edi
+;   jnz     label1; j label2
+; block1:
+;   movq    %rdi, %rax
+;   jmp     label3
+; block2:
+;   movl    $97, %eax
+;   jmp     label3
+; block3:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   testl %edi, %edi
+;   je 0x14
+; block2: ; offset 0xc
+;   movq %rdi, %rax
+;   jmp 0x19
+; block3: ; offset 0x14
+;   movl $0x61, %eax
+; block4: ; offset 0x19
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %cold_annotation(i32) -> i32 {
+block0(v0: i32):
+    brif v0, block1(v0), block2
+
+block1(v1: i32):
+    return v1
+
+block2 cold:
+    v2 = iconst.i32 97
+    jump block1(v2)
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   testl   %edi, %edi
+;   jnz     label1; j label2
+; block1:
+;   movq    %rdi, %rax
+;   jmp     label3
+; block3:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; block2:
+;   movl    $97, %eax
+;   jmp     label3
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   testl %edi, %edi
+;   je 0x14
+; block2: ; offset 0xc
+;   movq %rdi, %rax
+; block3: ; offset 0xf
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+; block4: ; offset 0x14
+;   movl $0x61, %eax
+;   jmp 0xf
+
diff --git a/cranelift/filetests/filetests/isa/x64/float-avx.clif b/cranelift/filetests/filetests/isa/x64/float-avx.clif
new file mode 100644
index 000000000..8626b3475
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/float-avx.clif
@@ -0,0 +1,598 @@
+test compile precise-output
+set enable_simd
+target x86_64 has_avx
+
+function %f32_add(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fadd v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vaddss  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vaddss %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64_add(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fadd v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vaddsd  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vaddsd %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32_sub(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fsub v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vsubss  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vsubss %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64_sub(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fsub v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vsubsd  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vsubsd %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32_mul(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fmul v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmulss  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmulss %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64_mul(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fmul v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmulsd  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmulsd %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32_div(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fdiv v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vdivss  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vdivss %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64_div(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fdiv v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vdivsd  %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vdivsd %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32_min(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fmin_pseudo v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vminss  %xmm1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vminss %xmm0, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64_min(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fmin_pseudo v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vminsd  %xmm1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vminsd %xmm0, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32_max(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fmax_pseudo v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmaxss  %xmm1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmaxss %xmm0, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64_max(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fmax_pseudo v0, v1
+  return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vmaxsd  %xmm1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vmaxsd %xmm0, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32x4_sqrt(f32x4) -> f32x4 {
+block0(v0: f32x4):
+  v1 = sqrt v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vsqrtps %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vsqrtps %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64x2_sqrt(f64x2) -> f64x2 {
+block0(v0: f64x2):
+  v1 = sqrt v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vsqrtpd %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vsqrtpd %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32x4_floor(f32x4) -> f32x4 {
+block0(v0: f32x4):
+  v1 = floor v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vroundps $1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vroundps $1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f64x2_floor(f64x2) -> f64x2 {
+block0(v0: f64x2):
+  v1 = floor v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vroundpd $1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vroundpd $1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %fcvt_low_from_sint(i32x4) -> f64x2 {
+block0(v0: i32x4):
+  v1 = fcvt_low_from_sint.f64x2 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vcvtdq2pd %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vcvtdq2pd %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %fcvt_from_uint(i32x4) -> f32x4 {
+block0(v0: i32x4):
+  v1 = fcvt_from_uint.f32x4 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpslld  %xmm0, $16, %xmm2
+;   vpsrld  %xmm2, $16, %xmm4
+;   vpsubd  %xmm0, %xmm4, %xmm6
+;   vcvtdq2ps %xmm4, %xmm8
+;   vpsrld  %xmm6, $1, %xmm10
+;   vcvtdq2ps %xmm10, %xmm12
+;   vaddps  %xmm12, %xmm12, %xmm14
+;   vaddps  %xmm14, %xmm8, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpslld $0x10, %xmm0, %xmm2
+;   vpsrld $0x10, %xmm2, %xmm4
+;   vpsubd %xmm4, %xmm0, %xmm6
+;   vcvtdq2ps %xmm4, %xmm8
+;   vpsrld $1, %xmm6, %xmm10
+;   vcvtdq2ps %xmm10, %xmm12
+;   vaddps %xmm12, %xmm12, %xmm14
+;   vaddps %xmm8, %xmm14, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %fvdemote(f64x2) -> f32x4 {
+block0(v0: f64x2):
+  v1 = fvdemote v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vcvtpd2ps %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vcvtpd2ps %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %fvpromote_low(f32x4) -> f64x2 {
+block0(v0: f32x4):
+  v1 = fvpromote_low v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vcvtps2pd %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vcvtps2pd %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %fcvt_to_sint_sat(f32x4) -> i32x4 {
+block0(v0: f32x4):
+  v1 = fcvt_to_sint_sat.i32x4 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vcmpps  $0 %xmm0, %xmm0, %xmm2
+;   vandps  %xmm0, %xmm2, %xmm4
+;   vpxor   %xmm2, %xmm4, %xmm6
+;   vcvttps2dq %xmm4, %xmm8
+;   vpand   %xmm8, %xmm6, %xmm10
+;   vpsrad  %xmm10, $31, %xmm12
+;   vpxor   %xmm12, %xmm8, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vcmpeqps %xmm0, %xmm0, %xmm2
+;   vandps %xmm2, %xmm0, %xmm4
+;   vpxor %xmm4, %xmm2, %xmm6
+;   vcvttps2dq %xmm4, %xmm8
+;   vpand %xmm6, %xmm8, %xmm10
+;   vpsrad $0x1f, %xmm10, %xmm12
+;   vpxor %xmm8, %xmm12, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %fcvt_to_sint_sat_snarrow(f64x2) -> i32x4 {
+block0(v0: f64x2):
+  v1 = fcvt_to_sint_sat.i64x2 v0
+  v2 = vconst.i64x2 0x00
+  v3 = snarrow v1, v2
+  return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vcmppd  $0 %xmm0, %xmm0, %xmm2
+;   movupd  const(0), %xmm4
+;   vandps  %xmm2, %xmm4, %xmm6
+;   vminpd  %xmm0, %xmm6, %xmm8
+;   vcvttpd2dq %xmm8, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vcmpeqpd %xmm0, %xmm0, %xmm2
+;   movupd 0x1f(%rip), %xmm4
+;   vandps %xmm4, %xmm2, %xmm6
+;   vminpd %xmm6, %xmm0, %xmm8
+;   vcvttpd2dq %xmm8, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, %al
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
index fa6ceda2c..323280260 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
@@ -1213,7 +1213,7 @@ block0(v0: i8x16):
 ;   movq    %rsp, %rbp
 ; block0:
 ;   vpalignr $8 %xmm0, %xmm0, %xmm2
-;   pmovzxbw %xmm2, %xmm0
+;   vpmovzxbw %xmm2, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -1224,7 +1224,7 @@ block0(v0: i8x16):
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
 ;   vpalignr $8, %xmm0, %xmm0, %xmm2
-;   pmovzxbw %xmm2, %xmm0
+;   vpmovzxbw %xmm2, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -1359,7 +1359,7 @@ block0(v0: f64x2):
 ;   vmaxpd  %xmm0, %xmm2, %xmm4
 ;   movupd  const(0), %xmm6
 ;   vminpd  %xmm4, %xmm6, %xmm8
-;   roundpd $3, %xmm8, %xmm10
+;   vroundpd $3, %xmm8, %xmm10
 ;   movupd  const(1), %xmm12
 ;   vaddpd  %xmm10, %xmm12, %xmm14
 ;   vshufps $136 %xmm14, %xmm2, %xmm0
@@ -1376,8 +1376,8 @@ block0(v0: f64x2):
 ;   vmaxpd %xmm2, %xmm0, %xmm4
 ;   movupd 0x2c(%rip), %xmm6
 ;   vminpd %xmm6, %xmm4, %xmm8
-;   roundpd $3, %xmm8, %xmm10
-;   movupd 0x28(%rip), %xmm12
+;   vroundpd $3, %xmm8, %xmm10
+;   movupd 0x29(%rip), %xmm12
 ;   vaddpd %xmm12, %xmm10, %xmm14
 ;   vshufps $0x88, %xmm2, %xmm14, %xmm0
 ;   movq %rbp, %rsp
@@ -1388,7 +1388,8 @@ block0(v0: f64x2):
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
 ;   addb %al, (%rax)
-;   addb %ah, %al
+;   addb %al, (%rax)
+;   loopne 0x43
 
 function %i8x16_shl(i8x16, i32) -> i8x16 {
 block0(v0: i8x16, v1: i32):
@@ -1884,3 +1885,78 @@ block0(v0: i64x2):
 ;   popq %rbp
 ;   retq
 
+function %i8x16_abs(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = iabs v0
+    return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpabsb  %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpabsb %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %i16x8_abs(i16x8) -> i16x8 {
+block0(v0: i16x8):
+    v1 = iabs v0
+    return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpabsw  %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpabsw %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %i32x4_abs(i32x4) -> i32x4 {
+block0(v0: i32x4):
+    v1 = iabs v0
+    return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpabsd  %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpabsd %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-load-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-load-avx.clif
new file mode 100644
index 000000000..df4f25a99
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/simd-load-avx.clif
@@ -0,0 +1,154 @@
+test compile precise-output
+set enable_simd
+target x86_64 has_avx
+
+function %sload8x8(i64) -> i16x8 {
+block0(v0: i64):
+  v1 = sload8x8 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpmovsxbw 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpmovsxbw (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %uload8x8(i64) -> i16x8 {
+block0(v0: i64):
+  v1 = uload8x8 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpmovzxbw 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpmovzxbw (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %sload16x4(i64) -> i32x4 {
+block0(v0: i64):
+  v1 = sload16x4 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpmovsxwd 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpmovsxwd (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %uload16x4(i64) -> i32x4 {
+block0(v0: i64):
+  v1 = uload16x4 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpmovzxwd 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpmovzxwd (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %sload32x2(i64) -> i64x2 {
+block0(v0: i64):
+  v1 = sload32x2 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpmovsxdq 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpmovsxdq (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %uload32x2(i64) -> i64x2 {
+block0(v0: i64):
+  v1 = uload32x2 v0
+  return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpmovzxdq 0(%rdi), %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpmovzxdq (%rdi), %xmm0 ; trap: heap_oob
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+