Repository: bytecodealliance/wasmtime
Pull request #5807 (2023-02-16 20:43:58)

winch: Add support for the `<i32|i64>.div_*` instructions
This change is divided into two main commits

- 2e1ea496ea9a87341056ef63a375764db618c5f6, which performs the necessary refactoring to support the  introduction of the division instructions.
- 2e1ea496ea9a87341056ef63a375764db618c5f6, introduces support for the instructions themselves. 

I opted to include both commits in a single change to demonstrate the objective behind the refactor, but I'm happy to split them into two different changes if desired. 

I added a detailed commit message to each to make reviewing easier. 

<!--

Please ensure that the following steps are all taken care of before submitting
the PR.

- [ ] This has been discussed in issue #..., or if not, please tell us why
  here.
- [ ] A short description of what this does, why it is needed; if the
  description becomes long, the matter should probably be discussed in an issue
  first.
- [ ] This PR contains test cases, if meaningful.
- [ ] A reviewer from the core maintainer team has been assigned for this PR.
  If you don't know who could review this, please indicate so. The list of
  suggested reviewers on the right can help you.

Please ensure all communication adheres to the [code of
conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
-->



Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 6bdcd18b4..07008be3f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -54,6 +54,8 @@ jobs:
   # incompatible licences
   cargo_deny:
     name: Cargo deny
+    needs: determine
+    if: needs.determine.outputs.audit
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -76,6 +78,8 @@ jobs:
   # Ensure dependencies are vetted. See https://mozilla.github.io/cargo-vet/
   cargo_vet:
     name: Cargo vet
+    needs: determine
+    if: needs.determine.outputs.audit
     runs-on: ubuntu-latest
     env:
       CARGO_VET_VERSION: 0.3.1
@@ -116,6 +120,7 @@ jobs:
       test-matrix: ${{ steps.calculate.outputs.test-matrix }}
       test-capi: ${{ steps.calculate.outputs.test-capi }}
       build-fuzz: ${{ steps.calculate.outputs.build-fuzz }}
+      audit: ${{ steps.calculate.outputs.audit }}
     steps:
     - uses: actions/checkout@v3
     - id: calculate
@@ -143,6 +148,9 @@ jobs:
           if grep -q fuzz names.log; then
             echo build-fuzz=true >> $GITHUB_OUTPUT
           fi
+          if grep -q Cargo.lock names.log; then
+            echo audit=true >> $GITHUB_OUTPUT
+          fi
         fi
         matrix="$(node ./ci/build-test-matrix.js ./commits.log ./names.log $run_full)"
         echo "test-matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
@@ -152,6 +160,7 @@ jobs:
             echo run-full=true >> $GITHUB_OUTPUT
             echo test-capi=true >> $GITHUB_OUTPUT
             echo build-fuzz=true >> $GITHUB_OUTPUT
+            echo audit=true >> $GITHUB_OUTPUT
         fi
 
   # Build all documentation of Wasmtime, including the C API documentation,
@@ -439,7 +448,7 @@ jobs:
     - run: |
         sudo apt-get update && sudo apt-get install -y gdb lldb llvm
         cargo test test_debug_dwarf -- --ignored --test-threads 1
-      if: matrix.os == 'ubuntu-latest' && matrix.target == ''
+      if: matrix.os == 'ubuntu-latest' && matrix.target == ''&& needs.determine.outputs.run-full
       env:
         RUST_BACKTRACE: 1
 
@@ -670,6 +679,7 @@ jobs:
       - bench
       - meta_deterministic_check
       - verify-publish
+      - determine
     if: always()
     steps:
     - name: Dump needs context
diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml
index 4da009f8b..c7bde9931 100644
--- a/.github/workflows/triage.yml
+++ b/.github/workflows/triage.yml
@@ -4,11 +4,11 @@ on:
   issues:
     types: ["labeled"]
   schedule:
-    # Run pull request triage every 5 minutes. Ideally, this would be on
+    # Run pull request triage once an hour. Ideally, this would be on
     # "labeled" types of pull request events, but that doesn't work if the pull
     # request is from another fork. For example, see
     # https://github.com/actions/labeler/issues/12
-    - cron: '*/5 * * * *'
+    - cron: '42 * * * *'
 
 concurrency:
   group: issue-triage
diff --git a/cranelift/codegen/src/dominator_tree.rs b/cranelift/codegen/src/dominator_tree.rs
index 23ccb1783..89335bce1 100644
--- a/cranelift/codegen/src/dominator_tree.rs
+++ b/cranelift/codegen/src/dominator_tree.rs
@@ -2,7 +2,6 @@
 
 use crate::entity::SecondaryMap;
 use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
-use crate::inst_predicates;
 use crate::ir::{Block, ExpandedProgramPoint, Function, Inst, Layout, ProgramOrder, Value};
 use crate::packed_option::PackedOption;
 use crate::timing;
@@ -16,8 +15,7 @@ use core::mem;
 const STRIDE: u32 = 4;
 
 /// Special RPO numbers used during `compute_postorder`.
-const DONE: u32 = 1;
-const SEEN: u32 = 2;
+const SEEN: u32 = 1;
 
 /// Dominator tree node. We keep one of these per block.
 #[derive(Clone, Default)]
@@ -36,6 +34,12 @@ struct DomNode {
     idom: PackedOption<Inst>,
 }
 
+/// DFT stack state marker for computing the cfg postorder.
+enum Visit {
+    First,
+    Last,
+}
+
 /// The dominator tree for a single function.
 pub struct DominatorTree {
     nodes: SecondaryMap<Block, DomNode>,
@@ -44,7 +48,7 @@ pub struct DominatorTree {
     postorder: Vec<Block>,
 
     /// Scratch memory used by `compute_postorder()`.
-    stack: Vec<Block>,
+    stack: Vec<(Visit, Block)>,
 
     valid: bool,
 }
@@ -275,93 +279,64 @@ impl DominatorTree {
 
         // This algorithm is a depth first traversal (DFT) of the control flow graph, computing a
         // post-order of the blocks that are reachable form the entry block. A DFT post-order is not
-        // unique. The specific order we get is controlled by two factors:
-        //
-        // 1. The order each node's children are visited, and
-        // 2. The method used for pruning graph edges to get a tree.
-        //
-        // There are two ways of viewing the CFG as a graph:
-        //
-        // 1. Each block is a node, with outgoing edges for all the branches in the block.
-        // 2. Each basic block is a node, with outgoing edges for the single branch at the end of
-        //    the BB. (A block is a linear sequence of basic blocks).
-        //
-        // The first graph is a contraction of the second one. We want to compute a block post-order
-        // that is compatible both graph interpretations. That is, if you compute a BB post-order
-        // and then remove those BBs that do not correspond to block headers, you get a post-order of
-        // the block graph.
-        //
-        // Node child order:
-        //
-        //     In the BB graph, we always go down the fall-through path first and follow the branch
-        //     destination second.
-        //
-        //     In the block graph, this is equivalent to visiting block successors in a bottom-up
-        //     order, starting from the destination of the block's terminating jump, ending at the
-        //     destination of the first branch in the block.
-        //
-        // Edge pruning:
+        // unique. The specific order we get is controlled by the order each node's children are
+        // visited.
         //
-        //     In the BB graph, we keep an edge to a block the first time we visit the *source* side
-        //     of the edge. Any subsequent edges to the same block are pruned.
+        // We view the CFG as a graph where each `BlockCall` value of a terminating branch
+        // instruction is an edge. A consequence of this is that we visit successor nodes in the
+        // reverse order specified by the branch instruction that terminates the basic block.
+        // (Reversed because we are using a stack to control traversal, and push the successors in
+        // the order the branch instruction specifies -- there's no good reason for this particular
+        // order.)
         //
-        //     The equivalent tree is reached in the block graph by keeping the first edge to a block
-        //     in a top-down traversal of the successors. (And then visiting edges in a bottom-up
-        //     order).
-        //
-        // This pruning method makes it possible to compute the DFT without storing lots of
-        // information about the progress through a block.
-
         // During this algorithm only, use `rpo_number` to hold the following state:
         //
-        //   0:    block has not yet been reached in the pre-order.
-        //   SEEN: block has been pushed on the stack but successors not yet pushed.
-        //   DONE: Successors pushed.
+        //   0:    block has not yet had its first visit
+        //   SEEN: block has been visited at least once, implying that all of its successors are on
+        //         the stack
 
         match func.layout.entry_block() {
             Some(block) => {
-                self.stack.push(block);
-                self.nodes[block].rpo_number = SEEN;
+                self.stack.push((Visit::First, block));
             }
             None => return,
         }
 
-        while let Some(block) = self.stack.pop() {
-            match self.nodes[block].rpo_number {
-                SEEN => {
-                    // This is the first time we pop the block, so we need to scan its successors and
-                    // then revisit it.
-                    self.nodes[block].rpo_number = DONE;
-                    self.stack.push(block);
-                    self.push_successors(func, block);
+        while let Some((visit, block)) = self.stack.pop() {
+            match visit {
+                Visit::First => {
+                    if self.nodes[block].rpo_number == 0 {
+                        // This is the first time we pop the block, so we need to scan its
+                        // successors and then revisit it.
+                        self.nodes[block].rpo_number = SEEN;
+                        self.stack.push((Visit::Last, block));
+                        if let Some(inst) = func.stencil.layout.last_inst(block) {
+                            for block in func.stencil.dfg.insts[inst]
+                                .branch_destination(&func.stencil.dfg.jump_tables)
+                                .iter()
+                            {
+                                let succ = block.block(&func.stencil.dfg.value_lists);
+
+                                // This is purely an optimization to avoid additional iterations of
+                                // the loop, and is not required; it's merely inlining the check
+                                // from the outer conditional of this case to avoid the extra loop
+                                // iteration.
+                                if self.nodes[succ].rpo_number == 0 {
+                                    self.stack.push((Visit::First, succ))
+                                }
+                            }
+                        }
+                    }
                 }
-                DONE => {
-                    // This is the second time we pop the block, so all successors have been
-                    // processed.
+
+                Visit::Last => {
+                    // We've finished all this node's successors.
                     self.postorder.push(block);
                 }
-                _ => unreachable!(),
             }
         }
     }
 
-    /// Push `block` successors onto `self.stack`, filtering out those that have already been seen.
-    ///
-    /// The successors are pushed in program order which is important to get a split-invariant
-    /// post-order. Split-invariant means that if a block is split in two, we get the same
-    /// post-order except for the insertion of the new block header at the split point.
-    fn push_successors(&mut self, func: &Function, block: Block) {
-        inst_predicates::visit_block_succs(func, block, |_, succ, _| self.push_if_unseen(succ))
-    }
-
-    /// Push `block` onto `self.stack` if it has not already been seen.
-    fn push_if_unseen(&mut self, block: Block) {
-        if self.nodes[block].rpo_number == 0 {
-            self.nodes[block].rpo_number = SEEN;
-            self.stack.push(block);
-        }
-    }
-
     /// Build a dominator tree from a control flow graph using Keith D. Cooper's
     /// "Simple, Fast Dominator Algorithm."
     fn compute_domtree(&mut self, func: &Function, cfg: &ControlFlowGraph) {
@@ -728,7 +703,7 @@ mod tests {
         //   } block3:jump block1
         // } block3
 
-        assert_eq!(dt.cfg_postorder(), &[block2, block0, block1, block3]);
+        assert_eq!(dt.cfg_postorder(), &[block0, block2, block1, block3]);
 
         assert_eq!(cur.func.layout.entry_block().unwrap(), block3);
         assert_eq!(dt.idom(block3), None);
diff --git a/cranelift/codegen/src/egraph.rs b/cranelift/codegen/src/egraph.rs
index 02bfeb013..2cbe412e3 100644
--- a/cranelift/codegen/src/egraph.rs
+++ b/cranelift/codegen/src/egraph.rs
@@ -7,7 +7,7 @@ use crate::dominator_tree::DominatorTree;
 use crate::egraph::domtree::DomTreeWithChildren;
 use crate::egraph::elaborate::Elaborator;
 use crate::fx::FxHashSet;
-use crate::inst_predicates::{is_mergeable_for_egraph, is_pure_for_egraph};
+use crate::inst_predicates::is_pure_for_egraph;
 use crate::ir::{
     DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
 };
@@ -285,41 +285,10 @@ where
     fn optimize_skeleton_inst(&mut self, inst: Inst) -> bool {
         self.stats.skeleton_inst += 1;
 
-        // First, can we try to deduplicate? We need to keep some copy
-        // of the instruction around because it's side-effecting, but
-        // we may be able to reuse an earlier instance of it.
-        if is_mergeable_for_egraph(self.func, inst) {
-            let result = self.func.dfg.inst_results(inst)[0];
-            trace!(" -> mergeable side-effecting op {}", inst);
-            let inst = NewOrExistingInst::Existing(inst);
-            let gvn_context = GVNContext {
-                union_find: self.eclasses,
-                value_lists: &self.func.dfg.value_lists,
-            };
-
-            // Does this instruction already exist? If so, add entries to
-            // the value-map to rewrite uses of its results to the results
-            // of the original (existing) instruction. If not, optimize
-            // the new instruction.
-            let key = inst.get_inst_key(&self.func.dfg);
-            if let Some(&orig_result) = self.gvn_map.get(&key, &gvn_context) {
-                // Hit in GVN map -- reuse value.
-                self.value_to_opt_value[result] = orig_result;
-                self.eclasses.union(orig_result, result);
-                trace!(" -> merges result {} to {}", result, orig_result);
-                true
-            } else {
-                // Otherwise, insert it into the value-map.
-                self.value_to_opt_value[result] = result;
-                self.gvn_map.insert(key, result, &gvn_context);
-                trace!(" -> inserts as new (no GVN)");
-                false
-            }
-        }
-        // Otherwise, if a load or store, process it with the alias
-        // analysis to see if we can optimize it (rewrite in terms of
-        // an earlier load or stored value).
-        else if let Some(new_result) =
+        // If a load or store, process it with the alias analysis to see
+        // if we can optimize it (rewrite in terms of an earlier load or
+        // stored value).
+        if let Some(new_result) =
             self.alias_analysis
                 .process_inst(self.func, self.alias_analysis_state, inst)
         {
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index f82f41daa..9b284e8b8 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -73,25 +73,6 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
     has_one_result && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
 }
 
-/// Can the given instruction be merged into another copy of itself?
-/// These instructions may have side-effects, but as long as we retain
-/// the first instance of the instruction, the second and further
-/// instances are redundant if they would produce the same trap or
-/// result.
-pub fn is_mergeable_for_egraph(func: &Function, inst: Inst) -> bool {
-    let op = func.dfg.insts[inst].opcode();
-    // We can only merge one-result operators due to the way that GVN
-    // is structured in the egraph implementation.
-    let has_one_result = func.dfg.inst_results(inst).len() == 1;
-    has_one_result
-        // Loads/stores are handled by alias analysis and not
-        // otherwise mergeable.
-        && !op.can_load()
-        && !op.can_store()
-        // Can only have idempotent side-effects.
-        && (!has_side_effect(func, inst) || op.side_effects_idempotent())
-}
-
 /// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,
 /// but not the get_pinned_reg opcode?
 pub fn has_lowering_side_effect(func: &Function, inst: Inst) -> bool {
diff --git a/cranelift/codegen/src/isa/x64/encoding/rex.rs b/cranelift/codegen/src/isa/x64/encoding/rex.rs
index 41ae596eb..f48fea830 100644
--- a/cranelift/codegen/src/isa/x64/encoding/rex.rs
+++ b/cranelift/codegen/src/isa/x64/encoding/rex.rs
@@ -312,18 +312,50 @@ pub(crate) fn emit_std_enc_mem(
 
     prefixes.emit(sink);
 
+    // After prefixes, first emit the REX byte depending on the kind of
+    // addressing mode that's being used.
     match *mem_e {
-        Amode::ImmReg { simm32, base, .. } => {
-            // First, the REX byte.
+        Amode::ImmReg { base, .. } => {
             let enc_e = int_reg_enc(base);
             rex.emit_two_op(sink, enc_g, enc_e);
+        }
 
-            // Now the opcode(s).  These include any other prefixes the caller
-            // hands to us.
-            while num_opcodes > 0 {
-                num_opcodes -= 1;
-                sink.put1(((opcodes >> (num_opcodes << 3)) & 0xFF) as u8);
-            }
+        Amode::ImmRegRegShift {
+            base: reg_base,
+            index: reg_index,
+            ..
+        } => {
+            let enc_base = int_reg_enc(*reg_base);
+            let enc_index = int_reg_enc(*reg_index);
+            rex.emit_three_op(sink, enc_g, enc_index, enc_base);
+        }
+
+        Amode::RipRelative { .. } => {
+            // note REX.B = 0.
+            rex.emit_two_op(sink, enc_g, 0);
+        }
+    }
+
+    // Now the opcode(s).  These include any other prefixes the caller
+    // hands to us.
+    while num_opcodes > 0 {
+        num_opcodes -= 1;
+        sink.put1(((opcodes >> (num_opcodes << 3)) & 0xFF) as u8);
+    }
+
+    // And finally encode the mod/rm bytes and all further information.
+    emit_modrm_sib_disp(sink, enc_g, mem_e, bytes_at_end)
+}
+
+pub(crate) fn emit_modrm_sib_disp(
+    sink: &mut MachBuffer<Inst>,
+    enc_g: u8,
+    mem_e: &Amode,
+    bytes_at_end: u8,
+) {
+    match *mem_e {
+        Amode::ImmReg { simm32, base, .. } => {
+            let enc_e = int_reg_enc(base);
 
             // Now the mod/rm and associated immediates.  This is
             // significantly complicated due to the multiple special cases.
@@ -377,15 +409,6 @@ pub(crate) fn emit_std_enc_mem(
             let enc_base = int_reg_enc(*reg_base);
             let enc_index = int_reg_enc(*reg_index);
 
-            // The rex byte.
-            rex.emit_three_op(sink, enc_g, enc_index, enc_base);
-
-            // All other prefixes and opcodes.
-            while num_opcodes > 0 {
-                num_opcodes -= 1;
-                sink.put1(((opcodes >> (num_opcodes << 3)) & 0xFF) as u8);
-            }
-
             // modrm, SIB, immediates.
             if low8_will_sign_extend_to_32(simm32) && enc_index != regs::ENC_RSP {
                 sink.put1(encode_modrm(1, enc_g & 7, 4));
@@ -401,16 +424,6 @@ pub(crate) fn emit_std_enc_mem(
         }
 
         Amode::RipRelative { ref target } => {
-            // First, the REX byte, with REX.B = 0.
-            rex.emit_two_op(sink, enc_g, 0);
-
-            // Now the opcode(s).  These include any other prefixes the caller
-            // hands to us.
-            while num_opcodes > 0 {
-                num_opcodes -= 1;
-                sink.put1(((opcodes >> (num_opcodes << 3)) & 0xFF) as u8);
-            }
-
             // RIP-relative is mod=00, rm=101.
             sink.put1(encode_modrm(0, enc_g & 7, 0b101));
 
diff --git a/cranelift/codegen/src/isa/x64/encoding/vex.rs b/cranelift/codegen/src/isa/x64/encoding/vex.rs
index 3aa9bb8d5..df2c92169 100644
--- a/cranelift/codegen/src/isa/x64/encoding/vex.rs
+++ b/cranelift/codegen/src/isa/x64/encoding/vex.rs
@@ -4,7 +4,10 @@
 use super::evex::Register;
 use super::rex::{LegacyPrefixes, OpcodeMap};
 use super::ByteSink;
-use crate::isa::x64::encoding::rex::encode_modrm;
+use crate::isa::x64::args::Amode;
+use crate::isa::x64::encoding::rex;
+use crate::isa::x64::inst::Inst;
+use crate::machinst::MachBuffer;
 
 /// Constructs a VEX-encoded instruction using a builder pattern. This approach makes it visually
 /// easier to transform something the manual's syntax, `VEX.128.66.0F 73 /7 ib` to code:
@@ -16,11 +19,29 @@ pub struct VexInstruction {
     opcode: u8,
     w: bool,
     reg: u8,
-    rm: Register,
+    rm: RegisterOrAmode,
     vvvv: Option<Register>,
     imm: Option<u8>,
 }
 
+#[allow(missing_docs)]
+pub enum RegisterOrAmode {
+    Register(Register),
+    Amode(Amode),
+}
+
+impl From<u8> for RegisterOrAmode {
+    fn from(reg: u8) -> Self {
+        RegisterOrAmode::Register(reg.into())
+    }
+}
+
+impl From<Amode> for RegisterOrAmode {
+    fn from(amode: Amode) -> Self {
+        RegisterOrAmode::Amode(amode)
+    }
+}
+
 impl Default for VexInstruction {
     fn default() -> Self {
         Self {
@@ -30,7 +51,7 @@ impl Default for VexInstruction {
             opcode: 0x00,
             w: false,
             reg: 0x00,
-            rm: Register::default(),
+            rm: RegisterOrAmode::Register(Register::default()),
             vvvv: None,
             imm: None,
         }
@@ -105,12 +126,12 @@ impl VexInstruction {
         self
     }
 
-    /// Set the register to use for the `rm` bits; many instructions use this as the "read from
-    /// register/memory" operand. Currently this does not support memory addressing (TODO).Setting
-    /// this affects both the ModRM byte (`rm` section) and the VEX prefix (the extension bits for
-    /// register encodings > 8).
+    /// Set the register to use for the `rm` bits; many instructions use this
+    /// as the "read from register/memory" operand. Setting this affects both
+    /// the ModRM byte (`rm` section) and the VEX prefix (the extension bits
+    /// for register encodings > 8).
     #[inline(always)]
-    pub fn rm(mut self, reg: impl Into<Register>) -> Self {
+    pub fn rm(mut self, reg: impl Into<RegisterOrAmode>) -> Self {
         self.rm = reg.into();
         self
     }
@@ -150,15 +171,33 @@ impl VexInstruction {
     /// The X bit in encoded format (inverted).
     #[inline(always)]
     fn x_bit(&self) -> u8 {
-        // TODO
-        (!0) & 1
+        let reg = match &self.rm {
+            RegisterOrAmode::Register(_) => 0,
+            RegisterOrAmode::Amode(Amode::ImmReg { .. }) => 0,
+            RegisterOrAmode::Amode(Amode::ImmRegRegShift { index, .. }) => {
+                index.to_real_reg().unwrap().hw_enc()
+            }
+            RegisterOrAmode::Amode(Amode::RipRelative { .. }) => 0,
+        };
+
+        !(reg >> 3) & 1
     }
 
     /// The B bit in encoded format (inverted).
     #[inline(always)]
     fn b_bit(&self) -> u8 {
-        let rm: u8 = self.rm.into();
-        (!(rm >> 3)) & 1
+        let reg = match &self.rm {
+            RegisterOrAmode::Register(r) => (*r).into(),
+            RegisterOrAmode::Amode(Amode::ImmReg { base, .. }) => {
+                base.to_real_reg().unwrap().hw_enc()
+            }
+            RegisterOrAmode::Amode(Amode::ImmRegRegShift { base, .. }) => {
+                base.to_real_reg().unwrap().hw_enc()
+            }
+            RegisterOrAmode::Amode(Amode::RipRelative { .. }) => 0,
+        };
+
+        !(reg >> 3) & 1
     }
 
     /// Is the 2 byte prefix available for this instruction?
@@ -176,6 +215,7 @@ impl VexInstruction {
         // encoded by the three-byte form of VEX
         !(self.map == OpcodeMap::_0F3A || self.map == OpcodeMap::_0F38)
     }
+
     /// The last byte of the 2byte and 3byte prefixes is mostly the same, share the common
     /// encoding logic here.
     #[inline(always)]
@@ -225,8 +265,8 @@ impl VexInstruction {
         sink.put1(last_byte);
     }
 
-    /// Emit the VEX-encoded instruction to the code sink:
-    pub fn encode<CS: ByteSink + ?Sized>(&self, sink: &mut CS) {
+    /// Emit the VEX-encoded instruction to the provided buffer.
+    pub fn encode(&self, sink: &mut MachBuffer<Inst>) {
         // 2/3 byte prefix
         if self.use_2byte_prefix() {
             self.encode_2byte_prefix(sink);
@@ -237,13 +277,21 @@ impl VexInstruction {
         // 1 Byte Opcode
         sink.put1(self.opcode);
 
-        // 1 ModRM Byte
-        // Not all instructions use Reg as a reg, some use it as an extension of the opcode.
-        let rm: u8 = self.rm.into();
-        sink.put1(encode_modrm(3, self.reg & 7, rm & 7));
-
-        // TODO: 0/1 byte SIB
-        // TODO: 0/1/2/4 bytes DISP
+        match &self.rm {
+            // Not all instructions use Reg as a reg, some use it as an extension
+            // of the opcode.
+            RegisterOrAmode::Register(reg) => {
+                let rm: u8 = (*reg).into();
+                sink.put1(rex::encode_modrm(3, self.reg & 7, rm & 7));
+            }
+            // For address-based modes reuse the logic from the `rex` module
+            // for the modrm and trailing bytes since VEX uses the same
+            // encoding.
+            RegisterOrAmode::Amode(amode) => {
+                let bytes_at_end = if self.imm.is_some() { 1 } else { 0 };
+                rex::emit_modrm_sib_disp(sink, self.reg & 7, amode, bytes_at_end);
+            }
+        }
 
         // Optional 1 Byte imm
         if let Some(imm) = self.imm {
@@ -278,8 +326,9 @@ impl Default for VexVectorLength {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::isa::x64::inst::args::Gpr;
     use crate::isa::x64::inst::regs;
-    use std::vec::Vec;
+    use crate::opts::MemFlags;
 
     #[test]
     fn vpslldq() {
@@ -288,7 +337,7 @@ mod tests {
 
         let dst = regs::xmm1().to_real_reg().unwrap().hw_enc();
         let src = regs::xmm2().to_real_reg().unwrap().hw_enc();
-        let mut sink0 = Vec::new();
+        let mut sink = MachBuffer::new();
 
         VexInstruction::new()
             .length(VexVectorLength::V128)
@@ -299,9 +348,10 @@ mod tests {
             .vvvv(dst)
             .rm(src)
             .imm(0x17)
-            .encode(&mut sink0);
+            .encode(&mut sink);
 
-        assert_eq!(sink0, vec![0xc5, 0xf1, 0x73, 0xfa, 0x17]);
+        let bytes = sink.finish().data;
+        assert_eq!(bytes.as_slice(), [0xc5, 0xf1, 0x73, 0xfa, 0x17]);
     }
 
     #[test]
@@ -314,7 +364,7 @@ mod tests {
         let a = regs::xmm2().to_real_reg().unwrap().hw_enc();
         let b = regs::xmm3().to_real_reg().unwrap().hw_enc();
         let c = regs::xmm4().to_real_reg().unwrap().hw_enc();
-        let mut sink0 = Vec::new();
+        let mut sink = MachBuffer::new();
 
         VexInstruction::new()
             .length(VexVectorLength::V128)
@@ -326,9 +376,10 @@ mod tests {
             .vvvv(a)
             .rm(b)
             .imm_reg(c)
-            .encode(&mut sink0);
+            .encode(&mut sink);
 
-        assert_eq!(sink0, vec![0xc4, 0xe3, 0x69, 0x4b, 0xcb, 0x40]);
+        let bytes = sink.finish().data;
+        assert_eq!(bytes.as_slice(), [0xc4, 0xe3, 0x69, 0x4b, 0xcb, 0x40]);
     }
 
     #[test]
@@ -339,7 +390,7 @@ mod tests {
         let dst = regs::xmm10().to_real_reg().unwrap().hw_enc();
         let a = regs::xmm11().to_real_reg().unwrap().hw_enc();
         let b = regs::xmm12().to_real_reg().unwrap().hw_enc();
-        let mut sink0 = Vec::new();
+        let mut sink = MachBuffer::new();
 
         VexInstruction::new()
             .length(VexVectorLength::V256)
@@ -350,8 +401,91 @@ mod tests {
             .vvvv(a)
             .rm(b)
             .imm(4)
-            .encode(&mut sink0);
+            .encode(&mut sink);
+
+        let bytes = sink.finish().data;
+        assert_eq!(bytes.as_slice(), [0xc4, 0x41, 0x24, 0xc2, 0xd4, 0x04]);
+    }
+
+    #[test]
+    fn vandnps() {
+        // VEX.128.0F 55 /r
+        // VANDNPS xmm0, xmm1, xmm2
+
+        let dst = regs::xmm2().to_real_reg().unwrap().hw_enc();
+        let src1 = regs::xmm1().to_real_reg().unwrap().hw_enc();
+        let src2 = regs::xmm0().to_real_reg().unwrap().hw_enc();
+        let mut sink = MachBuffer::new();
+
+        VexInstruction::new()
+            .length(VexVectorLength::V128)
+            .prefix(LegacyPrefixes::None)
+            .map(OpcodeMap::_0F)
+            .opcode(0x55)
+            .reg(dst)
+            .vvvv(src1)
+            .rm(src2)
+            .encode(&mut sink);
+
+        let bytes = sink.finish().data;
+        assert_eq!(bytes.as_slice(), [0xc5, 0xf0, 0x55, 0xd0]);
+    }
+
+    #[test]
+    fn vandnps_mem() {
+        // VEX.128.0F 55 /r
+        // VANDNPS 10(%r13), xmm1, xmm2
+
+        let dst = regs::xmm2().to_real_reg().unwrap().hw_enc();
+        let src1 = regs::xmm1().to_real_reg().unwrap().hw_enc();
+        let src2 = Amode::ImmReg {
+            base: regs::r13(),
+            flags: MemFlags::trusted(),
+            simm32: 10,
+        };
+        let mut sink = MachBuffer::new();
+
+        VexInstruction::new()
+            .length(VexVectorLength::V128)
+            .prefix(LegacyPrefixes::None)
+            .map(OpcodeMap::_0F)
+            .opcode(0x55)
+            .reg(dst)
+            .vvvv(src1)
+            .rm(src2)
+            .encode(&mut sink);
+
+        let bytes = sink.finish().data;
+        assert_eq!(bytes.as_slice(), [0xc4, 0xc1, 0x70, 0x55, 0x55, 0x0a]);
+    }
+
+    #[test]
+    fn vandnps_more_mem() {
+        // VEX.128.0F 55 /r
+        // VANDNPS 100(%rax,%r13,4), xmm1, xmm2
+
+        let dst = regs::xmm2().to_real_reg().unwrap().hw_enc();
+        let src1 = regs::xmm1().to_real_reg().unwrap().hw_enc();
+        let src2 = Amode::ImmRegRegShift {
+            base: Gpr::new(regs::rax()).unwrap(),
+            index: Gpr::new(regs::r13()).unwrap(),
+            flags: MemFlags::trusted(),
+            simm32: 100,
+            shift: 2,
+        };
+        let mut sink = MachBuffer::new();
+
+        VexInstruction::new()
+            .length(VexVectorLength::V128)
+            .prefix(LegacyPrefixes::None)
+            .map(OpcodeMap::_0F)
+            .opcode(0x55)
+            .reg(dst)
+            .vvvv(src1)
+            .rm(src2)
+            .encode(&mut sink);
 
-        assert_eq!(sink0, vec![0xc4, 0x41, 0x24, 0xc2, 0xd4, 0x04]);
+        let bytes = sink.finish().data;
+        assert_eq!(bytes.as_slice(), [0xc4, 0xa1, 0x70, 0x55, 0x54, 0xa8, 100]);
     }
 }
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 715d9fd87..f0f3c70d0 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -227,8 +227,24 @@
          (mask Xmm)
          (dst WritableXmm))
 
-       ;; XMM (scalar or vector) binary op that relies on the VEX prefix.
-       (XmmRmRVex (op AvxOpcode)
+       ;; XMM (scalar or vector) binary op that relies on the VEX prefix and
+       ;; has two inputs.
+       (XmmRmiRVex (op AvxOpcode)
+                   (src1 Xmm)
+                   (src2 XmmMemImm)
+                   (dst WritableXmm))
+
+       ;; XMM (scalar or vector) ternary op that relies on the VEX prefix and
+       ;; has two dynamic inputs plus one immediate input.
+       (XmmRmRImmVex (op AvxOpcode)
+                     (src1 Xmm)
+                     (src2 XmmMem)
+                     (dst WritableXmm)
+                     (imm u8))
+
+       ;; XMM (scalar or vector) ternary op that relies on the VEX prefix and
+       ;; has three dynamic inputs.
+       (XmmRmRVex3 (op AvxOpcode)
                    (src1 Xmm)
                    (src2 Xmm)
                    (src3 XmmMem)
@@ -1132,11 +1148,16 @@
 (decl cc_nz_or_z (CC) CC)
 (extern extractor cc_nz_or_z cc_nz_or_z)
 
-(type AvxOpcode extern
+(type AvxOpcode
       (enum Vfmadd213ss
             Vfmadd213sd
             Vfmadd213ps
-            Vfmadd213pd))
+            Vfmadd213pd
+            Vminps
+            Vorps
+            Vandnps
+            Vcmpps
+            Vpsrld))
 
 (type Avx512Opcode extern
       (enum Vcvtudq2ps
@@ -1226,6 +1247,10 @@
 (decl xmm_to_xmm_mem_imm (Xmm) XmmMemImm)
 (extern constructor xmm_to_xmm_mem_imm xmm_to_xmm_mem_imm)
 
+;; Convert an `XmmMem` into an `XmmMemImm`.
+(decl xmm_mem_to_xmm_mem_imm (XmmMem) XmmMemImm)
+(extern constructor xmm_mem_to_xmm_mem_imm xmm_mem_to_xmm_mem_imm)
+
 ;; Allocate a new temporary GPR register.
 (decl temp_writable_gpr () WritableGpr)
 (extern constructor temp_writable_gpr temp_writable_gpr)
@@ -1438,6 +1463,9 @@
 (decl use_sse41 (bool) Type)
 (extern extractor infallible use_sse41 use_sse41)
 
+(decl pure has_avx () bool)
+(extern constructor has_avx has_avx)
+
 ;;;; Helpers for Merging and Sinking Immediates/Loads  ;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Extract a constant `Imm8Reg.Imm8` from a value operand.
@@ -2285,8 +2313,11 @@
 
 ;; Helper for creating `orps` instructions.
 (decl x64_orps (Xmm XmmMem) Xmm)
-(rule (x64_orps src1 src2)
+(rule 0 (x64_orps src1 src2)
       (xmm_rm_r (SseOpcode.Orps) src1 src2))
+(rule 1 (x64_orps src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vorps) src1 src2))
 
 ;; Helper for creating `orpd` instructions.
 (decl x64_orpd (Xmm XmmMem) Xmm)
@@ -2360,8 +2391,11 @@
 
 ;; Helper for creating `andnps` instructions.
 (decl x64_andnps (Xmm XmmMem) Xmm)
-(rule (x64_andnps src1 src2)
+(rule 0 (x64_andnps src1 src2)
       (xmm_rm_r (SseOpcode.Andnps) src1 src2))
+(rule 1 (x64_andnps src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vandnps) src1 src2))
 
 ;; Helper for creating `andnpd` instructions.
 (decl x64_andnpd (Xmm XmmMem) Xmm)
@@ -2602,12 +2636,18 @@
 (rule (x64_cmpp $F64X2 x y imm) (x64_cmppd x y imm))
 
 (decl x64_cmpps (Xmm XmmMem FcmpImm) Xmm)
-(rule (x64_cmpps src1 src2 imm)
+(rule 0 (x64_cmpps src1 src2 imm)
       (xmm_rm_r_imm (SseOpcode.Cmpps)
                     src1
                     src2
                     (encode_fcmp_imm imm)
                     (OperandSize.Size32)))
+(rule 1 (x64_cmpps src1 src2 imm)
+      (if-let $true (has_avx))
+      (xmm_rmr_imm_vex (AvxOpcode.Vcmpps)
+                       src1
+                       src2
+                       (encode_fcmp_imm imm)))
 
 ;; Note that `Size32` is intentional despite this being used for 64-bit
 ;; operations, since this presumably induces the correct encoding of the
@@ -2858,8 +2898,11 @@
 
 ;; Helper for creating `psrld` instructions.
 (decl x64_psrld (Xmm XmmMemImm) Xmm)
-(rule (x64_psrld src1 src2)
+(rule 0 (x64_psrld src1 src2)
       (xmm_rmi_xmm (SseOpcode.Psrld) src1 src2))
+(rule 1 (x64_psrld src1 src2)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vpsrld) src1 src2))
 
 ;; Helper for creating `psrlq` instructions.
 (decl x64_psrlq (Xmm XmmMemImm) Xmm)
@@ -3070,10 +3113,11 @@
 
 ;; Helper for creating `minps` instructions.
 (decl x64_minps (Xmm Xmm) Xmm)
-(rule (x64_minps x y)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Minps) x y dst))))
-        dst))
+(rule 0 (x64_minps x y)
+      (xmm_rm_r (SseOpcode.Minps) x y))
+(rule 1 (x64_minps x y)
+      (if-let $true (has_avx))
+      (xmm_rmir_vex (AvxOpcode.Vminps) x y))
 
 ;; Helper for creating `minpd` instructions.
 (decl x64_minpd (Xmm Xmm) Xmm)
@@ -3101,15 +3145,25 @@
       (xmm_rm_r (SseOpcode.Maxpd) x y))
 
 
-;; Helper for creating `MInst.XmmRmRVex` instructions.
-(decl xmm_rmr_vex (AvxOpcode Xmm Xmm XmmMem) Xmm)
-(rule (xmm_rmr_vex op src1 src2 src3)
+;; Helper for creating `MInst.XmmRmiRVex` instructions.
+(decl xmm_rmir_vex (AvxOpcode Xmm XmmMemImm) Xmm)
+(rule (xmm_rmir_vex op src1 src2)
       (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.XmmRmRVex op
-                                           src1
-                                           src2
-                                           src3
-                                           dst))))
+            (_ Unit (emit (MInst.XmmRmiRVex op src1 src2 dst))))
+        dst))
+
+;; Helper for creating `MInst.XmmRmRImmVex` instructions.
+(decl xmm_rmr_imm_vex (AvxOpcode Xmm XmmMem u8) Xmm)
+(rule (xmm_rmr_imm_vex op src1 src2 imm)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmRmRImmVex op src1 src2 dst imm))))
+        dst))
+
+;; Helper for creating `MInst.XmmRmRVex3` instructions.
+(decl xmm_rmr_vex3 (AvxOpcode Xmm Xmm XmmMem) Xmm)
+(rule (xmm_rmr_vex3 op src1 src2 src3)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmRmRVex3 op src1 src2 src3 dst))))
         dst))
 
 ;; Helper for creating `vfmadd213ss` instructions.
@@ -3117,28 +3171,28 @@
 ; but we don't support VEX memory encodings yet
 (decl x64_vfmadd213ss (Xmm Xmm Xmm) Xmm)
 (rule (x64_vfmadd213ss x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213ss) x y z))
+      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213ss) x y z))
 
 ;; Helper for creating `vfmadd213sd` instructions.
 ; TODO: This should have the (Xmm Xmm XmmMem) signature
 ; but we don't support VEX memory encodings yet
 (decl x64_vfmadd213sd (Xmm Xmm Xmm) Xmm)
 (rule (x64_vfmadd213sd x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213sd) x y z))
+      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213sd) x y z))
 
 ;; Helper for creating `vfmadd213ps` instructions.
 ; TODO: This should have the (Xmm Xmm XmmMem) signature
 ; but we don't support VEX memory encodings yet
 (decl x64_vfmadd213ps (Xmm Xmm Xmm) Xmm)
 (rule (x64_vfmadd213ps x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213ps) x y z))
+      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213ps) x y z))
 
 ;; Helper for creating `vfmadd213pd` instructions.
 ; TODO: This should have the (Xmm Xmm XmmMem) signature
 ; but we don't support VEX memory encodings yet
 (decl x64_vfmadd213pd (Xmm Xmm Xmm) Xmm)
 (rule (x64_vfmadd213pd x y z)
-      (xmm_rmr_vex (AvxOpcode.Vfmadd213pd) x y z))
+      (xmm_rmr_vex3 (AvxOpcode.Vfmadd213pd) x y z))
 
 
 ;; Helper for creating `sqrtss` instructions.
@@ -3836,6 +3890,7 @@
 (convert RegMemImm XmmMemImm mov_rmi_to_xmm)
 (convert Xmm XmmMem xmm_to_xmm_mem)
 (convert Xmm XmmMemImm xmm_to_xmm_mem_imm)
+(convert XmmMem XmmMemImm xmm_mem_to_xmm_mem_imm)
 (convert XmmMem RegMem xmm_mem_to_reg_mem)
 (convert WritableXmm Xmm writable_xmm_to_xmm)
 (convert WritableXmm WritableReg writable_xmm_to_reg)
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 76787bfb6..c4de650e4 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -165,6 +165,12 @@ macro_rules! newtype_of_reg {
             }
         }
 
+        impl From<$newtype_reg_mem> for $newtype_reg_mem_imm {
+            fn from(r: $newtype_reg_mem) -> Self {
+                $newtype_reg_mem_imm(r.0.into())
+            }
+        }
+
         impl $newtype_reg_mem_imm {
             /// Construct this newtype from the given `RegMemImm`, or return
             /// `None` if the `RegMemImm` is not a valid instance of this
@@ -631,6 +637,15 @@ impl RegMemImm {
     }
 }
 
+impl From<RegMem> for RegMemImm {
+    fn from(rm: RegMem) -> RegMemImm {
+        match rm {
+            RegMem::Reg { reg } => RegMemImm::Reg { reg },
+            RegMem::Mem { addr } => RegMemImm::Mem { addr },
+        }
+    }
+}
+
 impl PrettyPrint for RegMemImm {
     fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
         match self {
@@ -730,6 +745,12 @@ impl RegMem {
     }
 }
 
+impl From<Reg> for RegMem {
+    fn from(reg: Reg) -> RegMem {
+        RegMem::Reg { reg }
+    }
+}
+
 impl From<Writable<Reg>> for RegMem {
     fn from(r: Writable<Reg>) -> Self {
         RegMem::reg(r.to_reg())
@@ -884,6 +905,7 @@ pub(crate) enum InstructionSet {
     #[allow(dead_code)] // never constructed (yet).
     BMI2,
     FMA,
+    AVX,
     AVX512BITALG,
     AVX512DQ,
     AVX512F,
@@ -1477,14 +1499,7 @@ impl fmt::Display for SseOpcode {
     }
 }
 
-#[derive(Clone, PartialEq)]
-#[allow(missing_docs)]
-pub enum AvxOpcode {
-    Vfmadd213ss,
-    Vfmadd213sd,
-    Vfmadd213ps,
-    Vfmadd213pd,
-}
+pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;
 
 impl AvxOpcode {
     /// Which `InstructionSet`s support the opcode?
@@ -1494,25 +1509,20 @@ impl AvxOpcode {
             | AvxOpcode::Vfmadd213sd
             | AvxOpcode::Vfmadd213ps
             | AvxOpcode::Vfmadd213pd => smallvec![InstructionSet::FMA],
+            AvxOpcode::Vminps
+            | AvxOpcode::Vorps
+            | AvxOpcode::Vandnps
+            | AvxOpcode::Vcmpps
+            | AvxOpcode::Vpsrld => {
+                smallvec![InstructionSet::AVX]
+            }
         }
     }
 }
 
-impl fmt::Debug for AvxOpcode {
-    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        let name = match self {
-            AvxOpcode::Vfmadd213ss => "vfmadd213ss",
-            AvxOpcode::Vfmadd213sd => "vfmadd213sd",
-            AvxOpcode::Vfmadd213ps => "vfmadd213ps",
-            AvxOpcode::Vfmadd213pd => "vfmadd213pd",
-        };
-        write!(fmt, "{}", name)
-    }
-}
-
 impl fmt::Display for AvxOpcode {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        fmt::Debug::fmt(self, f)
+        format!("{self:?}").to_lowercase().fmt(f)
     }
 }
 
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 830565eff..9d984edab 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -8,7 +8,7 @@ use crate::isa::x64::encoding::rex::{
     low8_will_sign_extend_to_32, low8_will_sign_extend_to_64, reg_enc, LegacyPrefixes, OpcodeMap,
     RexFlags,
 };
-use crate::isa::x64::encoding::vex::{VexInstruction, VexVectorLength};
+use crate::isa::x64::encoding::vex::{RegisterOrAmode, VexInstruction, VexVectorLength};
 use crate::isa::x64::inst::args::*;
 use crate::isa::x64::inst::*;
 use crate::machinst::{inst_common, MachBuffer, MachInstEmit, MachLabel, Reg, Writable};
@@ -121,6 +121,7 @@ pub(crate) fn emit(
             InstructionSet::BMI1 => info.isa_flags.use_bmi1(),
             InstructionSet::BMI2 => info.isa_flags.has_bmi2(),
             InstructionSet::FMA => info.isa_flags.has_fma(),
+            InstructionSet::AVX => info.isa_flags.has_avx(),
             InstructionSet::AVX512BITALG => info.isa_flags.has_avx512bitalg(),
             InstructionSet::AVX512DQ => info.isa_flags.has_avx512dq(),
             InstructionSet::AVX512F => info.isa_flags.has_avx512f(),
@@ -1991,7 +1992,94 @@ pub(crate) fn emit(
             }
         }
 
-        Inst::XmmRmRVex {
+        Inst::XmmRmiRVex {
+            op,
+            src1,
+            src2,
+            dst,
+        } => {
+            let dst = allocs.next(dst.to_reg().to_reg());
+            let src1 = allocs.next(src1.to_reg());
+            let src2 = src2.clone().to_reg_mem_imm().with_allocs(allocs);
+
+            let src2 = match src2 {
+                // For opcodes where one of the operands is an immediate the
+                // encoding is a bit different, notably the usage of
+                // `opcode_ext`, so handle that specially here.
+                RegMemImm::Imm { simm32 } => {
+                    let (opcode, opcode_ext, prefix) = match op {
+                        AvxOpcode::Vpsrld => (0x72, 2, LegacyPrefixes::_66),
+                        _ => panic!("unexpected avx opcode with immediate {op:?}"),
+                    };
+                    VexInstruction::new()
+                        .length(VexVectorLength::V128)
+                        .prefix(prefix)
+                        .map(OpcodeMap::_0F)
+                        .opcode(opcode)
+                        .opcode_ext(opcode_ext)
+                        .vvvv(dst.to_real_reg().unwrap().hw_enc())
+                        .prefix(LegacyPrefixes::_66)
+                        .rm(src1.to_real_reg().unwrap().hw_enc())
+                        .imm(simm32.try_into().unwrap())
+                        .encode(sink);
+                    return;
+                }
+                RegMemImm::Reg { reg } => {
+                    RegisterOrAmode::Register(reg.to_real_reg().unwrap().hw_enc().into())
+                }
+                RegMemImm::Mem { addr } => RegisterOrAmode::Amode(addr.finalize(state, sink)),
+            };
+            let (prefix, opcode) = match op {
+                AvxOpcode::Vminps => (LegacyPrefixes::None, 0x5D),
+                AvxOpcode::Vandnps => (LegacyPrefixes::None, 0x55),
+                AvxOpcode::Vorps => (LegacyPrefixes::None, 0x56),
+                AvxOpcode::Vpsrld => (LegacyPrefixes::_66, 0xD2),
+                _ => panic!("unexpected rmir vex opcode {op:?}"),
+            };
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .prefix(prefix)
+                .opcode(opcode)
+                .map(OpcodeMap::_0F)
+                .reg(dst.to_real_reg().unwrap().hw_enc())
+                .vvvv(src1.to_real_reg().unwrap().hw_enc())
+                .rm(src2)
+                .encode(sink);
+        }
+
+        Inst::XmmRmRImmVex {
+            op,
+            src1,
+            src2,
+            dst,
+            imm,
+        } => {
+            let dst = allocs.next(dst.to_reg().to_reg());
+            let src1 = allocs.next(src1.to_reg());
+            let src2 = src2.clone().to_reg_mem().with_allocs(allocs);
+
+            let (w, opcode) = match op {
+                AvxOpcode::Vcmpps => (false, 0xC2),
+                _ => unreachable!(),
+            };
+
+            match src2 {
+                RegMem::Reg { reg: src } => VexInstruction::new()
+                    .length(VexVectorLength::V128)
+                    .prefix(LegacyPrefixes::None)
+                    .map(OpcodeMap::_0F)
+                    .w(w)
+                    .opcode(opcode)
+                    .reg(dst.to_real_reg().unwrap().hw_enc())
+                    .rm(src.to_real_reg().unwrap().hw_enc())
+                    .vvvv(src1.to_real_reg().unwrap().hw_enc())
+                    .imm(*imm)
+                    .encode(sink),
+                _ => todo!(),
+            };
+        }
+
+        Inst::XmmRmRVex3 {
             op,
             src1,
             src2,
@@ -2009,6 +2097,7 @@ pub(crate) fn emit(
                 AvxOpcode::Vfmadd213sd => (true, 0xA9),
                 AvxOpcode::Vfmadd213ps => (false, 0xA8),
                 AvxOpcode::Vfmadd213pd => (true, 0xA8),
+                _ => unreachable!(),
             };
 
             match src3 {
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 6716ec21b..a1056b859 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -142,7 +142,9 @@ impl Inst {
             | Inst::XmmRmREvex { op, .. }
             | Inst::XmmRmREvex3 { op, .. } => op.available_from(),
 
-            Inst::XmmRmRVex { op, .. } => op.available_from(),
+            Inst::XmmRmiRVex { op, .. }
+            | Inst::XmmRmRVex3 { op, .. }
+            | Inst::XmmRmRImmVex { op, .. } => op.available_from(),
         }
     }
 }
@@ -303,11 +305,11 @@ impl Inst {
     }
 
     #[cfg(test)]
-    pub(crate) fn xmm_rm_r_vex(op: AvxOpcode, src3: RegMem, src2: Reg, dst: Writable<Reg>) -> Self {
+    pub(crate) fn xmm_rmr_vex3(op: AvxOpcode, src3: RegMem, src2: Reg, dst: Writable<Reg>) -> Self {
         src3.assert_regclass_is(RegClass::Float);
         debug_assert!(src2.class() == RegClass::Float);
         debug_assert!(dst.to_reg().class() == RegClass::Float);
-        Inst::XmmRmRVex {
+        Inst::XmmRmRVex3 {
             op,
             src3: XmmMem::new(src3).unwrap(),
             src2: Xmm::new(src2).unwrap(),
@@ -988,7 +990,42 @@ impl PrettyPrint for Inst {
                 )
             }
 
-            Inst::XmmRmRVex {
+            Inst::XmmRmiRVex {
+                op,
+                src1,
+                src2,
+                dst,
+                ..
+            } => {
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
+                let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
+                let src2 = src2.pretty_print(8, allocs);
+
+                format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
+            }
+
+            Inst::XmmRmRImmVex {
+                op,
+                src1,
+                src2,
+                dst,
+                imm,
+                ..
+            } => {
+                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
+                let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
+                let src2 = src2.pretty_print(8, allocs);
+
+                format!(
+                    "{} ${imm} {}, {}, {}",
+                    ljustify(op.to_string()),
+                    src1,
+                    src2,
+                    dst
+                )
+            }
+
+            Inst::XmmRmRVex3 {
                 op,
                 src1,
                 src2,
@@ -1892,7 +1929,21 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_reuse_def(dst.to_writable_reg(), 0);
             src2.get_operands(collector);
         }
-        Inst::XmmRmRVex {
+        Inst::XmmRmiRVex {
+            src1, src2, dst, ..
+        } => {
+            collector.reg_def(dst.to_writable_reg());
+            collector.reg_use(src1.to_reg());
+            src2.get_operands(collector);
+        }
+        Inst::XmmRmRImmVex {
+            src1, src2, dst, ..
+        } => {
+            collector.reg_def(dst.to_writable_reg());
+            collector.reg_use(src1.to_reg());
+            src2.get_operands(collector);
+        }
+        Inst::XmmRmRVex3 {
             op,
             src1,
             src2,
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 26766dc3d..8adbf9446 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -185,6 +185,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
         imm.encode()
     }
 
+    #[inline]
+    fn has_avx(&mut self) -> bool {
+        self.backend.x64_flags.has_avx()
+    }
+
     #[inline]
     fn avx512vl_enabled(&mut self, _: Type) -> bool {
         self.backend.x64_flags.use_avx512vl_simd()
@@ -439,6 +444,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
         r.into()
     }
 
+    #[inline]
+    fn xmm_mem_to_xmm_mem_imm(&mut self, r: &XmmMem) -> XmmMemImm {
+        r.clone().into()
+    }
+
     #[inline]
     fn temp_writable_gpr(&mut self) -> WritableGpr {
         Writable::from_reg(Gpr::new(self.temp_writable_reg(I64).to_reg()).unwrap())
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index 5115c5b01..c199aa15a 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -10,8 +10,8 @@ use crate::fx::{FxHashMap, FxHashSet};
 use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
 use crate::ir::{
     ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function,
-    GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, RelSourceLoc,
-    Type, Value, ValueDef, ValueLabelAssignments, ValueLabelStart,
+    GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, RelSourceLoc, Type,
+    Value, ValueDef, ValueLabelAssignments, ValueLabelStart,
 };
 use crate::machinst::{
     writable_value_regs, BlockIndex, BlockLoweringOrder, Callee, LoweredBlock, MachLabel, Reg,
@@ -898,39 +898,29 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         bindex: BlockIndex,
         // Original CLIF block:
         block: Block,
-        branches: &SmallVec<[Inst; 2]>,
-        targets: &SmallVec<[MachLabel; 2]>,
+        branch: Inst,
+        targets: &[MachLabel],
     ) -> CodegenResult<()> {
         trace!(
-            "lower_clif_branches: block {} branches {:?} targets {:?}",
+            "lower_clif_branches: block {} branch {:?} targets {:?}",
             block,
-            branches,
+            branch,
             targets,
         );
-        // A block should end with at most two branches. The first may be a
-        // conditional branch; a conditional branch can be followed only by an
-        // unconditional branch or fallthrough. Otherwise, if only one branch,
-        // it may be an unconditional branch, a fallthrough, a return, or a
-        // trap. These conditions are verified by `is_block_basic()` during the
-        // verifier pass.
-        assert!(branches.len() <= 2);
-        if branches.len() == 2 {
-            assert!(self.data(branches[1]).opcode() == Opcode::Jump);
-        }
         // When considering code-motion opportunities, consider the current
-        // program point to be the first branch.
-        self.cur_inst = Some(branches[0]);
-        // Lower the first branch in ISLE.  This will automatically handle
-        // the second branch (if any) by emitting a two-way conditional branch.
+        // program point to be this branch.
+        self.cur_inst = Some(branch);
+
+        // Lower the branch in ISLE.
         backend
-            .lower_branch(self, branches[0], targets)
+            .lower_branch(self, branch, targets)
             .unwrap_or_else(|| {
                 panic!(
                     "should be implemented in ISLE: branch = `{}`",
-                    self.f.dfg.display_inst(branches[0]),
+                    self.f.dfg.display_inst(branch),
                 )
             });
-        let loc = self.srcloc(branches[0]);
+        let loc = self.srcloc(branch);
         self.finish_ir_inst(loc);
         // Add block param outputs for current block.
         self.lower_branch_blockparam_args(bindex);
@@ -967,26 +957,20 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         &self,
         bindex: BlockIndex,
         _bb: Block,
-        branches: &mut SmallVec<[Inst; 2]>,
         targets: &mut SmallVec<[MachLabel; 2]>,
-    ) {
-        branches.clear();
+    ) -> Option<Inst> {
         targets.clear();
         let mut last_inst = None;
         for &(inst, succ) in self.vcode.block_order().succ_indices(bindex) {
-            // Avoid duplicates: this ensures a br_table is only inserted once.
-            if last_inst != Some(inst) {
-                branches.push(inst);
-            } else {
-                debug_assert!(
-                    self.f.dfg.insts[inst].opcode() == Opcode::BrTable
-                        || self.f.dfg.insts[inst].opcode() == Opcode::Brif
-                );
-                debug_assert!(branches.len() == 1);
-            }
+            // Basic blocks may end in a single branch instruction, but those instructions may have
+            // multiple destinations. As such, all `inst` values in `succ_indices` must be the
+            // same, or this basic block would have multiple branch instructions present.
+            debug_assert!(last_inst.map_or(true, |prev| prev == inst));
             last_inst = Some(inst);
             targets.push(MachLabel::from_block(succ));
         }
+
+        last_inst
     }
 
     /// Lower the function.
@@ -1010,7 +994,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         self.vcode.set_entry(BlockIndex::new(0));
 
         // Reused vectors for branch lowering.
-        let mut branches: SmallVec<[Inst; 2]> = SmallVec::new();
         let mut targets: SmallVec<[MachLabel; 2]> = SmallVec::new();
 
         // get a copy of the lowered order; we hold this separately because we
@@ -1032,10 +1015,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
             // End branches.
             if let Some(bb) = lb.orig_block() {
-                self.collect_branches_and_targets(bindex, bb, &mut branches, &mut targets);
-                if branches.len() > 0 {
-                    self.lower_clif_branches(backend, bindex, bb, &branches, &targets)?;
-                    self.finish_ir_inst(self.srcloc(branches[0]));
+                if let Some(branch) = self.collect_branches_and_targets(bindex, bb, &mut targets) {
+                    self.lower_clif_branches(backend, bindex, bb, branch, &targets)?;
+                    self.finish_ir_inst(self.srcloc(branch));
                 }
             } else {
                 // If no orig block, this must be a pure edge block;
diff --git a/cranelift/codegen/src/opts/algebraic.isle b/cranelift/codegen/src/opts/algebraic.isle
index c25fb5a1c..da177603e 100644
--- a/cranelift/codegen/src/opts/algebraic.isle
+++ b/cranelift/codegen/src/opts/algebraic.isle
@@ -42,7 +42,7 @@
                       x))
       (subsume x))
 
-;; x*0 == 0*x == x.
+;; x*0 == 0*x == 0.
 (rule (simplify (imul ty
                       _
                       zero @ (iconst ty (u64_from_imm64 0))))
diff --git a/cranelift/fuzzgen/src/config.rs b/cranelift/fuzzgen/src/config.rs
index 971f6d27e..2ccdf9ef6 100644
--- a/cranelift/fuzzgen/src/config.rs
+++ b/cranelift/fuzzgen/src/config.rs
@@ -30,7 +30,8 @@ pub struct Config {
     pub switch_cases: RangeInclusive<usize>,
     pub switch_max_range_size: RangeInclusive<usize>,
 
-    pub funcrefs_per_function: RangeInclusive<usize>,
+    /// Number of distinct functions in the same testsuite that we allow calling per function.
+    pub usercalls: RangeInclusive<usize>,
 
     /// Stack slots.
     /// The combination of these two determines stack usage per function
@@ -79,7 +80,7 @@ impl Default for Config {
             switch_cases: 0..=64,
             // Ranges smaller than 2 don't make sense.
             switch_max_range_size: 2..=32,
-            funcrefs_per_function: 0..=8,
+            usercalls: 0..=8,
             static_stack_slots_per_function: 0..=8,
             static_stack_slot_size: 0..=128,
             // We need the mix of sizes that allows us to:
diff --git a/cranelift/fuzzgen/src/cranelift_arbitrary.rs b/cranelift/fuzzgen/src/cranelift_arbitrary.rs
new file mode 100644
index 000000000..ca59135ae
--- /dev/null
+++ b/cranelift/fuzzgen/src/cranelift_arbitrary.rs
@@ -0,0 +1,94 @@
+use crate::codegen::ir::{ArgumentExtension, ArgumentPurpose};
+use anyhow::Result;
+use cranelift::codegen::data_value::DataValue;
+use cranelift::codegen::ir::types::*;
+use cranelift::codegen::ir::{AbiParam, Signature, Type};
+use cranelift::codegen::isa::CallConv;
+
+use arbitrary::Unstructured;
+use cranelift::prelude::{Ieee32, Ieee64};
+
+/// A trait for generating random Cranelift datastructures.
+pub trait CraneliftArbitrary {
+    fn _type(&mut self) -> Result<Type>;
+    fn callconv(&mut self) -> Result<CallConv>;
+    fn abi_param(&mut self) -> Result<AbiParam>;
+    fn signature(&mut self, max_params: usize, max_rets: usize) -> Result<Signature>;
+    fn datavalue(&mut self, ty: Type) -> Result<DataValue>;
+}
+
+impl<'a> CraneliftArbitrary for &mut Unstructured<'a> {
+    fn _type(&mut self) -> Result<Type> {
+        // TODO: It would be nice if we could get these directly from cranelift
+        let scalars = [
+            I8, I16, I32, I64, I128, F32, F64,
+            // R32, R64,
+        ];
+        // TODO: vector types
+
+        let ty = self.choose(&scalars[..])?;
+        Ok(*ty)
+    }
+
+    fn callconv(&mut self) -> Result<CallConv> {
+        // TODO: Generate random CallConvs per target
+        Ok(CallConv::SystemV)
+    }
+
+    fn abi_param(&mut self) -> Result<AbiParam> {
+        let value_type = self._type()?;
+        // TODO: There are more argument purposes to be explored...
+        let purpose = ArgumentPurpose::Normal;
+        let extension = if value_type.is_int() {
+            *self.choose(&[
+                ArgumentExtension::Sext,
+                ArgumentExtension::Uext,
+                ArgumentExtension::None,
+            ])?
+        } else {
+            ArgumentExtension::None
+        };
+
+        Ok(AbiParam {
+            value_type,
+            purpose,
+            extension,
+        })
+    }
+
+    fn signature(&mut self, max_params: usize, max_rets: usize) -> Result<Signature> {
+        let callconv = self.callconv()?;
+        let mut sig = Signature::new(callconv);
+
+        for _ in 0..max_params {
+            sig.params.push(self.abi_param()?);
+        }
+
+        for _ in 0..max_rets {
+            sig.returns.push(self.abi_param()?);
+        }
+
+        Ok(sig)
+    }
+
+    fn datavalue(&mut self, ty: Type) -> Result<DataValue> {
+        Ok(match ty {
+            ty if ty.is_int() => {
+                let imm = match ty {
+                    I8 => self.arbitrary::<i8>()? as i128,
+                    I16 => self.arbitrary::<i16>()? as i128,
+                    I32 => self.arbitrary::<i32>()? as i128,
+                    I64 => self.arbitrary::<i64>()? as i128,
+                    I128 => self.arbitrary::<i128>()?,
+                    _ => unreachable!(),
+                };
+                DataValue::from_integer(imm, ty)?
+            }
+            // f{32,64}::arbitrary does not generate a bunch of important values
+            // such as Signaling NaN's / NaN's with payload, so generate floats from integers.
+            F32 => DataValue::F32(Ieee32::with_bits(self.arbitrary::<u32>()?)),
+            F64 => DataValue::F64(Ieee64::with_bits(self.arbitrary::<u64>()?)),
+            _ => unimplemented!(),
+        })
+    }
+}
diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs
index fb8871dff..028464d56 100644
--- a/cranelift/fuzzgen/src/function_generator.rs
+++ b/cranelift/fuzzgen/src/function_generator.rs
@@ -1,12 +1,13 @@
-use crate::codegen::ir::{ArgumentExtension, ArgumentPurpose};
 use crate::config::Config;
+use crate::cranelift_arbitrary::CraneliftArbitrary;
 use anyhow::Result;
 use arbitrary::{Arbitrary, Unstructured};
+use cranelift::codegen::data_value::DataValue;
 use cranelift::codegen::ir::instructions::InstructionFormat;
 use cranelift::codegen::ir::stackslot::StackSize;
 use cranelift::codegen::ir::{types::*, FuncRef, LibCall, UserExternalName, UserFuncName};
 use cranelift::codegen::ir::{
-    AbiParam, Block, ExternalName, Function, Opcode, Signature, StackSlot, Type, Value,
+    Block, ExternalName, Function, Opcode, Signature, StackSlot, Type, Value,
 };
 use cranelift::codegen::isa::CallConv;
 use cranelift::frontend::{FunctionBuilder, FunctionBuilderContext, Switch, Variable};
@@ -1279,16 +1280,6 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Call, &[], &[], insert_call),
 ];
 
-/// These libcalls need a interpreter implementation in `cranelift-fuzzgen.rs`
-const ALLOWED_LIBCALLS: &'static [LibCall] = &[
-    LibCall::CeilF32,
-    LibCall::CeilF64,
-    LibCall::FloorF32,
-    LibCall::FloorF64,
-    LibCall::TruncF32,
-    LibCall::TruncF64,
-];
-
 pub struct FunctionGenerator<'r, 'data>
 where
     'data: 'r,
@@ -1297,6 +1288,8 @@ where
     config: &'r Config,
     resources: Resources,
     target_triple: Triple,
+    name: UserFuncName,
+    signature: Signature,
 }
 
 #[derive(Debug, Clone)]
@@ -1325,6 +1318,8 @@ struct Resources {
     block_terminators: Vec<BlockTerminator>,
     func_refs: Vec<(Signature, FuncRef)>,
     stack_slots: Vec<(StackSlot, StackSize)>,
+    usercalls: Vec<(UserExternalName, Signature)>,
+    libcalls: Vec<LibCall>,
 }
 
 impl Resources {
@@ -1359,12 +1354,26 @@ impl<'r, 'data> FunctionGenerator<'r, 'data>
 where
     'data: 'r,
 {
-    pub fn new(u: &'r mut Unstructured<'data>, config: &'r Config, target_triple: Triple) -> Self {
+    pub fn new(
+        u: &'r mut Unstructured<'data>,
+        config: &'r Config,
+        target_triple: Triple,
+        name: UserFuncName,
+        signature: Signature,
+        usercalls: Vec<(UserExternalName, Signature)>,
+        libcalls: Vec<LibCall>,
+    ) -> Self {
         Self {
             u,
             config,
-            resources: Resources::default(),
+            resources: Resources {
+                usercalls,
+                libcalls,
+                ..Resources::default()
+            },
             target_triple,
+            name,
+            signature,
         }
     }
 
@@ -1373,65 +1382,12 @@ where
         Ok(self.u.int_in_range(param.clone())?)
     }
 
-    fn generate_callconv(&mut self) -> Result<CallConv> {
-        // TODO: Generate random CallConvs per target
-        Ok(CallConv::SystemV)
-    }
-
     fn system_callconv(&mut self) -> CallConv {
         // TODO: This currently only runs on linux, so this is the only choice
         // We should improve this once we generate flags and targets
         CallConv::SystemV
     }
 
-    fn generate_type(&mut self) -> Result<Type> {
-        // TODO: It would be nice if we could get these directly from cranelift
-        let scalars = [
-            I8, I16, I32, I64, I128, F32, F64,
-            // R32, R64,
-        ];
-        // TODO: vector types
-
-        let ty = self.u.choose(&scalars[..])?;
-        Ok(*ty)
-    }
-
-    fn generate_abi_param(&mut self) -> Result<AbiParam> {
-        let value_type = self.generate_type()?;
-        // TODO: There are more argument purposes to be explored...
-        let purpose = ArgumentPurpose::Normal;
-        let extension = if value_type.is_int() {
-            *self.u.choose(&[
-                ArgumentExtension::Sext,
-                ArgumentExtension::Uext,
-                ArgumentExtension::None,
-            ])?
-        } else {
-            ArgumentExtension::None
-        };
-
-        Ok(AbiParam {
-            value_type,
-            purpose,
-            extension,
-        })
-    }
-
-    fn generate_signature(&mut self) -> Result<Signature> {
-        let callconv = self.generate_callconv()?;
-        let mut sig = Signature::new(callconv);
-
-        for _ in 0..self.param(&self.config.signature_params)? {
-            sig.params.push(self.generate_abi_param()?);
-        }
-
-        for _ in 0..self.param(&self.config.signature_rets)? {
-            sig.returns.push(self.generate_abi_param()?);
-        }
-
-        Ok(sig)
-    }
-
     /// Finds a stack slot with size of at least n bytes
     fn stack_slot_with_size(&mut self, n: u32) -> Result<(StackSlot, StackSize)> {
         let first = self
@@ -1494,31 +1450,18 @@ where
 
     /// Generates an instruction(`iconst`/`fconst`/etc...) to introduce a constant value
     fn generate_const(&mut self, builder: &mut FunctionBuilder, ty: Type) -> Result<Value> {
-        Ok(match ty {
-            I128 => {
-                // See: https://github.com/bytecodealliance/wasmtime/issues/2906
-                let hi = builder.ins().iconst(I64, self.u.arbitrary::<i64>()?);
-                let lo = builder.ins().iconst(I64, self.u.arbitrary::<i64>()?);
+        Ok(match self.u.datavalue(ty)? {
+            DataValue::I8(i) => builder.ins().iconst(ty, i as i64),
+            DataValue::I16(i) => builder.ins().iconst(ty, i as i64),
+            DataValue::I32(i) => builder.ins().iconst(ty, i as i64),
+            DataValue::I64(i) => builder.ins().iconst(ty, i as i64),
+            DataValue::I128(i) => {
+                let hi = builder.ins().iconst(I64, (i >> 64) as i64);
+                let lo = builder.ins().iconst(I64, i as i64);
                 builder.ins().iconcat(lo, hi)
             }
-            ty if ty.is_int() => {
-                let imm64 = match ty {
-                    I8 => self.u.arbitrary::<i8>()? as i64,
-                    I16 => self.u.arbitrary::<i16>()? as i64,
-                    I32 => self.u.arbitrary::<i32>()? as i64,
-                    I64 => self.u.arbitrary::<i64>()?,
-                    _ => unreachable!(),
-                };
-                builder.ins().iconst(ty, imm64)
-            }
-            // f{32,64}::arbitrary does not generate a bunch of important values
-            // such as Signaling NaN's / NaN's with payload, so generate floats from integers.
-            F32 => builder
-                .ins()
-                .f32const(f32::from_bits(u32::arbitrary(self.u)?)),
-            F64 => builder
-                .ins()
-                .f64const(f64::from_bits(u64::arbitrary(self.u)?)),
+            DataValue::F32(f) => builder.ins().f32const(f),
+            DataValue::F64(f) => builder.ins().f64const(f),
             _ => unimplemented!(),
         })
     }
@@ -1650,34 +1593,38 @@ where
     }
 
     fn generate_funcrefs(&mut self, builder: &mut FunctionBuilder) -> Result<()> {
-        let count = self.param(&self.config.funcrefs_per_function)?;
-        for func_index in 0..count.try_into().unwrap() {
-            let (ext_name, sig) = if self.u.arbitrary::<bool>()? {
-                let user_func_ref = builder
-                    .func
-                    .declare_imported_user_function(UserExternalName {
-                        namespace: 0,
-                        index: func_index,
-                    });
+        let usercalls: Vec<(ExternalName, Signature)> = self
+            .resources
+            .usercalls
+            .iter()
+            .map(|(name, signature)| {
+                let user_func_ref = builder.func.declare_imported_user_function(name.clone());
                 let name = ExternalName::User(user_func_ref);
-                let signature = self.generate_signature()?;
+                (name, signature.clone())
+            })
+            .collect();
+
+        let lib_callconv = self.system_callconv();
+        let libcalls: Vec<(ExternalName, Signature)> = self
+            .resources
+            .libcalls
+            .iter()
+            .map(|libcall| {
+                let signature = libcall.signature(lib_callconv);
+                let name = ExternalName::LibCall(*libcall);
                 (name, signature)
-            } else {
-                let libcall = *self.u.choose(ALLOWED_LIBCALLS)?;
-                // TODO: Use [CallConv::for_libcall] once we generate flags.
-                let callconv = self.system_callconv();
-                let signature = libcall.signature(callconv);
-                (ExternalName::LibCall(libcall), signature)
-            };
+            })
+            .collect();
 
-            let sig_ref = builder.import_signature(sig.clone());
+        for (name, signature) in usercalls.into_iter().chain(libcalls) {
+            let sig_ref = builder.import_signature(signature.clone());
             let func_ref = builder.import_function(ExtFuncData {
-                name: ext_name,
+                name,
                 signature: sig_ref,
                 colocated: self.u.arbitrary()?,
             });
 
-            self.resources.func_refs.push((sig, func_ref));
+            self.resources.func_refs.push((signature, func_ref));
         }
 
         Ok(())
@@ -1727,7 +1674,7 @@ where
     }
 
     /// Creates a random amount of blocks in this function
-    fn generate_blocks(&mut self, builder: &mut FunctionBuilder, sig: &Signature) -> Result<()> {
+    fn generate_blocks(&mut self, builder: &mut FunctionBuilder) -> Result<()> {
         let extra_block_count = self.param(&self.config.blocks_per_function)?;
 
         // We must always have at least one block, so we generate the "extra" blocks and add 1 for
@@ -1751,7 +1698,10 @@ where
                 // a random signature;
                 if is_entry {
                     builder.append_block_params_for_function_params(block);
-                    Ok((block, sig.params.iter().map(|a| a.value_type).collect()))
+                    Ok((
+                        block,
+                        self.signature.params.iter().map(|a| a.value_type).collect(),
+                    ))
                 } else {
                     let sig = self.generate_block_signature()?;
                     sig.iter().for_each(|ty| {
@@ -1882,7 +1832,7 @@ where
 
         let mut params = Vec::with_capacity(param_count);
         for _ in 0..param_count {
-            params.push(self.generate_type()?);
+            params.push(self.u._type()?);
         }
         Ok(params)
     }
@@ -1902,7 +1852,7 @@ where
 
         // Create a pool of vars that are going to be used in this function
         for _ in 0..self.param(&self.config.vars_per_function)? {
-            let ty = self.generate_type()?;
+            let ty = self.u._type()?;
             let value = self.generate_const(builder, ty)?;
             vars.push((ty, value));
         }
@@ -1930,15 +1880,12 @@ where
     /// Because we generate all blocks and variables up front we already know everything that
     /// we need when generating instructions (i.e. jump targets / variables)
     pub fn generate(mut self) -> Result<Function> {
-        let sig = self.generate_signature()?;
-
         let mut fn_builder_ctx = FunctionBuilderContext::new();
-        // function name must be in a different namespace than TESTFILE_NAMESPACE (0)
-        let mut func = Function::with_name_signature(UserFuncName::user(1, 0), sig.clone());
+        let mut func = Function::with_name_signature(self.name.clone(), self.signature.clone());
 
         let mut builder = FunctionBuilder::new(&mut func, &mut fn_builder_ctx);
 
-        self.generate_blocks(&mut builder, &sig)?;
+        self.generate_blocks(&mut builder)?;
 
         // Function preamble
         self.generate_funcrefs(&mut builder)?;
diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs
index e5048f79e..ee0c36423 100644
--- a/cranelift/fuzzgen/src/lib.rs
+++ b/cranelift/fuzzgen/src/lib.rs
@@ -4,33 +4,32 @@ use crate::settings::{Flags, OptLevel};
 use anyhow::Result;
 use arbitrary::{Arbitrary, Unstructured};
 use cranelift::codegen::data_value::DataValue;
-use cranelift::codegen::ir::types::*;
-use cranelift::codegen::ir::Function;
+use cranelift::codegen::ir::{types::*, UserExternalName, UserFuncName};
+use cranelift::codegen::ir::{Function, LibCall};
 use cranelift::codegen::Context;
 use cranelift::prelude::isa;
 use cranelift::prelude::*;
+use cranelift_arbitrary::CraneliftArbitrary;
 use cranelift_native::builder_with_options;
 use std::fmt;
 use target_lexicon::{Architecture, Triple};
 
 mod config;
+mod cranelift_arbitrary;
 mod function_generator;
 mod passes;
 
-pub type TestCaseInput = Vec<DataValue>;
-
-/// Simple wrapper to generate a single Cranelift `Function`.
-#[derive(Debug)]
-pub struct SingleFunction(pub Function);
+/// These libcalls need a interpreter implementation in `cranelift-fuzzgen.rs`
+const ALLOWED_LIBCALLS: &'static [LibCall] = &[
+    LibCall::CeilF32,
+    LibCall::CeilF64,
+    LibCall::FloorF32,
+    LibCall::FloorF64,
+    LibCall::TruncF32,
+    LibCall::TruncF64,
+];
 
-impl<'a> Arbitrary<'a> for SingleFunction {
-    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
-        FuzzGen::new(u)
-            .generate_func(Triple::host())
-            .map_err(|_| arbitrary::Error::IncorrectFormat)
-            .map(Self)
-    }
-}
+pub type TestCaseInput = Vec<DataValue>;
 
 /// Print only non default flags.
 fn write_non_default_flags(f: &mut fmt::Formatter<'_>, flags: &settings::Flags) -> fmt::Result {
@@ -176,27 +175,6 @@ where
         }
     }
 
-    fn generate_datavalue(&mut self, ty: Type) -> Result<DataValue> {
-        Ok(match ty {
-            ty if ty.is_int() => {
-                let imm = match ty {
-                    I8 => self.u.arbitrary::<i8>()? as i128,
-                    I16 => self.u.arbitrary::<i16>()? as i128,
-                    I32 => self.u.arbitrary::<i32>()? as i128,
-                    I64 => self.u.arbitrary::<i64>()? as i128,
-                    I128 => self.u.arbitrary::<i128>()?,
-                    _ => unreachable!(),
-                };
-                DataValue::from_integer(imm, ty)?
-            }
-            // f{32,64}::arbitrary does not generate a bunch of important values
-            // such as Signaling NaN's / NaN's with payload, so generate floats from integers.
-            F32 => DataValue::F32(Ieee32::with_bits(u32::arbitrary(self.u)?)),
-            F64 => DataValue::F64(Ieee64::with_bits(u64::arbitrary(self.u)?)),
-            _ => unimplemented!(),
-        })
-    }
-
     fn generate_test_inputs(mut self, signature: &Signature) -> Result<Vec<TestCaseInput>> {
         let mut inputs = Vec::new();
 
@@ -209,7 +187,7 @@ where
             let test_args = signature
                 .params
                 .iter()
-                .map(|p| self.generate_datavalue(p.value_type))
+                .map(|p| self.u.datavalue(p.value_type))
                 .collect::<Result<TestCaseInput>>()?;
 
             inputs.push(test_args);
@@ -276,7 +254,38 @@ where
     }
 
     fn generate_func(&mut self, target_triple: Triple) -> Result<Function> {
-        let func = FunctionGenerator::new(&mut self.u, &self.config, target_triple).generate()?;
+        let max_params = self.u.int_in_range(self.config.signature_params.clone())?;
+        let max_rets = self.u.int_in_range(self.config.signature_rets.clone())?;
+        let sig = self.u.signature(max_params, max_rets)?;
+
+        // Function name must be in a different namespace than TESTFILE_NAMESPACE (0)
+        let fname = UserFuncName::user(1, 0);
+
+        // Generate the external functions that we allow calling in this function.
+        let usercalls = (0..self.u.int_in_range(self.config.usercalls.clone())?)
+            .map(|i| {
+                let max_params = self.u.int_in_range(self.config.signature_params.clone())?;
+                let max_rets = self.u.int_in_range(self.config.signature_rets.clone())?;
+                let sig = self.u.signature(max_params, max_rets)?;
+                let name = UserExternalName {
+                    namespace: 2,
+                    index: i as u32,
+                };
+                Ok((name, sig))
+            })
+            .collect::<Result<Vec<(UserExternalName, Signature)>>>()?;
+
+        let func = FunctionGenerator::new(
+            &mut self.u,
+            &self.config,
+            target_triple,
+            fname,
+            sig,
+            usercalls,
+            ALLOWED_LIBCALLS.to_vec(),
+        )
+        .generate()?;
+
         self.run_func_passes(func)
     }
 
@@ -321,6 +330,7 @@ where
         let supports_inline_probestack = match target_arch {
             Architecture::X86_64 => true,
             Architecture::Aarch64(_) => true,
+            Architecture::Riscv64(_) => true,
             _ => false,
         };
 
@@ -363,8 +373,9 @@ where
     pub fn generate_host_test(mut self) -> Result<TestCase> {
         // If we're generating test inputs as well as a function, then we're planning to execute
         // this function. That means that any function references in it need to exist. We don't yet
-        // have infrastructure for generating multiple functions, so just don't generate funcrefs.
-        self.config.funcrefs_per_function = 0..=0;
+        // have infrastructure for generating multiple functions, so just don't generate user call
+        // function references.
+        self.config.usercalls = 0..=0;
 
         // TestCase is meant to be consumed by a runner, so we make the assumption here that we're
         // generating a TargetIsa for the host.
diff --git a/crates/component-macro/src/bindgen.rs b/crates/component-macro/src/bindgen.rs
index c96e55e54..bb82b1750 100644
--- a/crates/component-macro/src/bindgen.rs
+++ b/crates/component-macro/src/bindgen.rs
@@ -53,7 +53,6 @@ impl Parse for Config {
             let content;
             syn::braced!(content in input);
             let fields = Punctuated::<Opt, Token![,]>::parse_terminated(&content)?;
-            let mut world = None;
             for field in fields.into_pairs() {
                 match field.into_value() {
                     Opt::Path(s) => {
diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs
index 12126990e..73b752bce 100644
--- a/crates/environ/src/compilation.rs
+++ b/crates/environ/src/compilation.rs
@@ -179,9 +179,9 @@ pub trait Compiler: Send + Sync {
 
     /// Appends a list of compiled functions to an in-memory object.
     ///
-    /// This function will receive the same `Box<dyn Ayn>` produced as part of
+    /// This function will receive the same `Box<dyn Any>` produced as part of
     /// compilation from functions like `compile_function`,
-    /// compile_host_to_wasm_trampoline`, and other component-related shims.
+    /// `compile_host_to_wasm_trampoline`, and other component-related shims.
     /// Internally this will take all of these functions and add information to
     /// the object such as:
     ///
diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rs
new file mode 100644
index 000000000..94b3da496
--- /dev/null
+++ b/winch/codegen/src/codegen/context.rs
@@ -0,0 +1,204 @@
+use crate::{
+    frame::Frame,
+    masm::{MacroAssembler, OperandSize, RegImm},
+    reg::Reg,
+    regalloc::RegAlloc,
+    stack::{Stack, Val},
+};
+
+/// The code generation context.
+/// The code generation context is made up of three
+/// essential data structures:
+///
+/// * The register allocator, in charge of keeping the inventory of register
+///   availability.
+/// * The value stack, which keeps track of the state of the values
+///   after each operation.
+/// * The current function's frame.
+///
+/// These data structures normally require cooperating with each other
+/// to perform most of the operations needed during the code
+/// generation process. The code generation context should
+/// be generally used as the single entry point to access
+/// the compound functionality provided by its elements.
+pub(crate) struct CodeGenContext<'a> {
+    /// The register allocator.
+    pub regalloc: RegAlloc,
+    /// The value stack.
+    pub stack: Stack,
+    /// The current function's frame.
+    pub frame: &'a Frame,
+}
+
+impl<'a> CodeGenContext<'a> {
+    /// Create a new code generation context.
+    pub fn new(regalloc: RegAlloc, stack: Stack, frame: &'a Frame) -> Self {
+        Self {
+            regalloc,
+            stack,
+            frame,
+        }
+    }
+
+    /// Request a specific general purpose register to the register allocator,
+    /// spilling if not available.
+    pub fn gpr<M: MacroAssembler>(&mut self, named: Reg, masm: &mut M) -> Reg {
+        self.regalloc.gpr(named, &mut |regalloc| {
+            Self::spill(&mut self.stack, regalloc, &self.frame, masm)
+        })
+    }
+
+    /// Request the next avaiable general purpose register to the register allocator,
+    /// spilling if no registers are available.
+    pub fn any_gpr<M: MacroAssembler>(&mut self, masm: &mut M) -> Reg {
+        self.regalloc
+            .any_gpr(&mut |regalloc| Self::spill(&mut self.stack, regalloc, &self.frame, masm))
+    }
+
+    /// Free the given general purpose register.
+    pub fn free_gpr(&mut self, reg: Reg) {
+        self.regalloc.free_gpr(reg);
+    }
+
+    /// Loads the stack top value into a register, if it isn't already one;
+    /// spilling if there are no registers available.
+    pub fn pop_to_reg<M: MacroAssembler>(&mut self, masm: &mut M, size: OperandSize) -> Reg {
+        if let Some(reg) = self.stack.pop_reg() {
+            return reg;
+        }
+
+        let dst = self.any_gpr(masm);
+        let val = self.stack.pop().expect("a value at stack top");
+        Self::move_val_to_reg(val, dst, masm, self.frame, size);
+        dst
+    }
+
+    /// Checks if the stack top contains the given register. The register
+    /// gets allocated otherwise, potentially causing a spill.
+    /// Once the requested register is allocated, the value at the top of the stack
+    /// gets loaded into the register.
+    pub fn pop_to_named_reg<M: MacroAssembler>(
+        &mut self,
+        masm: &mut M,
+        named: Reg,
+        size: OperandSize,
+    ) -> Reg {
+        if let Some(reg) = self.stack.pop_named_reg(named) {
+            return reg;
+        }
+
+        let dst = self.gpr(named, masm);
+        let val = self.stack.pop().expect("a value at stack top");
+        Self::move_val_to_reg(val, dst, masm, self.frame, size);
+        dst
+    }
+
+    fn move_val_to_reg<M: MacroAssembler>(
+        src: Val,
+        dst: Reg,
+        masm: &mut M,
+        frame: &Frame,
+        size: OperandSize,
+    ) {
+        match src {
+            Val::Reg(src) => masm.mov(RegImm::reg(src), RegImm::reg(dst), size),
+            Val::I32(imm) => masm.mov(RegImm::imm(imm.into()), RegImm::reg(dst), size),
+            Val::I64(imm) => masm.mov(RegImm::imm(imm), RegImm::reg(dst), size),
+            Val::Local(index) => {
+                let slot = frame
+                    .get_local(index)
+                    .expect(&format!("valid locat at index = {}", index));
+                let addr = masm.local_address(&slot);
+                masm.load(addr, dst, slot.ty.into());
+            }
+            v => panic!("Unsupported value {:?}", v),
+        };
+    }
+
+    /// Prepares arguments for emitting an i32 binary operation.
+    pub fn i32_binop<F, M>(&mut self, masm: &mut M, emit: &mut F)
+    where
+        F: FnMut(&mut M, RegImm, RegImm, OperandSize),
+        M: MacroAssembler,
+    {
+        let top = self.stack.peek().expect("value at stack top");
+
+        if top.is_i32_const() {
+            let val = self
+                .stack
+                .pop_i32_const()
+                .expect("i32 const value at stack top");
+            let reg = self.pop_to_reg(masm, OperandSize::S32);
+            emit(
+                masm,
+                RegImm::reg(reg),
+                RegImm::imm(val as i64),
+                OperandSize::S32,
+            );
+            self.stack.push(Val::reg(reg));
+        } else {
+            let src = self.pop_to_reg(masm, OperandSize::S32);
+            let dst = self.pop_to_reg(masm, OperandSize::S32);
+            emit(masm, dst.into(), src.into(), OperandSize::S32);
+            self.regalloc.free_gpr(src);
+            self.stack.push(Val::reg(dst));
+        }
+    }
+
+    /// Prepares arguments for emitting an i64 binary operation.
+    pub fn i64_binop<F, M>(&mut self, masm: &mut M, emit: &mut F)
+    where
+        F: FnMut(&mut M, RegImm, RegImm, OperandSize),
+        M: MacroAssembler,
+    {
+        let top = self.stack.peek().expect("value at stack top");
+        if top.is_i64_const() {
+            let val = self
+                .stack
+                .pop_i64_const()
+                .expect("i64 const value at stack top");
+            let reg = self.pop_to_reg(masm, OperandSize::S64);
+            emit(masm, RegImm::reg(reg), RegImm::imm(val), OperandSize::S64);
+            self.stack.push(Val::reg(reg));
+        } else {
+            let src = self.pop_to_reg(masm, OperandSize::S64);
+            let dst = self.pop_to_reg(masm, OperandSize::S64);
+            emit(masm, dst.into(), src.into(), OperandSize::S64);
+            self.regalloc.free_gpr(src);
+            self.stack.push(Val::reg(dst));
+        }
+    }
+
+    /// Spill locals and registers to memory.
+    // TODO optimize the spill range;
+    //
+    // At any point in the program, the stack
+    // might already contain Memory entries;
+    // we could effectively ignore that range;
+    // only focusing on the range that contains
+    // spillable values.
+    fn spill<M: MacroAssembler>(
+        stack: &mut Stack,
+        regalloc: &mut RegAlloc,
+        frame: &Frame,
+        masm: &mut M,
+    ) {
+        stack.inner_mut().iter_mut().for_each(|v| match v {
+            Val::Reg(r) => {
+                let offset = masm.push(*r);
+                regalloc.free_gpr(*r);
+                *v = Val::Memory(offset);
+            }
+            Val::Local(index) => {
+                let slot = frame.get_local(*index).expect("valid local at slot");
+                let addr = masm.local_address(&slot);
+                masm.load(addr, regalloc.scratch, slot.ty.into());
+                let offset = masm.push(regalloc.scratch);
+                *v = Val::Memory(offset);
+            }
+            v => {
+                println!("trying to spill something unknown {:?}", v);
+            }
+        });
+    }
+}
diff --git a/winch/codegen/src/codegen.rs b/winch/codegen/src/codegen/mod.rs
similarity index 52%
rename from winch/codegen/src/codegen.rs
rename to winch/codegen/src/codegen/mod.rs
index 7114db770..aa4ab537f 100644
--- a/winch/codegen/src/codegen.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -1,89 +1,12 @@
 use crate::{
     abi::{ABISig, ABI},
-    frame::Frame,
-    masm::{MacroAssembler, OperandSize, RegImm},
-    regalloc::RegAlloc,
-    stack::{Stack, Val},
+    masm::{MacroAssembler, OperandSize},
 };
 use anyhow::Result;
 use wasmparser::{BinaryReader, FuncValidator, ValType, ValidatorResources, VisitOperator};
 
-/// The code generation context.
-pub(crate) struct CodeGenContext<'a, M>
-where
-    M: MacroAssembler,
-{
-    pub masm: &'a mut M,
-    pub stack: Stack,
-    pub frame: &'a Frame,
-}
-
-impl<'a, M> CodeGenContext<'a, M>
-where
-    M: MacroAssembler,
-{
-    /// Create a new code generation context.
-    pub fn new(masm: &'a mut M, stack: Stack, frame: &'a Frame) -> Self {
-        Self { masm, stack, frame }
-    }
-
-    /// Prepares arguments for emitting an i32 binary operation.
-    pub fn i32_binop<F>(&mut self, regalloc: &mut RegAlloc, emit: &mut F)
-    where
-        F: FnMut(&mut M, RegImm, RegImm, OperandSize),
-    {
-        let top = self.stack.peek().expect("value at stack top");
-
-        if top.is_i32_const() {
-            let val = self
-                .stack
-                .pop_i32_const()
-                .expect("i32 const value at stack top");
-            let reg = regalloc.pop_to_reg(self, OperandSize::S32);
-            emit(
-                &mut self.masm,
-                RegImm::reg(reg),
-                RegImm::imm(val as i64),
-                OperandSize::S32,
-            );
-            self.stack.push(Val::reg(reg));
-        } else {
-            let src = regalloc.pop_to_reg(self, OperandSize::S32);
-            let dst = regalloc.pop_to_reg(self, OperandSize::S32);
-            emit(&mut self.masm, dst.into(), src.into(), OperandSize::S32);
-            regalloc.free_gpr(src);
-            self.stack.push(Val::reg(dst));
-        }
-    }
-
-    /// Prepares arguments for emitting an i64 binary operation.
-    pub fn i64_binop<F>(&mut self, regalloc: &mut RegAlloc, emit: &mut F)
-    where
-        F: FnMut(&mut M, RegImm, RegImm, OperandSize),
-    {
-        let top = self.stack.peek().expect("value at stack top");
-        if top.is_i64_const() {
-            let val = self
-                .stack
-                .pop_i64_const()
-                .expect("i64 const value at stack top");
-            let reg = regalloc.pop_to_reg(self, OperandSize::S64);
-            emit(
-                &mut self.masm,
-                RegImm::reg(reg),
-                RegImm::imm(val),
-                OperandSize::S64,
-            );
-            self.stack.push(Val::reg(reg));
-        } else {
-            let src = regalloc.pop_to_reg(self, OperandSize::S64);
-            let dst = regalloc.pop_to_reg(self, OperandSize::S64);
-            emit(&mut self.masm, dst.into(), src.into(), OperandSize::S64);
-            regalloc.free_gpr(src);
-            self.stack.push(Val::reg(dst));
-        }
-    }
-}
+mod context;
+pub(crate) use context::*;
 
 /// The code generation abstraction.
 pub(crate) struct CodeGen<'a, M>
@@ -97,22 +20,22 @@ where
     sig: ABISig,
 
     /// The code generation context.
-    pub context: CodeGenContext<'a, M>,
+    pub context: CodeGenContext<'a>,
 
-    /// The register allocator.
-    pub regalloc: RegAlloc,
+    /// The MacroAssembler.
+    pub masm: &'a mut M,
 }
 
 impl<'a, M> CodeGen<'a, M>
 where
     M: MacroAssembler,
 {
-    pub fn new<A: ABI>(context: CodeGenContext<'a, M>, sig: ABISig, regalloc: RegAlloc) -> Self {
+    pub fn new<A: ABI>(masm: &'a mut M, context: CodeGenContext<'a>, sig: ABISig) -> Self {
         Self {
             word_size: <A as ABI>::word_bytes(),
             sig,
             context,
-            regalloc,
+            masm,
         }
     }
 
@@ -131,10 +54,8 @@ where
 
     // TODO stack checks
     fn emit_start(&mut self) -> Result<()> {
-        self.context.masm.prologue();
-        self.context
-            .masm
-            .reserve_stack(self.context.frame.locals_size);
+        self.masm.prologue();
+        self.masm.reserve_stack(self.context.frame.locals_size);
         Ok(())
     }
 
@@ -145,10 +66,10 @@ where
     ) -> Result<()> {
         self.spill_register_arguments();
         let defined_locals_range = &self.context.frame.defined_locals_range;
-        self.context.masm.zero_mem_range(
+        self.masm.zero_mem_range(
             defined_locals_range.as_range(),
             self.word_size,
-            &mut self.regalloc,
+            &mut self.context.regalloc,
         );
 
         while !body.eof() {
@@ -185,7 +106,7 @@ where
     // Emit the usual function end instruction sequence.
     pub fn emit_end(&mut self) -> Result<()> {
         self.handle_abi_result();
-        self.context.masm.epilogue(self.context.frame.locals_size);
+        self.masm.epilogue(self.context.frame.locals_size);
         Ok(())
     }
 
@@ -206,14 +127,14 @@ where
                     .frame
                     .get_local(index as u32)
                     .expect("valid local slot at location");
-                let addr = self.context.masm.local_address(local);
+                let addr = self.masm.local_address(local);
                 let src = arg
                     .get_reg()
                     .expect("arg should be associated to a register");
 
                 match &ty {
-                    ValType::I32 => self.context.masm.store(src.into(), addr, OperandSize::S32),
-                    ValType::I64 => self.context.masm.store(src.into(), addr, OperandSize::S64),
+                    ValType::I32 => self.masm.store(src.into(), addr, OperandSize::S32),
+                    ValType::I64 => self.masm.store(src.into(), addr, OperandSize::S64),
                     _ => panic!("Unsupported type {:?}", ty),
                 }
             });
@@ -225,8 +146,8 @@ where
         }
         let named_reg = self.sig.result.result_reg();
         let reg = self
-            .regalloc
-            .pop_to_named_reg(&mut self.context, named_reg, OperandSize::S64);
-        self.regalloc.free_gpr(reg);
+            .context
+            .pop_to_named_reg(self.masm, named_reg, OperandSize::S64);
+        self.context.regalloc.free_gpr(reg);
     }
 }
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 687bee93f..89efe72cc 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -5,8 +5,9 @@ use super::{
 };
 use crate::{
     abi::local::LocalSlot,
+    codegen::CodeGenContext,
     isa::reg::Reg,
-    masm::{MacroAssembler as Masm, OperandSize, RegImm},
+    masm::{DivKind, MacroAssembler as Masm, OperandSize, RegImm},
 };
 use cranelift_codegen::{settings, Final, MachBufferFinalized};
 
@@ -151,6 +152,10 @@ impl Masm for MacroAssembler {
         todo!()
     }
 
+    fn div(&mut self, _context: &mut CodeGenContext, _kind: DivKind, _size: OperandSize) {
+        todo!()
+    }
+
     fn zero(&mut self, reg: Reg) {
         self.asm.load_constant(0, reg);
     }
diff --git a/winch/codegen/src/isa/aarch64/mod.rs b/winch/codegen/src/isa/aarch64/mod.rs
index 7e62e1aee..4f8d3048c 100644
--- a/winch/codegen/src/isa/aarch64/mod.rs
+++ b/winch/codegen/src/isa/aarch64/mod.rs
@@ -82,8 +82,8 @@ impl TargetIsa for Aarch64 {
         let frame = Frame::new(&abi_sig, &mut body, &mut validator, &abi)?;
         // TODO: Add floating point bitmask
         let regalloc = RegAlloc::new(RegSet::new(ALL_GPR, 0), scratch());
-        let codegen_context = CodeGenContext::new(&mut masm, stack, &frame);
-        let mut codegen = CodeGen::new::<abi::Aarch64ABI>(codegen_context, abi_sig, regalloc);
+        let codegen_context = CodeGenContext::new(regalloc, stack, &frame);
+        let mut codegen = CodeGen::new::<abi::Aarch64ABI>(&mut masm, codegen_context, abi_sig);
 
         codegen.emit(&mut body, validator)?;
         Ok(masm.finalize())
diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs
index 0f4e0376c..5b2286404 100644
--- a/winch/codegen/src/isa/x64/asm.rs
+++ b/winch/codegen/src/isa/x64/asm.rs
@@ -1,11 +1,14 @@
 //! Assembler library implementation for x64.
 
-use crate::{isa::reg::Reg, masm::OperandSize};
+use crate::{
+    isa::reg::Reg,
+    masm::{DivKind, OperandSize},
+};
 use cranelift_codegen::{
     isa::x64::{
         args::{
-            self, AluRmiROpcode, Amode, ExtMode, FromWritableReg, Gpr, GprMem, GprMemImm, RegMem,
-            RegMemImm, SyntheticAmode, WritableGpr,
+            self, AluRmiROpcode, Amode, DivOrRemKind, ExtMode, FromWritableReg, Gpr, GprMem,
+            GprMemImm, RegMem, RegMemImm, SyntheticAmode, WritableGpr,
         },
         settings as x64_settings, EmitInfo, EmitState, Inst,
     },
@@ -61,6 +64,15 @@ impl From<OperandSize> for args::OperandSize {
     }
 }
 
+impl From<DivKind> for DivOrRemKind {
+    fn from(kind: DivKind) -> Self {
+        match kind {
+            DivKind::Signed => DivOrRemKind::SignedDiv,
+            DivKind::Unsigned => DivOrRemKind::UnsignedDiv,
+        }
+    }
+}
+
 /// Low level assembler implementation for x64.
 pub(crate) struct Assembler {
     /// The machine instruction buffer.
@@ -262,6 +274,31 @@ impl Assembler {
         }
     }
 
+    /// Signed/unsigned division.
+    ///
+    /// Emits a sequence of instructions to ensure the correctness of
+    /// the division invariants.  This function assumes that the
+    /// caller has correctly allocated the dividend as `(rdx:rax)` and
+    /// accounted for the quotient to be stored in `rax`.
+    pub fn div(&mut self, divisor: Reg, dst: (Reg, Reg), kind: DivKind, size: OperandSize) {
+        let tmp = if size == OperandSize::S64 && kind == DivKind::Signed {
+            Some(regs::scratch())
+        } else {
+            None
+        };
+
+        self.emit(Inst::CheckedDivOrRemSeq {
+            kind: kind.into(),
+            size: size.into(),
+            divisor: divisor.into(),
+            dividend_lo: dst.0.into(),
+            dividend_hi: dst.1.into(),
+            dst_quotient: dst.0.into(),
+            dst_remainder: dst.1.into(),
+            tmp: tmp.map(|reg| reg.into()),
+        });
+    }
+
     /// Multiply immediate and register.
     pub fn mul_ir(&mut self, imm: i32, dst: Reg, size: OperandSize) {
         let imm = RegMemImm::imm(imm as u32);
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index 6c32e7b96..758e72fa5 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -1,11 +1,11 @@
 use super::{
     address::Address,
     asm::{Assembler, Operand},
-    regs::{rbp, rsp},
+    regs::{self, rbp, rsp},
 };
-use crate::abi::LocalSlot;
 use crate::isa::reg::Reg;
-use crate::masm::{MacroAssembler as Masm, OperandSize, RegImm};
+use crate::masm::{DivKind, MacroAssembler as Masm, OperandSize, RegImm};
+use crate::{abi::LocalSlot, codegen::CodeGenContext, stack::Val};
 use cranelift_codegen::{isa::x64::settings as x64_settings, settings, Final, MachBufferFinalized};
 
 /// x64 MacroAssembler.
@@ -152,6 +152,28 @@ impl Masm for MacroAssembler {
         self.asm.mul(src, dst, size);
     }
 
+    fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize) {
+        // Allocate rdx:rax.
+        let rdx = context.gpr(regs::rdx(), self);
+        let rax = context.gpr(regs::rax(), self);
+
+        // Allocate the divisor, which can be any gpr.
+        let divisor = context.pop_to_reg(self, size);
+
+        // Mark rax as allocatable.
+        context.regalloc.free_gpr(rax);
+        // Move the top value to rax.
+        let rax = context.pop_to_named_reg(self, rax, size);
+        self.asm.div(divisor, (rax, rdx), kind, size);
+
+        // Free the divisor and rdx.
+        context.free_gpr(divisor);
+        context.free_gpr(rdx);
+
+        // Push the quotient.
+        context.stack.push(Val::reg(rax));
+    }
+
     fn epilogue(&mut self, locals_size: u32) {
         assert!(self.sp_offset == locals_size);
 
diff --git a/winch/codegen/src/isa/x64/mod.rs b/winch/codegen/src/isa/x64/mod.rs
index 3f72c2c5f..e44f3470f 100644
--- a/winch/codegen/src/isa/x64/mod.rs
+++ b/winch/codegen/src/isa/x64/mod.rs
@@ -87,8 +87,8 @@ impl TargetIsa for X64 {
         let frame = Frame::new(&abi_sig, &mut body, &mut validator, &abi)?;
         // TODO Add in floating point bitmask
         let regalloc = RegAlloc::new(RegSet::new(ALL_GPR, 0), regs::scratch());
-        let codegen_context = CodeGenContext::new(&mut masm, stack, &frame);
-        let mut codegen = CodeGen::new::<abi::X64ABI>(codegen_context, abi_sig, regalloc);
+        let codegen_context = CodeGenContext::new(regalloc, stack, &frame);
+        let mut codegen = CodeGen::new::<abi::X64ABI>(&mut masm, codegen_context, abi_sig);
 
         codegen.emit(&mut body, validator)?;
 
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 56bc27897..1f4507be5 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -1,9 +1,18 @@
 use crate::abi::{align_to, LocalSlot};
+use crate::codegen::CodeGenContext;
 use crate::isa::reg::Reg;
 use crate::regalloc::RegAlloc;
 use cranelift_codegen::{Final, MachBufferFinalized};
 use std::{fmt::Debug, ops::Range};
 
+#[derive(Eq, PartialEq)]
+pub(crate) enum DivKind {
+    /// Signed division.
+    Signed,
+    /// Unsigned division.
+    Unsigned,
+}
+
 /// Operand size, in bits.
 #[derive(Copy, Clone, Eq, PartialEq)]
 pub(crate) enum OperandSize {
@@ -94,6 +103,19 @@ pub(crate) trait MacroAssembler {
     /// Perform multiplication operation.
     fn mul(&mut self, dst: RegImm, lhs: RegImm, rhs: RegImm, size: OperandSize);
 
+    /// Perform division operation.
+    /// Division is special in that some architectures have specific
+    /// expectations regarding the location of the instruction
+    /// arguments and regarding the location of the quotient /
+    /// remainder. To free the caller from having to deal with the
+    /// architecure specific contraints we give this function access
+    /// to the code generation context, allowing each implementation
+    /// to decide the lowering path.  For cases in which division is a
+    /// unconstrained binary operation, the caller can decide to use
+    /// the `CodeGenContext::i32_binop` or `CodeGenContext::i64_binop`
+    /// functions.
+    fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize);
+
     /// Push the register to the stack, returning the offset.
     fn push(&mut self, src: Reg) -> u32;
 
diff --git a/winch/codegen/src/regalloc.rs b/winch/codegen/src/regalloc.rs
index bdfd38486..9cd95213a 100644
--- a/winch/codegen/src/regalloc.rs
+++ b/winch/codegen/src/regalloc.rs
@@ -1,11 +1,4 @@
-use crate::{
-    codegen::CodeGenContext,
-    frame::Frame,
-    isa::reg::Reg,
-    masm::{MacroAssembler, OperandSize, RegImm},
-    regset::RegSet,
-    stack::Val,
-};
+use crate::{isa::reg::Reg, regset::RegSet};
 
 /// The register allocator.
 ///
@@ -30,79 +23,26 @@ impl RegAlloc {
         Self { regset, scratch }
     }
 
-    /// Loads the stack top value into a register, if it isn't already one;
-    /// spilling if there are no registers available.
-    pub fn pop_to_reg<M: MacroAssembler>(
-        &mut self,
-        context: &mut CodeGenContext<M>,
-        size: OperandSize,
-    ) -> Reg {
-        if let Some(reg) = context.stack.pop_reg() {
-            return reg;
-        }
-
-        let dst = self.any_gpr(context);
-        let val = context.stack.pop().expect("a value at stack top");
-        Self::move_val_to_reg(val, dst, context.masm, context.frame, size);
-        dst
-    }
-
-    /// Checks if the stack top contains the given register. The register
-    /// gets allocated otherwise, potentially causing a spill.
-    /// Once the requested register is allocated, the value at the top of the stack
-    /// gets loaded into the register.
-    pub fn pop_to_named_reg<M: MacroAssembler>(
-        &mut self,
-        context: &mut CodeGenContext<M>,
-        named: Reg,
-        size: OperandSize,
-    ) -> Reg {
-        if let Some(reg) = context.stack.pop_named_reg(named) {
-            return reg;
-        }
-
-        let dst = self.gpr(context, named);
-        let val = context.stack.pop().expect("a value at stack top");
-        Self::move_val_to_reg(val, dst, context.masm, context.frame, size);
-        dst
-    }
-
-    fn move_val_to_reg<M: MacroAssembler>(
-        src: Val,
-        dst: Reg,
-        masm: &mut M,
-        frame: &Frame,
-        size: OperandSize,
-    ) {
-        match src {
-            Val::Reg(src) => masm.mov(RegImm::reg(src), RegImm::reg(dst), size),
-            Val::I32(imm) => masm.mov(RegImm::imm(imm.into()), RegImm::reg(dst), size),
-            Val::I64(imm) => masm.mov(RegImm::imm(imm), RegImm::reg(dst), size),
-            Val::Local(index) => {
-                let slot = frame
-                    .get_local(index)
-                    .expect(&format!("valid locat at index = {}", index));
-                let addr = masm.local_address(&slot);
-                masm.load(addr, dst, slot.ty.into());
-            }
-            v => panic!("Unsupported value {:?}", v),
-        };
-    }
-
     /// Allocate the next available general purpose register,
     /// spilling if none available.
-    pub fn any_gpr<M: MacroAssembler>(&mut self, context: &mut CodeGenContext<M>) -> Reg {
+    pub fn any_gpr<F>(&mut self, spill: &mut F) -> Reg
+    where
+        F: FnMut(&mut RegAlloc),
+    {
         self.regset.any_gpr().unwrap_or_else(|| {
-            self.spill(context);
+            spill(self);
             self.regset.any_gpr().expect("any gpr to be available")
         })
     }
 
     /// Request a specific general purpose register,
     /// spilling if not available.
-    pub fn gpr<M: MacroAssembler>(&mut self, context: &mut CodeGenContext<M>, named: Reg) -> Reg {
+    pub fn gpr<F>(&mut self, named: Reg, spill: &mut F) -> Reg
+    where
+        F: FnMut(&mut RegAlloc),
+    {
         self.regset.gpr(named).unwrap_or_else(|| {
-            self.spill(context);
+            spill(self);
             self.regset
                 .gpr(named)
                 .expect(&format!("gpr {:?} to be available", named))
@@ -113,33 +53,4 @@ impl RegAlloc {
     pub fn free_gpr(&mut self, reg: Reg) {
         self.regset.free_gpr(reg);
     }
-
-    /// Spill locals and registers to memory.
-    // TODO optimize the spill range;
-    //
-    // At any point in the program, the stack
-    // might already contain Memory entries;
-    // we could effectively ignore that range;
-    // only focusing on the range that contains
-    // spillable values.
-    fn spill<M: MacroAssembler>(&mut self, context: &mut CodeGenContext<M>) {
-        context.stack.inner_mut().iter_mut().for_each(|v| match v {
-            Val::Reg(r) => {
-                let offset = context.masm.push(*r);
-                self.free_gpr(*r);
-                *v = Val::Memory(offset);
-            }
-            Val::Local(index) => {
-                let slot = context
-                    .frame
-                    .get_local(*index)
-                    .expect("valid local at slot");
-                let addr = context.masm.local_address(&slot);
-                context.masm.load(addr, self.scratch, slot.ty.into());
-                let offset = context.masm.push(self.scratch);
-                *v = Val::Memory(offset);
-            }
-            _ => {}
-        });
-    }
 }
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index f948fb94e..072c70404 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -5,7 +5,7 @@
 //! machine code emitter.
 
 use crate::codegen::CodeGen;
-use crate::masm::{MacroAssembler, OperandSize, RegImm};
+use crate::masm::{DivKind, MacroAssembler, OperandSize, RegImm};
 use crate::stack::Val;
 use wasmparser::ValType;
 use wasmparser::VisitOperator;
@@ -37,6 +37,10 @@ macro_rules! def_unsupported {
     (emit I64Add $($rest:tt)*) => {};
     (emit I32Sub $($rest:tt)*) => {};
     (emit I32Mul $($rest:tt)*) => {};
+    (emit I32DivS $($rest:tt)*) => {};
+    (emit I32DivU $($rest:tt)*) => {};
+    (emit I64DivS $($rest:tt)*) => {};
+    (emit I64DivU $($rest:tt)*) => {};
     (emit I64Mul $($rest:tt)*) => {};
     (emit I64Sub $($rest:tt)*) => {};
     (emit LocalGet $($rest:tt)*) => {};
@@ -62,46 +66,74 @@ where
 
     fn visit_i32_add(&mut self) {
         self.context
-            .i32_binop(&mut self.regalloc, &mut |masm: &mut M, dst, src, size| {
+            .i32_binop(self.masm, &mut |masm, dst, src, size| {
                 masm.add(dst, dst, src, size);
             });
     }
 
     fn visit_i64_add(&mut self) {
         self.context
-            .i64_binop(&mut self.regalloc, &mut |masm: &mut M, dst, src, size| {
+            .i64_binop(self.masm, &mut |masm, dst, src, size| {
                 masm.add(dst, dst, src, size);
             });
     }
 
     fn visit_i32_sub(&mut self) {
         self.context
-            .i32_binop(&mut self.regalloc, &mut |masm: &mut M, dst, src, size| {
+            .i32_binop(self.masm, &mut |masm, dst, src, size| {
                 masm.sub(dst, dst, src, size);
             });
     }
 
     fn visit_i64_sub(&mut self) {
         self.context
-            .i64_binop(&mut self.regalloc, &mut |masm: &mut M, dst, src, size| {
+            .i64_binop(self.masm, &mut |masm, dst, src, size| {
                 masm.sub(dst, dst, src, size);
             });
     }
 
     fn visit_i32_mul(&mut self) {
         self.context
-            .i32_binop(&mut self.regalloc, &mut |masm: &mut M, dst, src, size| {
+            .i32_binop(self.masm, &mut |masm, dst, src, size| {
                 masm.mul(dst, dst, src, size);
             });
     }
 
     fn visit_i64_mul(&mut self) {
         self.context
-            .i64_binop(&mut self.regalloc, &mut |masm: &mut M, dst, src, size| {
+            .i64_binop(self.masm, &mut |masm, dst, src, size| {
                 masm.mul(dst, dst, src, size);
             });
     }
 
+    fn visit_i32_div_s(&mut self) {
+        use DivKind::*;
+        use OperandSize::*;
+
+        self.masm.div(&mut self.context, Signed, S32);
+    }
+
+    fn visit_i32_div_u(&mut self) {
+        use DivKind::*;
+        use OperandSize::*;
+
+        self.masm.div(&mut self.context, Unsigned, S32);
+    }
+
+    fn visit_i64_div_s(&mut self) {
+        use DivKind::*;
+        use OperandSize::*;
+
+        self.masm.div(&mut self.context, Signed, S64);
+    }
+
+    fn visit_i64_div_u(&mut self) {
+        use DivKind::*;
+        use OperandSize::*;
+
+        self.masm.div(&mut self.context, Unsigned, S64);
+    }
+
     fn visit_end(&mut self) {}
 
     fn visit_local_get(&mut self, index: u32) {
@@ -124,10 +156,10 @@ where
             .get_local(index)
             .expect(&format!("vald local at slot = {}", index));
         let size: OperandSize = slot.ty.into();
-        let src = self.regalloc.pop_to_reg(context, size);
-        let addr = context.masm.local_address(&slot);
-        context.masm.store(RegImm::reg(src), addr, size);
-        self.regalloc.free_gpr(src);
+        let src = self.context.pop_to_reg(self.masm, size);
+        let addr = self.masm.local_address(&slot);
+        self.masm.store(RegImm::reg(src), addr, size);
+        self.context.regalloc.free_gpr(src);
     }
 
     wasmparser::for_each_operator!(def_unsupported);


Test patch:
diff --git a/ci/build-test-matrix.js b/ci/build-test-matrix.js
index c02752c5d..d0dd51b93 100644
--- a/ci/build-test-matrix.js
+++ b/ci/build-test-matrix.js
@@ -36,7 +36,8 @@ const array = [
   {
     "os": "ubuntu-latest",
     "name": "Test Linux x86_64",
-    "filter": "linux-x64"
+    "filter": "linux-x64",
+    "isa": "x64"
   },
   {
     "os": "macos-latest",
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 3b9250329..f63eb3e8e 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -3964,25 +3964,25 @@ fn test_x64_emit() {
     // XMM FMA
 
     insns.push((
-        Inst::xmm_rm_r_vex(AvxOpcode::Vfmadd213ss, RegMem::reg(xmm2), xmm1, w_xmm0),
+        Inst::xmm_rmr_vex3(AvxOpcode::Vfmadd213ss, RegMem::reg(xmm2), xmm1, w_xmm0),
         "C4E271A9C2",
         "vfmadd213ss %xmm0, %xmm1, %xmm2, %xmm0",
     ));
 
     insns.push((
-        Inst::xmm_rm_r_vex(AvxOpcode::Vfmadd213sd, RegMem::reg(xmm5), xmm4, w_xmm3),
+        Inst::xmm_rmr_vex3(AvxOpcode::Vfmadd213sd, RegMem::reg(xmm5), xmm4, w_xmm3),
         "C4E2D9A9DD",
         "vfmadd213sd %xmm3, %xmm4, %xmm5, %xmm3",
     ));
 
     insns.push((
-        Inst::xmm_rm_r_vex(AvxOpcode::Vfmadd213ps, RegMem::reg(xmm2), xmm1, w_xmm0),
+        Inst::xmm_rmr_vex3(AvxOpcode::Vfmadd213ps, RegMem::reg(xmm2), xmm1, w_xmm0),
         "C4E271A8C2",
         "vfmadd213ps %xmm0, %xmm1, %xmm2, %xmm0",
     ));
 
     insns.push((
-        Inst::xmm_rm_r_vex(AvxOpcode::Vfmadd213pd, RegMem::reg(xmm5), xmm4, w_xmm3),
+        Inst::xmm_rmr_vex3(AvxOpcode::Vfmadd213pd, RegMem::reg(xmm5), xmm4, w_xmm3),
         "C4E2D9A8DD",
         "vfmadd213pd %xmm3, %xmm4, %xmm5, %xmm3",
     ));
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif b/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif
index ea1f1f913..ce4e8d0e8 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitselect.clif
@@ -1,6 +1,6 @@
 test compile precise-output
 set enable_simd
-target x86_64 skylake
+target x86_64
 
 function %mask_from_icmp(i8x16, i8x16) -> i8x16 {
 block0(v0: i8x16, v1: i8x16):
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
new file mode 100644
index 000000000..eeb096b3e
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-avx.clif
@@ -0,0 +1,228 @@
+test compile precise-output
+set enable_simd
+target x86_64 has_avx
+
+function %mask_from_icmp(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+    v2 = fmin v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vminps  %xmm0, %xmm1, %xmm3
+;   vminps  %xmm1, %xmm0, %xmm5
+;   vorps   %xmm3, %xmm5, %xmm7
+;   vcmpps  $3 %xmm7, %xmm5, %xmm9
+;   vorps   %xmm7, %xmm9, %xmm11
+;   vpsrld  %xmm9, $10, %xmm13
+;   vandnps %xmm13, %xmm11, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vminps %xmm1, %xmm0, %xmm3
+;   vminps %xmm0, %xmm1, %xmm5
+;   vorps %xmm5, %xmm3, %xmm7
+;   vcmpunordps %xmm5, %xmm7, %xmm9
+;   vorps %xmm9, %xmm7, %xmm11
+;   vpsrld $0xa, %xmm9, %xmm13
+;   vandnps %xmm11, %xmm13, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %or_from_memory(f32x4, i64) -> f32x4 {
+block0(v0: f32x4, v1: i64):
+    v2 = load.f32x4 notrap aligned v1
+    v3 = bor v0, v2
+    return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movups  0(%rdi), %xmm4
+;   vorps   %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movups (%rdi), %xmm4
+;   vorps %xmm4, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %copysign_from_memory(i64) -> f32 {
+block0(v0: i64):
+    v1 = f32const 0.0
+    v2 = load.f32 notrap aligned v0
+    v3 = fcopysign v1, v2
+    return v3
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm7
+;   movl    $-2147483648, %ecx
+;   movd    %ecx, %xmm8
+;   vandnps %xmm8, const(0), %xmm9
+;   andps   %xmm8, %xmm7, %xmm8
+;   vorps   %xmm9, %xmm8, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm7
+;   movl $0x80000000, %ecx
+;   movd %ecx, %xmm8
+;   vandnps 0x16(%rip), %xmm8, %xmm9
+;   andps %xmm7, %xmm8
+;   vorps %xmm8, %xmm9, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+
+function %bor_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+    v2 = bor v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vorps   %xmm0, %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vorps %xmm1, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %band_not_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+    v2 = band_not v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vandnps %xmm1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vandnps %xmm0, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %i32x4_shr(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+    v2 = ushr v0, v1
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movq    %rdi, %rcx
+;   andq    %rcx, $31, %rcx
+;   movd    %ecx, %xmm5
+;   vpsrld  %xmm0, %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movq %rdi, %rcx
+;   andq $0x1f, %rcx
+;   movd %ecx, %xmm5
+;   vpsrld %xmm5, %xmm0, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f32x4_abs(f32x4) -> f32x4 {
+block0(v0: f32x4):
+    v1 = fabs v0
+    return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pcmpeqd %xmm2, %xmm2, %xmm2
+;   vpsrld  %xmm2, $1, %xmm4
+;   andps   %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pcmpeqd %xmm2, %xmm2
+;   vpsrld $1, %xmm2, %xmm4
+;   andps %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
index 2c56dfd3c..24950d773 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif
@@ -1,6 +1,6 @@
 test compile precise-output
 set enable_simd
-target x86_64 skylake
+target x86_64
 
 function %band_f32x4(f32x4, f32x4) -> f32x4 {
 block0(v0: f32x4, v1: f32x4):
diff --git a/cranelift/filetests/filetests/runtests/simd-arithmetic.clif b/cranelift/filetests/filetests/runtests/simd-arithmetic.clif
index 4bc1ac828..ec1e4ad01 100644
--- a/cranelift/filetests/filetests/runtests/simd-arithmetic.clif
+++ b/cranelift/filetests/filetests/runtests/simd-arithmetic.clif
@@ -4,6 +4,7 @@ test run
 target aarch64
 target s390x
 set enable_simd
+target x86_64
 target x86_64 skylake
 
 function %iadd_i32x4(i32x4, i32x4) -> i32x4 {
diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
deleted file mode 100644
index b6d4e88dc..000000000
--- a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory-egraph.wat
+++ /dev/null
@@ -1,92 +0,0 @@
-;;! target = "x86_64"
-;;!
-;;! optimize = true
-;;!
-;;! settings = [
-;;!   "enable_heap_access_spectre_mitigation=true",
-;;!   "opt_level=speed_and_size",
-;;!   "use_egraphs=true"
-;;! ]
-;;!
-;;! [globals.vmctx]
-;;! type = "i64"
-;;! vmctx = true
-;;!
-;;! [globals.heap_base]
-;;! type = "i64"
-;;! load = { base = "vmctx", offset = 0 }
-;;!
-;;! [globals.heap_bound]
-;;! type = "i64"
-;;! load = { base = "vmctx", offset = 8 }
-;;!
-;;! [[heaps]]
-;;! base = "heap_base"
-;;! min_size = 0
-;;! offset_guard_size = 0xffffffff
-;;! index_type = "i32"
-;;! style = { kind = "dynamic", bound = "heap_bound" }
-
-(module
-  (memory (export "memory") 0)
-  (func (export "load-without-offset") (param i32) (result i32 i32)
-    local.get 0
-    i32.load
-    local.get 0
-    i32.load
-  )
-  (func (export "load-with-offset") (param i32) (result i32 i32)
-    local.get 0
-    i32.load offset=1234
-    local.get 0
-    i32.load offset=1234
-  )
-)
-
-;; function u0:0(i32, i64 vmctx) -> i32, i32 fast {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned gv0+8
-;;     gv2 = load.i64 notrap aligned gv0
-;;
-;;                                 block0(v0: i32, v1: i64):
-;; @0057                               v4 = uextend.i64 v0
-;; @0057                               v5 = iconst.i64 4
-;; @0057                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 4
-;; @0057                               v7 = load.i64 notrap aligned v1+8
-;; @0057                               v8 = load.i64 notrap aligned v1
-;; @0057                               v11 = icmp ugt v6, v7
-;; @0057                               v10 = iconst.i64 0
-;; @0057                               v9 = iadd v8, v4
-;; @0057                               v12 = select_spectre_guard v11, v10, v9  ; v10 = 0
-;; @0057                               v13 = load.i32 little heap v12
-;;                                     v2 -> v13
-;; @005f                               jump block1
-;;
-;;                                 block1:
-;; @005f                               return v13, v13
-;; }
-;;
-;; function u0:1(i32, i64 vmctx) -> i32, i32 fast {
-;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned gv0+8
-;;     gv2 = load.i64 notrap aligned gv0
-;;
-;;                                 block0(v0: i32, v1: i64):
-;; @0064                               v4 = uextend.i64 v0
-;; @0064                               v5 = iconst.i64 1238
-;; @0064                               v6 = uadd_overflow_trap v4, v5, heap_oob  ; v5 = 1238
-;; @0064                               v7 = load.i64 notrap aligned v1+8
-;; @0064                               v8 = load.i64 notrap aligned v1
-;; @0064                               v12 = icmp ugt v6, v7
-;; @0064                               v11 = iconst.i64 0
-;; @0064                               v9 = iadd v8, v4
-;;                                     v26 = iconst.i64 1234
-;; @0064                               v10 = iadd v9, v26  ; v26 = 1234
-;; @0064                               v13 = select_spectre_guard v12, v11, v10  ; v11 = 0
-;; @0064                               v14 = load.i32 little heap v13
-;;                                     v2 -> v14
-;; @006e                               jump block1
-;;
-;;                                 block1:
-;; @006e                               return v14, v14
-;; }
diff --git a/winch/filetests/filetests/x64/i32_divs/const.wat b/winch/filetests/filetests/x64/i32_divs/const.wat
new file mode 100644
index 000000000..c32078198
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divs/const.wat
@@ -0,0 +1,25 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const 20)
+	(i32.const 10)
+	(i32.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b90a000000           	mov	ecx, 0xa
+;;    9:	 b814000000           	mov	eax, 0x14
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 83f9ff               	cmp	ecx, -1
+;;   1c:	 0f850e000000         	jne	0x30
+;;   22:	 81f800000080         	cmp	eax, 0x80000000
+;;   28:	 0f8502000000         	jne	0x30
+;;   2e:	 0f0b                 	ud2	
+;;   30:	 99                   	cdq	
+;;   31:	 f7f9                 	idiv	ecx
+;;   33:	 5d                   	pop	rbp
+;;   34:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divs/one_zero.wat b/winch/filetests/filetests/x64/i32_divs/one_zero.wat
new file mode 100644
index 000000000..1ef0d7885
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divs/one_zero.wat
@@ -0,0 +1,25 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const 1)
+	(i32.const 0)
+	(i32.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b900000000           	mov	ecx, 0
+;;    9:	 b801000000           	mov	eax, 1
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 83f9ff               	cmp	ecx, -1
+;;   1c:	 0f850e000000         	jne	0x30
+;;   22:	 81f800000080         	cmp	eax, 0x80000000
+;;   28:	 0f8502000000         	jne	0x30
+;;   2e:	 0f0b                 	ud2	
+;;   30:	 99                   	cdq	
+;;   31:	 f7f9                 	idiv	ecx
+;;   33:	 5d                   	pop	rbp
+;;   34:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divs/overflow.wat b/winch/filetests/filetests/x64/i32_divs/overflow.wat
new file mode 100644
index 000000000..7f2499a1b
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divs/overflow.wat
@@ -0,0 +1,25 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const 0x80000000)
+	(i32.const -1)
+	(i32.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b9ffffffff           	mov	ecx, 0xffffffff
+;;    9:	 b800000080           	mov	eax, 0x80000000
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 83f9ff               	cmp	ecx, -1
+;;   1c:	 0f850e000000         	jne	0x30
+;;   22:	 81f800000080         	cmp	eax, 0x80000000
+;;   28:	 0f8502000000         	jne	0x30
+;;   2e:	 0f0b                 	ud2	
+;;   30:	 99                   	cdq	
+;;   31:	 f7f9                 	idiv	ecx
+;;   33:	 5d                   	pop	rbp
+;;   34:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divs/params.wat b/winch/filetests/filetests/x64/i32_divs/params.wat
new file mode 100644
index 000000000..ec999077b
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divs/params.wat
@@ -0,0 +1,29 @@
+;;! target = "x86_64"
+
+(module
+    (func (param i32) (param i32) (result i32)
+	(local.get 0)
+	(local.get 1)
+	(i32.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 4883ec08             	sub	rsp, 8
+;;    8:	 897c2404             	mov	dword ptr [rsp + 4], edi
+;;    c:	 893424               	mov	dword ptr [rsp], esi
+;;    f:	 8b0c24               	mov	ecx, dword ptr [rsp]
+;;   12:	 8b442404             	mov	eax, dword ptr [rsp + 4]
+;;   16:	 83f900               	cmp	ecx, 0
+;;   19:	 0f8502000000         	jne	0x21
+;;   1f:	 0f0b                 	ud2	
+;;   21:	 83f9ff               	cmp	ecx, -1
+;;   24:	 0f850e000000         	jne	0x38
+;;   2a:	 81f800000080         	cmp	eax, 0x80000000
+;;   30:	 0f8502000000         	jne	0x38
+;;   36:	 0f0b                 	ud2	
+;;   38:	 99                   	cdq	
+;;   39:	 f7f9                 	idiv	ecx
+;;   3b:	 4883c408             	add	rsp, 8
+;;   3f:	 5d                   	pop	rbp
+;;   40:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divs/zero_zero.wat b/winch/filetests/filetests/x64/i32_divs/zero_zero.wat
new file mode 100644
index 000000000..807b7f7eb
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divs/zero_zero.wat
@@ -0,0 +1,25 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const 0)
+	(i32.const 0)
+	(i32.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b900000000           	mov	ecx, 0
+;;    9:	 b800000000           	mov	eax, 0
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 83f9ff               	cmp	ecx, -1
+;;   1c:	 0f850e000000         	jne	0x30
+;;   22:	 81f800000080         	cmp	eax, 0x80000000
+;;   28:	 0f8502000000         	jne	0x30
+;;   2e:	 0f0b                 	ud2	
+;;   30:	 99                   	cdq	
+;;   31:	 f7f9                 	idiv	ecx
+;;   33:	 5d                   	pop	rbp
+;;   34:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divu/const.wat b/winch/filetests/filetests/x64/i32_divu/const.wat
new file mode 100644
index 000000000..d7be12fd1
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divu/const.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const 20)
+	(i32.const 10)
+	(i32.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b90a000000           	mov	ecx, 0xa
+;;    9:	 b814000000           	mov	eax, 0x14
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 ba00000000           	mov	edx, 0
+;;   1e:	 f7f1                 	div	ecx
+;;   20:	 5d                   	pop	rbp
+;;   21:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divu/one_zero.wat b/winch/filetests/filetests/x64/i32_divu/one_zero.wat
new file mode 100644
index 000000000..8eb16cfb2
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divu/one_zero.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const 1)
+	(i32.const 0)
+	(i32.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b900000000           	mov	ecx, 0
+;;    9:	 b801000000           	mov	eax, 1
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 ba00000000           	mov	edx, 0
+;;   1e:	 f7f1                 	div	ecx
+;;   20:	 5d                   	pop	rbp
+;;   21:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divu/params.wat b/winch/filetests/filetests/x64/i32_divu/params.wat
new file mode 100644
index 000000000..f3a57b864
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divu/params.wat
@@ -0,0 +1,24 @@
+;;! target = "x86_64"
+
+(module
+    (func (param i32) (param i32) (result i32)
+	(local.get 0)
+	(local.get 1)
+	(i32.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 4883ec08             	sub	rsp, 8
+;;    8:	 897c2404             	mov	dword ptr [rsp + 4], edi
+;;    c:	 893424               	mov	dword ptr [rsp], esi
+;;    f:	 8b0c24               	mov	ecx, dword ptr [rsp]
+;;   12:	 8b442404             	mov	eax, dword ptr [rsp + 4]
+;;   16:	 83f900               	cmp	ecx, 0
+;;   19:	 0f8502000000         	jne	0x21
+;;   1f:	 0f0b                 	ud2	
+;;   21:	 ba00000000           	mov	edx, 0
+;;   26:	 f7f1                 	div	ecx
+;;   28:	 4883c408             	add	rsp, 8
+;;   2c:	 5d                   	pop	rbp
+;;   2d:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divu/signed.wat b/winch/filetests/filetests/x64/i32_divu/signed.wat
new file mode 100644
index 000000000..925c2a038
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divu/signed.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const -1)
+	(i32.const -1)
+	(i32.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b9ffffffff           	mov	ecx, 0xffffffff
+;;    9:	 b8ffffffff           	mov	eax, 0xffffffff
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 ba00000000           	mov	edx, 0
+;;   1e:	 f7f1                 	div	ecx
+;;   20:	 5d                   	pop	rbp
+;;   21:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i32_divu/zero_zero.wat b/winch/filetests/filetests/x64/i32_divu/zero_zero.wat
new file mode 100644
index 000000000..c67d1a06c
--- /dev/null
+++ b/winch/filetests/filetests/x64/i32_divu/zero_zero.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i32)
+	(i32.const 0)
+	(i32.const 0)
+	(i32.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 b900000000           	mov	ecx, 0
+;;    9:	 b800000000           	mov	eax, 0
+;;    e:	 83f900               	cmp	ecx, 0
+;;   11:	 0f8502000000         	jne	0x19
+;;   17:	 0f0b                 	ud2	
+;;   19:	 ba00000000           	mov	edx, 0
+;;   1e:	 f7f1                 	div	ecx
+;;   20:	 5d                   	pop	rbp
+;;   21:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divs/const.wat b/winch/filetests/filetests/x64/i64_divs/const.wat
new file mode 100644
index 000000000..840f1ad5f
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divs/const.wat
@@ -0,0 +1,27 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const 20)
+	(i64.const 10)
+	(i64.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c10a000000       	mov	rcx, 0xa
+;;    b:	 48c7c014000000       	mov	rax, 0x14
+;;   12:	 4883f900             	cmp	rcx, 0
+;;   16:	 0f8502000000         	jne	0x1e
+;;   1c:	 0f0b                 	ud2	
+;;   1e:	 4883f9ff             	cmp	rcx, -1
+;;   22:	 0f8515000000         	jne	0x3d
+;;   28:	 49bb0000000000000080 	
+;; 				movabs	r11, 0x8000000000000000
+;;   32:	 4c39d8               	cmp	rax, r11
+;;   35:	 0f8502000000         	jne	0x3d
+;;   3b:	 0f0b                 	ud2	
+;;   3d:	 4899                 	cqo	
+;;   3f:	 48f7f9               	idiv	rcx
+;;   42:	 5d                   	pop	rbp
+;;   43:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divs/one_zero.wat b/winch/filetests/filetests/x64/i64_divs/one_zero.wat
new file mode 100644
index 000000000..c274f531e
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divs/one_zero.wat
@@ -0,0 +1,27 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const 1)
+	(i64.const 0)
+	(i64.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c100000000       	mov	rcx, 0
+;;    b:	 48c7c001000000       	mov	rax, 1
+;;   12:	 4883f900             	cmp	rcx, 0
+;;   16:	 0f8502000000         	jne	0x1e
+;;   1c:	 0f0b                 	ud2	
+;;   1e:	 4883f9ff             	cmp	rcx, -1
+;;   22:	 0f8515000000         	jne	0x3d
+;;   28:	 49bb0000000000000080 	
+;; 				movabs	r11, 0x8000000000000000
+;;   32:	 4c39d8               	cmp	rax, r11
+;;   35:	 0f8502000000         	jne	0x3d
+;;   3b:	 0f0b                 	ud2	
+;;   3d:	 4899                 	cqo	
+;;   3f:	 48f7f9               	idiv	rcx
+;;   42:	 5d                   	pop	rbp
+;;   43:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divs/overflow.wat b/winch/filetests/filetests/x64/i64_divs/overflow.wat
new file mode 100644
index 000000000..18630116b
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divs/overflow.wat
@@ -0,0 +1,28 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const 0x8000000000000000)
+	(i64.const -1)
+	(i64.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c1ffffffff       	mov	rcx, 0xffffffffffffffff
+;;    b:	 48b80000000000000080 	
+;; 				movabs	rax, 0x8000000000000000
+;;   15:	 4883f900             	cmp	rcx, 0
+;;   19:	 0f8502000000         	jne	0x21
+;;   1f:	 0f0b                 	ud2	
+;;   21:	 4883f9ff             	cmp	rcx, -1
+;;   25:	 0f8515000000         	jne	0x40
+;;   2b:	 49bb0000000000000080 	
+;; 				movabs	r11, 0x8000000000000000
+;;   35:	 4c39d8               	cmp	rax, r11
+;;   38:	 0f8502000000         	jne	0x40
+;;   3e:	 0f0b                 	ud2	
+;;   40:	 4899                 	cqo	
+;;   42:	 48f7f9               	idiv	rcx
+;;   45:	 5d                   	pop	rbp
+;;   46:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divs/params.wat b/winch/filetests/filetests/x64/i64_divs/params.wat
new file mode 100644
index 000000000..d52e28cff
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divs/params.wat
@@ -0,0 +1,31 @@
+;;! target = "x86_64"
+
+(module
+    (func (param i64) (param i64) (result i64)
+	(local.get 0)
+	(local.get 1)
+	(i64.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 4883ec10             	sub	rsp, 0x10
+;;    8:	 48897c2408           	mov	qword ptr [rsp + 8], rdi
+;;    d:	 48893424             	mov	qword ptr [rsp], rsi
+;;   11:	 488b0c24             	mov	rcx, qword ptr [rsp]
+;;   15:	 488b442408           	mov	rax, qword ptr [rsp + 8]
+;;   1a:	 4883f900             	cmp	rcx, 0
+;;   1e:	 0f8502000000         	jne	0x26
+;;   24:	 0f0b                 	ud2	
+;;   26:	 4883f9ff             	cmp	rcx, -1
+;;   2a:	 0f8515000000         	jne	0x45
+;;   30:	 49bb0000000000000080 	
+;; 				movabs	r11, 0x8000000000000000
+;;   3a:	 4c39d8               	cmp	rax, r11
+;;   3d:	 0f8502000000         	jne	0x45
+;;   43:	 0f0b                 	ud2	
+;;   45:	 4899                 	cqo	
+;;   47:	 48f7f9               	idiv	rcx
+;;   4a:	 4883c410             	add	rsp, 0x10
+;;   4e:	 5d                   	pop	rbp
+;;   4f:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divs/zero_zero.wat b/winch/filetests/filetests/x64/i64_divs/zero_zero.wat
new file mode 100644
index 000000000..4a08ab105
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divs/zero_zero.wat
@@ -0,0 +1,27 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const 0)
+	(i64.const 0)
+	(i64.div_s)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c100000000       	mov	rcx, 0
+;;    b:	 48c7c000000000       	mov	rax, 0
+;;   12:	 4883f900             	cmp	rcx, 0
+;;   16:	 0f8502000000         	jne	0x1e
+;;   1c:	 0f0b                 	ud2	
+;;   1e:	 4883f9ff             	cmp	rcx, -1
+;;   22:	 0f8515000000         	jne	0x3d
+;;   28:	 49bb0000000000000080 	
+;; 				movabs	r11, 0x8000000000000000
+;;   32:	 4c39d8               	cmp	rax, r11
+;;   35:	 0f8502000000         	jne	0x3d
+;;   3b:	 0f0b                 	ud2	
+;;   3d:	 4899                 	cqo	
+;;   3f:	 48f7f9               	idiv	rcx
+;;   42:	 5d                   	pop	rbp
+;;   43:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divu/const.wat b/winch/filetests/filetests/x64/i64_divu/const.wat
new file mode 100644
index 000000000..3e6b33ba9
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divu/const.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const 20)
+	(i64.const 10)
+	(i64.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c10a000000       	mov	rcx, 0xa
+;;    b:	 48c7c014000000       	mov	rax, 0x14
+;;   12:	 4883f900             	cmp	rcx, 0
+;;   16:	 0f8502000000         	jne	0x1e
+;;   1c:	 0f0b                 	ud2	
+;;   1e:	 ba00000000           	mov	edx, 0
+;;   23:	 48f7f1               	div	rcx
+;;   26:	 5d                   	pop	rbp
+;;   27:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divu/one_zero.wat b/winch/filetests/filetests/x64/i64_divu/one_zero.wat
new file mode 100644
index 000000000..53bddb442
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divu/one_zero.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const 1)
+	(i64.const 0)
+	(i64.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c100000000       	mov	rcx, 0
+;;    b:	 48c7c001000000       	mov	rax, 1
+;;   12:	 4883f900             	cmp	rcx, 0
+;;   16:	 0f8502000000         	jne	0x1e
+;;   1c:	 0f0b                 	ud2	
+;;   1e:	 ba00000000           	mov	edx, 0
+;;   23:	 48f7f1               	div	rcx
+;;   26:	 5d                   	pop	rbp
+;;   27:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divu/params.wat b/winch/filetests/filetests/x64/i64_divu/params.wat
new file mode 100644
index 000000000..75aeddef3
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divu/params.wat
@@ -0,0 +1,24 @@
+;;! target = "x86_64"
+
+(module
+    (func (param i64) (param i64) (result i64)
+	(local.get 0)
+	(local.get 1)
+	(i64.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 4883ec10             	sub	rsp, 0x10
+;;    8:	 48897c2408           	mov	qword ptr [rsp + 8], rdi
+;;    d:	 48893424             	mov	qword ptr [rsp], rsi
+;;   11:	 488b0c24             	mov	rcx, qword ptr [rsp]
+;;   15:	 488b442408           	mov	rax, qword ptr [rsp + 8]
+;;   1a:	 4883f900             	cmp	rcx, 0
+;;   1e:	 0f8502000000         	jne	0x26
+;;   24:	 0f0b                 	ud2	
+;;   26:	 ba00000000           	mov	edx, 0
+;;   2b:	 48f7f1               	div	rcx
+;;   2e:	 4883c410             	add	rsp, 0x10
+;;   32:	 5d                   	pop	rbp
+;;   33:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divu/signed.wat b/winch/filetests/filetests/x64/i64_divu/signed.wat
new file mode 100644
index 000000000..6fb06fa86
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divu/signed.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const -1)
+	(i64.const -1)
+	(i64.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c1ffffffff       	mov	rcx, 0xffffffffffffffff
+;;    b:	 48c7c0ffffffff       	mov	rax, 0xffffffffffffffff
+;;   12:	 4883f900             	cmp	rcx, 0
+;;   16:	 0f8502000000         	jne	0x1e
+;;   1c:	 0f0b                 	ud2	
+;;   1e:	 ba00000000           	mov	edx, 0
+;;   23:	 48f7f1               	div	rcx
+;;   26:	 5d                   	pop	rbp
+;;   27:	 c3                   	ret	
diff --git a/winch/filetests/filetests/x64/i64_divu/zero_zero.wat b/winch/filetests/filetests/x64/i64_divu/zero_zero.wat
new file mode 100644
index 000000000..d0c087eee
--- /dev/null
+++ b/winch/filetests/filetests/x64/i64_divu/zero_zero.wat
@@ -0,0 +1,20 @@
+;;! target = "x86_64"
+
+(module
+    (func (result i64)
+	(i64.const 0)
+	(i64.const 0)
+	(i64.div_u)
+    )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c100000000       	mov	rcx, 0
+;;    b:	 48c7c000000000       	mov	rax, 0
+;;   12:	 4883f900             	cmp	rcx, 0
+;;   16:	 0f8502000000         	jne	0x1e
+;;   1c:	 0f0b                 	ud2	
+;;   1e:	 ba00000000           	mov	edx, 0
+;;   23:	 48f7f1               	div	rcx
+;;   26:	 5d                   	pop	rbp
+;;   27:	 c3                   	ret