Repository: bytecodealliance/wasmtime
Pull request #8493 (2024-04-27 01:55:34)

cranelift: Move VReg aliases to VRegAllocator
Now that #8486 landed, allowing us to resolve aliases in machine instructions, we have ensured that all VReg aliases are resolved by the time we're done building the VCode. Therefore we only need to keep track of the aliases map before that.

The VReg allocator is also dropped when we finish building the VCode, and it makes sense to track aliases there. This lets us maintain an invariant, that PCC facts are only stored on VRegs which are not aliased, while only reasoning locally within VRegAllocator.

I've changed the trace-log output to print the VCode immediately before it's finalized, along with key details in the VRegAllocator. This allows seeing the instructions before aliases are rewritten, although they're in reverse order at that point. There's another trace-log message somewhere else which logs the finalized VCode, so you can see both.

Previously, the initial capacity of the vreg_aliases map was set to ten times the number of basic blocks in the function. However we can make a better estimate based on the number of SSA values in the function, and use that to preallocate storage for other things in VRegAllocator too.

Keeping the aliases outside the VCode fixes previous borrow-checker challenges, which is a nice bonus.


Patch:
diff --git a/Cargo.lock b/Cargo.lock
index da9be3bda..8274bbb8c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -610,6 +610,7 @@ dependencies = [
  "log",
  "postcard",
  "regalloc2",
+ "rustc-hash",
  "serde",
  "serde_derive",
  "sha2",
@@ -818,12 +819,12 @@ dependencies = [
  "cranelift-object",
  "cranelift-reader",
  "filecheck",
- "fxhash",
  "indicatif",
  "log",
  "pretty_env_logger",
  "rayon",
  "regalloc2",
+ "rustc-hash",
  "serde",
  "similar",
  "target-lexicon",
diff --git a/Cargo.toml b/Cargo.toml
index 118c26787..075f300ea 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -292,6 +292,7 @@ url = "2.3.1"
 humantime = "2.0.0"
 postcard = { version = "1.0.8", default-features = false, features = ['alloc'] }
 criterion = { version = "0.5.0", default-features = false, features = ["html_reports", "rayon"] }
+rustc-hash = "1.1.0"
 
 # =============================================================================
 #
diff --git a/cranelift/Cargo.toml b/cranelift/Cargo.toml
index 26cb31be1..0d2b8fefa 100644
--- a/cranelift/Cargo.toml
+++ b/cranelift/Cargo.toml
@@ -43,7 +43,7 @@ clap = { workspace = true, features = ['default'] }
 similar = { workspace = true }
 toml = { workspace = true }
 serde = { workspace = true }
-fxhash = "0.2.1"
+rustc-hash  = { workspace = true }
 # Note that this just enables `trace-log` for `clif-util` and doesn't turn it on
 # for all of Cranelift, which would be bad.
 regalloc2 = { workspace = true, features = ["trace-log"] }
diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml
index f6d67951d..3860dc911 100644
--- a/cranelift/codegen/Cargo.toml
+++ b/cranelift/codegen/Cargo.toml
@@ -34,6 +34,7 @@ smallvec = { workspace = true }
 regalloc2 = { workspace = true, features = ["checker", "trace-log"] }
 souper-ir = { version = "2.1.0", optional = true }
 sha2 = { version = "0.10.2", optional = true }
+rustc-hash  = { workspace = true }
 # It is a goal of the cranelift-codegen crate to have minimal external dependencies.
 # Please don't add any unless they are essential to the task of creating binary
 # machine code. Integration tests that need external dependencies can be
diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rs
index 5b59f87a5..f0d5ce648 100644
--- a/cranelift/codegen/meta/src/isa/x86.rs
+++ b/cranelift/codegen/meta/src/isa/x86.rs
@@ -373,11 +373,23 @@ pub(crate) fn define() -> TargetIsa {
         "Zen (second generation) microarchitecture.",
         preset!(znver1),
     );
-    settings.add_preset(
+    let znver3 = settings.add_preset(
         "znver3",
         "Zen (third generation) microarchitecture.",
         preset!(znver2),
     );
+    settings.add_preset(
+        "znver4",
+        "Zen (fourth generation) microarchitecture.",
+        preset!(
+            znver3
+                && has_avx512bitalg
+                && has_avx512dq
+                && has_avx512f
+                && has_avx512vbmi
+                && has_avx512vl
+        ),
+    );
 
     // Generic
 
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rs
index 0f9eaf002..636c0a8e9 100644
--- a/cranelift/codegen/src/alias_analysis.rs
+++ b/cranelift/codegen/src/alias_analysis.rs
@@ -64,7 +64,6 @@
 use crate::{
     cursor::{Cursor, FuncCursor},
     dominator_tree::DominatorTree,
-    fx::{FxHashMap, FxHashSet},
     inst_predicates::{
         has_memory_fence_semantics, inst_addr_offset_type, inst_store_data, visit_block_succs,
     },
@@ -72,6 +71,7 @@ use crate::{
     trace,
 };
 use cranelift_entity::{packed_option::PackedOption, EntityRef};
+use rustc_hash::{FxHashMap, FxHashSet};
 
 /// For a given program point, the vector of last-store instruction
 /// indices for each disjoint category of abstract state.
diff --git a/cranelift/codegen/src/ctxhash.rs b/cranelift/codegen/src/ctxhash.rs
index a249202aa..74290877b 100644
--- a/cranelift/codegen/src/ctxhash.rs
+++ b/cranelift/codegen/src/ctxhash.rs
@@ -76,7 +76,7 @@ fn compute_hash<Ctx, K>(ctx: &Ctx, k: &K) -> u32
 where
     Ctx: CtxHash<K>,
 {
-    let mut hasher = crate::fx::FxHasher::default();
+    let mut hasher = rustc_hash::FxHasher::default();
     ctx.ctx_hash(&mut hasher, k);
     hasher.finish() as u32
 }
diff --git a/cranelift/codegen/src/egraph.rs b/cranelift/codegen/src/egraph.rs
index 6b139b1ba..02e287706 100644
--- a/cranelift/codegen/src/egraph.rs
+++ b/cranelift/codegen/src/egraph.rs
@@ -5,7 +5,6 @@ use crate::ctxhash::{CtxEq, CtxHash, CtxHashMap};
 use crate::cursor::{Cursor, CursorPosition, FuncCursor};
 use crate::dominator_tree::{DominatorTree, DominatorTreePreorder};
 use crate::egraph::elaborate::Elaborator;
-use crate::fx::FxHashSet;
 use crate::inst_predicates::{is_mergeable_for_egraph, is_pure_for_egraph};
 use crate::ir::pcc::Fact;
 use crate::ir::{
@@ -22,6 +21,7 @@ use core::cmp::Ordering;
 use cranelift_control::ControlPlane;
 use cranelift_entity::packed_option::ReservedValue;
 use cranelift_entity::SecondaryMap;
+use rustc_hash::FxHashSet;
 use smallvec::SmallVec;
 use std::hash::Hasher;
 
diff --git a/cranelift/codegen/src/egraph/elaborate.rs b/cranelift/codegen/src/egraph/elaborate.rs
index 2aca8bf64..a35c2ac25 100644
--- a/cranelift/codegen/src/egraph/elaborate.rs
+++ b/cranelift/codegen/src/egraph/elaborate.rs
@@ -4,7 +4,6 @@
 use super::cost::Cost;
 use super::Stats;
 use crate::dominator_tree::DominatorTreePreorder;
-use crate::fx::{FxHashMap, FxHashSet};
 use crate::hash_map::Entry as HashEntry;
 use crate::inst_predicates::is_pure_for_egraph;
 use crate::ir::{Block, Function, Inst, Value, ValueDef};
@@ -14,6 +13,7 @@ use crate::trace;
 use alloc::vec::Vec;
 use cranelift_control::ControlPlane;
 use cranelift_entity::{packed_option::ReservedValue, SecondaryMap};
+use rustc_hash::{FxHashMap, FxHashSet};
 use smallvec::{smallvec, SmallVec};
 
 pub(crate) struct Elaborator<'a> {
diff --git a/cranelift/codegen/src/fx.rs b/cranelift/codegen/src/fx.rs
deleted file mode 100644
index 8e8b28fc9..000000000
--- a/cranelift/codegen/src/fx.rs
+++ /dev/null
@@ -1,105 +0,0 @@
-// This file is taken from the Rust compiler: src/librustc_data_structures/fx.rs
-
-// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-use super::{HashMap, HashSet};
-use core::hash::{BuildHasherDefault, Hash, Hasher};
-use core::ops::BitXor;
-
-pub type FxHashMap<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher>>;
-pub type FxHashSet<V> = HashSet<V, BuildHasherDefault<FxHasher>>;
-
-#[allow(non_snake_case)]
-pub fn FxHashMap<K: Hash + Eq, V>() -> FxHashMap<K, V> {
-    HashMap::default()
-}
-
-/// A speedy hash algorithm for use within rustc. The hashmap in liballoc
-/// by default uses SipHash which isn't quite as speedy as we want. In the
-/// compiler we're not really worried about DOS attempts, so we use a fast
-/// non-cryptographic hash.
-///
-/// This is the same as the algorithm used by Firefox -- which is a homespun
-/// one not based on any widely-known algorithm -- though modified to produce
-/// 64-bit hash values instead of 32-bit hash values. It consistently
-/// out-performs an FNV-based hash within rustc itself -- the collision rate is
-/// similar or slightly worse than FNV, but the speed of the hash function
-/// itself is much higher because it works on up to 8 bytes at a time.
-pub struct FxHasher {
-    hash: usize,
-}
-
-#[cfg(target_pointer_width = "32")]
-const K: usize = 0x9e3779b9;
-#[cfg(target_pointer_width = "64")]
-const K: usize = 0x517cc1b727220a95;
-
-impl Default for FxHasher {
-    #[inline]
-    fn default() -> Self {
-        Self { hash: 0 }
-    }
-}
-
-impl FxHasher {
-    #[inline]
-    fn add_to_hash(&mut self, i: usize) {
-        self.hash = self.hash.rotate_left(5).bitxor(i).wrapping_mul(K);
-    }
-}
-
-impl Hasher for FxHasher {
-    #[inline]
-    fn write(&mut self, bytes: &[u8]) {
-        for byte in bytes {
-            let i = *byte;
-            self.add_to_hash(i as usize);
-        }
-    }
-
-    #[inline]
-    fn write_u8(&mut self, i: u8) {
-        self.add_to_hash(i as usize);
-    }
-
-    #[inline]
-    fn write_u16(&mut self, i: u16) {
-        self.add_to_hash(i as usize);
-    }
-
-    #[inline]
-    fn write_u32(&mut self, i: u32) {
-        self.add_to_hash(i as usize);
-    }
-
-    #[cfg(target_pointer_width = "32")]
-    #[inline]
-    fn write_u64(&mut self, i: u64) {
-        self.add_to_hash(i as usize);
-        self.add_to_hash((i >> 32) as usize);
-    }
-
-    #[cfg(target_pointer_width = "64")]
-    #[inline]
-    fn write_u64(&mut self, i: u64) {
-        self.add_to_hash(i as usize);
-    }
-
-    #[inline]
-    fn write_usize(&mut self, i: usize) {
-        self.add_to_hash(i);
-    }
-
-    #[inline]
-    fn finish(&self) -> u64 {
-        self.hash as u64
-    }
-}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 98b7abdee..7cffbce30 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -10,7 +10,7 @@ use crate::{settings, CodegenError, CodegenResult};
 use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
 
 use alloc::vec::Vec;
-use regalloc2::{PRegSet, VReg};
+use regalloc2::PRegSet;
 use smallvec::{smallvec, SmallVec};
 use std::fmt::Write;
 use std::string::{String, ToString};
@@ -396,10 +396,7 @@ impl Inst {
 //=============================================================================
 // Instructions: get_regs
 
-fn memarg_operands<F: Fn(VReg) -> VReg>(
-    memarg: &mut AMode,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn memarg_operands(memarg: &mut AMode, collector: &mut impl OperandVisitor) {
     // This should match `AMode::with_allocs()`.
     match memarg {
         AMode::Unscaled { rn, .. } | AMode::UnsignedOffset { rn, .. } => {
@@ -423,10 +420,7 @@ fn memarg_operands<F: Fn(VReg) -> VReg>(
     }
 }
 
-fn pairmemarg_operands<F: Fn(VReg) -> VReg>(
-    pairmemarg: &mut PairAMode,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn pairmemarg_operands(pairmemarg: &mut PairAMode, collector: &mut impl OperandVisitor) {
     // This should match `PairAMode::with_allocs()`.
     match pairmemarg {
         PairAMode::SignedOffset { reg, .. } => {
@@ -436,10 +430,7 @@ fn pairmemarg_operands<F: Fn(VReg) -> VReg>(
     }
 }
 
-fn aarch64_get_operands<F: Fn(VReg) -> VReg>(
-    inst: &mut Inst,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
     match inst {
         Inst::AluRRR { rd, rn, rm, .. } => {
             collector.reg_def(rd);
@@ -997,7 +988,7 @@ impl MachInst for Inst {
     // debugging.
     const TRAP_OPCODE: &'static [u8] = &0xc11f_u32.to_le_bytes();
 
-    fn get_operands<F: Fn(VReg) -> VReg>(&mut self, collector: &mut OperandCollector<'_, F>) {
+    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         aarch64_get_operands(self, collector);
     }
 
diff --git a/cranelift/codegen/src/isa/riscv64/inst/args.rs b/cranelift/codegen/src/isa/riscv64/inst/args.rs
index 42326222b..4a0487889 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/args.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/args.rs
@@ -127,10 +127,7 @@ impl AMode {
 
     /// Add the registers referenced by this AMode to `collector`.
     /// Keep this in sync with `with_allocs`.
-    pub(crate) fn get_operands<F: Fn(regalloc2::VReg) -> regalloc2::VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             AMode::RegOffset(reg, ..) => collector.reg_use(reg),
             // Registers used in these modes aren't allocatable.
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
index fe148b43c..5172b61c7 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -13,7 +13,7 @@ use crate::{settings, CodegenError, CodegenResult};
 pub use crate::ir::condcodes::FloatCC;
 
 use alloc::vec::Vec;
-use regalloc2::{PRegSet, RegClass, VReg};
+use regalloc2::{PRegSet, RegClass};
 use smallvec::{smallvec, SmallVec};
 use std::boxed::Box;
 use std::fmt::Write;
@@ -317,10 +317,7 @@ impl Inst {
 
 //=============================================================================
 
-fn vec_mask_operands<F: Fn(VReg) -> VReg>(
-    mask: &mut VecOpMasking,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn vec_mask_operands(mask: &mut VecOpMasking, collector: &mut impl OperandVisitor) {
     match mask {
         VecOpMasking::Enabled { reg } => {
             collector.reg_fixed_use(reg, pv_reg(0).into());
@@ -328,10 +325,7 @@ fn vec_mask_operands<F: Fn(VReg) -> VReg>(
         VecOpMasking::Disabled => {}
     }
 }
-fn vec_mask_late_operands<F: Fn(VReg) -> VReg>(
-    mask: &mut VecOpMasking,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn vec_mask_late_operands(mask: &mut VecOpMasking, collector: &mut impl OperandVisitor) {
     match mask {
         VecOpMasking::Enabled { reg } => {
             collector.reg_fixed_late_use(reg, pv_reg(0).into());
@@ -340,10 +334,7 @@ fn vec_mask_late_operands<F: Fn(VReg) -> VReg>(
     }
 }
 
-fn riscv64_get_operands<F: Fn(VReg) -> VReg>(
-    inst: &mut Inst,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
     match inst {
         Inst::Nop0 | Inst::Nop4 => {}
         Inst::BrTable {
@@ -789,7 +780,7 @@ impl MachInst for Inst {
         }
     }
 
-    fn get_operands<F: Fn(VReg) -> VReg>(&mut self, collector: &mut OperandCollector<'_, F>) {
+    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         riscv64_get_operands(self, collector);
     }
 
@@ -1552,8 +1543,8 @@ impl Inst {
                 format!("virtual_sp_offset_adj {:+}", amount)
             }
             &MInst::Mov { rd, rm, ty } => {
-                let rd = format_reg(rd.to_reg(), allocs);
                 let rm = format_reg(rm, allocs);
+                let rd = format_reg(rd.to_reg(), allocs);
 
                 let op = match ty {
                     F32 => "fmv.s",
diff --git a/cranelift/codegen/src/isa/riscv64/inst/vector.rs b/cranelift/codegen/src/isa/riscv64/inst/vector.rs
index dc04ba9ca..496896fb6 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/vector.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/vector.rs
@@ -4,7 +4,7 @@ use crate::isa::riscv64::lower::isle::generated_code::{
     VecAMode, VecAluOpRImm5, VecAluOpRR, VecAluOpRRImm5, VecAluOpRRR, VecAluOpRRRImm5, VecAvl,
     VecElementWidth, VecLmul, VecMaskMode, VecOpCategory, VecOpMasking, VecTailMode,
 };
-use crate::machinst::{OperandCollector, RegClass};
+use crate::machinst::{OperandVisitor, RegClass};
 use crate::Reg;
 use core::fmt;
 
@@ -1070,10 +1070,7 @@ impl VecAMode {
         }
     }
 
-    pub fn get_operands<F: Fn(regalloc2::VReg) -> regalloc2::VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             VecAMode::UnitStride { base, .. } => base.get_operands(collector),
         }
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index a59c06a93..1bb6b5fa0 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -8,7 +8,7 @@ use crate::machinst::*;
 use crate::{settings, CodegenError, CodegenResult};
 use alloc::boxed::Box;
 use alloc::vec::Vec;
-use regalloc2::{PRegSet, VReg};
+use regalloc2::PRegSet;
 use smallvec::SmallVec;
 use std::fmt::Write;
 use std::string::{String, ToString};
@@ -389,10 +389,7 @@ impl Inst {
 //=============================================================================
 // Instructions: get_regs
 
-fn memarg_operands<F: Fn(VReg) -> VReg>(
-    memarg: &mut MemArg,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn memarg_operands(memarg: &mut MemArg, collector: &mut impl OperandVisitor) {
     match memarg {
         MemArg::BXD12 { base, index, .. } | MemArg::BXD20 { base, index, .. } => {
             collector.reg_use(base);
@@ -409,10 +406,7 @@ fn memarg_operands<F: Fn(VReg) -> VReg>(
     collector.reg_fixed_nonallocatable(gpr_preg(1));
 }
 
-fn s390x_get_operands<F: Fn(VReg) -> VReg>(
-    inst: &mut Inst,
-    collector: &mut OperandCollector<'_, F>,
-) {
+fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl OperandVisitor>) {
     match inst {
         Inst::AluRRR { rd, rn, rm, .. } => {
             collector.reg_def(rd);
@@ -952,15 +946,17 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(
             memarg_operands(mem, collector);
         }
         Inst::Loop { body, .. } => {
-            for inst in body {
-                s390x_get_operands(inst, collector);
-            }
-
             // `reuse_def` constraints can't be permitted in a Loop instruction because the operand
             // index will always be relative to the Loop instruction, not the individual
             // instruction in the loop body. However, fixed-nonallocatable registers used with
             // instructions that would have emitted `reuse_def` constraints are fine.
-            debug_assert!(collector.no_reuse_def());
+            let mut collector = DenyReuseVisitor {
+                inner: collector.inner,
+                deny_reuse: true,
+            };
+            for inst in body {
+                s390x_get_operands(inst, &mut collector);
+            }
         }
         Inst::CondBreak { .. } => {}
         Inst::VirtualSPOffsetAdj { .. } => {}
@@ -971,6 +967,34 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(
     }
 }
 
+struct DenyReuseVisitor<'a, T> {
+    inner: &'a mut T,
+    deny_reuse: bool,
+}
+
+impl<T: OperandVisitor> OperandVisitor for DenyReuseVisitor<'_, T> {
+    fn add_operand(
+        &mut self,
+        reg: &mut Reg,
+        constraint: regalloc2::OperandConstraint,
+        kind: regalloc2::OperandKind,
+        pos: regalloc2::OperandPos,
+    ) {
+        debug_assert!(
+            !self.deny_reuse || !matches!(constraint, regalloc2::OperandConstraint::Reuse(_))
+        );
+        self.inner.add_operand(reg, constraint, kind, pos);
+    }
+
+    fn debug_assert_is_allocatable_preg(&self, reg: regalloc2::PReg, expected: bool) {
+        self.inner.debug_assert_is_allocatable_preg(reg, expected);
+    }
+
+    fn reg_clobbers(&mut self, regs: PRegSet) {
+        self.inner.reg_clobbers(regs);
+    }
+}
+
 //=============================================================================
 // Instructions: misc functions and external interface
 
@@ -979,8 +1003,14 @@ impl MachInst for Inst {
     type LabelUse = LabelUse;
     const TRAP_OPCODE: &'static [u8] = &[0, 0];
 
-    fn get_operands<F: Fn(VReg) -> VReg>(&mut self, collector: &mut OperandCollector<'_, F>) {
-        s390x_get_operands(self, collector);
+    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
+        s390x_get_operands(
+            self,
+            &mut DenyReuseVisitor {
+                inner: collector,
+                deny_reuse: false,
+            },
+        );
     }
 
     fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 82206958f..294387945 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -8,7 +8,6 @@ use crate::ir::MemFlags;
 use crate::isa::x64::inst::regs::pretty_print_reg;
 use crate::isa::x64::inst::Inst;
 use crate::machinst::*;
-use regalloc2::VReg;
 use smallvec::{smallvec, SmallVec};
 use std::fmt;
 use std::string::String;
@@ -166,10 +165,7 @@ macro_rules! newtype_of_reg {
                 }
 
                 #[allow(dead_code)] // Used by some newtypes and not others.
-                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-                    &mut self,
-                    collector: &mut OperandCollector<'_, F>,
-                ) {
+                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
                     self.0.get_operands(collector);
                 }
             }
@@ -234,10 +230,7 @@ macro_rules! newtype_of_reg {
                 }
 
                 #[allow(dead_code)] // Used by some newtypes and not others.
-                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-                    &mut self,
-                    collector: &mut OperandCollector<'_, F>,
-                ) {
+                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
                     self.0.get_operands(collector);
                 }
             }
@@ -370,10 +363,7 @@ impl Amode {
     }
 
     /// Add the registers mentioned by `self` to `collector`.
-    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             Amode::ImmReg { base, .. } => {
                 if *base != regs::rbp() && *base != regs::rsp() {
@@ -395,10 +385,7 @@ impl Amode {
     }
 
     /// Same as `get_operands`, but add the registers in the "late" phase.
-    pub(crate) fn get_operands_late<F: Fn(VReg) -> VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             Amode::ImmReg { base, .. } => {
                 collector.reg_late_use(base);
@@ -535,10 +522,7 @@ impl SyntheticAmode {
     }
 
     /// Add the registers mentioned by `self` to `collector`.
-    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             SyntheticAmode::Real(addr) => addr.get_operands(collector),
             SyntheticAmode::IncomingArg { .. } => {
@@ -552,10 +536,7 @@ impl SyntheticAmode {
     }
 
     /// Same as `get_operands`, but add the register in the "late" phase.
-    pub(crate) fn get_operands_late<F: Fn(VReg) -> VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             SyntheticAmode::Real(addr) => addr.get_operands_late(collector),
             SyntheticAmode::IncomingArg { .. } => {
@@ -686,10 +667,7 @@ impl RegMemImm {
     }
 
     /// Add the regs mentioned by `self` to `collector`.
-    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             Self::Reg { reg } => collector.reg_use(reg),
             Self::Mem { addr } => addr.get_operands(collector),
@@ -796,10 +774,7 @@ impl RegMem {
         }
     }
     /// Add the regs mentioned by `self` to `collector`.
-    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
-        &mut self,
-        collector: &mut OperandCollector<'_, F>,
-    ) {
+    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         match self {
             RegMem::Reg { reg } => collector.reg_use(reg),
             RegMem::Mem { addr, .. } => addr.get_operands(collector),
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 06793ae0b..ceb0d2962 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -11,7 +11,7 @@ use crate::isa::{CallConv, FunctionAlignment};
 use crate::{machinst::*, trace};
 use crate::{settings, CodegenError, CodegenResult};
 use alloc::boxed::Box;
-use regalloc2::{Allocation, PRegSet, VReg};
+use regalloc2::{Allocation, PRegSet};
 use smallvec::{smallvec, SmallVec};
 use std::fmt::{self, Write};
 use std::string::{String, ToString};
@@ -1895,7 +1895,7 @@ impl fmt::Debug for Inst {
     }
 }
 
-fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &mut Inst, collector: &mut OperandCollector<'_, F>) {
+fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
     // Note: because we need to statically know the indices of each
     // reg in the operands list in order to fetch its allocation
     // later, we put the variable-operand-count bits (the RegMem,
@@ -2523,7 +2523,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &mut Inst, collector: &mut Operan
 impl MachInst for Inst {
     type ABIMachineSpec = X64ABIMachineSpec;
 
-    fn get_operands<F: Fn(VReg) -> VReg>(&mut self, collector: &mut OperandCollector<'_, F>) {
+    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
         x64_get_operands(self, collector)
     }
 
diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs
index 7b8ea499b..6e66d31d5 100644
--- a/cranelift/codegen/src/lib.rs
+++ b/cranelift/codegen/src/lib.rs
@@ -18,9 +18,9 @@ extern crate alloc;
 extern crate std;
 
 #[cfg(not(feature = "std"))]
-use hashbrown::{hash_map, HashMap, HashSet};
+use hashbrown::{hash_map, HashMap};
 #[cfg(feature = "std")]
-use std::collections::{hash_map, HashMap, HashSet};
+use std::collections::{hash_map, HashMap};
 
 pub use crate::context::Context;
 pub use crate::value_label::{LabelValueLoc, ValueLabelsRanges, ValueLocRange};
@@ -70,7 +70,6 @@ mod context;
 mod ctxhash;
 mod dce;
 mod egraph;
-mod fx;
 mod inst_predicates;
 mod isle_prelude;
 mod iterators;
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index a5de3824d..1edec7269 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -105,7 +105,6 @@
 //! ABI. See each platform's `abi.rs` implementation for details.
 
 use crate::entity::SecondaryMap;
-use crate::fx::FxHashMap;
 use crate::ir::types::*;
 use crate::ir::{ArgumentExtension, ArgumentPurpose, Signature};
 use crate::isa::TargetIsa;
@@ -114,6 +113,7 @@ use crate::CodegenError;
 use crate::{ir, isa};
 use crate::{machinst::*, trace};
 use regalloc2::{MachineEnv, PReg, PRegSet};
+use rustc_hash::FxHashMap;
 use smallvec::smallvec;
 use std::collections::HashMap;
 use std::marker::PhantomData;
diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs
index 31f7eef2d..f385f5c26 100644
--- a/cranelift/codegen/src/machinst/blockorder.rs
+++ b/cranelift/codegen/src/machinst/blockorder.rs
@@ -62,10 +62,10 @@
 
 use crate::dominator_tree::DominatorTree;
 use crate::entity::SecondaryMap;
-use crate::fx::{FxHashMap, FxHashSet};
 use crate::inst_predicates::visit_block_succs;
 use crate::ir::{Block, Function, Inst, Opcode};
 use crate::{machinst::*, trace};
+use rustc_hash::{FxHashMap, FxHashSet};
 
 /// Mapping from CLIF BBs to VCode BBs.
 #[derive(Debug)]
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index bd965d17d..8a99cd83f 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -6,7 +6,6 @@
 // top of it, e.g. the side-effect/coloring analysis and the scan support.
 
 use crate::entity::SecondaryMap;
-use crate::fx::{FxHashMap, FxHashSet};
 use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
 use crate::ir::pcc::{Fact, FactContext, PccError, PccResult};
 use crate::ir::{
@@ -23,6 +22,7 @@ use crate::settings::Flags;
 use crate::{trace, CodegenResult};
 use alloc::vec::Vec;
 use cranelift_control::ControlPlane;
+use rustc_hash::{FxHashMap, FxHashSet};
 use smallvec::{smallvec, SmallVec};
 use std::fmt::Debug;
 
@@ -362,7 +362,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             VCodeBuildDirection::Backward,
         );
 
-        let mut vregs = VRegAllocator::new();
+        // We usually need two VRegs per instruction result, plus extras for
+        // various temporaries, but two per Value is a good starting point.
+        let mut vregs = VRegAllocator::with_capacity(f.dfg.num_values() * 2);
 
         let mut value_regs = SecondaryMap::with_default(ValueRegs::invalid());
 
@@ -784,20 +786,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
                     debug_assert_eq!(regs.len(), dsts.len());
                     for (&dst, &temp) in dsts.regs().iter().zip(regs.regs()) {
                         trace!("set vreg alias: {result:?} = {dst:?}, lowering = {temp:?}");
-                        self.vcode.set_vreg_alias(dst, temp);
-
-                        // If there was any PCC fact about the
-                        // original VReg, move it to the aliased reg
-                        // instead. Lookup goes through the alias, but
-                        // we want to preserve whatever was stated
-                        // about the vreg before its producer was
-                        // lowered.
-                        if let Some(fact) =
-                            self.vregs.take_fact(dst.to_virtual_reg().unwrap().into())
-                        {
-                            self.vregs
-                                .set_fact(temp.to_virtual_reg().unwrap().into(), fact);
-                        }
+                        self.vregs.set_vreg_alias(dst, temp);
                     }
                 }
             }
@@ -1096,8 +1085,12 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
         // Now that we've emitted all instructions into the
         // VCodeBuilder, let's build the VCode.
+        trace!(
+            "built vcode:\n{:?}Backwards {:?}",
+            &self.vregs,
+            &self.vcode.vcode
+        );
         let vcode = self.vcode.build(self.vregs);
-        trace!("built vcode: {:?}", vcode);
 
         Ok(vcode)
     }
diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs
index 1055203c3..435d7825d 100644
--- a/cranelift/codegen/src/machinst/mod.rs
+++ b/cranelift/codegen/src/machinst/mod.rs
@@ -96,7 +96,7 @@ pub trait MachInst: Clone + Debug {
 
     /// Return the registers referenced by this machine instruction along with
     /// the modes of reference (use, def, modify).
-    fn get_operands<F: Fn(VReg) -> VReg>(&mut self, collector: &mut OperandCollector<'_, F>);
+    fn get_operands(&mut self, collector: &mut impl OperandVisitor);
 
     /// If this is a simple move, return the (source, destination) tuple of registers.
     fn is_move(&self) -> Option<(Writable<Reg>, Reg)>;
diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rs
index 40091fede..5ea6efc37 100644
--- a/cranelift/codegen/src/machinst/reg.rs
+++ b/cranelift/codegen/src/machinst/reg.rs
@@ -310,45 +310,37 @@ impl<'a, F: Fn(VReg) -> VReg> OperandCollector<'a, F> {
         }
     }
 
-    /// Returns true if no reuse_def constraints have been added.
-    pub fn no_reuse_def(&self) -> bool {
-        !self.operands[self.operands_start..]
-            .iter()
-            .any(|operand| match operand.constraint() {
-                OperandConstraint::Reuse(_) => true,
-                _ => false,
-            })
-    }
-
-    fn is_allocatable_preg(&self, reg: PReg) -> bool {
-        self.allocatable.contains(reg)
+    /// Finish the operand collection and return the tuple giving the
+    /// range of indices in the flattened operand array, and the
+    /// clobber set.
+    pub fn finish(self) -> ((u32, u32), PRegSet) {
+        let start = self.operands_start as u32;
+        let end = self.operands.len() as u32;
+        ((start, end), self.clobbers)
     }
+}
 
-    /// Add an operand.
+pub trait OperandVisitor {
     fn add_operand(
         &mut self,
         reg: &mut Reg,
         constraint: OperandConstraint,
         kind: OperandKind,
         pos: OperandPos,
-    ) {
-        reg.0 = (self.renamer)(reg.0);
-        self.operands
-            .push(Operand::new(reg.0, constraint, kind, pos));
-    }
+    );
 
-    /// Finish the operand collection and return the tuple giving the
-    /// range of indices in the flattened operand array, and the
-    /// clobber set.
-    pub fn finish(self) -> ((u32, u32), PRegSet) {
-        let start = self.operands_start as u32;
-        let end = self.operands.len() as u32;
-        ((start, end), self.clobbers)
-    }
+    fn debug_assert_is_allocatable_preg(&self, _reg: PReg, _expected: bool) {}
 
+    /// Add a register clobber set. This is a set of registers that
+    /// are written by the instruction, so must be reserved (not used)
+    /// for the whole instruction, but are not used afterward.
+    fn reg_clobbers(&mut self, _regs: PRegSet) {}
+}
+
+pub trait OperandVisitorImpl: OperandVisitor {
     /// Add a use of a fixed, nonallocatable physical register.
-    pub fn reg_fixed_nonallocatable(&mut self, preg: PReg) {
-        debug_assert!(!self.is_allocatable_preg(preg));
+    fn reg_fixed_nonallocatable(&mut self, preg: PReg) {
+        self.debug_assert_is_allocatable_preg(preg, false);
         // Magic VReg which does not participate in register allocation.
         let mut reg = Reg(VReg::new(VReg::MAX, preg.class()));
         let constraint = OperandConstraint::FixedReg(preg);
@@ -358,19 +350,19 @@ impl<'a, F: Fn(VReg) -> VReg> OperandCollector<'a, F> {
 
     /// Add a register use, at the start of the instruction (`Before`
     /// position).
-    pub fn reg_use(&mut self, reg: &mut impl AsMut<Reg>) {
+    fn reg_use(&mut self, reg: &mut impl AsMut<Reg>) {
         self.reg_maybe_fixed(reg.as_mut(), OperandKind::Use, OperandPos::Early);
     }
 
     /// Add a register use, at the end of the instruction (`After` position).
-    pub fn reg_late_use(&mut self, reg: &mut impl AsMut<Reg>) {
+    fn reg_late_use(&mut self, reg: &mut impl AsMut<Reg>) {
         self.reg_maybe_fixed(reg.as_mut(), OperandKind::Use, OperandPos::Late);
     }
 
     /// Add a register def, at the end of the instruction (`After`
     /// position). Use only when this def will be written after all
     /// uses are read.
-    pub fn reg_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>) {
+    fn reg_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>) {
         self.reg_maybe_fixed(reg.reg.as_mut(), OperandKind::Def, OperandPos::Late);
     }
 
@@ -378,25 +370,25 @@ impl<'a, F: Fn(VReg) -> VReg> OperandCollector<'a, F> {
     /// beginning of the instruction, alongside all uses. Use this
     /// when the def may be written before all uses are read; the
     /// regalloc will ensure that it does not overwrite any uses.
-    pub fn reg_early_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>) {
+    fn reg_early_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>) {
         self.reg_maybe_fixed(reg.reg.as_mut(), OperandKind::Def, OperandPos::Early);
     }
 
     /// Add a register "fixed use", which ties a vreg to a particular
     /// RealReg at the end of the instruction.
-    pub fn reg_fixed_late_use(&mut self, reg: &mut impl AsMut<Reg>, rreg: Reg) {
+    fn reg_fixed_late_use(&mut self, reg: &mut impl AsMut<Reg>, rreg: Reg) {
         self.reg_fixed(reg.as_mut(), rreg, OperandKind::Use, OperandPos::Late);
     }
 
     /// Add a register "fixed use", which ties a vreg to a particular
     /// RealReg at this point.
-    pub fn reg_fixed_use(&mut self, reg: &mut impl AsMut<Reg>, rreg: Reg) {
+    fn reg_fixed_use(&mut self, reg: &mut impl AsMut<Reg>, rreg: Reg) {
         self.reg_fixed(reg.as_mut(), rreg, OperandKind::Use, OperandPos::Early);
     }
 
     /// Add a register "fixed def", which ties a vreg to a particular
     /// RealReg at this point.
-    pub fn reg_fixed_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>, rreg: Reg) {
+    fn reg_fixed_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>, rreg: Reg) {
         self.reg_fixed(reg.reg.as_mut(), rreg, OperandKind::Def, OperandPos::Late);
     }
 
@@ -404,10 +396,7 @@ impl<'a, F: Fn(VReg) -> VReg> OperandCollector<'a, F> {
     fn reg_fixed(&mut self, reg: &mut Reg, rreg: Reg, kind: OperandKind, pos: OperandPos) {
         debug_assert!(reg.is_virtual());
         let rreg = rreg.to_real_reg().expect("fixed reg is not a RealReg");
-        debug_assert!(
-            self.is_allocatable_preg(rreg.into()),
-            "{rreg:?} is not allocatable"
-        );
+        self.debug_assert_is_allocatable_preg(rreg.into(), true);
         let constraint = OperandConstraint::FixedReg(rreg.into());
         self.add_operand(reg, constraint, kind, pos);
     }
@@ -425,7 +414,7 @@ impl<'a, F: Fn(VReg) -> VReg> OperandCollector<'a, F> {
     /// Add a register def that reuses an earlier use-operand's
     /// allocation. The index of that earlier operand (relative to the
     /// current instruction's start of operands) must be known.
-    pub fn reg_reuse_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>, idx: usize) {
+    fn reg_reuse_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>, idx: usize) {
         let reg = reg.reg.as_mut();
         if let Some(rreg) = reg.to_real_reg() {
             // In some cases we see real register arguments to a reg_reuse_def
@@ -442,11 +431,33 @@ impl<'a, F: Fn(VReg) -> VReg> OperandCollector<'a, F> {
             self.add_operand(reg, constraint, OperandKind::Def, OperandPos::Late);
         }
     }
+}
 
-    /// Add a register clobber set. This is a set of registers that
-    /// are written by the instruction, so must be reserved (not used)
-    /// for the whole instruction, but are not used afterward.
-    pub fn reg_clobbers(&mut self, regs: PRegSet) {
+impl<T: OperandVisitor> OperandVisitorImpl for T {}
+
+impl<'a, F: Fn(VReg) -> VReg> OperandVisitor for OperandCollector<'a, F> {
+    fn add_operand(
+        &mut self,
+        reg: &mut Reg,
+        constraint: OperandConstraint,
+        kind: OperandKind,
+        pos: OperandPos,
+    ) {
+        reg.0 = (self.renamer)(reg.0);
+        self.operands
+            .push(Operand::new(reg.0, constraint, kind, pos));
+    }
+
+    fn debug_assert_is_allocatable_preg(&self, reg: PReg, expected: bool) {
+        debug_assert_eq!(
+            self.allocatable.contains(reg),
+            expected,
+            "{reg:?} should{} be allocatable",
+            if expected { "" } else { " not" }
+        );
+    }
+
+    fn reg_clobbers(&mut self, regs: PRegSet) {
         self.clobbers.union_from(regs);
     }
 }
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs
index 6ca3039cb..cc76d358b 100644
--- a/cranelift/codegen/src/machinst/vcode.rs
+++ b/cranelift/codegen/src/machinst/vcode.rs
@@ -17,7 +17,6 @@
 //! See the main module comment in `mod.rs` for more details on the VCode-based
 //! backend pipeline.
 
-use crate::fx::FxHashMap;
 use crate::ir::pcc::*;
 use crate::ir::{self, types, Constant, ConstantData, ValueLabel};
 use crate::machinst::*;
@@ -29,7 +28,9 @@ use regalloc2::{
     Edit, Function as RegallocFunction, InstOrEdit, InstRange, MachineEnv, Operand, OperandKind,
     PRegSet, RegClass,
 };
+use rustc_hash::FxHashMap;
 
+use core::mem::take;
 use cranelift_entity::{entity_impl, Keys};
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
@@ -133,20 +134,6 @@ pub struct VCode<I: VCodeInst> {
     /// corresponding to all of its successors.
     branch_block_arg_succ_range: Vec<(u32, u32)>,
 
-    /// VReg aliases. Each key in this table is translated to its
-    /// value when gathering Operands from instructions. Aliases are
-    /// not chased transitively (we do not further look up the
-    /// translated reg to see if it is another alias).
-    ///
-    /// We use these aliases to rename an instruction's expected
-    /// result vregs to the returned vregs from lowering, which are
-    /// usually freshly-allocated temps.
-    ///
-    /// Operands and branch arguments will already have been
-    /// translated through this alias table; but it helps to make
-    /// sense of instructions when pretty-printed, for example.
-    vreg_aliases: FxHashMap<regalloc2::VReg, regalloc2::VReg>,
-
     /// Block-order information.
     block_order: BlockLoweringOrder,
 
@@ -393,14 +380,6 @@ impl<I: VCodeInst> VCodeBuilder<I> {
         labels.push((last, inst, reg.into()));
     }
 
-    pub fn set_vreg_alias(&mut self, from: Reg, to: Reg) {
-        let from = from.into();
-        let resolved_to = self.vcode.resolve_vreg_alias(to.into());
-        // Disallow cycles (see below).
-        assert_ne!(resolved_to, from);
-        self.vcode.vreg_aliases.insert(from, resolved_to);
-    }
-
     /// Access the constants.
     pub fn constants(&mut self) -> &mut VCodeConstants {
         &mut self.vcode.constants
@@ -438,7 +417,7 @@ impl<I: VCodeInst> VCodeBuilder<I> {
     /// Called once, when a build in Backward order is complete, to
     /// perform the overall reversal (into final forward order) and
     /// finalize metadata accordingly.
-    fn reverse_and_finalize(&mut self) {
+    fn reverse_and_finalize(&mut self, vregs: &VRegAllocator<I>) {
         let n_insts = self.vcode.insts.len();
         if n_insts == 0 {
             return;
@@ -482,7 +461,7 @@ impl<I: VCodeInst> VCodeBuilder<I> {
         // Generate debug-value labels based on per-label maps.
         for (label, tuples) in &self.debug_info {
             for &(start, end, vreg) in tuples {
-                let vreg = self.vcode.resolve_vreg_alias(vreg);
+                let vreg = vregs.resolve_vreg_alias(vreg);
                 let fwd_start = translate(end);
                 let fwd_end = translate(start);
                 self.vcode
@@ -498,7 +477,7 @@ impl<I: VCodeInst> VCodeBuilder<I> {
             .sort_unstable_by_key(|(vreg, _, _, _)| *vreg);
     }
 
-    fn collect_operands(&mut self) {
+    fn collect_operands(&mut self, vregs: &VRegAllocator<I>) {
         let allocatable = PRegSet::from(self.vcode.machine_env());
         for (i, insn) in self.vcode.insts.iter_mut().enumerate() {
             // Push operands from the instruction onto the operand list.
@@ -512,10 +491,9 @@ impl<I: VCodeInst> VCodeBuilder<I> {
             // need to do the `match` over the instruction to visit
             // its register fields (which is slow, branchy code) once.
 
-            let vreg_aliases = &self.vcode.vreg_aliases;
             let mut op_collector =
                 OperandCollector::new(&mut self.vcode.operands, allocatable, |vreg| {
-                    VCode::<I>::resolve_vreg_alias_impl(vreg_aliases, vreg)
+                    vregs.resolve_vreg_alias(vreg)
                 });
             insn.get_operands(&mut op_collector);
             let (ops, clobbers) = op_collector.finish();
@@ -543,22 +521,22 @@ impl<I: VCodeInst> VCodeBuilder<I> {
 
         // Translate blockparam args via the vreg aliases table as well.
         for arg in &mut self.vcode.branch_block_args {
-            let new_arg = VCode::<I>::resolve_vreg_alias_impl(&self.vcode.vreg_aliases, *arg);
+            let new_arg = vregs.resolve_vreg_alias(*arg);
             trace!("operandcollector: block arg {:?} -> {:?}", arg, new_arg);
             *arg = new_arg;
         }
     }
 
     /// Build the final VCode.
-    pub fn build(mut self, vregs: VRegAllocator<I>) -> VCode<I> {
-        self.vcode.vreg_types = vregs.vreg_types;
-        self.vcode.facts = vregs.facts;
-        self.vcode.reftyped_vregs = vregs.reftyped_vregs;
+    pub fn build(mut self, mut vregs: VRegAllocator<I>) -> VCode<I> {
+        self.vcode.vreg_types = take(&mut vregs.vreg_types);
+        self.vcode.facts = take(&mut vregs.facts);
+        self.vcode.reftyped_vregs = take(&mut vregs.reftyped_vregs);
 
         if self.direction == VCodeBuildDirection::Backward {
-            self.reverse_and_finalize();
+            self.reverse_and_finalize(&vregs);
         }
-        self.collect_operands();
+        self.collect_operands(&vregs);
 
         // Apply register aliases to the `reftyped_vregs` list since this list
         // will be returned directly to `regalloc2` eventually and all
@@ -568,7 +546,7 @@ impl<I: VCodeInst> VCodeBuilder<I> {
         // Also note that `reftyped_vregs` can't have duplicates, so after the
         // aliases are applied duplicates are removed.
         for reg in self.vcode.reftyped_vregs.iter_mut() {
-            *reg = VCode::<I>::resolve_vreg_alias_impl(&self.vcode.vreg_aliases, *reg);
+            *reg = vregs.resolve_vreg_alias(*reg);
         }
         self.vcode.reftyped_vregs.sort();
         self.vcode.reftyped_vregs.dedup();
@@ -576,8 +554,37 @@ impl<I: VCodeInst> VCodeBuilder<I> {
         self.compute_preds_from_succs();
         self.vcode.debug_value_labels.sort_unstable();
 
-        // All aliases are resolved now, so remove them from the map.
-        self.vcode.vreg_aliases.clear();
+        // At this point, nothing in the vcode should mention any
+        // VReg which has been aliased. All the appropriate rewriting
+        // should have happened above. Just to be sure, let's
+        // double-check each field which has vregs.
+        // Note: can't easily check vcode.insts, resolved in collect_operands.
+        // Operands are resolved in collect_operands.
+        vregs.debug_assert_no_vreg_aliases(self.vcode.operands.iter().map(|op| op.vreg()));
+        // Currently block params are never aliased to another vreg.
+        vregs.debug_assert_no_vreg_aliases(self.vcode.block_params.iter().copied());
+        // Branch block args are resolved in collect_operands.
+        vregs.debug_assert_no_vreg_aliases(self.vcode.branch_block_args.iter().copied());
+        // Reftyped vregs are resolved above in this function.
+        vregs.debug_assert_no_vreg_aliases(self.vcode.reftyped_vregs.iter().copied());
+        // Debug value labels are resolved in reverse_and_finalize.
+        vregs.debug_assert_no_vreg_aliases(
+            self.vcode.debug_value_labels.iter().map(|&(vreg, ..)| vreg),
+        );
+        // Facts are resolved eagerly during set_vreg_alias.
+        vregs.debug_assert_no_vreg_aliases(
+            self.vcode
+                .facts
+                .iter()
+                .zip(&vregs.vreg_types)
+                .enumerate()
+                .filter(|(_, (fact, _))| fact.is_some())
+                .map(|(vreg, (_, &ty))| {
+                    let (regclasses, _) = I::rc_for_type(ty).unwrap();
+                    VReg::new(vreg, regclasses[0])
+                }),
+        );
+
         self.vcode
     }
 }
@@ -623,7 +630,6 @@ impl<I: VCodeInst> VCode<I> {
             reftyped_vregs: vec![],
             constants,
             debug_value_labels: vec![],
-            vreg_aliases: FxHashMap::with_capacity_and_hasher(10 * n_blocks, Default::default()),
             facts: vec![],
         }
     }
@@ -1207,39 +1213,6 @@ impl<I: VCodeInst> VCode<I> {
         self.block_order.lowered_order()[block.index()].orig_block()
     }
 
-    fn resolve_vreg_alias(&self, from: regalloc2::VReg) -> regalloc2::VReg {
-        Self::resolve_vreg_alias_impl(&self.vreg_aliases, from)
-    }
-
-    /// Implementation of alias resolution. Separate helper that does
-    /// not borrow `self` in order to allow working around borrowing
-    /// restrictions.
-    fn resolve_vreg_alias_impl(
-        aliases: &FxHashMap<regalloc2::VReg, regalloc2::VReg>,
-        from: regalloc2::VReg,
-    ) -> regalloc2::VReg {
-        // We prevent cycles from existing by resolving targets of
-        // aliases eagerly before setting them. If the target resolves
-        // to the origin of the alias, then a cycle would be created
-        // and the alias is disallowed. Because of the structure of
-        // SSA code (one instruction can refer to another's defs but
-        // not vice-versa, except indirectly through
-        // phis/blockparams), cycles should not occur as we use
-        // aliases to redirect vregs to the temps that actually define
-        // them.
-
-        let mut vreg = from;
-        while let Some(to) = aliases.get(&vreg) {
-            vreg = *to;
-        }
-        vreg
-    }
-
-    #[inline]
-    fn debug_assert_no_vreg_aliases(&self, mut list: impl Iterator<Item = VReg>) {
-        debug_assert!(list.all(|vreg| !self.vreg_aliases.contains_key(&vreg)));
-    }
-
     /// Get the type of a VReg.
     pub fn vreg_type(&self, vreg: VReg) -> Type {
         self.vreg_types[vreg.vreg()]
@@ -1247,13 +1220,11 @@ impl<I: VCodeInst> VCode<I> {
 
     /// Get the fact, if any, for a given VReg.
     pub fn vreg_fact(&self, vreg: VReg) -> Option<&Fact> {
-        self.debug_assert_no_vreg_aliases(core::iter::once(vreg));
         self.facts[vreg.vreg()].as_ref()
     }
 
     /// Set the fact for a given VReg.
     pub fn set_vreg_fact(&mut self, vreg: VReg, fact: Fact) {
-        self.debug_assert_no_vreg_aliases(core::iter::once(vreg));
         trace!("set fact on {}: {:?}", vreg, fact);
         self.facts[vreg.vreg()] = Some(fact);
     }
@@ -1311,11 +1282,7 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
         }
 
         let (start, end) = self.block_params_range[block.index()];
-        let ret = &self.block_params[start as usize..end as usize];
-        // Currently block params are never aliased to another vreg, but
-        // double-check just to be sure.
-        self.debug_assert_no_vreg_aliases(ret.iter().copied());
-        ret
+        &self.block_params[start as usize..end as usize]
     }
 
     fn branch_blockparams(&self, block: BlockIndex, _insn: InsnIndex, succ_idx: usize) -> &[VReg] {
@@ -1323,10 +1290,7 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
         let succ_ranges =
             &self.branch_block_arg_range[succ_range_start as usize..succ_range_end as usize];
         let (branch_block_args_start, branch_block_args_end) = succ_ranges[succ_idx];
-        let ret = &self.branch_block_args
-            [branch_block_args_start as usize..branch_block_args_end as usize];
-        self.debug_assert_no_vreg_aliases(ret.iter().copied());
-        ret
+        &self.branch_block_args[branch_block_args_start as usize..branch_block_args_end as usize]
     }
 
     fn is_ret(&self, insn: InsnIndex) -> bool {
@@ -1351,12 +1315,7 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
 
     fn inst_operands(&self, insn: InsnIndex) -> &[Operand] {
         let (start, end) = self.operand_ranges[insn.index()];
-        let ret = &self.operands[start as usize..end as usize];
-        // It should be true by construction that `Operand`s do not contain any
-        // aliased vregs since they're all collected and mapped when the VCode
-        // is itself constructed.
-        self.debug_assert_no_vreg_aliases(ret.iter().map(|op| op.vreg()));
-        ret
+        &self.operands[start as usize..end as usize]
     }
 
     fn inst_clobbers(&self, insn: InsnIndex) -> PRegSet {
@@ -1368,18 +1327,11 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
     }
 
     fn reftype_vregs(&self) -> &[VReg] {
-        let ret = &self.reftyped_vregs;
-        self.debug_assert_no_vreg_aliases(ret.iter().copied());
-        ret
+        &self.reftyped_vregs
     }
 
     fn debug_value_labels(&self) -> &[(VReg, InsnIndex, InsnIndex, u32)] {
-        // VRegs here are inserted into `debug_value_labels` after code is
-        // generated and aliases are fully defined, so double-check that
-        // aliases are not lingering.
-        let ret = &self.debug_value_labels;
-        self.debug_assert_no_vreg_aliases(ret.iter().map(|&(vreg, ..)| vreg));
-        ret
+        &self.debug_value_labels
     }
 
     fn spillslot_size(&self, regclass: RegClass) -> usize {
@@ -1394,12 +1346,9 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
     }
 }
 
-impl<I: VCodeInst> fmt::Debug for VCode<I> {
+impl<I: VCodeInst> Debug for VRegAllocator<I> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(f, "VCode {{")?;
-        writeln!(f, "  Entry block: {}", self.entry.index())?;
-
-        let mut state = Default::default();
+        writeln!(f, "VRegAllocator {{")?;
 
         let mut alias_keys = self.vreg_aliases.keys().cloned().collect::<Vec<_>>();
         alias_keys.sort_unstable();
@@ -1408,6 +1357,23 @@ impl<I: VCodeInst> fmt::Debug for VCode<I> {
             writeln!(f, "  {:?} := {:?}", Reg::from(key), Reg::from(*dest))?;
         }
 
+        for (vreg, fact) in self.facts.iter().enumerate() {
+            if let Some(fact) = fact {
+                writeln!(f, "  v{} ! {}", vreg, fact)?;
+            }
+        }
+
+        writeln!(f, "}}")
+    }
+}
+
+impl<I: VCodeInst> fmt::Debug for VCode<I> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "VCode {{")?;
+        writeln!(f, "  Entry block: {}", self.entry.index())?;
+
+        let mut state = Default::default();
+
         for block in 0..self.num_blocks() {
             let block = BlockIndex::new(block);
             writeln!(f, "Block {}:", block.index())?;
@@ -1418,12 +1384,6 @@ impl<I: VCodeInst> fmt::Debug for VCode<I> {
                 writeln!(f, "    (successor: Block {})", succ.index())?;
             }
             let (start, end) = self.block_ranges[block.index()];
-            writeln!(
-                f,
-                "    (instruction range: {} .. {})",
-                start.index(),
-                end.index()
-            )?;
             for inst in start.index()..end.index() {
                 writeln!(
                     f,
@@ -1431,10 +1391,12 @@ impl<I: VCodeInst> fmt::Debug for VCode<I> {
                     inst,
                     self.insts[inst].pretty_print_inst(&[], &mut state)
                 )?;
-                for operand in self.inst_operands(InsnIndex::new(inst)) {
-                    if operand.kind() == OperandKind::Def {
-                        if let Some(fact) = &self.facts[operand.vreg().vreg()] {
-                            writeln!(f, "    v{} ! {}", operand.vreg().vreg(), fact)?;
+                if !self.operands.is_empty() {
+                    for operand in self.inst_operands(InsnIndex::new(inst)) {
+                        if operand.kind() == OperandKind::Def {
+                            if let Some(fact) = &self.facts[operand.vreg().vreg()] {
+                                writeln!(f, "    v{} ! {}", operand.vreg().vreg(), fact)?;
+                            }
                         }
                     }
                 }
@@ -1456,6 +1418,14 @@ pub struct VRegAllocator<I> {
     /// reftype-status of each vreg) for efficient iteration.
     reftyped_vregs: Vec<VReg>,
 
+    /// VReg aliases. When the final VCode is built we rewrite all
+    /// uses of the keys in this table to their replacement values.
+    ///
+    /// We use these aliases to rename an instruction's expected
+    /// result vregs to the returned vregs from lowering, which are
+    /// usually freshly-allocated temps.
+    vreg_aliases: FxHashMap<regalloc2::VReg, regalloc2::VReg>,
+
     /// A deferred error, to be bubbled up to the top level of the
     /// lowering algorithm. We take this approach because we cannot
     /// currently propagate a `Result` upward through ISLE code (the
@@ -1471,12 +1441,16 @@ pub struct VRegAllocator<I> {
 
 impl<I: VCodeInst> VRegAllocator<I> {
     /// Make a new VRegAllocator.
-    pub fn new() -> Self {
+    pub fn with_capacity(capacity: usize) -> Self {
+        let capacity = first_user_vreg_index() + capacity;
+        let mut vreg_types = Vec::with_capacity(capacity);
+        vreg_types.resize(first_user_vreg_index(), types::INVALID);
         Self {
-            vreg_types: vec![types::INVALID; first_user_vreg_index()],
-            facts: vec![],
+            vreg_types,
             reftyped_vregs: vec![],
+            vreg_aliases: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
             deferred_error: None,
+            facts: Vec::with_capacity(capacity),
             _inst: core::marker::PhantomData::default(),
         }
     }
@@ -1547,25 +1521,58 @@ impl<I: VCodeInst> VRegAllocator<I> {
         }
     }
 
+    /// Rewrite any mention of `from` into `to`.
+    pub fn set_vreg_alias(&mut self, from: Reg, to: Reg) {
+        let from = from.into();
+        let resolved_to = self.resolve_vreg_alias(to.into());
+        // Disallow cycles (see below).
+        assert_ne!(resolved_to, from);
+
+        // Maintain the invariant that PCC facts only exist on vregs
+        // which aren't aliases. We want to preserve whatever was
+        // stated about the vreg before its producer was lowered.
+        if let Some(fact) = self.facts[from.vreg()].take() {
+            self.set_fact(resolved_to, fact);
+        }
+
+        let old_alias = self.vreg_aliases.insert(from, resolved_to);
+        debug_assert_eq!(old_alias, None);
+    }
+
+    fn resolve_vreg_alias(&self, mut vreg: regalloc2::VReg) -> regalloc2::VReg {
+        // We prevent cycles from existing by resolving targets of
+        // aliases eagerly before setting them. If the target resolves
+        // to the origin of the alias, then a cycle would be created
+        // and the alias is disallowed. Because of the structure of
+        // SSA code (one instruction can refer to another's defs but
+        // not vice-versa, except indirectly through
+        // phis/blockparams), cycles should not occur as we use
+        // aliases to redirect vregs to the temps that actually define
+        // them.
+        while let Some(to) = self.vreg_aliases.get(&vreg) {
+            vreg = *to;
+        }
+        vreg
+    }
+
+    #[inline]
+    fn debug_assert_no_vreg_aliases(&self, mut list: impl Iterator<Item = VReg>) {
+        debug_assert!(list.all(|vreg| !self.vreg_aliases.contains_key(&vreg)));
+    }
+
     /// Set the proof-carrying code fact on a given virtual register.
     ///
     /// Returns the old fact, if any (only one fact can be stored).
-    pub fn set_fact(&mut self, vreg: VirtualReg, fact: Fact) -> Option<Fact> {
+    fn set_fact(&mut self, vreg: regalloc2::VReg, fact: Fact) -> Option<Fact> {
         trace!("vreg {:?} has fact: {:?}", vreg, fact);
-        self.facts[vreg.index()].replace(fact)
-    }
-
-    /// Take (and remove) a fact about a VReg. Used when setting up
-    /// aliases: we want to move a fact from the alias vreg to the
-    /// aliased vreg, to preserve facts about a value that were stated
-    /// before we lowered its producer.
-    pub fn take_fact(&mut self, vreg: VirtualReg) -> Option<Fact> {
-        self.facts[vreg.index()].take()
+        debug_assert!(!self.vreg_aliases.contains_key(&vreg));
+        self.facts[vreg.vreg()].replace(fact)
     }
 
     /// Set a fact only if one doesn't already exist.
     pub fn set_fact_if_missing(&mut self, vreg: VirtualReg, fact: Fact) {
-        if self.facts[vreg.index()].is_none() {
+        let vreg = self.resolve_vreg_alias(vreg.into());
+        if self.facts[vreg.vreg()].is_none() {
             self.set_fact(vreg, fact);
         }
     }
@@ -1583,7 +1590,7 @@ impl<I: VCodeInst> VRegAllocator<I> {
         // into multiple VRegs.
         assert!(result.len() == 1 || fact.is_none());
         if let Some(fact) = fact {
-            self.set_fact(result.regs()[0].to_virtual_reg().unwrap(), fact);
+            self.set_fact(result.regs()[0].into(), fact);
         }
 
         Ok(result)
diff --git a/cranelift/codegen/src/remove_constant_phis.rs b/cranelift/codegen/src/remove_constant_phis.rs
index 5a8ab0235..483788e27 100644
--- a/cranelift/codegen/src/remove_constant_phis.rs
+++ b/cranelift/codegen/src/remove_constant_phis.rs
@@ -1,14 +1,13 @@
 //! A Constant-Phi-Node removal pass.
 
 use crate::dominator_tree::DominatorTree;
-use crate::fx::FxHashMap;
-use crate::fx::FxHashSet;
 use crate::ir;
 use crate::ir::Function;
 use crate::ir::{Block, BlockCall, Inst, Value};
 use crate::timing;
 use bumpalo::Bump;
 use cranelift_entity::SecondaryMap;
+use rustc_hash::{FxHashMap, FxHashSet};
 use smallvec::SmallVec;
 
 // A note on notation.  For the sake of clarity, this file uses the phrase
diff --git a/cranelift/codegen/src/scoped_hash_map.rs b/cranelift/codegen/src/scoped_hash_map.rs
index 7f64ae3c3..170a6140d 100644
--- a/cranelift/codegen/src/scoped_hash_map.rs
+++ b/cranelift/codegen/src/scoped_hash_map.rs
@@ -4,8 +4,8 @@
 //! container that has a concept of scopes that can be entered and exited, such that
 //! values inserted while inside a scope aren't visible outside the scope.
 
-use crate::fx::FxHashMap;
 use core::hash::Hash;
+use rustc_hash::FxHashMap;
 use smallvec::{smallvec, SmallVec};
 
 #[cfg(not(feature = "std"))]
@@ -90,7 +90,7 @@ where
     /// Creates an empty `ScopedHashMap`.
     pub fn new() -> Self {
         Self {
-            map: FxHashMap(),
+            map: FxHashMap::default(),
             generation: 0,
             generation_by_depth: smallvec![0],
         }
diff --git a/cranelift/src/souper_harvest.rs b/cranelift/src/souper_harvest.rs
index 6d74bb63b..65e21447b 100644
--- a/cranelift/src/souper_harvest.rs
+++ b/cranelift/src/souper_harvest.rs
@@ -82,7 +82,7 @@ pub fn run(options: &Options) -> Result<()> {
                     let i = lhs.find('\n').unwrap();
                     &lhs[i + 1..]
                 };
-                let hash = fxhash::hash(lhs.as_bytes());
+                let hash = hash(lhs.as_bytes());
                 if already_harvested.insert(hash) {
                     let output_path = output_dir.join(hash.to_string());
                     let mut output =
@@ -139,3 +139,11 @@ fn parse_input(path: PathBuf) -> Result<Vec<Function>> {
         .with_context(|| format!("parse error in {}", path.display()))?;
     Ok(funcs)
 }
+
+/// A convenience functon for a quick usize hash
+#[inline]
+pub fn hash<T: std::hash::Hash + ?Sized>(v: &T) -> usize {
+    let mut state = rustc_hash::FxHasher::default();
+    v.hash(&mut state);
+    std::hash::Hasher::finish(&state) as usize
+}
diff --git a/crates/c-api/CMakeLists.txt b/crates/c-api/CMakeLists.txt
index 432df8185..16b186f24 100644
--- a/crates/c-api/CMakeLists.txt
+++ b/crates/c-api/CMakeLists.txt
@@ -78,6 +78,9 @@ if(ANDROID)
 endif()
 include(ExternalProject)
 find_program(WASMTIME_CARGO_BINARY cargo)
+if(NOT WASMTIME_CARGO_BINARY)
+    message(FATAL_ERROR [["cargo" was not found. Ensure "cargo" is in PATH. Aborting...]])
+endif()
 ExternalProject_Add(
 	wasmtime-crate
 	DOWNLOAD_COMMAND ""
diff --git a/crates/c-api/src/table.rs b/crates/c-api/src/table.rs
index 69ec5a689..d7ff32458 100644
--- a/crates/c-api/src/table.rs
+++ b/crates/c-api/src/table.rs
@@ -4,7 +4,7 @@ use crate::{
 };
 use anyhow::anyhow;
 use std::mem::MaybeUninit;
-use wasmtime::{Extern, HeapType, Ref, RootScope, Table, TableType};
+use wasmtime::{Extern, Ref, RootScope, Table, TableType};
 
 #[derive(Clone)]
 #[repr(transparent)]
@@ -33,13 +33,8 @@ impl wasm_table_t {
 }
 
 fn option_wasm_ref_t_to_ref(r: Option<&wasm_ref_t>, table_ty: &TableType) -> Ref {
-    match (r.map(|r| r.r.clone()), table_ty.element().heap_type().top()) {
-        (Some(r), _) => r,
-        (None, HeapType::Func) => Ref::Func(None),
-        (None, HeapType::Extern) => Ref::Extern(None),
-        (None, HeapType::Any) => Ref::Any(None),
-        (None, ty) => unreachable!("not a top type: {ty:?}"),
-    }
+    r.map(|r| r.r.clone())
+        .unwrap_or_else(|| Ref::null(table_ty.element().heap_type()))
 }
 
 #[no_mangle]
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
index d51e7bd96..5d0c3f8f8 100644
--- a/crates/cli-flags/src/lib.rs
+++ b/crates/cli-flags/src/lib.rs
@@ -258,11 +258,11 @@ wasmtime_option_group! {
         pub cli: Option<bool>,
         /// Deprecated alias for `cli`
         pub common: Option<bool>,
-        /// Enable suport for WASI neural network API (experimental)
+        /// Enable support for WASI neural network API (experimental)
         pub nn: Option<bool>,
-        /// Enable suport for WASI threading API (experimental)
+        /// Enable support for WASI threading API (experimental)
         pub threads: Option<bool>,
-        /// Enable suport for WASI HTTP API (experimental)
+        /// Enable support for WASI HTTP API (experimental)
         pub http: Option<bool>,
         /// Inherit environment variables and file descriptors following the
         /// systemd listen fd specification (UNIX only)
diff --git a/crates/fuzzing/src/generators/value.rs b/crates/fuzzing/src/generators/value.rs
index 669b7cd25..f12c6d3fa 100644
--- a/crates/fuzzing/src/generators/value.rs
+++ b/crates/fuzzing/src/generators/value.rs
@@ -60,9 +60,9 @@ impl DiffValue {
                 ];
                 let bits = biased_arbitrary_value(u, known_f32_values)?;
 
-                // If the chosen bits are NAN then always use the canonical bit
-                // pattern of nan to enable better compatibility with engines
-                // where arbitrary nan patterns can't make their way into wasm
+                // If the chosen bits are NaN then always use the canonical bit
+                // pattern of NaN to enable better compatibility with engines
+                // where arbitrary NaN patterns can't make their way into wasm
                 // (e.g. v8 through JS can't do that).
                 let bits = if f32::from_bits(bits).is_nan() {
                     f32::NAN.to_bits()
@@ -85,7 +85,7 @@ impl DiffValue {
                     f64::MAX.to_bits(),
                 ];
                 let bits = biased_arbitrary_value(u, known_f64_values)?;
-                // See `f32` above for why canonical nan patterns are always
+                // See `f32` above for why canonical NaN patterns are always
                 // used.
                 let bits = if f64::from_bits(bits).is_nan() {
                     f64::NAN.to_bits()
@@ -302,7 +302,7 @@ impl TryFrom<wasmtime::ValType> for DiffValueType {
                 (true, HeapType::Any) => Ok(Self::AnyRef),
                 (true, HeapType::I31) => Ok(Self::AnyRef),
                 (true, HeapType::None) => Ok(Self::AnyRef),
-                _ => Err("non-funcref and non-externref reference types are not suported yet"),
+                _ => Err("non-funcref and non-externref reference types are not supported yet"),
             },
         }
     }
diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs
index 59cba7739..d88cd2c3a 100644
--- a/crates/fuzzing/src/oracles/dummy.rs
+++ b/crates/fuzzing/src/oracles/dummy.rs
@@ -1,6 +1,6 @@
 //! Dummy implementations of things that a Wasm module can import.
 
-use anyhow::bail;
+use anyhow::ensure;
 use wasmtime::*;
 
 /// Create a set of dummy functions/globals/etc for the given imports.
@@ -45,13 +45,13 @@ pub fn dummy_value(val_ty: ValType) -> Result<Val> {
         ValType::F32 => Val::F32(0),
         ValType::F64 => Val::F64(0),
         ValType::V128 => Val::V128(0.into()),
-        ValType::Ref(r) => match r.heap_type().top() {
-            _ if !r.is_nullable() => bail!("cannot construct a dummy value of type `{r}`"),
-            HeapType::Extern => Val::null_extern_ref(),
-            HeapType::Func => Val::null_func_ref(),
-            HeapType::Any => Val::null_any_ref(),
-            ty => unreachable!("not a top type: {ty:?}"),
-        },
+        ValType::Ref(r) => {
+            ensure!(
+                r.is_nullable(),
+                "cannot construct a dummy value of type `{r}`"
+            );
+            Val::null_ref(r.heap_type())
+        }
     })
 }
 
diff --git a/crates/runtime/src/component/resources.rs b/crates/runtime/src/component/resources.rs
index a94425362..d0a013321 100644
--- a/crates/runtime/src/component/resources.rs
+++ b/crates/runtime/src/component/resources.rs
@@ -1,7 +1,7 @@
 //! Implementation of the canonical-ABI related intrinsics for resources in the
 //! component model.
 //!
-//! This module contains all the relevant gory details of the details of the
+//! This module contains all the relevant gory details of the
 //! component model related to lifting and lowering resources. For example
 //! intrinsics like `resource.new` will bottom out in calling this file, and
 //! this is where resource tables are actually defined and modified.
diff --git a/crates/runtime/src/cow.rs b/crates/runtime/src/cow.rs
index 7dfece799..908e80d49 100644
--- a/crates/runtime/src/cow.rs
+++ b/crates/runtime/src/cow.rs
@@ -299,7 +299,7 @@ impl MemoryImageSlot {
     /// Create a new MemoryImageSlot. Assumes that there is an anonymous
     /// mmap backing in the given range to start.
     ///
-    /// The `accessible` parameter descibes how much of linear memory is
+    /// The `accessible` parameter describes how much of linear memory is
     /// already mapped as R/W with all zero-bytes. The `static_size` value is
     /// the maximum size of this image which `accessible` cannot grow beyond,
     /// and all memory from `accessible` from `static_size` should be mapped as
diff --git a/crates/runtime/src/export.rs b/crates/runtime/src/export.rs
index 23338ea42..bd05c1b40 100644
--- a/crates/runtime/src/export.rs
+++ b/crates/runtime/src/export.rs
@@ -29,9 +29,9 @@ pub struct ExportFunction {
     pub func_ref: NonNull<VMFuncRef>,
 }
 
-// It's part of the contract of using `ExportFunction` that synchronization
-// properties are upheld, so declare that despite the raw pointers inside this
-// is send/sync.
+// As part of the contract for using `ExportFunction`, synchronization
+// properties must be upheld. Therefore, despite containing raw pointers,
+// it is declared as Send/Sync.
 unsafe impl Send for ExportFunction {}
 unsafe impl Sync for ExportFunction {}
 
diff --git a/crates/wasi-nn/examples/classification-component-onnx/src/bindings.rs b/crates/wasi-nn/examples/classification-component-onnx/src/bindings.rs
index 811582889..4e19e88d6 100644
--- a/crates/wasi-nn/examples/classification-component-onnx/src/bindings.rs
+++ b/crates/wasi-nn/examples/classification-component-onnx/src/bindings.rs
@@ -3357,7 +3357,7 @@ pub mod wasi {
       /// This function does not return a `result`; polling in itself does not
       /// do any I/O so it doesn't fail. If any of the I/O sources identified by
       /// the pollables has an error, it is indicated by marking the source as
-      /// being reaedy for I/O.
+      /// being ready for I/O.
       pub fn poll(in_: &[&Pollable],) -> wit_bindgen::rt::vec::Vec::<u32>{
         
         #[allow(unused_imports)]
@@ -9168,4 +9168,4 @@ pub mod wasi {
           #[doc(hidden)]
           #[cfg(target_arch = "wasm32")]
           pub fn __link_section() {}
-          
\ No newline at end of file
+          
diff --git a/crates/wasmtime/src/runtime/component/types.rs b/crates/wasmtime/src/runtime/component/types.rs
index 9536327ad..a48f1d30e 100644
--- a/crates/wasmtime/src/runtime/component/types.rs
+++ b/crates/wasmtime/src/runtime/component/types.rs
@@ -425,7 +425,7 @@ impl ResultType {
         ResultType(Handle::new(index, ty))
     }
 
-    /// Retrieve the `ok` type parameter for this `option`.
+    /// Retrieve the `ok` type parameter for this `result`.
     pub fn ok(&self) -> Option<Type> {
         Some(Type::from(
             self.0.types[self.0.index].ok.as_ref()?,
@@ -433,7 +433,7 @@ impl ResultType {
         ))
     }
 
-    /// Retrieve the `err` type parameter for this `option`.
+    /// Retrieve the `err` type parameter for this `result`.
     pub fn err(&self) -> Option<Type> {
         Some(Type::from(
             self.0.types[self.0.index].err.as_ref()?,
diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs
index 64674ce6a..da9cfd952 100644
--- a/crates/wasmtime/src/runtime/externals/table.rs
+++ b/crates/wasmtime/src/runtime/externals/table.rs
@@ -167,14 +167,7 @@ impl Table {
                 }
 
                 runtime::TableElement::GcRef(None) => {
-                    match self._ty(&store).element().heap_type().top() {
-                        HeapType::Any => Some(Ref::Any(None)),
-                        HeapType::Extern => Some(Ref::Extern(None)),
-                        HeapType::Func => {
-                            unreachable!("never have TableElement::GcRef for func tables")
-                        }
-                        ty => unreachable!("not a top type: {ty:?}"),
-                    }
+                    Some(Ref::null(self._ty(&store).element().heap_type()))
                 }
 
                 #[cfg_attr(not(feature = "gc"), allow(unreachable_code, unused_variables))]
diff --git a/crates/wasmtime/src/runtime/linker.rs b/crates/wasmtime/src/runtime/linker.rs
index 08389207c..d9546071b 100644
--- a/crates/wasmtime/src/runtime/linker.rs
+++ b/crates/wasmtime/src/runtime/linker.rs
@@ -316,8 +316,6 @@ impl<T> Linker<T> {
         &mut self,
         module: &Module,
     ) -> anyhow::Result<()> {
-        use crate::HeapType;
-
         for import in module.imports() {
             if let Err(import_err) = self._get_by_import(&import) {
                 if let ExternType::Func(func_ty) = import_err.ty() {
@@ -343,12 +341,7 @@ impl<T> Linker<T> {
                                     ValType::V128 => Val::V128(0_u128.into()),
                                     ValType::Ref(r) => {
                                         debug_assert!(r.is_nullable());
-                                        match r.heap_type().top() {
-                                            HeapType::Func => Val::null_func_ref(),
-                                            HeapType::Extern => Val::null_extern_ref(),
-                                            HeapType::Any => Val::null_any_ref(),
-                                            ty => unreachable!("not a top type: {ty:?}"),
-                                        }
+                                        Val::null_ref(r.heap_type())
                                     }
                                 };
                             }
diff --git a/crates/wasmtime/src/runtime/types.rs b/crates/wasmtime/src/runtime/types.rs
index ec6bd9110..4d501b50c 100644
--- a/crates/wasmtime/src/runtime/types.rs
+++ b/crates/wasmtime/src/runtime/types.rs
@@ -619,6 +619,7 @@ impl HeapType {
     ///
     /// The returned heap type is a supertype of all types in this heap type's
     /// type hierarchy.
+    #[inline]
     pub fn top(&self) -> HeapType {
         match self {
             HeapType::Func | HeapType::ConcreteFunc(_) | HeapType::NoFunc => HeapType::Func,
diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs
index fc4a6d262..b06c697d1 100644
--- a/crates/wasmtime/src/runtime/values.rs
+++ b/crates/wasmtime/src/runtime/values.rs
@@ -78,13 +78,9 @@ macro_rules! accessors {
 
 impl Val {
     /// Returns the null reference for the given heap type.
-    pub fn null_ref(heap_type: HeapType) -> Val {
-        match heap_type.top() {
-            HeapType::Func => Val::FuncRef(None),
-            HeapType::Extern => Val::ExternRef(None),
-            HeapType::Any => Val::AnyRef(None),
-            _ => unreachable!(),
-        }
+    #[inline]
+    pub fn null_ref(heap_type: &HeapType) -> Val {
+        Ref::null(&heap_type).into()
     }
 
     /// Returns the null function reference value.
@@ -631,6 +627,17 @@ impl From<Option<Rooted<AnyRef>>> for Ref {
 }
 
 impl Ref {
+    /// Create a null reference to the given heap type.
+    #[inline]
+    pub fn null(heap_type: &HeapType) -> Self {
+        match heap_type.top() {
+            HeapType::Any => Ref::Any(None),
+            HeapType::Extern => Ref::Extern(None),
+            HeapType::Func => Ref::Func(None),
+            ty => unreachable!("not a heap type: {ty:?}"),
+        }
+    }
+
     /// Is this a null reference?
     #[inline]
     pub fn is_null(&self) -> bool {


Test patch:
diff --git a/cranelift/filetests/filetests/isa/riscv64/reftypes.clif b/cranelift/filetests/filetests/isa/riscv64/reftypes.clif
index c7da9a8f4..5229c6f29 100644
--- a/cranelift/filetests/filetests/isa/riscv64/reftypes.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/reftypes.clif
@@ -97,7 +97,7 @@ block3(v7: r64, v8: r64):
 ;   mv a3,a0
 ;   mv s5,a2
 ;   sd a1,16(nominal_sp)
-;   mv a3,a0
+;   mv a0,a3
 ;   mv s9,a3
 ;   load_sym a5,%f+0
 ;   sd s9,8(nominal_sp)