Repository: bytecodealliance/wasmtime
Pull request #8408 (2024-04-19 01:36:39)

x64: Swap operand order of `cmp` in internal representation
This PR is #8362 but for general purpose registers instead of xmm registers. Upon reading over the instructions in `inst.isle` it appears that `x64_cmp` and its variants is the only remaining (after #8362) instruction where the operands are not in left-to-right form. The purpose of this PR is to refactor this situation to make sure all instructions have left-to-right argument orderings in their instruction helpers.

This PR is broken down into a number of small commits. Each commit swaps just one method and then handles all callers of that method. By the end all callers of all `cmp` instructions should have their operand order swapped. Note that special care was taken in Winch where there were a few swaps-of-swaps. For example the `branch` helper previously had a variable called `lhs` which actually was the right-hand-side of a comparison. This caused no issues for code today as all operations were reflexive or with the same value, but I've documented and ensured that lhs is now indeed the left-hand-side.

My intention as well is that `./ci/run-tests.sh` passes at each individual commit here as well. CI won't test that, but if anyone's interested to check it should work.

github-actions[bot]: #### Subscribe to Label Action

cc @saulecabrera

<details>
This issue or pull request has been labeled: "cranelift", "cranelift:area:x64", "winch"

Thus the following users have been cc'd because of the following labels:

* saulecabrera: winch

To subscribe or unsubscribe from this label, edit the <code>.github/subscribe-to-label.json</code> configuration file.

[Learn more.](https://github.com/bytecodealliance/subscribe-to-label-action)
</details>

Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 48417841f..37086e8af 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -249,7 +249,7 @@ jobs:
         submodules: true
     - uses: ./.github/actions/install-rust
       with:
-        toolchain: nightly-2024-03-21
+        toolchain: nightly-2024-04-19
 
     # Build C API documentation
     - run: curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.3/doxygen-1.9.3.linux.bin.tar.gz/download | tar xzf -
@@ -452,9 +452,9 @@ jobs:
     # happen upstream. This is periodically updated through a PR.
     - uses: ./.github/actions/install-rust
       with:
-        toolchain: nightly-2024-03-21
+        toolchain: nightly-2024-04-19
 
-    # Ensure that fuzzers sitll build.
+    # Ensure that fuzzers still build.
     #
     # Install the OCaml packages necessary for fuzz targets that use the
     # `wasm-spec-interpreter`.
diff --git a/Cargo.lock b/Cargo.lock
index ea48ab4d5..de9ad7eab 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -849,7 +849,7 @@ dependencies = [
  "serde_derive",
  "smallvec",
  "target-lexicon",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmtime-types",
  "wat",
 ]
@@ -1350,9 +1350,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.2"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943"
+checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069"
 dependencies = [
  "bytes",
  "fnv",
@@ -1863,9 +1863,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.8"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
 dependencies = [
  "libc",
  "wasi",
@@ -2368,9 +2368,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.22.2"
+version = "0.22.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41"
+checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432"
 dependencies = [
  "log",
  "ring",
@@ -2756,7 +2756,7 @@ dependencies = [
  "cargo_metadata",
  "heck",
  "wasmtime",
- "wit-component 0.204.0",
+ "wit-component 0.205.0",
 ]
 
 [[package]]
@@ -3092,7 +3092,7 @@ name = "verify-component-adapter"
 version = "21.0.0"
 dependencies = [
  "anyhow",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wat",
 ]
 
@@ -3184,7 +3184,7 @@ dependencies = [
  "byte-array-literals",
  "object 0.33.0",
  "wasi",
- "wasm-encoder 0.204.0",
+ "wasm-encoder 0.205.0",
  "wit-bindgen",
 ]
 
@@ -3253,9 +3253,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-encoder"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "500cbde9b4d8dfc0335ec729d226dbf083e51e47501ac71e6addaed10ccb0a51"
+checksum = "90e95b3563d164f33c1cfb0a7efbd5940c37710019be10cd09f800fdec8b0e5c"
 dependencies = [
  "leb128",
 ]
@@ -3278,9 +3278,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-metadata"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f33b61754be27af5abf0fda890badb7c3265f7b634077157b4214704303ebbcc"
+checksum = "e9855ad6dd4d099fa8505f01fb0b84c7e3efd5f555207329e3ad938d5f394fe7"
 dependencies = [
  "anyhow",
  "indexmap 2.0.0",
@@ -3288,36 +3288,36 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "spdx",
- "wasm-encoder 0.204.0",
- "wasmparser 0.204.0",
+ "wasm-encoder 0.205.0",
+ "wasmparser 0.205.0",
 ]
 
 [[package]]
 name = "wasm-mutate"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34d24ec979ae72a27f3770a7db426e7f2d459e67ecf26983ca727675e4956037"
+checksum = "d92842509e0cc163a0f8f4a54e38ec180e18594b9ed3999d0a05108e70d25cb3"
 dependencies = [
  "egg",
  "log",
  "rand",
  "thiserror",
- "wasm-encoder 0.204.0",
- "wasmparser 0.204.0",
+ "wasm-encoder 0.205.0",
+ "wasmparser 0.205.0",
 ]
 
 [[package]]
 name = "wasm-smith"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1fe1d435b8a14ebf05c2669cf76d1ca1cef6b632320c8e0b0e703ff0259a328"
+checksum = "1218f393276f7a13f6c54467434633144b46c33ecbc3cbafdc938033f0315d1a"
 dependencies = [
  "anyhow",
  "arbitrary",
  "flagset",
  "indexmap 2.0.0",
  "leb128",
- "wasm-encoder 0.204.0",
+ "wasm-encoder 0.205.0",
 ]
 
 [[package]]
@@ -3373,9 +3373,9 @@ dependencies = [
 
 [[package]]
 name = "wasmparser"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1239d539d9b92489cf729d40df694a62b09f5159f2ab058f4de15d4f820d6585"
+checksum = "1d457bb52804242e09d55a306e53ddbc65d1d29ed83db6a4eea3ed412ee0cfdf"
 dependencies = [
  "bitflags 2.4.1",
  "indexmap 2.0.0",
@@ -3393,12 +3393,12 @@ dependencies = [
 
 [[package]]
 name = "wasmprinter"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce3e6a3111855bf59cb6a16a95e217c59eff1d30d9f09483039b81b0739c3128"
+checksum = "edcd1e00bc348d4b1c0abe913d86f4a89377cbd5c1c704d4eca5cf3492a27da8"
 dependencies = [
  "anyhow",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
 ]
 
 [[package]]
@@ -3430,8 +3430,8 @@ dependencies = [
  "target-lexicon",
  "tempfile",
  "wasi-common",
- "wasm-encoder 0.204.0",
- "wasmparser 0.204.0",
+ "wasm-encoder 0.205.0",
+ "wasmparser 0.205.0",
  "wasmtime-cache",
  "wasmtime-component-macro",
  "wasmtime-component-util",
@@ -3571,7 +3571,7 @@ dependencies = [
  "tracing",
  "walkdir",
  "wasi-common",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmtime",
  "wasmtime-cache",
  "wasmtime-cli-flags",
@@ -3585,10 +3585,10 @@ dependencies = [
  "wasmtime-wasi-nn",
  "wasmtime-wasi-threads",
  "wasmtime-wast",
- "wast 204.0.0",
+ "wast 205.0.0",
  "wat",
  "windows-sys 0.52.0",
- "wit-component 0.204.0",
+ "wit-component 0.205.0",
 ]
 
 [[package]]
@@ -3617,7 +3617,7 @@ dependencies = [
  "wasmtime",
  "wasmtime-component-util",
  "wasmtime-wit-bindgen",
- "wit-parser 0.204.0",
+ "wit-parser 0.205.0",
 ]
 
 [[package]]
@@ -3641,7 +3641,7 @@ dependencies = [
  "object 0.33.0",
  "target-lexicon",
  "thiserror",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmtime-environ",
  "wasmtime-versioned-export-macros",
 ]
@@ -3665,8 +3665,8 @@ dependencies = [
  "serde_derive",
  "target-lexicon",
  "thiserror",
- "wasm-encoder 0.204.0",
- "wasmparser 0.204.0",
+ "wasm-encoder 0.205.0",
+ "wasmparser 0.205.0",
  "wasmprinter",
  "wasmtime-component-util",
  "wasmtime-types",
@@ -3681,7 +3681,7 @@ dependencies = [
  "component-fuzz-util",
  "env_logger",
  "libfuzzer-sys",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmprinter",
  "wasmtime-environ",
  "wat",
@@ -3738,7 +3738,7 @@ dependencies = [
  "rand",
  "smallvec",
  "target-lexicon",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmtime",
  "wasmtime-fuzzing",
 ]
@@ -3758,12 +3758,12 @@ dependencies = [
  "target-lexicon",
  "tempfile",
  "v8",
- "wasm-encoder 0.204.0",
+ "wasm-encoder 0.205.0",
  "wasm-mutate",
  "wasm-smith",
  "wasm-spec-interpreter",
  "wasmi",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmprinter",
  "wasmtime",
  "wasmtime-wast",
@@ -3810,7 +3810,7 @@ dependencies = [
  "rand",
  "rustix",
  "sptr",
- "wasm-encoder 0.204.0",
+ "wasm-encoder 0.205.0",
  "wasmtime-asm-macros",
  "wasmtime-environ",
  "wasmtime-fiber",
@@ -3833,7 +3833,7 @@ dependencies = [
  "serde",
  "serde_derive",
  "thiserror",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
 ]
 
 [[package]]
@@ -3940,7 +3940,7 @@ dependencies = [
  "anyhow",
  "log",
  "wasmtime",
- "wast 204.0.0",
+ "wast 205.0.0",
 ]
 
 [[package]]
@@ -3952,7 +3952,7 @@ dependencies = [
  "gimli",
  "object 0.33.0",
  "target-lexicon",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmtime-cranelift",
  "wasmtime-environ",
  "winch-codegen",
@@ -3965,7 +3965,7 @@ dependencies = [
  "anyhow",
  "heck",
  "indexmap 2.0.0",
- "wit-parser 0.204.0",
+ "wit-parser 0.205.0",
 ]
 
 [[package]]
@@ -3983,24 +3983,24 @@ dependencies = [
 
 [[package]]
 name = "wast"
-version = "204.0.0"
+version = "205.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0e3de19692b3d4c2fa13775271a751935decf530ae59c408c9f0b510b4ead62"
+checksum = "441a6a195b3b5245e26d450bbcc91366c6b652382a22f63cbe3c73240e13b2bb"
 dependencies = [
  "bumpalo",
  "leb128",
  "memchr",
  "unicode-width",
- "wasm-encoder 0.204.0",
+ "wasm-encoder 0.205.0",
 ]
 
 [[package]]
 name = "wat"
-version = "1.204.0"
+version = "1.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4280322d523214024d03bc05e25bdda6088d5229d9515aecd78c5914b1f3e734"
+checksum = "19832624d606e7c6bf3cd4caa73578ecec5eac30c768269256d19c79900beb18"
 dependencies = [
- "wast 204.0.0",
+ "wast 205.0.0",
 ]
 
 [[package]]
@@ -4130,7 +4130,7 @@ dependencies = [
  "regalloc2",
  "smallvec",
  "target-lexicon",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
  "wasmtime-cranelift",
  "wasmtime-environ",
 ]
@@ -4381,9 +4381,9 @@ dependencies = [
 
 [[package]]
 name = "wit-component"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "657fab6fd5f9ad3e1b7fd65a76f06ff25cac7f4fd8f885715bed4147095a6c38"
+checksum = "74841e17b2c2bfbd8ca8c45190ef85dff1c5e8659f0e2d164802ffc87dfb18c3"
 dependencies = [
  "anyhow",
  "bitflags 2.4.1",
@@ -4392,10 +4392,10 @@ dependencies = [
  "serde",
  "serde_derive",
  "serde_json",
- "wasm-encoder 0.204.0",
- "wasm-metadata 0.204.0",
- "wasmparser 0.204.0",
- "wit-parser 0.204.0",
+ "wasm-encoder 0.205.0",
+ "wasm-metadata 0.205.0",
+ "wasmparser 0.205.0",
+ "wit-parser 0.205.0",
 ]
 
 [[package]]
@@ -4418,9 +4418,9 @@ dependencies = [
 
 [[package]]
 name = "wit-parser"
-version = "0.204.0"
+version = "0.205.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdc20f9284017574e942347e953e82fb3a0dccdfafa3beea7840b0bcbfa4fe00"
+checksum = "a3db34c7688c161ed7bd1b2f8055dca9fb2c15201db58754e9c48a0805f32e5f"
 dependencies = [
  "anyhow",
  "id-arena",
@@ -4431,7 +4431,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "unicode-xid",
- "wasmparser 0.204.0",
+ "wasmparser 0.205.0",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index ab812cbc6..f137a2e26 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -234,15 +234,15 @@ rustix = "0.38.31"
 wit-bindgen = { version = "0.22.0", default-features = false }
 
 # wasm-tools family:
-wasmparser = "0.204.0"
-wat = "1.204.0"
-wast = "204.0.0"
-wasmprinter = "0.204.0"
-wasm-encoder = "0.204.0"
-wasm-smith = "0.204.0"
-wasm-mutate = "0.204.0"
-wit-parser = "0.204.0"
-wit-component = "0.204.0"
+wasmparser = "0.205.0"
+wat = "1.205.0"
+wast = "205.0.0"
+wasmprinter = "0.205.0"
+wasm-encoder = "0.205.0"
+wasm-smith = "0.205.0"
+wasm-mutate = "0.205.0"
+wit-parser = "0.205.0"
+wit-component = "0.205.0"
 
 # Non-Bytecode Alliance maintained dependencies:
 # --------------------------
diff --git a/RELEASES.md b/RELEASES.md
index 84af9edc6..5b232971f 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -139,6 +139,22 @@ Unreleased.
 
 --------------------------------------------------------------------------------
 
+## 19.0.2
+
+Released 2024-04-11.
+
+* Fix a panic when compiling invalid components.
+  [#8323](https://github.com/bytecodealliance/wasmtime/issues/8323)
+
+* Fix `bindgen!` with `trappable_error_type` using unversioned/versioned
+  packages.
+  [#8305](https://github.com/bytecodealliance/wasmtime/pull/8305)
+
+* cranelift: Include clobbers and outgoing args in stack limit.
+  [#8301](https://github.com/bytecodealliance/wasmtime/pull/8301)
+
+--------------------------------------------------------------------------------
+
 ## 19.0.1
 
 Released 2024-04-02.
@@ -248,6 +264,24 @@ Released 2024-03-20
 
 --------------------------------------------------------------------------------
 
+## 18.0.4
+
+Released 2024-04-11
+
+### Fixed
+
+* Fix `bindgen!` with `trappable_error_type` using unversioned/versioned
+  packages.
+  [#8305](https://github.com/bytecodealliance/wasmtime/pull/8305)
+
+* cranelift: Include clobbers and outgoing args in stack limit.
+  [#8301](https://github.com/bytecodealliance/wasmtime/pull/8301)
+
+* Fix a panic when compiling invalid components.
+  [#8323](https://github.com/bytecodealliance/wasmtime/issues/8323)
+
+--------------------------------------------------------------------------------
+
 ## 18.0.3
 
 Released 2024-03-12
@@ -333,6 +367,20 @@ Released 2024-02-20
 
 --------------------------------------------------------------------------------
 
+## 17.0.3
+
+Released 2024-04-11
+
+### Fixed
+
+* cranelift: Include clobbers and outgoing args in stack limit.
+  [#8301](https://github.com/bytecodealliance/wasmtime/pull/8301)
+
+* Fix a panic when compiling invalid components.
+  [#8323](https://github.com/bytecodealliance/wasmtime/issues/8323)
+
+--------------------------------------------------------------------------------
+
 ## 17.0.2
 
 Released 2024-02-28
diff --git a/ci/build-build-matrix.js b/ci/build-build-matrix.js
index 799f78177..b51be1801 100644
--- a/ci/build-build-matrix.js
+++ b/ci/build-build-matrix.js
@@ -61,7 +61,7 @@ for (let build of array) {
   // Next generate a "min" build and add it to the builds list. Min builds
   // require Nightly rust due to some nightly build options that are configured.
   build.build += '-min';
-  build.rust = 'nightly-2024-03-21';
+  build.rust = 'nightly-2024-04-19';
   builds.push(build);
 }
 
diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs
index eebe2c0e3..9b72e239b 100644
--- a/cranelift/codegen/src/ir/instructions.rs
+++ b/cranelift/codegen/src/ir/instructions.rs
@@ -6,6 +6,7 @@
 //! A large part of this module is auto-generated from the instruction descriptions in the meta
 //! directory.
 
+use crate::constant_hash::Table;
 use alloc::vec::Vec;
 use core::fmt::{self, Display, Formatter};
 use core::ops::{Deref, DerefMut};
@@ -211,17 +212,7 @@ impl FromStr for Opcode {
 
     /// Parse an Opcode name from a string.
     fn from_str(s: &str) -> Result<Self, &'static str> {
-        use crate::constant_hash::{probe, simple_hash, Table};
-
-        impl<'a> Table<&'a str> for [Option<Opcode>] {
-            fn len(&self) -> usize {
-                self.len()
-            }
-
-            fn key(&self, idx: usize) -> Option<&'a str> {
-                self[idx].map(opcode_name)
-            }
-        }
+        use crate::constant_hash::{probe, simple_hash};
 
         match probe::<&str, [Option<Self>]>(&OPCODE_HASH_TABLE, s, simple_hash(s)) {
             Err(_) => Err("Unknown opcode"),
@@ -232,6 +223,16 @@ impl FromStr for Opcode {
     }
 }
 
+impl<'a> Table<&'a str> for [Option<Opcode>] {
+    fn len(&self) -> usize {
+        self.len()
+    }
+
+    fn key(&self, idx: usize) -> Option<&'a str> {
+        self[idx].map(opcode_name)
+    }
+}
+
 /// A variable list of `Value` operands used for function call arguments and passing arguments to
 /// basic blocks.
 #[derive(Clone, Debug)]
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index ea2672328..c23d75509 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -469,7 +469,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
 
     fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallInstVec<Self::I> {
         smallvec![
-            Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::reg(regs::rsp()), limit_reg),
+            Inst::cmp_rmi_r(OperandSize::Size64, limit_reg, RegMemImm::reg(regs::rsp())),
             Inst::TrapIf {
                 // NBE == "> unsigned"; args above are reversed; this tests limit_reg > rsp.
                 cc: CC::NBE,
@@ -1046,7 +1046,7 @@ impl X64CallSite {
                     distance: RelocDistance::Far,
                 });
                 ctx.emit(Inst::ReturnCallUnknown {
-                    callee: tmp2.to_writable_reg().into(),
+                    callee: tmp2.to_reg().to_reg(),
                     info,
                 });
             }
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 84c498808..41d036152 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -218,8 +218,8 @@
        ;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
        (CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8
                 (opcode CmpOpcode)
-                (src GprMemImm)
-                (dst Gpr))
+                (src1 Gpr)
+                (src2 GprMemImm))
 
        ;; Materializes the requested condition code in the destinaton reg.
        (Setcc (cc CC)
@@ -367,6 +367,10 @@
                     (dst WritableGpr)
                     (dst_size OperandSize))
 
+       ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
+       (XmmCmpRmRVex (op AvxOpcode)
+                     (src1 Xmm)
+                     (src2 XmmMem))
 
        ;; XMM (scalar or vector) binary op that relies on the EVEX
        ;; prefix. Takes two inputs.
@@ -541,7 +545,7 @@
                         (info BoxReturnCallInfo))
 
        ;; Tail call to an indirect destination.
-       (ReturnCallUnknown (callee RegMem)
+       (ReturnCallUnknown (callee Reg)
                           (info BoxReturnCallInfo))
 
        ;; A pseudo-instruction that captures register arguments in vregs.
@@ -1413,6 +1417,9 @@
             Vsqrtsd
             Vroundss
             Vroundsd
+            Vucomiss
+            Vucomisd
+            Vptest
           ))
 
 (type Avx512Opcode
@@ -2793,7 +2800,7 @@
         dst))
 
 ;; Helper for creating `MInst.CmpRmiR` instructions.
-(decl cmp_rmi_r (OperandSize CmpOpcode GprMemImm Gpr) ProducesFlags)
+(decl cmp_rmi_r (OperandSize CmpOpcode Gpr GprMemImm) ProducesFlags)
 (rule (cmp_rmi_r size opcode src1 src2)
       (ProducesFlags.ProducesFlagsSideEffect
        (MInst.CmpRmiR size
@@ -2802,14 +2809,14 @@
                       src2)))
 
 ;; Helper for creating `cmp` instructions.
-(decl x64_cmp (OperandSize GprMemImm Gpr) ProducesFlags)
+(decl x64_cmp (OperandSize Gpr GprMemImm) ProducesFlags)
 (rule (x64_cmp size src1 src2)
       (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2))
 
 ;; Helper for creating `cmp` instructions with an immediate.
-(decl x64_cmp_imm (OperandSize u32 Gpr) ProducesFlags)
+(decl x64_cmp_imm (OperandSize Gpr u32) ProducesFlags)
 (rule (x64_cmp_imm size src1 src2)
-      (cmp_rmi_r size (CmpOpcode.Cmp) (RegMemImm.Imm src1) src2))
+      (x64_cmp size src1 (RegMemImm.Imm src2)))
 
 ;; Helper for creating `MInst.XmmCmpRmR` instructions.
 (decl xmm_cmp_rm_r (SseOpcode Xmm XmmMem) ProducesFlags)
@@ -2817,17 +2824,27 @@
       (ProducesFlags.ProducesFlagsSideEffect
        (MInst.XmmCmpRmR opcode src1 src2)))
 
+;; Helper for creating `MInst.XmmCmpRmRVex` instructions.
+(decl xmm_cmp_rm_r_vex (AvxOpcode Xmm XmmMem) ProducesFlags)
+(rule (xmm_cmp_rm_r_vex opcode src1 src2)
+      (ProducesFlags.ProducesFlagsSideEffect
+       (MInst.XmmCmpRmRVex opcode src1 src2)))
+
 ;; Helper for creating floating-point comparison instructions (`UCOMIS[S|D]`).
-(decl x64_ucomis (Value Value) ProducesFlags)
-(rule (x64_ucomis src1 @ (value_type $F32) src2)
-      ;; N.B.: cmp can be generated more than once, so cannot do a
-      ;; load-op merge. So `put_in_xmm` for src1, not `put_in_xmm_mem`.
-      (xmm_cmp_rm_r (SseOpcode.Ucomiss) (put_in_xmm src1) (put_in_xmm src2)))
-(rule (x64_ucomis src1 @ (value_type $F64) src2)
-      (xmm_cmp_rm_r (SseOpcode.Ucomisd) (put_in_xmm src1) (put_in_xmm src2)))
+(decl x64_ucomis (Type Xmm XmmMem) ProducesFlags)
+(rule (x64_ucomis $F32 src1 src2)
+      (xmm_cmp_rm_r (SseOpcode.Ucomiss) src1 src2))
+(rule (x64_ucomis $F64 src1 src2)
+      (xmm_cmp_rm_r (SseOpcode.Ucomisd) src1 src2))
+(rule 1 (x64_ucomis $F32 src1 src2)
+        (if-let $true (use_avx))
+        (xmm_cmp_rm_r_vex (AvxOpcode.Vucomiss) src1 src2))
+(rule 1 (x64_ucomis $F64 src1 src2)
+        (if-let $true (use_avx))
+        (xmm_cmp_rm_r_vex (AvxOpcode.Vucomisd) src1 src2))
 
 ;; Helper for creating `test` instructions.
-(decl x64_test (OperandSize GprMemImm Gpr) ProducesFlags)
+(decl x64_test (OperandSize Gpr GprMemImm) ProducesFlags)
 (rule (x64_test size src1 src2)
       (cmp_rmi_r size (CmpOpcode.Test) src1 src2))
 
@@ -2835,6 +2852,9 @@
 (decl x64_ptest (Xmm XmmMem) ProducesFlags)
 (rule (x64_ptest src1 src2)
       (xmm_cmp_rm_r (SseOpcode.Ptest) src1 src2))
+(rule 1 (x64_ptest src1 src2)
+        (if-let $true (use_avx))
+        (xmm_cmp_rm_r_vex (AvxOpcode.Vptest) src1 src2))
 
 ;; Helper for creating `cmove` instructions. Note that these instructions do not
 ;; always result in a single emitted x86 instruction; e.g., XmmCmove uses jumps
@@ -4746,17 +4766,16 @@
 
 ;; For GPR-held values we only need to emit `CMP + SETCC`. We rely here on
 ;; Cranelift's verification that `a` and `b` are of the same type.
-;; Unfortunately for clarity, the registers are flipped here (TODO).
 (rule 0 (emit_cmp cc a @ (value_type ty) b)
       (let ((size OperandSize (raw_operand_size_of_type ty)))
-        (icmp_cond_result (x64_cmp size b a) cc)))
+        (icmp_cond_result (x64_cmp size a b) cc)))
 
 ;; As a special case, swap the arguments to the comparison when the LHS is a
 ;; constant. This ensures that we avoid moving the constant into a register when
 ;; performing the comparison.
 (rule 1 (emit_cmp cc (and (simm32_from_value a) (value_type ty)) b)
       (let ((size OperandSize (raw_operand_size_of_type ty)))
-        (icmp_cond_result (x64_cmp size a b) (intcc_swap_args cc))))
+        (icmp_cond_result (x64_cmp size b a) (intcc_swap_args cc))))
 
 ;; Special case: use the test instruction for comparisons with 0.
 (rule 2 (emit_cmp cc a @ (value_type ty) (u64_from_iconst 0))
@@ -4776,8 +4795,8 @@
             (a_hi Gpr (value_regs_get_gpr a 1))
             (b_lo Gpr (value_regs_get_gpr b 0))
             (b_hi Gpr (value_regs_get_gpr b 1))
-            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_lo a_lo) (x64_setcc (CC.Z))))
-            (cmp_hi Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_hi a_hi) (x64_setcc (CC.Z))))
+            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) a_lo b_lo) (x64_setcc (CC.Z))))
+            (cmp_hi Reg (with_flags_reg (x64_cmp (OperandSize.Size64) a_hi b_hi) (x64_setcc (CC.Z))))
             ;; At this point, `cmp_lo` and `cmp_hi` contain either 0 or 1 in the
             ;; lowest 8 bits--`SETcc` guarantees this. The upper bits may be
             ;; unchanged so we must compare against 1 below; this instruction
@@ -4789,7 +4808,7 @@
         ;; the halves `AND`s to 0, they were not equal, therefore we `SETcc`
         ;; with `NZ`.
         (icmp_cond_result
-          (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) cmp)
+          (x64_test (OperandSize.Size64) cmp (RegMemImm.Imm 1))
           (CC.NZ))))
 
 (rule 5 (emit_cmp (IntCC.NotEqual) a @ (value_type $I128) b)
@@ -4797,12 +4816,12 @@
             (a_hi Gpr (value_regs_get_gpr a 1))
             (b_lo Gpr (value_regs_get_gpr b 0))
             (b_hi Gpr (value_regs_get_gpr b 1))
-            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_lo a_lo) (x64_setcc (CC.NZ))))
-            (cmp_hi Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_hi a_hi) (x64_setcc (CC.NZ))))
+            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) a_lo b_lo) (x64_setcc (CC.NZ))))
+            (cmp_hi Reg (with_flags_reg (x64_cmp (OperandSize.Size64) a_hi b_hi) (x64_setcc (CC.NZ))))
             ;; See comments for `IntCC.Equal`.
             (cmp Reg (x64_or $I64 cmp_lo cmp_hi)))
            (icmp_cond_result
-             (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) cmp)
+             (x64_test (OperandSize.Size64) cmp (RegMemImm.Imm 1))
              (CC.NZ))))
 
 ;; Result = (a_hi <> b_hi) ||
@@ -4812,20 +4831,20 @@
             (a_hi Gpr (value_regs_get_gpr a 1))
             (b_lo Gpr (value_regs_get_gpr b 0))
             (b_hi Gpr (value_regs_get_gpr b 1))
-            (cmp_hi ValueRegs (with_flags (x64_cmp (OperandSize.Size64) b_hi a_hi)
+            (cmp_hi ValueRegs (with_flags (x64_cmp (OperandSize.Size64) a_hi b_hi)
                                        (consumes_flags_concat
                                                  (x64_setcc (intcc_without_eq cc))
                                                  (x64_setcc (CC.Z)))))
             (cc_hi Reg (value_regs_get cmp_hi 0))
             (eq_hi Reg (value_regs_get cmp_hi 1))
 
-            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) b_lo a_lo)
+            (cmp_lo Reg (with_flags_reg (x64_cmp (OperandSize.Size64) a_lo b_lo)
                                         (x64_setcc (intcc_unsigned cc))))
 
             (res_lo Reg (x64_and $I64 eq_hi cmp_lo))
             (res Reg (x64_or $I64 cc_hi res_lo)))
         (icmp_cond_result
-          (x64_test (OperandSize.Size64) (RegMemImm.Imm 1) res)
+          (x64_test (OperandSize.Size64) res (RegMemImm.Imm 1))
           (CC.NZ))))
 
 (type FcmpCondResult
@@ -4881,46 +4900,46 @@
 ;;  - equal assigns        Z = 1, P = 0, C = 0
 (decl emit_fcmp (FloatCC Value Value) FcmpCondResult)
 
-(rule (emit_fcmp (FloatCC.Equal) a @ (value_type (ty_scalar_float _)) b)
-      (FcmpCondResult.AndCondition (x64_ucomis a b) (CC.NP) (CC.Z)))
+(rule (emit_fcmp (FloatCC.Equal) a @ (value_type (ty_scalar_float ty)) b)
+      (FcmpCondResult.AndCondition (x64_ucomis ty a b) (CC.NP) (CC.Z)))
 
-(rule (emit_fcmp (FloatCC.NotEqual) a @ (value_type (ty_scalar_float _)) b)
-      (FcmpCondResult.OrCondition (x64_ucomis a b) (CC.P) (CC.NZ)))
+(rule (emit_fcmp (FloatCC.NotEqual) a @ (value_type (ty_scalar_float ty)) b)
+      (FcmpCondResult.OrCondition (x64_ucomis ty a b) (CC.P) (CC.NZ)))
 
 ;; Some scalar lowerings correspond to one condition code.
 
 (rule (emit_fcmp (FloatCC.Ordered) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.NP)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.NP)))
 (rule (emit_fcmp (FloatCC.Unordered) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.P)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.P)))
 (rule (emit_fcmp (FloatCC.OrderedNotEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.NZ)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.NZ)))
 (rule (emit_fcmp (FloatCC.UnorderedOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.Z)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.Z)))
 (rule (emit_fcmp (FloatCC.GreaterThan) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.NBE)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.NBE)))
 (rule (emit_fcmp (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.NB)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.NB)))
 (rule (emit_fcmp (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.B)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.B)))
 (rule (emit_fcmp (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
-      (FcmpCondResult.Condition (x64_ucomis a b) (CC.BE)))
+      (FcmpCondResult.Condition (x64_ucomis ty a b) (CC.BE)))
 
 ;; Other scalar lowerings are made possible by flipping the operands and
 ;; reversing the condition code.
 
 (rule (emit_fcmp (FloatCC.LessThan) a @ (value_type (ty_scalar_float ty)) b)
       ;; Same flags as `GreaterThan`.
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.NBE)))
+      (FcmpCondResult.Condition (x64_ucomis ty b a) (CC.NBE)))
 (rule (emit_fcmp (FloatCC.LessThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
       ;; Same flags as `GreaterThanOrEqual`.
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.NB)))
+      (FcmpCondResult.Condition (x64_ucomis ty b a) (CC.NB)))
 (rule (emit_fcmp (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_scalar_float ty)) b)
       ;; Same flags as `UnorderedOrLessThan`.
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.B)))
+      (FcmpCondResult.Condition (x64_ucomis ty b a) (CC.B)))
 (rule (emit_fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_scalar_float ty)) b)
       ;; Same flags as `UnorderedOrLessThanOrEqual`.
-      (FcmpCondResult.Condition (x64_ucomis b a) (CC.BE)))
+      (FcmpCondResult.Condition (x64_ucomis ty b a) (CC.BE)))
 
 ;;;; Type Guards ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 992e322b4..caf21dbbb 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -1814,7 +1814,10 @@ impl AvxOpcode {
             | AvxOpcode::Vsqrtsd
             | AvxOpcode::Vroundss
             | AvxOpcode::Vroundsd
-            | AvxOpcode::Vunpcklpd => {
+            | AvxOpcode::Vunpcklpd
+            | AvxOpcode::Vptest
+            | AvxOpcode::Vucomiss
+            | AvxOpcode::Vucomisd => {
                 smallvec![InstructionSet::AVX]
             }
 
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 481362b9e..f1bd39aee 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -765,7 +765,7 @@ pub(crate) fn emit(
 
             // Check if the divisor is -1, and if it isn't then immediately
             // go to the `idiv`.
-            let inst = Inst::cmp_rmi_r(size, RegMemImm::imm(0xffffffff), divisor);
+            let inst = Inst::cmp_rmi_r(size, divisor, RegMemImm::imm(0xffffffff));
             inst.emit(&[], sink, info, state);
             one_way_jmp(sink, CC::NZ, do_op);
 
@@ -1278,8 +1278,8 @@ pub(crate) fn emit(
 
         Inst::CmpRmiR {
             size,
-            src: src_e,
-            dst: reg_g,
+            src1: reg_g,
+            src2: src_e,
             opcode,
         } => {
             let reg_g = allocs.next(reg_g.to_reg());
@@ -1580,8 +1580,8 @@ pub(crate) fn emit(
             // cmp  rsp, tmp_reg
             let inst = Inst::cmp_rmi_r(
                 OperandSize::Size64,
-                RegMemImm::reg(regs::rsp()),
                 tmp.to_reg(),
+                RegMemImm::reg(regs::rsp()),
             );
             inst.emit(&[], sink, info, state);
 
@@ -1647,11 +1647,14 @@ pub(crate) fn emit(
             callee,
             info: call_info,
         } => {
-            let callee = callee.with_allocs(allocs);
+            let callee = allocs.next(*callee);
 
             emit_return_call_common_sequence(allocs, sink, info, state, &call_info);
 
-            Inst::JmpUnknown { target: callee }.emit(&[], sink, info, state);
+            Inst::JmpUnknown {
+                target: RegMem::reg(callee),
+            }
+            .emit(&[], sink, info, state);
             sink.add_call_site(ir::Opcode::ReturnCallIndirect);
         }
 
@@ -2855,6 +2858,32 @@ pub(crate) fn emit(
                 .encode(sink);
         }
 
+        Inst::XmmCmpRmRVex { op, src1, src2 } => {
+            let src1 = allocs.next(src1.to_reg());
+            let src2 = match src2.clone().to_reg_mem().with_allocs(allocs) {
+                RegMem::Reg { reg } => {
+                    RegisterOrAmode::Register(reg.to_real_reg().unwrap().hw_enc().into())
+                }
+                RegMem::Mem { addr } => RegisterOrAmode::Amode(addr.finalize(state, sink)),
+            };
+
+            let (prefix, map, opcode) = match op {
+                AvxOpcode::Vucomiss => (LegacyPrefixes::None, OpcodeMap::_0F, 0x2E),
+                AvxOpcode::Vucomisd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x2E),
+                AvxOpcode::Vptest => (LegacyPrefixes::_66, OpcodeMap::_0F38, 0x17),
+                _ => unimplemented!("Opcode {:?} not implemented", op),
+            };
+
+            VexInstruction::new()
+                .length(VexVectorLength::V128)
+                .prefix(prefix)
+                .map(map)
+                .opcode(opcode)
+                .rm(src2)
+                .reg(src1.to_real_reg().unwrap().hw_enc())
+                .encode(sink);
+        }
+
         Inst::XmmRmREvex {
             op,
             src1,
@@ -3306,7 +3335,7 @@ pub(crate) fn emit(
             // If x seen as a signed int64 is not negative, a signed-conversion will do the right
             // thing.
             // TODO use tst src, src here.
-            let inst = Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(0), src);
+            let inst = Inst::cmp_rmi_r(OperandSize::Size64, src, RegMemImm::imm(0));
             inst.emit(&[], sink, info, state);
 
             one_way_jmp(sink, CC::L, handle_negative);
@@ -3454,7 +3483,7 @@ pub(crate) fn emit(
             inst.emit(&[], sink, info, state);
 
             // Compare against 1, in case of overflow the dst operand was INT_MIN.
-            let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(1), dst);
+            let inst = Inst::cmp_rmi_r(*dst_size, dst, RegMemImm::imm(1));
             inst.emit(&[], sink, info, state);
 
             one_way_jmp(sink, CC::NO, done); // no overflow => done
@@ -3701,7 +3730,7 @@ pub(crate) fn emit(
             let inst = Inst::xmm_to_gpr(trunc_op, src, Writable::from_reg(dst), *dst_size);
             inst.emit(&[], sink, info, state);
 
-            let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst);
+            let inst = Inst::cmp_rmi_r(*dst_size, dst, RegMemImm::imm(0));
             inst.emit(&[], sink, info, state);
 
             one_way_jmp(sink, CC::NL, done); // if dst >= 0, jump to done
@@ -3738,7 +3767,7 @@ pub(crate) fn emit(
             let inst = Inst::xmm_to_gpr(trunc_op, tmp_xmm2, Writable::from_reg(dst), *dst_size);
             inst.emit(&[], sink, info, state);
 
-            let inst = Inst::cmp_rmi_r(*dst_size, RegMemImm::imm(0), dst);
+            let inst = Inst::cmp_rmi_r(*dst_size, dst, RegMemImm::imm(0));
             inst.emit(&[], sink, info, state);
 
             if *is_saturating {
@@ -3940,8 +3969,8 @@ pub(crate) fn emit(
                     // cmp %r_temp, %r_operand
                     let i3 = Inst::cmp_rmi_r(
                         OperandSize::from_ty(*ty),
-                        RegMemImm::reg(temp.to_reg()),
                         operand,
+                        RegMemImm::reg(temp.to_reg()),
                     );
                     i3.emit(&[], sink, info, state);
 
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 55407ee86..51eb8859a 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -191,7 +191,8 @@ impl Inst {
             | Inst::XmmToGprImmVex { op, .. }
             | Inst::XmmToGprVex { op, .. }
             | Inst::GprToXmmVex { op, .. }
-            | Inst::CvtIntToFloatVex { op, .. } => op.available_from(),
+            | Inst::CvtIntToFloatVex { op, .. }
+            | Inst::XmmCmpRmRVex { op, .. } => op.available_from(),
         }
     }
 }
@@ -489,13 +490,13 @@ impl Inst {
 
     /// Does a comparison of dst - src for operands of size `size`, as stated by the machine
     /// instruction semantics. Be careful with the order of parameters!
-    pub(crate) fn cmp_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst {
-        src.assert_regclass_is(RegClass::Int);
-        debug_assert_eq!(dst.class(), RegClass::Int);
+    pub(crate) fn cmp_rmi_r(size: OperandSize, src1: Reg, src2: RegMemImm) -> Inst {
+        src2.assert_regclass_is(RegClass::Int);
+        debug_assert_eq!(src1.class(), RegClass::Int);
         Inst::CmpRmiR {
             size,
-            src: GprMemImm::new(src).unwrap(),
-            dst: Gpr::new(dst).unwrap(),
+            src1: Gpr::new(src1).unwrap(),
+            src2: GprMemImm::new(src2).unwrap(),
             opcode: CmpOpcode::Cmp,
         }
     }
@@ -1351,6 +1352,12 @@ impl PrettyPrint for Inst {
                 format!("{op} {src1}, {src2}, {dst}")
             }
 
+            Inst::XmmCmpRmRVex { op, src1, src2 } => {
+                let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
+                let src2 = src2.pretty_print(8, allocs);
+                format!("{} {src2}, {src1}", ljustify(op.to_string()))
+            }
+
             Inst::CvtUint64ToFloatSeq {
                 src,
                 dst,
@@ -1565,18 +1572,18 @@ impl PrettyPrint for Inst {
 
             Inst::CmpRmiR {
                 size,
-                src,
-                dst,
+                src1,
+                src2,
                 opcode,
             } => {
-                let dst = pretty_print_reg(dst.to_reg(), size.to_bytes(), allocs);
-                let src = src.pretty_print(size.to_bytes(), allocs);
+                let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes(), allocs);
+                let src2 = src2.pretty_print(size.to_bytes(), allocs);
                 let op = match opcode {
                     CmpOpcode::Cmp => "cmp",
                     CmpOpcode::Test => "test",
                 };
                 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
-                format!("{op} {src}, {dst}")
+                format!("{op} {src2}, {src1}")
             }
 
             Inst::Setcc { cc, dst } => {
@@ -1690,7 +1697,7 @@ impl PrettyPrint for Inst {
                     new_stack_arg_size,
                     tmp,
                 } = &**info;
-                let callee = callee.pretty_print(8, allocs);
+                let callee = pretty_print_reg(*callee, 8, allocs);
                 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8, allocs);
                 let mut s =
                     format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}");
@@ -2159,6 +2166,10 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_use(src1.to_reg());
             src2.get_operands(collector);
         }
+        Inst::XmmCmpRmRVex { src1, src2, .. } => {
+            collector.reg_use(src1.to_reg());
+            src2.get_operands(collector);
+        }
         Inst::Imm { dst, .. } => {
             collector.reg_def(dst.to_writable_reg());
         }
@@ -2273,10 +2284,9 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
                 collector.reg_fixed_use(reg, regs::rcx());
             }
         }
-        Inst::CmpRmiR { src, dst, .. } => {
-            // N.B.: use, not def (cmp doesn't write its result).
-            collector.reg_use(dst.to_reg());
-            src.get_operands(collector);
+        Inst::CmpRmiR { src1, src2, .. } => {
+            collector.reg_use(src1.to_reg());
+            src2.get_operands(collector);
         }
         Inst::Setcc { dst, .. } => {
             collector.reg_def(dst.to_writable_reg());
@@ -2336,8 +2346,9 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             match dest {
                 RegMem::Reg { reg } if info.callee_conv == CallConv::Winch => {
                     // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
-                    // This shouldn't be a fixed register constraint.
-                    collector.reg_fixed_use(*reg, regs::r15())
+                    // This shouldn't be a fixed register constraint. r10 is caller-saved, so this
+                    // should be safe to use.
+                    collector.reg_fixed_use(*reg, regs::r10())
                 }
                 _ => dest.get_operands(collector),
             }
@@ -2352,7 +2363,7 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
 
         Inst::ReturnCallKnown { callee, info } => {
             let ReturnCallInfo { uses, tmp, .. } = &**info;
-            collector.reg_early_def(tmp.to_writable_reg());
+            collector.reg_fixed_def(tmp.to_writable_reg(), regs::r11());
             // Same as in the `Inst::CallKnown` branch.
             debug_assert_ne!(*callee, ExternalName::LibCall(LibCall::Probestack));
             for u in uses {
@@ -2362,8 +2373,15 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
 
         Inst::ReturnCallUnknown { callee, info } => {
             let ReturnCallInfo { uses, tmp, .. } = &**info;
-            callee.get_operands(collector);
-            collector.reg_early_def(tmp.to_writable_reg());
+
+            // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
+            // This shouldn't be a fixed register constraint, but it's not clear how to
+            // pick a register that won't be clobbered by the callee-save restore code
+            // emitted with a return_call_indirect. r10 is caller-saved, so this should be
+            // safe to use.
+            collector.reg_fixed_use(*callee, regs::r10());
+
+            collector.reg_fixed_def(tmp.to_writable_reg(), regs::r11());
             for u in uses {
                 collector.reg_fixed_use(u.vreg, u.preg);
             }
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index e180d35bb..15efff9ff 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -518,8 +518,8 @@
             (zero Gpr (imm $I64 0))
             ;; Nullify the carry if we are shifting in by a multiple of 128.
             (carry_ Gpr (with_flags_reg (x64_test (OperandSize.Size64)
-                                              (RegMemImm.Imm 127)
-                                              amt)
+                                              amt
+                                              (RegMemImm.Imm 127))
                                         (cmove $I64
                                                (CC.Z)
                                                zero
@@ -529,7 +529,7 @@
         ;; Combine the two shifted halves. However, if we are shifting by >= 64
         ;; (modulo 128), then the low bits are zero and the high bits are our
         ;; low bits.
-        (with_flags (x64_test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+        (with_flags (x64_test (OperandSize.Size64) amt (RegMemImm.Imm 64))
                     (consumes_flags_concat
                      (cmove $I64 (CC.Z) lo_shifted zero)
                      (cmove $I64 (CC.Z) hi_shifted_ lo_shifted)))))
@@ -629,14 +629,14 @@
             (zero Gpr (imm $I64 0))
 
             ;; Nullify the carry if we are shifting by a multiple of 128.
-            (carry_ Gpr (with_flags_reg (x64_test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
+            (carry_ Gpr (with_flags_reg (x64_test (OperandSize.Size64) amt (RegMemImm.Imm 127))
                                         (cmove $I64 (CC.Z) zero carry)))
             ;; Add the carry bits into the lo.
             (lo_shifted_ Gpr (x64_or $I64 carry_ lo_shifted)))
         ;; Combine the two shifted halves. However, if we are shifting by >= 64
         ;; (modulo 128), then the hi bits are zero and the lo bits are what
         ;; would otherwise be our hi bits.
-        (with_flags (x64_test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+        (with_flags (x64_test (OperandSize.Size64) amt (RegMemImm.Imm 64))
                     (consumes_flags_concat
                      (cmove $I64 (CC.Z) lo_shifted_ hi_shifted)
                      (cmove $I64 (CC.Z) hi_shifted zero)))))
@@ -739,7 +739,7 @@
                                  (imm $I64 64)
                                  amt)))
             ;; Nullify the carry if we are shifting by a multiple of 128.
-            (carry_ Gpr (with_flags_reg (x64_test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
+            (carry_ Gpr (with_flags_reg (x64_test (OperandSize.Size64) amt (RegMemImm.Imm 127))
                                         (cmove $I64 (CC.Z) (imm $I64 0) carry)))
             ;; Add the carry into the low half.
             (lo_shifted_ Gpr (x64_or $I64 lo_shifted carry_))
@@ -748,7 +748,7 @@
         ;; Combine the two shifted halves. However, if we are shifting by >= 64
         ;; (modulo 128), then the hi bits are all sign bits and the lo bits are
         ;; what would otherwise be our hi bits.
-        (with_flags (x64_test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+        (with_flags (x64_test (OperandSize.Size64) amt (RegMemImm.Imm 64))
                     (consumes_flags_concat
                      (cmove $I64 (CC.Z) lo_shifted_ hi_shifted)
                      (cmove $I64 (CC.Z) hi_shifted sign_bits)))))
@@ -1659,7 +1659,7 @@
             ;; allow for load-op merging, here we cannot do that.
             (x_reg Reg x)
             (y_reg Reg y))
-        (with_flags_reg (x64_cmp size x_reg y_reg)
+        (with_flags_reg (x64_cmp size y_reg x_reg)
                         (cmove ty cc y_reg x_reg))))
 
 (rule -1 (lower (has_type (fits_in_64 ty) (umin x y)))
@@ -2154,7 +2154,7 @@
                             (RegMemImm.Imm 64)))
             (result_lo Gpr
               (with_flags_reg
-               (x64_cmp_imm (OperandSize.Size64) 64 upper)
+               (x64_cmp_imm (OperandSize.Size64) upper 64)
                (cmove $I64 (CC.NZ) upper lower))))
         (value_regs result_lo (imm $I64 0))))
 
@@ -2191,7 +2191,7 @@
                             (RegMemImm.Imm 64)))
             (result_lo Gpr
               (with_flags_reg
-               (x64_cmp_imm (OperandSize.Size64) 64 lower)
+               (x64_cmp_imm (OperandSize.Size64) lower 64)
                (cmove $I64 (CC.Z) upper lower))))
         (value_regs result_lo (imm $I64 0))))
 
@@ -2452,7 +2452,7 @@
 ;; Null references are represented by the constant value `0`.
 (rule (lower (is_null src @ (value_type $R64)))
       (with_flags
-       (x64_cmp_imm (OperandSize.Size64) 0 src)
+       (x64_cmp_imm (OperandSize.Size64) src 0)
        (x64_setcc (CC.Z))))
 
 ;; Rules for `is_invalid` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2460,7 +2460,7 @@
 ;; Invalid references are represented by the constant value `-1`.
 (rule (lower (is_invalid src @ (value_type $R64)))
       (with_flags
-       (x64_cmp_imm (OperandSize.Size64) 0xffffffff src)  ;; simm32 0xffff_ffff is sign-extended to -1.
+       (x64_cmp_imm (OperandSize.Size64) src 0xffffffff)  ;; simm32 0xffff_ffff is sign-extended to -1.
        (x64_setcc (CC.Z))))
 
 
@@ -3332,11 +3332,11 @@
 (rule (cmp_zero_i128 (cc_nz_or_z cc) val)
       (let ((lo Gpr (value_regs_get_gpr val 0))
             (hi Gpr (value_regs_get_gpr val 1))
-            (lo_z Gpr (with_flags_reg (x64_cmp (OperandSize.Size64) (RegMemImm.Imm 0) lo)
+            (lo_z Gpr (with_flags_reg (x64_cmp_imm (OperandSize.Size64) lo 0)
                                       (x64_setcc (CC.Z))))
-            (hi_z Gpr (with_flags_reg (x64_cmp (OperandSize.Size64) (RegMemImm.Imm 0) hi)
+            (hi_z Gpr (with_flags_reg (x64_cmp_imm (OperandSize.Size64) hi 0)
                                       (x64_setcc (CC.Z)))))
-          (icmp_cond_result (x64_test (OperandSize.Size8) lo_z hi_z) cc)))
+          (icmp_cond_result (x64_test (OperandSize.Size8) hi_z lo_z) cc)))
 
 
 (decl cmp_zero_int_bool_ref (Value) ProducesFlags)
@@ -3353,7 +3353,7 @@
             (size_reg Reg (imm ty (u32_as_u64 jt_size)))
             (idx_reg Gpr (extend_to_gpr idx $I64 (ExtendKind.Zero)))
             (clamped_idx Reg (with_flags_reg
-              (x64_cmp size size_reg idx_reg)
+              (x64_cmp size idx_reg size_reg)
               (cmove ty (CC.B) idx_reg size_reg))))
       (emit_side_effect (jmp_table_seq ty clamped_idx default_target jt_targets))))
 
@@ -4639,7 +4639,7 @@
           (any_byte_zero Xmm (x64_pcmpeqb val (xmm_zero $I8X16)))
           (mask Gpr (x64_pmovmskb (OperandSize.Size32) any_byte_zero))
         )
-        (with_flags (x64_cmp (OperandSize.Size32) (RegMemImm.Imm 0xffff) mask)
+        (with_flags (x64_cmp_imm (OperandSize.Size32) mask 0xffff)
                     (x64_setcc (CC.NZ)))))
 
 ;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs
index e28cf05d1..bae1053e8 100644
--- a/cranelift/codegen/src/isa/x64/pcc.rs
+++ b/cranelift/codegen/src/isa/x64/pcc.rs
@@ -438,15 +438,18 @@ pub(crate) fn check(
         }
 
         Inst::CmpRmiR {
-            size, dst, ref src, ..
-        } => match <&RegMemImm>::from(src) {
+            size,
+            src1,
+            ref src2,
+            ..
+        } => match <&RegMemImm>::from(src2) {
             RegMemImm::Mem {
                 addr: SyntheticAmode::ConstantOffset(k),
             } => {
                 match vcode.constants.get(*k) {
                     VCodeConstantData::U64(bytes) => {
                         let value = u64::from_le_bytes(*bytes);
-                        let lhs = get_fact_or_default(vcode, dst.to_reg(), 64);
+                        let lhs = get_fact_or_default(vcode, src1.to_reg(), 64);
                         let rhs = Fact::constant(64, value);
                         state.cmp_flags = Some((lhs, rhs));
                     }
@@ -456,19 +459,19 @@ pub(crate) fn check(
             }
             RegMemImm::Mem { ref addr } => {
                 if let Some(rhs) = check_load(ctx, None, addr, vcode, size.to_type(), 64)? {
-                    let lhs = get_fact_or_default(vcode, dst.to_reg(), 64);
+                    let lhs = get_fact_or_default(vcode, src1.to_reg(), 64);
                     state.cmp_flags = Some((lhs, rhs));
                 }
                 Ok(())
             }
             RegMemImm::Reg { reg } => {
                 let rhs = get_fact_or_default(vcode, *reg, 64);
-                let lhs = get_fact_or_default(vcode, dst.to_reg(), 64);
+                let lhs = get_fact_or_default(vcode, src1.to_reg(), 64);
                 state.cmp_flags = Some((lhs, rhs));
                 Ok(())
             }
             RegMemImm::Imm { simm32 } => {
-                let lhs = get_fact_or_default(vcode, dst.to_reg(), 64);
+                let lhs = get_fact_or_default(vcode, src1.to_reg(), 64);
                 let rhs = Fact::constant(64, (*simm32 as i32) as i64 as u64);
                 state.cmp_flags = Some((lhs, rhs));
                 Ok(())
@@ -813,6 +816,18 @@ pub(crate) fn check(
             ensure_no_fact(vcode, dst.to_reg())
         }
 
+        Inst::XmmCmpRmRVex {
+            ref src1, ref src2, ..
+        } => {
+            match <&RegMem>::from(src2) {
+                RegMem::Mem { ref addr } => {
+                    check_load(ctx, None, addr, vcode, F32, 32)?;
+                }
+                RegMem::Reg { .. } => {}
+            }
+            ensure_no_fact(vcode, src1.to_reg())
+        }
+
         Inst::CallKnown { .. }
         | Inst::ReturnCallKnown { .. }
         | Inst::JmpKnown { .. }
@@ -826,10 +841,9 @@ pub(crate) fn check(
         | Inst::Ud2 { .. } => Ok(()),
         Inst::Rets { .. } => Ok(()),
 
+        Inst::ReturnCallUnknown { .. } => Ok(()),
+
         Inst::CallUnknown { ref dest, .. }
-        | Inst::ReturnCallUnknown {
-            callee: ref dest, ..
-        }
         | Inst::JmpUnknown {
             target: ref dest, ..
         } => match <&RegMem>::from(dest) {
diff --git a/cranelift/entity/src/lib.rs b/cranelift/entity/src/lib.rs
index b22eeb350..3d22df490 100644
--- a/cranelift/entity/src/lib.rs
+++ b/cranelift/entity/src/lib.rs
@@ -38,6 +38,9 @@ extern crate alloc;
 #[doc(hidden)]
 pub extern crate core as __core;
 
+use core::iter::FusedIterator;
+use core::ops::Range;
+
 /// A type wrapping a small integer index should implement `EntityRef` so it can be used as the key
 /// of an `SecondaryMap` or `SparseMap`.
 pub trait EntityRef: Copy + Eq {
@@ -49,6 +52,67 @@ pub trait EntityRef: Copy + Eq {
     fn index(self) -> usize;
 }
 
+/// Iterate over a `Range<E: EntityRef>`, yielding a sequence of `E` items.
+#[inline]
+pub fn iter_entity_range<E>(range: Range<E>) -> IterEntityRange<E>
+where
+    E: EntityRef,
+{
+    IterEntityRange {
+        range: range.start.index()..range.end.index(),
+        _phantom: core::marker::PhantomData,
+    }
+}
+
+/// Iterator type returned by `iter_entity_range`.
+pub struct IterEntityRange<E> {
+    range: Range<usize>,
+    _phantom: core::marker::PhantomData<E>,
+}
+
+impl<E> Iterator for IterEntityRange<E>
+where
+    E: EntityRef,
+{
+    type Item = E;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        let i = self.range.next()?;
+        Some(E::new(i))
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.range.size_hint()
+    }
+}
+
+impl<E> DoubleEndedIterator for IterEntityRange<E>
+where
+    E: EntityRef,
+{
+    #[inline]
+    fn next_back(&mut self) -> Option<Self::Item> {
+        let i = self.range.next_back()?;
+        Some(E::new(i))
+    }
+}
+
+impl<E> FusedIterator for IterEntityRange<E>
+where
+    E: EntityRef,
+    Range<usize>: FusedIterator,
+{
+}
+
+impl<E> ExactSizeIterator for IterEntityRange<E>
+where
+    E: EntityRef,
+    Range<usize>: ExactSizeIterator,
+{
+}
+
 /// Macro which provides the common implementation of a 32-bit entity reference.
 #[macro_export]
 macro_rules! entity_impl {
diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs
index 2e32e0e70..502279f24 100644
--- a/cranelift/wasm/src/code_translator.rs
+++ b/cranelift/wasm/src/code_translator.rs
@@ -2543,6 +2543,10 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
         | Operator::StructGet { .. } => {
             unimplemented!("GC operators not yet implemented")
         }
+
+        Operator::GlobalAtomicGet { .. } | Operator::GlobalAtomicSet { .. } => {
+            unimplemented!("shared-everything-threads not yet implemented")
+        }
     };
     Ok(())
 }
diff --git a/crates/cranelift/src/debug.rs b/crates/cranelift/src/debug.rs
index 1b10eb351..ca92d77b6 100644
--- a/crates/cranelift/src/debug.rs
+++ b/crates/cranelift/src/debug.rs
@@ -8,7 +8,7 @@ pub enum ModuleMemoryOffset {
     /// Offset to the defined memory.
     Defined(u32),
     /// Offset to the imported memory.
-    Imported(u32),
+    Imported(#[allow(dead_code)] u32),
 }
 
 pub use write_debuginfo::{emit_dwarf, DwarfSectionRelocTarget};
diff --git a/crates/cranelift/src/debug/transform/expression.rs b/crates/cranelift/src/debug/transform/expression.rs
index 3e09bff91..f78b3b745 100644
--- a/crates/cranelift/src/debug/transform/expression.rs
+++ b/crates/cranelift/src/debug/transform/expression.rs
@@ -3,7 +3,7 @@
 use super::address_transform::AddressTransform;
 use crate::debug::ModuleMemoryOffset;
 use anyhow::{Context, Error, Result};
-use cranelift_codegen::ir::{StackSlots, ValueLabel};
+use cranelift_codegen::ir::ValueLabel;
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::LabelValueLoc;
 use cranelift_codegen::ValueLabelsRanges;
@@ -19,7 +19,6 @@ use wasmtime_environ::{DefinedFuncIndex, EntityRef};
 pub struct FunctionFrameInfo<'a> {
     pub value_ranges: &'a ValueLabelsRanges,
     pub memory_offset: ModuleMemoryOffset,
-    pub sized_stack_slots: &'a StackSlots,
 }
 
 impl<'a> FunctionFrameInfo<'a> {
@@ -1206,15 +1205,12 @@ mod tests {
     fn test_debug_value_range_builder() {
         use super::ValueLabelRangesBuilder;
         use crate::debug::ModuleMemoryOffset;
-        use cranelift_codegen::ir::StackSlots;
         use wasmtime_environ::{DefinedFuncIndex, EntityRef};
 
         let addr_tr = create_mock_address_transform();
-        let sized_stack_slots = StackSlots::new();
         let (value_ranges, value_labels) = create_mock_value_ranges();
         let fi = FunctionFrameInfo {
             memory_offset: ModuleMemoryOffset::None,
-            sized_stack_slots: &sized_stack_slots,
             value_ranges: &value_ranges,
         };
 
diff --git a/crates/cranelift/src/debug/transform/utils.rs b/crates/cranelift/src/debug/transform/utils.rs
index 0ffd19062..ce7ac0e38 100644
--- a/crates/cranelift/src/debug/transform/utils.rs
+++ b/crates/cranelift/src/debug/transform/utils.rs
@@ -178,7 +178,6 @@ where
         let frame_info = FunctionFrameInfo {
             value_ranges: &func.value_labels_ranges,
             memory_offset: memory_offset.clone(),
-            sized_stack_slots: &func.sized_stack_slots,
         };
         Some(frame_info)
     } else {
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 5e02e721c..5ffcf2cda 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -18,8 +18,8 @@ use cranelift_wasm::{
 use std::mem;
 use wasmparser::Operator;
 use wasmtime_environ::{
-    BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, ModuleTranslation, ModuleType,
-    ModuleTypesBuilder, PtrSize, TableStyle, Tunables, TypeConvert, VMOffsets, WASM_PAGE_SIZE,
+    BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, ModuleTranslation, ModuleTypesBuilder,
+    PtrSize, TableStyle, Tunables, TypeConvert, VMOffsets, WASM_PAGE_SIZE,
 };
 use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
 
@@ -1204,7 +1204,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
                 // If `ty_index` matches `table_ty`, then this call is
                 // statically known to have the right type, so no checks are
                 // necessary.
-                let ModuleType::Function(specified_ty) = self.env.module.types[ty_index];
+                let specified_ty = self.env.module.types[ty_index];
                 if specified_ty == table_ty {
                     return CheckIndirectCallTypeSignature::StaticMatch {
                         may_be_null: table.table.wasm_ty.nullable,
@@ -1244,6 +1244,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
             // validation error to perform a call through a non-function table,
             // so these cases are dynamically not reachable.
             WasmHeapType::Concrete(EngineOrModuleTypeIndex::Engine(_))
+            | WasmHeapType::Concrete(EngineOrModuleTypeIndex::RecGroup(_))
             | WasmHeapType::Extern
             | WasmHeapType::Any
             | WasmHeapType::I31
@@ -1265,7 +1266,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
             base,
             i32::try_from(self.env.offsets.vmctx_type_ids_array()).unwrap(),
         );
-        let sig_index = self.env.module.types[ty_index].unwrap_function();
+        let sig_index = self.env.module.types[ty_index];
         let offset =
             i32::try_from(sig_index.as_u32().checked_mul(sig_id_type.bytes()).unwrap()).unwrap();
         let caller_sig_id = self
@@ -1390,11 +1391,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
 
 impl TypeConvert for FuncEnvironment<'_> {
     fn lookup_heap_type(&self, ty: wasmparser::UnpackedIndex) -> WasmHeapType {
-        wasmtime_environ::WasmparserTypeConverter {
-            module: self.module,
-            types: self.types,
-        }
-        .lookup_heap_type(ty)
+        wasmtime_environ::WasmparserTypeConverter::new(self.types, self.module).lookup_heap_type(ty)
     }
 }
 
@@ -2005,7 +2002,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         func: &mut ir::Function,
         index: TypeIndex,
     ) -> WasmResult<ir::SigRef> {
-        let index = self.module.types[index].unwrap_function();
+        let index = self.module.types[index];
         let sig = crate::wasm_call_signature(self.isa, &self.types[index], &self.tunables);
         Ok(func.import_signature(sig))
     }
diff --git a/crates/environ/examples/factc.rs b/crates/environ/examples/factc.rs
index 979e617b4..ac55c4296 100644
--- a/crates/environ/examples/factc.rs
+++ b/crates/environ/examples/factc.rs
@@ -82,8 +82,6 @@ impl Factc {
     fn execute(self) -> Result<()> {
         env_logger::init();
 
-        let mut types = ComponentTypesBuilder::default();
-
         // Manufactures a unique `CoreDef` so all function imports get unique
         // function imports.
         let mut next_def = 0;
@@ -118,9 +116,11 @@ impl Factc {
             }
         };
 
+        let mut validator = Validator::new();
+        let mut types = ComponentTypesBuilder::new(&validator);
+
         let mut adapters = Vec::new();
         let input = wat::parse_file(&self.input)?;
-        let mut validator = Validator::new();
         let wasm_types = validator
             .validate_all(&input)
             .context("failed to validate input wasm")?;
diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs
index 964ea41fa..a80266f44 100644
--- a/crates/environ/src/component/translate.rs
+++ b/crates/environ/src/component/translate.rs
@@ -1,4 +1,5 @@
 use crate::component::*;
+use crate::Module;
 use crate::ScopeVec;
 use crate::{
     EntityIndex, ModuleEnvironment, ModuleTranslation, ModuleTypesBuilder, PrimaryMap, Tunables,
@@ -14,6 +15,7 @@ use wasmparser::types::{
 };
 use wasmparser::{Chunk, ComponentImportName, Encoding, Parser, Payload, Validator};
 use wasmtime_types::ModuleInternedTypeIndex;
+use wasmtime_types::WasmResult;
 
 mod adapt;
 pub use self::adapt::*;
@@ -487,7 +489,7 @@ impl<'a, 'data> Translator<'a, 'data> {
                             let lower_ty = types.component_function_at(func_index);
                             let func = ComponentFuncIndex::from_u32(func_index);
                             let options = self.canonical_options(&options);
-                            let canonical_abi = self.core_func_signature(core_func_index);
+                            let canonical_abi = self.core_func_signature(core_func_index)?;
 
                             core_func_index += 1;
                             LocalInitializer::Lower {
@@ -499,19 +501,19 @@ impl<'a, 'data> Translator<'a, 'data> {
                         }
                         wasmparser::CanonicalFunction::ResourceNew { resource } => {
                             let resource = types.component_any_type_at(resource).unwrap_resource();
-                            let ty = self.core_func_signature(core_func_index);
+                            let ty = self.core_func_signature(core_func_index)?;
                             core_func_index += 1;
                             LocalInitializer::ResourceNew(resource, ty)
                         }
                         wasmparser::CanonicalFunction::ResourceDrop { resource } => {
                             let resource = types.component_any_type_at(resource).unwrap_resource();
-                            let ty = self.core_func_signature(core_func_index);
+                            let ty = self.core_func_signature(core_func_index)?;
                             core_func_index += 1;
                             LocalInitializer::ResourceDrop(resource, ty)
                         }
                         wasmparser::CanonicalFunction::ResourceRep { resource } => {
                             let resource = types.component_any_type_at(resource).unwrap_resource();
-                            let ty = self.core_func_signature(core_func_index);
+                            let ty = self.core_func_signature(core_func_index)?;
                             core_func_index += 1;
                             LocalInitializer::ResourceRep(resource, ty)
                         }
@@ -911,12 +913,14 @@ impl<'a, 'data> Translator<'a, 'data> {
         return ret;
     }
 
-    fn core_func_signature(&mut self, idx: u32) -> ModuleInternedTypeIndex {
+    /// Get the interned type index for the `index`th core function.
+    fn core_func_signature(&mut self, index: u32) -> WasmResult<ModuleInternedTypeIndex> {
         let types = self.validator.types(0).unwrap();
-        let id = types.core_function_at(idx);
-        let ty = types[id].unwrap_func();
-        let ty = self.types.convert_func_type(ty);
-        self.types.module_types_builder().wasm_func_type(id, ty)
+        let id = types.core_function_at(index);
+        let module = Module::default();
+        self.types
+            .module_types_builder()
+            .intern_type(&module, types, id)
     }
 }
 
diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rs
index d112fa808..aeb60da23 100644
--- a/crates/environ/src/component/translate/adapt.rs
+++ b/crates/environ/src/component/translate/adapt.rs
@@ -119,7 +119,6 @@ use crate::component::translate::*;
 use crate::fact;
 use crate::EntityType;
 use std::collections::HashSet;
-use wasmparser::WasmFeatures;
 
 /// Metadata information about a fused adapter.
 #[derive(Debug, Clone, Hash, Eq, PartialEq)]
@@ -212,12 +211,10 @@ impl<'data> Translator<'_, 'data> {
             // specifically enabled here since the adapter module is highly
             // likely to use that if anything is actually indirected through
             // memory.
-            let mut validator = Validator::new_with_features(
-                *self.validator.features() | WasmFeatures::MULTI_MEMORY,
-            );
+            self.validator.reset();
             let translation = ModuleEnvironment::new(
                 self.tunables,
-                &mut validator,
+                &mut self.validator,
                 self.types.module_types_builder(),
             )
             .translate(Parser::new(0), wasm)
diff --git a/crates/environ/src/component/types.rs b/crates/environ/src/component/types.rs
index 327d23b13..93d002baf 100644
--- a/crates/environ/src/component/types.rs
+++ b/crates/environ/src/component/types.rs
@@ -1,7 +1,7 @@
 use crate::component::{Export, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS};
 use crate::{
-    CompiledModuleInfo, EntityType, ModuleTypes, ModuleTypesBuilder, PrimaryMap, TypeConvert,
-    WasmHeapType, WasmValType,
+    CompiledModuleInfo, EntityType, Module, ModuleTypes, ModuleTypesBuilder, PrimaryMap,
+    TypeConvert, WasmHeapType, WasmValType,
 };
 use anyhow::{bail, Result};
 use cranelift_entity::EntityRef;
@@ -11,7 +11,7 @@ use std::collections::HashMap;
 use std::hash::{Hash, Hasher};
 use std::ops::Index;
 use wasmparser::names::KebabString;
-use wasmparser::types;
+use wasmparser::{types, Validator};
 use wasmtime_component_util::{DiscriminantSize, FlagsSize};
 use wasmtime_types::ModuleInternedTypeIndex;
 
@@ -255,13 +255,13 @@ pub struct ComponentTypes {
     results: PrimaryMap<TypeResultIndex, TypeResult>,
     resource_tables: PrimaryMap<TypeResourceTableIndex, TypeResourceTable>,
 
-    module_types: ModuleTypes,
+    module_types: Option<ModuleTypes>,
 }
 
 impl ComponentTypes {
     /// Returns the core wasm module types known within this component.
     pub fn module_types(&self) -> &ModuleTypes {
-        &self.module_types
+        self.module_types.as_ref().unwrap()
     }
 
     /// Returns the canonical ABI information about the specified type.
@@ -341,7 +341,7 @@ where
 {
     type Output = <ModuleTypes as Index<T>>::Output;
     fn index(&self, idx: T) -> &Self::Output {
-        self.module_types.index(idx)
+        self.module_types.as_ref().unwrap().index(idx)
     }
 }
 
@@ -359,7 +359,6 @@ where
 ///
 /// This contains tables to intern any component types found as well as
 /// managing building up core wasm [`ModuleTypes`] as well.
-#[derive(Default)]
 pub struct ComponentTypesBuilder {
     functions: HashMap<TypeFunc, TypeFuncIndex>,
     lists: HashMap<TypeList, TypeListIndex>,
@@ -398,6 +397,26 @@ macro_rules! intern_and_fill_flat_types {
 }
 
 impl ComponentTypesBuilder {
+    /// Construct a new `ComponentTypesBuilder` for use with the given validator.
+    pub fn new(validator: &Validator) -> Self {
+        Self {
+            module_types: ModuleTypesBuilder::new(validator),
+
+            functions: HashMap::default(),
+            lists: HashMap::default(),
+            records: HashMap::default(),
+            variants: HashMap::default(),
+            tuples: HashMap::default(),
+            enums: HashMap::default(),
+            flags: HashMap::default(),
+            options: HashMap::default(),
+            results: HashMap::default(),
+            component_types: ComponentTypes::default(),
+            type_info: TypeInformationCache::default(),
+            resources: ResourcesBuilder::default(),
+        }
+    }
+
     fn export_type_def(
         &mut self,
         static_modules: &PrimaryMap<StaticModuleIndex, CompiledModuleInfo>,
@@ -457,7 +476,7 @@ impl ComponentTypesBuilder {
         }
         let ty = self.component_types.components.push(component_ty);
 
-        self.component_types.module_types = self.module_types.finish();
+        self.component_types.module_types = Some(self.module_types.finish());
         (self.component_types, ty)
     }
 
@@ -517,6 +536,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         id: types::ComponentFuncTypeId,
     ) -> Result<TypeFuncIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let ty = &types[id];
         let params = ty
             .params
@@ -542,6 +562,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: types::ComponentEntityType,
     ) -> Result<TypeDef> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         Ok(match ty {
             types::ComponentEntityType::Module(id) => {
                 TypeDef::Module(self.convert_module(types, id)?)
@@ -574,6 +595,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         id: types::ComponentAnyTypeId,
     ) -> Result<TypeDef> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         Ok(match id {
             types::ComponentAnyTypeId::Defined(id) => {
                 TypeDef::Interface(self.defined_type(types, id)?)
@@ -598,6 +620,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         id: types::ComponentTypeId,
     ) -> Result<TypeComponentIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let ty = &types[id];
         let mut result = TypeComponent::default();
         for (name, ty) in ty.imports.iter() {
@@ -620,6 +643,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         id: types::ComponentInstanceTypeId,
     ) -> Result<TypeComponentInstanceIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let ty = &types[id];
         let mut result = TypeComponentInstance::default();
         for (name, ty) in ty.exports.iter() {
@@ -636,6 +660,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         id: types::ComponentCoreModuleTypeId,
     ) -> Result<TypeModuleIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let ty = &types[id];
         let mut result = TypeModule::default();
         for ((module, field), ty) in ty.imports.iter() {
@@ -657,16 +682,14 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: &types::EntityType,
     ) -> Result<EntityType> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         Ok(match ty {
-            types::EntityType::Func(idx) => {
-                let ty = types[*idx].unwrap_func();
-                let ty = self.convert_func_type(ty);
-                EntityType::Function(
-                    self.module_types_builder_mut()
-                        .wasm_func_type(*idx, ty)
-                        .into(),
-                )
-            }
+            types::EntityType::Func(id) => EntityType::Function({
+                let module = Module::default();
+                self.module_types_builder_mut()
+                    .intern_type(&module, types, *id)?
+                    .into()
+            }),
             types::EntityType::Table(ty) => EntityType::Table(self.convert_table_type(ty)),
             types::EntityType::Memory(ty) => EntityType::Memory(ty.clone().into()),
             types::EntityType::Global(ty) => EntityType::Global(self.convert_global_type(ty)),
@@ -679,6 +702,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         id: types::ComponentDefinedTypeId,
     ) -> Result<InterfaceType> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let ret = match &types[id] {
             types::ComponentDefinedType::Primitive(ty) => ty.into(),
             types::ComponentDefinedType::Record(e) => {
@@ -718,6 +742,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: &types::ComponentValType,
     ) -> Result<InterfaceType> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         match ty {
             types::ComponentValType::Primitive(p) => Ok(p.into()),
             types::ComponentValType::Type(id) => self.defined_type(types, *id),
@@ -729,6 +754,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: &types::RecordType,
     ) -> Result<TypeRecordIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let fields = ty
             .fields
             .iter()
@@ -752,6 +778,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: &types::VariantType,
     ) -> Result<TypeVariantIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let cases = ty
             .cases
             .iter()
@@ -783,6 +810,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: &types::TupleType,
     ) -> Result<TypeTupleIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let types = ty
             .types
             .iter()
@@ -822,6 +850,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: &types::ComponentValType,
     ) -> Result<TypeOptionIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let ty = self.valtype(types, ty)?;
         let (info, abi) = VariantInfo::new([None, Some(self.component_types.canonical_abi(&ty))]);
         Ok(self.add_option_type(TypeOption { ty, abi, info }))
@@ -833,6 +862,7 @@ impl ComponentTypesBuilder {
         ok: &Option<types::ComponentValType>,
         err: &Option<types::ComponentValType>,
     ) -> Result<TypeResultIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let ok = match ok {
             Some(ty) => Some(self.valtype(types, ty)?),
             None => None,
@@ -853,6 +883,7 @@ impl ComponentTypesBuilder {
         types: types::TypesRef<'_>,
         ty: &types::ComponentValType,
     ) -> Result<TypeListIndex> {
+        assert_eq!(types.id(), self.module_types.validator_id());
         let element = self.valtype(types, ty)?;
         Ok(self.add_list_type(TypeList { element }))
     }
diff --git a/crates/environ/src/fact/signature.rs b/crates/environ/src/fact/signature.rs
index 07660d4d0..37531dc8d 100644
--- a/crates/environ/src/fact/signature.rs
+++ b/crates/environ/src/fact/signature.rs
@@ -12,14 +12,6 @@ pub struct Signature {
     pub params: Vec<ValType>,
     /// Core wasm results.
     pub results: Vec<ValType>,
-    /// Indicator to whether parameters are indirect, meaning that the first
-    /// entry of `params` is a pointer type which all parameters are loaded
-    /// through.
-    pub params_indirect: bool,
-    /// Indicator whether results are passed indirectly. This may mean that
-    /// `results` is an `i32` or that `params` ends with an `i32` depending on
-    /// the `Context`.
-    pub results_indirect: bool,
 }
 
 impl ComponentTypesBuilder {
@@ -33,7 +25,6 @@ impl ComponentTypesBuilder {
         let ty = &self[options.ty];
         let ptr_ty = options.options.ptr();
 
-        let mut params_indirect = false;
         let mut params = match self.flatten_types(
             &options.options,
             MAX_FLAT_PARAMS,
@@ -41,12 +32,10 @@ impl ComponentTypesBuilder {
         ) {
             Some(list) => list,
             None => {
-                params_indirect = true;
                 vec![ptr_ty]
             }
         };
 
-        let mut results_indirect = false;
         let results = match self.flatten_types(
             &options.options,
             MAX_FLAT_RESULTS,
@@ -54,7 +43,6 @@ impl ComponentTypesBuilder {
         ) {
             Some(list) => list,
             None => {
-                results_indirect = true;
                 match context {
                     // For a lifted function too-many-results gets translated to a
                     // returned pointer where results are read from. The callee
@@ -70,12 +58,7 @@ impl ComponentTypesBuilder {
                 }
             }
         };
-        Signature {
-            params,
-            results,
-            params_indirect,
-            results_indirect,
-        }
+        Signature { params, results }
     }
 
     /// Pushes the flat version of a list of component types into a final result
diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs
index e4f22f0db..bc63c1ceb 100644
--- a/crates/environ/src/module.rs
+++ b/crates/environ/src/module.rs
@@ -476,26 +476,6 @@ pub enum TableElementExpression {
     Null,
 }
 
-/// Different types that can appear in a module.
-///
-/// Note that each of these variants are intended to index further into a
-/// separate table.
-#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
-#[allow(missing_docs)]
-pub enum ModuleType {
-    Function(ModuleInternedTypeIndex),
-}
-
-impl ModuleType {
-    /// Asserts this is a `ModuleType::Function`, returning the underlying
-    /// `SignatureIndex`.
-    pub fn unwrap_function(&self) -> ModuleInternedTypeIndex {
-        match self {
-            ModuleType::Function(f) => *f,
-        }
-    }
-}
-
 /// A translated WebAssembly module, excluding the function bodies and
 /// memory initializers.
 #[derive(Default, Debug, Serialize, Deserialize)]
@@ -528,7 +508,7 @@ pub struct Module {
     pub passive_data_map: BTreeMap<DataIndex, Range<u32>>,
 
     /// Types declared in the wasm module.
-    pub types: PrimaryMap<TypeIndex, ModuleType>,
+    pub types: PrimaryMap<TypeIndex, ModuleInternedTypeIndex>,
 
     /// Number of imported or aliased functions in the module.
     pub num_imported_funcs: usize,
diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs
index 77e96db9c..0c119ae74 100644
--- a/crates/environ/src/module_environ.rs
+++ b/crates/environ/src/module_environ.rs
@@ -1,6 +1,6 @@
 use crate::module::{
     FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, MemoryPlan, Module,
-    ModuleType, TableElementExpression, TablePlan, TableSegment, TableSegmentElements,
+    TableElementExpression, TablePlan, TableSegment, TableSegmentElements,
 };
 use crate::{
     DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, FuncIndex, GlobalIndex,
@@ -15,11 +15,10 @@ use std::collections::HashMap;
 use std::mem;
 use std::path::PathBuf;
 use std::sync::Arc;
-use wasmparser::types::{CoreTypeId, Types};
 use wasmparser::{
-    CompositeType, CustomSectionReader, DataKind, ElementItems, ElementKind, Encoding,
-    ExternalKind, FuncToValidate, FunctionBody, NameSectionReader, Naming, Operator, Parser,
-    Payload, TypeRef, Validator, ValidatorResources, WasmFeatures,
+    types::Types, CustomSectionReader, DataKind, ElementItems, ElementKind, Encoding, ExternalKind,
+    FuncToValidate, FunctionBody, NameSectionReader, Naming, Operator, Parser, Payload, TypeRef,
+    Validator, ValidatorResources, WasmFeatures,
 };
 use wasmtime_types::ModuleInternedTypeIndex;
 
@@ -236,14 +235,56 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
 
             Payload::TypeSection(types) => {
                 self.validator.type_section(&types)?;
-                let num = usize::try_from(types.count()).unwrap();
-                self.result.module.types.reserve(num);
-                self.types.reserve_wasm_signatures(num);
-
-                for i in 0..types.count() {
-                    let types = self.validator.types(0).unwrap();
-                    let ty = types.core_type_at(i);
-                    self.declare_type(ty.unwrap_sub())?;
+
+                let count = types.count();
+                let capacity = usize::try_from(count).unwrap();
+                self.result.module.types.reserve(capacity);
+                self.types.reserve_wasm_signatures(capacity);
+
+                // Iterate over each *rec group* -- not type -- defined in the
+                // types section. Rec groups are the unit of canonicalization
+                // and therefore the unit at which we need to process at a
+                // time. `wasmparser` has already done the hard work of
+                // de-duplicating and canonicalizing the rec groups within the
+                // module for us, we just need to translate them into our data
+                // structures. Note that, if the Wasm defines duplicate rec
+                // groups, we need copy the duplicates over (shallowly) as well,
+                // so that our types index space doesn't have holes.
+                let mut type_index = 0;
+                for _ in 0..count {
+                    let validator_types = self.validator.types(0).unwrap();
+
+                    // Get the rec group for the current type index, which is
+                    // always the first type defined in a rec group.
+                    let core_type_id = validator_types.core_type_at(type_index).unwrap_sub();
+                    log::trace!(
+                        "about to intern rec group for {core_type_id:?} = {:?}",
+                        validator_types[core_type_id]
+                    );
+                    let rec_group_id = validator_types.rec_group_id_of(core_type_id);
+                    debug_assert_eq!(
+                        validator_types
+                            .rec_group_elements(rec_group_id)
+                            .position(|id| id == core_type_id),
+                        Some(0)
+                    );
+
+                    // Intern the rec group and then fill in this module's types
+                    // index space.
+                    let interned = self.types.intern_rec_group(
+                        &self.result.module,
+                        validator_types,
+                        rec_group_id,
+                    )?;
+                    let elems = self.types.rec_group_elements(interned);
+                    let len = elems.len();
+                    self.result.module.types.reserve(len);
+                    for ty in elems {
+                        self.result.module.types.push(ty);
+                    }
+
+                    // Advance `type_index` to the start of the next rec group.
+                    type_index += u32::try_from(len).unwrap();
                 }
             }
 
@@ -258,11 +299,11 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
                     let ty = match import.ty {
                         TypeRef::Func(index) => {
                             let index = TypeIndex::from_u32(index);
-                            let sig_index = self.result.module.types[index].unwrap_function();
+                            let interned_index = self.result.module.types[index];
                             self.result.module.num_imported_funcs += 1;
                             self.result.debuginfo.wasm_file.imported_func_count += 1;
                             EntityType::Function(wasmtime_types::EngineOrModuleTypeIndex::Module(
-                                sig_index,
+                                interned_index,
                             ))
                         }
                         TypeRef::Memory(ty) => {
@@ -294,8 +335,8 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
                 for entry in functions {
                     let sigindex = entry?;
                     let ty = TypeIndex::from_u32(sigindex);
-                    let sig_index = self.result.module.types[ty].unwrap_function();
-                    self.result.module.push_function(sig_index);
+                    let interned_index = self.result.module.types[ty];
+                    self.result.module.push_function(interned_index);
                 }
             }
 
@@ -842,25 +883,6 @@ and for re-adding support for interface types you can see this issue:
         self.result.module.num_escaped_funcs += 1;
     }
 
-    fn declare_type(&mut self, id: CoreTypeId) -> WasmResult<()> {
-        let types = self.validator.types(0).unwrap();
-        let ty = &types[id];
-        assert!(ty.is_final);
-        assert!(ty.supertype_idx.is_none());
-        match &ty.composite_type {
-            CompositeType::Func(ty) => {
-                let wasm = self.convert_func_type(ty);
-                let sig_index = self.types.wasm_func_type(id, wasm);
-                self.result
-                    .module
-                    .types
-                    .push(ModuleType::Function(sig_index));
-            }
-            CompositeType::Array(_) | CompositeType::Struct(_) => unimplemented!(),
-        }
-        Ok(())
-    }
-
     /// Parses the Name section of the wasm module.
     fn name_section(&mut self, names: NameSectionReader<'data>) -> WasmResult<()> {
         for subsection in names {
@@ -933,11 +955,7 @@ and for re-adding support for interface types you can see this issue:
 
 impl TypeConvert for ModuleEnvironment<'_, '_> {
     fn lookup_heap_type(&self, index: wasmparser::UnpackedIndex) -> WasmHeapType {
-        WasmparserTypeConverter {
-            types: &self.types,
-            module: &self.result.module,
-        }
-        .lookup_heap_type(index)
+        WasmparserTypeConverter::new(&self.types, &self.result.module).lookup_heap_type(index)
     }
 }
 
diff --git a/crates/environ/src/module_types.rs b/crates/environ/src/module_types.rs
index 06d11e772..693175e8f 100644
--- a/crates/environ/src/module_types.rs
+++ b/crates/environ/src/module_types.rs
@@ -1,10 +1,15 @@
-use crate::{Module, ModuleType, PrimaryMap, TypeConvert, WasmFuncType, WasmHeapType};
+use crate::{Module, PrimaryMap, TypeConvert, TypeIndex, WasmFuncType, WasmHeapType};
+use cranelift_entity::EntityRef;
 use serde_derive::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::ops::Index;
-use wasmparser::types::CoreTypeId;
-use wasmparser::UnpackedIndex;
-use wasmtime_types::{EngineOrModuleTypeIndex, ModuleInternedTypeIndex, TypeIndex};
+use std::{
+    collections::HashMap,
+    ops::{Index, Range},
+};
+use wasmparser::{CompositeType, UnpackedIndex, Validator, ValidatorId};
+use wasmtime_types::{
+    wasm_unsupported, EngineOrModuleTypeIndex, ModuleInternedRecGroupIndex,
+    ModuleInternedTypeIndex, WasmResult,
+};
 
 /// All types used in a core wasm module.
 ///
@@ -14,17 +19,42 @@ use wasmtime_types::{EngineOrModuleTypeIndex, ModuleInternedTypeIndex, TypeIndex
 /// Note that accesing this type is primarily done through the `Index`
 /// implementations for this type.
 #[derive(Default, Serialize, Deserialize)]
-#[allow(missing_docs)]
 pub struct ModuleTypes {
+    rec_groups: PrimaryMap<ModuleInternedRecGroupIndex, Range<ModuleInternedTypeIndex>>,
     wasm_types: PrimaryMap<ModuleInternedTypeIndex, WasmFuncType>,
 }
 
 impl ModuleTypes {
     /// Returns an iterator over all the wasm function signatures found within
     /// this module.
-    pub fn wasm_types(&self) -> impl Iterator<Item = (ModuleInternedTypeIndex, &WasmFuncType)> {
+    pub fn wasm_types(
+        &self,
+    ) -> impl ExactSizeIterator<Item = (ModuleInternedTypeIndex, &WasmFuncType)> {
         self.wasm_types.iter()
     }
+
+    /// Get the type at the specified index.
+    pub fn get(&self, ty: ModuleInternedTypeIndex) -> &WasmFuncType {
+        &self.wasm_types[ty]
+    }
+
+    /// Get an iterator over all recursion groups defined in this module and
+    /// their elements.
+    pub fn rec_groups(
+        &self,
+    ) -> impl ExactSizeIterator<Item = (ModuleInternedRecGroupIndex, Range<ModuleInternedTypeIndex>)> + '_
+    {
+        self.rec_groups.iter().map(|(k, v)| (k, v.clone()))
+    }
+
+    /// Get the elements within an already-defined rec group.
+    pub fn rec_group_elements(
+        &self,
+        rec_group: ModuleInternedRecGroupIndex,
+    ) -> impl ExactSizeIterator<Item = ModuleInternedTypeIndex> {
+        let range = &self.rec_groups[rec_group];
+        (range.start.as_u32()..range.end.as_u32()).map(|i| ModuleInternedTypeIndex::from_u32(i))
+    }
 }
 
 impl Index<ModuleInternedTypeIndex> for ModuleTypes {
@@ -35,38 +65,235 @@ impl Index<ModuleInternedTypeIndex> for ModuleTypes {
     }
 }
 
+/// A type marking the start of a recursion group's definition.
+///
+/// This is initialized by `ModuleTypesBuilder::start_rec_group` and then
+/// finished in `ModuleTypes::end_rec_group` after all of the types in the rec
+/// group have been defined.
+struct RecGroupStart {
+    rec_group_index: ModuleInternedRecGroupIndex,
+    start: ModuleInternedTypeIndex,
+    end: ModuleInternedTypeIndex,
+}
+
 /// A builder for [`ModuleTypes`].
-#[derive(Default)]
-#[allow(missing_docs)]
 pub struct ModuleTypesBuilder {
+    /// The ID of the validator that this builder is configured for. Using a
+    /// different validator, or multiple validators, with this builder would
+    /// result in silliness because our `wasmparser::types::*Id`s are only
+    /// unique within the context of a particular validator. Getting this wrong
+    /// could result in generating calls to functions of the wrong type, for
+    /// example. So therefore we always assert that a builder instances is only
+    /// ever paired with a particular validator context.
+    validator_id: ValidatorId,
+
+    /// The canonicalized and deduplicated set of types we are building.
     types: ModuleTypes,
-    interned_func_types: HashMap<WasmFuncType, ModuleInternedTypeIndex>,
-    wasmparser_to_wasmtime: HashMap<CoreTypeId, ModuleInternedTypeIndex>,
+
+    /// A map from already-interned `wasmparser` types to their corresponding
+    /// Wasmtime type.
+    wasmparser_to_wasmtime: HashMap<wasmparser::types::CoreTypeId, ModuleInternedTypeIndex>,
+
+    /// The set of recursion groups we have already seen and interned.
+    already_seen: HashMap<wasmparser::types::RecGroupId, ModuleInternedRecGroupIndex>,
+
+    /// If we are in the middle of defining a recursion group, this is the
+    /// metadata about the recursion group we started defining.
+    defining_rec_group: Option<RecGroupStart>,
 }
 
 impl ModuleTypesBuilder {
+    /// Construct a new `ModuleTypesBuilder` using the given validator.
+    pub fn new(validator: &Validator) -> Self {
+        Self {
+            validator_id: validator.id(),
+            types: ModuleTypes::default(),
+            wasmparser_to_wasmtime: HashMap::default(),
+            already_seen: HashMap::default(),
+            defining_rec_group: None,
+        }
+    }
+
     /// Reserves space for `amt` more type signatures.
     pub fn reserve_wasm_signatures(&mut self, amt: usize) {
         self.types.wasm_types.reserve(amt);
+        self.wasmparser_to_wasmtime.reserve(amt);
+        self.already_seen.reserve(amt);
     }
 
-    /// Interns the `sig` specified and returns a unique `SignatureIndex` that
-    /// can be looked up within [`ModuleTypes`] to recover the [`WasmFuncType`]
-    /// at runtime.
-    pub fn wasm_func_type(&mut self, id: CoreTypeId, sig: WasmFuncType) -> ModuleInternedTypeIndex {
-        let sig = self.intern_func_type(sig);
-        self.wasmparser_to_wasmtime.insert(id, sig);
-        sig
+    /// Get the id of the validator that this builder is configured for.
+    pub fn validator_id(&self) -> ValidatorId {
+        self.validator_id
     }
 
-    fn intern_func_type(&mut self, sig: WasmFuncType) -> ModuleInternedTypeIndex {
-        if let Some(idx) = self.interned_func_types.get(&sig) {
-            return *idx;
+    /// Intern a recursion group and all of its types into this builder.
+    ///
+    /// If the recursion group has already been interned, then it is reused.
+    ///
+    /// Panics if given types from a different validator than the one that this
+    /// builder is associated with.
+    pub fn intern_rec_group(
+        &mut self,
+        module: &Module,
+        validator_types: wasmparser::types::TypesRef<'_>,
+        rec_group_id: wasmparser::types::RecGroupId,
+    ) -> WasmResult<ModuleInternedRecGroupIndex> {
+        assert_eq!(validator_types.id(), self.validator_id);
+
+        if let Some(interned) = self.already_seen.get(&rec_group_id) {
+            return Ok(*interned);
         }
 
-        let idx = self.types.wasm_types.push(sig.clone());
-        self.interned_func_types.insert(sig, idx);
-        return idx;
+        self.define_new_rec_group(module, validator_types, rec_group_id)
+    }
+
+    /// Define a new recursion group that we haven't already interned.
+    fn define_new_rec_group(
+        &mut self,
+        module: &Module,
+        validator_types: wasmparser::types::TypesRef<'_>,
+        rec_group_id: wasmparser::types::RecGroupId,
+    ) -> WasmResult<ModuleInternedRecGroupIndex> {
+        assert_eq!(validator_types.id(), self.validator_id);
+
+        self.start_rec_group(
+            validator_types,
+            validator_types.rec_group_elements(rec_group_id),
+        );
+
+        for id in validator_types.rec_group_elements(rec_group_id) {
+            let ty = &validator_types[id];
+            if ty.supertype_idx.is_some() {
+                return Err(wasm_unsupported!("wasm gc: explicit subtyping"));
+            }
+            match &ty.composite_type {
+                CompositeType::Func(ty) => {
+                    let wasm_ty = WasmparserTypeConverter::new(self, module).convert_func_type(ty);
+                    self.wasm_func_type_in_rec_group(id, wasm_ty);
+                }
+                CompositeType::Array(_) => return Err(wasm_unsupported!("wasm gc: array types")),
+                CompositeType::Struct(_) => return Err(wasm_unsupported!("wasm gc: struct types")),
+            }
+        }
+
+        Ok(self.end_rec_group(rec_group_id))
+    }
+
+    /// Start defining a recursion group.
+    fn start_rec_group(
+        &mut self,
+        validator_types: wasmparser::types::TypesRef<'_>,
+        elems: impl ExactSizeIterator<Item = wasmparser::types::CoreTypeId>,
+    ) {
+        log::trace!("Starting rec group of length {}", elems.len());
+
+        assert!(self.defining_rec_group.is_none());
+        assert_eq!(validator_types.id(), self.validator_id);
+
+        // Eagerly define the reverse map's entries for this rec group's types
+        // so that we can use them when converting `wasmparser` types to our
+        // types.
+        let len = elems.len();
+        for (i, wasmparser_id) in elems.enumerate() {
+            let interned = ModuleInternedTypeIndex::new(self.types.wasm_types.len() + i);
+            log::trace!(
+                "Reserving {interned:?} for {wasmparser_id:?} = {:?}",
+                validator_types[wasmparser_id]
+            );
+
+            let old_entry = self.wasmparser_to_wasmtime.insert(wasmparser_id, interned);
+            debug_assert_eq!(
+                old_entry, None,
+                "should not have already inserted {wasmparser_id:?}"
+            );
+        }
+
+        self.defining_rec_group = Some(RecGroupStart {
+            rec_group_index: self.types.rec_groups.next_key(),
+            start: self.types.wasm_types.next_key(),
+            end: ModuleInternedTypeIndex::new(self.types.wasm_types.len() + len),
+        });
+    }
+
+    /// Finish defining a recursion group.
+    fn end_rec_group(
+        &mut self,
+        rec_group_id: wasmparser::types::RecGroupId,
+    ) -> ModuleInternedRecGroupIndex {
+        let RecGroupStart {
+            rec_group_index,
+            start,
+            end,
+        } = self
+            .defining_rec_group
+            .take()
+            .expect("should be defining a rec group");
+
+        log::trace!("Ending rec group {start:?}..{end:?}");
+
+        debug_assert!(start.index() < self.types.wasm_types.len());
+        debug_assert_eq!(
+            end,
+            self.types.wasm_types.next_key(),
+            "should have defined the number of types declared in `start_rec_group`"
+        );
+
+        let idx = self.types.rec_groups.push(start..end);
+        debug_assert_eq!(idx, rec_group_index);
+
+        self.already_seen.insert(rec_group_id, rec_group_index);
+        rec_group_index
+    }
+
+    /// Intern a type into this builder and get its Wasmtime index.
+    ///
+    /// This will intern not only the single given type, but the type's entire
+    /// rec group. This helper method is provided as a convenience so that
+    /// callers don't have to get the type's rec group, intern the rec group,
+    /// and then look up the Wasmtime index for the original type themselves.
+    pub fn intern_type(
+        &mut self,
+        module: &Module,
+        validator_types: wasmparser::types::TypesRef<'_>,
+        id: wasmparser::types::CoreTypeId,
+    ) -> WasmResult<ModuleInternedTypeIndex> {
+        assert!(self.defining_rec_group.is_none());
+        assert_eq!(validator_types.id(), self.validator_id);
+
+        let rec_group_id = validator_types.rec_group_id_of(id);
+        debug_assert!(validator_types
+            .rec_group_elements(rec_group_id)
+            .any(|e| e == id));
+
+        let interned_rec_group = self.intern_rec_group(module, validator_types, rec_group_id)?;
+
+        let interned_type = self.wasmparser_to_wasmtime[&id];
+        debug_assert!(self
+            .rec_group_elements(interned_rec_group)
+            .any(|e| e == interned_type));
+
+        Ok(interned_type)
+    }
+
+    /// Define a new Wasm function type while we are defining a rec group.
+    fn wasm_func_type_in_rec_group(
+        &mut self,
+        id: wasmparser::types::CoreTypeId,
+        func_ty: WasmFuncType,
+    ) -> ModuleInternedTypeIndex {
+        assert!(
+            self.defining_rec_group.is_some(),
+            "must be defining a rec group to define new types"
+        );
+
+        let module_interned_index = self.types.wasm_types.push(func_ty);
+        debug_assert_eq!(
+            self.wasmparser_to_wasmtime.get(&id),
+            Some(&module_interned_index),
+            "should have reserved the right module-interned index for this wasmparser type already"
+        );
+
+        module_interned_index
     }
 
     /// Returns the result [`ModuleTypes`] of this builder.
@@ -74,6 +301,14 @@ impl ModuleTypesBuilder {
         self.types
     }
 
+    /// Get the elements within an already-defined rec group.
+    pub fn rec_group_elements(
+        &self,
+        rec_group: ModuleInternedRecGroupIndex,
+    ) -> impl ExactSizeIterator<Item = ModuleInternedTypeIndex> {
+        self.types.rec_group_elements(rec_group)
+    }
+
     /// Returns an iterator over all the wasm function signatures found within
     /// this module.
     pub fn wasm_signatures(
@@ -95,10 +330,17 @@ where
     }
 }
 
-#[allow(missing_docs)]
+/// A convert from `wasmparser` types to Wasmtime types.
 pub struct WasmparserTypeConverter<'a> {
-    pub types: &'a ModuleTypesBuilder,
-    pub module: &'a Module,
+    types: &'a ModuleTypesBuilder,
+    module: &'a Module,
+}
+
+impl<'a> WasmparserTypeConverter<'a> {
+    /// Construct a new type converter from `wasmparser` types to Wasmtime types.
+    pub fn new(types: &'a ModuleTypesBuilder, module: &'a Module) -> Self {
+        Self { types, module }
+    }
 }
 
 impl TypeConvert for WasmparserTypeConverter<'_> {
@@ -108,15 +350,12 @@ impl TypeConvert for WasmparserTypeConverter<'_> {
                 let signature = self.types.wasmparser_to_wasmtime[&id];
                 WasmHeapType::Concrete(EngineOrModuleTypeIndex::Module(signature))
             }
-            UnpackedIndex::RecGroup(_) => unreachable!(),
-            UnpackedIndex::Module(i) => {
-                let i = TypeIndex::from_u32(i);
-                match self.module.types[i] {
-                    ModuleType::Function(sig) => {
-                        WasmHeapType::Concrete(EngineOrModuleTypeIndex::Module(sig))
-                    }
-                }
+            UnpackedIndex::Module(module_index) => {
+                let module_index = TypeIndex::from_u32(module_index);
+                let interned_index = self.module.types[module_index];
+                WasmHeapType::Concrete(EngineOrModuleTypeIndex::Module(interned_index))
             }
+            UnpackedIndex::RecGroup(_) => unreachable!(),
         }
     }
 }
diff --git a/crates/runtime/src/send_sync_ptr.rs b/crates/runtime/src/send_sync_ptr.rs
index 7dcea5815..f4975fef0 100644
--- a/crates/runtime/src/send_sync_ptr.rs
+++ b/crates/runtime/src/send_sync_ptr.rs
@@ -93,7 +93,7 @@ impl<T: ?Sized> Copy for SendSyncPtr<T> {}
 impl<T: ?Sized> PartialEq for SendSyncPtr<T> {
     #[inline]
     fn eq(&self, other: &SendSyncPtr<T>) -> bool {
-        self.0 == other.0
+        std::ptr::eq(self.0.as_ptr(), other.0.as_ptr())
     }
 }
 
diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs
index 81d1c5e70..b9c7a67fb 100644
--- a/crates/types/src/lib.rs
+++ b/crates/types/src/lib.rs
@@ -5,7 +5,7 @@ pub use wasmparser;
 
 use cranelift_entity::entity_impl;
 use serde_derive::{Deserialize, Serialize};
-use std::fmt;
+use std::{fmt, ops::Range};
 
 mod error;
 pub use error::*;
@@ -29,10 +29,28 @@ pub trait TypeTrace {
     where
         F: FnMut(&mut EngineOrModuleTypeIndex) -> Result<(), E>;
 
+    /// Trace all `VMSharedTypeIndex` edges, ignoring other edges.
+    fn trace_engine_indices<F, E>(&self, func: &mut F) -> Result<(), E>
+    where
+        F: FnMut(VMSharedTypeIndex) -> Result<(), E>,
+    {
+        self.trace(&mut |idx| match idx {
+            EngineOrModuleTypeIndex::Engine(idx) => func(idx),
+            EngineOrModuleTypeIndex::Module(_) | EngineOrModuleTypeIndex::RecGroup(_) => Ok(()),
+        })
+    }
+
     /// Canonicalize `self` by rewriting all type references inside `self` from
     /// module-level interned type indices to engine-level interned type
     /// indices.
-    fn canonicalize<F>(&mut self, module_to_engine: &mut F)
+    ///
+    /// This produces types that are suitable for usage by the runtime (only
+    /// contains `VMSharedTypeIndex` type references).
+    ///
+    /// This does not produce types that are suitable for hash consing types
+    /// (must have recgroup-relative indices for type indices referencing other
+    /// types in the same recgroup).
+    fn canonicalize_for_runtime_usage<F>(&mut self, module_to_engine: &mut F)
     where
         F: FnMut(ModuleInternedTypeIndex) -> VMSharedTypeIndex,
     {
@@ -43,9 +61,72 @@ pub trait TypeTrace {
                 *idx = EngineOrModuleTypeIndex::Engine(engine_index);
                 Ok(())
             }
+            EngineOrModuleTypeIndex::RecGroup(_) => {
+                panic!("should not already be canonicalized for hash consing")
+            }
         })
         .unwrap()
     }
+
+    /// Is this type canonicalized for runtime usage?
+    fn is_canonicalized_for_runtime_usage(&self) -> bool {
+        self.trace(&mut |idx| match idx {
+            EngineOrModuleTypeIndex::Engine(_) => Ok(()),
+            EngineOrModuleTypeIndex::Module(_) | EngineOrModuleTypeIndex::RecGroup(_) => Err(()),
+        })
+        .is_ok()
+    }
+
+    /// Canonicalize `self` by rewriting all type references inside `self` from
+    /// module-level interned type indices to either engine-level interned type
+    /// indices or recgroup-relative indices.
+    ///
+    /// This produces types that are suitable for hash consing and deduplicating
+    /// recgroups (types may have recgroup-relative indices for references to
+    /// other types within the same recgroup).
+    ///
+    /// This does *not* produce types that are suitable for usage by the runtime
+    /// (only contain `VMSharedTypeIndex` type references).
+    fn canonicalize_for_hash_consing<F>(
+        &mut self,
+        rec_group_range: Range<ModuleInternedTypeIndex>,
+        module_to_engine: &mut F,
+    ) where
+        F: FnMut(ModuleInternedTypeIndex) -> VMSharedTypeIndex,
+    {
+        self.trace_mut::<_, ()>(&mut |idx| match *idx {
+            EngineOrModuleTypeIndex::Engine(_) => Ok(()),
+            EngineOrModuleTypeIndex::Module(module_index) => {
+                *idx = if rec_group_range.start <= module_index {
+                    // Any module index within the recursion group gets
+                    // translated into a recgroup-relative index.
+                    debug_assert!(module_index < rec_group_range.end);
+                    let relative = module_index.as_u32() - rec_group_range.start.as_u32();
+                    let relative = RecGroupRelativeTypeIndex::from_u32(relative);
+                    EngineOrModuleTypeIndex::RecGroup(relative)
+                } else {
+                    // Cross-group indices are translated directly into
+                    // `VMSharedTypeIndex`es.
+                    debug_assert!(module_index < rec_group_range.start);
+                    EngineOrModuleTypeIndex::Engine(module_to_engine(module_index))
+                };
+                Ok(())
+            }
+            EngineOrModuleTypeIndex::RecGroup(_) => {
+                panic!("should not already be canonicalized for hash consing")
+            }
+        })
+        .unwrap()
+    }
+
+    /// Is this type canonicalized for hash consing?
+    fn is_canonicalized_for_hash_consing(&self) -> bool {
+        self.trace(&mut |idx| match idx {
+            EngineOrModuleTypeIndex::Engine(_) | EngineOrModuleTypeIndex::RecGroup(_) => Ok(()),
+            EngineOrModuleTypeIndex::Module(_) => Err(()),
+        })
+        .is_ok()
+    }
 }
 
 /// WebAssembly value type -- equivalent of `wasmparser::ValType`.
@@ -193,9 +274,15 @@ pub enum EngineOrModuleTypeIndex {
     /// An index within an engine, canonicalized among all modules that can
     /// interact with each other.
     Engine(VMSharedTypeIndex),
+
     /// An index within the current Wasm module, canonicalized within just this
     /// current module.
     Module(ModuleInternedTypeIndex),
+
+    /// An index within the containing type's rec group. This is only used when
+    /// hashing and canonicalizing rec groups, and should never appear outside
+    /// of the engine's type registry.
+    RecGroup(RecGroupRelativeTypeIndex),
 }
 
 impl From<ModuleInternedTypeIndex> for EngineOrModuleTypeIndex {
@@ -209,6 +296,7 @@ impl fmt::Display for EngineOrModuleTypeIndex {
         match self {
             Self::Engine(i) => write!(f, "(engine {})", i.bits()),
             Self::Module(i) => write!(f, "(module {})", i.as_u32()),
+            Self::RecGroup(i) => write!(f, "(recgroup {})", i.as_u32()),
         }
     }
 }
@@ -223,14 +311,14 @@ impl EngineOrModuleTypeIndex {
     pub fn as_engine_type_index(self) -> Option<VMSharedTypeIndex> {
         match self {
             Self::Engine(e) => Some(e),
-            Self::Module(_) => None,
+            Self::RecGroup(_) | Self::Module(_) => None,
         }
     }
 
     /// Get the underlying engine-level type index, or panic.
     pub fn unwrap_engine_type_index(self) -> VMSharedTypeIndex {
         self.as_engine_type_index()
-            .expect("`unwrap_engine_type_index` on module type index")
+            .unwrap_or_else(|| panic!("`unwrap_engine_type_index` on {self:?}"))
     }
 
     /// Is this an module-level type index?
@@ -242,14 +330,33 @@ impl EngineOrModuleTypeIndex {
     pub fn as_module_type_index(self) -> Option<ModuleInternedTypeIndex> {
         match self {
             Self::Module(e) => Some(e),
-            Self::Engine(_) => None,
+            Self::RecGroup(_) | Self::Engine(_) => None,
         }
     }
 
     /// Get the underlying module-level type index, or panic.
     pub fn unwrap_module_type_index(self) -> ModuleInternedTypeIndex {
         self.as_module_type_index()
-            .expect("`unwrap_module_type_index` on engine type index")
+            .unwrap_or_else(|| panic!("`unwrap_module_type_index` on {self:?}"))
+    }
+
+    /// Is this an recgroup-level type index?
+    pub fn is_rec_group_type_index(self) -> bool {
+        matches!(self, Self::RecGroup(_))
+    }
+
+    /// Get the underlying recgroup-level type index, if any.
+    pub fn as_rec_group_type_index(self) -> Option<RecGroupRelativeTypeIndex> {
+        match self {
+            Self::RecGroup(r) => Some(r),
+            Self::Module(_) | Self::Engine(_) => None,
+        }
+    }
+
+    /// Get the underlying module-level type index, or panic.
+    pub fn unwrap_rec_group_type_index(self) -> RecGroupRelativeTypeIndex {
+        self.as_rec_group_type_index()
+            .unwrap_or_else(|| panic!("`unwrap_rec_group_type_index` on {self:?}"))
     }
 }
 
@@ -406,6 +513,44 @@ impl WasmFuncType {
     }
 }
 
+/// A recursive type group.
+///
+/// Types within a recgroup can have forward references to each other, which
+/// allows for cyclic types, for example a function `$f` that returns a
+/// reference to a function `$g` which returns a reference to a function `$f`:
+///
+/// ```ignore
+/// (rec (type (func $f (result (ref null $g))))
+///      (type (func $g (result (ref null $f)))))
+/// ```
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
+pub struct WasmRecGroup {
+    /// The types inside of this recgroup.
+    pub types: Box<[WasmFuncType]>,
+}
+
+impl TypeTrace for WasmRecGroup {
+    fn trace<F, E>(&self, func: &mut F) -> Result<(), E>
+    where
+        F: FnMut(EngineOrModuleTypeIndex) -> Result<(), E>,
+    {
+        for ty in self.types.iter() {
+            ty.trace(func)?;
+        }
+        Ok(())
+    }
+
+    fn trace_mut<F, E>(&mut self, func: &mut F) -> Result<(), E>
+    where
+        F: FnMut(&mut EngineOrModuleTypeIndex) -> Result<(), E>,
+    {
+        for ty in self.types.iter_mut() {
+            ty.trace_mut(func)?;
+        }
+        Ok(())
+    }
+}
+
 /// Index type of a function (imported or defined) inside the WebAssembly module.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
 pub struct FuncIndex(u32);
@@ -451,11 +596,32 @@ entity_impl!(GlobalIndex);
 pub struct MemoryIndex(u32);
 entity_impl!(MemoryIndex);
 
+/// Index type of a canonicalized recursive type group inside a WebAssembly
+/// module (as opposed to canonicalized within the whole engine).
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
+pub struct ModuleInternedRecGroupIndex(u32);
+entity_impl!(ModuleInternedRecGroupIndex);
+
+/// Index type of a canonicalized recursive type group inside the whole engine
+/// (as opposed to canonicalized within just a single Wasm module).
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
+pub struct EngineInternedRecGroupIndex(u32);
+entity_impl!(EngineInternedRecGroupIndex);
+
 /// Index type of a type (imported or defined) inside the WebAssembly module.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
 pub struct TypeIndex(u32);
 entity_impl!(TypeIndex);
 
+/// A canonicalized type index referencing a type within a single recursion
+/// group from another type within that same recursion group.
+///
+/// This is only suitable for use when hash consing and deduplicating rec
+/// groups.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
+pub struct RecGroupRelativeTypeIndex(u32);
+entity_impl!(RecGroupRelativeTypeIndex);
+
 /// A canonicalized type index for a type within a single WebAssembly module.
 ///
 /// Note that this is deduplicated only at the level of a single WebAssembly
diff --git a/crates/wasi-common/src/snapshots/preview_1.rs b/crates/wasi-common/src/snapshots/preview_1.rs
index b73d3a5ec..a58312651 100644
--- a/crates/wasi-common/src/snapshots/preview_1.rs
+++ b/crates/wasi-common/src/snapshots/preview_1.rs
@@ -340,13 +340,17 @@ impl wasi_snapshot_preview1::WasiSnapshotPreview1 for WasiCtx {
                 return Ok(0);
             }
         } else {
-            // Convert all of the unsafe guest slices to safe ones--this uses
-            // Wiggle's internal borrow checker to ensure no overlaps. We assume
-            // here that, because the memory is not shared, there are no other
-            // threads to access it while it is written to.
+            // Convert the first unsafe guest slice into a safe one--Wiggle
+            // can only track mutable borrows for an entire region, and converting
+            // all guest pointers to slices would cause a runtime borrow-checking
+            // error. As read is allowed to return less than the requested amount,
+            // it's valid (though not as efficient) for us to only perform the
+            // read of the first buffer.
             let mut guest_slices: Vec<wiggle::GuestSliceMut<u8>> = iovs
                 .into_iter()
+                .filter(|iov| iov.len() > 0)
                 .map(|iov| Ok(iov.as_slice_mut()?.unwrap()))
+                .take(1)
                 .collect::<Result<_, Error>>()?;
 
             // Read directly into the Wasm memory.
diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs
index c2994850b..15e316e31 100644
--- a/crates/wasmtime/src/compile.rs
+++ b/crates/wasmtime/src/compile.rs
@@ -34,8 +34,8 @@ use wasmtime_environ::component::Translator;
 use wasmtime_environ::{
     BuiltinFunctionIndex, CompiledFunctionInfo, CompiledModuleInfo, Compiler, DefinedFuncIndex,
     FinishedObject, FunctionBodyData, ModuleEnvironment, ModuleInternedTypeIndex,
-    ModuleTranslation, ModuleType, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap,
-    RelocationTarget, StaticModuleIndex, WasmFunctionInfo,
+    ModuleTranslation, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap, RelocationTarget,
+    StaticModuleIndex, WasmFunctionInfo,
 };
 
 mod code_builder;
@@ -68,9 +68,9 @@ pub(crate) fn build_artifacts<T: FinishedObject>(
     // about the wasm module. This is where the WebAssembly is parsed and
     // validated. Afterwards `types` will have all the type information for
     // this module.
-    let mut validator = wasmparser::Validator::new_with_features(engine.config().features.clone());
     let parser = wasmparser::Parser::new(0);
-    let mut types = Default::default();
+    let mut validator = wasmparser::Validator::new_with_features(engine.config().features.clone());
+    let mut types = ModuleTypesBuilder::new(&validator);
     let mut translation = ModuleEnvironment::new(tunables, &mut validator, &mut types)
         .translate(parser, wasm)
         .context("failed to parse WebAssembly module")?;
@@ -122,7 +122,9 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
     engine: &Engine,
     binary: &[u8],
 ) -> Result<(T, Option<wasmtime_environ::component::ComponentArtifacts>)> {
-    use wasmtime_environ::component::{CompiledComponentInfo, ComponentArtifacts};
+    use wasmtime_environ::component::{
+        CompiledComponentInfo, ComponentArtifacts, ComponentTypesBuilder,
+    };
     use wasmtime_environ::ScopeVec;
 
     let tunables = engine.tunables();
@@ -130,7 +132,7 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
 
     let scope = ScopeVec::new();
     let mut validator = wasmparser::Validator::new_with_features(engine.config().features.clone());
-    let mut types = Default::default();
+    let mut types = ComponentTypesBuilder::new(&validator);
     let (component, mut module_translations) =
         Translator::new(tunables, &mut validator, &mut types, &scope)
             .translate(binary)
@@ -481,9 +483,7 @@ impl<'a> CompileInputs<'a> {
                 }
             }
 
-            sigs.extend(translation.module.types.iter().map(|(_, ty)| match ty {
-                ModuleType::Function(ty) => *ty,
-            }));
+            sigs.extend(translation.module.types.iter().map(|(_, ty)| *ty));
         }
 
         for signature in sigs {
@@ -825,9 +825,7 @@ impl FunctionIndices {
                     .module
                     .types
                     .iter()
-                    .map(|(_, ty)| match ty {
-                        ModuleType::Function(ty) => *ty,
-                    })
+                    .map(|(_, ty)| *ty)
                     .collect::<BTreeSet<_>>();
                 let wasm_to_native_trampolines = unique_and_sorted_sigs
                     .iter()
diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rs
index 6ade89cd4..7a10c3cc6 100644
--- a/crates/wasmtime/src/runtime/component/instance.rs
+++ b/crates/wasmtime/src/runtime/component/instance.rs
@@ -319,11 +319,11 @@ impl<'a> Instantiator<'a> {
         // below, but the funcrefs can all be configured here.
         for (idx, sig) in env_component.trampolines.iter() {
             let ptrs = self.component.trampoline_ptrs(idx);
-            let signature = self
-                .component
-                .signatures()
-                .shared_type(*sig)
-                .expect("found unregistered signature");
+            let signature = match self.component.signatures().shared_type(*sig) {
+                Some(s) => s,
+                None => panic!("found unregistered signature: {sig:?}"),
+            };
+
             self.data.state.set_trampoline(
                 idx,
                 ptrs.wasm_call,
@@ -468,8 +468,8 @@ impl<'a> Instantiator<'a> {
             // made, so double-check this property of wasmtime in debug mode.
 
             if cfg!(debug_assertions) {
-                let (_, _, expected) = imports.next().unwrap();
-                self.assert_type_matches(store, module, arg, expected);
+                let (imp_module, imp_name, expected) = imports.next().unwrap();
+                self.assert_type_matches(store, module, arg, imp_module, imp_name, expected);
             }
 
             // The unsafety here should be ok since the `export` is loaded
@@ -490,6 +490,8 @@ impl<'a> Instantiator<'a> {
         store: &mut StoreOpaque,
         module: &Module,
         arg: &CoreDef,
+        imp_module: &str,
+        imp_name: &str,
         expected: EntityType,
     ) {
         let export = self.data.lookup_def(store, arg);
@@ -502,9 +504,18 @@ impl<'a> Instantiator<'a> {
             let expected = match expected.unwrap_func() {
                 EngineOrModuleTypeIndex::Engine(e) => Some(e),
                 EngineOrModuleTypeIndex::Module(m) => module.signatures().shared_type(m),
+                EngineOrModuleTypeIndex::RecGroup(_) => unreachable!(),
             };
             let actual = unsafe { f.func_ref.as_ref().type_index };
-            assert_eq!(expected, Some(actual));
+            assert_eq!(
+                expected,
+                Some(actual),
+                "type mismatch for import {imp_module:?} {imp_name:?}!!!\n\n\
+                 expected {:#?}\n\n\
+                 found {:#?}",
+                expected.and_then(|e| store.engine().signatures().borrow(e)),
+                store.engine().signatures().borrow(actual)
+            );
             return;
         }
 
diff --git a/crates/wasmtime/src/runtime/component/linker.rs b/crates/wasmtime/src/runtime/component/linker.rs
index 5c3847082..4a885f027 100644
--- a/crates/wasmtime/src/runtime/component/linker.rs
+++ b/crates/wasmtime/src/runtime/component/linker.rs
@@ -195,7 +195,6 @@ impl<T> Linker<T> {
 
     fn typecheck<'a>(&'a self, component: &'a Component) -> Result<TypeChecker<'a>> {
         let mut cx = TypeChecker {
-            component: component.env_component(),
             types: component.types(),
             strings: &self.strings,
             imported_resources: Default::default(),
diff --git a/crates/wasmtime/src/runtime/component/matching.rs b/crates/wasmtime/src/runtime/component/matching.rs
index 8fb2c02c7..ed97896dc 100644
--- a/crates/wasmtime/src/runtime/component/matching.rs
+++ b/crates/wasmtime/src/runtime/component/matching.rs
@@ -15,7 +15,6 @@ use wasmtime_runtime::component::ComponentInstance;
 
 pub struct TypeChecker<'a> {
     pub types: &'a Arc<ComponentTypes>,
-    pub component: &'a wasmtime_environ::component::Component,
     pub strings: &'a Strings,
     pub imported_resources: Arc<PrimaryMap<ResourceIndex, ResourceType>>,
 }
diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs
index 2b471c161..5f78ca9ad 100644
--- a/crates/wasmtime/src/runtime/externals/global.rs
+++ b/crates/wasmtime/src/runtime/externals/global.rs
@@ -227,7 +227,7 @@ impl Global {
         wasmtime_export
             .global
             .wasm_ty
-            .canonicalize(&mut |module_index| {
+            .canonicalize_for_runtime_usage(&mut |module_index| {
                 wasmtime_runtime::Instance::from_vmctx(wasmtime_export.vmctx, |instance| {
                     instance.engine_type_index(module_index)
                 })
diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs
index 648362178..81b386f76 100644
--- a/crates/wasmtime/src/runtime/externals/table.rs
+++ b/crates/wasmtime/src/runtime/externals/table.rs
@@ -403,7 +403,7 @@ impl Table {
             .table
             .table
             .wasm_ty
-            .canonicalize(&mut |module_index| {
+            .canonicalize_for_runtime_usage(&mut |module_index| {
                 wasmtime_runtime::Instance::from_vmctx(wasmtime_export.vmctx, |instance| {
                     instance.engine_type_index(module_index)
                 })
diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs
index 5d55fb888..c018d864d 100644
--- a/crates/wasmtime/src/runtime/instance.rs
+++ b/crates/wasmtime/src/runtime/instance.rs
@@ -982,7 +982,7 @@ fn typecheck<I>(
     }
     let cx = matching::MatchCx::new(module.engine());
     for ((name, field, mut expected_ty), actual) in env_module.imports().zip(imports) {
-        expected_ty.canonicalize(&mut |module_index| {
+        expected_ty.canonicalize_for_runtime_usage(&mut |module_index| {
             module.signatures().shared_type(module_index).unwrap()
         });
 
diff --git a/crates/wasmtime/src/runtime/trampoline/table.rs b/crates/wasmtime/src/runtime/trampoline/table.rs
index 5950b51ae..4128c8fc4 100644
--- a/crates/wasmtime/src/runtime/trampoline/table.rs
+++ b/crates/wasmtime/src/runtime/trampoline/table.rs
@@ -2,7 +2,7 @@ use crate::store::{InstanceId, StoreOpaque};
 use crate::trampoline::create_handle;
 use crate::TableType;
 use anyhow::Result;
-use wasmtime_environ::{EngineOrModuleTypeIndex, EntityIndex, Module, TypeTrace};
+use wasmtime_environ::{EntityIndex, Module, TypeTrace};
 
 pub fn create_table(store: &mut StoreOpaque, table: &TableType) -> Result<InstanceId> {
     let mut module = Module::new();
@@ -10,17 +10,11 @@ pub fn create_table(store: &mut StoreOpaque, table: &TableType) -> Result<Instan
     let wasmtime_table = table.wasmtime_table().clone();
     let tunables = store.engine().tunables();
 
-    if cfg!(debug_assertions) {
-        wasmtime_table
-            .wasm_ty
-            .trace(&mut |idx| match idx {
-                EngineOrModuleTypeIndex::Engine(_) => Ok(()),
-                EngineOrModuleTypeIndex::Module(module_idx) => Err(format!(
-                    "found module-level canonicalized type index: {module_idx:?}"
-                )),
-            })
-            .expect("element type should be engine-level canonicalized");
-    }
+    debug_assert!(
+        wasmtime_table.wasm_ty.is_canonicalized_for_runtime_usage(),
+        "should be canonicalized for runtime usage: {:?}",
+        wasmtime_table.wasm_ty
+    );
 
     let table_plan = wasmtime_environ::TablePlan::for_table(wasmtime_table, tunables);
     let table_id = module.table_plans.push(table_plan);
diff --git a/crates/wasmtime/src/runtime/type_registry.rs b/crates/wasmtime/src/runtime/type_registry.rs
index a8c1b291b..4262a40eb 100644
--- a/crates/wasmtime/src/runtime/type_registry.rs
+++ b/crates/wasmtime/src/runtime/type_registry.rs
@@ -1,7 +1,7 @@
 //! Implement a registry of types: function, struct, and array definitions.
 //!
 //! Helps implement fast indirect call signature checking, reference type
-//! casting, and etc.
+//! downcasting, and etc...
 
 use crate::Engine;
 use std::{
@@ -9,7 +9,7 @@ use std::{
     collections::{HashMap, HashSet},
     fmt::Debug,
     hash::{Hash, Hasher},
-    ops::Deref,
+    ops::Range,
     sync::{
         atomic::{
             AtomicUsize,
@@ -19,18 +19,13 @@ use std::{
     },
 };
 use wasmtime_environ::{
-    EngineOrModuleTypeIndex, ModuleInternedTypeIndex, ModuleTypes, PrimaryMap, TypeTrace,
-    VMSharedTypeIndex, WasmFuncType,
+    iter_entity_range, EngineOrModuleTypeIndex, ModuleInternedTypeIndex, ModuleTypes, PrimaryMap,
+    TypeTrace, VMSharedTypeIndex, WasmFuncType, WasmRecGroup,
 };
 use wasmtime_slab::{Id as SlabId, Slab};
 
 // ### Notes on the Lifetime Management of Types
 //
-// (The below refers to recursion groups even though at the time of writing we
-// don't support Wasm GC yet, which introduced recursion groups. Until that
-// time, you can think of each type implicitly being in a singleton recursion
-// group, so types and recursion groups are effectively one to one.)
-//
 // All defined types from all Wasm modules loaded into Wasmtime are interned
 // into their engine's `TypeRegistry`.
 //
@@ -38,18 +33,18 @@ use wasmtime_slab::{Id as SlabId, Slab};
 // cared about canonicalizing types so that `call_indirect` was fast and we
 // didn't waste memory on many copies of the same function type definition.
 // Function types could only take and return simple scalars (i32/f64/etc...) and
-// there were no type-to-type references. We could simply deduplicate types and
-// reference count their entries in the registry.
+// there were no type-to-type references. We could simply deduplicate function
+// types and reference count their entries in the registry.
 //
 // The typed function references and GC proposals change everything. The former
 // introduced function types that take a reference to a function of another
 // specific type. This is a type-to-type reference. The latter introduces struct
 // and array types that can have references to other struct, array, and function
 // types, as well as recursion groups that allow cyclic references between
-// types. Now type canonicalization additionally enables fast type checks
-// *across* modules: so that two modules which define the same struct type, for
-// example, can pass instances of that struct type to each other, and we can
-// quickly check that those instances are in fact of the expected types.
+// types. Now type canonicalization additionally enables fast type checks and
+// downcasts *across* modules: so that two modules which define the same struct
+// type, for example, can pass instances of that struct type to each other, and
+// we can quickly check that those instances are in fact of the expected types.
 //
 // But how do we manage the lifetimes of types that can reference other types as
 // Wasm modules are dynamically loaded and unloaded from the engine? These
@@ -87,6 +82,7 @@ use wasmtime_slab::{Id as SlabId, Slab};
 /// when dropped.
 pub struct TypeCollection {
     engine: Engine,
+    rec_groups: Vec<RecGroupEntry>,
     types: PrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,
     reverse_types: HashMap<VMSharedTypeIndex, ModuleInternedTypeIndex>,
 }
@@ -95,10 +91,12 @@ impl Debug for TypeCollection {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let TypeCollection {
             engine: _,
+            rec_groups,
             types,
             reverse_types: _,
         } = self;
         f.debug_struct("TypeCollection")
+            .field("rec_groups", rec_groups)
             .field("types", types)
             .finish_non_exhaustive()
     }
@@ -109,11 +107,12 @@ impl TypeCollection {
     pub fn new_for_module(engine: &Engine, types: &ModuleTypes) -> Self {
         let engine = engine.clone();
         let registry = engine.signatures();
-        let types = registry.0.write().unwrap().register_for_module(types);
+        let (rec_groups, types) = registry.0.write().unwrap().register_module_types(types);
         let reverse_types = types.iter().map(|(k, v)| (*v, k)).collect();
 
         Self {
             engine,
+            rec_groups,
             types,
             reverse_types,
         }
@@ -142,7 +141,7 @@ impl TypeCollection {
 
 impl Drop for TypeCollection {
     fn drop(&mut self) {
-        if !self.types.is_empty() {
+        if !self.rec_groups.is_empty() {
             self.engine
                 .signatures()
                 .0
@@ -173,14 +172,22 @@ fn slab_id_to_shared_type_index(id: SlabId) -> VMSharedTypeIndex {
 /// Dereferences to its underlying `WasmFuncType`.
 pub struct RegisteredType {
     engine: Engine,
-    entry: Entry,
+    entry: RecGroupEntry,
+    ty: Arc<WasmFuncType>,
+    index: VMSharedTypeIndex,
 }
 
 impl Debug for RegisteredType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let RegisteredType { engine: _, entry } = self;
+        let RegisteredType {
+            engine: _,
+            entry: _,
+            ty,
+            index,
+        } = self;
         f.debug_struct("RegisteredType")
-            .field("entry", entry)
+            .field("index", index)
+            .field("ty", ty)
             .finish_non_exhaustive()
     }
 }
@@ -191,6 +198,8 @@ impl Clone for RegisteredType {
         RegisteredType {
             engine: self.engine.clone(),
             entry: self.entry.clone(),
+            ty: self.ty.clone(),
+            index: self.index,
         }
     }
 }
@@ -203,7 +212,7 @@ impl Drop for RegisteredType {
                 .0
                 .write()
                 .unwrap()
-                .unregister_entry(self.entry.0.index);
+                .unregister_entry(self.entry.clone());
         }
     }
 }
@@ -212,7 +221,7 @@ impl std::ops::Deref for RegisteredType {
     type Target = WasmFuncType;
 
     fn deref(&self) -> &Self::Target {
-        &self.entry.0.ty
+        &self.ty
     }
 }
 
@@ -223,11 +232,9 @@ impl PartialEq for RegisteredType {
         if cfg!(debug_assertions) {
             if eq {
                 assert!(Engine::same(&self.engine, &other.engine));
+                assert_eq!(self.ty, other.ty);
             } else {
-                assert!(
-                    self.entry.0.index != other.entry.0.index
-                        || !Engine::same(&self.engine, &other.engine)
-                );
+                assert!(self.ty != other.ty || !Engine::same(&self.engine, &other.engine));
             }
         }
 
@@ -248,23 +255,29 @@ impl RegisteredType {
     /// Constructs a new `RegisteredType`, registering the given type with the
     /// engine's `TypeRegistry`.
     pub fn new(engine: &Engine, ty: WasmFuncType) -> RegisteredType {
-        let entry = {
+        let (entry, index, ty) = {
+            log::trace!("RegisteredType::new({ty:?})");
+
             let mut inner = engine.signatures().0.write().unwrap();
 
-            log::trace!("RegisteredType::new({ty:?})");
+            // It shouldn't be possible for users to construct non-canonical
+            // types via the embedding API, and the only other types they can
+            // get are already-canonicalized types from modules, so we shouldn't
+            // ever get non-canonical types here. Furthermore, this is only
+            // called internally to Wasmtime, so we shouldn't ever have an
+            // engine mismatch; those should be caught earlier.
+            inner.assert_canonicalized_for_runtime_usage_in_this_registry(&ty);
 
-            // It shouldn't be possible for users to construct non-canonical types
-            // via the embedding API, and the only other types they can get are
-            // already-canonicalized types from modules, so we shouldn't ever get
-            // non-canonical types here.
-            assert!(
-                inner.is_canonicalized(&ty),
-                "ty is not already canonicalized: {ty:?}"
-            );
+            let entry = inner.register_singleton_rec_group(ty);
 
-            inner.register_canonicalized(ty)
+            let index = entry.0.shared_type_indices[0];
+            let id = shared_type_index_to_slab_id(index);
+            let ty = inner.types[id].clone();
+
+            (entry, index, ty)
         };
-        RegisteredType::from_parts(engine.clone(), entry)
+
+        RegisteredType::from_parts(engine.clone(), entry, index, ty)
     }
 
     /// Create an owning handle to the given index's associated type.
@@ -275,10 +288,12 @@ impl RegisteredType {
     /// Returns `None` if `index` is not registered in the given engine's
     /// registry.
     pub fn root(engine: &Engine, index: VMSharedTypeIndex) -> Option<RegisteredType> {
-        let entry = {
+        let (entry, ty) = {
             let id = shared_type_index_to_slab_id(index);
             let inner = engine.signatures().0.read().unwrap();
-            let e = inner.entries.get(id)?;
+
+            let ty = inner.types.get(id)?.clone();
+            let entry = inner.type_to_rec_group[&index].clone();
 
             // NB: make sure to incref while the lock is held to prevent:
             //
@@ -286,26 +301,36 @@ impl RegisteredType {
             // * Other thread: drops `RegisteredType` for entry E, decref
             //   reaches zero, write locks registry, unregisters entry
             // * This thread: increfs entry, but it isn't in the registry anymore
-            e.incref("RegisteredType::root");
+            entry.incref("RegisteredType::root");
 
-            e.clone()
+            (entry, ty)
         };
 
-        Some(RegisteredType::from_parts(engine.clone(), entry))
+        Some(RegisteredType::from_parts(engine.clone(), entry, index, ty))
     }
 
     /// Construct a new `RegisteredType`.
     ///
     /// It is the caller's responsibility to ensure that the entry's reference
     /// count has already been incremented.
-    fn from_parts(engine: Engine, entry: Entry) -> Self {
+    fn from_parts(
+        engine: Engine,
+        entry: RecGroupEntry,
+        index: VMSharedTypeIndex,
+        ty: Arc<WasmFuncType>,
+    ) -> Self {
         debug_assert!(entry.0.registrations.load(Acquire) != 0);
-        RegisteredType { engine, entry }
+        RegisteredType {
+            engine,
+            entry,
+            ty,
+            index,
+        }
     }
 
     /// Get this registered type's index.
     pub fn index(&self) -> VMSharedTypeIndex {
-        self.entry.0.index
+        self.index
     }
 
     /// Get the engine whose registry this type is registered within.
@@ -314,55 +339,66 @@ impl RegisteredType {
     }
 }
 
-/// A Wasm function type, its `VMSharedTypeIndex`, and its registration count.
-#[derive(Debug)]
-struct EntryInner {
-    ty: WasmFuncType,
-    index: VMSharedTypeIndex,
-    registrations: AtomicUsize,
-}
-
+/// An entry in the type registry.
+///
 /// Implements `Borrow`, `Eq`, and `Hash` by forwarding to the underlying Wasm
-/// function type, so that this can be a hash consing key in
-/// `TypeRegistryInner::map`.
-#[derive(Clone, Debug)]
-struct Entry(Arc<EntryInner>);
+/// rec group, so that this can be a hash consing key. (We can't use
+/// `Arc<RecGroupEntryInner>` directly for this purpose because `Arc<T>` doesn't
+/// implement `Borrow<U>` when `T: Borrow<U>`).
+#[derive(Clone)]
+struct RecGroupEntry(Arc<RecGroupEntryInner>);
 
-impl Deref for Entry {
-    type Target = WasmFuncType;
+impl Debug for RecGroupEntry {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        struct Ptr<'a, P>(&'a P);
+        impl<P: std::fmt::Pointer> Debug for Ptr<'_, P> {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                write!(f, "{:#p}", *self.0)
+            }
+        }
 
-    fn deref(&self) -> &Self::Target {
-        &self.0.ty
+        f.debug_struct("RecGroupEntry")
+            .field("ptr", &Ptr(&self.0))
+            .field("shared_type_indices", &self.0.shared_type_indices)
+            .field("hash_consing_key", &self.0.hash_consing_key)
+            .field("registrations", &self.0.registrations.load(Acquire))
+            .finish()
     }
 }
 
-impl PartialEq for Entry {
+struct RecGroupEntryInner {
+    /// The Wasm rec group, canonicalized for hash consing.
+    hash_consing_key: WasmRecGroup,
+    shared_type_indices: Box<[VMSharedTypeIndex]>,
+    registrations: AtomicUsize,
+}
+
+impl PartialEq for RecGroupEntry {
     fn eq(&self, other: &Self) -> bool {
-        self.0.ty == other.0.ty
+        self.0.hash_consing_key == other.0.hash_consing_key
     }
 }
 
-impl Eq for Entry {}
+impl Eq for RecGroupEntry {}
 
-impl Hash for Entry {
+impl Hash for RecGroupEntry {
     fn hash<H: Hasher>(&self, state: &mut H) {
-        self.0.ty.hash(state);
+        self.0.hash_consing_key.hash(state);
     }
 }
 
-impl Borrow<WasmFuncType> for Entry {
-    fn borrow(&self) -> &WasmFuncType {
-        &self.0.ty
+impl Borrow<WasmRecGroup> for RecGroupEntry {
+    fn borrow(&self) -> &WasmRecGroup {
+        &self.0.hash_consing_key
     }
 }
 
-impl Entry {
+impl RecGroupEntry {
     /// Increment the registration count.
     fn incref(&self, why: &str) {
         let old_count = self.0.registrations.fetch_add(1, AcqRel);
         log::trace!(
-            "increment registration count for {:?} (registrations -> {}): {why}",
-            self.0.index,
+            "increment registration count for {self:?} (registrations -> {}): {why}",
             old_count + 1
         );
     }
@@ -374,8 +410,7 @@ impl Entry {
         let old_count = self.0.registrations.fetch_sub(1, AcqRel);
         debug_assert_ne!(old_count, 0);
         log::trace!(
-            "decrement registration count for {:?} (registrations -> {}): {why}",
-            self.0.index,
+            "decrement registration count for {self:?} (registrations -> {}): {why}",
             old_count - 1
         );
         old_count == 1
@@ -384,156 +419,272 @@ impl Entry {
 
 #[derive(Debug, Default)]
 struct TypeRegistryInner {
-    // A map from the Wasm function type to a `VMSharedTypeIndex`, for all
-    // the Wasm function types we have already registered.
-    map: HashSet<Entry>,
+    // A hash map from a canonicalized-for-hash-consing rec group to its
+    // `VMSharedTypeIndex`es.
+    //
+    // There is an entry in this map for every rec group we have already
+    // registered. Before registering new rec groups, we first check this map to
+    // see if we've already registered an identical rec group that we should
+    // reuse instead.
+    hash_consing_map: HashSet<RecGroupEntry>,
 
     // A map from `VMSharedTypeIndex::bits()` to the type index's associated
     // Wasm type.
-    entries: Slab<Entry>,
+    //
+    // These types are always canonicalized for runtime usage.
+    types: Slab<Arc<WasmFuncType>>,
+
+    // A map that lets you walk backwards from a `VMSharedTypeIndex` to its
+    // `RecGroupEntry`.
+    type_to_rec_group: HashMap<VMSharedTypeIndex, RecGroupEntry>,
 
     // An explicit stack of entries that we are in the middle of dropping. Used
     // to avoid recursion when dropping a type that is holding the last
     // reference to another type, etc...
-    drop_stack: Vec<VMSharedTypeIndex>,
+    drop_stack: Vec<RecGroupEntry>,
 }
 
 impl TypeRegistryInner {
-    fn register_for_module(
+    fn register_module_types(
         &mut self,
         types: &ModuleTypes,
-    ) -> PrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex> {
-        log::trace!("Registering module types");
-        let mut map = PrimaryMap::default();
-        for (idx, ty) in types.wasm_types() {
-            let mut ty = ty.clone();
-            self.canonicalize(&map, &mut ty);
-            let entry = self.register_canonicalized(ty);
-            let map_idx = map.push(entry.0.index);
-            assert_eq!(idx, map_idx);
-        }
-        map
-    }
+    ) -> (
+        Vec<RecGroupEntry>,
+        PrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,
+    ) {
+        log::trace!("Start registering module types");
 
-    /// Is the given type canonicalized for this registry?
-    fn is_canonicalized(&self, ty: &WasmFuncType) -> bool {
-        let result = ty.trace::<_, ()>(&mut |index| match index {
-            EngineOrModuleTypeIndex::Module(_) => Err(()),
-            EngineOrModuleTypeIndex::Engine(id) => {
-                let id = shared_type_index_to_slab_id(id);
-                assert!(
-                    self.entries.contains(id),
-                    "canonicalized in a different engine? {ty:?}"
-                );
-                Ok(())
+        let mut entries = Vec::with_capacity(types.rec_groups().len());
+        let mut map = PrimaryMap::<ModuleInternedTypeIndex, VMSharedTypeIndex>::with_capacity(
+            types.wasm_types().len(),
+        );
+
+        for (_rec_group_index, module_group) in types.rec_groups() {
+            let entry = self.register_rec_group(
+                &map,
+                module_group.clone(),
+                iter_entity_range(module_group.clone()).map(|ty| types.get(ty).clone()),
+            );
+
+            for (module_ty, engine_ty) in
+                iter_entity_range(module_group).zip(entry.0.shared_type_indices.iter())
+            {
+                let module_ty2 = map.push(*engine_ty);
+                assert_eq!(module_ty, module_ty2);
             }
-        });
-        result.is_ok()
+
+            entries.push(entry);
+        }
+
+        log::trace!("End registering module types");
+
+        (entries, map)
     }
 
-    /// Canonicalize a type, such that its type-to-type references are via
-    /// engine-level `VMSharedTypeIndex`es rather than module-local via
-    /// `ModuleInternedTypeIndex`es.
+    /// Register a rec group in this registry.
     ///
-    /// This makes the type suitable for deduplication in the registry.
+    /// The rec group may be either module-level canonical (i.e. straight from
+    /// `wasmparser`) or engine-level canonicalized for runtime usage in this
+    /// registry. It may *not* be engine-level canonicalized for hash consing or
+    /// engine-level canonicalized for a different type registry instance.
     ///
-    /// Panics on already-canonicalized types. They might be canonicalized for
-    /// another engine's registry, and we wouldn't know how to recanonicalize
-    /// them for this registry.
-    fn canonicalize(
-        &self,
-        module_to_shared: &PrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,
-        ty: &mut WasmFuncType,
-    ) {
-        ty.canonicalize(&mut |module_index| module_to_shared[module_index]);
-        debug_assert!(self.is_canonicalized(ty))
-    }
-
-    /// Add a new type to this registry.
+    /// If this rec group is determined to be a duplicate of an
+    /// already-registered rec group, the existing rec group is reused.
     ///
-    /// The type must be canonicalized and must not already exist in the
-    /// registry.
+    /// Parameters:
     ///
-    /// Initializes the new entry's registration count to one, and callers
-    /// should not further increment the registration count.
-    fn register_new(&mut self, ty: WasmFuncType) -> Entry {
-        assert!(
-            self.is_canonicalized(&ty),
-            "ty is not already canonicalized: {ty:?}"
-        );
+    /// * `map`: A map that we use to canonicalize inter-group type references
+    ///   from module-canonical to engine-canonical indices. This must contain
+    ///   entries for each inter-group type reference that this rec group
+    ///   contains.
+    ///
+    /// * `range`: The range of (module-level) types defined by this rec
+    ///   group. This is used to determine which type references inside this rec
+    ///   group are inter- vs intra-group.
+    ///
+    /// * `types`: The types defined within this rec group. Must have the same
+    ///   length as `range`.
+    ///
+    /// The returned entry will have already had its reference count incremented
+    /// on behalf of callers.
+    fn register_rec_group(
+        &mut self,
+        map: &PrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,
+        range: Range<ModuleInternedTypeIndex>,
+        types: impl ExactSizeIterator<Item = WasmFuncType>,
+    ) -> RecGroupEntry {
+        debug_assert_eq!(iter_entity_range(range.clone()).len(), types.len());
+
+        let mut non_canon_types = Vec::with_capacity(types.len());
+        let hash_consing_key = WasmRecGroup {
+            types: types
+                .zip(iter_entity_range(range.clone()))
+                .map(|(mut ty, module_index)| {
+                    non_canon_types.push((module_index, ty.clone()));
+                    ty.canonicalize_for_hash_consing(range.clone(), &mut |idx| {
+                        debug_assert!(idx < range.clone().start);
+                        map[idx]
+                    });
+                    ty
+                })
+                .collect::<Box<[_]>>(),
+        };
 
-        // Increment the ref count of each existing type that is referenced from
-        // this new type. Those types shouldn't be dropped while this type is
-        // still alive.
-        ty.trace::<_, ()>(&mut |idx| match idx {
-            EngineOrModuleTypeIndex::Engine(id) => {
-                let i = shared_type_index_to_slab_id(id);
-                let e = &self.entries[i];
-                e.incref("new type references existing type in TypeRegistryInner::register_new");
-                Ok(())
-            }
-            EngineOrModuleTypeIndex::Module(_) => unreachable!("should be canonicalized"),
-        })
-        .unwrap();
+        // If we've already registered this rec group before, reuse it.
+        if let Some(entry) = self.hash_consing_map.get(&hash_consing_key) {
+            entry.incref(
+                "hash consed to already-registered type in `TypeRegistryInner::register_rec_group`",
+            );
+            return entry.clone();
+        }
 
-        let id = self.entries.next_id();
-        let index = slab_id_to_shared_type_index(id);
-        log::trace!("create {index:?} = {ty:?} (registrations -> 1)");
-        let entry = Entry(Arc::new(EntryInner {
-            ty,
-            index,
+        // Inter-group edges: increment the referenced group's ref
+        // count, because these other rec groups shouldn't be dropped
+        // while this rec group is still alive.
+        hash_consing_key
+            .trace_engine_indices::<_, ()>(&mut |index| {
+                let entry = &self.type_to_rec_group[&index];
+                entry.incref(
+                    "new cross-group type reference to existing type in `register_rec_group`",
+                );
+                Ok(())
+            })
+            .unwrap();
+
+        // Register the individual types.
+        //
+        // Note that we can't update the reverse type-to-rec-group map until
+        // after we've constructed the `RecGroupEntry`, since that map needs to
+        // the fully-constructed entry for its values.
+        let module_rec_group_start = range.start;
+        let engine_rec_group_start = u32::try_from(self.types.len()).unwrap();
+        let shared_type_indices = non_canon_types
+            .into_iter()
+            .map(|(module_index, mut ty)| {
+                ty.canonicalize_for_runtime_usage(&mut |idx| {
+                    if idx < module_rec_group_start {
+                        map[idx]
+                    } else {
+                        let rec_group_offset = idx.as_u32() - module_rec_group_start.as_u32();
+                        VMSharedTypeIndex::from_u32(engine_rec_group_start + rec_group_offset)
+                    }
+                });
+                self.insert_one_type_from_rec_group(module_index, ty)
+            })
+            .collect();
+
+        let entry = RecGroupEntry(Arc::new(RecGroupEntryInner {
+            hash_consing_key,
+            shared_type_indices,
             registrations: AtomicUsize::new(1),
         }));
-        let is_new_entry = self.map.insert(entry.clone());
-        assert!(is_new_entry);
+        log::trace!("create new entry {entry:?} (registrations -> 1)");
 
-        let id = self.entries.alloc(entry.clone());
-        assert_eq!(id, shared_type_index_to_slab_id(index));
+        let is_new_entry = self.hash_consing_map.insert(entry.clone());
+        debug_assert!(is_new_entry);
+
+        // Now that we've construct the entry, we can update the reverse
+        // type-to-rec-group map.
+        for ty in entry.0.shared_type_indices.iter() {
+            let old_entry = self.type_to_rec_group.insert(*ty, entry.clone());
+            debug_assert!(old_entry.is_none());
+        }
 
         entry
     }
 
-    /// Register the given canonicalized type, incrementing its reference count.
-    fn register_canonicalized(&mut self, ty: WasmFuncType) -> Entry {
+    /// Is the given type canonicalized for runtime usage this registry?
+    fn assert_canonicalized_for_runtime_usage_in_this_registry(&self, ty: &WasmFuncType) {
+        ty.trace::<_, ()>(&mut |index| match index {
+            EngineOrModuleTypeIndex::RecGroup(_) | EngineOrModuleTypeIndex::Module(_) => {
+                panic!("not canonicalized for runtime usage: {ty:?}")
+            }
+            EngineOrModuleTypeIndex::Engine(idx) => {
+                let id = shared_type_index_to_slab_id(idx);
+                assert!(
+                    self.types.contains(id),
+                    "canonicalized in a different engine? {ty:?}"
+                );
+                Ok(())
+            }
+        })
+        .unwrap();
+    }
+
+    /// Insert a new type as part of registering a new rec group.
+    ///
+    /// The type must be canonicalized for runtime usage in this registry and
+    /// its rec group must be a new one that we are currently registering, not
+    /// an already-registered rec group.
+    fn insert_one_type_from_rec_group(
+        &mut self,
+        module_index: ModuleInternedTypeIndex,
+        ty: WasmFuncType,
+    ) -> VMSharedTypeIndex {
+        // Despite being canonicalized for runtime usage, this type may still
+        // have forward references to other types in the rec group we haven't
+        // yet registered. Therefore, we can't use our usual
+        // `assert_canonicalized_for_runtime_usage_in_this_registry` helper here
+        // as that will see the forward references and think they must be
+        // references to types in other registries.
         assert!(
-            self.is_canonicalized(&ty),
-            "type is not already canonicalized: {ty:?}"
+            ty.is_canonicalized_for_runtime_usage(),
+            "type is not canonicalized for runtime usage: {ty:?}"
         );
 
-        if let Some(entry) = self.map.get(&ty) {
-            entry.incref(
-                "registering already-registered type in TypeRegistryInner::register_canonicalized",
-            );
-            entry.clone()
-        } else {
-            self.register_new(ty)
-        }
+        let id = self.types.alloc(Arc::new(ty));
+        let engine_index = slab_id_to_shared_type_index(id);
+        log::trace!(
+            "registered type {module_index:?} as {engine_index:?} = {:?}",
+            &self.types[id]
+        );
+        engine_index
+    }
+
+    /// Register a rec group consisting of a single type.
+    ///
+    /// The type must already be canonicalized for runtime usage in this
+    /// registry.
+    ///
+    /// The returned entry will have already had its reference count incremented
+    /// on behalf of callers.
+    fn register_singleton_rec_group(&mut self, ty: WasmFuncType) -> RecGroupEntry {
+        self.assert_canonicalized_for_runtime_usage_in_this_registry(&ty);
+
+        // This type doesn't have any module-level type references, since it is
+        // already canonicalized for runtime usage in this registry, so an empty
+        // map suffices.
+        let map = PrimaryMap::default();
+
+        // This must have `range.len() == 1`, even though we know this type
+        // doesn't have any intra-group type references, to satisfy
+        // `register_rec_group`'s preconditions.
+        let range = ModuleInternedTypeIndex::from_bits(u32::MAX - 1)
+            ..ModuleInternedTypeIndex::from_bits(u32::MAX);
+
+        self.register_rec_group(&map, range, std::iter::once(ty))
     }
 
+    /// Unregister all of a type collection's rec groups.
     fn unregister_type_collection(&mut self, collection: &TypeCollection) {
-        for (_, id) in collection.types.iter() {
-            let i = shared_type_index_to_slab_id(*id);
-            let e = &self.entries[i];
-            if e.decref("TypeRegistryInner::unregister_type_collection") {
-                self.unregister_entry(*id);
+        for entry in &collection.rec_groups {
+            if entry.decref("TypeRegistryInner::unregister_type_collection") {
+                self.unregister_entry(entry.clone());
             }
         }
     }
 
-    /// Remove an entry from the registry.
+    /// Remove a zero-refcount entry from the registry.
     ///
     /// This does *not* decrement the entry's registration count, it should
-    /// instead be invoked after a previous decrement operation observed zero
-    /// remaining registrations.
-    fn unregister_entry(&mut self, index: VMSharedTypeIndex) {
-        log::trace!("unregistering {index:?}");
-
+    /// instead be invoked only after a previous decrement operation observed
+    /// zero remaining registrations.
+    fn unregister_entry(&mut self, entry: RecGroupEntry) {
         debug_assert!(self.drop_stack.is_empty());
-        self.drop_stack.push(index);
+        self.drop_stack.push(entry);
 
-        while let Some(index) = self.drop_stack.pop() {
-            let slab_id = shared_type_index_to_slab_id(index);
-            let entry = &self.entries[slab_id];
+        while let Some(entry) = self.drop_stack.pop() {
+            log::trace!("Start unregistering {entry:?}");
 
             // We need to double check whether the entry is still at zero
             // registrations: Between the time that we observed a zero and
@@ -548,37 +699,52 @@ impl TypeRegistryInner {
             let registrations = entry.0.registrations.load(Acquire);
             if registrations != 0 {
                 log::trace!(
-                    "{index:?} was concurrently resurrected and no longer has zero \
-                     registrations (registrations -> {registrations})"
+                    "{entry:?} was concurrently resurrected and no longer has \
+                     zero registrations (registrations -> {registrations})",
                 );
                 continue;
             }
 
-            // Decrement any other types that this type was
-            // (shallowly/non-transitively) keeping alive.
+            // Decrement any other types that this type was shallowly
+            // (i.e. non-transitively) referencing and keeping alive. If this
+            // was the last thing keeping them registered, its okay to
+            // unregister them as well now.
+            debug_assert!(entry.0.hash_consing_key.is_canonicalized_for_hash_consing());
             entry
                 .0
-                .ty
-                .trace::<_, ()>(&mut |child_index| match child_index {
-                    EngineOrModuleTypeIndex::Engine(child_index) => {
-                        let child_slab_id = shared_type_index_to_slab_id(child_index);
-                        let child_entry = &self.entries[child_slab_id];
-                        if child_entry.decref(
-                            "referenced by unregistered type in TypeCollection::unregister_entry",
-                        ) {
-                            self.drop_stack.push(child_index);
-                        }
-                        Ok(())
-                    }
-                    EngineOrModuleTypeIndex::Module(_) => {
-                        unreachable!("should be canonicalized")
+                .hash_consing_key
+                .trace_engine_indices::<_, ()>(&mut |other_index| {
+                    let other_entry = self.type_to_rec_group[&other_index].clone();
+                    if other_entry.decref(
+                        "referenced by dropped entry in \
+                         `TypeCollection::unregister_entry`",
+                    ) {
+                        self.drop_stack.push(other_entry);
                     }
+                    Ok(())
                 })
                 .unwrap();
 
-            log::trace!("removing {index:?} from registry");
-            self.map.remove(entry);
-            self.entries.dealloc(slab_id);
+            // Remove the entry from the hash-consing map. If we register a
+            // duplicate definition of this rec group again in the future, it
+            // will be as if it is the first time it has ever been registered,
+            // and it will be inserted into the hash-consing map again at that
+            // time.
+            self.hash_consing_map.remove(&entry);
+
+            // Similarly, remove the rec group's types from the registry, as
+            // well as their entries from the reverse type-to-rec-group map.
+            for ty in entry.0.shared_type_indices.iter() {
+                log::trace!("removing {ty:?} from registry");
+
+                let removed_entry = self.type_to_rec_group.remove(ty);
+                debug_assert_eq!(removed_entry.unwrap(), entry);
+
+                let id = shared_type_index_to_slab_id(*ty);
+                self.types.dealloc(id);
+            }
+
+            log::trace!("End unregistering {entry:?}");
         }
     }
 }
@@ -588,13 +754,27 @@ impl TypeRegistryInner {
 #[cfg(debug_assertions)]
 impl Drop for TypeRegistryInner {
     fn drop(&mut self) {
+        let TypeRegistryInner {
+            hash_consing_map,
+            types,
+            type_to_rec_group,
+            drop_stack,
+        } = self;
+        assert!(
+            hash_consing_map.is_empty(),
+            "type registry not empty: hash consing map is not empty"
+        );
+        assert!(
+            types.is_empty(),
+            "type registry not empty: types slab is not empty"
+        );
         assert!(
-            self.map.is_empty(),
-            "type registry not empty: still have registered types in self.map"
+            type_to_rec_group.is_empty(),
+            "type registry not empty: type-to-rec-group map is not empty"
         );
         assert!(
-            self.entries.is_empty(),
-            "type registry not empty: not all entries are vacant"
+            drop_stack.is_empty(),
+            "type registry not empty: drop stack is not empty"
         );
     }
 }
@@ -620,9 +800,9 @@ impl TypeRegistry {
     /// still using the resulting value! Use the `RegisteredType::root`
     /// constructor if you need to ensure that property and you don't have some
     /// other mechanism already keeping the type registered.
-    pub fn borrow(&self, index: VMSharedTypeIndex) -> Option<impl Deref<Target = WasmFuncType>> {
+    pub fn borrow(&self, index: VMSharedTypeIndex) -> Option<Arc<WasmFuncType>> {
         let id = shared_type_index_to_slab_id(index);
         let inner = self.0.read().unwrap();
-        inner.entries.get(id).cloned()
+        inner.types.get(id).cloned()
     }
 }
diff --git a/crates/wasmtime/src/runtime/types.rs b/crates/wasmtime/src/runtime/types.rs
index bf978db76..1cca5e5a9 100644
--- a/crates/wasmtime/src/runtime/types.rs
+++ b/crates/wasmtime/src/runtime/types.rs
@@ -671,7 +671,8 @@ impl HeapType {
             WasmHeapType::Concrete(EngineOrModuleTypeIndex::Engine(idx)) => {
                 HeapType::Concrete(FuncType::from_shared_type_index(engine, *idx))
             }
-            WasmHeapType::Concrete(EngineOrModuleTypeIndex::Module(_)) => {
+            WasmHeapType::Concrete(EngineOrModuleTypeIndex::Module(_))
+            | WasmHeapType::Concrete(EngineOrModuleTypeIndex::RecGroup(_)) => {
                 panic!("HeapType::from_wasm_type on non-canonical heap type")
             }
         }
@@ -769,6 +770,7 @@ impl ExternType {
                 EngineOrModuleTypeIndex::Module(m) => {
                     FuncType::from_wasm_func_type(engine, types[*m].clone()).into()
                 }
+                EngineOrModuleTypeIndex::RecGroup(_) => unreachable!(),
             },
             EntityType::Global(ty) => GlobalType::from_wasmtime_global(engine, ty).into(),
             EntityType::Memory(ty) => MemoryType::from_wasmtime_memory(ty).into(),
@@ -1089,16 +1091,10 @@ impl TableType {
     pub fn new(element: RefType, min: u32, max: Option<u32>) -> TableType {
         let wasm_ty = element.to_wasm_type();
 
-        if cfg!(debug_assertions) {
-            wasm_ty
-                .trace(&mut |idx| match idx {
-                    EngineOrModuleTypeIndex::Engine(_) => Ok(()),
-                    EngineOrModuleTypeIndex::Module(module_idx) => Err(format!(
-                        "found module-level canonicalized type index: {module_idx:?}"
-                    )),
-                })
-                .expect("element type should be engine-level canonicalized");
-        }
+        debug_assert!(
+            wasm_ty.is_canonicalized_for_runtime_usage(),
+            "should be canonicalized for runtime usage: {wasm_ty:?}"
+        );
 
         TableType {
             element,
diff --git a/crates/wasmtime/src/runtime/types/matching.rs b/crates/wasmtime/src/runtime/types/matching.rs
index 9b12cae8a..45504aceb 100644
--- a/crates/wasmtime/src/runtime/types/matching.rs
+++ b/crates/wasmtime/src/runtime/types/matching.rs
@@ -1,8 +1,8 @@
 use crate::{linker::DefinitionType, Engine, FuncType};
 use anyhow::{anyhow, bail, Result};
 use wasmtime_environ::{
-    EngineOrModuleTypeIndex, EntityType, Global, Memory, ModuleTypes, Table, TypeTrace,
-    VMSharedTypeIndex, WasmFuncType, WasmHeapType, WasmRefType, WasmValType,
+    EntityType, Global, Memory, ModuleTypes, Table, TypeTrace, VMSharedTypeIndex, WasmFuncType,
+    WasmHeapType, WasmRefType, WasmValType,
 };
 
 pub struct MatchCx<'a> {
@@ -245,25 +245,14 @@ fn match_ref(expected: WasmRefType, actual: WasmRefType, desc: &str) -> Result<(
 fn match_ty(expected: WasmValType, actual: WasmValType, desc: &str) -> Result<()> {
     // Assert that both our types are engine-level canonicalized. We can't
     // compare types otherwise.
-    if cfg!(debug_assertions) {
-        expected
-            .trace(&mut |idx| match idx {
-                EngineOrModuleTypeIndex::Engine(_) => Ok(()),
-                EngineOrModuleTypeIndex::Module(mod_idx) => {
-                    Err(format!("found module-level index: {mod_idx:?}"))
-                }
-            })
-            .expect("expected type should be engine-level canonicalized");
-
-        actual
-            .trace(&mut |idx| match idx {
-                EngineOrModuleTypeIndex::Engine(_) => Ok(()),
-                EngineOrModuleTypeIndex::Module(mod_idx) => {
-                    Err(format!("found module-level index: {mod_idx:?}"))
-                }
-            })
-            .expect("actual type should be engine-level canonicalized");
-    }
+    debug_assert!(
+        expected.is_canonicalized_for_runtime_usage(),
+        "expected type should be canonicalized for runtime usage: {expected:?}"
+    );
+    debug_assert!(
+        actual.is_canonicalized_for_runtime_usage(),
+        "actual type should be canonicalized for runtime usage: {actual:?}"
+    );
 
     match (actual, expected) {
         (WasmValType::Ref(actual), WasmValType::Ref(expected)) => match_ref(expected, actual, desc),
@@ -274,25 +263,14 @@ fn match_ty(expected: WasmValType, actual: WasmValType, desc: &str) -> Result<()
 fn equal_ty(expected: WasmValType, actual: WasmValType, desc: &str) -> Result<()> {
     // Assert that both our types are engine-level canonicalized. We can't
     // compare types otherwise.
-    if cfg!(debug_assertions) {
-        expected
-            .trace(&mut |idx| match idx {
-                EngineOrModuleTypeIndex::Engine(_) => Ok(()),
-                EngineOrModuleTypeIndex::Module(mod_idx) => {
-                    Err(format!("found module-level index: {mod_idx:?}"))
-                }
-            })
-            .expect("expected type should be engine-level canonicalized");
-
-        actual
-            .trace(&mut |idx| match idx {
-                EngineOrModuleTypeIndex::Engine(_) => Ok(()),
-                EngineOrModuleTypeIndex::Module(mod_idx) => {
-                    Err(format!("found module-level index: {mod_idx:?}"))
-                }
-            })
-            .expect("actual type should be engine-level canonicalized");
-    }
+    debug_assert!(
+        expected.is_canonicalized_for_runtime_usage(),
+        "expected type should be canonicalized for runtime usage: {expected:?}"
+    );
+    debug_assert!(
+        actual.is_canonicalized_for_runtime_usage(),
+        "actual type should be canonicalized for runtime usage: {actual:?}"
+    );
 
     if expected == actual {
         return Ok(());
diff --git a/examples/externref.c b/examples/externref.c
index d29ea2b97..79fa82841 100644
--- a/examples/externref.c
+++ b/examples/externref.c
@@ -101,76 +101,82 @@ int main() {
   void *data = wasmtime_externref_data(context, externref);
   assert(strcmp((char *)data, "Hello, World!") == 0);
 
-  printf("Touching `externref` table...\n");
-
   wasmtime_extern_t item;
-
-  // Lookup the `table` export.
-  ok = wasmtime_instance_export_get(context, &instance, "table",
-                                    strlen("table"), &item);
-  assert(ok);
-  assert(item.kind == WASMTIME_EXTERN_TABLE);
-
-  // Set `table[3]` to our `externref`.
   wasmtime_val_t externref_val;
   externref_val.kind = WASMTIME_EXTERNREF;
   externref_val.of.externref = externref;
-  error = wasmtime_table_set(context, &item.of.table, 3, &externref_val);
-  if (error != NULL)
-    exit_with_error("failed to set table", error, NULL);
 
-  // `table[3]` should now be our `externref`.
-  wasmtime_val_t elem;
-  ok = wasmtime_table_get(context, &item.of.table, 3, &elem);
-  assert(ok);
-  assert(elem.kind == WASMTIME_EXTERNREF);
-  assert(strcmp((char *)wasmtime_externref_data(context, elem.of.externref),
-                "Hello, World!") == 0);
-  wasmtime_val_delete(context, &elem);
+  // Lookup the `table` export.
+  printf("Touching `externref` table...\n");
+  {
+    ok = wasmtime_instance_export_get(context, &instance, "table",
+                                      strlen("table"), &item);
+    assert(ok);
+    assert(item.kind == WASMTIME_EXTERN_TABLE);
+
+    // Set `table[3]` to our `externref`.
+    error = wasmtime_table_set(context, &item.of.table, 3, &externref_val);
+    if (error != NULL)
+      exit_with_error("failed to set table", error, NULL);
+
+    // `table[3]` should now be our `externref`.
+    wasmtime_val_t elem;
+    ok = wasmtime_table_get(context, &item.of.table, 3, &elem);
+    assert(ok);
+    assert(elem.kind == WASMTIME_EXTERNREF);
+    assert(strcmp((char *)wasmtime_externref_data(context, elem.of.externref),
+                  "Hello, World!") == 0);
+    wasmtime_val_delete(context, &elem);
+  }
 
   printf("Touching `externref` global...\n");
 
   // Lookup the `global` export.
-  ok = wasmtime_instance_export_get(context, &instance, "global",
-                                    strlen("global"), &item);
-  assert(ok);
-  assert(item.kind == WASMTIME_EXTERN_GLOBAL);
-
-  // Set the global to our `externref`.
-  error = wasmtime_global_set(context, &item.of.global, &externref_val);
-  if (error != NULL)
-    exit_with_error("failed to set global", error, NULL);
-
-  // Get the global, and it should return our `externref` again.
-  wasmtime_val_t global_val;
-  wasmtime_global_get(context, &item.of.global, &global_val);
-  assert(global_val.kind == WASMTIME_EXTERNREF);
-  assert(strcmp((char *)wasmtime_externref_data(context, elem.of.externref),
-                "Hello, World!") == 0);
-  wasmtime_val_delete(context, &global_val);
+  {
+    ok = wasmtime_instance_export_get(context, &instance, "global",
+                                      strlen("global"), &item);
+    assert(ok);
+    assert(item.kind == WASMTIME_EXTERN_GLOBAL);
+
+    // Set the global to our `externref`.
+    error = wasmtime_global_set(context, &item.of.global, &externref_val);
+    if (error != NULL)
+      exit_with_error("failed to set global", error, NULL);
+
+    // Get the global, and it should return our `externref` again.
+    wasmtime_val_t global_val;
+    wasmtime_global_get(context, &item.of.global, &global_val);
+    assert(global_val.kind == WASMTIME_EXTERNREF);
+    assert(strcmp((char *)wasmtime_externref_data(context,
+                                                  global_val.of.externref),
+                  "Hello, World!") == 0);
+    wasmtime_val_delete(context, &global_val);
+  }
 
   printf("Calling `externref` func...\n");
 
   // Lookup the `func` export.
-  ok = wasmtime_instance_export_get(context, &instance, "func", strlen("func"),
-                                    &item);
-  assert(ok);
-  assert(item.kind == WASMTIME_EXTERN_FUNC);
-
-  // And call it!
-  wasmtime_val_t results[1];
-  error = wasmtime_func_call(context, &item.of.func, &externref_val, 1, results,
-                             1, &trap);
-  if (error != NULL || trap != NULL)
-    exit_with_error("failed to call function", error, trap);
-
-  // `func` returns the same reference we gave it, so `results[0]` should be our
-  // `externref`.
-  assert(results[0].kind == WASMTIME_EXTERNREF);
-  assert(
-      strcmp((char *)wasmtime_externref_data(context, results[0].of.externref),
-             "Hello, World!") == 0);
-  wasmtime_val_delete(context, &results[0]);
+  {
+    ok = wasmtime_instance_export_get(context, &instance, "func",
+                                      strlen("func"), &item);
+    assert(ok);
+    assert(item.kind == WASMTIME_EXTERN_FUNC);
+
+    // And call it!
+    wasmtime_val_t results[1];
+    error = wasmtime_func_call(context, &item.of.func, &externref_val, 1,
+                               results, 1, &trap);
+    if (error != NULL || trap != NULL)
+      exit_with_error("failed to call function", error, trap);
+
+    // `func` returns the same reference we gave it, so `results[0]` should be
+    // our `externref`.
+    assert(results[0].kind == WASMTIME_EXTERNREF);
+    assert(strcmp((char *)wasmtime_externref_data(context,
+                                                  results[0].of.externref),
+                  "Hello, World!") == 0);
+    wasmtime_val_delete(context, &results[0]);
+  }
   wasmtime_val_delete(context, &externref_val);
 
   // We can GC any now-unused references to our externref that the store is
diff --git a/supply-chain/config.toml b/supply-chain/config.toml
index 66021af73..eb2b7081d 100644
--- a/supply-chain/config.toml
+++ b/supply-chain/config.toml
@@ -315,6 +315,10 @@ version = "0.3.19"
 criteria = "safe-to-deploy"
 notes = "we are exempting tokio, hyper, and their tightly coupled dependencies by the same authors, expecting that the authors at aws will publish attestions we can import at some point soon"
 
+[[exemptions.h2]]
+version = "0.4.4"
+criteria = "safe-to-deploy"
+
 [[exemptions.hermit-abi]]
 version = "0.2.0"
 criteria = "safe-to-deploy"
@@ -388,6 +392,10 @@ version = "0.8.6"
 criteria = "safe-to-deploy"
 notes = "we are exempting tokio, hyper, and their tightly coupled dependencies by the same authors, expecting that the authors at aws will publish attestions we can import at some point soon"
 
+[[exemptions.mio]]
+version = "0.8.11"
+criteria = "safe-to-deploy"
+
 [[exemptions.num_cpus]]
 version = "1.13.1"
 criteria = "safe-to-deploy"
@@ -493,6 +501,10 @@ criteria = "safe-to-deploy"
 version = "0.22.2"
 criteria = "safe-to-deploy"
 
+[[exemptions.rustls]]
+version = "0.22.4"
+criteria = "safe-to-deploy"
+
 [[exemptions.rustls-pki-types]]
 version = "1.3.1"
 criteria = "safe-to-deploy"
diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock
index 263bfd6e6..61311c79f 100644
--- a/supply-chain/imports.lock
+++ b/supply-chain/imports.lock
@@ -1190,6 +1190,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasm-encoder]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasm-metadata]]
 version = "0.201.0"
 when = "2024-02-27"
@@ -1214,6 +1220,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasm-metadata]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasm-mutate]]
 version = "0.201.0"
 when = "2024-02-27"
@@ -1238,6 +1250,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasm-mutate]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasm-smith]]
 version = "0.201.0"
 when = "2024-02-27"
@@ -1262,6 +1280,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasm-smith]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmparser]]
 version = "0.201.0"
 when = "2024-02-27"
@@ -1286,6 +1310,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmparser]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmprinter]]
 version = "0.201.0"
 when = "2024-02-27"
@@ -1310,6 +1340,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmprinter]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime]]
 version = "18.0.2"
 when = "2024-02-28"
@@ -1622,6 +1658,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wast]]
+version = "205.0.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wat]]
 version = "1.201.0"
 when = "2024-02-27"
@@ -1646,6 +1688,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wat]]
+version = "1.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wiggle]]
 version = "18.0.2"
 when = "2024-02-28"
@@ -1940,6 +1988,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wit-component]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wit-parser]]
 version = "0.201.0"
 when = "2024-02-27"
@@ -1964,6 +2018,12 @@ when = "2024-04-16"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wit-parser]]
+version = "0.205.0"
+when = "2024-04-18"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.witx]]
 version = "0.9.1"
 when = "2021-06-22"
diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rs
index 377bbcb72..6818c30f8 100644
--- a/winch/codegen/src/codegen/env.rs
+++ b/winch/codegen/src/codegen/env.rs
@@ -197,8 +197,7 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
                 BlockSig::new(control::BlockType::single(ty))
             }
             FuncType(idx) => {
-                let sig_index =
-                    self.translation.module.types[TypeIndex::from_u32(idx)].unwrap_function();
+                let sig_index = self.translation.module.types[TypeIndex::from_u32(idx)];
                 let sig = &self.types[sig_index];
                 BlockSig::new(control::BlockType::func(sig.clone()))
             }
@@ -340,7 +339,7 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
             }
             Callee::FuncRef(idx) => {
                 let val = || {
-                    let sig_index = self.translation.module.types[*idx].unwrap_function();
+                    let sig_index = self.translation.module.types[*idx];
                     let ty = &self.types[sig_index];
                     let sig = wasm_sig::<A>(ty);
                     sig
@@ -392,11 +391,8 @@ pub(crate) struct TypeConverter<'a, 'data: 'a> {
 
 impl TypeConvert for TypeConverter<'_, '_> {
     fn lookup_heap_type(&self, idx: wasmparser::UnpackedIndex) -> WasmHeapType {
-        wasmtime_environ::WasmparserTypeConverter {
-            module: &self.translation.module,
-            types: self.types,
-        }
-        .lookup_heap_type(idx)
+        wasmtime_environ::WasmparserTypeConverter::new(self.types, &self.translation.module)
+            .lookup_heap_type(idx)
     }
 }
 
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 3017dc812..73abb56f8 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -323,7 +323,7 @@ where
         let ptr_size: OperandSize = self.env.ptr_type().into();
         let sig_index_bytes = self.env.vmoffsets.size_of_vmshared_type_index();
         let sig_size = OperandSize::from_bytes(sig_index_bytes);
-        let sig_index = self.env.translation.module.types[type_index].unwrap_function();
+        let sig_index = self.env.translation.module.types[type_index];
         let sig_offset = sig_index
             .as_u32()
             .checked_mul(sig_index_bytes.into())
@@ -356,7 +356,7 @@ where
         );
 
         // Typecheck.
-        self.masm.cmp(callee_id.into(), caller_id, OperandSize::S32);
+        self.masm.cmp(caller_id, callee_id.into(), OperandSize::S32);
         self.masm.trapif(IntCmpKind::Ne, TrapCode::BadSignature);
         self.context.free_reg(callee_id);
         self.context.free_reg(caller_id);
@@ -492,8 +492,8 @@ where
 
         self.masm.branch(
             IntCmpKind::Ne,
-            elem_value.into(),
             elem_value,
+            elem_value.into(),
             defined,
             ptr_type.into(),
         );
@@ -613,8 +613,8 @@ where
                     |masm, bounds, _| {
                         let bounds_reg = bounds.as_typed_reg().reg;
                         masm.cmp(
-                            bounds_reg.into(),
                             index_offset_and_access_size.into(),
+                            bounds_reg.into(),
                             heap.ty.into(),
                         );
                         IntCmpKind::GtU
@@ -697,8 +697,8 @@ where
                         let adjusted_bounds = bounds.as_u64() - offset_with_access_size;
                         let index_reg = index.as_typed_reg().reg;
                         masm.cmp(
-                            RegImm::i64(adjusted_bounds as i64),
                             index_reg,
+                            RegImm::i64(adjusted_bounds as i64),
                             heap.ty.into(),
                         );
                         IntCmpKind::GtU
@@ -776,7 +776,7 @@ where
             .address_at_reg(base, table_data.current_elems_offset);
         let bound_size = table_data.current_elements_size;
         self.masm.load(bound_addr, bound, bound_size.into());
-        self.masm.cmp(bound.into(), index, bound_size);
+        self.masm.cmp(index, bound.into(), bound_size);
         self.masm
             .trapif(IntCmpKind::GeU, TrapCode::TableOutOfBounds);
 
@@ -801,7 +801,7 @@ where
         if self.env.table_access_spectre_mitigation() {
             // Perform a bounds check and override the value of the
             // table element address in case the index is out of bounds.
-            self.masm.cmp(bound.into(), index, OperandSize::S32);
+            self.masm.cmp(index, bound.into(), OperandSize::S32);
             self.masm.cmov(tmp, base, IntCmpKind::GeU, ptr_size);
         }
         self.context.free_reg(bound);
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 6fab3ae89..5bd13dcb8 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -452,7 +452,7 @@ impl Masm for MacroAssembler {
         todo!()
     }
 
-    fn cmp(&mut self, _src: RegImm, _dest: Reg, _size: OperandSize) {
+    fn cmp(&mut self, _src1: Reg, _src2: RegImm, _size: OperandSize) {
         todo!()
     }
 
@@ -495,8 +495,8 @@ impl Masm for MacroAssembler {
     fn branch(
         &mut self,
         _kind: IntCmpKind,
-        _lhs: RegImm,
-        _rhs: Reg,
+        _lhs: Reg,
+        _rhs: RegImm,
         _taken: MachLabel,
         _size: OperandSize,
     ) {
diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs
index 8d380ea85..d3b2e221b 100644
--- a/winch/codegen/src/isa/x64/asm.rs
+++ b/winch/codegen/src/isa/x64/asm.rs
@@ -835,8 +835,8 @@ impl Assembler {
             DivKind::Signed => {
                 self.emit(Inst::CmpRmiR {
                     size: size.into(),
-                    src: GprMemImm::new(RegMemImm::imm(0)).unwrap(),
-                    dst: divisor.into(),
+                    src1: divisor.into(),
+                    src2: GprMemImm::new(RegMemImm::imm(0)).unwrap(),
                     opcode: CmpOpcode::Cmp,
                 });
                 self.emit(Inst::TrapIf {
@@ -975,23 +975,23 @@ impl Assembler {
         });
     }
 
-    pub fn cmp_ir(&mut self, imm: i32, dst: Reg, size: OperandSize) {
+    pub fn cmp_ir(&mut self, src1: Reg, imm: i32, size: OperandSize) {
         let imm = RegMemImm::imm(imm as u32);
 
         self.emit(Inst::CmpRmiR {
             size: size.into(),
             opcode: CmpOpcode::Cmp,
-            src: GprMemImm::new(imm).expect("valid immediate"),
-            dst: dst.into(),
+            src1: src1.into(),
+            src2: GprMemImm::new(imm).expect("valid immediate"),
         });
     }
 
-    pub fn cmp_rr(&mut self, src: Reg, dst: Reg, size: OperandSize) {
+    pub fn cmp_rr(&mut self, src1: Reg, src2: Reg, size: OperandSize) {
         self.emit(Inst::CmpRmiR {
             size: size.into(),
             opcode: CmpOpcode::Cmp,
-            src: src.into(),
-            dst: dst.into(),
+            src1: src1.into(),
+            src2: src2.into(),
         });
     }
 
@@ -1025,12 +1025,12 @@ impl Assembler {
     }
 
     /// Emit a test instruction with two register operands.
-    pub fn test_rr(&mut self, src: Reg, dst: Reg, size: OperandSize) {
+    pub fn test_rr(&mut self, src1: Reg, src2: Reg, size: OperandSize) {
         self.emit(Inst::CmpRmiR {
             size: size.into(),
             opcode: CmpOpcode::Test,
-            src: src.into(),
-            dst: dst.into(),
+            src1: src1.into(),
+            src2: src2.into(),
         })
     }
 
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index 2300253b7..6197ce8bc 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -93,7 +93,7 @@ impl Masm for MacroAssembler {
 
         self.add_stack_max(scratch);
 
-        self.asm.cmp_rr(regs::rsp(), scratch, self.ptr_size);
+        self.asm.cmp_rr(scratch, regs::rsp(), self.ptr_size);
         self.asm.trapif(IntCmpKind::GtU, TrapCode::StackOverflow);
 
         // Emit unwind info.
@@ -627,25 +627,25 @@ impl Masm for MacroAssembler {
         Address::offset(reg, offset)
     }
 
-    fn cmp(&mut self, src: RegImm, dst: Reg, size: OperandSize) {
-        match src {
+    fn cmp(&mut self, src1: Reg, src2: RegImm, size: OperandSize) {
+        match src2 {
             RegImm::Imm(imm) => {
                 if let Some(v) = imm.to_i32() {
-                    self.asm.cmp_ir(v, dst, size);
+                    self.asm.cmp_ir(src1, v, size);
                 } else {
                     let scratch = regs::scratch();
                     self.load_constant(&imm, scratch, size);
-                    self.asm.cmp_rr(scratch, dst, size);
+                    self.asm.cmp_rr(src1, scratch, size);
                 }
             }
-            RegImm::Reg(src) => {
-                self.asm.cmp_rr(src, dst, size);
+            RegImm::Reg(src2) => {
+                self.asm.cmp_rr(src1, src2, size);
             }
         }
     }
 
     fn cmp_with_set(&mut self, src: RegImm, dst: Reg, kind: IntCmpKind, size: OperandSize) {
-        self.cmp(src, dst, size);
+        self.cmp(dst, src, size);
         self.asm.setcc(kind, dst);
     }
 
@@ -748,20 +748,20 @@ impl Masm for MacroAssembler {
     fn branch(
         &mut self,
         kind: IntCmpKind,
-        lhs: RegImm,
-        rhs: Reg,
+        lhs: Reg,
+        rhs: RegImm,
         taken: MachLabel,
         size: OperandSize,
     ) {
         use IntCmpKind::*;
 
         match &(lhs, rhs) {
-            (RegImm::Reg(rlhs), rrhs) => {
+            (rlhs, RegImm::Reg(rrhs)) => {
                 // If the comparision kind is zero or not zero and both operands
                 // are the same register, emit a test instruction. Else we emit
                 // a normal comparison.
                 if (kind == Eq || kind == Ne) && (rlhs == rrhs) {
-                    self.asm.test_rr(*rrhs, *rlhs, size);
+                    self.asm.test_rr(*rlhs, *rrhs, size);
                 } else {
                     self.cmp(lhs, rhs, size);
                 }
@@ -959,7 +959,7 @@ impl Masm for MacroAssembler {
         let max = default_index;
         let size = OperandSize::S32;
         self.asm.mov_ir(max as u64, tmp, size);
-        self.asm.cmp_rr(index, tmp, size);
+        self.asm.cmp_rr(tmp, index, size);
         self.asm.cmov(tmp, index, IntCmpKind::LtU, size);
 
         let default = targets[default_index];
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 5b86dc8fd..4f18842a1 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -717,11 +717,20 @@ pub(crate) trait MacroAssembler {
     /// Calculate remainder.
     fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize);
 
-    /// Compare src and dst and put the result in dst.
-    fn cmp(&mut self, src: RegImm, dest: Reg, size: OperandSize);
+    /// Compares `src1` against `src2` for the side effect of setting processor
+    /// flags.
+    ///
+    /// Note that `src1` is the left-hand-side of the comparison and `src2` is
+    /// the right-hand-side, so if testing `a < b` then `src1 == a` and
+    /// `src2 == b`
+    fn cmp(&mut self, src1: Reg, src2: RegImm, size: OperandSize);
 
     /// Compare src and dst and put the result in dst.
     /// This function will potentially emit a series of instructions.
+    ///
+    /// The initial value in `dst` is the left-hand-side of the comparison and
+    /// the initial value in `src` is the right-hand-side of the comparison.
+    /// That means for `a < b` then `dst == a` and `src == b`.
     fn cmp_with_set(&mut self, src: RegImm, dst: Reg, kind: IntCmpKind, size: OperandSize);
 
     /// Compare floats in src1 and src2 and put the result in dst.
@@ -881,8 +890,8 @@ pub(crate) trait MacroAssembler {
     fn branch(
         &mut self,
         kind: IntCmpKind,
-        lhs: RegImm,
-        rhs: Reg,
+        lhs: Reg,
+        rhs: RegImm,
         taken: MachLabel,
         size: OperandSize,
     );
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index c02abb763..88fa594c3 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -1885,7 +1885,7 @@ where
         let val2 = self.context.pop_to_reg(self.masm, None);
         let val1 = self.context.pop_to_reg(self.masm, None);
         self.masm
-            .cmp(RegImm::i32(0), cond.reg.into(), OperandSize::S32);
+            .cmp(cond.reg.into(), RegImm::i32(0), OperandSize::S32);
         // Conditionally move val1 to val2 if the the comparision is
         // not zero.
         self.masm


Test patch:
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 7f4682966..454e768c0 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -3241,25 +3241,25 @@ fn test_x64_emit() {
     // ========================================================
     // CmpRMIR
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::reg(r15), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size64, rdx, RegMemImm::reg(r15)),
         "4C39FA",
         "cmpq    %r15, %rdx",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::reg(rcx), r8),
+        Inst::cmp_rmi_r(OperandSize::Size64, r8, RegMemImm::reg(rcx)),
         "4939C8",
         "cmpq    %rcx, %r8",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::reg(rcx), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size64, rsi, RegMemImm::reg(rcx)),
         "4839CE",
         "cmpq    %rcx, %rsi",
     ));
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size64,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rdx,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "483B5763",
         "cmpq    99(%rdi), %rdx",
@@ -3267,8 +3267,8 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size64,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             r8,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "4C3B4763",
         "cmpq    99(%rdi), %r8",
@@ -3276,48 +3276,48 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size64,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rsi,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "483B7763",
         "cmpq    99(%rdi), %rsi",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(76543210), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size64, rdx, RegMemImm::imm(76543210)),
         "4881FAEAF48F04",
         "cmpq    $76543210, %rdx",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(-76543210i32 as u32), r8),
+        Inst::cmp_rmi_r(OperandSize::Size64, r8, RegMemImm::imm(-76543210i32 as u32)),
         "4981F8160B70FB",
         "cmpq    $-76543210, %r8",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(76543210), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size64, rsi, RegMemImm::imm(76543210)),
         "4881FEEAF48F04",
         "cmpq    $76543210, %rsi",
     ));
     //
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size32, RegMemImm::reg(r15), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size32, rdx, RegMemImm::reg(r15)),
         "4439FA",
         "cmpl    %r15d, %edx",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size32, RegMemImm::reg(rcx), r8),
+        Inst::cmp_rmi_r(OperandSize::Size32, r8, RegMemImm::reg(rcx)),
         "4139C8",
         "cmpl    %ecx, %r8d",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size32, RegMemImm::reg(rcx), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size32, rsi, RegMemImm::reg(rcx)),
         "39CE",
         "cmpl    %ecx, %esi",
     ));
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size32,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rdx,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "3B5763",
         "cmpl    99(%rdi), %edx",
@@ -3325,8 +3325,8 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size32,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             r8,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "443B4763",
         "cmpl    99(%rdi), %r8d",
@@ -3334,48 +3334,48 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size32,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rsi,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "3B7763",
         "cmpl    99(%rdi), %esi",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size32, RegMemImm::imm(76543210), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size32, rdx, RegMemImm::imm(76543210)),
         "81FAEAF48F04",
         "cmpl    $76543210, %edx",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size32, RegMemImm::imm(-76543210i32 as u32), r8),
+        Inst::cmp_rmi_r(OperandSize::Size32, r8, RegMemImm::imm(-76543210i32 as u32)),
         "4181F8160B70FB",
         "cmpl    $-76543210, %r8d",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size32, RegMemImm::imm(76543210), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size32, rsi, RegMemImm::imm(76543210)),
         "81FEEAF48F04",
         "cmpl    $76543210, %esi",
     ));
     //
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size16, RegMemImm::reg(r15), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size16, rdx, RegMemImm::reg(r15)),
         "664439FA",
         "cmpw    %r15w, %dx",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size16, RegMemImm::reg(rcx), r8),
+        Inst::cmp_rmi_r(OperandSize::Size16, r8, RegMemImm::reg(rcx)),
         "664139C8",
         "cmpw    %cx, %r8w",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size16, RegMemImm::reg(rcx), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size16, rsi, RegMemImm::reg(rcx)),
         "6639CE",
         "cmpw    %cx, %si",
     ));
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size16,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rdx,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "663B5763",
         "cmpw    99(%rdi), %dx",
@@ -3383,8 +3383,8 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size16,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             r8,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "66443B4763",
         "cmpw    99(%rdi), %r8w",
@@ -3392,48 +3392,48 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size16,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rsi,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "663B7763",
         "cmpw    99(%rdi), %si",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size16, RegMemImm::imm(23210), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size16, rdx, RegMemImm::imm(23210)),
         "6681FAAA5A",
         "cmpw    $23210, %dx",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size16, RegMemImm::imm(-7654i32 as u32), r8),
+        Inst::cmp_rmi_r(OperandSize::Size16, r8, RegMemImm::imm(-7654i32 as u32)),
         "664181F81AE2",
         "cmpw    $-7654, %r8w",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size16, RegMemImm::imm(7654), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size16, rsi, RegMemImm::imm(7654)),
         "6681FEE61D",
         "cmpw    $7654, %si",
     ));
     //
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r15), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size8, rdx, RegMemImm::reg(r15)),
         "4438FA",
         "cmpb    %r15b, %dl",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rcx), r8),
+        Inst::cmp_rmi_r(OperandSize::Size8, r8, RegMemImm::reg(rcx)),
         "4138C8",
         "cmpb    %cl, %r8b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rcx), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size8, rsi, RegMemImm::reg(rcx)),
         "4038CE",
         "cmpb    %cl, %sil",
     ));
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size8,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rdx,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "3A5763",
         "cmpb    99(%rdi), %dl",
@@ -3441,8 +3441,8 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size8,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             r8,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "443A4763",
         "cmpb    99(%rdi), %r8b",
@@ -3450,115 +3450,115 @@ fn test_x64_emit() {
     insns.push((
         Inst::cmp_rmi_r(
             OperandSize::Size8,
-            RegMemImm::mem(Amode::imm_reg(99, rdi)),
             rsi,
+            RegMemImm::mem(Amode::imm_reg(99, rdi)),
         ),
         "403A7763",
         "cmpb    99(%rdi), %sil",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::imm(70), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size8, rdx, RegMemImm::imm(70)),
         "80FA46",
         "cmpb    $70, %dl",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::imm(-76i32 as u32), r8),
+        Inst::cmp_rmi_r(OperandSize::Size8, r8, RegMemImm::imm(-76i32 as u32)),
         "4180F8B4",
         "cmpb    $-76, %r8b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::imm(76), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size8, rsi, RegMemImm::imm(76)),
         "4080FE4C",
         "cmpb    $76, %sil",
     ));
     // Extra byte-cases (paranoia!) for cmp_rmi_r for first operand = R
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rax), rbx),
+        Inst::cmp_rmi_r(OperandSize::Size8, rbx, RegMemImm::reg(rax)),
         "38C3",
         "cmpb    %al, %bl",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rbx), rax),
+        Inst::cmp_rmi_r(OperandSize::Size8, rax, RegMemImm::reg(rbx)),
         "38D8",
         "cmpb    %bl, %al",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rcx), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size8, rdx, RegMemImm::reg(rcx)),
         "38CA",
         "cmpb    %cl, %dl",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rcx), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size8, rsi, RegMemImm::reg(rcx)),
         "4038CE",
         "cmpb    %cl, %sil",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rcx), r10),
+        Inst::cmp_rmi_r(OperandSize::Size8, r10, RegMemImm::reg(rcx)),
         "4138CA",
         "cmpb    %cl, %r10b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rcx), r14),
+        Inst::cmp_rmi_r(OperandSize::Size8, r14, RegMemImm::reg(rcx)),
         "4138CE",
         "cmpb    %cl, %r14b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rbp), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size8, rdx, RegMemImm::reg(rbp)),
         "4038EA",
         "cmpb    %bpl, %dl",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rbp), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size8, rsi, RegMemImm::reg(rbp)),
         "4038EE",
         "cmpb    %bpl, %sil",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rbp), r10),
+        Inst::cmp_rmi_r(OperandSize::Size8, r10, RegMemImm::reg(rbp)),
         "4138EA",
         "cmpb    %bpl, %r10b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(rbp), r14),
+        Inst::cmp_rmi_r(OperandSize::Size8, r14, RegMemImm::reg(rbp)),
         "4138EE",
         "cmpb    %bpl, %r14b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r9), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size8, rdx, RegMemImm::reg(r9)),
         "4438CA",
         "cmpb    %r9b, %dl",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r9), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size8, rsi, RegMemImm::reg(r9)),
         "4438CE",
         "cmpb    %r9b, %sil",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r9), r10),
+        Inst::cmp_rmi_r(OperandSize::Size8, r10, RegMemImm::reg(r9)),
         "4538CA",
         "cmpb    %r9b, %r10b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r9), r14),
+        Inst::cmp_rmi_r(OperandSize::Size8, r14, RegMemImm::reg(r9)),
         "4538CE",
         "cmpb    %r9b, %r14b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r13), rdx),
+        Inst::cmp_rmi_r(OperandSize::Size8, rdx, RegMemImm::reg(r13)),
         "4438EA",
         "cmpb    %r13b, %dl",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r13), rsi),
+        Inst::cmp_rmi_r(OperandSize::Size8, rsi, RegMemImm::reg(r13)),
         "4438EE",
         "cmpb    %r13b, %sil",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r13), r10),
+        Inst::cmp_rmi_r(OperandSize::Size8, r10, RegMemImm::reg(r13)),
         "4538EA",
         "cmpb    %r13b, %r10b",
     ));
     insns.push((
-        Inst::cmp_rmi_r(OperandSize::Size8, RegMemImm::reg(r13), r14),
+        Inst::cmp_rmi_r(OperandSize::Size8, r14, RegMemImm::reg(r13)),
         "4538EE",
         "cmpb    %r13b, %r14b",
     ));
diff --git a/cranelift/filetests/filetests/isa/x64/branches.clif b/cranelift/filetests/filetests/isa/x64/branches.clif
index 57ab4de3f..fbbd876b8 100644
--- a/cranelift/filetests/filetests/isa/x64/branches.clif
+++ b/cranelift/filetests/filetests/isa/x64/branches.clif
@@ -630,15 +630,13 @@ block202:
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movl    $1112539136, %eax
-;   movd    %eax, %xmm4
-;   ucomiss %xmm4, %xmm0
+;   ucomiss const(1), %xmm0
 ;   jp      label2
 ;   jnz     label2; j label1
 ; block1:
 ;   jmp     label5
 ; block2:
-;   ucomiss %xmm4, %xmm0
+;   ucomiss const(0), %xmm0
 ;   jnp     label4; j label3
 ; block3:
 ;   jmp     label5
@@ -654,20 +652,36 @@ block202:
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movl $0x42500000, %eax
-;   movd %eax, %xmm4
-;   ucomiss %xmm4, %xmm0
-;   jp 0x1c
-;   je 0x27
-; block2: ; offset 0x1c
-;   ucomiss %xmm4, %xmm0
-;   jp 0x27
-; block3: ; offset 0x25
+;   ucomiss 0x25(%rip), %xmm0
+;   jp 0x17
+;   je 0x26
+; block2: ; offset 0x17
+;   ucomiss 0x22(%rip), %xmm0
+;   jp 0x26
+; block3: ; offset 0x24
 ;   ud2 ; trap: heap_oob
-; block4: ; offset 0x27
+; block4: ; offset 0x26
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %dl, 0x42(%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   pushq %rax
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
 
 function %br_i8_icmp(i32, i32) -> i32 {
 block0(v0: i32, v1: i32):
diff --git a/cranelift/filetests/filetests/isa/x64/f64-branches-avx.clif b/cranelift/filetests/filetests/isa/x64/f64-branches-avx.clif
new file mode 100644
index 000000000..f40100bbf
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/f64-branches-avx.clif
@@ -0,0 +1,105 @@
+test compile precise-output
+target x86_64 has_avx
+
+function %f3(f32, f32) -> i32 {
+block0(v0: f32, v1: f32):
+  v2 = fcmp eq v0, v1
+  brif v2, block1, block2
+
+block1:
+  v3 = iconst.i32 1
+  return v3
+
+block2:
+  v4 = iconst.i32 2
+  return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vucomiss %xmm1, %xmm0
+;   jp      label1
+;   jnz     label1; j label2
+; block1:
+;   movl    $2, %eax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; block2:
+;   movl    $1, %eax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vucomiss %xmm1, %xmm0
+;   jp 0x14
+;   je 0x1e
+; block2: ; offset 0x14
+;   movl $2, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+; block3: ; offset 0x1e
+;   movl $1, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f3(f64, f64) -> i32 {
+block0(v0: f64, v1: f64):
+  v2 = fcmp eq v0, v1
+  brif v2, block1, block2
+
+block1:
+  v3 = iconst.i32 1
+  return v3
+
+block2:
+  v4 = iconst.i32 2
+  return v4
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vucomisd %xmm1, %xmm0
+;   jp      label1
+;   jnz     label1; j label2
+; block1:
+;   movl    $2, %eax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; block2:
+;   movl    $1, %eax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vucomisd %xmm1, %xmm0
+;   jp 0x14
+;   je 0x1e
+; block2: ; offset 0x14
+;   movl $2, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+; block3: ; offset 0x1e
+;   movl $1, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
index ba16a98c9..22ba0bb94 100644
--- a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
+++ b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
@@ -42,18 +42,18 @@ block0(v0: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   load_ext_name %callee_i64+0, %rcx
-;   return_call_unknown %rcx (0) tmp=%rdx %rdi=%rdi
+;   load_ext_name %callee_i64+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11 %rdi=%rdi
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movabsq $0, %rcx ; reloc_external Abs8 %callee_i64 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %callee_i64 0
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rcx
+;   jmpq *%r10
 
 ;;;; Test colocated tail calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -70,18 +70,18 @@ block0(v0: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   load_ext_name %callee_i64+0, %rcx
-;   return_call_unknown %rcx (0) tmp=%rdx %rdi=%rdi
+;   load_ext_name %callee_i64+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11 %rdi=%rdi
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   leaq (%rip), %rcx ; reloc_external CallPCRel4 %callee_i64 -4
+;   leaq (%rip), %r10 ; reloc_external CallPCRel4 %callee_i64 -4
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rcx
+;   jmpq *%r10
 
 ;;;; Test passing `f64`s ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -139,18 +139,18 @@ block0(v0: f64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   load_ext_name %callee_f64+0, %rcx
-;   return_call_unknown %rcx (0) tmp=%rdx %xmm0=%xmm0
+;   load_ext_name %callee_f64+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11 %xmm0=%xmm0
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movabsq $0, %rcx ; reloc_external Abs8 %callee_f64 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %callee_f64 0
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rcx
+;   jmpq *%r10
 
 ;;;; Test passing `i8`s ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -195,18 +195,18 @@ block0(v0: i8):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   load_ext_name %callee_i8+0, %rcx
-;   return_call_unknown %rcx (0) tmp=%rdx %rdi=%rdi
+;   load_ext_name %callee_i8+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11 %rdi=%rdi
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movabsq $0, %rcx ; reloc_external Abs8 %callee_i8 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %callee_i8 0
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rcx
+;   jmpq *%r10
 
 ;;;; Test passing many arguments on stack ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -335,7 +335,7 @@ block0:
 ;   movq    rsp(72 + virtual offset), %r8
 ;   movq    rsp(64 + virtual offset), %r9
 ;   movq    rsp(0 + virtual offset), %r10
-;   return_call_unknown %r10 (160) tmp=%rax %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
+;   return_call_unknown %r10 (160) tmp=%r11 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
diff --git a/cranelift/filetests/filetests/isa/x64/return-call.clif b/cranelift/filetests/filetests/isa/x64/return-call.clif
index 8ffb1c2fd..113d495bc 100644
--- a/cranelift/filetests/filetests/isa/x64/return-call.clif
+++ b/cranelift/filetests/filetests/isa/x64/return-call.clif
@@ -40,18 +40,18 @@ block0(v0: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   load_ext_name %callee_i64+0, %rcx
-;   return_call_unknown %rcx (0) tmp=%rdx %rdi=%rdi
+;   load_ext_name %callee_i64+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11 %rdi=%rdi
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movabsq $0, %rcx ; reloc_external Abs8 %callee_i64 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %callee_i64 0
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rcx
+;   jmpq *%r10
 
 ;;;; Test colocated tail calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -66,7 +66,7 @@ block0(v0: i64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   return_call_known TestCase(%callee_i64) (0) tmp=%rax %rdi=%rdi
+;   return_call_known TestCase(%callee_i64) (0) tmp=%r11 %rdi=%rdi
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
@@ -131,18 +131,18 @@ block0(v0: f64):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   load_ext_name %callee_f64+0, %rcx
-;   return_call_unknown %rcx (0) tmp=%rdx %xmm0=%xmm0
+;   load_ext_name %callee_f64+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11 %xmm0=%xmm0
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movabsq $0, %rcx ; reloc_external Abs8 %callee_f64 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %callee_f64 0
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rcx
+;   jmpq *%r10
 
 ;;;; Test passing `i8`s ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -185,18 +185,18 @@ block0(v0: i8):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   load_ext_name %callee_i8+0, %rcx
-;   return_call_unknown %rcx (0) tmp=%rdx %rdi=%rdi
+;   load_ext_name %callee_i8+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11 %rdi=%rdi
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   movabsq $0, %rcx ; reloc_external Abs8 %callee_i8 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %callee_i8 0
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rcx
+;   jmpq *%r10
 
 ;;;; Test passing fewer arguments on the stack ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -243,7 +243,7 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32, v7: i32, v
 ;   movq    rbp(stack args max - 24), %r9
 ;   movq    rbp(stack args max - 16), %rax
 ;   movl    %eax, rbp(stack args max - 16)
-;   return_call_known TestCase(%one_stack_arg) (16) tmp=%r10 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
+;   return_call_known TestCase(%one_stack_arg) (16) tmp=%r11 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
@@ -260,8 +260,8 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32, v7: i32, v
 ;   movl %eax, 0x20(%rbp)
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   movq (%rsp), %r10
-;   movq %r10, 0x10(%rsp)
+;   movq (%rsp), %r11
+;   movq %r11, 0x10(%rsp)
 ;   addq $0x10, %rsp
 ;   jmp 0x35 ; reloc_external CallPCRel4 %one_stack_arg -4
 
@@ -279,7 +279,7 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32, v7: i32, v
 ;   movq    rbp(stack args max - 32), %r10
 ;   movq    rbp(stack args max - 24), %rsi
 ;   movq    rbp(stack args max - 16), %rdi
-;   return_call_known TestCase(%callee_i8) (0) tmp=%rdx %rdi=%rdi
+;   return_call_known TestCase(%callee_i8) (0) tmp=%r11 %rdi=%rdi
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
@@ -291,8 +291,8 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32, v7: i32, v
 ;   movq 0x20(%rbp), %rdi
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   movq (%rsp), %rdx
-;   movq %rdx, 0x20(%rsp)
+;   movq (%rsp), %r11
+;   movq %r11, 0x20(%rsp)
 ;   addq $0x20, %rsp
 ;   jmp 0x26 ; reloc_external CallPCRel4 %callee_i8 -4
 
@@ -325,7 +325,7 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32):
 ;   movl    %esi, rbp(stack args max - 16)
 ;   movq    %rsi, %rdi
 ;   movq    %r10, %rsi
-;   return_call_known TestCase(%call_one_stack_arg) (32) tmp=%r10 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
+;   return_call_known TestCase(%call_one_stack_arg) (32) tmp=%r11 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
@@ -531,14 +531,14 @@ block0:
 ;   movq    %rsi, rbp(stack args max - 16)
 ;   movq    rsp(0 + virtual offset), %rsi
 ;   movq    %rsi, rbp(stack args max - 8)
-;   load_ext_name %tail_callee_stack_args+0, %rax
+;   load_ext_name %tail_callee_stack_args+0, %r10
 ;   movq    rsp(72 + virtual offset), %rcx
 ;   movq    rsp(80 + virtual offset), %rdx
 ;   movq    rsp(88 + virtual offset), %rsi
 ;   movq    rsp(96 + virtual offset), %rdi
 ;   movq    rsp(64 + virtual offset), %r8
 ;   movq    rsp(56 + virtual offset), %r9
-;   return_call_unknown %rax (160) tmp=%r10 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
+;   return_call_unknown %r10 (160) tmp=%r11 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
@@ -623,7 +623,7 @@ block0:
 ;   movq %rsi, 0xa0(%rbp)
 ;   movq (%rsp), %rsi
 ;   movq %rsi, 0xa8(%rbp)
-;   movabsq $0, %rax ; reloc_external Abs8 %tail_callee_stack_args 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %tail_callee_stack_args 0
 ;   movq 0x48(%rsp), %rcx
 ;   movq 0x50(%rsp), %rdx
 ;   movq 0x58(%rsp), %rsi
@@ -638,5 +638,5 @@ block0:
 ;   addq $0xa0, %rsp
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   jmpq *%rax
+;   jmpq *%r10
 
diff --git a/cranelift/filetests/filetests/isa/x64/simd-logical-compile-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-logical-compile-avx.clif
new file mode 100644
index 000000000..f30b20acc
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/simd-logical-compile-avx.clif
@@ -0,0 +1,62 @@
+test compile precise-output
+target x86_64 sse42 has_avx
+
+function %vany_true_i32x4(i32x4) -> i8 {
+block0(v0: i32x4):
+    v1 = vany_true v0
+    return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vptest  %xmm0, %xmm0
+;   setnz   %al
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vptest %xmm0, %xmm0
+;   setne %al
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %vall_true_i64x2(i64x2) -> i8 {
+block0(v0: i64x2):
+    v1 = vall_true v0
+    return v1
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   uninit  %xmm2
+;   vpxor   %xmm2, %xmm2, %xmm4
+;   vpcmpeqq %xmm0, %xmm4, %xmm6
+;   vptest  %xmm6, %xmm6
+;   setz    %al
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpxor %xmm2, %xmm2, %xmm4
+;   vpcmpeqq %xmm4, %xmm0, %xmm6
+;   vptest %xmm6, %xmm6
+;   sete %al
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/tail-call-conv.clif b/cranelift/filetests/filetests/isa/x64/tail-call-conv.clif
index 93ccab77f..7a2243a0f 100644
--- a/cranelift/filetests/filetests/isa/x64/tail-call-conv.clif
+++ b/cranelift/filetests/filetests/isa/x64/tail-call-conv.clif
@@ -895,7 +895,6 @@ block0:
 ;   popq %rbp
 ;   retq
 
-
 ;; Test that tail calls that shrink the argument area don't clobber the location
 ;; of an indirect jump
 
@@ -916,8 +915,8 @@ block0(v0: f64, v1: f64, v2: i8, v3: i32, v4: i128, v5: i32, v6: i128, v7: i32,
 ;   movq    rbp(stack args max - 24), %rax
 ;   movq    rbp(stack args max - 16), %rdx
 ;   movq    rbp(stack args max - 8), %r9
-;   load_ext_name %callee_simple+0, %rsi
-;   return_call_unknown %rsi (0) tmp=%rdi
+;   load_ext_name %callee_simple+0, %r10
+;   return_call_unknown %r10 (0) tmp=%r11
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
@@ -928,11 +927,11 @@ block0(v0: f64, v1: f64, v2: i8, v3: i32, v4: i128, v5: i32, v6: i128, v7: i32,
 ;   movq 0x18(%rbp), %rax
 ;   movq 0x20(%rbp), %rdx
 ;   movq 0x28(%rbp), %r9
-;   movabsq $0, %rsi ; reloc_external Abs8 %callee_simple 0
+;   movabsq $0, %r10 ; reloc_external Abs8 %callee_simple 0
 ;   movq %rbp, %rsp
 ;   popq %rbp
-;   movq (%rsp), %rdi
-;   movq %rdi, 0x20(%rsp)
+;   movq (%rsp), %r11
+;   movq %r11, 0x20(%rsp)
 ;   addq $0x20, %rsp
-;   jmpq *%rsi
+;   jmpq *%r10
 
diff --git a/cranelift/filetests/filetests/isa/x64/very-carefully-sink-loads-in-float-compares.clif b/cranelift/filetests/filetests/isa/x64/very-carefully-sink-loads-in-float-compares.clif
new file mode 100644
index 000000000..01c9d4db2
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/very-carefully-sink-loads-in-float-compares.clif
@@ -0,0 +1,499 @@
+test compile precise-output
+target x86_64
+
+;; This test ensures that Cranelift does not attempt to too eagerly sink loads
+;; into float comparison operations. When a single `fcmp` is used multiple times
+;; it'll regenerate `ucomiss` instructions, for example, but if this duplication
+;; happens then it shouldn't have a load sunk into it because that would load
+;; at two different points in the program, possibly seeing two different
+;; results.
+;;
+;; Exactly how this guarantee is provided has changed a bit over time with
+;; Cranelift. Originally in #3934 this was done explicitly in ISLE by avoiding
+;; the use of `put_in_xmm_mem`. Later in #4061 that ended up removing the need
+;; for the safeguards in #3934. Today there are no guards in ISLE for this any
+;; more and this test is intended to serve as a reference-test for notifying
+;; reviewers if anything changes.
+
+function %select_cond_rhs(f32, i64, i32, i32) -> i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v0, v4
+  v6 = select v5, v2, v3
+  return v6
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   ucomiss 0(%rdi), %xmm0
+;   cmovpl  %edx, %esi, %esi
+;   movq    %rsi, %rax
+;   cmovnzl %edx, %eax, %eax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   ucomiss (%rdi), %xmm0
+;   cmovpl %edx, %esi
+;   movq %rsi, %rax
+;   cmovnel %edx, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+
+function %select_cond_lhs(f32, i64, i32, i32) -> i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v4, v0
+  v6 = select v5, v2, v3
+  return v6
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm7
+;   ucomiss %xmm0, %xmm7
+;   cmovpl  %edx, %esi, %esi
+;   movq    %rsi, %rax
+;   cmovnzl %edx, %eax, %eax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm7
+;   ucomiss %xmm0, %xmm7
+;   cmovpl %edx, %esi
+;   movq %rsi, %rax
+;   cmovnel %edx, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %select_cond_used_twice_rhs(f32, i64, i32, i32) -> i32, i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  ;; this load should NOT sink into either `select` below
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v0, v4
+  v6 = select v5, v2, v3
+  v7 = select v5, v3, v2
+  return v6, v7
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm1
+;   ucomiss %xmm1, %xmm0
+;   movq    %rsi, %rax
+;   cmovpl  %edx, %eax, %eax
+;   cmovnzl %edx, %eax, %eax
+;   ucomiss %xmm1, %xmm0
+;   cmovpl  %esi, %edx, %edx
+;   cmovnzl %esi, %edx, %edx
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm1
+;   ucomiss %xmm1, %xmm0
+;   movq %rsi, %rax
+;   cmovpl %edx, %eax
+;   cmovnel %edx, %eax
+;   ucomiss %xmm1, %xmm0
+;   cmovpl %esi, %edx
+;   cmovnel %esi, %edx
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+
+function %select_cond_used_twice_lhs(f32, i64, i32, i32) -> i32, i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  ;; this load should NOT sink into either `select` below
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v4, v0
+  v6 = select v5, v2, v3
+  v7 = select v5, v3, v2
+  return v6, v7
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm1
+;   ucomiss %xmm0, %xmm1
+;   movq    %rsi, %rax
+;   cmovpl  %edx, %eax, %eax
+;   cmovnzl %edx, %eax, %eax
+;   ucomiss %xmm0, %xmm1
+;   cmovpl  %esi, %edx, %edx
+;   cmovnzl %esi, %edx, %edx
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm1
+;   ucomiss %xmm0, %xmm1
+;   movq %rsi, %rax
+;   cmovpl %edx, %eax
+;   cmovnel %edx, %eax
+;   ucomiss %xmm0, %xmm1
+;   cmovpl %esi, %edx
+;   cmovnel %esi, %edx
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %brif_cond_rhs(f32, i64, i32, i32) -> i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v0, v4
+  brif v5, block1(v2), block1(v3)
+block1(v8: i32):
+  return v8
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm1
+;   ucomiss %xmm1, %xmm0
+;   jp      label2
+;   jnz     label2; j label1
+; block1:
+;   movq    %rsi, %rax
+;   jmp     label3
+; block2:
+;   movq    %rdx, %rax
+;   jmp     label3
+; block3:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm1
+;   ucomiss %xmm1, %xmm0
+;   jp 0x1f
+;   jne 0x1f
+; block2: ; offset 0x17
+;   movq %rsi, %rax
+;   jmp 0x22
+; block3: ; offset 0x1f
+;   movq %rdx, %rax
+; block4: ; offset 0x22
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %brif_cond_lhs(f32, i64, i32, i32) -> i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v4, v0
+  brif v5, block1(v2), block1(v3)
+block1(v8: i32):
+  return v8
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm1
+;   ucomiss %xmm0, %xmm1
+;   jp      label2
+;   jnz     label2; j label1
+; block1:
+;   movq    %rsi, %rax
+;   jmp     label3
+; block2:
+;   movq    %rdx, %rax
+;   jmp     label3
+; block3:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm1
+;   ucomiss %xmm0, %xmm1
+;   jp 0x1f
+;   jne 0x1f
+; block2: ; offset 0x17
+;   movq %rsi, %rax
+;   jmp 0x22
+; block3: ; offset 0x1f
+;   movq %rdx, %rax
+; block4: ; offset 0x22
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %brif_cond_used_twice_rhs(f32, i64, i32, i32) -> i32, i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v0, v4
+  brif v5, block1(v2), block1(v3)
+block1(v6: i32):
+  brif v5, block2(v3), block2(v2)
+block2(v7: i32):
+  return v6, v7
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm3
+;   ucomiss %xmm3, %xmm0
+;   jp      label2
+;   jnz     label2; j label1
+; block1:
+;   movq    %rsi, %rax
+;   jmp     label3
+; block2:
+;   movq    %rdx, %rax
+;   jmp     label3
+; block3:
+;   ucomiss %xmm3, %xmm0
+;   jp      label5
+;   jnz     label5; j label4
+; block4:
+;   jmp     label6
+; block5:
+;   movq    %rsi, %rdx
+;   jmp     label6
+; block6:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm3
+;   ucomiss %xmm3, %xmm0
+;   jp 0x1f
+;   jne 0x1f
+; block2: ; offset 0x17
+;   movq %rsi, %rax
+;   jmp 0x22
+; block3: ; offset 0x1f
+;   movq %rdx, %rax
+; block4: ; offset 0x22
+;   ucomiss %xmm3, %xmm0
+;   jp 0x31
+;   je 0x34
+; block5: ; offset 0x31
+;   movq %rsi, %rdx
+; block6: ; offset 0x34
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %brif_cond_used_twice_lhs(f32, i64, i32, i32) -> i32, i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v4, v0
+  brif v5, block1(v2), block1(v3)
+block1(v6: i32):
+  brif v5, block2(v3), block2(v2)
+block2(v7: i32):
+  return v6, v7
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm3
+;   ucomiss %xmm0, %xmm3
+;   jp      label2
+;   jnz     label2; j label1
+; block1:
+;   movq    %rsi, %rax
+;   jmp     label3
+; block2:
+;   movq    %rdx, %rax
+;   jmp     label3
+; block3:
+;   ucomiss %xmm0, %xmm3
+;   jp      label5
+;   jnz     label5; j label4
+; block4:
+;   jmp     label6
+; block5:
+;   movq    %rsi, %rdx
+;   jmp     label6
+; block6:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm3
+;   ucomiss %xmm0, %xmm3
+;   jp 0x1f
+;   jne 0x1f
+; block2: ; offset 0x17
+;   movq %rsi, %rax
+;   jmp 0x22
+; block3: ; offset 0x1f
+;   movq %rdx, %rax
+; block4: ; offset 0x22
+;   ucomiss %xmm0, %xmm3
+;   jp 0x31
+;   je 0x34
+; block5: ; offset 0x31
+;   movq %rsi, %rdx
+; block6: ; offset 0x34
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %mix_select_and_brif_rhs(f32, i64, i32, i32) -> i32, i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v0, v4
+  v6 = select v5, v2, v3
+  brif v5, block1(v3), block1(v2)
+block1(v7: i32):
+  return v6, v7
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm2
+;   ucomiss %xmm2, %xmm0
+;   movq    %rsi, %rax
+;   cmovpl  %edx, %eax, %eax
+;   cmovnzl %edx, %eax, %eax
+;   ucomiss %xmm2, %xmm0
+;   jp      label2
+;   jnz     label2; j label1
+; block1:
+;   jmp     label3
+; block2:
+;   movq    %rsi, %rdx
+;   jmp     label3
+; block3:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm2
+;   ucomiss %xmm2, %xmm0
+;   movq %rsi, %rax
+;   cmovpl %edx, %eax
+;   cmovnel %edx, %eax
+;   ucomiss %xmm2, %xmm0
+;   jp 0x23
+;   je 0x26
+; block2: ; offset 0x23
+;   movq %rsi, %rdx
+; block3: ; offset 0x26
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %mix_select_and_brif_lhs(f32, i64, i32, i32) -> i32, i32 {
+block0(v0: f32, v1: i64, v2: i32, v3: i32):
+  v4 = load.f32 notrap aligned v1
+  v5 = fcmp eq v4, v0
+  v6 = select v5, v2, v3
+  brif v5, block1(v3), block1(v2)
+block1(v7: i32):
+  return v6, v7
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movss   0(%rdi), %xmm2
+;   ucomiss %xmm0, %xmm2
+;   movq    %rsi, %rax
+;   cmovpl  %edx, %eax, %eax
+;   cmovnzl %edx, %eax, %eax
+;   ucomiss %xmm0, %xmm2
+;   jp      label2
+;   jnz     label2; j label1
+; block1:
+;   jmp     label3
+; block2:
+;   movq    %rsi, %rdx
+;   jmp     label3
+; block3:
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movss (%rdi), %xmm2
+;   ucomiss %xmm0, %xmm2
+;   movq %rsi, %rax
+;   cmovpl %edx, %eax
+;   cmovnel %edx, %eax
+;   ucomiss %xmm0, %xmm2
+;   jp 0x23
+;   je 0x26
+; block2: ; offset 0x23
+;   movq %rsi, %rdx
+; block3: ; offset 0x26
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/winch.clif b/cranelift/filetests/filetests/isa/x64/winch.clif
index 808b73d57..8832531b7 100644
--- a/cranelift/filetests/filetests/isa/x64/winch.clif
+++ b/cranelift/filetests/filetests/isa/x64/winch.clif
@@ -38,8 +38,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   subq    %rsp, $16, %rsp
 ; block0:
 ;   movq    %rdi, rsp(0 + virtual offset)
-;   load_ext_name %g+0, %r15
-;   call    *%r15
+;   load_ext_name %g+0, %r10
+;   call    *%r10
 ;   movq    rsp(0 + virtual offset), %rax
 ;   addq    %rsp, $16, %rsp
 ;   movq    %rbp, %rsp
@@ -53,8 +53,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   subq $0x10, %rsp
 ; block1: ; offset 0x8
 ;   movq %rdi, (%rsp)
-;   movabsq $0, %r15 ; reloc_external Abs8 %g 0
-;   callq *%r15
+;   movabsq $0, %r10 ; reloc_external Abs8 %g 0
+;   callq *%r10
 ;   movq (%rsp), %rax
 ;   addq $0x10, %rsp
 ;   movq %rbp, %rsp
@@ -81,8 +81,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   movq    %r15, 48(%rsp)
 ; block0:
 ;   movq    %rdi, rsp(0 + virtual offset)
-;   load_ext_name %g+0, %r15
-;   call    *%r15
+;   load_ext_name %g+0, %r10
+;   call    *%r10
 ;   movq    rsp(0 + virtual offset), %rax
 ;   movq    16(%rsp), %rbx
 ;   movq    24(%rsp), %r12
@@ -106,8 +106,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   movq %r15, 0x30(%rsp)
 ; block1: ; offset 0x21
 ;   movq %rdi, (%rsp)
-;   movabsq $0, %r15 ; reloc_external Abs8 %g 0
-;   callq *%r15
+;   movabsq $0, %r10 ; reloc_external Abs8 %g 0
+;   callq *%r10
 ;   movq (%rsp), %rax
 ;   movq 0x10(%rsp), %rbx
 ;   movq 0x18(%rsp), %r12
@@ -135,8 +135,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   movq    %rdi, %rax
 ;   movq    %r9, %rdi
 ;   movq    %rax, %r9
-;   load_ext_name %g+0, %r15
-;   call    *%r15
+;   load_ext_name %g+0, %r10
+;   call    *%r10
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -149,8 +149,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   movq %rdi, %rax
 ;   movq %r9, %rdi
 ;   movq %rax, %r9
-;   movabsq $0, %r15 ; reloc_external Abs8 %g 0
-;   callq *%r15
+;   movabsq $0, %r10 ; reloc_external Abs8 %g 0
+;   callq *%r10
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -177,8 +177,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   movq    %rdi, %rax
 ;   movq    %r9, %rdi
 ;   movq    %rax, %r9
-;   load_ext_name %g+0, %r15
-;   call    *%r15
+;   load_ext_name %g+0, %r10
+;   call    *%r10
 ;   movq    0(%rsp), %rbx
 ;   movq    8(%rsp), %r12
 ;   movq    16(%rsp), %r13
@@ -203,8 +203,8 @@ block0(v0:i64, v1:i64, v2:i64, v3:i64, v4:i64, v5:i64):
 ;   movq %rdi, %rax
 ;   movq %r9, %rdi
 ;   movq %rax, %r9
-;   movabsq $0, %r15 ; reloc_external Abs8 %g 0
-;   callq *%r15
+;   movabsq $0, %r10 ; reloc_external Abs8 %g 0
+;   callq *%r10
 ;   movq (%rsp), %rbx
 ;   movq 8(%rsp), %r12
 ;   movq 0x10(%rsp), %r13
@@ -236,12 +236,12 @@ block0:
 ;   movq    %r14, 40(%rsp)
 ;   movq    %r15, 48(%rsp)
 ; block0:
-;   load_ext_name userextname0+0, %r15
-;   movq    %r15, rsp(0 + virtual offset)
-;   movq    rsp(0 + virtual offset), %r15
-;   call    *%r15
-;   movq    rsp(0 + virtual offset), %r15
-;   call    *%r15
+;   load_ext_name userextname0+0, %r10
+;   movq    %r10, rsp(0 + virtual offset)
+;   movq    rsp(0 + virtual offset), %r10
+;   call    *%r10
+;   movq    rsp(0 + virtual offset), %r10
+;   call    *%r10
 ;   movq    16(%rsp), %rbx
 ;   movq    24(%rsp), %r12
 ;   movq    32(%rsp), %r13
@@ -263,12 +263,12 @@ block0:
 ;   movq %r14, 0x28(%rsp)
 ;   movq %r15, 0x30(%rsp)
 ; block1: ; offset 0x21
-;   movabsq $0, %r15 ; reloc_external Abs8 u2:0 0
-;   movq %r15, (%rsp)
-;   movq (%rsp), %r15
-;   callq *%r15
-;   movq (%rsp), %r15
-;   callq *%r15
+;   movabsq $0, %r10 ; reloc_external Abs8 u2:0 0
+;   movq %r10, (%rsp)
+;   movq (%rsp), %r10
+;   callq *%r10
+;   movq (%rsp), %r10
+;   callq *%r10
 ;   movq 0x10(%rsp), %rbx
 ;   movq 0x18(%rsp), %r12
 ;   movq 0x20(%rsp), %r13
@@ -301,8 +301,8 @@ block0(v0:i64):
 ;   movq    %r15, 48(%rsp)
 ; block0:
 ;   lea     0(%rsp), %rdi
-;   load_ext_name %g+0, %r15
-;   call    *%r15
+;   load_ext_name %g+0, %r10
+;   call    *%r10
 ;   movq    4(%rsp), %rax
 ;   movq    0(%rsp), %r11
 ;   andl    %eax, %r11d, %eax
@@ -328,8 +328,8 @@ block0(v0:i64):
 ;   movq %r15, 0x30(%rsp)
 ; block1: ; offset 0x21
 ;   leaq (%rsp), %rdi
-;   movabsq $0, %r15 ; reloc_external Abs8 %g 0
-;   callq *%r15
+;   movabsq $0, %r10 ; reloc_external Abs8 %g 0
+;   callq *%r10
 ;   movq 4(%rsp), %rax
 ;   movq (%rsp), %r11
 ;   andl %r11d, %eax
diff --git a/cranelift/filetests/src/concurrent.rs b/cranelift/filetests/src/concurrent.rs
index 5d07874c7..027a81233 100644
--- a/cranelift/filetests/src/concurrent.rs
+++ b/cranelift/filetests/src/concurrent.rs
@@ -21,7 +21,6 @@ struct Request(usize, PathBuf);
 pub enum Reply {
     Starting {
         jobid: usize,
-        thread_num: usize,
     },
     Done {
         jobid: usize,
@@ -145,7 +144,7 @@ fn worker_thread(
 
                 // Tell them we're starting this job.
                 // The receiver should always be present for this as long as we have jobs.
-                replies.send(Reply::Starting { jobid, thread_num }).unwrap();
+                replies.send(Reply::Starting { jobid }).unwrap();
 
                 let result = catch_unwind(|| runone::run(path.as_path(), None, None))
                     .unwrap_or_else(|e| {
diff --git a/cranelift/filetests/src/subtest.rs b/cranelift/filetests/src/subtest.rs
index 24c0d8a6b..a0761ad9b 100644
--- a/cranelift/filetests/src/subtest.rs
+++ b/cranelift/filetests/src/subtest.rs
@@ -29,6 +29,7 @@ pub struct Context<'a> {
     pub isa: Option<&'a dyn TargetIsa>,
 
     /// Full path to the file containing the test.
+    #[allow(dead_code)]
     pub file_path: &'a str,
 
     /// Context used to update the original `file_path` in-place with its test
diff --git a/crates/component-macro/tests/codegen.rs b/crates/component-macro/tests/codegen.rs
index 76e228203..e646528c3 100644
--- a/crates/component-macro/tests/codegen.rs
+++ b/crates/component-macro/tests/codegen.rs
@@ -1,3 +1,5 @@
+#![allow(dead_code)]
+
 macro_rules! gentest {
     ($id:ident $name:tt $path:tt) => {
         mod $id {
diff --git a/crates/test-programs/src/bin/preview1_file_pread_pwrite.rs b/crates/test-programs/src/bin/preview1_file_pread_pwrite.rs
index ff1630f43..77061e721 100644
--- a/crates/test-programs/src/bin/preview1_file_pread_pwrite.rs
+++ b/crates/test-programs/src/bin/preview1_file_pread_pwrite.rs
@@ -165,7 +165,7 @@ unsafe fn test_file_pwrite_and_file_pos(dir_fd: wasi::Fd) {
         buf: buf.as_ptr(),
         buf_len: buf.len(),
     };
-    let n = wasi::fd_pwrite(file_fd, &mut [ciovec], 50).expect("writing bytes at offset 2");
+    let n = wasi::fd_pwrite(file_fd, &mut [ciovec], 50).expect("writing bytes at offset 50");
     assert_eq!(n, 1);
 
     assert_eq!(wasi::fd_tell(file_fd).unwrap(), 0);
diff --git a/crates/test-programs/src/bin/preview1_file_read_write.rs b/crates/test-programs/src/bin/preview1_file_read_write.rs
new file mode 100644
index 000000000..f32825a27
--- /dev/null
+++ b/crates/test-programs/src/bin/preview1_file_read_write.rs
@@ -0,0 +1,209 @@
+use std::{env, process};
+use test_programs::preview1::open_scratch_directory;
+
+unsafe fn test_file_read_write(dir_fd: wasi::Fd) {
+    // Create a file in the scratch directory.
+    let file_fd = wasi::path_open(
+        dir_fd,
+        0,
+        "file",
+        wasi::OFLAGS_CREAT,
+        wasi::RIGHTS_FD_READ | wasi::RIGHTS_FD_WRITE,
+        0,
+        0,
+    )
+    .expect("opening a file");
+    assert!(
+        file_fd > libc::STDERR_FILENO as wasi::Fd,
+        "file descriptor range check",
+    );
+
+    let contents = &[0u8, 1, 2, 3];
+    let ciovec = wasi::Ciovec {
+        buf: contents.as_ptr() as *const _,
+        buf_len: contents.len(),
+    };
+    let mut nwritten = wasi::fd_write(file_fd, &mut [ciovec]).expect("writing bytes at offset 0");
+    assert_eq!(nwritten, 4, "nwritten bytes check");
+
+    let contents = &mut [0u8; 4];
+    let iovec = wasi::Iovec {
+        buf: contents.as_mut_ptr() as *mut _,
+        buf_len: contents.len(),
+    };
+    wasi::fd_seek(file_fd, 0, wasi::WHENCE_SET).expect("seeking to offset 0");
+    let mut nread = wasi::fd_read(file_fd, &[iovec]).expect("reading bytes at offset 0");
+    assert_eq!(nread, 4, "nread bytes check");
+    assert_eq!(contents, &[0u8, 1, 2, 3], "written bytes equal read bytes");
+
+    // Write all the data through multiple iovecs.
+    //
+    // Note that this needs to be done with a loop, because some
+    // platforms do not support writing multiple iovecs at once.
+    // See https://github.com/rust-lang/rust/issues/74825.
+    let contents = &[0u8, 1, 2, 3];
+    let mut offset = 0usize;
+    wasi::fd_seek(file_fd, 0, wasi::WHENCE_SET).expect("seeking to offset 0");
+    loop {
+        let mut ciovecs: Vec<wasi::Ciovec> = Vec::new();
+        let mut remaining = contents.len() - offset;
+        if remaining > 2 {
+            ciovecs.push(wasi::Ciovec {
+                buf: contents[offset..].as_ptr() as *const _,
+                buf_len: 2,
+            });
+            remaining -= 2;
+        }
+        ciovecs.push(wasi::Ciovec {
+            buf: contents[contents.len() - remaining..].as_ptr() as *const _,
+            buf_len: remaining,
+        });
+
+        nwritten = wasi::fd_write(file_fd, ciovecs.as_slice()).expect("writing bytes at offset 0");
+
+        offset += nwritten;
+        if offset == contents.len() {
+            break;
+        }
+    }
+    assert_eq!(offset, 4, "nread bytes check");
+
+    // Read all the data through multiple iovecs.
+    //
+    // Note that this needs to be done with a loop, because some
+    // platforms do not support reading multiple iovecs at once.
+    // See https://github.com/rust-lang/rust/issues/74825.
+    let contents = &mut [0u8; 4];
+    let mut offset = 0usize;
+    wasi::fd_seek(file_fd, 0, wasi::WHENCE_SET).expect("seeking to offset 0");
+    loop {
+        let buffer = &mut [0u8; 4];
+        let iovecs = &[
+            wasi::Iovec {
+                buf: buffer.as_mut_ptr() as *mut _,
+                buf_len: 2,
+            },
+            wasi::Iovec {
+                buf: buffer[2..].as_mut_ptr() as *mut _,
+                buf_len: 2,
+            },
+        ];
+        nread = wasi::fd_read(file_fd, iovecs).expect("reading bytes at offset 0");
+        if nread == 0 {
+            break;
+        }
+        contents[offset..offset + nread].copy_from_slice(&buffer[0..nread]);
+        offset += nread;
+    }
+    assert_eq!(offset, 4, "nread bytes check");
+    assert_eq!(contents, &[0u8, 1, 2, 3], "file cursor was overwritten");
+
+    let contents = &mut [0u8; 4];
+    let iovec = wasi::Iovec {
+        buf: contents.as_mut_ptr() as *mut _,
+        buf_len: contents.len(),
+    };
+    wasi::fd_seek(file_fd, 2, wasi::WHENCE_SET).expect("seeking to offset 2");
+    nread = wasi::fd_read(file_fd, &[iovec]).expect("reading bytes at offset 2");
+    assert_eq!(nread, 2, "nread bytes check");
+    assert_eq!(contents, &[2u8, 3, 0, 0], "file cursor was overwritten");
+
+    let contents = &[1u8, 0];
+    let ciovec = wasi::Ciovec {
+        buf: contents.as_ptr() as *const _,
+        buf_len: contents.len(),
+    };
+    wasi::fd_seek(file_fd, 2, wasi::WHENCE_SET).expect("seeking to offset 2");
+    nwritten = wasi::fd_write(file_fd, &mut [ciovec]).expect("writing bytes at offset 2");
+    assert_eq!(nwritten, 2, "nwritten bytes check");
+
+    let contents = &mut [0u8; 4];
+    let iovec = wasi::Iovec {
+        buf: contents.as_mut_ptr() as *mut _,
+        buf_len: contents.len(),
+    };
+    wasi::fd_seek(file_fd, 0, wasi::WHENCE_SET).expect("seeking to offset 0");
+    nread = wasi::fd_read(file_fd, &[iovec]).expect("reading bytes at offset 0");
+    assert_eq!(nread, 4, "nread bytes check");
+    assert_eq!(contents, &[0u8, 1, 1, 0], "file cursor was overwritten");
+
+    wasi::fd_close(file_fd).expect("closing a file");
+    wasi::path_unlink_file(dir_fd, "file").expect("removing a file");
+}
+
+unsafe fn test_file_write_and_file_pos(dir_fd: wasi::Fd) {
+    let path = "file2";
+    let file_fd = wasi::path_open(
+        dir_fd,
+        0,
+        path,
+        wasi::OFLAGS_CREAT,
+        wasi::RIGHTS_FD_READ | wasi::RIGHTS_FD_WRITE,
+        0,
+        0,
+    )
+    .expect("opening a file");
+    assert!(
+        file_fd > libc::STDERR_FILENO as wasi::Fd,
+        "file descriptor range check",
+    );
+
+    // Perform a 0-sized pwrite at an offset beyond the end of the file. Unix
+    // semantics should pop out where nothing is actually written and the size
+    // of the file isn't modified.
+    assert_eq!(wasi::fd_tell(file_fd).unwrap(), 0);
+    let ciovec = wasi::Ciovec {
+        buf: [].as_ptr(),
+        buf_len: 0,
+    };
+    wasi::fd_seek(file_fd, 2, wasi::WHENCE_SET).expect("seeking to offset 2");
+    let n = wasi::fd_write(file_fd, &mut [ciovec]).expect("writing bytes at offset 2");
+    assert_eq!(n, 0);
+
+    assert_eq!(wasi::fd_tell(file_fd).unwrap(), 2);
+    let stat = wasi::fd_filestat_get(file_fd).unwrap();
+    assert_eq!(stat.size, 0);
+
+    // Now write a single byte and make sure it actually works
+    let buf = [0];
+    let ciovec = wasi::Ciovec {
+        buf: buf.as_ptr(),
+        buf_len: buf.len(),
+    };
+    wasi::fd_seek(file_fd, 50, wasi::WHENCE_SET).expect("seeking to offset 50");
+    let n = wasi::fd_write(file_fd, &mut [ciovec]).expect("writing bytes at offset 50");
+    assert_eq!(n, 1);
+
+    assert_eq!(wasi::fd_tell(file_fd).unwrap(), 51);
+    let stat = wasi::fd_filestat_get(file_fd).unwrap();
+    assert_eq!(stat.size, 51);
+
+    wasi::fd_close(file_fd).expect("closing a file");
+    wasi::path_unlink_file(dir_fd, path).expect("removing a file");
+}
+
+fn main() {
+    let mut args = env::args();
+    let prog = args.next().unwrap();
+    let arg = if let Some(arg) = args.next() {
+        arg
+    } else {
+        eprintln!("usage: {} <scratch directory>", prog);
+        process::exit(1);
+    };
+
+    // Open scratch directory
+    let dir_fd = match open_scratch_directory(&arg) {
+        Ok(dir_fd) => dir_fd,
+        Err(err) => {
+            eprintln!("{}", err);
+            process::exit(1)
+        }
+    };
+
+    // Run the tests.
+    unsafe {
+        test_file_read_write(dir_fd);
+        test_file_write_and_file_pos(dir_fd);
+    }
+}
diff --git a/crates/wasi-common/tests/all/async_.rs b/crates/wasi-common/tests/all/async_.rs
index 70f187f78..75f43973f 100644
--- a/crates/wasi-common/tests/all/async_.rs
+++ b/crates/wasi-common/tests/all/async_.rs
@@ -133,6 +133,10 @@ async fn preview1_file_pread_pwrite() {
     run(PREVIEW1_FILE_PREAD_PWRITE, true).await.unwrap()
 }
 #[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn preview1_file_read_write() {
+    run(PREVIEW1_FILE_READ_WRITE, true).await.unwrap()
+}
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
 async fn preview1_file_seek_tell() {
     run(PREVIEW1_FILE_SEEK_TELL, true).await.unwrap()
 }
diff --git a/crates/wasi-common/tests/all/sync.rs b/crates/wasi-common/tests/all/sync.rs
index ccf0a5783..bc5fd348f 100644
--- a/crates/wasi-common/tests/all/sync.rs
+++ b/crates/wasi-common/tests/all/sync.rs
@@ -125,6 +125,10 @@ fn preview1_file_pread_pwrite() {
     run(PREVIEW1_FILE_PREAD_PWRITE, true).unwrap()
 }
 #[test_log::test]
+fn preview1_file_read_write() {
+    run(PREVIEW1_FILE_READ_WRITE, true).unwrap()
+}
+#[test_log::test]
 fn preview1_file_seek_tell() {
     run(PREVIEW1_FILE_SEEK_TELL, true).unwrap()
 }
diff --git a/crates/wasi/tests/all/async_.rs b/crates/wasi/tests/all/async_.rs
index cec59bbe1..de8cb47b8 100644
--- a/crates/wasi/tests/all/async_.rs
+++ b/crates/wasi/tests/all/async_.rs
@@ -99,6 +99,12 @@ async fn preview1_file_pread_pwrite() {
         .unwrap()
 }
 #[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn preview1_file_read_write() {
+    run(PREVIEW1_FILE_READ_WRITE_COMPONENT, false)
+        .await
+        .unwrap()
+}
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
 async fn preview1_file_seek_tell() {
     run(PREVIEW1_FILE_SEEK_TELL_COMPONENT, false).await.unwrap()
 }
diff --git a/crates/wasi/tests/all/preview1.rs b/crates/wasi/tests/all/preview1.rs
index 544e21424..6bd21d8e4 100644
--- a/crates/wasi/tests/all/preview1.rs
+++ b/crates/wasi/tests/all/preview1.rs
@@ -86,6 +86,10 @@ async fn preview1_file_pread_pwrite() {
     run(PREVIEW1_FILE_PREAD_PWRITE, false).await.unwrap()
 }
 #[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn preview1_file_read_write() {
+    run(PREVIEW1_FILE_READ_WRITE, false).await.unwrap()
+}
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
 async fn preview1_file_seek_tell() {
     run(PREVIEW1_FILE_SEEK_TELL, false).await.unwrap()
 }
diff --git a/crates/wasi/tests/all/sync.rs b/crates/wasi/tests/all/sync.rs
index 54528dd67..25da8c537 100644
--- a/crates/wasi/tests/all/sync.rs
+++ b/crates/wasi/tests/all/sync.rs
@@ -91,6 +91,10 @@ fn preview1_file_pread_pwrite() {
     run(PREVIEW1_FILE_PREAD_PWRITE_COMPONENT, false).unwrap()
 }
 #[test_log::test]
+fn preview1_file_read_write() {
+    run(PREVIEW1_FILE_READ_WRITE_COMPONENT, false).unwrap()
+}
+#[test_log::test]
 fn preview1_file_seek_tell() {
     run(PREVIEW1_FILE_SEEK_TELL_COMPONENT, false).unwrap()
 }
diff --git a/tests/all/component_model/bindgen.rs b/tests/all/component_model/bindgen.rs
index 764836491..0a6bc3636 100644
--- a/tests/all/component_model/bindgen.rs
+++ b/tests/all/component_model/bindgen.rs
@@ -1,4 +1,5 @@
 #![cfg(not(miri))]
+#![allow(dead_code)]
 
 use super::engine;
 use anyhow::Result;
diff --git a/tests/all/component_model/bindgen/ownership.rs b/tests/all/component_model/bindgen/ownership.rs
index 68ebab857..b16690a34 100644
--- a/tests/all/component_model/bindgen/ownership.rs
+++ b/tests/all/component_model/bindgen/ownership.rs
@@ -114,8 +114,9 @@ fn component() -> String {
     )
 }
 
-#[test]
-fn owning() -> Result<()> {
+mod owning {
+    use super::*;
+
     wasmtime::component::bindgen!({
         inline: "
         package inline:inline;
@@ -157,33 +158,36 @@ fn owning() -> Result<()> {
         }
     }
 
-    let engine = engine();
-    let component = Component::new(&engine, component())?;
+    #[test]
+    fn owning() -> Result<()> {
+        let engine = engine();
+        let component = Component::new(&engine, component())?;
 
-    let linker = Linker::new(&engine);
-    let mut store = Store::new(&engine, ());
-    let (test, _) = Test::instantiate(&mut store, &component, &linker)?;
+        let linker = Linker::new(&engine);
+        let mut store = Store::new(&engine, ());
+        let (test, _) = Test::instantiate(&mut store, &component, &linker)?;
 
-    let value = vec![vec!["a".to_owned(), "b".to_owned()]];
-    assert_eq!(value, test.lists().call_foo(&mut store, &value)?);
+        let value = vec![vec!["a".to_owned(), "b".to_owned()]];
+        assert_eq!(value, test.lists().call_foo(&mut store, &value)?);
 
-    let value = exports::thing_in::Thing {
-        name: "thing 1".to_owned(),
-        value: vec!["some value".to_owned(), "another value".to_owned()],
-    };
-    test.thing_in().call_bar(&mut store, &value)?;
+        let value = exports::thing_in::Thing {
+            name: "thing 1".to_owned(),
+            value: vec!["some value".to_owned(), "another value".to_owned()],
+        };
+        test.thing_in().call_bar(&mut store, &value)?;
 
-    let value = exports::thing_in_and_out::Thing {
-        name: "thing 1".to_owned(),
-        value: vec!["some value".to_owned(), "another value".to_owned()],
-    };
-    assert_eq!(value, test.thing_in_and_out().call_baz(&mut store, &value)?);
+        let value = exports::thing_in_and_out::Thing {
+            name: "thing 1".to_owned(),
+            value: vec!["some value".to_owned(), "another value".to_owned()],
+        };
+        assert_eq!(value, test.thing_in_and_out().call_baz(&mut store, &value)?);
 
-    Ok(())
+        Ok(())
+    }
 }
 
-#[test]
-fn borrowing_no_duplication() -> Result<()> {
+mod borrowing_no_duplication {
+    use super::*;
     wasmtime::component::bindgen!({
         inline: "
         package inline:inline;
@@ -227,33 +231,37 @@ fn borrowing_no_duplication() -> Result<()> {
         }
     }
 
-    let engine = engine();
-    let component = Component::new(&engine, component())?;
+    #[test]
+    fn borrowing_no_duplication() -> Result<()> {
+        let engine = engine();
+        let component = Component::new(&engine, component())?;
 
-    let linker = Linker::new(&engine);
-    let mut store = Store::new(&engine, ());
-    let (test, _) = Test::instantiate(&mut store, &component, &linker)?;
+        let linker = Linker::new(&engine);
+        let mut store = Store::new(&engine, ());
+        let (test, _) = Test::instantiate(&mut store, &component, &linker)?;
 
-    let value = &[&["a", "b"] as &[_]] as &[_];
-    assert_eq!(value, test.lists().call_foo(&mut store, value)?);
+        let value = &[&["a", "b"] as &[_]] as &[_];
+        assert_eq!(value, test.lists().call_foo(&mut store, value)?);
 
-    let value = exports::thing_in::Thing {
-        name: "thing 1",
-        value: &["some value", "another value"],
-    };
-    test.thing_in().call_bar(&mut store, value)?;
+        let value = exports::thing_in::Thing {
+            name: "thing 1",
+            value: &["some value", "another value"],
+        };
+        test.thing_in().call_bar(&mut store, value)?;
 
-    let value = exports::thing_in_and_out::Thing {
-        name: "thing 1".to_owned(),
-        value: vec!["some value".to_owned(), "another value".to_owned()],
-    };
-    assert_eq!(value, test.thing_in_and_out().call_baz(&mut store, &value)?);
+        let value = exports::thing_in_and_out::Thing {
+            name: "thing 1".to_owned(),
+            value: vec!["some value".to_owned(), "another value".to_owned()],
+        };
+        assert_eq!(value, test.thing_in_and_out().call_baz(&mut store, &value)?);
 
-    Ok(())
+        Ok(())
+    }
 }
 
-#[test]
-fn borrowing_with_duplication() -> Result<()> {
+mod borrowing_with_duplication {
+    use super::*;
+
     wasmtime::component::bindgen!({
         inline: "
         package inline:inline;
@@ -297,33 +305,36 @@ fn borrowing_with_duplication() -> Result<()> {
         }
     }
 
-    let engine = engine();
-    let component = Component::new(&engine, component())?;
-
-    let linker = Linker::new(&engine);
-    let mut store = Store::new(&engine, ());
-    let (test, _) = Test::instantiate(&mut store, &component, &linker)?;
-
-    let value = &[&["a", "b"] as &[_]] as &[_];
-    assert_eq!(value, test.lists().call_foo(&mut store, value)?);
-
-    let value = exports::thing_in::Thing {
-        name: "thing 1",
-        value: &["some value", "another value"],
-    };
-    test.thing_in().call_bar(&mut store, value)?;
-
-    let value = exports::thing_in_and_out::ThingParam {
-        name: "thing 1",
-        value: &["some value", "another value"],
-    };
-    assert_eq!(
-        exports::thing_in_and_out::ThingResult {
-            name: "thing 1".to_owned(),
-            value: vec!["some value".to_owned(), "another value".to_owned()],
-        },
-        test.thing_in_and_out().call_baz(&mut store, value)?
-    );
-
-    Ok(())
+    #[test]
+    fn borrowing_with_duplication() -> Result<()> {
+        let engine = engine();
+        let component = Component::new(&engine, component())?;
+
+        let linker = Linker::new(&engine);
+        let mut store = Store::new(&engine, ());
+        let (test, _) = Test::instantiate(&mut store, &component, &linker)?;
+
+        let value = &[&["a", "b"] as &[_]] as &[_];
+        assert_eq!(value, test.lists().call_foo(&mut store, value)?);
+
+        let value = exports::thing_in::Thing {
+            name: "thing 1",
+            value: &["some value", "another value"],
+        };
+        test.thing_in().call_bar(&mut store, value)?;
+
+        let value = exports::thing_in_and_out::ThingParam {
+            name: "thing 1",
+            value: &["some value", "another value"],
+        };
+        assert_eq!(
+            exports::thing_in_and_out::ThingResult {
+                name: "thing 1".to_owned(),
+                value: vec!["some value".to_owned(), "another value".to_owned()],
+            },
+            test.thing_in_and_out().call_baz(&mut store, value)?
+        );
+
+        Ok(())
+    }
 }
diff --git a/tests/all/wast.rs b/tests/all/wast.rs
index 96f975b71..c7f94c144 100644
--- a/tests/all/wast.rs
+++ b/tests/all/wast.rs
@@ -23,7 +23,8 @@ fn run_wast(wast: &str, strategy: Strategy, pooling: bool) -> anyhow::Result<()>
     let wast = Path::new(wast);
 
     let memory64 = feature_found(wast, "memory64");
-    let multi_memory = feature_found(wast, "multi-memory");
+    let multi_memory =
+        feature_found(wast, "multi-memory") || feature_found(wast, "component-model");
     let threads = feature_found(wast, "threads");
     let gc = feature_found(wast, "gc");
     let function_references = gc || feature_found(wast, "function-references");
diff --git a/tests/misc_testsuite/gc/rec-group-funcs.wast b/tests/misc_testsuite/gc/rec-group-funcs.wast
new file mode 100644
index 000000000..3f8b27983
--- /dev/null
+++ b/tests/misc_testsuite/gc/rec-group-funcs.wast
@@ -0,0 +1,38 @@
+;; Test that we properly canonicalize function types across modules, at the
+;; engine level. We rely on this canonicalization to make cross-module imports
+;; work among other things.
+
+(module $m1
+  ;; A pair of recursive types.
+  (rec (type $type_a (sub final (func (result i32 (ref null $type_b)))))
+       (type $type_b (sub final (func (result i32 (ref null $type_a))))))
+
+  (func (export "func_a") (type $type_a)
+    i32.const 1234
+    ref.null $type_b
+  )
+
+  (func (export "func_b") (type $type_b)
+    i32.const 4321
+    ref.null $type_a
+  )
+)
+(register "m1")
+
+(module $m2
+  ;; The same pair of recursive types.
+  (rec (type $type_a (sub final (func (result i32 (ref null $type_b)))))
+       (type $type_b (sub final (func (result i32 (ref null $type_a))))))
+
+  (import "m1" "func_a" (func $func_a (type $type_a)))
+  (import "m1" "func_b" (func $func_b (type $type_b)))
+
+  (func (export "call") (result i32 i32)
+    call $func_a
+    drop
+    call $func_b
+    drop
+  )
+)
+
+(assert_return (invoke "call") (i32.const 1234) (i32.const 4321))