Repository: bytecodealliance/wasmtime
Pull request #9251 (2024-09-15 22:00:31)

Pulley: tail calls
Sorry for the delay, I was on holiday.
The proof of concept works, but codegen for `become` is not actually implemented yet and so triggers an ICE:
```
error: internal compiler error: /rustc/9b72238eb813e9d06e9e9d270168512fbffd7ee7/compiler/rustc_codegen_ssa/src/mir/block.rs:1372:17: `TailCall` terminator is not yet supported by `rustc_codegen_ssa`
   --> pulley/src/interp/interp_loop.rs:85:25
    |
85  |                           become (next_handler.fun)($state, $pc);
    |                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
```

So for now `return` is used instead of `become` and we must hope that LLVM will be smart enough to recongize the tail call

github-actions[bot]: #### Subscribe to Label Action

cc @fitzgen

<details>
This issue or pull request has been labeled: "pulley"

Thus the following users have been cc'd because of the following labels:

* fitzgen: pulley

To subscribe or unsubscribe from this label, edit the <code>.github/subscribe-to-label.json</code> configuration file.

[Learn more.](https://github.com/bytecodealliance/subscribe-to-label-action)
</details>
alexcrichton: Thanks for this! I might jump in with a small recommendation on overall structure / CI here -- with a Cargo feature controlling this it unfortunately doesn't play well with our "test with all features enabled" in CI well because it enables the feature when a stable compiler is in use. To fix this I might recommend `#[cfg(pulley_tail_call)]` perhaps? That would require setting `RUSTFLAGS` in CI but that could be added to the "Monolith Checks" perhaps as a single `cargo check` of pulley on nightly? That would also enable this to land despite codegen support not being in upstream rust-lang/rust yet.
fitzgen: Thanks! I'll take a look either later today or tomorrow.

Mildly surprised about `become` not working anymore, as it was working when I did a few experiments around the time of the RFC. I guess rustc is doing a new ssa thing in the midend and hasn't extended support to `become` yet?
fitzgen: FWIW, it looks like https://github.com/rust-lang/rust/issues/127520 was filed about this regression, but closed as WONTFIX, so I also cross-posted the bug to the `explicit_tail_calls` tracking issue: https://github.com/rust-lang/rust/issues/112788#issuecomment-2353680606
WaffleLapkin: @fitzgen I'm not sure how it could have worked for you -- `become` was never fully implemented. (maybe you tried to use it when it was just an alternative syntax for `return`?)

either way the reason why https://github.com/rust-lang/rust/issues/127520 was WONTFIXED is that tails calls are simply not implemented yet, they are an incomplete feature. The error even says as much:
> `TailCall` terminator is not yet supported by `rustc_codegen_ssa`

there is no reason to bring any more awareness to this issue -- it will be fixed naturally once the implementation is complete.
fitzgen: > Once #9274 lands, is this ready to merge as well?

@Kmeakin, now that #9274 landed, is this ready to rebase and land as well, or are there still more things left to do first?
Kmeakin: > > Once #9274 lands, is this ready to merge as well?
> 
> @Kmeakin, now that #9274 landed, is this ready to rebase and land as well, or are there still more things left to do first?

I think it is ready to land, I don't have anything else I want to add to it.

Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c6212b485..233a29617 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -552,6 +552,13 @@ jobs:
       with:
         toolchain: wasmtime-ci-pinned-nightly
 
+    # Check that `pulley-interpreter` compiles with tail calls enabled.  Don't
+    # actually run the tests with tail calls enabled, because they are not yet
+    # implemented in rustc and cause an ICE.
+    - run: cargo check -p pulley-interpreter
+      env:
+        RUSTFLAGS: "--cfg pulley_tail_calls"
+
     # Ensure that fuzzers still build.
     #
     # Install the OCaml packages necessary for fuzz targets that use the
diff --git a/Cargo.lock b/Cargo.lock
index dbc562771..9d110b50d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -251,7 +251,7 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
 
 [[package]]
 name = "byte-array-literals"
-version = "26.0.0"
+version = "27.0.0"
 
 [[package]]
 name = "byteorder"
@@ -618,7 +618,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "cranelift-codegen",
  "cranelift-frontend",
@@ -631,14 +631,14 @@ dependencies = [
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-bitset"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "arbitrary",
  "serde",
@@ -647,7 +647,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "anyhow",
  "bumpalo",
@@ -679,25 +679,25 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.113.0"
+version = "0.114.0"
 
 [[package]]
 name = "cranelift-control"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "cranelift-bitset",
  "serde",
@@ -736,7 +736,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "cranelift-codegen",
  "env_logger 0.11.5",
@@ -761,7 +761,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-interpreter"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "cranelift-codegen",
  "cranelift-entity",
@@ -775,7 +775,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "codespan-reporting",
  "log",
@@ -784,7 +784,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "anyhow",
  "cranelift",
@@ -805,7 +805,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -817,7 +817,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -826,7 +826,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -841,7 +841,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-reader"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -851,7 +851,7 @@ dependencies = [
 
 [[package]]
 name = "cranelift-serde"
-version = "0.113.0"
+version = "0.114.0"
 dependencies = [
  "clap",
  "cranelift-codegen",
@@ -1093,7 +1093,7 @@ checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
 
 [[package]]
 name = "embedding"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "dlmalloc",
@@ -1915,7 +1915,7 @@ dependencies = [
 
 [[package]]
 name = "min-platform-host"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "libloading",
@@ -2227,7 +2227,7 @@ dependencies = [
 
 [[package]]
 name = "pulley-interpreter"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "arbitrary",
  "cranelift-bitset",
@@ -3255,7 +3255,7 @@ version = "0.1.0"
 
 [[package]]
 name = "verify-component-adapter"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "wasmparser",
@@ -3305,7 +3305,7 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasi-common"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "bitflags 2.6.0",
@@ -3345,7 +3345,7 @@ dependencies = [
 
 [[package]]
 name = "wasi-preview1-component-adapter"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "bitflags 2.6.0",
  "byte-array-literals",
@@ -3539,7 +3539,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "addr2line",
  "anyhow",
@@ -3600,14 +3600,14 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-asm-macros"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "cfg-if",
 ]
 
 [[package]]
 name = "wasmtime-bench-api"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "cap-std",
@@ -3623,14 +3623,14 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-c-api"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "wasmtime-c-api-impl",
 ]
 
 [[package]]
 name = "wasmtime-c-api-impl"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "cap-std",
@@ -3648,7 +3648,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-c-api-macros"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3656,7 +3656,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-cache"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "base64 0.21.0",
@@ -3678,7 +3678,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-cli"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3748,7 +3748,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-cli-flags"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -3761,7 +3761,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-component-macro"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "component-macro-test-helpers",
@@ -3781,11 +3781,11 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-component-util"
-version = "26.0.0"
+version = "27.0.0"
 
 [[package]]
 name = "wasmtime-cranelift"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "cfg-if",
@@ -3808,7 +3808,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-environ"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -3850,7 +3850,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-explorer"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "capstone",
@@ -3865,7 +3865,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-fiber"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "backtrace",
@@ -3937,7 +3937,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-jit-debug"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "object",
  "once_cell",
@@ -3947,7 +3947,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "cfg-if",
@@ -3957,7 +3957,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-slab"
-version = "26.0.0"
+version = "27.0.0"
 
 [[package]]
 name = "wasmtime-test-macros"
@@ -3971,7 +3971,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-versioned-export-macros"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3980,7 +3980,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wasi"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -4013,7 +4013,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wasi-http"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -4039,7 +4039,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wasi-keyvalue"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "test-programs-artifacts",
@@ -4050,7 +4050,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wasi-nn"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "cap-std",
@@ -4070,7 +4070,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wasi-runtime-config"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "test-programs-artifacts",
@@ -4081,7 +4081,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wasi-threads"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "log",
@@ -4093,7 +4093,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wast"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "log",
@@ -4103,7 +4103,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-winch"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -4118,7 +4118,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wit-bindgen"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "heck 0.5.0",
@@ -4128,7 +4128,7 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-wmemcheck"
-version = "26.0.0"
+version = "27.0.0"
 
 [[package]]
 name = "wast"
@@ -4206,7 +4206,7 @@ dependencies = [
 
 [[package]]
 name = "wiggle"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -4223,7 +4223,7 @@ dependencies = [
 
 [[package]]
 name = "wiggle-generate"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "heck 0.5.0",
@@ -4236,7 +4236,7 @@ dependencies = [
 
 [[package]]
 name = "wiggle-macro"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4291,7 +4291,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "winch-codegen"
-version = "26.0.0"
+version = "27.0.0"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
diff --git a/Cargo.toml b/Cargo.toml
index 314d520f2..8fa098ae0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -160,7 +160,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "26.0.0"
+version = "27.0.0"
 authors = ["The Wasmtime Project Developers"]
 edition = "2021"
 # Wasmtime's current policy is that this number can be no larger than the
@@ -176,6 +176,9 @@ unused_import_braces = 'warn'
 unused-lifetimes = 'warn'
 unused-macro-rules = 'warn'
 
+# Don't warn about unknown cfg condition in `#[cfg(pulley_tail_calls)]`
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(pulley_tail_calls)'] }
+
 [workspace.lints.clippy]
 # The default set of lints in Clippy is viewed as "too noisy" right now so
 # they're all turned off by default. Selective lints are then enabled below as
@@ -192,60 +195,60 @@ unnecessary_cast = 'warn'
 
 [workspace.dependencies]
 arbitrary = { version = "1.3.1" }
-wasmtime-wmemcheck = { path = "crates/wmemcheck", version = "=26.0.0" }
-wasmtime = { path = "crates/wasmtime", version = "26.0.0", default-features = false }
-wasmtime-c-api-macros = { path = "crates/c-api-macros", version = "=26.0.0" }
-wasmtime-cache = { path = "crates/cache", version = "=26.0.0" }
-wasmtime-cli-flags = { path = "crates/cli-flags", version = "=26.0.0" }
-wasmtime-cranelift = { path = "crates/cranelift", version = "=26.0.0" }
-wasmtime-winch = { path = "crates/winch", version = "=26.0.0" }
-wasmtime-environ = { path = "crates/environ", version = "=26.0.0" }
-wasmtime-explorer = { path = "crates/explorer", version = "=26.0.0" }
-wasmtime-fiber = { path = "crates/fiber", version = "=26.0.0" }
-wasmtime-jit-debug = { path = "crates/jit-debug", version = "=26.0.0" }
-wasmtime-wast = { path = "crates/wast", version = "=26.0.0" }
-wasmtime-wasi = { path = "crates/wasi", version = "26.0.0", default-features = false }
-wasmtime-wasi-http = { path = "crates/wasi-http", version = "=26.0.0", default-features = false }
-wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "26.0.0" }
-wasmtime-wasi-runtime-config = { path = "crates/wasi-runtime-config", version = "26.0.0" }
-wasmtime-wasi-keyvalue = { path = "crates/wasi-keyvalue", version = "26.0.0" }
-wasmtime-wasi-threads = { path = "crates/wasi-threads", version = "26.0.0" }
-wasmtime-component-util = { path = "crates/component-util", version = "=26.0.0" }
-wasmtime-component-macro = { path = "crates/component-macro", version = "=26.0.0" }
-wasmtime-asm-macros = { path = "crates/asm-macros", version = "=26.0.0" }
-wasmtime-versioned-export-macros = { path = "crates/versioned-export-macros", version = "=26.0.0" }
-wasmtime-slab = { path = "crates/slab", version = "=26.0.0" }
+wasmtime-wmemcheck = { path = "crates/wmemcheck", version = "=27.0.0" }
+wasmtime = { path = "crates/wasmtime", version = "27.0.0", default-features = false }
+wasmtime-c-api-macros = { path = "crates/c-api-macros", version = "=27.0.0" }
+wasmtime-cache = { path = "crates/cache", version = "=27.0.0" }
+wasmtime-cli-flags = { path = "crates/cli-flags", version = "=27.0.0" }
+wasmtime-cranelift = { path = "crates/cranelift", version = "=27.0.0" }
+wasmtime-winch = { path = "crates/winch", version = "=27.0.0" }
+wasmtime-environ = { path = "crates/environ", version = "=27.0.0" }
+wasmtime-explorer = { path = "crates/explorer", version = "=27.0.0" }
+wasmtime-fiber = { path = "crates/fiber", version = "=27.0.0" }
+wasmtime-jit-debug = { path = "crates/jit-debug", version = "=27.0.0" }
+wasmtime-wast = { path = "crates/wast", version = "=27.0.0" }
+wasmtime-wasi = { path = "crates/wasi", version = "27.0.0", default-features = false }
+wasmtime-wasi-http = { path = "crates/wasi-http", version = "=27.0.0", default-features = false }
+wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "27.0.0" }
+wasmtime-wasi-runtime-config = { path = "crates/wasi-runtime-config", version = "27.0.0" }
+wasmtime-wasi-keyvalue = { path = "crates/wasi-keyvalue", version = "27.0.0" }
+wasmtime-wasi-threads = { path = "crates/wasi-threads", version = "27.0.0" }
+wasmtime-component-util = { path = "crates/component-util", version = "=27.0.0" }
+wasmtime-component-macro = { path = "crates/component-macro", version = "=27.0.0" }
+wasmtime-asm-macros = { path = "crates/asm-macros", version = "=27.0.0" }
+wasmtime-versioned-export-macros = { path = "crates/versioned-export-macros", version = "=27.0.0" }
+wasmtime-slab = { path = "crates/slab", version = "=27.0.0" }
 component-test-util = { path = "crates/misc/component-test-util" }
 component-fuzz-util = { path = "crates/misc/component-fuzz-util" }
-wiggle = { path = "crates/wiggle", version = "=26.0.0", default-features = false }
-wiggle-macro = { path = "crates/wiggle/macro", version = "=26.0.0" }
-wiggle-generate = { path = "crates/wiggle/generate", version = "=26.0.0" }
-wasi-common = { path = "crates/wasi-common", version = "=26.0.0", default-features = false }
+wiggle = { path = "crates/wiggle", version = "=27.0.0", default-features = false }
+wiggle-macro = { path = "crates/wiggle/macro", version = "=27.0.0" }
+wiggle-generate = { path = "crates/wiggle/generate", version = "=27.0.0" }
+wasi-common = { path = "crates/wasi-common", version = "=27.0.0", default-features = false }
 wasmtime-fuzzing = { path = "crates/fuzzing" }
-wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=26.0.0" }
-wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=26.0.0" }
+wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=27.0.0" }
+wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=27.0.0" }
 test-programs-artifacts = { path = 'crates/test-programs/artifacts' }
 
-pulley-interpreter = { path = 'pulley', version = "=26.0.0" }
+pulley-interpreter = { path = 'pulley', version = "=27.0.0" }
 pulley-interpreter-fuzz = { path = 'pulley/fuzz' }
 
-cranelift-codegen = { path = "cranelift/codegen", version = "0.113.0", default-features = false, features = ["std", "unwind"] }
-cranelift-frontend = { path = "cranelift/frontend", version = "0.113.0" }
-cranelift-entity = { path = "cranelift/entity", version = "0.113.0" }
-cranelift-native = { path = "cranelift/native", version = "0.113.0" }
-cranelift-module = { path = "cranelift/module", version = "0.113.0" }
-cranelift-interpreter = { path = "cranelift/interpreter", version = "0.113.0" }
-cranelift-reader = { path = "cranelift/reader", version = "0.113.0" }
+cranelift-codegen = { path = "cranelift/codegen", version = "0.114.0", default-features = false, features = ["std", "unwind"] }
+cranelift-frontend = { path = "cranelift/frontend", version = "0.114.0" }
+cranelift-entity = { path = "cranelift/entity", version = "0.114.0" }
+cranelift-native = { path = "cranelift/native", version = "0.114.0" }
+cranelift-module = { path = "cranelift/module", version = "0.114.0" }
+cranelift-interpreter = { path = "cranelift/interpreter", version = "0.114.0" }
+cranelift-reader = { path = "cranelift/reader", version = "0.114.0" }
 cranelift-filetests = { path = "cranelift/filetests" }
-cranelift-object = { path = "cranelift/object", version = "0.113.0" }
-cranelift-jit = { path = "cranelift/jit", version = "0.113.0" }
+cranelift-object = { path = "cranelift/object", version = "0.114.0" }
+cranelift-jit = { path = "cranelift/jit", version = "0.114.0" }
 cranelift-fuzzgen = { path = "cranelift/fuzzgen" }
-cranelift-bforest = { path = "cranelift/bforest", version = "0.113.0" }
-cranelift-bitset = { path = "cranelift/bitset", version = "0.113.0" }
-cranelift-control = { path = "cranelift/control", version = "0.113.0" }
-cranelift = { path = "cranelift/umbrella", version = "0.113.0" }
+cranelift-bforest = { path = "cranelift/bforest", version = "0.114.0" }
+cranelift-bitset = { path = "cranelift/bitset", version = "0.114.0" }
+cranelift-control = { path = "cranelift/control", version = "0.114.0" }
+cranelift = { path = "cranelift/umbrella", version = "0.114.0" }
 
-winch-codegen = { path = "winch/codegen", version = "=26.0.0" }
+winch-codegen = { path = "winch/codegen", version = "=27.0.0" }
 
 wasi-preview1-component-adapter = { path = "crates/wasi-preview1-component-adapter" }
 byte-array-literals = { path = "crates/wasi-preview1-component-adapter/byte-array-literals" }
diff --git a/RELEASES.md b/RELEASES.md
index 0e87cd88e..2eda2069e 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -1,4 +1,4 @@
-## 26.0.0
+## 27.0.0
 
 Unreleased.
 
@@ -12,6 +12,7 @@ Release notes for previous releases of Wasmtime can be found on the respective
 release branches of the Wasmtime repository.
 
 <!-- ARCHIVE_START -->
+* [26.0.x](https://github.com/bytecodealliance/wasmtime/blob/release-26.0.0/RELEASES.md)
 * [25.0.x](https://github.com/bytecodealliance/wasmtime/blob/release-25.0.0/RELEASES.md)
 * [24.0.x](https://github.com/bytecodealliance/wasmtime/blob/release-24.0.0/RELEASES.md)
 * [23.0.x](https://github.com/bytecodealliance/wasmtime/blob/release-23.0.0/RELEASES.md)
diff --git a/cranelift/bforest/Cargo.toml b/cranelift/bforest/Cargo.toml
index 17f7c6575..9e90bc73e 100644
--- a/cranelift/bforest/Cargo.toml
+++ b/cranelift/bforest/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-bforest"
-version = "0.113.0"
+version = "0.114.0"
 description = "A forest of B+-trees"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift-bforest"
diff --git a/cranelift/bitset/Cargo.toml b/cranelift/bitset/Cargo.toml
index b72fda390..62e14d3eb 100644
--- a/cranelift/bitset/Cargo.toml
+++ b/cranelift/bitset/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-bitset"
-version = "0.113.0"
+version = "0.114.0"
 description = "Various bitset stuff for use inside Cranelift"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift-bitset"
diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml
index e7ccb5882..8d0aad235 100644
--- a/cranelift/codegen/Cargo.toml
+++ b/cranelift/codegen/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-codegen"
-version = "0.113.0"
+version = "0.114.0"
 description = "Low-level code generator library"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift-codegen"
@@ -20,7 +20,7 @@ workspace = true
 anyhow = { workspace = true, optional = true, features = ['std'] }
 bumpalo = "3"
 capstone = { workspace = true, optional = true }
-cranelift-codegen-shared = { path = "./shared", version = "0.113.0" }
+cranelift-codegen-shared = { path = "./shared", version = "0.114.0" }
 cranelift-entity = { workspace = true }
 cranelift-bforest = { workspace = true }
 cranelift-bitset = { workspace = true }
@@ -49,8 +49,8 @@ similar = "2.1.0"
 env_logger = { workspace = true }
 
 [build-dependencies]
-cranelift-codegen-meta = { path = "meta", version = "0.113.0" }
-cranelift-isle = { path = "../isle/isle", version = "=0.113.0" }
+cranelift-codegen-meta = { path = "meta", version = "0.114.0" }
+cranelift-isle = { path = "../isle/isle", version = "=0.114.0" }
 
 [features]
 default = ["std", "unwind", "host-arch", "timing"]
diff --git a/cranelift/codegen/meta/Cargo.toml b/cranelift/codegen/meta/Cargo.toml
index 047f35e6c..281c66561 100644
--- a/cranelift/codegen/meta/Cargo.toml
+++ b/cranelift/codegen/meta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "cranelift-codegen-meta"
 authors = ["The Cranelift Project Developers"]
-version = "0.113.0"
+version = "0.114.0"
 description = "Metaprogram for cranelift-codegen code generator library"
 license = "Apache-2.0 WITH LLVM-exception"
 repository = "https://github.com/bytecodealliance/wasmtime"
@@ -16,4 +16,4 @@ workspace = true
 rustdoc-args = [ "--document-private-items" ]
 
 [dependencies]
-cranelift-codegen-shared = { path = "../shared", version = "0.113.0" }
+cranelift-codegen-shared = { path = "../shared", version = "0.114.0" }
diff --git a/cranelift/codegen/shared/Cargo.toml b/cranelift/codegen/shared/Cargo.toml
index 62f563d46..b82ca52d8 100644
--- a/cranelift/codegen/shared/Cargo.toml
+++ b/cranelift/codegen/shared/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-codegen-shared"
-version = "0.113.0"
+version = "0.114.0"
 description = "For code shared between cranelift-codegen-meta and cranelift-codegen"
 license = "Apache-2.0 WITH LLVM-exception"
 repository = "https://github.com/bytecodealliance/wasmtime"
diff --git a/cranelift/control/Cargo.toml b/cranelift/control/Cargo.toml
index f6354051d..19cf67a42 100644
--- a/cranelift/control/Cargo.toml
+++ b/cranelift/control/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-control"
-version = "0.113.0"
+version = "0.114.0"
 description = "White-box fuzz testing framework"
 license = "Apache-2.0 WITH LLVM-exception"
 repository = "https://github.com/bytecodealliance/wasmtime"
diff --git a/cranelift/entity/Cargo.toml b/cranelift/entity/Cargo.toml
index 6f30f6f23..4d14bf9e9 100644
--- a/cranelift/entity/Cargo.toml
+++ b/cranelift/entity/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-entity"
-version = "0.113.0"
+version = "0.114.0"
 description = "Data structures using entity references as mapping keys"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift-entity"
diff --git a/cranelift/frontend/Cargo.toml b/cranelift/frontend/Cargo.toml
index ca8662afa..f4e65172c 100644
--- a/cranelift/frontend/Cargo.toml
+++ b/cranelift/frontend/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-frontend"
-version = "0.113.0"
+version = "0.114.0"
 description = "Cranelift IR builder helper"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift-frontend"
diff --git a/cranelift/interpreter/Cargo.toml b/cranelift/interpreter/Cargo.toml
index 9561216db..7ae9ecd8f 100644
--- a/cranelift/interpreter/Cargo.toml
+++ b/cranelift/interpreter/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cranelift-interpreter"
-version = "0.113.0"
+version = "0.114.0"
 authors = ["The Cranelift Project Developers"]
 description = "Interpret Cranelift IR"
 repository = "https://github.com/bytecodealliance/wasmtime"
diff --git a/cranelift/isle/isle/Cargo.toml b/cranelift/isle/isle/Cargo.toml
index 05519e797..b50c2b253 100644
--- a/cranelift/isle/isle/Cargo.toml
+++ b/cranelift/isle/isle/Cargo.toml
@@ -7,7 +7,7 @@ license = "Apache-2.0 WITH LLVM-exception"
 name = "cranelift-isle"
 readme = "../README.md"
 repository = "https://github.com/bytecodealliance/wasmtime/tree/main/cranelift/isle"
-version = "0.113.0"
+version = "0.114.0"
 
 [lints]
 workspace = true
diff --git a/cranelift/jit/Cargo.toml b/cranelift/jit/Cargo.toml
index bc42305a5..d5f68e470 100644
--- a/cranelift/jit/Cargo.toml
+++ b/cranelift/jit/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cranelift-jit"
-version = "0.113.0"
+version = "0.114.0"
 authors = ["The Cranelift Project Developers"]
 description = "A JIT library backed by Cranelift"
 repository = "https://github.com/bytecodealliance/wasmtime"
diff --git a/cranelift/module/Cargo.toml b/cranelift/module/Cargo.toml
index e2c8e10d5..6be645908 100644
--- a/cranelift/module/Cargo.toml
+++ b/cranelift/module/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cranelift-module"
-version = "0.113.0"
+version = "0.114.0"
 authors = ["The Cranelift Project Developers"]
 description = "Support for linking functions and data with Cranelift"
 repository = "https://github.com/bytecodealliance/wasmtime"
diff --git a/cranelift/native/Cargo.toml b/cranelift/native/Cargo.toml
index 3b94c0d13..f41bb6694 100644
--- a/cranelift/native/Cargo.toml
+++ b/cranelift/native/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cranelift-native"
-version = "0.113.0"
+version = "0.114.0"
 authors = ["The Cranelift Project Developers"]
 description = "Support for targeting the host with Cranelift"
 documentation = "https://docs.rs/cranelift-native"
diff --git a/cranelift/object/Cargo.toml b/cranelift/object/Cargo.toml
index 3e8384552..dba53977c 100644
--- a/cranelift/object/Cargo.toml
+++ b/cranelift/object/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cranelift-object"
-version = "0.113.0"
+version = "0.114.0"
 authors = ["The Cranelift Project Developers"]
 description = "Emit Cranelift output to native object files with `object`"
 repository = "https://github.com/bytecodealliance/wasmtime"
diff --git a/cranelift/reader/Cargo.toml b/cranelift/reader/Cargo.toml
index deb3644ae..fba788c08 100644
--- a/cranelift/reader/Cargo.toml
+++ b/cranelift/reader/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift-reader"
-version = "0.113.0"
+version = "0.114.0"
 description = "Cranelift textual IR reader"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift-reader"
diff --git a/cranelift/serde/Cargo.toml b/cranelift/serde/Cargo.toml
index 5d1fefd42..8a783f380 100644
--- a/cranelift/serde/Cargo.toml
+++ b/cranelift/serde/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cranelift-serde"
-version = "0.113.0"
+version = "0.114.0"
 authors = ["The Cranelift Project Developers"]
 description = "Serializer/Deserializer for Cranelift IR"
 repository = "https://github.com/bytecodealliance/wasmtime"
diff --git a/cranelift/umbrella/Cargo.toml b/cranelift/umbrella/Cargo.toml
index 7e6d2d1ba..f13bb7ed5 100644
--- a/cranelift/umbrella/Cargo.toml
+++ b/cranelift/umbrella/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["The Cranelift Project Developers"]
 name = "cranelift"
-version = "0.113.0"
+version = "0.114.0"
 description = "Umbrella for commonly-used cranelift crates"
 license = "Apache-2.0 WITH LLVM-exception"
 documentation = "https://docs.rs/cranelift"
diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h
index 30b35b07d..f8c25fca3 100644
--- a/crates/c-api/include/wasmtime.h
+++ b/crates/c-api/include/wasmtime.h
@@ -206,11 +206,11 @@
 /**
  * \brief Wasmtime version string.
  */
-#define WASMTIME_VERSION "26.0.0"
+#define WASMTIME_VERSION "27.0.0"
 /**
  * \brief Wasmtime major version number.
  */
-#define WASMTIME_VERSION_MAJOR 26
+#define WASMTIME_VERSION_MAJOR 27
 /**
  * \brief Wasmtime minor version number.
  */
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 953e37fda..db0a4225e 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -2020,36 +2020,19 @@ impl<'module_environment> crate::translate::FuncEnvironment
     fn translate_array_copy(
         &mut self,
         builder: &mut FunctionBuilder,
-        dst_array_type_index: TypeIndex,
+        _dst_array_type_index: TypeIndex,
         dst_array: ir::Value,
         dst_index: ir::Value,
-        src_array_type_index: TypeIndex,
+        _src_array_type_index: TypeIndex,
         src_array: ir::Value,
         src_index: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
         let libcall = gc::builtins::array_copy(self, builder.func)?;
         let vmctx = self.vmctx_val(&mut builder.cursor());
-        let dst_array_type_index = self.module.types[dst_array_type_index];
-        let dst_array_type_index = builder
-            .ins()
-            .iconst(I32, i64::from(dst_array_type_index.as_u32()));
-        let src_array_type_index = self.module.types[src_array_type_index];
-        let src_array_type_index = builder
-            .ins()
-            .iconst(I32, i64::from(src_array_type_index.as_u32()));
         builder.ins().call(
             libcall,
-            &[
-                vmctx,
-                dst_array_type_index,
-                dst_array,
-                dst_index,
-                src_array_type_index,
-                src_array,
-                src_index,
-                len,
-            ],
+            &[vmctx, dst_array, dst_index, src_array, src_index, len],
         );
         Ok(())
     }
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index e904cb8c8..65e035135 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -148,10 +148,8 @@ macro_rules! foreach_builtin_function {
             #[cfg(feature = "gc")]
             array_copy(
                 vmctx: vmctx,
-                dst_array_interned_type_index: i32,
                 dst_array: reference,
                 dst_index: i32,
-                src_array_interned_type_index: i32,
                 src_array: reference,
                 src_index: i32,
                 len: i32
diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
index 0dc4d4cec..22661caea 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
@@ -568,7 +568,7 @@ impl ArrayRef {
         Ok(gc_ref.as_arrayref_unchecked())
     }
 
-    fn layout(&self, store: &AutoAssertNoGc<'_>) -> Result<GcArrayLayout> {
+    pub(crate) fn layout(&self, store: &AutoAssertNoGc<'_>) -> Result<GcArrayLayout> {
         assert!(self.comes_from_same_store(&store));
         let type_index = self.type_index(store)?;
         let layout = store
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index 6f6fec630..436ea220d 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -905,6 +905,14 @@ impl<T: GcRef> Rooted<T> {
         b: &impl RootedGcRef<T>,
     ) -> Result<bool> {
         let store = store.as_context().0;
+        Self::_ref_eq(store, a, b)
+    }
+
+    pub(crate) fn _ref_eq(
+        store: &StoreOpaque,
+        a: &impl RootedGcRef<T>,
+        b: &impl RootedGcRef<T>,
+    ) -> Result<bool> {
         let a = a.try_gc_ref(store)?;
         let b = b.try_gc_ref(store)?;
         Ok(a == b)
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 4b7a00330..38eef67ca 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1614,6 +1614,8 @@ impl StoreOpaque {
             return;
         }
 
+        log::trace!("============ Begin GC ===========");
+
         // Take the GC roots out of `self` so we can borrow it mutably but still
         // call mutable methods on `self`.
         let mut roots = core::mem::take(&mut self.gc_roots_list);
@@ -1624,6 +1626,8 @@ impl StoreOpaque {
         // Restore the GC roots for the next GC.
         roots.clear();
         self.gc_roots_list = roots;
+
+        log::trace!("============ End GC ===========");
     }
 
     #[inline]
@@ -1662,6 +1666,8 @@ impl StoreOpaque {
             return;
         }
 
+        log::trace!("============ Begin Async GC ===========");
+
         // Take the GC roots out of `self` so we can borrow it mutably but still
         // call mutable methods on `self`.
         let mut roots = std::mem::take(&mut self.gc_roots_list);
@@ -1674,6 +1680,8 @@ impl StoreOpaque {
         // Restore the GC roots for the next GC.
         roots.clear();
         self.gc_roots_list = roots;
+
+        log::trace!("============ End Async GC ===========");
     }
 
     #[inline]
diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs
index a1bf7f0a0..0ffab2720 100644
--- a/crates/wasmtime/src/runtime/vm/const_expr.rs
+++ b/crates/wasmtime/src/runtime/vm/const_expr.rs
@@ -38,11 +38,7 @@ impl<'a> ConstEvalContext<'a> {
                 .defined_or_imported_global_ptr(index)
                 .as_ref()
                 .unwrap();
-            let mut gc_store = store.unwrap_gc_store_mut();
-            Ok(global.to_val_raw(
-                &mut gc_store,
-                self.instance.env_module().globals[index].wasm_ty,
-            ))
+            global.to_val_raw(store, self.instance.env_module().globals[index].wasm_ty)
         }
     }
 
@@ -169,6 +165,8 @@ impl ConstExprEvaluator {
         context: &mut ConstEvalContext<'_>,
         expr: &ConstExpr,
     ) -> Result<ValRaw> {
+        log::trace!("evaluating const expr: {:?}", expr);
+
         self.stack.clear();
 
         let mut store = (*context.instance.store()).store_opaque_mut();
@@ -372,6 +370,7 @@ impl ConstExprEvaluator {
         }
 
         if self.stack.len() == 1 {
+            log::trace!("const expr evaluated to {:?}", self.stack[0]);
             Ok(self.stack[0])
         } else {
             bail!(
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index 7500a68fc..b2f020c85 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -23,6 +23,7 @@ pub use i31::*;
 use crate::prelude::*;
 use crate::runtime::vm::GcHeapAllocationIndex;
 use core::alloc::Layout;
+use core::mem::MaybeUninit;
 use core::ptr;
 use core::{any::Any, num::NonZeroUsize};
 use wasmtime_environ::{GcArrayLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
@@ -96,6 +97,19 @@ impl GcStore {
         }
     }
 
+    /// Write the `source` GC reference into the uninitialized `destination`
+    /// slot, performing write barriers as necessary.
+    pub fn init_gc_ref(
+        &mut self,
+        destination: &mut MaybeUninit<Option<VMGcRef>>,
+        source: Option<&VMGcRef>,
+    ) {
+        // Initialize the destination to `None`, at which point the regular GC
+        // write barrier is safe to reuse.
+        let destination = destination.write(None);
+        self.write_gc_ref(destination, source);
+    }
+
     /// Write the `source` GC reference into the `destination` slot, performing
     /// write barriers as necessary.
     pub fn write_gc_ref(&mut self, destination: &mut Option<VMGcRef>, source: Option<&VMGcRef>) {
@@ -205,6 +219,18 @@ impl GcStore {
         self.gc_heap.gc_object_data(gc_ref)
     }
 
+    /// Get the object datas for the given pair of object references.
+    ///
+    /// Panics if `a` and `b` are the same reference or either is out of bounds.
+    pub fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        assert_ne!(a, b);
+        self.gc_heap.gc_object_data_pair(a, b)
+    }
+
     /// Allocate an uninitialized array with the given type index.
     ///
     /// This does NOT check that the index is currently allocated in the types
@@ -299,6 +325,13 @@ unsafe impl GcHeap for DisabledGcHeap {
     fn gc_object_data(&mut self, _gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_> {
         unreachable!()
     }
+    fn gc_object_data_pair(
+        &mut self,
+        _a: &VMGcRef,
+        _b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        unreachable!()
+    }
     fn alloc_uninit_array(
         &mut self,
         _ty: VMSharedTypeIndex,
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
index 2c74ccbfd..19328387e 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
@@ -133,6 +133,28 @@ impl<'a> VMGcObjectDataMut<'a> {
         val.write_le(into.try_into().unwrap());
     }
 
+    /// Get a slice of this object's data.
+    ///
+    /// Panics on out-of-bounds accesses.
+    #[inline]
+    pub fn slice(&self, offset: u32, len: u32) -> &[u8] {
+        let start = usize::try_from(offset).unwrap();
+        let len = usize::try_from(len).unwrap();
+        let end = start.checked_add(len).unwrap();
+        self.data.get(start..end).expect("out of bounds slice")
+    }
+
+    /// Get a mutable slice of this object's data.
+    ///
+    /// Panics on out-of-bounds accesses.
+    #[inline]
+    pub fn slice_mut(&mut self, offset: u32, len: u32) -> &mut [u8] {
+        let start = usize::try_from(offset).unwrap();
+        let len = usize::try_from(len).unwrap();
+        let end = start.checked_add(len).unwrap();
+        self.data.get_mut(start..end).expect("out of bounds slice")
+    }
+
     /// Copy the given slice into this object's data at the given offset.
     ///
     /// Panics on out-of-bounds accesses.
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index ebf5e9b46..02c65b68b 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -49,7 +49,7 @@ use crate::runtime::vm::{
     ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
     GcProgress, GcRootsIter, GcRuntime, Mmap, TypedGcRef, VMExternRef, VMGcHeader, VMGcRef,
 };
-use core::ops::{Deref, DerefMut};
+use core::ops::{Deref, DerefMut, Range};
 use core::{
     alloc::Layout,
     any::Any,
@@ -132,6 +132,16 @@ impl DrcHeap {
             .dealloc(gc_ref.as_heap_index().unwrap(), layout);
     }
 
+    fn object_range(&self, gc_ref: &VMGcRef) -> Range<usize> {
+        let start = gc_ref.as_heap_index().unwrap().get();
+        let start = usize::try_from(start).unwrap();
+        let size = self
+            .index::<VMDrcHeader>(gc_ref.as_typed_unchecked())
+            .object_size();
+        let end = start.checked_add(size).unwrap();
+        start..end
+    }
+
     /// Index into this heap and get a shared reference to the `T` that `gc_ref`
     /// points to.
     ///
@@ -617,16 +627,42 @@ unsafe impl GcHeap for DrcHeap {
     }
 
     fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_> {
-        let start = gc_ref.as_heap_index().unwrap().get();
-        let start = usize::try_from(start).unwrap();
-        let size = self
-            .index::<VMDrcHeader>(gc_ref.as_typed_unchecked())
-            .object_size();
-        let end = start + size;
-        let data = &mut self.heap_slice_mut()[start..end];
+        let range = self.object_range(gc_ref);
+        let data = &mut self.heap_slice_mut()[range];
         VMGcObjectDataMut::new(data)
     }
 
+    fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        assert_ne!(a, b);
+
+        let a_range = self.object_range(a);
+        let b_range = self.object_range(b);
+
+        // Assert that the two objects do not overlap.
+        assert!(a_range.start <= a_range.end);
+        assert!(b_range.start <= b_range.end);
+        assert!(a_range.end <= b_range.start || b_range.end <= a_range.start);
+
+        let (a_data, b_data) = if a_range.start < b_range.start {
+            let (a_half, b_half) = self.heap_slice_mut().split_at_mut(b_range.start);
+            let b_len = b_range.end - b_range.start;
+            (&mut a_half[a_range], &mut b_half[..b_len])
+        } else {
+            let (b_half, a_half) = self.heap_slice_mut().split_at_mut(a_range.start);
+            let a_len = a_range.end - a_range.start;
+            (&mut a_half[..a_len], &mut b_half[b_range])
+        };
+
+        (
+            VMGcObjectDataMut::new(a_data),
+            VMGcObjectDataMut::new(b_data),
+        )
+    }
+
     fn alloc_uninit_array(
         &mut self,
         ty: VMSharedTypeIndex,
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
index fa89efb81..539ff5b95 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
@@ -285,15 +285,15 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
     /// Panics on out-of-bounds accesses.
     fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_>;
 
-    /// Allocate a GC-managed array of the given type and length.
-    ///
-    /// The array's elements are left uninitialized. It is the caller's
-    /// responsibility to initialize them before exposing the array to Wasm or
-    /// triggering a GC. Failure to do this is memory safe, but may result in
-    /// general failures such as panics or incorrect results.
-    ///
-    /// Return values:
+    /// Get a pair of mutable borrows of the given objects' data.
     ///
+    /// Panics if `a == b` or on out-of-bounds accesses.
+    fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>);
+
     /// * `Ok(Some(_))`: The allocation was successful.
     ///
     /// * `Ok(None)`: There is currently no available space for this
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index b1e878c59..d707bbc09 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -1318,34 +1318,12 @@ impl Instance {
             ptr = ptr.add(1);
         }
 
-        // Initialize the defined globals
-        let mut const_evaluator = ConstExprEvaluator::default();
-        self.initialize_vmctx_globals(&mut const_evaluator, module);
-    }
-
-    unsafe fn initialize_vmctx_globals(
-        &mut self,
-        const_evaluator: &mut ConstExprEvaluator,
-        module: &Module,
-    ) {
-        for (index, init) in module.global_initializers.iter() {
-            let mut context = ConstEvalContext::new(self);
-            let raw = const_evaluator
-                .eval(&mut context, init)
-                .expect("should be a valid const expr");
-
-            let to = self.global_ptr(index);
-            let wasm_ty = module.globals[module.global_index(index)].wasm_ty;
-
-            #[cfg(feature = "wmemcheck")]
-            if index.index() == 0 && wasm_ty == wasmtime_environ::WasmValType::I32 {
-                if let Some(wmemcheck) = &mut self.wmemcheck_state {
-                    let size = usize::try_from(raw.get_i32()).unwrap();
-                    wmemcheck.set_stack_size(size);
-                }
-            }
-
-            ptr::write(to, VMGlobalDefinition::from_val_raw(wasm_ty, raw));
+        // Zero-initialize the globals so that nothing is uninitialized memory
+        // after this function returns. The globals are actually initialized
+        // with their const expression initializers after the instance is fully
+        // allocated.
+        for (index, _init) in module.global_initializers.iter() {
+            ptr::write(self.global_ptr(index), VMGlobalDefinition::new());
         }
     }
 
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
index f93592378..1fe374517 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
@@ -6,6 +6,8 @@ use crate::runtime::vm::memory::Memory;
 use crate::runtime::vm::mpk::ProtectionKey;
 use crate::runtime::vm::table::Table;
 use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo, VMFuncRef, VMGcRef, VMStore};
+use crate::store::AutoAssertNoGc;
+use crate::vm::VMGlobalDefinition;
 use core::{any::Any, mem, ptr};
 use wasmtime_environ::{
     DefinedMemoryIndex, DefinedTableIndex, HostPtr, InitMemory, MemoryInitialization,
@@ -555,27 +557,28 @@ fn check_table_init_bounds(instance: &mut Instance, module: &Module) -> Result<(
     Ok(())
 }
 
-fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
-    let mut const_evaluator = ConstExprEvaluator::default();
-
+fn initialize_tables(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
     for (table, init) in module.table_initialization.initial_values.iter() {
         match init {
             // Tables are always initially null-initialized at this time
             TableInitialValue::Null { precomputed: _ } => {}
 
             TableInitialValue::Expr(expr) => {
-                let mut context = ConstEvalContext::new(instance);
                 let raw = unsafe {
                     const_evaluator
-                        .eval(&mut context, expr)
+                        .eval(context, expr)
                         .expect("const expression should be valid")
                 };
                 let idx = module.table_index(table);
-                let table = unsafe { instance.get_defined_table(table).as_mut().unwrap() };
+                let table = unsafe { context.instance.get_defined_table(table).as_mut().unwrap() };
                 match module.table_plans[idx].table.ref_type.heap_type.top() {
                     WasmHeapTopType::Extern => {
                         let gc_ref = VMGcRef::from_raw_u32(raw.get_externref());
-                        let gc_store = unsafe { (*instance.store()).unwrap_gc_store_mut() };
+                        let gc_store = unsafe { (*context.instance.store()).gc_store_mut()? };
                         let items = (0..table.size())
                             .map(|_| gc_ref.as_ref().map(|r| gc_store.clone_gc_ref(r)));
                         table.init_gc_refs(0, items).err2anyhow()?;
@@ -583,7 +586,7 @@ fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
 
                     WasmHeapTopType::Any => {
                         let gc_ref = VMGcRef::from_raw_u32(raw.get_anyref());
-                        let gc_store = unsafe { (*instance.store()).unwrap_gc_store_mut() };
+                        let gc_store = unsafe { (*context.instance.store()).gc_store_mut()? };
                         let items = (0..table.size())
                             .map(|_| gc_ref.as_ref().map(|r| gc_store.clone_gc_ref(r)));
                         table.init_gc_refs(0, items).err2anyhow()?;
@@ -607,15 +610,15 @@ fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
     // iterates over all segments (Segments mode) or leftover
     // segments (FuncTable mode) to initialize.
     for segment in module.table_initialization.segments.iter() {
-        let mut context = ConstEvalContext::new(instance);
         let start = unsafe {
             const_evaluator
-                .eval(&mut context, &segment.offset)
+                .eval(context, &segment.offset)
                 .expect("const expression should be valid")
         };
-        instance
+        context
+            .instance
             .table_init_segment(
-                &mut const_evaluator,
+                const_evaluator,
                 segment.table_index,
                 &segment.elements,
                 start.get_u64(),
@@ -666,7 +669,11 @@ fn check_memory_init_bounds(
     Ok(())
 }
 
-fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
+fn initialize_memories(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
     // Delegates to the `init_memory` method which is sort of a duplicate of
     // `instance.memory_init_segment` but is used at compile-time in other
     // contexts so is shared here to have only one method of memory
@@ -676,18 +683,18 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
     // so errors only happen if an out-of-bounds segment is found, in which case
     // a trap is returned.
 
-    struct InitMemoryAtInstantiation<'a> {
-        instance: &'a mut Instance,
+    struct InitMemoryAtInstantiation<'a, 'b> {
         module: &'a Module,
-        const_evaluator: ConstExprEvaluator,
+        context: &'a mut ConstEvalContext<'b>,
+        const_evaluator: &'a mut ConstExprEvaluator,
     }
 
-    impl InitMemory for InitMemoryAtInstantiation<'_> {
+    impl InitMemory for InitMemoryAtInstantiation<'_, '_> {
         fn memory_size_in_bytes(
             &mut self,
             memory: wasmtime_environ::MemoryIndex,
         ) -> Result<u64, SizeOverflow> {
-            let len = self.instance.get_memory(memory).current_length();
+            let len = self.context.instance.get_memory(memory).current_length();
             let len = u64::try_from(len).unwrap();
             Ok(len)
         }
@@ -697,11 +704,10 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
             memory: wasmtime_environ::MemoryIndex,
             expr: &wasmtime_environ::ConstExpr,
         ) -> Option<u64> {
-            let mut context = ConstEvalContext::new(self.instance);
-            let val = unsafe { self.const_evaluator.eval(&mut context, expr) }
+            let val = unsafe { self.const_evaluator.eval(self.context, expr) }
                 .expect("const expression should be valid");
             Some(
-                match self.instance.env_module().memory_plans[memory]
+                match self.context.instance.env_module().memory_plans[memory]
                     .memory
                     .idx_type
                 {
@@ -721,14 +727,14 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
             // pre-initializing it via mmap magic, then this initializer can be
             // skipped entirely.
             if let Some(memory_index) = self.module.defined_memory_index(memory_index) {
-                if !self.instance.memories[memory_index].1.needs_init() {
+                if !self.context.instance.memories[memory_index].1.needs_init() {
                     return true;
                 }
             }
-            let memory = self.instance.get_memory(memory_index);
+            let memory = self.context.instance.get_memory(memory_index);
 
             unsafe {
-                let src = self.instance.wasm_data(init.data.clone());
+                let src = self.context.instance.wasm_data(init.data.clone());
                 let offset = usize::try_from(init.offset).unwrap();
                 let dst = memory.base.add(offset);
 
@@ -746,9 +752,9 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
     let ok = module
         .memory_initialization
         .init_memory(&mut InitMemoryAtInstantiation {
-            instance,
             module,
-            const_evaluator: ConstExprEvaluator::default(),
+            context,
+            const_evaluator,
         });
     if !ok {
         return Err(Trap::MemoryOutOfBounds).err2anyhow();
@@ -771,6 +777,46 @@ fn check_init_bounds(instance: &mut Instance, module: &Module) -> Result<()> {
     Ok(())
 }
 
+fn initialize_globals(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
+    assert!(core::ptr::eq(&**context.instance.env_module(), module));
+
+    for (index, init) in module.global_initializers.iter() {
+        let raw = unsafe {
+            const_evaluator
+                .eval(context, init)
+                .expect("should be a valid const expr")
+        };
+
+        let to = context.instance.global_ptr(index);
+        let wasm_ty = module.globals[module.global_index(index)].wasm_ty;
+
+        #[cfg(feature = "wmemcheck")]
+        if index.as_bits() == 0 && wasm_ty == wasmtime_environ::WasmValType::I32 {
+            if let Some(wmemcheck) = &mut context.instance.wmemcheck_state {
+                let size = usize::try_from(raw.get_i32()).unwrap();
+                wmemcheck.set_stack_size(size);
+            }
+        }
+
+        let store = unsafe { (*context.instance.store()).store_opaque_mut() };
+        let mut store = AutoAssertNoGc::new(store);
+
+        // This write is safe because we know we have the correct module for
+        // this instance and its vmctx due to the assert above.
+        unsafe {
+            ptr::write(
+                to,
+                VMGlobalDefinition::from_val_raw(&mut store, wasm_ty, raw)?,
+            )
+        };
+    }
+    Ok(())
+}
+
 pub(super) fn initialize_instance(
     instance: &mut Instance,
     module: &Module,
@@ -784,11 +830,12 @@ pub(super) fn initialize_instance(
         check_init_bounds(instance, module)?;
     }
 
-    // Initialize the tables
-    initialize_tables(instance, module)?;
+    let mut context = ConstEvalContext::new(instance);
+    let mut const_evaluator = ConstExprEvaluator::default();
 
-    // Initialize the memories
-    initialize_memories(instance, &module)?;
+    initialize_globals(&mut context, &mut const_evaluator, module)?;
+    initialize_tables(&mut context, &mut const_evaluator, module)?;
+    initialize_memories(&mut context, &mut const_evaluator, &module)?;
 
     Ok(())
 }
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 01519dd5e..15189d37e 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -840,6 +840,7 @@ unsafe fn array_init_elem(
         // Convert the raw GC ref into a `Rooted<ArrayRef>`.
         let array =
             VMGcRef::from_raw_u32(array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+        let array = (*instance.store()).unwrap_gc_store_mut().clone_gc_ref(&array);
         let array = {
             let mut no_gc = AutoAssertNoGc::new((*instance.store()).store_opaque_mut());
             ArrayRef::from_cloned_gc_ref(&mut no_gc, array)
@@ -918,18 +919,79 @@ unsafe fn array_init_elem(
     })
 }
 
+// TODO: Specialize this libcall for only non-GC array elements, so we never
+// have to do GC barriers and their associated indirect calls through the `dyn
+// GcHeap`. Instead, implement those copies inline in Wasm code. Then, use bulk
+// `memcpy`-style APIs to do the actual copies here.
 #[cfg(feature = "gc")]
 unsafe fn array_copy(
-    _instance: &mut Instance,
-    _dst_array_type_index: u32,
-    _dst_array: u32,
-    _dst_index: u32,
-    _src_array_type_index: u32,
-    _src_array: u32,
-    _src_index: u32,
-    _len: u32,
+    instance: &mut Instance,
+    dst_array: u32,
+    dst: u32,
+    src_array: u32,
+    src: u32,
+    len: u32,
 ) -> Result<()> {
-    bail!("the `array.copy` instruction is not yet implemented")
+    use crate::{store::AutoAssertNoGc, ArrayRef, OpaqueRootScope};
+
+    log::trace!(
+            "array.copy(dst_array={dst_array:#x}, dst_index={dst}, src_array={src_array:#x}, src_index={src}, len={len})",
+        );
+
+    let store = (*instance.store()).store_opaque_mut();
+    let mut store = OpaqueRootScope::new(store);
+    let mut store = AutoAssertNoGc::new(&mut store);
+
+    // Convert the raw GC refs into `Rooted<ArrayRef>`s.
+    let dst_array =
+        VMGcRef::from_raw_u32(dst_array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+    let dst_array = store.unwrap_gc_store_mut().clone_gc_ref(&dst_array);
+    let dst_array = ArrayRef::from_cloned_gc_ref(&mut store, dst_array);
+    let src_array =
+        VMGcRef::from_raw_u32(src_array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+    let src_array = store.unwrap_gc_store_mut().clone_gc_ref(&src_array);
+    let src_array = ArrayRef::from_cloned_gc_ref(&mut store, src_array);
+
+    // Bounds check the destination array's elements.
+    let dst_array_len = dst_array._len(&store)?;
+    if dst
+        .checked_add(len)
+        .ok_or_else(|| Trap::ArrayOutOfBounds.into_anyhow())?
+        > dst_array_len
+    {
+        return Err(Trap::ArrayOutOfBounds.into_anyhow());
+    }
+
+    // Bounds check the source array's elements.
+    let src_array_len = src_array._len(&store)?;
+    if src
+        .checked_add(len)
+        .ok_or_else(|| Trap::ArrayOutOfBounds.into_anyhow())?
+        > src_array_len
+    {
+        return Err(Trap::ArrayOutOfBounds.into_anyhow());
+    }
+
+    let mut store = AutoAssertNoGc::new(&mut store);
+    // If `src_array` and `dst_array` are the same array, then we are
+    // potentially doing an overlapping copy, so make sure to copy elements in
+    // the order that doesn't clobber the source elements before they are
+    // copied. If they are different arrays, the order doesn't matter, but we
+    // simply don't bother checking.
+    if src > dst {
+        for i in 0..len {
+            let src_elem = src_array._get(&mut store, src + i)?;
+            let dst_i = dst + i;
+            dst_array._set(&mut store, dst_i, src_elem)?;
+        }
+    } else {
+        for i in (0..len).rev() {
+            let src_elem = src_array._get(&mut store, src + i)?;
+            let dst_i = dst + i;
+            dst_array._set(&mut store, dst_i, src_elem)?;
+        }
+    }
+    Ok(())
 }
 
 // Implementation of `memory.atomic.notify` for locally defined memories.
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 26e72a84f..59ae7ba0f 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -4,12 +4,14 @@
 mod vm_host_func_context;
 
 pub use self::vm_host_func_context::VMArrayCallHostFuncContext;
+use crate::prelude::*;
 use crate::runtime::vm::{GcStore, VMGcRef};
+use crate::store::StoreOpaque;
 use core::cell::UnsafeCell;
 use core::ffi::c_void;
 use core::fmt;
 use core::marker;
-use core::mem;
+use core::mem::{self, MaybeUninit};
 use core::ptr::{self, NonNull};
 use core::sync::atomic::{AtomicUsize, Ordering};
 use sptr::Strict;
@@ -423,7 +425,11 @@ impl VMGlobalDefinition {
     /// # Unsafety
     ///
     /// This raw value's type must match the given `WasmValType`.
-    pub unsafe fn from_val_raw(wasm_ty: WasmValType, raw: ValRaw) -> Self {
+    pub unsafe fn from_val_raw(
+        store: &mut StoreOpaque,
+        wasm_ty: WasmValType,
+        raw: ValRaw,
+    ) -> Result<Self> {
         let mut global = Self::new();
         match wasm_ty {
             WasmValType::I32 => *global.as_i32_mut() = raw.get_i32(),
@@ -433,13 +439,17 @@ impl VMGlobalDefinition {
             WasmValType::V128 => *global.as_u128_mut() = raw.get_v128(),
             WasmValType::Ref(r) => match r.heap_type.top() {
                 WasmHeapTopType::Extern => {
-                    global.init_gc_ref(VMGcRef::from_raw_u32(raw.get_externref()))
+                    let r = VMGcRef::from_raw_u32(raw.get_externref());
+                    global.init_gc_ref(store.gc_store_mut()?, r.as_ref())
+                }
+                WasmHeapTopType::Any => {
+                    let r = VMGcRef::from_raw_u32(raw.get_anyref());
+                    global.init_gc_ref(store.gc_store_mut()?, r.as_ref())
                 }
-                WasmHeapTopType::Any => global.init_gc_ref(VMGcRef::from_raw_u32(raw.get_anyref())),
                 WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(),
             },
         }
-        global
+        Ok(global)
     }
 
     /// Get this global's value as a `ValRaw`.
@@ -447,25 +457,31 @@ impl VMGlobalDefinition {
     /// # Unsafety
     ///
     /// This global's value's type must match the given `WasmValType`.
-    pub unsafe fn to_val_raw(&self, gc_store: &mut GcStore, wasm_ty: WasmValType) -> ValRaw {
-        match wasm_ty {
+    pub unsafe fn to_val_raw(
+        &self,
+        store: &mut StoreOpaque,
+        wasm_ty: WasmValType,
+    ) -> Result<ValRaw> {
+        Ok(match wasm_ty {
             WasmValType::I32 => ValRaw::i32(*self.as_i32()),
             WasmValType::I64 => ValRaw::i64(*self.as_i64()),
             WasmValType::F32 => ValRaw::f32(*self.as_f32_bits()),
             WasmValType::F64 => ValRaw::f64(*self.as_f64_bits()),
             WasmValType::V128 => ValRaw::v128(*self.as_u128()),
             WasmValType::Ref(r) => match r.heap_type.top() {
-                WasmHeapTopType::Extern => ValRaw::externref(
-                    self.as_gc_ref()
-                        .map_or(0, |r| gc_store.clone_gc_ref(r).as_raw_u32()),
-                ),
-                WasmHeapTopType::Any => ValRaw::anyref(
-                    self.as_gc_ref()
-                        .map_or(0, |r| gc_store.clone_gc_ref(r).as_raw_u32()),
-                ),
+                WasmHeapTopType::Extern => ValRaw::externref(match self.as_gc_ref() {
+                    Some(r) => store.gc_store_mut()?.clone_gc_ref(r).as_raw_u32(),
+                    None => 0,
+                }),
+                WasmHeapTopType::Any => ValRaw::anyref({
+                    match self.as_gc_ref() {
+                        Some(r) => store.gc_store_mut()?.clone_gc_ref(r).as_raw_u32(),
+                        None => 0,
+                    }
+                }),
                 WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()),
             },
-        }
+        })
     }
 
     /// Return a reference to the value as an i32.
@@ -577,10 +593,16 @@ impl VMGlobalDefinition {
     }
 
     /// Initialize a global to the given GC reference.
-    pub unsafe fn init_gc_ref(&mut self, gc_ref: Option<VMGcRef>) {
+    pub unsafe fn init_gc_ref(&mut self, gc_store: &mut GcStore, gc_ref: Option<&VMGcRef>) {
         assert!(cfg!(feature = "gc") || gc_ref.is_none());
-        let raw_ptr = self.storage.as_mut().as_mut_ptr().cast::<Option<VMGcRef>>();
-        ptr::write(raw_ptr, gc_ref);
+
+        let dest = &mut *(self
+            .storage
+            .as_mut()
+            .as_mut_ptr()
+            .cast::<MaybeUninit<Option<VMGcRef>>>());
+
+        gc_store.init_gc_ref(dest, gc_ref)
     }
 
     /// Write a GC reference into this global value.
diff --git a/pulley/Cargo.toml b/pulley/Cargo.toml
index 9d95056fd..bc27619e7 100644
--- a/pulley/Cargo.toml
+++ b/pulley/Cargo.toml
@@ -27,7 +27,7 @@ arbitrary = ["dep:arbitrary", "arbitrary/derive", "std", "cranelift-bitset/arbit
 encode = []
 decode = []
 disas = ["decode"]
-interp = ["decode"]
+interp = ["decode", "encode"]
 
 [package.metadata.docs.rs]
 all-features = true
diff --git a/pulley/src/decode.rs b/pulley/src/decode.rs
index 1a0c8b202..bcd570172 100644
--- a/pulley/src/decode.rs
+++ b/pulley/src/decode.rs
@@ -260,7 +260,8 @@ impl BytecodeStream for UnsafeBytecodeStream {
 
 /// Anything that can be decoded from a bytecode stream, e.g. opcodes,
 /// immediates, registers, etc...
-trait Decode: Sized {
+pub trait Decode: Sized {
+    /// Decode this type from the given bytecode stream.
     fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
     where
         T: BytecodeStream;
@@ -377,6 +378,32 @@ impl Decode for PcRelOffset {
     }
 }
 
+impl Decode for Opcode {
+    fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
+    where
+        T: BytecodeStream,
+    {
+        let byte = u8::decode(bytecode)?;
+        match Opcode::new(byte) {
+            Some(v) => Ok(v),
+            None => Err(bytecode.invalid_opcode(byte)),
+        }
+    }
+}
+
+impl Decode for ExtendedOpcode {
+    fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
+    where
+        T: BytecodeStream,
+    {
+        let word = u16::decode(bytecode)?;
+        match ExtendedOpcode::new(word) {
+            Some(v) => Ok(v),
+            None => Err(bytecode.invalid_extended_opcode(word)),
+        }
+    }
+}
+
 impl<R: Reg> Decode for BinaryOperands<R> {
     fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
     where
@@ -655,3 +682,53 @@ macro_rules! define_extended_decoder {
     };
 }
 for_each_extended_op!(define_extended_decoder);
+
+/// Unwrap a `Result<T, Uninhabited>`.
+/// Always succeeds, since `Uninhabited` is uninhabited.
+pub fn unwrap_uninhabited<T>(res: Result<T, Uninhabited>) -> T {
+    match res {
+        Ok(ok) => ok,
+
+        // Nightly rust warns that this pattern is unreachable, but stable rust
+        // doesn't.
+        #[allow(unreachable_patterns)]
+        Err(err) => match err {},
+    }
+}
+
+/// Functions for decoding the operands of an instruction, assuming the opcode
+/// has already been decoded.
+pub mod operands {
+    use super::*;
+
+    macro_rules! define_operands_decoder {
+        (
+            $(
+                $( #[$attr:meta] )*
+                    $snake_name:ident = $name:ident $( {
+                    $(
+                        $( #[$field_attr:meta] )*
+                        $field:ident : $field_ty:ty
+                    ),*
+                } )? ;
+            )*
+        ) => {
+            $(
+                #[allow(unused_variables)]
+                #[allow(missing_docs)]
+                pub fn $snake_name<T: BytecodeStream>(pc: &mut T) -> Result<($($($field_ty,)*)?), T::Error> {
+                    Ok((($($((<$field_ty>::decode(pc))?,)*)?)))
+                }
+            )*
+        };
+    }
+
+    for_each_op!(define_operands_decoder);
+
+    #[allow(missing_docs)]
+    pub fn extended<T: BytecodeStream>(pc: &mut T) -> Result<(ExtendedOpcode,), T::Error> {
+        Ok((ExtendedOpcode::decode(pc)?,))
+    }
+
+    for_each_extended_op!(define_operands_decoder);
+}
diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs
index 24b3e2a59..49e3a1bb3 100644
--- a/pulley/src/interp.rs
+++ b/pulley/src/interp.rs
@@ -4,19 +4,22 @@ use crate::decode::*;
 use crate::imms::*;
 use crate::regs::*;
 use crate::ExtendedOpcode;
+use crate::Opcode;
 use alloc::string::ToString;
 use alloc::{vec, vec::Vec};
 use core::fmt;
 use core::mem;
+use core::ops::ControlFlow;
 use core::ops::{Index, IndexMut};
 use core::ptr::{self, NonNull};
 use sptr::Strict;
 
+mod interp_loop;
+
 const DEFAULT_STACK_SIZE: usize = 1 << 20; // 1 MiB
 
 /// A virtual machine for interpreting Pulley bytecode.
 pub struct Vm {
-    decoder: Decoder,
     state: MachineState,
 }
 
@@ -35,7 +38,6 @@ impl Vm {
     /// Create a new virtual machine with the given stack.
     pub fn with_stack(stack: Vec<u8>) -> Self {
         Self {
-            decoder: Decoder::new(),
             state: MachineState::with_stack(stack),
         }
     }
@@ -120,29 +122,11 @@ impl Vm {
     }
 
     unsafe fn run(&mut self, pc: NonNull<u8>) -> Result<(), NonNull<u8>> {
-        let mut visitor = InterpreterVisitor {
-            state: &mut self.state,
-            pc: UnsafeBytecodeStream::new(pc),
-        };
-
-        loop {
-            let continuation = self.decoder.decode_one(&mut visitor).unwrap();
-
-            // Really wish we had `feature(explicit_tail_calls)`...
-            match continuation {
-                Continuation::Continue => {
-                    continue;
-                }
-
-                // Out-of-line slow paths marked `cold` and `inline(never)` to
-                // improve codegen.
-                Continuation::Trap => {
-                    let pc = visitor.pc.as_ptr();
-                    return self.trap(pc);
-                }
-                Continuation::ReturnToHost => return self.return_to_host(),
-                Continuation::HostCall => return self.host_call(),
-            }
+        let mut bytecode = UnsafeBytecodeStream::new(pc);
+        match interp_loop::interpreter_loop(self, &mut bytecode) {
+            Done::ReturnToHost => self.return_to_host(),
+            Done::Trap(pc) => self.trap(pc),
+            Done::HostCall => self.host_call(),
         }
     }
 
@@ -271,6 +255,14 @@ impl From<VRegVal> for Val {
 #[derive(Copy, Clone)]
 pub struct XRegVal(XRegUnion);
 
+impl PartialEq for XRegVal {
+    fn eq(&self, other: &Self) -> bool {
+        self.get_u64() == other.get_u64()
+    }
+}
+
+impl Eq for XRegVal {}
+
 impl fmt::Debug for XRegVal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("XRegVal")
@@ -304,6 +296,9 @@ impl Default for XRegVal {
 
 #[allow(missing_docs)]
 impl XRegVal {
+    /// Sentinel return address that signals the end of the call stack.
+    pub const HOST_RETURN_ADDR: Self = Self(XRegUnion { i64: -1 });
+
     pub fn new_i32(x: i32) -> Self {
         let mut val = XRegVal::default();
         val.set_i32(x);
@@ -566,8 +561,8 @@ impl MachineState {
         let sp = sp.as_mut_ptr();
         let sp = unsafe { sp.add(len) };
         state[XReg::sp] = XRegVal::new_ptr(sp);
-        state[XReg::fp] = XRegVal::new_i64(-1);
-        state[XReg::lr] = XRegVal::new_i64(-1);
+        state[XReg::fp] = XRegVal::HOST_RETURN_ADDR;
+        state[XReg::lr] = XRegVal::HOST_RETURN_ADDR;
 
         state
     }
@@ -588,588 +583,16 @@ impl MachineState {
     }
 }
 
-enum Continuation {
-    Continue,
+/// The reason the interpreter loop terminated.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum Done {
+    /// A `ret` instruction was executed and the call stack was empty. This is
+    /// how the loop normally ends.
     ReturnToHost,
-    Trap,
+
+    /// A `trap` instruction was executed at the given PC.
+    Trap(NonNull<u8>),
 
     #[allow(dead_code)]
     HostCall,
 }
-
-struct InterpreterVisitor<'a> {
-    state: &'a mut MachineState,
-    pc: UnsafeBytecodeStream,
-}
-
-impl InterpreterVisitor<'_> {
-    #[inline(always)]
-    fn pc_rel_jump(&mut self, offset: PcRelOffset, inst_size: isize) -> Continuation {
-        let offset = isize::try_from(i32::from(offset)).unwrap();
-        self.pc = unsafe { self.pc.offset(offset - inst_size) };
-        Continuation::Continue
-    }
-}
-
-#[doc(hidden)]
-impl OpVisitor for InterpreterVisitor<'_> {
-    type BytecodeStream = UnsafeBytecodeStream;
-
-    fn bytecode(&mut self) -> &mut Self::BytecodeStream {
-        &mut self.pc
-    }
-
-    type Return = Continuation;
-
-    fn ret(&mut self) -> Self::Return {
-        if self.state[XReg::lr].get_u64() == u64::MAX {
-            Continuation::ReturnToHost
-        } else {
-            let return_addr = self.state[XReg::lr].get_ptr();
-            self.pc = unsafe { UnsafeBytecodeStream::new(NonNull::new_unchecked(return_addr)) };
-            // log::trace!("returning to {return_addr:#p}");
-            Continuation::Continue
-        }
-    }
-
-    fn call(&mut self, offset: PcRelOffset) -> Self::Return {
-        let return_addr = self.pc.as_ptr();
-        self.state[XReg::lr].set_ptr(return_addr.as_ptr());
-        self.pc_rel_jump(offset, 5)
-    }
-
-    fn jump(&mut self, offset: PcRelOffset) -> Self::Return {
-        self.pc_rel_jump(offset, 5)
-    }
-
-    fn br_if(&mut self, cond: XReg, offset: PcRelOffset) -> Self::Return {
-        let cond = self.state[cond].get_u64();
-        if cond != 0 {
-            self.pc_rel_jump(offset, 6)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_not(&mut self, cond: XReg, offset: PcRelOffset) -> Self::Return {
-        let cond = self.state[cond].get_u64();
-        if cond == 0 {
-            self.pc_rel_jump(offset, 6)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xeq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a == b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xneq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a != b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslt32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i32();
-        let b = self.state[b].get_i32();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslteq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i32();
-        let b = self.state[b].get_i32();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xult32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xulteq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xeq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a == b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xneq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a != b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslt64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i64();
-        let b = self.state[b].get_i64();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslteq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i64();
-        let b = self.state[b].get_i64();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xult64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xulteq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn xmov(&mut self, dst: XReg, src: XReg) -> Self::Return {
-        let val = self.state[src];
-        self.state[dst] = val;
-        Continuation::Continue
-    }
-
-    fn fmov(&mut self, dst: FReg, src: FReg) -> Self::Return {
-        let val = self.state[src];
-        self.state[dst] = val;
-        Continuation::Continue
-    }
-
-    fn vmov(&mut self, dst: VReg, src: VReg) -> Self::Return {
-        let val = self.state[src];
-        self.state[dst] = val;
-        Continuation::Continue
-    }
-
-    fn xconst8(&mut self, dst: XReg, imm: i8) -> Self::Return {
-        self.state[dst].set_i64(i64::from(imm));
-        Continuation::Continue
-    }
-
-    fn xconst16(&mut self, dst: XReg, imm: i16) -> Self::Return {
-        self.state[dst].set_i64(i64::from(imm));
-        Continuation::Continue
-    }
-
-    fn xconst32(&mut self, dst: XReg, imm: i32) -> Self::Return {
-        self.state[dst].set_i64(i64::from(imm));
-        Continuation::Continue
-    }
-
-    fn xconst64(&mut self, dst: XReg, imm: i64) -> Self::Return {
-        self.state[dst].set_i64(imm);
-        Continuation::Continue
-    }
-
-    fn xadd32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u32(a.wrapping_add(b));
-        Continuation::Continue
-    }
-
-    fn xadd64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(a.wrapping_add(b));
-        Continuation::Continue
-    }
-
-    fn xeq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a == b));
-        Continuation::Continue
-    }
-
-    fn xneq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a != b));
-        Continuation::Continue
-    }
-
-    fn xslt64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i64();
-        let b = self.state[operands.src2].get_i64();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xslteq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i64();
-        let b = self.state[operands.src2].get_i64();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn xult64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xulteq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn xeq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a == b));
-        Continuation::Continue
-    }
-
-    fn xneq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a != b));
-        Continuation::Continue
-    }
-
-    fn xslt32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i32();
-        let b = self.state[operands.src2].get_i32();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xslteq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i32();
-        let b = self.state[operands.src2].get_i32();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn xult32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xulteq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn load32_u(&mut self, dst: XReg, ptr: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u32>();
-        let val = unsafe { ptr::read_unaligned(ptr) };
-        self.state[dst].set_u64(u64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_s(&mut self, dst: XReg, ptr: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<i32>();
-        let val = unsafe { ptr::read_unaligned(ptr) };
-        self.state[dst].set_i64(i64::from(val));
-        Continuation::Continue
-    }
-
-    fn load64(&mut self, dst: XReg, ptr: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u64>();
-        let val = unsafe { ptr::read_unaligned(ptr) };
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn load32_u_offset8(&mut self, dst: XReg, ptr: XReg, offset: i8) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset.into())
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(u64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_s_offset8(&mut self, dst: XReg, ptr: XReg, offset: i8) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<i32>()
-                .byte_offset(offset.into())
-                .read_unaligned()
-        };
-        self.state[dst].set_i64(i64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_u_offset64(&mut self, dst: XReg, ptr: XReg, offset: i64) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset as isize)
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(u64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_s_offset64(&mut self, dst: XReg, ptr: XReg, offset: i64) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<i32>()
-                .byte_offset(offset as isize)
-                .read_unaligned()
-        };
-        self.state[dst].set_i64(i64::from(val));
-        Continuation::Continue
-    }
-
-    fn load64_offset8(&mut self, dst: XReg, ptr: XReg, offset: i8) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset.into())
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn load64_offset64(&mut self, dst: XReg, ptr: XReg, offset: i64) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset as isize)
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn store32(&mut self, ptr: XReg, src: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u32>();
-        let val = self.state[src].get_u32();
-        unsafe {
-            ptr::write_unaligned(ptr, val);
-        }
-        Continuation::Continue
-    }
-
-    fn store64(&mut self, ptr: XReg, src: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u64>();
-        let val = self.state[src].get_u64();
-        unsafe {
-            ptr::write_unaligned(ptr, val);
-        }
-        Continuation::Continue
-    }
-
-    fn store32_offset8(&mut self, ptr: XReg, offset: i8, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u32();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset.into())
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn store64_offset8(&mut self, ptr: XReg, offset: i8, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u64();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset.into())
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn store32_offset64(&mut self, ptr: XReg, offset: i64, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u32();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset as isize)
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn store64_offset64(&mut self, ptr: XReg, offset: i64, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u64();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset as isize)
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn xpush32(&mut self, src: XReg) -> Self::Return {
-        self.state.push(self.state[src].get_u32());
-        Continuation::Continue
-    }
-
-    fn xpush32_many(&mut self, srcs: RegSet<XReg>) -> Self::Return {
-        for src in srcs {
-            self.xpush32(src);
-        }
-        Continuation::Continue
-    }
-
-    fn xpush64(&mut self, src: XReg) -> Self::Return {
-        self.state.push(self.state[src].get_u64());
-        Continuation::Continue
-    }
-
-    fn xpush64_many(&mut self, srcs: RegSet<XReg>) -> Self::Return {
-        for src in srcs {
-            self.xpush64(src);
-        }
-        Continuation::Continue
-    }
-
-    fn xpop32(&mut self, dst: XReg) -> Self::Return {
-        let val = self.state.pop();
-        self.state[dst].set_u32(val);
-        Continuation::Continue
-    }
-
-    fn xpop32_many(&mut self, dsts: RegSet<XReg>) -> Self::Return {
-        for dst in dsts.into_iter().rev() {
-            self.xpop32(dst);
-        }
-        Continuation::Continue
-    }
-
-    fn xpop64(&mut self, dst: XReg) -> Self::Return {
-        let val = self.state.pop();
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn xpop64_many(&mut self, dsts: RegSet<XReg>) -> Self::Return {
-        for dst in dsts.into_iter().rev() {
-            self.xpop64(dst);
-        }
-        Continuation::Continue
-    }
-
-    /// `push lr; push fp; fp = sp`
-    fn push_frame(&mut self) -> Self::Return {
-        self.state.push(self.state[XReg::lr].get_ptr::<u8>());
-        self.state.push(self.state[XReg::fp].get_ptr::<u8>());
-        self.state[XReg::fp] = self.state[XReg::sp];
-        Continuation::Continue
-    }
-
-    /// `sp = fp; pop fp; pop lr`
-    fn pop_frame(&mut self) -> Self::Return {
-        self.state[XReg::sp] = self.state[XReg::fp];
-        let fp = self.state.pop();
-        let lr = self.state.pop();
-        self.state[XReg::fp].set_ptr::<u8>(fp);
-        self.state[XReg::lr].set_ptr::<u8>(lr);
-        Continuation::Continue
-    }
-
-    fn bitcast_int_from_float_32(&mut self, dst: XReg, src: FReg) -> Self::Return {
-        let val = self.state[src].get_f32();
-        self.state[dst].set_u64(u32::from_ne_bytes(val.to_ne_bytes()).into());
-        Continuation::Continue
-    }
-
-    fn bitcast_int_from_float_64(&mut self, dst: XReg, src: FReg) -> Self::Return {
-        let val = self.state[src].get_f64();
-        self.state[dst].set_u64(u64::from_ne_bytes(val.to_ne_bytes()));
-        Continuation::Continue
-    }
-
-    fn bitcast_float_from_int_32(&mut self, dst: FReg, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u32();
-        self.state[dst].set_f32(f32::from_ne_bytes(val.to_ne_bytes()));
-        Continuation::Continue
-    }
-
-    fn bitcast_float_from_int_64(&mut self, dst: FReg, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u64();
-        self.state[dst].set_f64(f64::from_ne_bytes(val.to_ne_bytes()));
-        Continuation::Continue
-    }
-}
-
-impl ExtendedOpVisitor for InterpreterVisitor<'_> {
-    fn nop(&mut self) -> Self::Return {
-        Continuation::Continue
-    }
-
-    fn trap(&mut self) -> Self::Return {
-        Continuation::Trap
-    }
-
-    fn get_sp(&mut self, dst: XReg) -> Self::Return {
-        let sp = self.state[XReg::sp].get_u64();
-        self.state[dst].set_u64(sp);
-        Continuation::Continue
-    }
-}
diff --git a/pulley/src/interp/interp_loop.rs b/pulley/src/interp/interp_loop.rs
new file mode 100644
index 000000000..c17cba3ca
--- /dev/null
+++ b/pulley/src/interp/interp_loop.rs
@@ -0,0 +1,669 @@
+use super::*;
+
+#[derive(Copy, Clone)]
+pub struct OpcodeHandler {
+    /// The type of non tail-recursive opcode handlers: return
+    /// `ControlFlow::Continue` with the next handler to call, or
+    /// `ControlFlow::Done` with the reason to stop.
+    #[cfg(not(pulley_tail_calls))]
+    fun: fn(&mut MachineState, &mut UnsafeBytecodeStream) -> ControlFlow<Done, OpcodeHandler>,
+
+    /// The type of tail-recursive opcode handlers: instead of returning
+    /// `ControwFlow::Continue`, tail call the next handler directly; so
+    /// `ControlFlow::Continue` is uninhabited.
+    #[cfg(pulley_tail_calls)]
+    fun: fn(&mut MachineState, &mut UnsafeBytecodeStream) -> Done,
+}
+
+#[cfg(not(pulley_tail_calls))]
+pub fn interpreter_loop(vm: &mut Vm, bytecode: &mut UnsafeBytecodeStream) -> Done {
+    let opcode = Opcode::decode(bytecode).unwrap();
+    let mut handler = OPCODE_HANDLER_TABLE[opcode as usize];
+
+    // As tight as we can get the interpreter loop without tail calls: while
+    // the handlers keep returning the next handler to call, call it.
+    loop {
+        match (handler.fun)(&mut vm.state, bytecode) {
+            ControlFlow::Continue(next_handler) => handler = next_handler,
+            ControlFlow::Break(done) => return done,
+        }
+    }
+}
+
+/// The extra indirection through a macro is necessary to avoid a compiler error
+/// when compiling without `#![feature(explicit_tail_calls)]` enabled (via
+/// `--cfg pulley_tail_calls`).
+///
+/// It seems rustc first parses the the function, encounters `become` and emits
+/// an error about using an unstable keyword on a stable compiler, then applies
+/// `#[cfg(...)` after parsing to disable the function.
+///
+/// Macro bodies are just bags of tokens; the body is not parsed until after
+/// they are expanded, and this macro is only expanded when `pulley_tail_calls`
+/// is enabled.
+#[cfg_attr(not(pulley_tail_calls), allow(unused_macros))]
+macro_rules! tail_call {
+    ($e:expr) => {
+        become $e
+    };
+}
+
+#[cfg(pulley_tail_calls)]
+pub fn interpreter_loop(vm: &mut Vm, bytecode: &mut UnsafeBytecodeStream) -> Done {
+    let opcode = Opcode::decode(bytecode).unwrap();
+    let handler = OPCODE_HANDLER_TABLE[opcode as usize];
+
+    // The ideal interpreter loop: a bunch of opcode handlers tail calling
+    // each other!
+    tail_call!((handler.fun)(&mut vm.state, bytecode));
+}
+
+/// Wrap the business logic of each handler with the boilerplate of decoding
+/// operands and dispatching to next handler/exiting the loop.
+macro_rules! define_opcode_handlers {
+    (
+    $(
+        fn $name:ident (
+            $state:ident : &mut MachineState,
+            $bytecode:ident : &mut UnsafeBytecodeStream$(,)?
+            $($field:ident : $field_ty:ty),*
+        ) $body:block
+    )*
+    ) => {
+        $(
+            #[cfg(not(pulley_tail_calls))]
+            pub fn $name($state: &mut MachineState, $bytecode: &mut UnsafeBytecodeStream) -> ControlFlow<Done, OpcodeHandler> {
+                let ($($field,)*) = crate::decode::unwrap_uninhabited(crate::decode::operands::$name($bytecode));
+                match $body {
+                    ControlFlow::Continue(()) => {
+                        // Decode the next handler and return it so that `run`
+                        // can call it.
+                        let next_opcode = Opcode::decode($bytecode).unwrap();
+                        let next_handler = OPCODE_HANDLER_TABLE[next_opcode as usize];
+                        ControlFlow::Continue(next_handler)
+                    }
+                    ControlFlow::Break(done) => ControlFlow::Break(done),
+                }
+            }
+
+            #[cfg(pulley_tail_calls)]
+            pub fn $name($state: &mut MachineState, $bytecode: &mut UnsafeBytecodeStream) -> Done {
+                let ($($field,)*) = crate::decode::unwrap_uninhabited(crate::decode::operands::$name($bytecode));
+                match $body {
+                    ControlFlow::Continue(()) => {
+                        // Decode the next handler and return it so that `run`
+                        // can call it.
+                        let next_opcode = Opcode::decode($bytecode).unwrap();
+                        let next_handler = OPCODE_HANDLER_TABLE[next_opcode as usize];
+                        tail_call!((next_handler.fun)($state, $bytecode));
+                    }
+                    ControlFlow::Break(done) => done,
+                }
+            }
+        )*
+    };
+}
+
+/// Define the table of opcode handlers.
+macro_rules! opcode_handler_table_entry {
+    (
+        $(
+            $( #[$attr:meta] )*
+                $snake_name:ident = $name:ident $( {
+                $(
+                    $( #[$field_attr:meta] )*
+                    $field:ident : $field_ty:ty
+                ),*
+            } )? ;
+        )*
+    ) => {[ $(OpcodeHandler { fun: $snake_name },)* OpcodeHandler { fun: extended }]};
+}
+
+/// Add one to account for `ExtendedOp`.
+const NUM_OPCODES: usize = Opcode::MAX as usize + 1;
+static OPCODE_HANDLER_TABLE: [OpcodeHandler; NUM_OPCODES] =
+    for_each_op!(opcode_handler_table_entry);
+
+#[inline]
+fn pc_rel_jump(pc: &mut UnsafeBytecodeStream, offset: PcRelOffset, inst_size: isize) {
+    let offset = isize::try_from(i32::from(offset)).unwrap();
+    *pc = unsafe { pc.offset(offset - inst_size) };
+}
+
+define_opcode_handlers! {
+    fn ret(state: &mut MachineState, pc: &mut UnsafeBytecodeStream) {
+        if state[XReg::lr] == XRegVal::HOST_RETURN_ADDR {
+            ControlFlow::Break(Done::ReturnToHost)
+        } else {
+            let return_addr = state[XReg::lr].get_ptr();
+            *pc = unsafe { UnsafeBytecodeStream::new(NonNull::new_unchecked(return_addr)) };
+            ControlFlow::Continue(())
+        }
+    }
+
+    fn call(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, offset: PcRelOffset) {
+        let return_addr = pc.as_ptr();
+        state[XReg::lr].set_ptr(return_addr.as_ptr());
+        pc_rel_jump(pc, offset, 5);
+        ControlFlow::Continue(())
+    }
+
+    fn jump(_state: &mut MachineState, pc: &mut UnsafeBytecodeStream, offset: PcRelOffset) {
+        pc_rel_jump(pc, offset, 5);
+        ControlFlow::Continue(())
+    }
+
+    fn br_if(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, cond: XReg, offset: PcRelOffset) {
+        let cond = state[cond].get_u64();
+        if cond != 0 {
+            pc_rel_jump(pc, offset, 6)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_not(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, cond: XReg, offset: PcRelOffset) {
+        let cond = state[cond].get_u64();
+        if cond == 0 {
+            pc_rel_jump(pc, offset, 6)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xeq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a == b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xneq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a != b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslt32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i32();
+        let b = state[b].get_i32();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i32();
+        let b = state[b].get_i32();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xult32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xulteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xeq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a == b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xneq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a != b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslt64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i64();
+        let b = state[b].get_i64();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i64();
+        let b = state[b].get_i64();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xult64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xulteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xmov(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, src: XReg) {
+        let val = state[src];
+        state[dst] = val;
+        ControlFlow::Continue(())
+    }
+
+    fn fmov(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: FReg, src: FReg) {
+        let val = state[src];
+        state[dst] = val;
+        ControlFlow::Continue(())
+    }
+
+    fn vmov(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: VReg, src: VReg) {
+        let val = state[src];
+        state[dst] = val;
+        ControlFlow::Continue(())
+    }
+
+    fn xconst8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i8) {
+        state[dst].set_i64(i64::from(imm));
+        ControlFlow::Continue(())
+    }
+
+    fn xconst16(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i16) {
+        state[dst].set_i64(i64::from(imm));
+        ControlFlow::Continue(())
+    }
+
+    fn xconst32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i32) {
+        state[dst].set_i64(i64::from(imm));
+        ControlFlow::Continue(())
+    }
+
+    fn xconst64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i64) {
+        state[dst].set_i64(imm);
+        ControlFlow::Continue(())
+    }
+
+    fn xadd32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u32(a.wrapping_add(b));
+        ControlFlow::Continue(())
+    }
+
+    fn xadd64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(a.wrapping_add(b));
+        ControlFlow::Continue(())
+    }
+
+    fn xeq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a == b));
+        ControlFlow::Continue(())
+    }
+
+    fn xneq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a != b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslt64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i64();
+        let b = state[operands.src2].get_i64();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i64();
+        let b = state[operands.src2].get_i64();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn xult64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xulteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn xeq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a == b));
+        ControlFlow::Continue(())
+    }
+
+    fn xneq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a != b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslt32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i32();
+        let b = state[operands.src2].get_i32();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i32();
+        let b = state[operands.src2].get_i32();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn xult32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xulteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_u(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg) {
+        let ptr = state[ptr].get_ptr::<u32>();
+        let val = unsafe { ptr::read_unaligned(ptr) };
+        state[dst].set_u64(u64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_s(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg) {
+        let ptr = state[ptr].get_ptr::<i32>();
+        let val = unsafe { ptr::read_unaligned(ptr) };
+        state[dst].set_i64(i64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg) {
+        let ptr = state[ptr].get_ptr::<u64>();
+        let val = unsafe { ptr::read_unaligned(ptr) };
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn load32_u_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i8) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset.into())
+                .read_unaligned()
+        };
+        state[dst].set_u64(u64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_s_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i8) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<i32>()
+                .byte_offset(offset.into())
+                .read_unaligned()
+        };
+        state[dst].set_i64(i64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_u_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i64) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset as isize)
+                .read_unaligned()
+        };
+        state[dst].set_u64(u64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_s_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i64) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<i32>()
+                .byte_offset(offset as isize)
+                .read_unaligned()
+        };
+        state[dst].set_i64(i64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load64_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i8) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset.into())
+                .read_unaligned()
+        };
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn load64_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i64) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset as isize)
+                .read_unaligned()
+        };
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn store32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, src: XReg) {
+        let ptr = state[ptr].get_ptr::<u32>();
+        let val = state[src].get_u32();
+        unsafe {
+            ptr::write_unaligned(ptr, val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, src: XReg) {
+        let ptr = state[ptr].get_ptr::<u64>();
+        let val = state[src].get_u64();
+        unsafe {
+            ptr::write_unaligned(ptr, val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store32_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i8, src: XReg) {
+        let val = state[src].get_u32();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset.into())
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store64_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i8, src: XReg) {
+        let val = state[src].get_u64();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset.into())
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store32_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i64, src: XReg) {
+        let val = state[src].get_u32();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset as isize)
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store64_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i64, src: XReg) {
+        let val = state[src].get_u64();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset as isize)
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpush32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, src: XReg) {
+        state.push(state[src].get_u32());
+        ControlFlow::Continue(())
+    }
+
+    fn xpush32_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, srcs: RegSet<XReg>) {
+        for src in srcs {
+            state.push(state[src].get_u32());
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpush64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, src: XReg) {
+        state.push(state[src].get_u64());
+        ControlFlow::Continue(())
+    }
+
+    fn xpush64_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, srcs: RegSet<XReg>) {
+        for src in srcs {
+            state.push(state[src].get_u64());
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpop32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg) {
+        let val = state.pop();
+        state[dst].set_u32(val);
+        ControlFlow::Continue(())
+    }
+
+    fn xpop32_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dsts: RegSet<XReg>) {
+        for dst in dsts.into_iter().rev() {
+            let val = state.pop();
+            state[dst].set_u32(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpop64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg) {
+        let val = state.pop();
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn xpop64_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dsts: RegSet<XReg>) {
+        for dst in dsts.into_iter().rev() {
+            let val = state.pop();
+            state[dst].set_u64(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn push_frame(state: &mut MachineState, pc: &mut UnsafeBytecodeStream) {
+        state.push(state[XReg::lr].get_ptr::<u8>());
+        state.push(state[XReg::fp].get_ptr::<u8>());
+        state[XReg::fp] = state[XReg::sp];
+        ControlFlow::Continue(())
+    }
+
+    fn pop_frame(state: &mut MachineState, pc: &mut UnsafeBytecodeStream) {
+        state[XReg::sp] = state[XReg::fp];
+        let fp = state.pop();
+        let lr = state.pop();
+        state[XReg::fp].set_ptr::<u8>(fp);
+        state[XReg::lr].set_ptr::<u8>(lr);
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_int_from_float_32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, src: FReg) {
+        let val = state[src].get_f32();
+        state[dst].set_u64(u32::from_ne_bytes(val.to_ne_bytes()).into());
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_int_from_float_64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, src: FReg) {
+        let val = state[src].get_f64();
+        state[dst].set_u64(u64::from_ne_bytes(val.to_ne_bytes()));
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_float_from_int_32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: FReg, src: XReg) {
+        let val = state[src].get_u32();
+        state[dst].set_f32(f32::from_ne_bytes(val.to_ne_bytes()));
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_float_from_int_64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: FReg, src: XReg) {
+        let val = state[src].get_u64();
+        state[dst].set_f64(f64::from_ne_bytes(val.to_ne_bytes()));
+        ControlFlow::Continue(())
+    }
+
+    fn extended(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, opcode: ExtendedOpcode) {
+        match opcode {
+            ExtendedOpcode::Nop => ControlFlow::Continue(()),
+            ExtendedOpcode::Trap => ControlFlow::Break(Done::Trap(pc.as_ptr())),
+            ExtendedOpcode::GetSp => {
+                let (dst,) = crate::decode::unwrap_uninhabited(crate::decode::operands::get_sp(pc));
+                let sp = state[XReg::sp].get_u64();
+                state[dst].set_u64(sp);
+                ControlFlow::Continue(())
+            }
+        }
+    }
+}
diff --git a/pulley/src/lib.rs b/pulley/src/lib.rs
index f792b83df..b80a5646e 100644
--- a/pulley/src/lib.rs
+++ b/pulley/src/lib.rs
@@ -1,6 +1,8 @@
 //! The pulley bytecode for fast interpreters.
 
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![cfg_attr(pulley_tail_calls, feature(explicit_tail_calls))]
+#![cfg_attr(pulley_tail_calls, allow(incomplete_features, unstable_features))]
 #![deny(missing_docs)]
 #![no_std]
 
diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock
index a3d879a51..6d2a0f320 100644
--- a/supply-chain/imports.lock
+++ b/supply-chain/imports.lock
@@ -13,6 +13,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-bforest]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -25,6 +29,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-bforest]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-bitset]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -37,6 +45,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-bitset]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-codegen]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -49,6 +61,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-codegen]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-codegen-meta]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -61,6 +77,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-codegen-meta]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-codegen-shared]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -73,6 +93,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-codegen-shared]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-control]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -85,6 +109,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-control]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-entity]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -97,6 +125,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-entity]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-frontend]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -109,6 +141,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-frontend]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-interpreter]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -121,6 +157,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-interpreter]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-isle]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -133,6 +173,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-isle]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-jit]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -145,6 +189,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-jit]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-module]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -157,6 +205,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-module]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-native]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -169,6 +221,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-native]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-object]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -181,6 +237,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-object]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-reader]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -193,6 +253,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-reader]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-serde]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -205,6 +269,10 @@ audited_as = "0.110.1"
 version = "0.113.0"
 audited_as = "0.111.0"
 
+[[unpublished.cranelift-serde]]
+version = "0.114.0"
+audited_as = "0.112.1"
+
 [[unpublished.cranelift-wasm]]
 version = "0.111.0"
 audited_as = "0.110.1"
@@ -225,6 +293,10 @@ audited_as = "0.1.1"
 version = "26.0.0"
 audited_as = "0.1.1"
 
+[[unpublished.pulley-interpreter]]
+version = "27.0.0"
+audited_as = "0.1.1"
+
 [[unpublished.wasi-common]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -237,6 +309,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasi-common]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -249,6 +325,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-asm-macros]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -261,6 +341,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-asm-macros]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-cache]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -273,6 +357,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-cache]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-cli]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -285,6 +373,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-cli]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-cli-flags]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -297,6 +389,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-cli-flags]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-component-macro]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -309,6 +405,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-component-macro]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-component-util]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -321,6 +421,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-component-util]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-cranelift]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -333,6 +437,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-cranelift]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-environ]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -345,6 +453,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-environ]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-explorer]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -357,6 +469,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-explorer]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-fiber]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -369,6 +485,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-fiber]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-jit-debug]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -381,6 +501,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-jit-debug]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-jit-icache-coherence]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -393,6 +517,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-jit-icache-coherence]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-slab]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -405,6 +533,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-slab]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-types]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -429,6 +561,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wasi]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wasi-http]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -441,6 +577,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wasi-http]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wasi-keyvalue]]
 version = "25.0.0"
 audited_as = "24.0.0"
@@ -449,6 +589,10 @@ audited_as = "24.0.0"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wasi-keyvalue]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wasi-nn]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -461,6 +605,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wasi-nn]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wasi-runtime-config]]
 version = "25.0.0"
 audited_as = "24.0.0"
@@ -469,6 +617,10 @@ audited_as = "24.0.0"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wasi-runtime-config]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wasi-threads]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -481,6 +633,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wasi-threads]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wast]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -493,6 +649,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wast]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-winch]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -505,6 +665,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-winch]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wit-bindgen]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -517,6 +681,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wit-bindgen]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wasmtime-wmemcheck]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -529,6 +697,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wasmtime-wmemcheck]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wiggle]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -541,6 +713,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wiggle]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wiggle-generate]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -553,6 +729,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wiggle-generate]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wiggle-macro]]
 version = "24.0.0"
 audited_as = "23.0.1"
@@ -565,6 +745,10 @@ audited_as = "23.0.1"
 version = "26.0.0"
 audited_as = "24.0.0"
 
+[[unpublished.wiggle-macro]]
+version = "27.0.0"
+audited_as = "25.0.1"
+
 [[unpublished.wiggle-test]]
 version = "0.0.0"
 audited_as = "0.1.0"
@@ -585,6 +769,10 @@ audited_as = "0.22.0"
 version = "26.0.0"
 audited_as = "0.23.1"
 
+[[unpublished.winch-codegen]]
+version = "27.0.0"
+audited_as = "0.23.1"
+
 [[publisher.aho-corasick]]
 version = "1.0.2"
 when = "2023-06-04"
@@ -877,6 +1065,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-bforest]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -889,6 +1083,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-bforest]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-bitset]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -901,6 +1101,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-bitset]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-codegen]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -913,6 +1119,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-codegen]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-codegen-meta]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -925,6 +1137,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-codegen-meta]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-codegen-shared]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -937,6 +1155,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-codegen-shared]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-control]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -949,6 +1173,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-control]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-entity]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -961,6 +1191,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-entity]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-frontend]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -973,6 +1209,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-frontend]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-interpreter]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -985,6 +1227,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-interpreter]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-isle]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -997,6 +1245,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-isle]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-jit]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -1009,6 +1263,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-jit]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-module]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -1021,6 +1281,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-module]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-native]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -1033,6 +1299,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-native]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-object]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -1045,6 +1317,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-object]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-reader]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -1057,6 +1335,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-reader]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.cranelift-serde]]
 version = "0.110.1"
 when = "2024-07-22"
@@ -1069,6 +1353,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.cranelift-serde]]
+version = "0.112.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.derive_arbitrary]]
 version = "1.3.0"
 when = "2023-03-13"
@@ -1548,6 +1838,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasi-common]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasm-bindgen]]
 version = "0.2.87"
 when = "2023-06-12"
@@ -1715,6 +2011,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-asm-macros]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1727,6 +2029,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-asm-macros]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-cache]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1739,6 +2047,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-cache]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-cli]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1751,6 +2065,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-cli]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-cli-flags]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1763,6 +2083,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-cli-flags]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-component-macro]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1775,6 +2101,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-component-macro]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-component-util]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1787,6 +2119,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-component-util]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-cranelift]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1799,6 +2137,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-cranelift]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-environ]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1811,6 +2155,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-environ]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-explorer]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1823,6 +2173,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-explorer]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-fiber]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1835,6 +2191,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-fiber]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-jit-debug]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1847,6 +2209,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-jit-debug]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-jit-icache-coherence]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1859,6 +2227,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-jit-icache-coherence]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-slab]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1871,6 +2245,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-slab]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wasi]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1883,6 +2263,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wasi]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wasi-http]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1895,12 +2281,24 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wasi-http]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wasi-keyvalue]]
 version = "24.0.0"
 when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wasi-keyvalue]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wasi-nn]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1913,12 +2311,24 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wasi-nn]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wasi-runtime-config]]
 version = "24.0.0"
 when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wasi-runtime-config]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wasi-threads]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1931,6 +2341,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wasi-threads]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wast]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1943,6 +2359,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wast]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-winch]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1955,6 +2377,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-winch]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wit-bindgen]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1967,6 +2395,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wit-bindgen]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wasmtime-wmemcheck]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -1979,6 +2413,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wasmtime-wmemcheck]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wast]]
 version = "214.0.0"
 when = "2024-07-16"
@@ -2051,6 +2491,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wiggle]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wiggle-generate]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -2063,6 +2509,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wiggle-generate]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wiggle-macro]]
 version = "23.0.1"
 when = "2024-07-22"
@@ -2075,6 +2527,12 @@ when = "2024-08-20"
 user-id = 73222
 user-login = "wasmtime-publish"
 
+[[publisher.wiggle-macro]]
+version = "25.0.1"
+when = "2024-09-24"
+user-id = 73222
+user-login = "wasmtime-publish"
+
 [[publisher.wiggle-test]]
 version = "0.1.0"
 when = "2020-03-12"


Test patch:
diff --git a/crates/wasi-nn/tests/check/openvino.rs b/crates/wasi-nn/tests/check/openvino.rs
index 4bf05ba09..d773c1e3c 100644
--- a/crates/wasi-nn/tests/check/openvino.rs
+++ b/crates/wasi-nn/tests/check/openvino.rs
@@ -23,7 +23,7 @@ pub fn is_installed() -> Result<()> {
 pub fn are_artifacts_available() -> Result<()> {
     let _exclusively_retrieve_artifacts = DOWNLOAD_LOCK.lock().unwrap();
     const BASE_URL: &str =
-        "https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/mobilenet";
+        "https://github.com/intel/openvino-rs/raw/72d75601e9be394b3e8c7ff28313d66ef53ff358/crates/openvino/tests/fixtures/mobilenet";
     let artifacts_dir = artifacts_dir();
     if !artifacts_dir.is_dir() {
         fs::create_dir(&artifacts_dir)?;
diff --git a/tests/wast.rs b/tests/wast.rs
index 933dfc33b..e3f498558 100644
--- a/tests/wast.rs
+++ b/tests/wast.rs
@@ -202,7 +202,6 @@ fn should_fail(test: &Path, strategy: Strategy) -> bool {
         }
     }
     let unsupported_gc_tests = [
-        "array_copy.wast",
         "binary_gc.wast",
         "br_on_cast_fail.wast",
         "br_on_cast.wast",
@@ -210,8 +209,6 @@ fn should_fail(test: &Path, strategy: Strategy) -> bool {
         "ref_cast.wast",
         "ref_eq.wast",
         "ref_test.wast",
-        "return_call_indirect.wast",
-        "return_call.wast",
         "table_sub.wast",
         "type_canon.wast",
         "type_equivalence.wast",
@@ -281,7 +278,7 @@ fn run_wast(wast: &Path, strategy: Strategy, pooling: bool) -> anyhow::Result<()
     let function_references = gc || memory64 || feature_found(wast, "function-references");
     let reference_types = !(threads && feature_found(wast, "proposals"));
     let relaxed_simd = feature_found(wast, "relaxed-simd");
-    let tail_call = feature_found(wast, "tail-call") || feature_found(wast, "function-references");
+    let tail_call = function_references || feature_found(wast, "tail-call");
     let use_shared_memory = feature_found_src(&wast_bytes, "shared_memory")
         || feature_found_src(&wast_bytes, "shared)");
     let extended_const = feature_found(wast, "extended-const") || memory64;