Repository: bytecodealliance/wasmtime
Pull request #9391 (2024-10-07 19:07:09)

Run the nightly CI job when `pulley` files are touched in a PR
After https://github.com/bytecodealliance/wasmtime/pull/9251, we will check that Pulley builds with rustc's experimental tail calls feature enabled in the nightly CI job. So if we touch any pulley source files in a PR, we should also run that job in the PR's CI.

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->



Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c6212b485..37679b505 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -199,6 +199,7 @@ jobs:
       build-matrix: ${{ steps.calculate.outputs.build-matrix }}
       test-capi: ${{ steps.calculate.outputs.test-capi }}
       test-nightly: ${{ steps.calculate.outputs.test-nightly }}
+      test-miri: ${{ steps.calculate.outputs.test-miri }}
       audit: ${{ steps.calculate.outputs.audit }}
       preview1-adapter: ${{ steps.calculate.outputs.preview1-adapter }}
       run-dwarf: ${{ steps.calculate.outputs.run-dwarf }}
@@ -246,6 +247,10 @@ jobs:
           if grep -q debug names.log; then
             echo run-dwarf=true >> $GITHUB_OUTPUT
           fi
+          if grep -q pulley names.log; then
+            echo test-nightly=true >> $GITHUB_OUTPUT
+            echo test-miri=true >> $GITHUB_OUTPUT
+          fi
         fi
         matrix="$(node ./ci/build-test-matrix.js ./commits.log ./names.log $run_full)"
         echo "test-matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
@@ -258,6 +263,7 @@ jobs:
             echo run-full=true >> $GITHUB_OUTPUT
             echo test-capi=true >> $GITHUB_OUTPUT
             echo test-nightly=true >> $GITHUB_OUTPUT
+            echo test-miri=true >> $GITHUB_OUTPUT
             echo audit=true >> $GITHUB_OUTPUT
             echo preview1-adapter=true >> $GITHUB_OUTPUT
             echo run-dwarf=true >> $GITHUB_OUTPUT
@@ -552,6 +558,13 @@ jobs:
       with:
         toolchain: wasmtime-ci-pinned-nightly
 
+    # Check that `pulley-interpreter` compiles with tail calls enabled.  Don't
+    # actually run the tests with tail calls enabled, because they are not yet
+    # implemented in rustc and cause an ICE.
+    - run: cargo check -p pulley-interpreter
+      env:
+        RUSTFLAGS: "--cfg pulley_tail_calls"
+
     # Ensure that fuzzers still build.
     #
     # Install the OCaml packages necessary for fuzz targets that use the
@@ -1027,7 +1040,7 @@ jobs:
           - "wasmtime-environ"
           - "pulley-interpreter --all-features"
     needs: determine
-    if: needs.determine.outputs.run-full && github.repository == 'bytecodealliance/wasmtime'
+    if: needs.determine.outputs.test-miri && github.repository == 'bytecodealliance/wasmtime'
     name: Miri
     runs-on: ubuntu-latest
     env:
diff --git a/Cargo.toml b/Cargo.toml
index 486ff69d4..8fa098ae0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -176,6 +176,9 @@ unused_import_braces = 'warn'
 unused-lifetimes = 'warn'
 unused-macro-rules = 'warn'
 
+# Don't warn about unknown cfg condition in `#[cfg(pulley_tail_calls)]`
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(pulley_tail_calls)'] }
+
 [workspace.lints.clippy]
 # The default set of lints in Clippy is viewed as "too noisy" right now so
 # they're all turned off by default. Selective lints are then enabled below as
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 953e37fda..db0a4225e 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -2020,36 +2020,19 @@ impl<'module_environment> crate::translate::FuncEnvironment
     fn translate_array_copy(
         &mut self,
         builder: &mut FunctionBuilder,
-        dst_array_type_index: TypeIndex,
+        _dst_array_type_index: TypeIndex,
         dst_array: ir::Value,
         dst_index: ir::Value,
-        src_array_type_index: TypeIndex,
+        _src_array_type_index: TypeIndex,
         src_array: ir::Value,
         src_index: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
         let libcall = gc::builtins::array_copy(self, builder.func)?;
         let vmctx = self.vmctx_val(&mut builder.cursor());
-        let dst_array_type_index = self.module.types[dst_array_type_index];
-        let dst_array_type_index = builder
-            .ins()
-            .iconst(I32, i64::from(dst_array_type_index.as_u32()));
-        let src_array_type_index = self.module.types[src_array_type_index];
-        let src_array_type_index = builder
-            .ins()
-            .iconst(I32, i64::from(src_array_type_index.as_u32()));
         builder.ins().call(
             libcall,
-            &[
-                vmctx,
-                dst_array_type_index,
-                dst_array,
-                dst_index,
-                src_array_type_index,
-                src_array,
-                src_index,
-                len,
-            ],
+            &[vmctx, dst_array, dst_index, src_array, src_index, len],
         );
         Ok(())
     }
diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs
index 2e9d8489d..c332edb22 100644
--- a/crates/cranelift/src/translate/code_translator.rs
+++ b/crates/cranelift/src/translate/code_translator.rs
@@ -2723,9 +2723,14 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
             let (array, index, elem) = state.pop3();
             environ.translate_array_set(builder, array_type_index, array, index, elem)?;
         }
+        Operator::RefEq => {
+            let (r1, r2) = state.pop2();
+            let eq = builder.ins().icmp(ir::condcodes::IntCC::Equal, r1, r2);
+            let eq = builder.ins().uextend(ir::types::I32, eq);
+            state.push1(eq);
+        }
 
-        Operator::RefEq
-        | Operator::RefTestNonNull { .. }
+        Operator::RefTestNonNull { .. }
         | Operator::RefTestNullable { .. }
         | Operator::RefCastNonNull { .. }
         | Operator::RefCastNullable { .. }
@@ -2733,7 +2738,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
         | Operator::BrOnCastFail { .. }
         | Operator::AnyConvertExtern
         | Operator::ExternConvertAny => {
-            return Err(wasm_unsupported!("GC operators are not yet implemented"));
+            return Err(wasm_unsupported!("GC operator not yet implemented: {op:?}"));
         }
 
         Operator::GlobalAtomicGet { .. }
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index e904cb8c8..65e035135 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -148,10 +148,8 @@ macro_rules! foreach_builtin_function {
             #[cfg(feature = "gc")]
             array_copy(
                 vmctx: vmctx,
-                dst_array_interned_type_index: i32,
                 dst_array: reference,
                 dst_index: i32,
-                src_array_interned_type_index: i32,
                 src_array: reference,
                 src_index: i32,
                 len: i32
diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
index 0dc4d4cec..22661caea 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
@@ -568,7 +568,7 @@ impl ArrayRef {
         Ok(gc_ref.as_arrayref_unchecked())
     }
 
-    fn layout(&self, store: &AutoAssertNoGc<'_>) -> Result<GcArrayLayout> {
+    pub(crate) fn layout(&self, store: &AutoAssertNoGc<'_>) -> Result<GcArrayLayout> {
         assert!(self.comes_from_same_store(&store));
         let type_index = self.type_index(store)?;
         let layout = store
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index 6f6fec630..436ea220d 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -905,6 +905,14 @@ impl<T: GcRef> Rooted<T> {
         b: &impl RootedGcRef<T>,
     ) -> Result<bool> {
         let store = store.as_context().0;
+        Self::_ref_eq(store, a, b)
+    }
+
+    pub(crate) fn _ref_eq(
+        store: &StoreOpaque,
+        a: &impl RootedGcRef<T>,
+        b: &impl RootedGcRef<T>,
+    ) -> Result<bool> {
         let a = a.try_gc_ref(store)?;
         let b = b.try_gc_ref(store)?;
         Ok(a == b)
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 4b7a00330..38eef67ca 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1614,6 +1614,8 @@ impl StoreOpaque {
             return;
         }
 
+        log::trace!("============ Begin GC ===========");
+
         // Take the GC roots out of `self` so we can borrow it mutably but still
         // call mutable methods on `self`.
         let mut roots = core::mem::take(&mut self.gc_roots_list);
@@ -1624,6 +1626,8 @@ impl StoreOpaque {
         // Restore the GC roots for the next GC.
         roots.clear();
         self.gc_roots_list = roots;
+
+        log::trace!("============ End GC ===========");
     }
 
     #[inline]
@@ -1662,6 +1666,8 @@ impl StoreOpaque {
             return;
         }
 
+        log::trace!("============ Begin Async GC ===========");
+
         // Take the GC roots out of `self` so we can borrow it mutably but still
         // call mutable methods on `self`.
         let mut roots = std::mem::take(&mut self.gc_roots_list);
@@ -1674,6 +1680,8 @@ impl StoreOpaque {
         // Restore the GC roots for the next GC.
         roots.clear();
         self.gc_roots_list = roots;
+
+        log::trace!("============ End Async GC ===========");
     }
 
     #[inline]
diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs
index a1bf7f0a0..0ffab2720 100644
--- a/crates/wasmtime/src/runtime/vm/const_expr.rs
+++ b/crates/wasmtime/src/runtime/vm/const_expr.rs
@@ -38,11 +38,7 @@ impl<'a> ConstEvalContext<'a> {
                 .defined_or_imported_global_ptr(index)
                 .as_ref()
                 .unwrap();
-            let mut gc_store = store.unwrap_gc_store_mut();
-            Ok(global.to_val_raw(
-                &mut gc_store,
-                self.instance.env_module().globals[index].wasm_ty,
-            ))
+            global.to_val_raw(store, self.instance.env_module().globals[index].wasm_ty)
         }
     }
 
@@ -169,6 +165,8 @@ impl ConstExprEvaluator {
         context: &mut ConstEvalContext<'_>,
         expr: &ConstExpr,
     ) -> Result<ValRaw> {
+        log::trace!("evaluating const expr: {:?}", expr);
+
         self.stack.clear();
 
         let mut store = (*context.instance.store()).store_opaque_mut();
@@ -372,6 +370,7 @@ impl ConstExprEvaluator {
         }
 
         if self.stack.len() == 1 {
+            log::trace!("const expr evaluated to {:?}", self.stack[0]);
             Ok(self.stack[0])
         } else {
             bail!(
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index 7500a68fc..b2f020c85 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -23,6 +23,7 @@ pub use i31::*;
 use crate::prelude::*;
 use crate::runtime::vm::GcHeapAllocationIndex;
 use core::alloc::Layout;
+use core::mem::MaybeUninit;
 use core::ptr;
 use core::{any::Any, num::NonZeroUsize};
 use wasmtime_environ::{GcArrayLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
@@ -96,6 +97,19 @@ impl GcStore {
         }
     }
 
+    /// Write the `source` GC reference into the uninitialized `destination`
+    /// slot, performing write barriers as necessary.
+    pub fn init_gc_ref(
+        &mut self,
+        destination: &mut MaybeUninit<Option<VMGcRef>>,
+        source: Option<&VMGcRef>,
+    ) {
+        // Initialize the destination to `None`, at which point the regular GC
+        // write barrier is safe to reuse.
+        let destination = destination.write(None);
+        self.write_gc_ref(destination, source);
+    }
+
     /// Write the `source` GC reference into the `destination` slot, performing
     /// write barriers as necessary.
     pub fn write_gc_ref(&mut self, destination: &mut Option<VMGcRef>, source: Option<&VMGcRef>) {
@@ -205,6 +219,18 @@ impl GcStore {
         self.gc_heap.gc_object_data(gc_ref)
     }
 
+    /// Get the object datas for the given pair of object references.
+    ///
+    /// Panics if `a` and `b` are the same reference or either is out of bounds.
+    pub fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        assert_ne!(a, b);
+        self.gc_heap.gc_object_data_pair(a, b)
+    }
+
     /// Allocate an uninitialized array with the given type index.
     ///
     /// This does NOT check that the index is currently allocated in the types
@@ -299,6 +325,13 @@ unsafe impl GcHeap for DisabledGcHeap {
     fn gc_object_data(&mut self, _gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_> {
         unreachable!()
     }
+    fn gc_object_data_pair(
+        &mut self,
+        _a: &VMGcRef,
+        _b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        unreachable!()
+    }
     fn alloc_uninit_array(
         &mut self,
         _ty: VMSharedTypeIndex,
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
index 2c74ccbfd..19328387e 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
@@ -133,6 +133,28 @@ impl<'a> VMGcObjectDataMut<'a> {
         val.write_le(into.try_into().unwrap());
     }
 
+    /// Get a slice of this object's data.
+    ///
+    /// Panics on out-of-bounds accesses.
+    #[inline]
+    pub fn slice(&self, offset: u32, len: u32) -> &[u8] {
+        let start = usize::try_from(offset).unwrap();
+        let len = usize::try_from(len).unwrap();
+        let end = start.checked_add(len).unwrap();
+        self.data.get(start..end).expect("out of bounds slice")
+    }
+
+    /// Get a mutable slice of this object's data.
+    ///
+    /// Panics on out-of-bounds accesses.
+    #[inline]
+    pub fn slice_mut(&mut self, offset: u32, len: u32) -> &mut [u8] {
+        let start = usize::try_from(offset).unwrap();
+        let len = usize::try_from(len).unwrap();
+        let end = start.checked_add(len).unwrap();
+        self.data.get_mut(start..end).expect("out of bounds slice")
+    }
+
     /// Copy the given slice into this object's data at the given offset.
     ///
     /// Panics on out-of-bounds accesses.
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index ebf5e9b46..02c65b68b 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -49,7 +49,7 @@ use crate::runtime::vm::{
     ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
     GcProgress, GcRootsIter, GcRuntime, Mmap, TypedGcRef, VMExternRef, VMGcHeader, VMGcRef,
 };
-use core::ops::{Deref, DerefMut};
+use core::ops::{Deref, DerefMut, Range};
 use core::{
     alloc::Layout,
     any::Any,
@@ -132,6 +132,16 @@ impl DrcHeap {
             .dealloc(gc_ref.as_heap_index().unwrap(), layout);
     }
 
+    fn object_range(&self, gc_ref: &VMGcRef) -> Range<usize> {
+        let start = gc_ref.as_heap_index().unwrap().get();
+        let start = usize::try_from(start).unwrap();
+        let size = self
+            .index::<VMDrcHeader>(gc_ref.as_typed_unchecked())
+            .object_size();
+        let end = start.checked_add(size).unwrap();
+        start..end
+    }
+
     /// Index into this heap and get a shared reference to the `T` that `gc_ref`
     /// points to.
     ///
@@ -617,16 +627,42 @@ unsafe impl GcHeap for DrcHeap {
     }
 
     fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_> {
-        let start = gc_ref.as_heap_index().unwrap().get();
-        let start = usize::try_from(start).unwrap();
-        let size = self
-            .index::<VMDrcHeader>(gc_ref.as_typed_unchecked())
-            .object_size();
-        let end = start + size;
-        let data = &mut self.heap_slice_mut()[start..end];
+        let range = self.object_range(gc_ref);
+        let data = &mut self.heap_slice_mut()[range];
         VMGcObjectDataMut::new(data)
     }
 
+    fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        assert_ne!(a, b);
+
+        let a_range = self.object_range(a);
+        let b_range = self.object_range(b);
+
+        // Assert that the two objects do not overlap.
+        assert!(a_range.start <= a_range.end);
+        assert!(b_range.start <= b_range.end);
+        assert!(a_range.end <= b_range.start || b_range.end <= a_range.start);
+
+        let (a_data, b_data) = if a_range.start < b_range.start {
+            let (a_half, b_half) = self.heap_slice_mut().split_at_mut(b_range.start);
+            let b_len = b_range.end - b_range.start;
+            (&mut a_half[a_range], &mut b_half[..b_len])
+        } else {
+            let (b_half, a_half) = self.heap_slice_mut().split_at_mut(a_range.start);
+            let a_len = a_range.end - a_range.start;
+            (&mut a_half[..a_len], &mut b_half[b_range])
+        };
+
+        (
+            VMGcObjectDataMut::new(a_data),
+            VMGcObjectDataMut::new(b_data),
+        )
+    }
+
     fn alloc_uninit_array(
         &mut self,
         ty: VMSharedTypeIndex,
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
index fa89efb81..539ff5b95 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
@@ -285,15 +285,15 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
     /// Panics on out-of-bounds accesses.
     fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_>;
 
-    /// Allocate a GC-managed array of the given type and length.
-    ///
-    /// The array's elements are left uninitialized. It is the caller's
-    /// responsibility to initialize them before exposing the array to Wasm or
-    /// triggering a GC. Failure to do this is memory safe, but may result in
-    /// general failures such as panics or incorrect results.
-    ///
-    /// Return values:
+    /// Get a pair of mutable borrows of the given objects' data.
     ///
+    /// Panics if `a == b` or on out-of-bounds accesses.
+    fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>);
+
     /// * `Ok(Some(_))`: The allocation was successful.
     ///
     /// * `Ok(None)`: There is currently no available space for this
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index b1e878c59..d707bbc09 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -1318,34 +1318,12 @@ impl Instance {
             ptr = ptr.add(1);
         }
 
-        // Initialize the defined globals
-        let mut const_evaluator = ConstExprEvaluator::default();
-        self.initialize_vmctx_globals(&mut const_evaluator, module);
-    }
-
-    unsafe fn initialize_vmctx_globals(
-        &mut self,
-        const_evaluator: &mut ConstExprEvaluator,
-        module: &Module,
-    ) {
-        for (index, init) in module.global_initializers.iter() {
-            let mut context = ConstEvalContext::new(self);
-            let raw = const_evaluator
-                .eval(&mut context, init)
-                .expect("should be a valid const expr");
-
-            let to = self.global_ptr(index);
-            let wasm_ty = module.globals[module.global_index(index)].wasm_ty;
-
-            #[cfg(feature = "wmemcheck")]
-            if index.index() == 0 && wasm_ty == wasmtime_environ::WasmValType::I32 {
-                if let Some(wmemcheck) = &mut self.wmemcheck_state {
-                    let size = usize::try_from(raw.get_i32()).unwrap();
-                    wmemcheck.set_stack_size(size);
-                }
-            }
-
-            ptr::write(to, VMGlobalDefinition::from_val_raw(wasm_ty, raw));
+        // Zero-initialize the globals so that nothing is uninitialized memory
+        // after this function returns. The globals are actually initialized
+        // with their const expression initializers after the instance is fully
+        // allocated.
+        for (index, _init) in module.global_initializers.iter() {
+            ptr::write(self.global_ptr(index), VMGlobalDefinition::new());
         }
     }
 
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
index f93592378..1fe374517 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
@@ -6,6 +6,8 @@ use crate::runtime::vm::memory::Memory;
 use crate::runtime::vm::mpk::ProtectionKey;
 use crate::runtime::vm::table::Table;
 use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo, VMFuncRef, VMGcRef, VMStore};
+use crate::store::AutoAssertNoGc;
+use crate::vm::VMGlobalDefinition;
 use core::{any::Any, mem, ptr};
 use wasmtime_environ::{
     DefinedMemoryIndex, DefinedTableIndex, HostPtr, InitMemory, MemoryInitialization,
@@ -555,27 +557,28 @@ fn check_table_init_bounds(instance: &mut Instance, module: &Module) -> Result<(
     Ok(())
 }
 
-fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
-    let mut const_evaluator = ConstExprEvaluator::default();
-
+fn initialize_tables(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
     for (table, init) in module.table_initialization.initial_values.iter() {
         match init {
             // Tables are always initially null-initialized at this time
             TableInitialValue::Null { precomputed: _ } => {}
 
             TableInitialValue::Expr(expr) => {
-                let mut context = ConstEvalContext::new(instance);
                 let raw = unsafe {
                     const_evaluator
-                        .eval(&mut context, expr)
+                        .eval(context, expr)
                         .expect("const expression should be valid")
                 };
                 let idx = module.table_index(table);
-                let table = unsafe { instance.get_defined_table(table).as_mut().unwrap() };
+                let table = unsafe { context.instance.get_defined_table(table).as_mut().unwrap() };
                 match module.table_plans[idx].table.ref_type.heap_type.top() {
                     WasmHeapTopType::Extern => {
                         let gc_ref = VMGcRef::from_raw_u32(raw.get_externref());
-                        let gc_store = unsafe { (*instance.store()).unwrap_gc_store_mut() };
+                        let gc_store = unsafe { (*context.instance.store()).gc_store_mut()? };
                         let items = (0..table.size())
                             .map(|_| gc_ref.as_ref().map(|r| gc_store.clone_gc_ref(r)));
                         table.init_gc_refs(0, items).err2anyhow()?;
@@ -583,7 +586,7 @@ fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
 
                     WasmHeapTopType::Any => {
                         let gc_ref = VMGcRef::from_raw_u32(raw.get_anyref());
-                        let gc_store = unsafe { (*instance.store()).unwrap_gc_store_mut() };
+                        let gc_store = unsafe { (*context.instance.store()).gc_store_mut()? };
                         let items = (0..table.size())
                             .map(|_| gc_ref.as_ref().map(|r| gc_store.clone_gc_ref(r)));
                         table.init_gc_refs(0, items).err2anyhow()?;
@@ -607,15 +610,15 @@ fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
     // iterates over all segments (Segments mode) or leftover
     // segments (FuncTable mode) to initialize.
     for segment in module.table_initialization.segments.iter() {
-        let mut context = ConstEvalContext::new(instance);
         let start = unsafe {
             const_evaluator
-                .eval(&mut context, &segment.offset)
+                .eval(context, &segment.offset)
                 .expect("const expression should be valid")
         };
-        instance
+        context
+            .instance
             .table_init_segment(
-                &mut const_evaluator,
+                const_evaluator,
                 segment.table_index,
                 &segment.elements,
                 start.get_u64(),
@@ -666,7 +669,11 @@ fn check_memory_init_bounds(
     Ok(())
 }
 
-fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
+fn initialize_memories(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
     // Delegates to the `init_memory` method which is sort of a duplicate of
     // `instance.memory_init_segment` but is used at compile-time in other
     // contexts so is shared here to have only one method of memory
@@ -676,18 +683,18 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
     // so errors only happen if an out-of-bounds segment is found, in which case
     // a trap is returned.
 
-    struct InitMemoryAtInstantiation<'a> {
-        instance: &'a mut Instance,
+    struct InitMemoryAtInstantiation<'a, 'b> {
         module: &'a Module,
-        const_evaluator: ConstExprEvaluator,
+        context: &'a mut ConstEvalContext<'b>,
+        const_evaluator: &'a mut ConstExprEvaluator,
     }
 
-    impl InitMemory for InitMemoryAtInstantiation<'_> {
+    impl InitMemory for InitMemoryAtInstantiation<'_, '_> {
         fn memory_size_in_bytes(
             &mut self,
             memory: wasmtime_environ::MemoryIndex,
         ) -> Result<u64, SizeOverflow> {
-            let len = self.instance.get_memory(memory).current_length();
+            let len = self.context.instance.get_memory(memory).current_length();
             let len = u64::try_from(len).unwrap();
             Ok(len)
         }
@@ -697,11 +704,10 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
             memory: wasmtime_environ::MemoryIndex,
             expr: &wasmtime_environ::ConstExpr,
         ) -> Option<u64> {
-            let mut context = ConstEvalContext::new(self.instance);
-            let val = unsafe { self.const_evaluator.eval(&mut context, expr) }
+            let val = unsafe { self.const_evaluator.eval(self.context, expr) }
                 .expect("const expression should be valid");
             Some(
-                match self.instance.env_module().memory_plans[memory]
+                match self.context.instance.env_module().memory_plans[memory]
                     .memory
                     .idx_type
                 {
@@ -721,14 +727,14 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
             // pre-initializing it via mmap magic, then this initializer can be
             // skipped entirely.
             if let Some(memory_index) = self.module.defined_memory_index(memory_index) {
-                if !self.instance.memories[memory_index].1.needs_init() {
+                if !self.context.instance.memories[memory_index].1.needs_init() {
                     return true;
                 }
             }
-            let memory = self.instance.get_memory(memory_index);
+            let memory = self.context.instance.get_memory(memory_index);
 
             unsafe {
-                let src = self.instance.wasm_data(init.data.clone());
+                let src = self.context.instance.wasm_data(init.data.clone());
                 let offset = usize::try_from(init.offset).unwrap();
                 let dst = memory.base.add(offset);
 
@@ -746,9 +752,9 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
     let ok = module
         .memory_initialization
         .init_memory(&mut InitMemoryAtInstantiation {
-            instance,
             module,
-            const_evaluator: ConstExprEvaluator::default(),
+            context,
+            const_evaluator,
         });
     if !ok {
         return Err(Trap::MemoryOutOfBounds).err2anyhow();
@@ -771,6 +777,46 @@ fn check_init_bounds(instance: &mut Instance, module: &Module) -> Result<()> {
     Ok(())
 }
 
+fn initialize_globals(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
+    assert!(core::ptr::eq(&**context.instance.env_module(), module));
+
+    for (index, init) in module.global_initializers.iter() {
+        let raw = unsafe {
+            const_evaluator
+                .eval(context, init)
+                .expect("should be a valid const expr")
+        };
+
+        let to = context.instance.global_ptr(index);
+        let wasm_ty = module.globals[module.global_index(index)].wasm_ty;
+
+        #[cfg(feature = "wmemcheck")]
+        if index.as_bits() == 0 && wasm_ty == wasmtime_environ::WasmValType::I32 {
+            if let Some(wmemcheck) = &mut context.instance.wmemcheck_state {
+                let size = usize::try_from(raw.get_i32()).unwrap();
+                wmemcheck.set_stack_size(size);
+            }
+        }
+
+        let store = unsafe { (*context.instance.store()).store_opaque_mut() };
+        let mut store = AutoAssertNoGc::new(store);
+
+        // This write is safe because we know we have the correct module for
+        // this instance and its vmctx due to the assert above.
+        unsafe {
+            ptr::write(
+                to,
+                VMGlobalDefinition::from_val_raw(&mut store, wasm_ty, raw)?,
+            )
+        };
+    }
+    Ok(())
+}
+
 pub(super) fn initialize_instance(
     instance: &mut Instance,
     module: &Module,
@@ -784,11 +830,12 @@ pub(super) fn initialize_instance(
         check_init_bounds(instance, module)?;
     }
 
-    // Initialize the tables
-    initialize_tables(instance, module)?;
+    let mut context = ConstEvalContext::new(instance);
+    let mut const_evaluator = ConstExprEvaluator::default();
 
-    // Initialize the memories
-    initialize_memories(instance, &module)?;
+    initialize_globals(&mut context, &mut const_evaluator, module)?;
+    initialize_tables(&mut context, &mut const_evaluator, module)?;
+    initialize_memories(&mut context, &mut const_evaluator, &module)?;
 
     Ok(())
 }
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 01519dd5e..15189d37e 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -840,6 +840,7 @@ unsafe fn array_init_elem(
         // Convert the raw GC ref into a `Rooted<ArrayRef>`.
         let array =
             VMGcRef::from_raw_u32(array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+        let array = (*instance.store()).unwrap_gc_store_mut().clone_gc_ref(&array);
         let array = {
             let mut no_gc = AutoAssertNoGc::new((*instance.store()).store_opaque_mut());
             ArrayRef::from_cloned_gc_ref(&mut no_gc, array)
@@ -918,18 +919,79 @@ unsafe fn array_init_elem(
     })
 }
 
+// TODO: Specialize this libcall for only non-GC array elements, so we never
+// have to do GC barriers and their associated indirect calls through the `dyn
+// GcHeap`. Instead, implement those copies inline in Wasm code. Then, use bulk
+// `memcpy`-style APIs to do the actual copies here.
 #[cfg(feature = "gc")]
 unsafe fn array_copy(
-    _instance: &mut Instance,
-    _dst_array_type_index: u32,
-    _dst_array: u32,
-    _dst_index: u32,
-    _src_array_type_index: u32,
-    _src_array: u32,
-    _src_index: u32,
-    _len: u32,
+    instance: &mut Instance,
+    dst_array: u32,
+    dst: u32,
+    src_array: u32,
+    src: u32,
+    len: u32,
 ) -> Result<()> {
-    bail!("the `array.copy` instruction is not yet implemented")
+    use crate::{store::AutoAssertNoGc, ArrayRef, OpaqueRootScope};
+
+    log::trace!(
+            "array.copy(dst_array={dst_array:#x}, dst_index={dst}, src_array={src_array:#x}, src_index={src}, len={len})",
+        );
+
+    let store = (*instance.store()).store_opaque_mut();
+    let mut store = OpaqueRootScope::new(store);
+    let mut store = AutoAssertNoGc::new(&mut store);
+
+    // Convert the raw GC refs into `Rooted<ArrayRef>`s.
+    let dst_array =
+        VMGcRef::from_raw_u32(dst_array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+    let dst_array = store.unwrap_gc_store_mut().clone_gc_ref(&dst_array);
+    let dst_array = ArrayRef::from_cloned_gc_ref(&mut store, dst_array);
+    let src_array =
+        VMGcRef::from_raw_u32(src_array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+    let src_array = store.unwrap_gc_store_mut().clone_gc_ref(&src_array);
+    let src_array = ArrayRef::from_cloned_gc_ref(&mut store, src_array);
+
+    // Bounds check the destination array's elements.
+    let dst_array_len = dst_array._len(&store)?;
+    if dst
+        .checked_add(len)
+        .ok_or_else(|| Trap::ArrayOutOfBounds.into_anyhow())?
+        > dst_array_len
+    {
+        return Err(Trap::ArrayOutOfBounds.into_anyhow());
+    }
+
+    // Bounds check the source array's elements.
+    let src_array_len = src_array._len(&store)?;
+    if src
+        .checked_add(len)
+        .ok_or_else(|| Trap::ArrayOutOfBounds.into_anyhow())?
+        > src_array_len
+    {
+        return Err(Trap::ArrayOutOfBounds.into_anyhow());
+    }
+
+    let mut store = AutoAssertNoGc::new(&mut store);
+    // If `src_array` and `dst_array` are the same array, then we are
+    // potentially doing an overlapping copy, so make sure to copy elements in
+    // the order that doesn't clobber the source elements before they are
+    // copied. If they are different arrays, the order doesn't matter, but we
+    // simply don't bother checking.
+    if src > dst {
+        for i in 0..len {
+            let src_elem = src_array._get(&mut store, src + i)?;
+            let dst_i = dst + i;
+            dst_array._set(&mut store, dst_i, src_elem)?;
+        }
+    } else {
+        for i in (0..len).rev() {
+            let src_elem = src_array._get(&mut store, src + i)?;
+            let dst_i = dst + i;
+            dst_array._set(&mut store, dst_i, src_elem)?;
+        }
+    }
+    Ok(())
 }
 
 // Implementation of `memory.atomic.notify` for locally defined memories.
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 26e72a84f..59ae7ba0f 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -4,12 +4,14 @@
 mod vm_host_func_context;
 
 pub use self::vm_host_func_context::VMArrayCallHostFuncContext;
+use crate::prelude::*;
 use crate::runtime::vm::{GcStore, VMGcRef};
+use crate::store::StoreOpaque;
 use core::cell::UnsafeCell;
 use core::ffi::c_void;
 use core::fmt;
 use core::marker;
-use core::mem;
+use core::mem::{self, MaybeUninit};
 use core::ptr::{self, NonNull};
 use core::sync::atomic::{AtomicUsize, Ordering};
 use sptr::Strict;
@@ -423,7 +425,11 @@ impl VMGlobalDefinition {
     /// # Unsafety
     ///
     /// This raw value's type must match the given `WasmValType`.
-    pub unsafe fn from_val_raw(wasm_ty: WasmValType, raw: ValRaw) -> Self {
+    pub unsafe fn from_val_raw(
+        store: &mut StoreOpaque,
+        wasm_ty: WasmValType,
+        raw: ValRaw,
+    ) -> Result<Self> {
         let mut global = Self::new();
         match wasm_ty {
             WasmValType::I32 => *global.as_i32_mut() = raw.get_i32(),
@@ -433,13 +439,17 @@ impl VMGlobalDefinition {
             WasmValType::V128 => *global.as_u128_mut() = raw.get_v128(),
             WasmValType::Ref(r) => match r.heap_type.top() {
                 WasmHeapTopType::Extern => {
-                    global.init_gc_ref(VMGcRef::from_raw_u32(raw.get_externref()))
+                    let r = VMGcRef::from_raw_u32(raw.get_externref());
+                    global.init_gc_ref(store.gc_store_mut()?, r.as_ref())
+                }
+                WasmHeapTopType::Any => {
+                    let r = VMGcRef::from_raw_u32(raw.get_anyref());
+                    global.init_gc_ref(store.gc_store_mut()?, r.as_ref())
                 }
-                WasmHeapTopType::Any => global.init_gc_ref(VMGcRef::from_raw_u32(raw.get_anyref())),
                 WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(),
             },
         }
-        global
+        Ok(global)
     }
 
     /// Get this global's value as a `ValRaw`.
@@ -447,25 +457,31 @@ impl VMGlobalDefinition {
     /// # Unsafety
     ///
     /// This global's value's type must match the given `WasmValType`.
-    pub unsafe fn to_val_raw(&self, gc_store: &mut GcStore, wasm_ty: WasmValType) -> ValRaw {
-        match wasm_ty {
+    pub unsafe fn to_val_raw(
+        &self,
+        store: &mut StoreOpaque,
+        wasm_ty: WasmValType,
+    ) -> Result<ValRaw> {
+        Ok(match wasm_ty {
             WasmValType::I32 => ValRaw::i32(*self.as_i32()),
             WasmValType::I64 => ValRaw::i64(*self.as_i64()),
             WasmValType::F32 => ValRaw::f32(*self.as_f32_bits()),
             WasmValType::F64 => ValRaw::f64(*self.as_f64_bits()),
             WasmValType::V128 => ValRaw::v128(*self.as_u128()),
             WasmValType::Ref(r) => match r.heap_type.top() {
-                WasmHeapTopType::Extern => ValRaw::externref(
-                    self.as_gc_ref()
-                        .map_or(0, |r| gc_store.clone_gc_ref(r).as_raw_u32()),
-                ),
-                WasmHeapTopType::Any => ValRaw::anyref(
-                    self.as_gc_ref()
-                        .map_or(0, |r| gc_store.clone_gc_ref(r).as_raw_u32()),
-                ),
+                WasmHeapTopType::Extern => ValRaw::externref(match self.as_gc_ref() {
+                    Some(r) => store.gc_store_mut()?.clone_gc_ref(r).as_raw_u32(),
+                    None => 0,
+                }),
+                WasmHeapTopType::Any => ValRaw::anyref({
+                    match self.as_gc_ref() {
+                        Some(r) => store.gc_store_mut()?.clone_gc_ref(r).as_raw_u32(),
+                        None => 0,
+                    }
+                }),
                 WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()),
             },
-        }
+        })
     }
 
     /// Return a reference to the value as an i32.
@@ -577,10 +593,16 @@ impl VMGlobalDefinition {
     }
 
     /// Initialize a global to the given GC reference.
-    pub unsafe fn init_gc_ref(&mut self, gc_ref: Option<VMGcRef>) {
+    pub unsafe fn init_gc_ref(&mut self, gc_store: &mut GcStore, gc_ref: Option<&VMGcRef>) {
         assert!(cfg!(feature = "gc") || gc_ref.is_none());
-        let raw_ptr = self.storage.as_mut().as_mut_ptr().cast::<Option<VMGcRef>>();
-        ptr::write(raw_ptr, gc_ref);
+
+        let dest = &mut *(self
+            .storage
+            .as_mut()
+            .as_mut_ptr()
+            .cast::<MaybeUninit<Option<VMGcRef>>>());
+
+        gc_store.init_gc_ref(dest, gc_ref)
     }
 
     /// Write a GC reference into this global value.
diff --git a/pulley/Cargo.toml b/pulley/Cargo.toml
index 9d95056fd..bc27619e7 100644
--- a/pulley/Cargo.toml
+++ b/pulley/Cargo.toml
@@ -27,7 +27,7 @@ arbitrary = ["dep:arbitrary", "arbitrary/derive", "std", "cranelift-bitset/arbit
 encode = []
 decode = []
 disas = ["decode"]
-interp = ["decode"]
+interp = ["decode", "encode"]
 
 [package.metadata.docs.rs]
 all-features = true
diff --git a/pulley/src/decode.rs b/pulley/src/decode.rs
index 1a0c8b202..bcd570172 100644
--- a/pulley/src/decode.rs
+++ b/pulley/src/decode.rs
@@ -260,7 +260,8 @@ impl BytecodeStream for UnsafeBytecodeStream {
 
 /// Anything that can be decoded from a bytecode stream, e.g. opcodes,
 /// immediates, registers, etc...
-trait Decode: Sized {
+pub trait Decode: Sized {
+    /// Decode this type from the given bytecode stream.
     fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
     where
         T: BytecodeStream;
@@ -377,6 +378,32 @@ impl Decode for PcRelOffset {
     }
 }
 
+impl Decode for Opcode {
+    fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
+    where
+        T: BytecodeStream,
+    {
+        let byte = u8::decode(bytecode)?;
+        match Opcode::new(byte) {
+            Some(v) => Ok(v),
+            None => Err(bytecode.invalid_opcode(byte)),
+        }
+    }
+}
+
+impl Decode for ExtendedOpcode {
+    fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
+    where
+        T: BytecodeStream,
+    {
+        let word = u16::decode(bytecode)?;
+        match ExtendedOpcode::new(word) {
+            Some(v) => Ok(v),
+            None => Err(bytecode.invalid_extended_opcode(word)),
+        }
+    }
+}
+
 impl<R: Reg> Decode for BinaryOperands<R> {
     fn decode<T>(bytecode: &mut T) -> Result<Self, T::Error>
     where
@@ -655,3 +682,53 @@ macro_rules! define_extended_decoder {
     };
 }
 for_each_extended_op!(define_extended_decoder);
+
+/// Unwrap a `Result<T, Uninhabited>`.
+/// Always succeeds, since `Uninhabited` is uninhabited.
+pub fn unwrap_uninhabited<T>(res: Result<T, Uninhabited>) -> T {
+    match res {
+        Ok(ok) => ok,
+
+        // Nightly rust warns that this pattern is unreachable, but stable rust
+        // doesn't.
+        #[allow(unreachable_patterns)]
+        Err(err) => match err {},
+    }
+}
+
+/// Functions for decoding the operands of an instruction, assuming the opcode
+/// has already been decoded.
+pub mod operands {
+    use super::*;
+
+    macro_rules! define_operands_decoder {
+        (
+            $(
+                $( #[$attr:meta] )*
+                    $snake_name:ident = $name:ident $( {
+                    $(
+                        $( #[$field_attr:meta] )*
+                        $field:ident : $field_ty:ty
+                    ),*
+                } )? ;
+            )*
+        ) => {
+            $(
+                #[allow(unused_variables)]
+                #[allow(missing_docs)]
+                pub fn $snake_name<T: BytecodeStream>(pc: &mut T) -> Result<($($($field_ty,)*)?), T::Error> {
+                    Ok((($($((<$field_ty>::decode(pc))?,)*)?)))
+                }
+            )*
+        };
+    }
+
+    for_each_op!(define_operands_decoder);
+
+    #[allow(missing_docs)]
+    pub fn extended<T: BytecodeStream>(pc: &mut T) -> Result<(ExtendedOpcode,), T::Error> {
+        Ok((ExtendedOpcode::decode(pc)?,))
+    }
+
+    for_each_extended_op!(define_operands_decoder);
+}
diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs
index 24b3e2a59..49e3a1bb3 100644
--- a/pulley/src/interp.rs
+++ b/pulley/src/interp.rs
@@ -4,19 +4,22 @@ use crate::decode::*;
 use crate::imms::*;
 use crate::regs::*;
 use crate::ExtendedOpcode;
+use crate::Opcode;
 use alloc::string::ToString;
 use alloc::{vec, vec::Vec};
 use core::fmt;
 use core::mem;
+use core::ops::ControlFlow;
 use core::ops::{Index, IndexMut};
 use core::ptr::{self, NonNull};
 use sptr::Strict;
 
+mod interp_loop;
+
 const DEFAULT_STACK_SIZE: usize = 1 << 20; // 1 MiB
 
 /// A virtual machine for interpreting Pulley bytecode.
 pub struct Vm {
-    decoder: Decoder,
     state: MachineState,
 }
 
@@ -35,7 +38,6 @@ impl Vm {
     /// Create a new virtual machine with the given stack.
     pub fn with_stack(stack: Vec<u8>) -> Self {
         Self {
-            decoder: Decoder::new(),
             state: MachineState::with_stack(stack),
         }
     }
@@ -120,29 +122,11 @@ impl Vm {
     }
 
     unsafe fn run(&mut self, pc: NonNull<u8>) -> Result<(), NonNull<u8>> {
-        let mut visitor = InterpreterVisitor {
-            state: &mut self.state,
-            pc: UnsafeBytecodeStream::new(pc),
-        };
-
-        loop {
-            let continuation = self.decoder.decode_one(&mut visitor).unwrap();
-
-            // Really wish we had `feature(explicit_tail_calls)`...
-            match continuation {
-                Continuation::Continue => {
-                    continue;
-                }
-
-                // Out-of-line slow paths marked `cold` and `inline(never)` to
-                // improve codegen.
-                Continuation::Trap => {
-                    let pc = visitor.pc.as_ptr();
-                    return self.trap(pc);
-                }
-                Continuation::ReturnToHost => return self.return_to_host(),
-                Continuation::HostCall => return self.host_call(),
-            }
+        let mut bytecode = UnsafeBytecodeStream::new(pc);
+        match interp_loop::interpreter_loop(self, &mut bytecode) {
+            Done::ReturnToHost => self.return_to_host(),
+            Done::Trap(pc) => self.trap(pc),
+            Done::HostCall => self.host_call(),
         }
     }
 
@@ -271,6 +255,14 @@ impl From<VRegVal> for Val {
 #[derive(Copy, Clone)]
 pub struct XRegVal(XRegUnion);
 
+impl PartialEq for XRegVal {
+    fn eq(&self, other: &Self) -> bool {
+        self.get_u64() == other.get_u64()
+    }
+}
+
+impl Eq for XRegVal {}
+
 impl fmt::Debug for XRegVal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("XRegVal")
@@ -304,6 +296,9 @@ impl Default for XRegVal {
 
 #[allow(missing_docs)]
 impl XRegVal {
+    /// Sentinel return address that signals the end of the call stack.
+    pub const HOST_RETURN_ADDR: Self = Self(XRegUnion { i64: -1 });
+
     pub fn new_i32(x: i32) -> Self {
         let mut val = XRegVal::default();
         val.set_i32(x);
@@ -566,8 +561,8 @@ impl MachineState {
         let sp = sp.as_mut_ptr();
         let sp = unsafe { sp.add(len) };
         state[XReg::sp] = XRegVal::new_ptr(sp);
-        state[XReg::fp] = XRegVal::new_i64(-1);
-        state[XReg::lr] = XRegVal::new_i64(-1);
+        state[XReg::fp] = XRegVal::HOST_RETURN_ADDR;
+        state[XReg::lr] = XRegVal::HOST_RETURN_ADDR;
 
         state
     }
@@ -588,588 +583,16 @@ impl MachineState {
     }
 }
 
-enum Continuation {
-    Continue,
+/// The reason the interpreter loop terminated.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum Done {
+    /// A `ret` instruction was executed and the call stack was empty. This is
+    /// how the loop normally ends.
     ReturnToHost,
-    Trap,
+
+    /// A `trap` instruction was executed at the given PC.
+    Trap(NonNull<u8>),
 
     #[allow(dead_code)]
     HostCall,
 }
-
-struct InterpreterVisitor<'a> {
-    state: &'a mut MachineState,
-    pc: UnsafeBytecodeStream,
-}
-
-impl InterpreterVisitor<'_> {
-    #[inline(always)]
-    fn pc_rel_jump(&mut self, offset: PcRelOffset, inst_size: isize) -> Continuation {
-        let offset = isize::try_from(i32::from(offset)).unwrap();
-        self.pc = unsafe { self.pc.offset(offset - inst_size) };
-        Continuation::Continue
-    }
-}
-
-#[doc(hidden)]
-impl OpVisitor for InterpreterVisitor<'_> {
-    type BytecodeStream = UnsafeBytecodeStream;
-
-    fn bytecode(&mut self) -> &mut Self::BytecodeStream {
-        &mut self.pc
-    }
-
-    type Return = Continuation;
-
-    fn ret(&mut self) -> Self::Return {
-        if self.state[XReg::lr].get_u64() == u64::MAX {
-            Continuation::ReturnToHost
-        } else {
-            let return_addr = self.state[XReg::lr].get_ptr();
-            self.pc = unsafe { UnsafeBytecodeStream::new(NonNull::new_unchecked(return_addr)) };
-            // log::trace!("returning to {return_addr:#p}");
-            Continuation::Continue
-        }
-    }
-
-    fn call(&mut self, offset: PcRelOffset) -> Self::Return {
-        let return_addr = self.pc.as_ptr();
-        self.state[XReg::lr].set_ptr(return_addr.as_ptr());
-        self.pc_rel_jump(offset, 5)
-    }
-
-    fn jump(&mut self, offset: PcRelOffset) -> Self::Return {
-        self.pc_rel_jump(offset, 5)
-    }
-
-    fn br_if(&mut self, cond: XReg, offset: PcRelOffset) -> Self::Return {
-        let cond = self.state[cond].get_u64();
-        if cond != 0 {
-            self.pc_rel_jump(offset, 6)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_not(&mut self, cond: XReg, offset: PcRelOffset) -> Self::Return {
-        let cond = self.state[cond].get_u64();
-        if cond == 0 {
-            self.pc_rel_jump(offset, 6)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xeq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a == b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xneq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a != b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslt32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i32();
-        let b = self.state[b].get_i32();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslteq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i32();
-        let b = self.state[b].get_i32();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xult32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xulteq32(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u32();
-        let b = self.state[b].get_u32();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xeq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a == b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xneq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a != b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslt64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i64();
-        let b = self.state[b].get_i64();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xslteq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_i64();
-        let b = self.state[b].get_i64();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xult64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a < b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn br_if_xulteq64(&mut self, a: XReg, b: XReg, offset: PcRelOffset) -> Self::Return {
-        let a = self.state[a].get_u64();
-        let b = self.state[b].get_u64();
-        if a <= b {
-            self.pc_rel_jump(offset, 7)
-        } else {
-            Continuation::Continue
-        }
-    }
-
-    fn xmov(&mut self, dst: XReg, src: XReg) -> Self::Return {
-        let val = self.state[src];
-        self.state[dst] = val;
-        Continuation::Continue
-    }
-
-    fn fmov(&mut self, dst: FReg, src: FReg) -> Self::Return {
-        let val = self.state[src];
-        self.state[dst] = val;
-        Continuation::Continue
-    }
-
-    fn vmov(&mut self, dst: VReg, src: VReg) -> Self::Return {
-        let val = self.state[src];
-        self.state[dst] = val;
-        Continuation::Continue
-    }
-
-    fn xconst8(&mut self, dst: XReg, imm: i8) -> Self::Return {
-        self.state[dst].set_i64(i64::from(imm));
-        Continuation::Continue
-    }
-
-    fn xconst16(&mut self, dst: XReg, imm: i16) -> Self::Return {
-        self.state[dst].set_i64(i64::from(imm));
-        Continuation::Continue
-    }
-
-    fn xconst32(&mut self, dst: XReg, imm: i32) -> Self::Return {
-        self.state[dst].set_i64(i64::from(imm));
-        Continuation::Continue
-    }
-
-    fn xconst64(&mut self, dst: XReg, imm: i64) -> Self::Return {
-        self.state[dst].set_i64(imm);
-        Continuation::Continue
-    }
-
-    fn xadd32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u32(a.wrapping_add(b));
-        Continuation::Continue
-    }
-
-    fn xadd64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(a.wrapping_add(b));
-        Continuation::Continue
-    }
-
-    fn xeq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a == b));
-        Continuation::Continue
-    }
-
-    fn xneq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a != b));
-        Continuation::Continue
-    }
-
-    fn xslt64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i64();
-        let b = self.state[operands.src2].get_i64();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xslteq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i64();
-        let b = self.state[operands.src2].get_i64();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn xult64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xulteq64(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u64();
-        let b = self.state[operands.src2].get_u64();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn xeq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a == b));
-        Continuation::Continue
-    }
-
-    fn xneq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a != b));
-        Continuation::Continue
-    }
-
-    fn xslt32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i32();
-        let b = self.state[operands.src2].get_i32();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xslteq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_i32();
-        let b = self.state[operands.src2].get_i32();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn xult32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a < b));
-        Continuation::Continue
-    }
-
-    fn xulteq32(&mut self, operands: BinaryOperands<XReg>) -> Self::Return {
-        let a = self.state[operands.src1].get_u32();
-        let b = self.state[operands.src2].get_u32();
-        self.state[operands.dst].set_u64(u64::from(a <= b));
-        Continuation::Continue
-    }
-
-    fn load32_u(&mut self, dst: XReg, ptr: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u32>();
-        let val = unsafe { ptr::read_unaligned(ptr) };
-        self.state[dst].set_u64(u64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_s(&mut self, dst: XReg, ptr: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<i32>();
-        let val = unsafe { ptr::read_unaligned(ptr) };
-        self.state[dst].set_i64(i64::from(val));
-        Continuation::Continue
-    }
-
-    fn load64(&mut self, dst: XReg, ptr: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u64>();
-        let val = unsafe { ptr::read_unaligned(ptr) };
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn load32_u_offset8(&mut self, dst: XReg, ptr: XReg, offset: i8) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset.into())
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(u64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_s_offset8(&mut self, dst: XReg, ptr: XReg, offset: i8) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<i32>()
-                .byte_offset(offset.into())
-                .read_unaligned()
-        };
-        self.state[dst].set_i64(i64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_u_offset64(&mut self, dst: XReg, ptr: XReg, offset: i64) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset as isize)
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(u64::from(val));
-        Continuation::Continue
-    }
-
-    fn load32_s_offset64(&mut self, dst: XReg, ptr: XReg, offset: i64) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<i32>()
-                .byte_offset(offset as isize)
-                .read_unaligned()
-        };
-        self.state[dst].set_i64(i64::from(val));
-        Continuation::Continue
-    }
-
-    fn load64_offset8(&mut self, dst: XReg, ptr: XReg, offset: i8) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset.into())
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn load64_offset64(&mut self, dst: XReg, ptr: XReg, offset: i64) -> Self::Return {
-        let val = unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset as isize)
-                .read_unaligned()
-        };
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn store32(&mut self, ptr: XReg, src: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u32>();
-        let val = self.state[src].get_u32();
-        unsafe {
-            ptr::write_unaligned(ptr, val);
-        }
-        Continuation::Continue
-    }
-
-    fn store64(&mut self, ptr: XReg, src: XReg) -> Self::Return {
-        let ptr = self.state[ptr].get_ptr::<u64>();
-        let val = self.state[src].get_u64();
-        unsafe {
-            ptr::write_unaligned(ptr, val);
-        }
-        Continuation::Continue
-    }
-
-    fn store32_offset8(&mut self, ptr: XReg, offset: i8, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u32();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset.into())
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn store64_offset8(&mut self, ptr: XReg, offset: i8, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u64();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset.into())
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn store32_offset64(&mut self, ptr: XReg, offset: i64, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u32();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u32>()
-                .byte_offset(offset as isize)
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn store64_offset64(&mut self, ptr: XReg, offset: i64, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u64();
-        unsafe {
-            self.state[ptr]
-                .get_ptr::<u64>()
-                .byte_offset(offset as isize)
-                .write_unaligned(val);
-        }
-        Continuation::Continue
-    }
-
-    fn xpush32(&mut self, src: XReg) -> Self::Return {
-        self.state.push(self.state[src].get_u32());
-        Continuation::Continue
-    }
-
-    fn xpush32_many(&mut self, srcs: RegSet<XReg>) -> Self::Return {
-        for src in srcs {
-            self.xpush32(src);
-        }
-        Continuation::Continue
-    }
-
-    fn xpush64(&mut self, src: XReg) -> Self::Return {
-        self.state.push(self.state[src].get_u64());
-        Continuation::Continue
-    }
-
-    fn xpush64_many(&mut self, srcs: RegSet<XReg>) -> Self::Return {
-        for src in srcs {
-            self.xpush64(src);
-        }
-        Continuation::Continue
-    }
-
-    fn xpop32(&mut self, dst: XReg) -> Self::Return {
-        let val = self.state.pop();
-        self.state[dst].set_u32(val);
-        Continuation::Continue
-    }
-
-    fn xpop32_many(&mut self, dsts: RegSet<XReg>) -> Self::Return {
-        for dst in dsts.into_iter().rev() {
-            self.xpop32(dst);
-        }
-        Continuation::Continue
-    }
-
-    fn xpop64(&mut self, dst: XReg) -> Self::Return {
-        let val = self.state.pop();
-        self.state[dst].set_u64(val);
-        Continuation::Continue
-    }
-
-    fn xpop64_many(&mut self, dsts: RegSet<XReg>) -> Self::Return {
-        for dst in dsts.into_iter().rev() {
-            self.xpop64(dst);
-        }
-        Continuation::Continue
-    }
-
-    /// `push lr; push fp; fp = sp`
-    fn push_frame(&mut self) -> Self::Return {
-        self.state.push(self.state[XReg::lr].get_ptr::<u8>());
-        self.state.push(self.state[XReg::fp].get_ptr::<u8>());
-        self.state[XReg::fp] = self.state[XReg::sp];
-        Continuation::Continue
-    }
-
-    /// `sp = fp; pop fp; pop lr`
-    fn pop_frame(&mut self) -> Self::Return {
-        self.state[XReg::sp] = self.state[XReg::fp];
-        let fp = self.state.pop();
-        let lr = self.state.pop();
-        self.state[XReg::fp].set_ptr::<u8>(fp);
-        self.state[XReg::lr].set_ptr::<u8>(lr);
-        Continuation::Continue
-    }
-
-    fn bitcast_int_from_float_32(&mut self, dst: XReg, src: FReg) -> Self::Return {
-        let val = self.state[src].get_f32();
-        self.state[dst].set_u64(u32::from_ne_bytes(val.to_ne_bytes()).into());
-        Continuation::Continue
-    }
-
-    fn bitcast_int_from_float_64(&mut self, dst: XReg, src: FReg) -> Self::Return {
-        let val = self.state[src].get_f64();
-        self.state[dst].set_u64(u64::from_ne_bytes(val.to_ne_bytes()));
-        Continuation::Continue
-    }
-
-    fn bitcast_float_from_int_32(&mut self, dst: FReg, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u32();
-        self.state[dst].set_f32(f32::from_ne_bytes(val.to_ne_bytes()));
-        Continuation::Continue
-    }
-
-    fn bitcast_float_from_int_64(&mut self, dst: FReg, src: XReg) -> Self::Return {
-        let val = self.state[src].get_u64();
-        self.state[dst].set_f64(f64::from_ne_bytes(val.to_ne_bytes()));
-        Continuation::Continue
-    }
-}
-
-impl ExtendedOpVisitor for InterpreterVisitor<'_> {
-    fn nop(&mut self) -> Self::Return {
-        Continuation::Continue
-    }
-
-    fn trap(&mut self) -> Self::Return {
-        Continuation::Trap
-    }
-
-    fn get_sp(&mut self, dst: XReg) -> Self::Return {
-        let sp = self.state[XReg::sp].get_u64();
-        self.state[dst].set_u64(sp);
-        Continuation::Continue
-    }
-}
diff --git a/pulley/src/interp/interp_loop.rs b/pulley/src/interp/interp_loop.rs
new file mode 100644
index 000000000..c17cba3ca
--- /dev/null
+++ b/pulley/src/interp/interp_loop.rs
@@ -0,0 +1,669 @@
+use super::*;
+
+#[derive(Copy, Clone)]
+pub struct OpcodeHandler {
+    /// The type of non tail-recursive opcode handlers: return
+    /// `ControlFlow::Continue` with the next handler to call, or
+    /// `ControlFlow::Done` with the reason to stop.
+    #[cfg(not(pulley_tail_calls))]
+    fun: fn(&mut MachineState, &mut UnsafeBytecodeStream) -> ControlFlow<Done, OpcodeHandler>,
+
+    /// The type of tail-recursive opcode handlers: instead of returning
+    /// `ControwFlow::Continue`, tail call the next handler directly; so
+    /// `ControlFlow::Continue` is uninhabited.
+    #[cfg(pulley_tail_calls)]
+    fun: fn(&mut MachineState, &mut UnsafeBytecodeStream) -> Done,
+}
+
+#[cfg(not(pulley_tail_calls))]
+pub fn interpreter_loop(vm: &mut Vm, bytecode: &mut UnsafeBytecodeStream) -> Done {
+    let opcode = Opcode::decode(bytecode).unwrap();
+    let mut handler = OPCODE_HANDLER_TABLE[opcode as usize];
+
+    // As tight as we can get the interpreter loop without tail calls: while
+    // the handlers keep returning the next handler to call, call it.
+    loop {
+        match (handler.fun)(&mut vm.state, bytecode) {
+            ControlFlow::Continue(next_handler) => handler = next_handler,
+            ControlFlow::Break(done) => return done,
+        }
+    }
+}
+
+/// The extra indirection through a macro is necessary to avoid a compiler error
+/// when compiling without `#![feature(explicit_tail_calls)]` enabled (via
+/// `--cfg pulley_tail_calls`).
+///
+/// It seems rustc first parses the the function, encounters `become` and emits
+/// an error about using an unstable keyword on a stable compiler, then applies
+/// `#[cfg(...)` after parsing to disable the function.
+///
+/// Macro bodies are just bags of tokens; the body is not parsed until after
+/// they are expanded, and this macro is only expanded when `pulley_tail_calls`
+/// is enabled.
+#[cfg_attr(not(pulley_tail_calls), allow(unused_macros))]
+macro_rules! tail_call {
+    ($e:expr) => {
+        become $e
+    };
+}
+
+#[cfg(pulley_tail_calls)]
+pub fn interpreter_loop(vm: &mut Vm, bytecode: &mut UnsafeBytecodeStream) -> Done {
+    let opcode = Opcode::decode(bytecode).unwrap();
+    let handler = OPCODE_HANDLER_TABLE[opcode as usize];
+
+    // The ideal interpreter loop: a bunch of opcode handlers tail calling
+    // each other!
+    tail_call!((handler.fun)(&mut vm.state, bytecode));
+}
+
+/// Wrap the business logic of each handler with the boilerplate of decoding
+/// operands and dispatching to next handler/exiting the loop.
+macro_rules! define_opcode_handlers {
+    (
+    $(
+        fn $name:ident (
+            $state:ident : &mut MachineState,
+            $bytecode:ident : &mut UnsafeBytecodeStream$(,)?
+            $($field:ident : $field_ty:ty),*
+        ) $body:block
+    )*
+    ) => {
+        $(
+            #[cfg(not(pulley_tail_calls))]
+            pub fn $name($state: &mut MachineState, $bytecode: &mut UnsafeBytecodeStream) -> ControlFlow<Done, OpcodeHandler> {
+                let ($($field,)*) = crate::decode::unwrap_uninhabited(crate::decode::operands::$name($bytecode));
+                match $body {
+                    ControlFlow::Continue(()) => {
+                        // Decode the next handler and return it so that `run`
+                        // can call it.
+                        let next_opcode = Opcode::decode($bytecode).unwrap();
+                        let next_handler = OPCODE_HANDLER_TABLE[next_opcode as usize];
+                        ControlFlow::Continue(next_handler)
+                    }
+                    ControlFlow::Break(done) => ControlFlow::Break(done),
+                }
+            }
+
+            #[cfg(pulley_tail_calls)]
+            pub fn $name($state: &mut MachineState, $bytecode: &mut UnsafeBytecodeStream) -> Done {
+                let ($($field,)*) = crate::decode::unwrap_uninhabited(crate::decode::operands::$name($bytecode));
+                match $body {
+                    ControlFlow::Continue(()) => {
+                        // Decode the next handler and return it so that `run`
+                        // can call it.
+                        let next_opcode = Opcode::decode($bytecode).unwrap();
+                        let next_handler = OPCODE_HANDLER_TABLE[next_opcode as usize];
+                        tail_call!((next_handler.fun)($state, $bytecode));
+                    }
+                    ControlFlow::Break(done) => done,
+                }
+            }
+        )*
+    };
+}
+
+/// Define the table of opcode handlers.
+macro_rules! opcode_handler_table_entry {
+    (
+        $(
+            $( #[$attr:meta] )*
+                $snake_name:ident = $name:ident $( {
+                $(
+                    $( #[$field_attr:meta] )*
+                    $field:ident : $field_ty:ty
+                ),*
+            } )? ;
+        )*
+    ) => {[ $(OpcodeHandler { fun: $snake_name },)* OpcodeHandler { fun: extended }]};
+}
+
+/// Add one to account for `ExtendedOp`.
+const NUM_OPCODES: usize = Opcode::MAX as usize + 1;
+static OPCODE_HANDLER_TABLE: [OpcodeHandler; NUM_OPCODES] =
+    for_each_op!(opcode_handler_table_entry);
+
+#[inline]
+fn pc_rel_jump(pc: &mut UnsafeBytecodeStream, offset: PcRelOffset, inst_size: isize) {
+    let offset = isize::try_from(i32::from(offset)).unwrap();
+    *pc = unsafe { pc.offset(offset - inst_size) };
+}
+
+define_opcode_handlers! {
+    fn ret(state: &mut MachineState, pc: &mut UnsafeBytecodeStream) {
+        if state[XReg::lr] == XRegVal::HOST_RETURN_ADDR {
+            ControlFlow::Break(Done::ReturnToHost)
+        } else {
+            let return_addr = state[XReg::lr].get_ptr();
+            *pc = unsafe { UnsafeBytecodeStream::new(NonNull::new_unchecked(return_addr)) };
+            ControlFlow::Continue(())
+        }
+    }
+
+    fn call(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, offset: PcRelOffset) {
+        let return_addr = pc.as_ptr();
+        state[XReg::lr].set_ptr(return_addr.as_ptr());
+        pc_rel_jump(pc, offset, 5);
+        ControlFlow::Continue(())
+    }
+
+    fn jump(_state: &mut MachineState, pc: &mut UnsafeBytecodeStream, offset: PcRelOffset) {
+        pc_rel_jump(pc, offset, 5);
+        ControlFlow::Continue(())
+    }
+
+    fn br_if(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, cond: XReg, offset: PcRelOffset) {
+        let cond = state[cond].get_u64();
+        if cond != 0 {
+            pc_rel_jump(pc, offset, 6)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_not(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, cond: XReg, offset: PcRelOffset) {
+        let cond = state[cond].get_u64();
+        if cond == 0 {
+            pc_rel_jump(pc, offset, 6)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xeq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a == b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xneq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a != b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslt32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i32();
+        let b = state[b].get_i32();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i32();
+        let b = state[b].get_i32();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xult32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xulteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u32();
+        let b = state[b].get_u32();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xeq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a == b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xneq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a != b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslt64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i64();
+        let b = state[b].get_i64();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xslteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_i64();
+        let b = state[b].get_i64();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xult64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a < b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn br_if_xulteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, a: XReg, b: XReg, offset: PcRelOffset) {
+        let a = state[a].get_u64();
+        let b = state[b].get_u64();
+        if a <= b {
+            pc_rel_jump(pc, offset, 7)
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xmov(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, src: XReg) {
+        let val = state[src];
+        state[dst] = val;
+        ControlFlow::Continue(())
+    }
+
+    fn fmov(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: FReg, src: FReg) {
+        let val = state[src];
+        state[dst] = val;
+        ControlFlow::Continue(())
+    }
+
+    fn vmov(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: VReg, src: VReg) {
+        let val = state[src];
+        state[dst] = val;
+        ControlFlow::Continue(())
+    }
+
+    fn xconst8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i8) {
+        state[dst].set_i64(i64::from(imm));
+        ControlFlow::Continue(())
+    }
+
+    fn xconst16(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i16) {
+        state[dst].set_i64(i64::from(imm));
+        ControlFlow::Continue(())
+    }
+
+    fn xconst32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i32) {
+        state[dst].set_i64(i64::from(imm));
+        ControlFlow::Continue(())
+    }
+
+    fn xconst64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, imm: i64) {
+        state[dst].set_i64(imm);
+        ControlFlow::Continue(())
+    }
+
+    fn xadd32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u32(a.wrapping_add(b));
+        ControlFlow::Continue(())
+    }
+
+    fn xadd64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(a.wrapping_add(b));
+        ControlFlow::Continue(())
+    }
+
+    fn xeq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a == b));
+        ControlFlow::Continue(())
+    }
+
+    fn xneq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a != b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslt64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i64();
+        let b = state[operands.src2].get_i64();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i64();
+        let b = state[operands.src2].get_i64();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn xult64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xulteq64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u64();
+        let b = state[operands.src2].get_u64();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn xeq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a == b));
+        ControlFlow::Continue(())
+    }
+
+    fn xneq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a != b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslt32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i32();
+        let b = state[operands.src2].get_i32();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xslteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_i32();
+        let b = state[operands.src2].get_i32();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn xult32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a < b));
+        ControlFlow::Continue(())
+    }
+
+    fn xulteq32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, operands: BinaryOperands<XReg>) {
+        let a = state[operands.src1].get_u32();
+        let b = state[operands.src2].get_u32();
+        state[operands.dst].set_u64(u64::from(a <= b));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_u(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg) {
+        let ptr = state[ptr].get_ptr::<u32>();
+        let val = unsafe { ptr::read_unaligned(ptr) };
+        state[dst].set_u64(u64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_s(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg) {
+        let ptr = state[ptr].get_ptr::<i32>();
+        let val = unsafe { ptr::read_unaligned(ptr) };
+        state[dst].set_i64(i64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg) {
+        let ptr = state[ptr].get_ptr::<u64>();
+        let val = unsafe { ptr::read_unaligned(ptr) };
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn load32_u_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i8) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset.into())
+                .read_unaligned()
+        };
+        state[dst].set_u64(u64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_s_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i8) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<i32>()
+                .byte_offset(offset.into())
+                .read_unaligned()
+        };
+        state[dst].set_i64(i64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_u_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i64) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset as isize)
+                .read_unaligned()
+        };
+        state[dst].set_u64(u64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load32_s_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i64) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<i32>()
+                .byte_offset(offset as isize)
+                .read_unaligned()
+        };
+        state[dst].set_i64(i64::from(val));
+        ControlFlow::Continue(())
+    }
+
+    fn load64_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i8) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset.into())
+                .read_unaligned()
+        };
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn load64_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, ptr: XReg, offset: i64) {
+        let val = unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset as isize)
+                .read_unaligned()
+        };
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn store32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, src: XReg) {
+        let ptr = state[ptr].get_ptr::<u32>();
+        let val = state[src].get_u32();
+        unsafe {
+            ptr::write_unaligned(ptr, val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, src: XReg) {
+        let ptr = state[ptr].get_ptr::<u64>();
+        let val = state[src].get_u64();
+        unsafe {
+            ptr::write_unaligned(ptr, val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store32_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i8, src: XReg) {
+        let val = state[src].get_u32();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset.into())
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store64_offset8(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i8, src: XReg) {
+        let val = state[src].get_u64();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset.into())
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store32_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i64, src: XReg) {
+        let val = state[src].get_u32();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u32>()
+                .byte_offset(offset as isize)
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn store64_offset64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, ptr: XReg, offset: i64, src: XReg) {
+        let val = state[src].get_u64();
+        unsafe {
+            state[ptr]
+                .get_ptr::<u64>()
+                .byte_offset(offset as isize)
+                .write_unaligned(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpush32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, src: XReg) {
+        state.push(state[src].get_u32());
+        ControlFlow::Continue(())
+    }
+
+    fn xpush32_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, srcs: RegSet<XReg>) {
+        for src in srcs {
+            state.push(state[src].get_u32());
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpush64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, src: XReg) {
+        state.push(state[src].get_u64());
+        ControlFlow::Continue(())
+    }
+
+    fn xpush64_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, srcs: RegSet<XReg>) {
+        for src in srcs {
+            state.push(state[src].get_u64());
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpop32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg) {
+        let val = state.pop();
+        state[dst].set_u32(val);
+        ControlFlow::Continue(())
+    }
+
+    fn xpop32_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dsts: RegSet<XReg>) {
+        for dst in dsts.into_iter().rev() {
+            let val = state.pop();
+            state[dst].set_u32(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn xpop64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg) {
+        let val = state.pop();
+        state[dst].set_u64(val);
+        ControlFlow::Continue(())
+    }
+
+    fn xpop64_many(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dsts: RegSet<XReg>) {
+        for dst in dsts.into_iter().rev() {
+            let val = state.pop();
+            state[dst].set_u64(val);
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn push_frame(state: &mut MachineState, pc: &mut UnsafeBytecodeStream) {
+        state.push(state[XReg::lr].get_ptr::<u8>());
+        state.push(state[XReg::fp].get_ptr::<u8>());
+        state[XReg::fp] = state[XReg::sp];
+        ControlFlow::Continue(())
+    }
+
+    fn pop_frame(state: &mut MachineState, pc: &mut UnsafeBytecodeStream) {
+        state[XReg::sp] = state[XReg::fp];
+        let fp = state.pop();
+        let lr = state.pop();
+        state[XReg::fp].set_ptr::<u8>(fp);
+        state[XReg::lr].set_ptr::<u8>(lr);
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_int_from_float_32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, src: FReg) {
+        let val = state[src].get_f32();
+        state[dst].set_u64(u32::from_ne_bytes(val.to_ne_bytes()).into());
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_int_from_float_64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: XReg, src: FReg) {
+        let val = state[src].get_f64();
+        state[dst].set_u64(u64::from_ne_bytes(val.to_ne_bytes()));
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_float_from_int_32(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: FReg, src: XReg) {
+        let val = state[src].get_u32();
+        state[dst].set_f32(f32::from_ne_bytes(val.to_ne_bytes()));
+        ControlFlow::Continue(())
+    }
+
+    fn bitcast_float_from_int_64(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, dst: FReg, src: XReg) {
+        let val = state[src].get_u64();
+        state[dst].set_f64(f64::from_ne_bytes(val.to_ne_bytes()));
+        ControlFlow::Continue(())
+    }
+
+    fn extended(state: &mut MachineState, pc: &mut UnsafeBytecodeStream, opcode: ExtendedOpcode) {
+        match opcode {
+            ExtendedOpcode::Nop => ControlFlow::Continue(()),
+            ExtendedOpcode::Trap => ControlFlow::Break(Done::Trap(pc.as_ptr())),
+            ExtendedOpcode::GetSp => {
+                let (dst,) = crate::decode::unwrap_uninhabited(crate::decode::operands::get_sp(pc));
+                let sp = state[XReg::sp].get_u64();
+                state[dst].set_u64(sp);
+                ControlFlow::Continue(())
+            }
+        }
+    }
+}
diff --git a/pulley/src/lib.rs b/pulley/src/lib.rs
index f792b83df..b80a5646e 100644
--- a/pulley/src/lib.rs
+++ b/pulley/src/lib.rs
@@ -1,6 +1,8 @@
 //! The pulley bytecode for fast interpreters.
 
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![cfg_attr(pulley_tail_calls, feature(explicit_tail_calls))]
+#![cfg_attr(pulley_tail_calls, allow(incomplete_features, unstable_features))]
 #![deny(missing_docs)]
 #![no_std]
 


Test patch:
diff --git a/tests/wast.rs b/tests/wast.rs
index e84479774..e3f09012f 100644
--- a/tests/wast.rs
+++ b/tests/wast.rs
@@ -202,13 +202,11 @@ fn should_fail(test: &Path, strategy: Strategy) -> bool {
         }
     }
     let unsupported_gc_tests = [
-        "array_copy.wast",
         "binary_gc.wast",
         "br_on_cast_fail.wast",
         "br_on_cast.wast",
         "extern.wast",
         "ref_cast.wast",
-        "ref_eq.wast",
         "ref_test.wast",
         "table_sub.wast",
         "type_canon.wast",