Repository: bytecodealliance/wasmtime
Pull request #9389 (2024-10-07 18:19:45)

Implement the `array.copy` Wasm GC instruction
This also involved fixing some `VMGcRef` initialization and GC barrier code for globals. That in turn involved making global initialization fallible, which meant that it needed to be pulled out of `vmctx` initialization and put into instance initialization.

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->



Patch:
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 953e37fda..db0a4225e 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -2020,36 +2020,19 @@ impl<'module_environment> crate::translate::FuncEnvironment
     fn translate_array_copy(
         &mut self,
         builder: &mut FunctionBuilder,
-        dst_array_type_index: TypeIndex,
+        _dst_array_type_index: TypeIndex,
         dst_array: ir::Value,
         dst_index: ir::Value,
-        src_array_type_index: TypeIndex,
+        _src_array_type_index: TypeIndex,
         src_array: ir::Value,
         src_index: ir::Value,
         len: ir::Value,
     ) -> WasmResult<()> {
         let libcall = gc::builtins::array_copy(self, builder.func)?;
         let vmctx = self.vmctx_val(&mut builder.cursor());
-        let dst_array_type_index = self.module.types[dst_array_type_index];
-        let dst_array_type_index = builder
-            .ins()
-            .iconst(I32, i64::from(dst_array_type_index.as_u32()));
-        let src_array_type_index = self.module.types[src_array_type_index];
-        let src_array_type_index = builder
-            .ins()
-            .iconst(I32, i64::from(src_array_type_index.as_u32()));
         builder.ins().call(
             libcall,
-            &[
-                vmctx,
-                dst_array_type_index,
-                dst_array,
-                dst_index,
-                src_array_type_index,
-                src_array,
-                src_index,
-                len,
-            ],
+            &[vmctx, dst_array, dst_index, src_array, src_index, len],
         );
         Ok(())
     }
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index e904cb8c8..65e035135 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -148,10 +148,8 @@ macro_rules! foreach_builtin_function {
             #[cfg(feature = "gc")]
             array_copy(
                 vmctx: vmctx,
-                dst_array_interned_type_index: i32,
                 dst_array: reference,
                 dst_index: i32,
-                src_array_interned_type_index: i32,
                 src_array: reference,
                 src_index: i32,
                 len: i32
diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
index 0dc4d4cec..22661caea 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
@@ -568,7 +568,7 @@ impl ArrayRef {
         Ok(gc_ref.as_arrayref_unchecked())
     }
 
-    fn layout(&self, store: &AutoAssertNoGc<'_>) -> Result<GcArrayLayout> {
+    pub(crate) fn layout(&self, store: &AutoAssertNoGc<'_>) -> Result<GcArrayLayout> {
         assert!(self.comes_from_same_store(&store));
         let type_index = self.type_index(store)?;
         let layout = store
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index 6f6fec630..436ea220d 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -905,6 +905,14 @@ impl<T: GcRef> Rooted<T> {
         b: &impl RootedGcRef<T>,
     ) -> Result<bool> {
         let store = store.as_context().0;
+        Self::_ref_eq(store, a, b)
+    }
+
+    pub(crate) fn _ref_eq(
+        store: &StoreOpaque,
+        a: &impl RootedGcRef<T>,
+        b: &impl RootedGcRef<T>,
+    ) -> Result<bool> {
         let a = a.try_gc_ref(store)?;
         let b = b.try_gc_ref(store)?;
         Ok(a == b)
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 4b7a00330..38eef67ca 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1614,6 +1614,8 @@ impl StoreOpaque {
             return;
         }
 
+        log::trace!("============ Begin GC ===========");
+
         // Take the GC roots out of `self` so we can borrow it mutably but still
         // call mutable methods on `self`.
         let mut roots = core::mem::take(&mut self.gc_roots_list);
@@ -1624,6 +1626,8 @@ impl StoreOpaque {
         // Restore the GC roots for the next GC.
         roots.clear();
         self.gc_roots_list = roots;
+
+        log::trace!("============ End GC ===========");
     }
 
     #[inline]
@@ -1662,6 +1666,8 @@ impl StoreOpaque {
             return;
         }
 
+        log::trace!("============ Begin Async GC ===========");
+
         // Take the GC roots out of `self` so we can borrow it mutably but still
         // call mutable methods on `self`.
         let mut roots = std::mem::take(&mut self.gc_roots_list);
@@ -1674,6 +1680,8 @@ impl StoreOpaque {
         // Restore the GC roots for the next GC.
         roots.clear();
         self.gc_roots_list = roots;
+
+        log::trace!("============ End Async GC ===========");
     }
 
     #[inline]
diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs
index a1bf7f0a0..0ffab2720 100644
--- a/crates/wasmtime/src/runtime/vm/const_expr.rs
+++ b/crates/wasmtime/src/runtime/vm/const_expr.rs
@@ -38,11 +38,7 @@ impl<'a> ConstEvalContext<'a> {
                 .defined_or_imported_global_ptr(index)
                 .as_ref()
                 .unwrap();
-            let mut gc_store = store.unwrap_gc_store_mut();
-            Ok(global.to_val_raw(
-                &mut gc_store,
-                self.instance.env_module().globals[index].wasm_ty,
-            ))
+            global.to_val_raw(store, self.instance.env_module().globals[index].wasm_ty)
         }
     }
 
@@ -169,6 +165,8 @@ impl ConstExprEvaluator {
         context: &mut ConstEvalContext<'_>,
         expr: &ConstExpr,
     ) -> Result<ValRaw> {
+        log::trace!("evaluating const expr: {:?}", expr);
+
         self.stack.clear();
 
         let mut store = (*context.instance.store()).store_opaque_mut();
@@ -372,6 +370,7 @@ impl ConstExprEvaluator {
         }
 
         if self.stack.len() == 1 {
+            log::trace!("const expr evaluated to {:?}", self.stack[0]);
             Ok(self.stack[0])
         } else {
             bail!(
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index 7500a68fc..b2f020c85 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -23,6 +23,7 @@ pub use i31::*;
 use crate::prelude::*;
 use crate::runtime::vm::GcHeapAllocationIndex;
 use core::alloc::Layout;
+use core::mem::MaybeUninit;
 use core::ptr;
 use core::{any::Any, num::NonZeroUsize};
 use wasmtime_environ::{GcArrayLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
@@ -96,6 +97,19 @@ impl GcStore {
         }
     }
 
+    /// Write the `source` GC reference into the uninitialized `destination`
+    /// slot, performing write barriers as necessary.
+    pub fn init_gc_ref(
+        &mut self,
+        destination: &mut MaybeUninit<Option<VMGcRef>>,
+        source: Option<&VMGcRef>,
+    ) {
+        // Initialize the destination to `None`, at which point the regular GC
+        // write barrier is safe to reuse.
+        let destination = destination.write(None);
+        self.write_gc_ref(destination, source);
+    }
+
     /// Write the `source` GC reference into the `destination` slot, performing
     /// write barriers as necessary.
     pub fn write_gc_ref(&mut self, destination: &mut Option<VMGcRef>, source: Option<&VMGcRef>) {
@@ -205,6 +219,18 @@ impl GcStore {
         self.gc_heap.gc_object_data(gc_ref)
     }
 
+    /// Get the object datas for the given pair of object references.
+    ///
+    /// Panics if `a` and `b` are the same reference or either is out of bounds.
+    pub fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        assert_ne!(a, b);
+        self.gc_heap.gc_object_data_pair(a, b)
+    }
+
     /// Allocate an uninitialized array with the given type index.
     ///
     /// This does NOT check that the index is currently allocated in the types
@@ -299,6 +325,13 @@ unsafe impl GcHeap for DisabledGcHeap {
     fn gc_object_data(&mut self, _gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_> {
         unreachable!()
     }
+    fn gc_object_data_pair(
+        &mut self,
+        _a: &VMGcRef,
+        _b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        unreachable!()
+    }
     fn alloc_uninit_array(
         &mut self,
         _ty: VMSharedTypeIndex,
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
index 2c74ccbfd..19328387e 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
@@ -133,6 +133,28 @@ impl<'a> VMGcObjectDataMut<'a> {
         val.write_le(into.try_into().unwrap());
     }
 
+    /// Get a slice of this object's data.
+    ///
+    /// Panics on out-of-bounds accesses.
+    #[inline]
+    pub fn slice(&self, offset: u32, len: u32) -> &[u8] {
+        let start = usize::try_from(offset).unwrap();
+        let len = usize::try_from(len).unwrap();
+        let end = start.checked_add(len).unwrap();
+        self.data.get(start..end).expect("out of bounds slice")
+    }
+
+    /// Get a mutable slice of this object's data.
+    ///
+    /// Panics on out-of-bounds accesses.
+    #[inline]
+    pub fn slice_mut(&mut self, offset: u32, len: u32) -> &mut [u8] {
+        let start = usize::try_from(offset).unwrap();
+        let len = usize::try_from(len).unwrap();
+        let end = start.checked_add(len).unwrap();
+        self.data.get_mut(start..end).expect("out of bounds slice")
+    }
+
     /// Copy the given slice into this object's data at the given offset.
     ///
     /// Panics on out-of-bounds accesses.
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index ebf5e9b46..02c65b68b 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -49,7 +49,7 @@ use crate::runtime::vm::{
     ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
     GcProgress, GcRootsIter, GcRuntime, Mmap, TypedGcRef, VMExternRef, VMGcHeader, VMGcRef,
 };
-use core::ops::{Deref, DerefMut};
+use core::ops::{Deref, DerefMut, Range};
 use core::{
     alloc::Layout,
     any::Any,
@@ -132,6 +132,16 @@ impl DrcHeap {
             .dealloc(gc_ref.as_heap_index().unwrap(), layout);
     }
 
+    fn object_range(&self, gc_ref: &VMGcRef) -> Range<usize> {
+        let start = gc_ref.as_heap_index().unwrap().get();
+        let start = usize::try_from(start).unwrap();
+        let size = self
+            .index::<VMDrcHeader>(gc_ref.as_typed_unchecked())
+            .object_size();
+        let end = start.checked_add(size).unwrap();
+        start..end
+    }
+
     /// Index into this heap and get a shared reference to the `T` that `gc_ref`
     /// points to.
     ///
@@ -617,16 +627,42 @@ unsafe impl GcHeap for DrcHeap {
     }
 
     fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_> {
-        let start = gc_ref.as_heap_index().unwrap().get();
-        let start = usize::try_from(start).unwrap();
-        let size = self
-            .index::<VMDrcHeader>(gc_ref.as_typed_unchecked())
-            .object_size();
-        let end = start + size;
-        let data = &mut self.heap_slice_mut()[start..end];
+        let range = self.object_range(gc_ref);
+        let data = &mut self.heap_slice_mut()[range];
         VMGcObjectDataMut::new(data)
     }
 
+    fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>) {
+        assert_ne!(a, b);
+
+        let a_range = self.object_range(a);
+        let b_range = self.object_range(b);
+
+        // Assert that the two objects do not overlap.
+        assert!(a_range.start <= a_range.end);
+        assert!(b_range.start <= b_range.end);
+        assert!(a_range.end <= b_range.start || b_range.end <= a_range.start);
+
+        let (a_data, b_data) = if a_range.start < b_range.start {
+            let (a_half, b_half) = self.heap_slice_mut().split_at_mut(b_range.start);
+            let b_len = b_range.end - b_range.start;
+            (&mut a_half[a_range], &mut b_half[..b_len])
+        } else {
+            let (b_half, a_half) = self.heap_slice_mut().split_at_mut(a_range.start);
+            let a_len = a_range.end - a_range.start;
+            (&mut a_half[..a_len], &mut b_half[b_range])
+        };
+
+        (
+            VMGcObjectDataMut::new(a_data),
+            VMGcObjectDataMut::new(b_data),
+        )
+    }
+
     fn alloc_uninit_array(
         &mut self,
         ty: VMSharedTypeIndex,
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
index fa89efb81..539ff5b95 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
@@ -285,15 +285,15 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
     /// Panics on out-of-bounds accesses.
     fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> VMGcObjectDataMut<'_>;
 
-    /// Allocate a GC-managed array of the given type and length.
-    ///
-    /// The array's elements are left uninitialized. It is the caller's
-    /// responsibility to initialize them before exposing the array to Wasm or
-    /// triggering a GC. Failure to do this is memory safe, but may result in
-    /// general failures such as panics or incorrect results.
-    ///
-    /// Return values:
+    /// Get a pair of mutable borrows of the given objects' data.
     ///
+    /// Panics if `a == b` or on out-of-bounds accesses.
+    fn gc_object_data_pair(
+        &mut self,
+        a: &VMGcRef,
+        b: &VMGcRef,
+    ) -> (VMGcObjectDataMut<'_>, VMGcObjectDataMut<'_>);
+
     /// * `Ok(Some(_))`: The allocation was successful.
     ///
     /// * `Ok(None)`: There is currently no available space for this
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index b1e878c59..d707bbc09 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -1318,34 +1318,12 @@ impl Instance {
             ptr = ptr.add(1);
         }
 
-        // Initialize the defined globals
-        let mut const_evaluator = ConstExprEvaluator::default();
-        self.initialize_vmctx_globals(&mut const_evaluator, module);
-    }
-
-    unsafe fn initialize_vmctx_globals(
-        &mut self,
-        const_evaluator: &mut ConstExprEvaluator,
-        module: &Module,
-    ) {
-        for (index, init) in module.global_initializers.iter() {
-            let mut context = ConstEvalContext::new(self);
-            let raw = const_evaluator
-                .eval(&mut context, init)
-                .expect("should be a valid const expr");
-
-            let to = self.global_ptr(index);
-            let wasm_ty = module.globals[module.global_index(index)].wasm_ty;
-
-            #[cfg(feature = "wmemcheck")]
-            if index.index() == 0 && wasm_ty == wasmtime_environ::WasmValType::I32 {
-                if let Some(wmemcheck) = &mut self.wmemcheck_state {
-                    let size = usize::try_from(raw.get_i32()).unwrap();
-                    wmemcheck.set_stack_size(size);
-                }
-            }
-
-            ptr::write(to, VMGlobalDefinition::from_val_raw(wasm_ty, raw));
+        // Zero-initialize the globals so that nothing is uninitialized memory
+        // after this function returns. The globals are actually initialized
+        // with their const expression initializers after the instance is fully
+        // allocated.
+        for (index, _init) in module.global_initializers.iter() {
+            ptr::write(self.global_ptr(index), VMGlobalDefinition::new());
         }
     }
 
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
index f93592378..1fe374517 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
@@ -6,6 +6,8 @@ use crate::runtime::vm::memory::Memory;
 use crate::runtime::vm::mpk::ProtectionKey;
 use crate::runtime::vm::table::Table;
 use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo, VMFuncRef, VMGcRef, VMStore};
+use crate::store::AutoAssertNoGc;
+use crate::vm::VMGlobalDefinition;
 use core::{any::Any, mem, ptr};
 use wasmtime_environ::{
     DefinedMemoryIndex, DefinedTableIndex, HostPtr, InitMemory, MemoryInitialization,
@@ -555,27 +557,28 @@ fn check_table_init_bounds(instance: &mut Instance, module: &Module) -> Result<(
     Ok(())
 }
 
-fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
-    let mut const_evaluator = ConstExprEvaluator::default();
-
+fn initialize_tables(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
     for (table, init) in module.table_initialization.initial_values.iter() {
         match init {
             // Tables are always initially null-initialized at this time
             TableInitialValue::Null { precomputed: _ } => {}
 
             TableInitialValue::Expr(expr) => {
-                let mut context = ConstEvalContext::new(instance);
                 let raw = unsafe {
                     const_evaluator
-                        .eval(&mut context, expr)
+                        .eval(context, expr)
                         .expect("const expression should be valid")
                 };
                 let idx = module.table_index(table);
-                let table = unsafe { instance.get_defined_table(table).as_mut().unwrap() };
+                let table = unsafe { context.instance.get_defined_table(table).as_mut().unwrap() };
                 match module.table_plans[idx].table.ref_type.heap_type.top() {
                     WasmHeapTopType::Extern => {
                         let gc_ref = VMGcRef::from_raw_u32(raw.get_externref());
-                        let gc_store = unsafe { (*instance.store()).unwrap_gc_store_mut() };
+                        let gc_store = unsafe { (*context.instance.store()).gc_store_mut()? };
                         let items = (0..table.size())
                             .map(|_| gc_ref.as_ref().map(|r| gc_store.clone_gc_ref(r)));
                         table.init_gc_refs(0, items).err2anyhow()?;
@@ -583,7 +586,7 @@ fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
 
                     WasmHeapTopType::Any => {
                         let gc_ref = VMGcRef::from_raw_u32(raw.get_anyref());
-                        let gc_store = unsafe { (*instance.store()).unwrap_gc_store_mut() };
+                        let gc_store = unsafe { (*context.instance.store()).gc_store_mut()? };
                         let items = (0..table.size())
                             .map(|_| gc_ref.as_ref().map(|r| gc_store.clone_gc_ref(r)));
                         table.init_gc_refs(0, items).err2anyhow()?;
@@ -607,15 +610,15 @@ fn initialize_tables(instance: &mut Instance, module: &Module) -> Result<()> {
     // iterates over all segments (Segments mode) or leftover
     // segments (FuncTable mode) to initialize.
     for segment in module.table_initialization.segments.iter() {
-        let mut context = ConstEvalContext::new(instance);
         let start = unsafe {
             const_evaluator
-                .eval(&mut context, &segment.offset)
+                .eval(context, &segment.offset)
                 .expect("const expression should be valid")
         };
-        instance
+        context
+            .instance
             .table_init_segment(
-                &mut const_evaluator,
+                const_evaluator,
                 segment.table_index,
                 &segment.elements,
                 start.get_u64(),
@@ -666,7 +669,11 @@ fn check_memory_init_bounds(
     Ok(())
 }
 
-fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
+fn initialize_memories(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
     // Delegates to the `init_memory` method which is sort of a duplicate of
     // `instance.memory_init_segment` but is used at compile-time in other
     // contexts so is shared here to have only one method of memory
@@ -676,18 +683,18 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
     // so errors only happen if an out-of-bounds segment is found, in which case
     // a trap is returned.
 
-    struct InitMemoryAtInstantiation<'a> {
-        instance: &'a mut Instance,
+    struct InitMemoryAtInstantiation<'a, 'b> {
         module: &'a Module,
-        const_evaluator: ConstExprEvaluator,
+        context: &'a mut ConstEvalContext<'b>,
+        const_evaluator: &'a mut ConstExprEvaluator,
     }
 
-    impl InitMemory for InitMemoryAtInstantiation<'_> {
+    impl InitMemory for InitMemoryAtInstantiation<'_, '_> {
         fn memory_size_in_bytes(
             &mut self,
             memory: wasmtime_environ::MemoryIndex,
         ) -> Result<u64, SizeOverflow> {
-            let len = self.instance.get_memory(memory).current_length();
+            let len = self.context.instance.get_memory(memory).current_length();
             let len = u64::try_from(len).unwrap();
             Ok(len)
         }
@@ -697,11 +704,10 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
             memory: wasmtime_environ::MemoryIndex,
             expr: &wasmtime_environ::ConstExpr,
         ) -> Option<u64> {
-            let mut context = ConstEvalContext::new(self.instance);
-            let val = unsafe { self.const_evaluator.eval(&mut context, expr) }
+            let val = unsafe { self.const_evaluator.eval(self.context, expr) }
                 .expect("const expression should be valid");
             Some(
-                match self.instance.env_module().memory_plans[memory]
+                match self.context.instance.env_module().memory_plans[memory]
                     .memory
                     .idx_type
                 {
@@ -721,14 +727,14 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
             // pre-initializing it via mmap magic, then this initializer can be
             // skipped entirely.
             if let Some(memory_index) = self.module.defined_memory_index(memory_index) {
-                if !self.instance.memories[memory_index].1.needs_init() {
+                if !self.context.instance.memories[memory_index].1.needs_init() {
                     return true;
                 }
             }
-            let memory = self.instance.get_memory(memory_index);
+            let memory = self.context.instance.get_memory(memory_index);
 
             unsafe {
-                let src = self.instance.wasm_data(init.data.clone());
+                let src = self.context.instance.wasm_data(init.data.clone());
                 let offset = usize::try_from(init.offset).unwrap();
                 let dst = memory.base.add(offset);
 
@@ -746,9 +752,9 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<()> {
     let ok = module
         .memory_initialization
         .init_memory(&mut InitMemoryAtInstantiation {
-            instance,
             module,
-            const_evaluator: ConstExprEvaluator::default(),
+            context,
+            const_evaluator,
         });
     if !ok {
         return Err(Trap::MemoryOutOfBounds).err2anyhow();
@@ -771,6 +777,46 @@ fn check_init_bounds(instance: &mut Instance, module: &Module) -> Result<()> {
     Ok(())
 }
 
+fn initialize_globals(
+    context: &mut ConstEvalContext<'_>,
+    const_evaluator: &mut ConstExprEvaluator,
+    module: &Module,
+) -> Result<()> {
+    assert!(core::ptr::eq(&**context.instance.env_module(), module));
+
+    for (index, init) in module.global_initializers.iter() {
+        let raw = unsafe {
+            const_evaluator
+                .eval(context, init)
+                .expect("should be a valid const expr")
+        };
+
+        let to = context.instance.global_ptr(index);
+        let wasm_ty = module.globals[module.global_index(index)].wasm_ty;
+
+        #[cfg(feature = "wmemcheck")]
+        if index.as_bits() == 0 && wasm_ty == wasmtime_environ::WasmValType::I32 {
+            if let Some(wmemcheck) = &mut context.instance.wmemcheck_state {
+                let size = usize::try_from(raw.get_i32()).unwrap();
+                wmemcheck.set_stack_size(size);
+            }
+        }
+
+        let store = unsafe { (*context.instance.store()).store_opaque_mut() };
+        let mut store = AutoAssertNoGc::new(store);
+
+        // This write is safe because we know we have the correct module for
+        // this instance and its vmctx due to the assert above.
+        unsafe {
+            ptr::write(
+                to,
+                VMGlobalDefinition::from_val_raw(&mut store, wasm_ty, raw)?,
+            )
+        };
+    }
+    Ok(())
+}
+
 pub(super) fn initialize_instance(
     instance: &mut Instance,
     module: &Module,
@@ -784,11 +830,12 @@ pub(super) fn initialize_instance(
         check_init_bounds(instance, module)?;
     }
 
-    // Initialize the tables
-    initialize_tables(instance, module)?;
+    let mut context = ConstEvalContext::new(instance);
+    let mut const_evaluator = ConstExprEvaluator::default();
 
-    // Initialize the memories
-    initialize_memories(instance, &module)?;
+    initialize_globals(&mut context, &mut const_evaluator, module)?;
+    initialize_tables(&mut context, &mut const_evaluator, module)?;
+    initialize_memories(&mut context, &mut const_evaluator, &module)?;
 
     Ok(())
 }
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 01519dd5e..15189d37e 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -840,6 +840,7 @@ unsafe fn array_init_elem(
         // Convert the raw GC ref into a `Rooted<ArrayRef>`.
         let array =
             VMGcRef::from_raw_u32(array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+        let array = (*instance.store()).unwrap_gc_store_mut().clone_gc_ref(&array);
         let array = {
             let mut no_gc = AutoAssertNoGc::new((*instance.store()).store_opaque_mut());
             ArrayRef::from_cloned_gc_ref(&mut no_gc, array)
@@ -918,18 +919,79 @@ unsafe fn array_init_elem(
     })
 }
 
+// TODO: Specialize this libcall for only non-GC array elements, so we never
+// have to do GC barriers and their associated indirect calls through the `dyn
+// GcHeap`. Instead, implement those copies inline in Wasm code. Then, use bulk
+// `memcpy`-style APIs to do the actual copies here.
 #[cfg(feature = "gc")]
 unsafe fn array_copy(
-    _instance: &mut Instance,
-    _dst_array_type_index: u32,
-    _dst_array: u32,
-    _dst_index: u32,
-    _src_array_type_index: u32,
-    _src_array: u32,
-    _src_index: u32,
-    _len: u32,
+    instance: &mut Instance,
+    dst_array: u32,
+    dst: u32,
+    src_array: u32,
+    src: u32,
+    len: u32,
 ) -> Result<()> {
-    bail!("the `array.copy` instruction is not yet implemented")
+    use crate::{store::AutoAssertNoGc, ArrayRef, OpaqueRootScope};
+
+    log::trace!(
+            "array.copy(dst_array={dst_array:#x}, dst_index={dst}, src_array={src_array:#x}, src_index={src}, len={len})",
+        );
+
+    let store = (*instance.store()).store_opaque_mut();
+    let mut store = OpaqueRootScope::new(store);
+    let mut store = AutoAssertNoGc::new(&mut store);
+
+    // Convert the raw GC refs into `Rooted<ArrayRef>`s.
+    let dst_array =
+        VMGcRef::from_raw_u32(dst_array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+    let dst_array = store.unwrap_gc_store_mut().clone_gc_ref(&dst_array);
+    let dst_array = ArrayRef::from_cloned_gc_ref(&mut store, dst_array);
+    let src_array =
+        VMGcRef::from_raw_u32(src_array).ok_or_else(|| Trap::NullReference.into_anyhow())?;
+    let src_array = store.unwrap_gc_store_mut().clone_gc_ref(&src_array);
+    let src_array = ArrayRef::from_cloned_gc_ref(&mut store, src_array);
+
+    // Bounds check the destination array's elements.
+    let dst_array_len = dst_array._len(&store)?;
+    if dst
+        .checked_add(len)
+        .ok_or_else(|| Trap::ArrayOutOfBounds.into_anyhow())?
+        > dst_array_len
+    {
+        return Err(Trap::ArrayOutOfBounds.into_anyhow());
+    }
+
+    // Bounds check the source array's elements.
+    let src_array_len = src_array._len(&store)?;
+    if src
+        .checked_add(len)
+        .ok_or_else(|| Trap::ArrayOutOfBounds.into_anyhow())?
+        > src_array_len
+    {
+        return Err(Trap::ArrayOutOfBounds.into_anyhow());
+    }
+
+    let mut store = AutoAssertNoGc::new(&mut store);
+    // If `src_array` and `dst_array` are the same array, then we are
+    // potentially doing an overlapping copy, so make sure to copy elements in
+    // the order that doesn't clobber the source elements before they are
+    // copied. If they are different arrays, the order doesn't matter, but we
+    // simply don't bother checking.
+    if src > dst {
+        for i in 0..len {
+            let src_elem = src_array._get(&mut store, src + i)?;
+            let dst_i = dst + i;
+            dst_array._set(&mut store, dst_i, src_elem)?;
+        }
+    } else {
+        for i in (0..len).rev() {
+            let src_elem = src_array._get(&mut store, src + i)?;
+            let dst_i = dst + i;
+            dst_array._set(&mut store, dst_i, src_elem)?;
+        }
+    }
+    Ok(())
 }
 
 // Implementation of `memory.atomic.notify` for locally defined memories.
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 26e72a84f..59ae7ba0f 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -4,12 +4,14 @@
 mod vm_host_func_context;
 
 pub use self::vm_host_func_context::VMArrayCallHostFuncContext;
+use crate::prelude::*;
 use crate::runtime::vm::{GcStore, VMGcRef};
+use crate::store::StoreOpaque;
 use core::cell::UnsafeCell;
 use core::ffi::c_void;
 use core::fmt;
 use core::marker;
-use core::mem;
+use core::mem::{self, MaybeUninit};
 use core::ptr::{self, NonNull};
 use core::sync::atomic::{AtomicUsize, Ordering};
 use sptr::Strict;
@@ -423,7 +425,11 @@ impl VMGlobalDefinition {
     /// # Unsafety
     ///
     /// This raw value's type must match the given `WasmValType`.
-    pub unsafe fn from_val_raw(wasm_ty: WasmValType, raw: ValRaw) -> Self {
+    pub unsafe fn from_val_raw(
+        store: &mut StoreOpaque,
+        wasm_ty: WasmValType,
+        raw: ValRaw,
+    ) -> Result<Self> {
         let mut global = Self::new();
         match wasm_ty {
             WasmValType::I32 => *global.as_i32_mut() = raw.get_i32(),
@@ -433,13 +439,17 @@ impl VMGlobalDefinition {
             WasmValType::V128 => *global.as_u128_mut() = raw.get_v128(),
             WasmValType::Ref(r) => match r.heap_type.top() {
                 WasmHeapTopType::Extern => {
-                    global.init_gc_ref(VMGcRef::from_raw_u32(raw.get_externref()))
+                    let r = VMGcRef::from_raw_u32(raw.get_externref());
+                    global.init_gc_ref(store.gc_store_mut()?, r.as_ref())
+                }
+                WasmHeapTopType::Any => {
+                    let r = VMGcRef::from_raw_u32(raw.get_anyref());
+                    global.init_gc_ref(store.gc_store_mut()?, r.as_ref())
                 }
-                WasmHeapTopType::Any => global.init_gc_ref(VMGcRef::from_raw_u32(raw.get_anyref())),
                 WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(),
             },
         }
-        global
+        Ok(global)
     }
 
     /// Get this global's value as a `ValRaw`.
@@ -447,25 +457,31 @@ impl VMGlobalDefinition {
     /// # Unsafety
     ///
     /// This global's value's type must match the given `WasmValType`.
-    pub unsafe fn to_val_raw(&self, gc_store: &mut GcStore, wasm_ty: WasmValType) -> ValRaw {
-        match wasm_ty {
+    pub unsafe fn to_val_raw(
+        &self,
+        store: &mut StoreOpaque,
+        wasm_ty: WasmValType,
+    ) -> Result<ValRaw> {
+        Ok(match wasm_ty {
             WasmValType::I32 => ValRaw::i32(*self.as_i32()),
             WasmValType::I64 => ValRaw::i64(*self.as_i64()),
             WasmValType::F32 => ValRaw::f32(*self.as_f32_bits()),
             WasmValType::F64 => ValRaw::f64(*self.as_f64_bits()),
             WasmValType::V128 => ValRaw::v128(*self.as_u128()),
             WasmValType::Ref(r) => match r.heap_type.top() {
-                WasmHeapTopType::Extern => ValRaw::externref(
-                    self.as_gc_ref()
-                        .map_or(0, |r| gc_store.clone_gc_ref(r).as_raw_u32()),
-                ),
-                WasmHeapTopType::Any => ValRaw::anyref(
-                    self.as_gc_ref()
-                        .map_or(0, |r| gc_store.clone_gc_ref(r).as_raw_u32()),
-                ),
+                WasmHeapTopType::Extern => ValRaw::externref(match self.as_gc_ref() {
+                    Some(r) => store.gc_store_mut()?.clone_gc_ref(r).as_raw_u32(),
+                    None => 0,
+                }),
+                WasmHeapTopType::Any => ValRaw::anyref({
+                    match self.as_gc_ref() {
+                        Some(r) => store.gc_store_mut()?.clone_gc_ref(r).as_raw_u32(),
+                        None => 0,
+                    }
+                }),
                 WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()),
             },
-        }
+        })
     }
 
     /// Return a reference to the value as an i32.
@@ -577,10 +593,16 @@ impl VMGlobalDefinition {
     }
 
     /// Initialize a global to the given GC reference.
-    pub unsafe fn init_gc_ref(&mut self, gc_ref: Option<VMGcRef>) {
+    pub unsafe fn init_gc_ref(&mut self, gc_store: &mut GcStore, gc_ref: Option<&VMGcRef>) {
         assert!(cfg!(feature = "gc") || gc_ref.is_none());
-        let raw_ptr = self.storage.as_mut().as_mut_ptr().cast::<Option<VMGcRef>>();
-        ptr::write(raw_ptr, gc_ref);
+
+        let dest = &mut *(self
+            .storage
+            .as_mut()
+            .as_mut_ptr()
+            .cast::<MaybeUninit<Option<VMGcRef>>>());
+
+        gc_store.init_gc_ref(dest, gc_ref)
     }
 
     /// Write a GC reference into this global value.


Test patch:
diff --git a/tests/wast.rs b/tests/wast.rs
index 933dfc33b..e3f498558 100644
--- a/tests/wast.rs
+++ b/tests/wast.rs
@@ -202,7 +202,6 @@ fn should_fail(test: &Path, strategy: Strategy) -> bool {
         }
     }
     let unsupported_gc_tests = [
-        "array_copy.wast",
         "binary_gc.wast",
         "br_on_cast_fail.wast",
         "br_on_cast.wast",
@@ -210,8 +209,6 @@ fn should_fail(test: &Path, strategy: Strategy) -> bool {
         "ref_cast.wast",
         "ref_eq.wast",
         "ref_test.wast",
-        "return_call_indirect.wast",
-        "return_call.wast",
         "table_sub.wast",
         "type_canon.wast",
         "type_equivalence.wast",
@@ -281,7 +278,7 @@ fn run_wast(wast: &Path, strategy: Strategy, pooling: bool) -> anyhow::Result<()
     let function_references = gc || memory64 || feature_found(wast, "function-references");
     let reference_types = !(threads && feature_found(wast, "proposals"));
     let relaxed_simd = feature_found(wast, "relaxed-simd");
-    let tail_call = feature_found(wast, "tail-call") || feature_found(wast, "function-references");
+    let tail_call = function_references || feature_found(wast, "tail-call");
     let use_shared_memory = feature_found_src(&wast_bytes, "shared_memory")
         || feature_found_src(&wast_bytes, "shared)");
     let extended_const = feature_found(wast, "extended-const") || memory64;