Repository: bytecodealliance/wasmtime
Pull request #5311 (2022-11-21 22:47:22)

Refactor shared memory internals, expose embedder methods
This commit refactors the internals of `wasmtime_runtime::SharedMemory` a bit to expose the necessary functions to invoke from the `wasmtime::SharedMemory` layer. Notably some items are moved out of the `RwLock` from prior, such as the type and the `VMMemoryDefinition`. Additionally the organization around the `atomic_*` methods has been redone to ensure that the `wasmtime`-layer abstraction has a single method to call into which everything else uses as well.

<!--

Please ensure that the following steps are all taken care of before submitting
the PR.

- [ ] This has been discussed in issue #..., or if not, please tell us why
  here.
- [ ] A short description of what this does, why it is needed; if the
  description becomes long, the matter should probably be discussed in an issue
  first.
- [ ] This PR contains test cases, if meaningful.
- [ ] A reviewer from the core maintainer team has been assigned for this PR.
  If you don't know who could review this, please indicate so. The list of
  suggested reviewers on the right can help you.

Please ensure all communication adheres to the [code of
conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
-->


alexcrichton: cc @haraldh 
github-actions[bot]: #### Subscribe to Label Action

cc @peterhuene

<details>
This issue or pull request has been labeled: "wasmtime:api"

Thus the following users have been cc'd because of the following labels:

* peterhuene: wasmtime:api

To subscribe or unsubscribe from this label, edit the <code>.github/subscribe-to-label.json</code> configuration file.

[Learn more.](https://github.com/bytecodealliance/subscribe-to-label-action)
</details>

Patch:
diff --git a/RELEASES.md b/RELEASES.md
index 6cc80f22c..f99a433a6 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -12,7 +12,7 @@ Unreleased.
 
 ## 3.0.0
 
-Unreleased.
+Released 2022-11-21
 
 ### Added
 
diff --git a/benches/wasi.rs b/benches/wasi.rs
index 3b8f28863..ae8a7c8c1 100644
--- a/benches/wasi.rs
+++ b/benches/wasi.rs
@@ -1,7 +1,7 @@
 //! Measure some common WASI call scenarios.
 
 use criterion::{criterion_group, criterion_main, Criterion};
-use std::time::Instant;
+use std::{fs::File, path::Path, time::Instant};
 use wasmtime::{Engine, Linker, Module, Store, TypedFunc};
 use wasmtime_wasi::{sync::WasiCtxBuilder, WasiCtx};
 
@@ -9,6 +9,15 @@ criterion_group!(benches, bench_wasi);
 criterion_main!(benches);
 
 fn bench_wasi(c: &mut Criterion) {
+    let _ = env_logger::try_init();
+
+    // Build a zero-filled test file if it does not yet exist.
+    let test_file = Path::new("benches/wasi/test.bin");
+    if !test_file.is_file() {
+        let file = File::create(test_file).unwrap();
+        file.set_len(4096).unwrap();
+    }
+
     // Benchmark each `*.wat` file in the `wasi` directory.
     for file in std::fs::read_dir("benches/wasi").unwrap() {
         let path = file.unwrap().path();
@@ -67,5 +76,11 @@ fn wasi_context() -> WasiCtx {
         "--flag4".to_string(),
     ])
     .unwrap()
+    .preopened_dir(
+        wasmtime_wasi::Dir::open_ambient_dir("benches/wasi", wasmtime_wasi::ambient_authority())
+            .unwrap(),
+        "/",
+    )
+    .unwrap()
     .build()
 }
diff --git a/benches/wasi/.gitignore b/benches/wasi/.gitignore
new file mode 100644
index 000000000..a0c61d0c3
--- /dev/null
+++ b/benches/wasi/.gitignore
@@ -0,0 +1,1 @@
+test.bin
diff --git a/benches/wasi/open-file.wat b/benches/wasi/open-file.wat
new file mode 100644
index 000000000..390562ee8
--- /dev/null
+++ b/benches/wasi/open-file.wat
@@ -0,0 +1,53 @@
+;; Repeatedly open and close `test.bin`.
+(module
+    (import "wasi_snapshot_preview1" "path_open"
+        (func $__wasi_path_open (param i32 i32 i32 i32 i32 i64 i64 i32 i32) (result i32)))
+    (import "wasi_snapshot_preview1" "fd_read"
+        (func $__wasi_fd_read (param i32 i32 i32 i32) (result i32)))
+    (import "wasi_snapshot_preview1" "fd_close"
+        (func $__wasi_fd_close (param i32) (result i32)))
+    (func (export "run") (param $iters i64) (result i64)
+        (local $i i64)
+        (local.set $i (i64.const 0))
+        (loop $cont
+            ;; Open the file `test.bin` under the same directory as this WAT
+            ;; file; this assumes some prior set up of the preopens in
+            ;; `wasi.rs`. See https://github.com/WebAssembly/WASI/blob/d8da230b/phases/snapshot/witx/wasi_snapshot_preview1.witx#L346.
+            (call $__wasi_path_open
+                ;; The fd of the preopen under which to search for the file;
+                ;; the first three are the `std*` ones.
+                (i32.const 3)
+                ;; The lookup flags (i.e., whether to follow symlinks).
+                (i32.const 0)
+                ;; The path to the file under the initial fd.
+                (i32.const 0)
+                (i32.const 8)
+                ;; The open flags; in this case we will only attempt to read but
+                ;; this may attempt to create the file if it does not exist, see
+                ;; https://github.com/WebAssembly/WASI/blob/d8da230b/phases/snapshot/witxtypenames.witx#L444).
+                (i32.const 0)
+                ;; The base rights and the inheriting rights: here we only set
+                ;; the bits for the FD_READ and FD_READDIR capabilities.
+                (i64.const 0x2002)
+                (i64.const 0x2002)
+                ;; The file descriptor flags (e.g., whether to append, sync,
+                ;; etc.); see https://github.com/WebAssembly/WASI/blob/d8da230b/phases/snapshot/witx/typenames.witx#L385
+                (i32.const 0)
+                ;; The address at which to store the opened fd (if the call
+                ;; succeeds)
+                (i32.const 16))
+            (if (then unreachable))
+
+            ;; Close the open file handle we stored at offset 16.
+            (call $__wasi_fd_close (i32.load (i32.const 16)))
+            (if (then unreachable))
+
+            ;; Continue looping until $i reaches $iters.
+            (local.set $i (i64.add (local.get $i) (i64.const 1)))
+            (br_if $cont (i64.lt_u (local.get $i) (local.get $iters)))
+        )
+        (local.get $i)
+    )
+    (data (i32.const 0) "test.bin")
+    (memory (export "memory") 1)
+)
diff --git a/benches/wasi/read-dir.wat b/benches/wasi/read-dir.wat
new file mode 100644
index 000000000..6d9004a71
--- /dev/null
+++ b/benches/wasi/read-dir.wat
@@ -0,0 +1,41 @@
+;; Read the directory entries of the preopened directory.
+(module
+    (import "wasi_snapshot_preview1" "fd_readdir"
+        (func $__wasi_fd_readdir (param i32 i32 i32 i64 i32) (result i32)))
+    (func (export "run") (param $iters i64) (result i64)
+        (local $i i64)
+        (local.set $i (i64.const 0))
+
+        (if (i32.ne (i32.load (i32.const 0)) (i32.const 0))
+            (then unreachable))
+
+        (loop $cont
+            ;; Read the file into the sole iovec buffer.
+            (call $__wasi_fd_readdir
+                ;; The fd of the preopened directory; the first three are the
+                ;; `std*` ones.
+                (i32.const 3)
+                ;; The buffer address at which to store the entries and the
+                ;; length of the buffer.
+                (i32.const 16)
+                (i32.const 4096)
+                ;; The location at which to start reading entries in the
+                ;; directory; here we start at the first entry.
+                (i64.const 0)
+                ;; The address at which to store the number of bytes read.
+                (i32.const 8))
+            (drop)
+
+            ;; Check that we indeed read at least 380 bytes of directory
+            ;; entries.
+            (if (i32.lt_u (i32.load (i32.const 8)) (i32.const 300))
+               (then unreachable))
+
+            ;; Continue looping until $i reaches $iters.
+            (local.set $i (i64.add (local.get $i) (i64.const 1)))
+            (br_if $cont (i64.lt_u (local.get $i) (local.get $iters)))
+        )
+        (local.get $i)
+    )
+    (memory (export "memory") 1)
+)
diff --git a/benches/wasi/read-file.wat b/benches/wasi/read-file.wat
new file mode 100644
index 000000000..9e3d2bb5b
--- /dev/null
+++ b/benches/wasi/read-file.wat
@@ -0,0 +1,78 @@
+;; Repeatedly read the contents of `test.bin`.
+(module
+    (import "wasi_snapshot_preview1" "path_open"
+        (func $__wasi_path_open (param i32 i32 i32 i32 i32 i64 i64 i32 i32) (result i32)))
+    (import "wasi_snapshot_preview1" "fd_read"
+        (func $__wasi_fd_read (param i32 i32 i32 i32) (result i32)))
+    (import "wasi_snapshot_preview1" "fd_close"
+        (func $__wasi_fd_close (param i32) (result i32)))
+    (func (export "run") (param $iters i64) (result i64)
+        (local $i i64)
+        (local.set $i (i64.const 0))
+
+        ;; Set up the iovec list; the memory usage for this module should be:
+        ;; - offset 0 => file name
+        ;; - offset 16 => the opened file descriptor
+        ;; - offset 24 => the number of read bytes
+        ;; - offset 32 => the iovec list
+        ;; - offset 48 => the first (and only) iovec buffer
+        (i32.store (i32.const 32) (i32.const 48))
+        (i32.store (i32.const 36) (i32.const 4096))
+
+        (loop $cont
+            ;; Open the file `test.bin` under the same directory as this WAT
+            ;; file; this assumes some prior set up of the preopens in
+            ;; `wasi.rs`. See https://github.com/WebAssembly/WASI/blob/d8da230b/phases/snapshot/witx/wasi_snapshot_preview1.witx#L346.
+            (call $__wasi_path_open
+                ;; The fd of the preopen under which to search for the file;
+                ;; the first three are the `std*` ones.
+                (i32.const 3)
+                ;; The lookup flags (i.e., whether to follow symlinks).
+                (i32.const 0)
+                ;; The path to the file under the initial fd.
+                (i32.const 0)
+                (i32.const 8)
+                ;; The open flags; in this case we will only attempt to read but
+                ;; this may attempt to create the file if it does not exist, see
+                ;; https://github.com/WebAssembly/WASI/blob/d8da230b/phases/snapshot/witxtypenames.witx#L444).
+                (i32.const 0)
+                ;; The base rights and the inheriting rights: here we only set
+                ;; the bits for the FD_READ and FD_READDIR capabilities.
+                (i64.const 0x2002)
+                (i64.const 0x2002)
+                ;; The file descriptor flags (e.g., whether to append, sync,
+                ;; etc.); see https://github.com/WebAssembly/WASI/blob/d8da230b/phases/snapshot/witx/typenames.witx#L385
+                (i32.const 0)
+                ;; The address at which to store the opened fd (if the call
+                ;; succeeds)
+                (i32.const 16))
+            (if (then unreachable))
+
+            ;; Read the file into the sole iovec buffer.
+            (call $__wasi_fd_read
+                ;; The now-open fd stored at offset 16.
+                (i32.load (i32.const 16))
+                ;; The address and size of the list of iovecs; here we only use
+                ;; a list of a single iovec set up outside the loop.
+                (i32.const 32)
+                (i32.const 1)
+                ;; The address at which to store the number of bytes read.
+                (i32.const 24))
+            (if (then unreachable))
+            ;; Check that we indeed read 4096 bytes.
+            (if (i32.ne (i32.load (i32.const 24)) (i32.const 4096))
+                (then unreachable))
+
+            ;; Close the open file handle we stored at offset 16.
+            (call $__wasi_fd_close (i32.load (i32.const 16)))
+            (if (then unreachable))
+
+            ;; Continue looping until $i reaches $iters.
+            (local.set $i (i64.add (local.get $i) (i64.const 1)))
+            (br_if $cont (i64.lt_u (local.get $i) (local.get $iters)))
+        )
+        (local.get $i)
+    )
+    (data (i32.const 0) "test.bin")
+    (memory (export "memory") 1)
+)
diff --git a/cranelift/codegen/meta/src/shared/formats.rs b/cranelift/codegen/meta/src/shared/formats.rs
index 8c54c1916..e9d517561 100644
--- a/cranelift/codegen/meta/src/shared/formats.rs
+++ b/cranelift/codegen/meta/src/shared/formats.rs
@@ -16,6 +16,8 @@ pub(crate) struct Formats {
     pub(crate) float_compare: Rc<InstructionFormat>,
     pub(crate) func_addr: Rc<InstructionFormat>,
     pub(crate) heap_addr: Rc<InstructionFormat>,
+    pub(crate) heap_load: Rc<InstructionFormat>,
+    pub(crate) heap_store: Rc<InstructionFormat>,
     pub(crate) int_compare: Rc<InstructionFormat>,
     pub(crate) int_compare_imm: Rc<InstructionFormat>,
     pub(crate) int_add_trap: Rc<InstructionFormat>,
@@ -206,6 +208,17 @@ impl Formats {
                 .imm_with_name("size", &imm.uimm8)
                 .build(),
 
+            heap_load: Builder::new("HeapLoad").imm(&imm.heap_imm).value().build(),
+
+            heap_store: Builder::new("HeapStore")
+                // We have more fields for this instruction than
+                // `InstructionData` can hold without growing in size, so we
+                // push the immediates out into a side table.
+                .imm(&imm.heap_imm)
+                .value()
+                .value()
+                .build(),
+
             // Accessing a WebAssembly table.
             table_addr: Builder::new("TableAddr")
                 .imm(&entities.table)
diff --git a/cranelift/codegen/meta/src/shared/immediates.rs b/cranelift/codegen/meta/src/shared/immediates.rs
index 6e1e93347..9e9cdb196 100644
--- a/cranelift/codegen/meta/src/shared/immediates.rs
+++ b/cranelift/codegen/meta/src/shared/immediates.rs
@@ -59,6 +59,9 @@ pub(crate) struct Immediates {
     /// Flags for memory operations like `load` and `store`.
     pub memflags: OperandKind,
 
+    /// A reference to out-of-line immediates for heap accesses.
+    pub heap_imm: OperandKind,
+
     /// A trap code indicating the reason for trapping.
     ///
     /// The Rust enum type also has a `User(u16)` variant for user-provided trap codes.
@@ -182,6 +185,13 @@ impl Immediates {
             },
 
             memflags: new_imm("flags", "ir::MemFlags", "Memory operation flags"),
+
+            heap_imm: new_imm(
+                "heap_imm",
+                "ir::HeapImm",
+                "Reference to out-of-line heap access immediates",
+            ),
+
             trapcode: {
                 let mut trapcode_values = HashMap::new();
                 trapcode_values.insert("stk_ovf", "StackOverflow");
diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs
index f6aad71b7..91a3099f5 100755
--- a/cranelift/codegen/meta/src/shared/instructions.rs
+++ b/cranelift/codegen/meta/src/shared/instructions.rs
@@ -1155,6 +1155,55 @@ pub(crate) fn define(
         .operands_out(vec![addr]),
     );
 
+    let heap_imm = &Operand::new("heap_imm", &imm.heap_imm);
+    let index =
+        &Operand::new("index", HeapOffset).with_doc("Dynamic index (in bytes) into the heap");
+    let a = &Operand::new("a", Mem).with_doc("The value loaded from the heap");
+
+    ig.push(
+        Inst::new(
+            "heap_load",
+            r#"
+            Load a value from the given heap at address ``index + offset``,
+            trapping on out-of-bounds accesses.
+
+            Checks that ``index + offset .. index + offset + sizeof(a)`` is
+            within the heap's bounds, trapping if it is not. Otherwise, when
+            that range is in bounds, loads the value from the heap.
+
+            Traps on ``index + offset + sizeof(a)`` overflow.
+            "#,
+            &formats.heap_load,
+        )
+        .operands_in(vec![heap_imm, index])
+        .operands_out(vec![a])
+        .can_load(true)
+        .can_trap(true),
+    );
+
+    let a = &Operand::new("a", Mem).with_doc("The value stored into the heap");
+
+    ig.push(
+        Inst::new(
+            "heap_store",
+            r#"
+            Store ``a`` into the given heap at address ``index + offset``,
+            trapping on out-of-bounds accesses.
+
+            Checks that ``index + offset .. index + offset + sizeof(a)`` is
+            within the heap's bounds, trapping if it is not. Otherwise, when
+            that range is in bounds, stores the value into the heap.
+
+            Traps on ``index + offset + sizeof(a)`` overflow.
+            "#,
+            &formats.heap_store,
+        )
+        .operands_in(vec![heap_imm, index, a])
+        .operands_out(vec![])
+        .can_store(true)
+        .can_trap(true),
+    );
+
     // Note this instruction is marked as having other side-effects, so GVN won't try to hoist it,
     // which would result in it being subject to spilling. While not hoisting would generally hurt
     // performance, since a computed value used many times may need to be regenerated before each
diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs
index e0818df11..48eb4fa91 100644
--- a/cranelift/codegen/src/ir/dfg.rs
+++ b/cranelift/codegen/src/ir/dfg.rs
@@ -4,11 +4,12 @@ use crate::entity::{self, PrimaryMap, SecondaryMap};
 use crate::ir;
 use crate::ir::builder::ReplaceBuilder;
 use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes};
+use crate::ir::immediates::HeapImmData;
 use crate::ir::instructions::{BranchInfo, CallInfo, InstructionData};
 use crate::ir::{types, ConstantData, ConstantPool, Immediate};
 use crate::ir::{
-    Block, DynamicType, FuncRef, Inst, SigRef, Signature, Type, Value, ValueLabelAssignments,
-    ValueList, ValueListPool,
+    Block, DynamicType, FuncRef, HeapImm, Inst, SigRef, Signature, Type, Value,
+    ValueLabelAssignments, ValueList, ValueListPool,
 };
 use crate::ir::{ExtFuncData, RelSourceLoc};
 use crate::packed_option::ReservedValue;
@@ -83,6 +84,9 @@ pub struct DataFlowGraph {
 
     /// Stores large immediates that otherwise will not fit on InstructionData
     pub immediates: PrimaryMap<Immediate, ConstantData>,
+
+    /// Out-of-line heap access immediates that don't fit in `InstructionData`.
+    pub heap_imms: PrimaryMap<HeapImm, HeapImmData>,
 }
 
 impl DataFlowGraph {
@@ -101,6 +105,7 @@ impl DataFlowGraph {
             values_labels: None,
             constants: ConstantPool::new(),
             immediates: PrimaryMap::new(),
+            heap_imms: PrimaryMap::new(),
         }
     }
 
diff --git a/cranelift/codegen/src/ir/entities.rs b/cranelift/codegen/src/ir/entities.rs
index 315ea2183..dc63d0ec3 100644
--- a/cranelift/codegen/src/ir/entities.rs
+++ b/cranelift/codegen/src/ir/entities.rs
@@ -394,6 +394,34 @@ impl Heap {
     }
 }
 
+/// An opaque reference to some out-of-line immediates for `heap_{load,store}`
+/// instructions.
+///
+/// These immediates are too large to store in
+/// [`InstructionData`](super::instructions::InstructionData) and therefore must
+/// be tracked separately in
+/// [`DataFlowGraph::heap_imms`](super::dfg::DataFlowGraph). `HeapImm` provides
+/// a way to reference values stored there.
+///
+/// While the order is stable, it is arbitrary.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub struct HeapImm(u32);
+entity_impl!(HeapImm, "heap_imm");
+
+impl HeapImm {
+    /// Create a new `HeapImm` reference from its number.
+    ///
+    /// This method is for use by the parser.
+    pub fn with_number(n: u32) -> Option<Self> {
+        if n < u32::MAX {
+            Some(Self(n))
+        } else {
+            None
+        }
+    }
+}
+
 /// An opaque reference to a [WebAssembly
 /// table](https://developer.mozilla.org/en-US/docs/WebAssembly/Understanding_the_text_format#WebAssembly_tables).
 ///
diff --git a/cranelift/codegen/src/ir/immediates.rs b/cranelift/codegen/src/ir/immediates.rs
index 3b3f70323..54f737b19 100644
--- a/cranelift/codegen/src/ir/immediates.rs
+++ b/cranelift/codegen/src/ir/immediates.rs
@@ -4,6 +4,7 @@
 //! Each type here should have a corresponding definition in the
 //! `cranelift-codegen/meta/src/shared/immediates` crate in the meta language.
 
+use crate::ir;
 use alloc::vec::Vec;
 use core::cmp::Ordering;
 use core::convert::TryFrom;
@@ -1177,6 +1178,18 @@ impl Not for Ieee64 {
     }
 }
 
+/// Out-of-line heap access immediates.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub struct HeapImmData {
+    /// The memory flags for the heap access.
+    pub flags: ir::MemFlags,
+    /// The heap being accessed.
+    pub heap: ir::Heap,
+    /// The static offset added to the heap access's index.
+    pub offset: Uimm32,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs
index 52a4b6037..23f952738 100644
--- a/cranelift/codegen/src/ir/mod.rs
+++ b/cranelift/codegen/src/ir/mod.rs
@@ -37,8 +37,8 @@ pub use crate::ir::constant::{ConstantData, ConstantPool};
 pub use crate::ir::dfg::{DataFlowGraph, ValueDef};
 pub use crate::ir::dynamic_type::{dynamic_to_fixed, DynamicTypeData, DynamicTypes};
 pub use crate::ir::entities::{
-    Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, Immediate, Inst,
-    JumpTable, SigRef, StackSlot, Table, UserExternalNameRef, Value,
+    Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Heap, HeapImm, Immediate,
+    Inst, JumpTable, SigRef, StackSlot, Table, UserExternalNameRef, Value,
 };
 pub use crate::ir::extfunc::{
     AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature,
diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
index 1fcf7446b..433db8407 100644
--- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
@@ -139,8 +139,8 @@ pub(crate) fn lower_insn_to_regs(
             panic!("Direct stack memory access not supported; should not be used by Wasm");
         }
 
-        Opcode::HeapAddr => {
-            panic!("heap_addr should have been removed by legalization!");
+        Opcode::HeapLoad | Opcode::HeapStore | Opcode::HeapAddr => {
+            panic!("heap access instructions should have been removed by legalization!");
         }
 
         Opcode::TableAddr => {
diff --git a/cranelift/codegen/src/isa/s390x/lower.rs b/cranelift/codegen/src/isa/s390x/lower.rs
index cfc73c7f6..79765d017 100644
--- a/cranelift/codegen/src/isa/s390x/lower.rs
+++ b/cranelift/codegen/src/isa/s390x/lower.rs
@@ -212,8 +212,8 @@ impl LowerBackend for S390xBackend {
             Opcode::StackLoad | Opcode::StackStore => {
                 panic!("Direct stack memory access not supported; should not be used by Wasm");
             }
-            Opcode::HeapAddr => {
-                panic!("heap_addr should have been removed by legalization!");
+            Opcode::HeapLoad | Opcode::HeapStore | Opcode::HeapAddr => {
+                panic!("heap access instructions should have been removed by legalization!");
             }
             Opcode::TableAddr => {
                 panic!("table_addr should have been removed by legalization!");
diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs
index da0fb6efa..f5ea23c0a 100644
--- a/cranelift/codegen/src/isa/x64/lower.rs
+++ b/cranelift/codegen/src/isa/x64/lower.rs
@@ -549,8 +549,8 @@ fn lower_insn_to_regs(
             panic!("global_value should have been removed by legalization!");
         }
 
-        Opcode::HeapAddr => {
-            panic!("heap_addr should have been removed by legalization!");
+        Opcode::HeapLoad | Opcode::HeapStore | Opcode::HeapAddr => {
+            panic!("heap access instructions should have been removed by legalization!");
         }
 
         Opcode::TableAddr => {
diff --git a/cranelift/codegen/src/opts.rs b/cranelift/codegen/src/opts.rs
index a64f32c0d..7e77370e2 100644
--- a/cranelift/codegen/src/opts.rs
+++ b/cranelift/codegen/src/opts.rs
@@ -9,8 +9,8 @@ pub use crate::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, Uimm32, Uimm64,
 pub use crate::ir::types::*;
 pub use crate::ir::{
     dynamic_to_fixed, AtomicRmwOp, Block, Constant, DynamicStackSlot, FuncRef, GlobalValue, Heap,
-    Immediate, InstructionImms, JumpTable, MemFlags, Opcode, StackSlot, Table, TrapCode, Type,
-    Value,
+    HeapImm, Immediate, InstructionImms, JumpTable, MemFlags, Opcode, StackSlot, Table, TrapCode,
+    Type, Value,
 };
 use crate::isle_common_prelude_methods;
 use crate::machinst::isle::*;
diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs
index d47817e09..535fe0d99 100644
--- a/cranelift/codegen/src/verifier/mod.rs
+++ b/cranelift/codegen/src/verifier/mod.rs
@@ -64,6 +64,7 @@ use crate::entity::SparseSet;
 use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
 use crate::ir;
 use crate::ir::entities::AnyEntity;
+use crate::ir::immediates::HeapImmData;
 use crate::ir::instructions::{BranchInfo, CallInfo, InstructionFormat, ResolvedConstraint};
 use crate::ir::{
     types, ArgumentPurpose, Block, Constant, DynamicStackSlot, FuncRef, Function, GlobalValue,
@@ -678,6 +679,10 @@ impl<'a> Verifier<'a> {
             UnaryGlobalValue { global_value, .. } => {
                 self.verify_global_value(inst, global_value, errors)?;
             }
+            HeapLoad { heap_imm, .. } | HeapStore { heap_imm, .. } => {
+                let HeapImmData { heap, .. } = self.func.dfg.heap_imms[heap_imm];
+                self.verify_heap(inst, heap, errors)?;
+            }
             HeapAddr { heap, .. } => {
                 self.verify_heap(inst, heap, errors)?;
             }
diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs
index 725ca46c4..58d73e4ba 100644
--- a/cranelift/codegen/src/write.rs
+++ b/cranelift/codegen/src/write.rs
@@ -5,6 +5,7 @@
 
 use crate::entity::SecondaryMap;
 use crate::ir::entities::AnyEntity;
+use crate::ir::immediates::{HeapImmData, Uimm32};
 use crate::ir::{Block, DataFlowGraph, Function, Inst, SigRef, Type, Value, ValueDef};
 use crate::packed_option::ReservedValue;
 use alloc::string::{String, ToString};
@@ -476,6 +477,47 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
             dynamic_stack_slot,
             ..
         } => write!(w, " {}, {}", arg, dynamic_stack_slot),
+        HeapLoad {
+            opcode: _,
+            heap_imm,
+            arg,
+        } => {
+            let HeapImmData {
+                flags,
+                heap,
+                offset,
+            } = dfg.heap_imms[heap_imm];
+            write!(
+                w,
+                " {heap} {flags} {arg}{optional_offset}",
+                optional_offset = if offset == Uimm32::from(0) {
+                    "".to_string()
+                } else {
+                    format!("+{offset}")
+                }
+            )
+        }
+        HeapStore {
+            opcode: _,
+            heap_imm,
+            args,
+        } => {
+            let HeapImmData {
+                flags,
+                heap,
+                offset,
+            } = dfg.heap_imms[heap_imm];
+            let [index, value] = args;
+            write!(
+                w,
+                " {heap} {flags} {index}{optional_offset}, {value}",
+                optional_offset = if offset == Uimm32::from(0) {
+                    "".to_string()
+                } else {
+                    format!("+{offset}")
+                }
+            )
+        }
         HeapAddr {
             heap,
             arg,
diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs
index 15c9d9e28..12c268322 100644
--- a/cranelift/interpreter/src/step.rs
+++ b/cranelift/interpreter/src/step.rs
@@ -6,6 +6,7 @@ use crate::state::{MemoryError, State};
 use crate::value::{Value, ValueConversionKind, ValueError, ValueResult};
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
+use cranelift_codegen::ir::immediates::HeapImmData;
 use cranelift_codegen::ir::{
     types, AbiParam, Block, ExternalName, FuncRef, Function, InstructionData, Opcode, TrapCode,
     Type, Value as ValueRef,
@@ -484,6 +485,59 @@ where
         }
         Opcode::SymbolValue => unimplemented!("SymbolValue"),
         Opcode::TlsValue => unimplemented!("TlsValue"),
+        Opcode::HeapLoad => {
+            if let InstructionData::HeapLoad { heap_imm, arg, .. } = inst {
+                let HeapImmData {
+                    flags,
+                    heap,
+                    offset,
+                } = state.get_current_function().dfg.heap_imms[heap_imm];
+                let offset = i128::from(u32::from(offset));
+                let addr_ty = state.get_current_function().dfg.value_type(arg);
+                let index = state.get_value(arg).unwrap().into_int().unwrap();
+                let load_ty = inst_context.controlling_type().unwrap();
+                let load_size = i128::from(load_ty.bytes());
+                assign_or_memtrap(AddressSize::try_from(addr_ty).and_then(|addr_size| {
+                    let heap_address = index.saturating_add(offset).saturating_add(load_size);
+                    let heap_address =
+                        u64::try_from(heap_address).map_err(|e| MemoryError::OutOfBoundsLoad {
+                            addr: Address::try_from(0).unwrap(),
+                            load_size: load_size as usize,
+                        })?;
+                    let address = state.heap_address(addr_size, heap, heap_address)?;
+                    state.checked_load(address, load_ty)
+                }))
+            } else {
+                unreachable!()
+            }
+        }
+        Opcode::HeapStore => {
+            if let InstructionData::HeapStore { heap_imm, args, .. } = inst {
+                let HeapImmData {
+                    flags,
+                    heap,
+                    offset,
+                } = state.get_current_function().dfg.heap_imms[heap_imm];
+                let offset = i128::from(u32::from(offset));
+                let addr_ty = state.get_current_function().dfg.value_type(args[0]);
+                let index = state.get_value(args[0]).unwrap().into_int().unwrap();
+                let value = state.get_value(args[1]).unwrap();
+                let store_ty = state.get_current_function().dfg.value_type(args[1]);
+                let store_size = i128::from(store_ty.bytes());
+                continue_or_memtrap(AddressSize::try_from(addr_ty).and_then(|addr_size| {
+                    let heap_address = index.saturating_add(offset).saturating_add(store_size);
+                    let heap_address =
+                        u64::try_from(heap_address).map_err(|e| MemoryError::OutOfBoundsStore {
+                            addr: Address::try_from(0).unwrap(),
+                            store_size: store_size as usize,
+                        })?;
+                    let address = state.heap_address(addr_size, heap, heap_address)?;
+                    state.checked_store(address, value)
+                }))
+            } else {
+                unreachable!()
+            }
+        }
         Opcode::HeapAddr => {
             if let InstructionData::HeapAddr {
                 heap,
diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs
index ff85acec7..baf0bc0a9 100644
--- a/cranelift/reader/src/parser.rs
+++ b/cranelift/reader/src/parser.rs
@@ -11,7 +11,9 @@ use crate::testfile::{Comment, Details, Feature, TestFile};
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::entity::{EntityRef, PrimaryMap};
 use cranelift_codegen::ir::entities::{AnyEntity, DynamicType};
-use cranelift_codegen::ir::immediates::{Ieee32, Ieee64, Imm64, Offset32, Uimm32, Uimm64};
+use cranelift_codegen::ir::immediates::{
+    HeapImmData, Ieee32, Ieee64, Imm64, Offset32, Uimm32, Uimm64,
+};
 use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs};
 use cranelift_codegen::ir::types::INVALID;
 use cranelift_codegen::ir::types::*;
@@ -876,6 +878,19 @@ impl<'a> Parser<'a> {
         }
     }
 
+    // Match and consume an optional uimm32 offset immediate.
+    //
+    // Note that this will match an empty string as an empty offset, and that if an offset is
+    // present, it must contain a plus sign.
+    fn optional_uimm32_offset(&mut self) -> ParseResult<Uimm32> {
+        match self.token() {
+            Some(Token::Integer(x)) if x.starts_with('+') => {
+                self.match_uimm32("expected a uimm32 offset immediate")
+            }
+            _ => Ok(Uimm32::from(0)),
+        }
+    }
+
     // Match and consume a u8 immediate.
     // This is used for lane numbers in SIMD vectors.
     fn match_uimm8(&mut self, err_msg: &str) -> ParseResult<u8> {
@@ -2976,6 +2991,42 @@ impl<'a> Parser<'a> {
                     size,
                 }
             }
+            InstructionFormat::HeapLoad => {
+                let heap = self.match_heap("expected heap identifier")?;
+                ctx.check_heap(heap, self.loc)?;
+                let flags = self.optional_memflags();
+                let arg = self.match_value("expected SSA value heap index")?;
+                let offset = self.optional_uimm32_offset()?;
+                let heap_imm = ctx.function.dfg.heap_imms.push(HeapImmData {
+                    flags,
+                    heap,
+                    offset,
+                });
+                InstructionData::HeapLoad {
+                    opcode,
+                    heap_imm,
+                    arg,
+                }
+            }
+            InstructionFormat::HeapStore => {
+                let heap = self.match_heap("expected heap identifier")?;
+                ctx.check_heap(heap, self.loc)?;
+                let flags = self.optional_memflags();
+                let index = self.match_value("expected SSA value heap index")?;
+                let offset = self.optional_uimm32_offset()?;
+                self.match_token(Token::Comma, "expected ',' between operands")?;
+                let value = self.match_value("expected SSA value to store")?;
+                let heap_imm = ctx.function.dfg.heap_imms.push(HeapImmData {
+                    flags,
+                    heap,
+                    offset,
+                });
+                InstructionData::HeapStore {
+                    opcode,
+                    heap_imm,
+                    args: [index, value],
+                }
+            }
             InstructionFormat::TableAddr => {
                 let table = self.match_table("expected table identifier")?;
                 ctx.check_table(table, self.loc)?;
diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs
index e8d9ed929..9e4f88ecb 100644
--- a/crates/runtime/src/instance.rs
+++ b/crates/runtime/src/instance.rs
@@ -966,8 +966,8 @@ impl Instance {
                 let def_ptr = self.memories[defined_memory_index]
                     .as_shared_memory()
                     .unwrap()
-                    .vmmemory_ptr_mut();
-                ptr::write(ptr, def_ptr);
+                    .vmmemory_ptr();
+                ptr::write(ptr, def_ptr.cast_mut());
             } else {
                 ptr::write(owned_ptr, self.memories[defined_memory_index].vmmemory());
                 ptr::write(ptr, owned_ptr);
diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs
index 325e34740..2c1db36ad 100644
--- a/crates/runtime/src/lib.rs
+++ b/crates/runtime/src/lib.rs
@@ -231,3 +231,17 @@ pub fn page_size() -> usize {
         unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }
     }
 }
+
+/// Result of [`Memory::atomic_wait32`] and [`Memory::atomic_wait64`]
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum WaitResult {
+    /// Indicates that a `wait` completed by being awoken by a different thread.
+    /// This means the thread went to sleep and didn't time out.
+    Ok = 0,
+    /// Indicates that `wait` did not complete and instead returned due to the
+    /// value in memory not matching the expected value.
+    Mismatch = 1,
+    /// Indicates that `wait` completed with a timeout, meaning that the
+    /// original value matched as expected but nothing ever called `notify`.
+    TimedOut = 2,
+}
diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs
index 384e7f190..e6bebc25d 100644
--- a/crates/runtime/src/libcalls.rs
+++ b/crates/runtime/src/libcalls.rs
@@ -57,7 +57,7 @@
 use crate::externref::VMExternRef;
 use crate::table::{Table, TableElementType};
 use crate::vmcontext::{VMCallerCheckedAnyfunc, VMContext};
-use crate::{SharedMemory, TrapReason};
+use crate::TrapReason;
 use anyhow::Result;
 use std::mem;
 use std::ptr::{self, NonNull};
@@ -436,25 +436,12 @@ unsafe fn memory_atomic_notify(
     memory_index: u32,
     addr_index: u64,
     count: u32,
-) -> Result<u32, TrapReason> {
+) -> Result<u32, Trap> {
     let memory = MemoryIndex::from_u32(memory_index);
     let instance = (*vmctx).instance_mut();
     instance
-        .get_memory(memory)
-        .validate_addr(addr_index, 4, 4)?;
-
-    let shared_mem = instance.get_runtime_memory(memory).as_shared_memory();
-
-    if count == 0 {
-        return Ok(0);
-    }
-
-    let unparked_threads = shared_mem.map_or(0, |shared_mem| {
-        // SAFETY: checked `addr_index` above
-        unsafe { shared_mem.unchecked_atomic_notify(addr_index, count) }
-    });
-
-    Ok(unparked_threads)
+        .get_runtime_memory(memory)
+        .atomic_notify(addr_index, count)
 }
 
 // Implementation of `memory.atomic.wait32` for locally defined memories.
@@ -464,24 +451,14 @@ unsafe fn memory_atomic_wait32(
     addr_index: u64,
     expected: u32,
     timeout: u64,
-) -> Result<u32, TrapReason> {
+) -> Result<u32, Trap> {
     // convert timeout to Instant, before any wait happens on locking
     let timeout = (timeout as i64 >= 0).then(|| Instant::now() + Duration::from_nanos(timeout));
-
     let memory = MemoryIndex::from_u32(memory_index);
     let instance = (*vmctx).instance_mut();
-    let addr = instance
-        .get_memory(memory)
-        .validate_addr(addr_index, 4, 4)?;
-
-    let shared_mem: SharedMemory = instance
+    Ok(instance
         .get_runtime_memory(memory)
-        .as_shared_memory()
-        .ok_or(Trap::AtomicWaitNonSharedMemory)?;
-
-    // SAFETY: checked `addr_index` above
-    let res = unsafe { shared_mem.unchecked_atomic_wait32(addr_index, addr, expected, timeout) };
-    Ok(res)
+        .atomic_wait32(addr_index, expected, timeout)? as u32)
 }
 
 // Implementation of `memory.atomic.wait64` for locally defined memories.
@@ -491,24 +468,14 @@ unsafe fn memory_atomic_wait64(
     addr_index: u64,
     expected: u64,
     timeout: u64,
-) -> Result<u32, TrapReason> {
+) -> Result<u32, Trap> {
     // convert timeout to Instant, before any wait happens on locking
     let timeout = (timeout as i64 >= 0).then(|| Instant::now() + Duration::from_nanos(timeout));
-
     let memory = MemoryIndex::from_u32(memory_index);
     let instance = (*vmctx).instance_mut();
-    let addr = instance
-        .get_memory(memory)
-        .validate_addr(addr_index, 8, 8)?;
-
-    let shared_mem: SharedMemory = instance
+    Ok(instance
         .get_runtime_memory(memory)
-        .as_shared_memory()
-        .ok_or(Trap::AtomicWaitNonSharedMemory)?;
-
-    // SAFETY: checked `addr_index` above
-    let res = unsafe { shared_mem.unchecked_atomic_wait64(addr_index, addr, expected, timeout) };
-    Ok(res)
+        .atomic_wait64(addr_index, expected, timeout)? as u32)
 }
 
 // Hook for when an instance runs out of fuel.
diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs
index 5ba8bc52e..3cbcd125d 100644
--- a/crates/runtime/src/memory.rs
+++ b/crates/runtime/src/memory.rs
@@ -3,11 +3,9 @@
 //! `RuntimeLinearMemory` is to WebAssembly linear memories what `Table` is to WebAssembly tables.
 
 use crate::mmap::Mmap;
-use crate::parking_spot::{ParkResult, ParkingSpot};
+use crate::parking_spot::ParkingSpot;
 use crate::vmcontext::VMMemoryDefinition;
-use crate::MemoryImage;
-use crate::MemoryImageSlot;
-use crate::Store;
+use crate::{MemoryImage, MemoryImageSlot, Store, WaitResult};
 use anyhow::Error;
 use anyhow::{bail, format_err, Result};
 use std::convert::TryFrom;
@@ -152,8 +150,7 @@ pub trait RuntimeLinearMemory: Send + Sync {
     /// `RuntimeMemoryCreator::new_memory()`.
     fn needs_init(&self) -> bool;
 
-    /// For the pooling allocator, we must be able to downcast this trait to its
-    /// underlying structure.
+    /// Used for optional dynamic downcasting.
     fn as_any_mut(&mut self) -> &mut dyn std::any::Any;
 }
 
@@ -436,6 +433,13 @@ impl RuntimeLinearMemory for StaticMemory {
 #[derive(Clone)]
 pub struct SharedMemory(Arc<SharedMemoryInner>);
 
+struct SharedMemoryInner {
+    memory: RwLock<Box<dyn RuntimeLinearMemory>>,
+    spot: ParkingSpot,
+    ty: wasmtime_environ::Memory,
+    def: LongTermVMMemoryDefinition,
+}
+
 impl SharedMemory {
     /// Construct a new [`SharedMemory`].
     pub fn new(plan: MemoryPlan) -> Result<Self> {
@@ -460,15 +464,17 @@ impl SharedMemory {
             memory.as_any_mut().type_id() != std::any::TypeId::of::<SharedMemory>(),
             "cannot re-wrap a shared memory"
         );
-        let def = LongTermVMMemoryDefinition(memory.vmmemory());
-        let spot = ParkingSpot::default();
-        let inner = RwLock::new(SharedMemoryInnerRwLocked { memory, ty, def });
-        Ok(Self(Arc::new(SharedMemoryInner { spot, lock: inner })))
+        Ok(Self(Arc::new(SharedMemoryInner {
+            ty,
+            spot: ParkingSpot::default(),
+            def: LongTermVMMemoryDefinition(memory.vmmemory()),
+            memory: RwLock::new(memory),
+        })))
     }
 
     /// Return the memory type for this [`SharedMemory`].
     pub fn ty(&self) -> wasmtime_environ::Memory {
-        self.0.lock.read().unwrap().ty
+        self.0.ty
     }
 
     /// Convert this shared memory into a [`Memory`].
@@ -476,33 +482,52 @@ impl SharedMemory {
         Memory(Box::new(self))
     }
 
-    /// Return a mutable pointer to the shared memory's [VMMemoryDefinition].
-    pub fn vmmemory_ptr_mut(&mut self) -> *mut VMMemoryDefinition {
-        &self.0.lock.read().unwrap().def.0 as *const _ as *mut _
-    }
-
     /// Return a pointer to the shared memory's [VMMemoryDefinition].
     pub fn vmmemory_ptr(&self) -> *const VMMemoryDefinition {
-        &self.0.lock.read().unwrap().def.0 as *const _
+        &self.0.def.0
     }
 
-    /// Implementation of `memory.atomic.notify` for this shared memory.
-    pub fn atomic_notify(&self, addr_index: u64, count: u32) -> Result<u32, Trap> {
-        let definition = &self.0.lock.read().unwrap().def.0;
-        definition.validate_addr(addr_index, 4, 4)?;
-        // SAFETY: checked `addr_index` above
-        Ok(unsafe { self.unchecked_atomic_notify(addr_index, count) })
+    /// Same as `RuntimeLinearMemory::grow`, except with `&self`.
+    pub fn grow(
+        &self,
+        delta_pages: u64,
+        store: Option<&mut dyn Store>,
+    ) -> Result<Option<(usize, usize)>, Error> {
+        let mut memory = self.0.memory.write().unwrap();
+        let result = memory.grow(delta_pages, store)?;
+        if let Some((_old_size_in_bytes, new_size_in_bytes)) = result {
+            // Store the new size to the `VMMemoryDefinition` for JIT-generated
+            // code (and runtime functions) to access. No other code can be
+            // growing this memory due to the write lock, but code in other
+            // threads could have access to this shared memory and we want them
+            // to see the most consistent version of the `current_length`; a
+            // weaker consistency is possible if we accept them seeing an older,
+            // smaller memory size (assumption: memory only grows) but presently
+            // we are aiming for accuracy.
+            //
+            // Note that it could be possible to access a memory address that is
+            // now-valid due to changes to the page flags in `grow` above but
+            // beyond the `memory.size` that we are about to assign to. In these
+            // and similar cases, discussion in the thread proposal concluded
+            // that: "multiple accesses in one thread racing with another
+            // thread's `memory.grow` that are in-bounds only after the grow
+            // commits may independently succeed or trap" (see
+            // https://github.com/WebAssembly/threads/issues/26#issuecomment-433930711).
+            // In other words, some non-determinism is acceptable when using
+            // `memory.size` on work being done by `memory.grow`.
+            self.0
+                .def
+                .0
+                .current_length
+                .store(new_size_in_bytes, Ordering::SeqCst);
+        }
+        Ok(result)
     }
 
-    /// Unchecked implementation of `memory.atomic.notify` for this shared memory.
-    ///
-    /// # Safety
-    /// The caller must ensure that `addr_index` is a valid address in the memory.
-    pub unsafe fn unchecked_atomic_notify(&self, addr_index: u64, count: u32) -> u32 {
-        if count == 0 {
-            return 0;
-        }
-        self.0.spot.unpark(addr_index, count)
+    /// Implementation of `memory.atomic.notify` for this shared memory.
+    pub fn atomic_notify(&self, addr_index: u64, count: u32) -> Result<u32, Trap> {
+        validate_atomic_addr(&self.0.def.0, addr_index, 4, 4)?;
+        Ok(self.0.spot.unpark(addr_index, count))
     }
 
     /// Implementation of `memory.atomic.wait32` for this shared memory.
@@ -511,36 +536,18 @@ impl SharedMemory {
         addr_index: u64,
         expected: u32,
         timeout: Option<Instant>,
-    ) -> Result<u32, Trap> {
-        let definition = &self.0.lock.read().unwrap().def.0;
-        let addr = definition.validate_addr(addr_index, 4, 4)?;
-        // SAFETY: checked `addr` above
-        Ok(unsafe { self.unchecked_atomic_wait32(addr_index, addr, expected, timeout) })
-    }
-
-    /// Unchecked implementation of `memory.atomic.wait32` for this shared memory.
-    ///
-    /// # Safety
-    /// The caller must ensure that `addr` is a valid address in the memory.
-    pub unsafe fn unchecked_atomic_wait32(
-        &self,
-        addr_index: u64,
-        addr: *const u8,
-        expected: u32,
-        timeout: Option<Instant>,
-    ) -> u32 {
-        // SAFETY: `addr_index` was validated by `validate_addr` above.
+    ) -> Result<WaitResult, Trap> {
+        let addr = validate_atomic_addr(&self.0.def.0, addr_index, 4, 4)?;
+        // SAFETY: `addr_index` was validated by `validate_atomic_addr` above.
+        assert!(std::mem::size_of::<AtomicU32>() == 4);
+        assert!(std::mem::align_of::<AtomicU32>() <= 4);
         let atomic = unsafe { &*(addr as *const AtomicU32) };
 
         // We want the sequential consistency of `SeqCst` to ensure that the `load` sees the value that the `notify` will/would see.
         // All WASM atomic operations are also `SeqCst`.
         let validate = || atomic.load(Ordering::SeqCst) == expected;
 
-        match self.0.spot.park(addr_index, validate, timeout) {
-            ParkResult::Unparked => 0,
-            ParkResult::Invalid => 1,
-            ParkResult::TimedOut => 2,
-        }
+        Ok(self.0.spot.park(addr_index, validate, timeout))
     }
 
     /// Implementation of `memory.atomic.wait64` for this shared memory.
@@ -549,49 +556,21 @@ impl SharedMemory {
         addr_index: u64,
         expected: u64,
         timeout: Option<Instant>,
-    ) -> Result<u32, Trap> {
-        let definition = &self.0.lock.read().unwrap().def.0;
-        let addr = definition.validate_addr(addr_index, 8, 8)?;
-        // SAFETY: checked `addr` above
-        Ok(unsafe { self.unchecked_atomic_wait64(addr_index, addr, expected, timeout) })
-    }
-    /// Unchecked implementation of `memory.atomic.wait64` for this shared memory.
-    ///
-    /// # Safety
-    /// The caller must ensure that `addr` is a valid address in the memory.
-    pub unsafe fn unchecked_atomic_wait64(
-        &self,
-        addr_index: u64,
-        addr: *const u8,
-        expected: u64,
-        timeout: Option<Instant>,
-    ) -> u32 {
-        // SAFETY: `addr_index` was validated by `validate_addr` above.
+    ) -> Result<WaitResult, Trap> {
+        let addr = validate_atomic_addr(&self.0.def.0, addr_index, 8, 8)?;
+        // SAFETY: `addr_index` was validated by `validate_atomic_addr` above.
+        assert!(std::mem::size_of::<AtomicU64>() == 8);
+        assert!(std::mem::align_of::<AtomicU64>() <= 8);
         let atomic = unsafe { &*(addr as *const AtomicU64) };
 
         // We want the sequential consistency of `SeqCst` to ensure that the `load` sees the value that the `notify` will/would see.
         // All WASM atomic operations are also `SeqCst`.
         let validate = || atomic.load(Ordering::SeqCst) == expected;
 
-        match self.0.spot.park(addr_index, validate, timeout) {
-            ParkResult::Unparked => 0,
-            ParkResult::Invalid => 1,
-            ParkResult::TimedOut => 2,
-        }
+        Ok(self.0.spot.park(addr_index, validate, timeout))
     }
 }
 
-struct SharedMemoryInnerRwLocked {
-    memory: Box<dyn RuntimeLinearMemory>,
-    ty: wasmtime_environ::Memory,
-    def: LongTermVMMemoryDefinition,
-}
-
-struct SharedMemoryInner {
-    lock: RwLock<SharedMemoryInnerRwLocked>,
-    spot: ParkingSpot,
-}
-
 /// Shared memory needs some representation of a `VMMemoryDefinition` for
 /// JIT-generated code to access. This structure owns the base pointer and
 /// length to the actual memory and we share this definition across threads by:
@@ -607,11 +586,11 @@ unsafe impl Sync for LongTermVMMemoryDefinition {}
 /// Proxy all calls through the [`RwLock`].
 impl RuntimeLinearMemory for SharedMemory {
     fn byte_size(&self) -> usize {
-        self.0.lock.read().unwrap().memory.byte_size()
+        self.0.memory.read().unwrap().byte_size()
     }
 
     fn maximum_byte_size(&self) -> Option<usize> {
-        self.0.lock.read().unwrap().memory.maximum_byte_size()
+        self.0.memory.read().unwrap().maximum_byte_size()
     }
 
     fn grow(
@@ -619,39 +598,11 @@ impl RuntimeLinearMemory for SharedMemory {
         delta_pages: u64,
         store: Option<&mut dyn Store>,
     ) -> Result<Option<(usize, usize)>, Error> {
-        let mut inner = self.0.lock.write().unwrap();
-        let result = inner.memory.grow(delta_pages, store)?;
-        if let Some((_old_size_in_bytes, new_size_in_bytes)) = result {
-            // Store the new size to the `VMMemoryDefinition` for JIT-generated
-            // code (and runtime functions) to access. No other code can be
-            // growing this memory due to the write lock, but code in other
-            // threads could have access to this shared memory and we want them
-            // to see the most consistent version of the `current_length`; a
-            // weaker consistency is possible if we accept them seeing an older,
-            // smaller memory size (assumption: memory only grows) but presently
-            // we are aiming for accuracy.
-            //
-            // Note that it could be possible to access a memory address that is
-            // now-valid due to changes to the page flags in `grow` above but
-            // beyond the `memory.size` that we are about to assign to. In these
-            // and similar cases, discussion in the thread proposal concluded
-            // that: "multiple accesses in one thread racing with another
-            // thread's `memory.grow` that are in-bounds only after the grow
-            // commits may independently succeed or trap" (see
-            // https://github.com/WebAssembly/threads/issues/26#issuecomment-433930711).
-            // In other words, some non-determinism is acceptable when using
-            // `memory.size` on work being done by `memory.grow`.
-            inner
-                .def
-                .0
-                .current_length
-                .store(new_size_in_bytes, Ordering::SeqCst);
-        }
-        Ok(result)
+        SharedMemory::grow(self, delta_pages, store)
     }
 
     fn grow_to(&mut self, size: usize) -> Result<()> {
-        self.0.lock.write().unwrap().memory.grow_to(size)
+        self.0.memory.write().unwrap().grow_to(size)
     }
 
     fn vmmemory(&mut self) -> VMMemoryDefinition {
@@ -663,7 +614,7 @@ impl RuntimeLinearMemory for SharedMemory {
     }
 
     fn needs_init(&self) -> bool {
-        self.0.lock.read().unwrap().memory.needs_init()
+        self.0.memory.read().unwrap().needs_init()
     }
 
     fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
@@ -872,12 +823,78 @@ impl Memory {
 
     /// If the [Memory] is a [SharedMemory], unwrap it and return a clone to
     /// that shared memory.
-    pub fn as_shared_memory(&mut self) -> Option<SharedMemory> {
+    pub fn as_shared_memory(&mut self) -> Option<&mut SharedMemory> {
         let as_any = self.0.as_any_mut();
         if let Some(m) = as_any.downcast_mut::<SharedMemory>() {
-            Some(m.clone())
+            Some(m)
         } else {
             None
         }
     }
+
+    /// Implementation of `memory.atomic.notify` for all memories.
+    pub fn atomic_notify(&mut self, addr: u64, count: u32) -> Result<u32, Trap> {
+        match self.0.as_any_mut().downcast_mut::<SharedMemory>() {
+            Some(m) => m.atomic_notify(addr, count),
+            None => {
+                validate_atomic_addr(&self.vmmemory(), addr, 4, 4)?;
+                Ok(0)
+            }
+        }
+    }
+
+    /// Implementation of `memory.atomic.wait32` for all memories.
+    pub fn atomic_wait32(
+        &mut self,
+        addr: u64,
+        expected: u32,
+        deadline: Option<Instant>,
+    ) -> Result<WaitResult, Trap> {
+        match self.0.as_any_mut().downcast_mut::<SharedMemory>() {
+            Some(m) => m.atomic_wait32(addr, expected, deadline),
+            None => {
+                validate_atomic_addr(&self.vmmemory(), addr, 4, 4)?;
+                Err(Trap::AtomicWaitNonSharedMemory)
+            }
+        }
+    }
+
+    /// Implementation of `memory.atomic.wait64` for all memories.
+    pub fn atomic_wait64(
+        &mut self,
+        addr: u64,
+        expected: u64,
+        deadline: Option<Instant>,
+    ) -> Result<WaitResult, Trap> {
+        match self.0.as_any_mut().downcast_mut::<SharedMemory>() {
+            Some(m) => m.atomic_wait64(addr, expected, deadline),
+            None => {
+                validate_atomic_addr(&self.vmmemory(), addr, 8, 8)?;
+                Err(Trap::AtomicWaitNonSharedMemory)
+            }
+        }
+    }
+}
+
+/// In the configurations where bounds checks were elided in JIT code (because
+/// we are using static memories with virtual memory guard pages) this manual
+/// check is here so we don't segfault from Rust. For other configurations,
+/// these checks are required anyways.
+fn validate_atomic_addr(
+    def: &VMMemoryDefinition,
+    addr: u64,
+    access_size: u64,
+    access_alignment: u64,
+) -> Result<*mut u8, Trap> {
+    debug_assert!(access_alignment.is_power_of_two());
+    if !(addr % access_alignment == 0) {
+        return Err(Trap::HeapMisaligned);
+    }
+
+    let length = u64::try_from(def.current_length()).unwrap();
+    if !(addr.saturating_add(access_size) < length) {
+        return Err(Trap::MemoryOutOfBounds);
+    }
+
+    Ok(def.base.wrapping_add(addr as usize))
 }
diff --git a/crates/runtime/src/parking_spot.rs b/crates/runtime/src/parking_spot.rs
index a958d6d75..dee1019b2 100644
--- a/crates/runtime/src/parking_spot.rs
+++ b/crates/runtime/src/parking_spot.rs
@@ -14,23 +14,11 @@
 #![deny(missing_docs)]
 #![deny(unsafe_code)]
 
+use crate::WaitResult;
 use std::collections::BTreeMap;
 use std::sync::{Arc, Condvar, Mutex};
 use std::time::Instant;
 
-/// Result of a park operation.
-#[derive(Copy, Clone, Eq, PartialEq, Debug)]
-pub enum ParkResult {
-    /// Unparked by another thread.
-    Unparked,
-
-    /// The validation callback returned false.
-    Invalid,
-
-    /// The timeout expired.
-    TimedOut,
-}
-
 #[derive(Default, Debug)]
 struct Spot {
     /// The number of threads parked on this spot.
@@ -57,7 +45,7 @@ impl ParkingSpot {
     /// `unpark_all` or `unpark` with the same key, the current thread will be unparked.
     ///
     /// The `validate` callback is called before parking.
-    /// If it returns `false`, the thread is not parked and `ParkResult::Invalid` is returned.
+    /// If it returns `false`, the thread is not parked and `WaitResult::Mismatch` is returned.
     ///
     /// The `timeout` argument specifies the maximum amount of time the thread will be parked.
     pub fn park(
@@ -65,7 +53,7 @@ impl ParkingSpot {
         key: u64,
         validate: impl FnOnce() -> bool,
         timeout: impl Into<Option<Instant>>,
-    ) -> ParkResult {
+    ) -> WaitResult {
         self.park_inner(key, validate, timeout.into())
     }
 
@@ -74,7 +62,7 @@ impl ParkingSpot {
         key: u64,
         validate: impl FnOnce() -> bool,
         timeout: Option<Instant>,
-    ) -> ParkResult {
+    ) -> WaitResult {
         let mut inner = self
             .inner
             .lock()
@@ -82,7 +70,7 @@ impl ParkingSpot {
 
         // check validation with lock held
         if !validate() {
-            return ParkResult::Invalid;
+            return WaitResult::Mismatch;
         }
 
         // clone the condvar, so we can move the lock
@@ -115,7 +103,14 @@ impl ParkingSpot {
 
             let spot = inner.get_mut(&key).expect("failed to get spot");
 
-            if !timed_out {
+            if timed_out {
+                if let Some(timeout) = timeout {
+                    if Instant::now() < timeout {
+                        // Did not sleep long enough, try again.
+                        continue;
+                    }
+                }
+            } else {
                 if spot.to_unpark == 0 {
                     continue;
                 }
@@ -136,10 +131,10 @@ impl ParkingSpot {
             }
 
             if timed_out {
-                return ParkResult::TimedOut;
+                return WaitResult::TimedOut;
             }
 
-            return ParkResult::Unparked;
+            return WaitResult::Ok;
         }
     }
 
@@ -147,6 +142,9 @@ impl ParkingSpot {
     ///
     /// Returns the number of threads that were actually unparked.
     pub fn unpark(&self, key: u64, n: u32) -> u32 {
+        if n == 0 {
+            return 0;
+        }
         let mut num_unpark = 0;
 
         self.with_lot(key, |spot| {
diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs
index f954d70e0..ae15342ea 100644
--- a/crates/runtime/src/vmcontext.rs
+++ b/crates/runtime/src/vmcontext.rs
@@ -12,7 +12,7 @@ use std::ptr::NonNull;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::u32;
 pub use vm_host_func_context::VMHostFuncContext;
-use wasmtime_environ::{DefinedMemoryIndex, Trap};
+use wasmtime_environ::DefinedMemoryIndex;
 
 pub const VMCONTEXT_MAGIC: u32 = u32::from_le_bytes(*b"core");
 
@@ -248,30 +248,6 @@ impl VMMemoryDefinition {
             current_length: other.current_length().into(),
         }
     }
-
-    /// In the configurations where bounds checks were elided in JIT code (because
-    /// we are using static memories with virtual memory guard pages) this manual
-    /// check is here so we don't segfault from Rust. For other configurations,
-    /// these checks are required anyways.
-    pub fn validate_addr(
-        &self,
-        addr: u64,
-        access_size: u64,
-        access_alignment: u64,
-    ) -> Result<*const u8, Trap> {
-        debug_assert!(access_alignment.is_power_of_two());
-        if !(addr % access_alignment == 0) {
-            return Err(Trap::HeapMisaligned);
-        }
-
-        let length = u64::try_from(self.current_length()).unwrap();
-        if !(addr.saturating_add(access_size) < length) {
-            return Err(Trap::MemoryOutOfBounds);
-        }
-
-        // SAFETY: checked above that the address is in bounds
-        Ok(unsafe { self.base.add(addr as usize) })
-    }
 }
 
 #[cfg(test)]
diff --git a/crates/wasmtime/src/memory.rs b/crates/wasmtime/src/memory.rs
index 4d5cc788b..61d5ef3a2 100644
--- a/crates/wasmtime/src/memory.rs
+++ b/crates/wasmtime/src/memory.rs
@@ -1,13 +1,17 @@
 use crate::store::{StoreData, StoreOpaque, Stored};
 use crate::trampoline::generate_memory_export;
+use crate::Trap;
 use crate::{AsContext, AsContextMut, Engine, MemoryType, StoreContext, StoreContextMut};
 use anyhow::{bail, Result};
 use std::cell::UnsafeCell;
 use std::convert::TryFrom;
 use std::slice;
+use std::time::Instant;
 use wasmtime_environ::MemoryPlan;
 use wasmtime_runtime::{RuntimeLinearMemory, VMMemoryImport};
 
+pub use wasmtime_runtime::WaitResult;
+
 /// Error for out of bounds [`Memory`] access.
 #[derive(Debug)]
 #[non_exhaustive]
@@ -780,7 +784,7 @@ impl SharedMemory {
     /// the maximum limits of this memory. A
     /// [`ResourceLimiter`](crate::ResourceLimiter) is another example of
     /// preventing a memory to grow.
-    pub fn grow(&mut self, delta: u64) -> Result<u64> {
+    pub fn grow(&self, delta: u64) -> Result<u64> {
         match self.0.grow(delta, None)? {
             Some((old_size, _new_size)) => {
                 // For shared memory, the `VMMemoryDefinition` is updated inside
@@ -791,6 +795,87 @@ impl SharedMemory {
         }
     }
 
+    /// Equivalent of the WebAssembly `memory.atomic.notify` instruction for
+    /// this shared memory.
+    ///
+    /// This method allows embedders to notify threads blocked on the specified
+    /// `addr`, an index into wasm linear memory. Threads could include
+    /// wasm threads blocked on a `memory.atomic.wait*` instruction or embedder
+    /// threads blocked on [`SharedMemory::atomic_wait32`], for example.
+    ///
+    /// The `count` argument is the number of threads to wake up.
+    ///
+    /// This function returns the number of threads awoken.
+    ///
+    /// # Errors
+    ///
+    /// This function will return an error if `addr` is not within bounds or
+    /// not aligned to a 4-byte boundary.
+    pub fn atomic_notify(&self, addr: u64, count: u32) -> Result<u32, Trap> {
+        self.0.atomic_notify(addr, count)
+    }
+
+    /// Equivalent of the WebAssembly `memory.atomic.wait32` instruction for
+    /// this shared memory.
+    ///
+    /// This method allows embedders to block the current thread until notified
+    /// via the `memory.atomic.notify` instruction or the
+    /// [`SharedMemory::atomic_notify`] method, enabling synchronization with
+    /// the wasm guest as desired.
+    ///
+    /// The `expected` argument is the expected 32-bit value to be stored at
+    /// the byte address `addr` specified. The `addr` specified is an index
+    /// into this linear memory.
+    ///
+    /// The optional `timeout` argument is the point in time after which the
+    /// calling thread is guaranteed to be woken up. Blocking will not occur
+    /// past this point.
+    ///
+    /// This function returns one of three possible values:
+    ///
+    /// * `WaitResult::Ok` - this function, loaded the value at `addr`, found
+    ///   it was equal to `expected`, and then blocked (all as one atomic
+    ///   operation). The thread was then awoken with a `memory.atomic.notify`
+    ///   instruction or the [`SharedMemory::atomic_notify`] method.
+    /// * `WaitResult::Mismatch` - the value at `addr` was loaded but was not
+    ///   equal to `expected` so the thread did not block and immediately
+    ///   returned.
+    /// * `WaitResult::TimedOut` - all the steps of `Ok` happened, except this
+    ///   thread was woken up due to a timeout.
+    ///
+    /// This function will not return due to spurious wakeups.
+    ///
+    /// # Errors
+    ///
+    /// This function will return an error if `addr` is not within bounds or
+    /// not aligned to a 4-byte boundary.
+    pub fn atomic_wait32(
+        &self,
+        addr: u64,
+        expected: u32,
+        timeout: Option<Instant>,
+    ) -> Result<WaitResult, Trap> {
+        self.0.atomic_wait32(addr, expected, timeout)
+    }
+
+    /// Equivalent of the WebAssembly `memory.atomic.wait64` instruction for
+    /// this shared memory.
+    ///
+    /// For more information see [`SharedMemory::atomic_wait32`].
+    ///
+    /// # Errors
+    ///
+    /// Returns the same error as [`SharedMemory::atomic_wait32`] except that
+    /// the specified address must be 8-byte aligned instead of 4-byte aligned.
+    pub fn atomic_wait64(
+        &self,
+        addr: u64,
+        expected: u64,
+        timeout: Option<Instant>,
+    ) -> Result<WaitResult, Trap> {
+        self.0.atomic_wait64(addr, expected, timeout)
+    }
+
     /// Return a reference to the [`Engine`] used to configure the shared
     /// memory.
     pub(crate) fn engine(&self) -> &Engine {
@@ -824,7 +909,8 @@ impl SharedMemory {
             .unwrap();
         let shared_memory = memory
             .as_shared_memory()
-            .expect("unable to convert from a shared memory");
+            .expect("unable to convert from a shared memory")
+            .clone();
         Self(shared_memory, store.engine().clone())
     }
 }


Test patch:
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif
new file mode 100644
index 000000000..52081f193
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_no_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif
new file mode 100644
index 000000000..52081f193
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i32_yes_guards_yes_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif
new file mode 100644
index 000000000..db20c1168
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_no_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif
new file mode 100644
index 000000000..db20c1168
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_dynamic_i64_yes_guards_yes_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    gv2 = load.i64 notrap aligned gv0+8
+    heap0 = dynamic gv1, min 0x1000, bound gv2, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: dynamic, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif
new file mode 100644
index 000000000..42bd0eb49
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_no_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif
new file mode 100644
index 000000000..42bd0eb49
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i32_yes_guards_yes_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i32, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+
+block0(v0: i64, v1: i32, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i32, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i32
+    fn0 = %do_store(i64, i32, i32)
+
+block0(v0: i64, v1: i32, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif
new file mode 100644
index 000000000..dcb08f9c9
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_no_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif
new file mode 100644
index 000000000..dcb08f9c9
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/heap_load_store_static_i64_yes_guards_yes_spectre.clif
@@ -0,0 +1,40 @@
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=true
+
+function %do_store(i64 vmctx, i64, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+
+block0(v0: i64, v1: i64, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, i64, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    
+    heap0 = static gv1, min 0x1000, bound 0x1000, offset_guard 0xffff_ffff, index_type i64
+    fn0 = %do_store(i64, i64, i32)
+
+block0(v0: i64, v1: i64, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: static, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
diff --git a/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh b/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh
new file mode 100755
index 000000000..694ee30ae
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/make-heap-load-store-tests.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# This script generates the `heap_load_store_*.clif` test files.
+#
+# Usage:
+#
+#     $ ./make-heap-load-store-tests.sh
+
+set -e
+cd $(dirname "$0")
+
+function generate_one_test() {
+    local kind=$1
+    local index_type=$2
+    local guard=$3
+    local spectre=$4
+
+    local enable_spectre=true
+    if [[ spectre == "no" ]]; then
+        enable_spectre=false
+    fi
+
+    local have_guards=yes
+    if [[ guard == "0" ]]; then
+        have_guards=no
+    fi
+
+    local gv2=""
+    local bound=0x1000
+    if [[ $kind == "dynamic" ]]; then
+        gv2="gv2 = load.i64 notrap aligned gv0+8"
+        bound=gv2
+    fi
+
+    local filename="heap_load_store_${kind}_${index_type}_${have_guards}_guards_${spectre}_spectre.clif"
+    echo "Generating $filename"
+
+    cat <<EOF > "$filename"
+;; !!! GENERATED BY 'make-heap-load-store-tests.sh' DO NOT EDIT !!!
+
+test interpret
+;; test run
+;; target x86_64
+;; target s390x
+;; target aarch64
+;; target riscv64
+
+set enable_heap_access_spectre_mitigation=${enable_spectre}
+
+function %do_store(i64 vmctx, ${index_type}, i32) {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    $gv2
+    heap0 = ${kind} gv1, min 0x1000, bound ${bound}, offset_guard ${guard}, index_type ${index_type}
+
+block0(v0: i64, v1: ${index_type}, v2: i32):
+    heap_store.i32 heap0 little v1+4, v2
+    return
+}
+
+function %test(i64 vmctx, ${index_type}, i32) -> i32 {
+    gv0 = vmctx
+    gv1 = load.i64 notrap aligned gv0+0
+    $gv2
+    heap0 = ${kind} gv1, min 0x1000, bound ${bound}, offset_guard ${guard}, index_type ${index_type}
+    fn0 = %do_store(i64, ${index_type}, i32)
+
+block0(v0: i64, v1: ${index_type}, v2: i32):
+    call fn0(v0, v1, v2)
+    v3 = heap_load.i32 heap0 little v1+4
+    return v3
+}
+; heap: ${kind}, size=0x1000, ptr=vmctx+0, bound=vmctx+8
+; run: %test(0, 0) == 0
+; run: %test(0, -1) == -1
+; run: %test(16, 1) == 1
+; run: %test(16, -1) == -1
+; run: %test(2049, 0xaabb_ccdd) == 0xaabb_ccdd
+EOF
+}
+
+for spectre in yes no; do
+    for guard in 0 0xffff_ffff; do
+        for index_type in i32 i64; do
+            for kind in static dynamic; do
+                generate_one_test $kind $index_type $guard $spectre
+            done
+        done
+    done
+done
diff --git a/tests/all/memory.rs b/tests/all/memory.rs
index 281a2e7ab..2f7663db9 100644
--- a/tests/all/memory.rs
+++ b/tests/all/memory.rs
@@ -1,5 +1,7 @@
 use anyhow::Result;
 use rayon::prelude::*;
+use std::sync::atomic::{AtomicU32, Ordering::SeqCst};
+use std::time::{Duration, Instant};
 use wasmtime::*;
 
 fn module(engine: &Engine) -> Result<Module> {
@@ -462,3 +464,105 @@ fn memory64_maximum_minimum() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn shared_memory_basics() -> Result<()> {
+    let engine = Engine::default();
+    assert!(SharedMemory::new(&engine, MemoryType::new(1, None)).is_err());
+    assert!(SharedMemory::new(&engine, MemoryType::new(1, Some(1))).is_err());
+    assert!(SharedMemory::new(&engine, MemoryType::new64(1, None)).is_err());
+    assert!(SharedMemory::new(&engine, MemoryType::new64(1, Some(1))).is_err());
+    assert!(SharedMemory::new(&engine, MemoryType::shared(1, 0)).is_err());
+
+    let memory = SharedMemory::new(&engine, MemoryType::shared(1, 1))?;
+    assert!(memory.ty().is_shared());
+    assert_eq!(memory.ty().minimum(), 1);
+    assert_eq!(memory.ty().maximum(), Some(1));
+    assert_eq!(memory.size(), 1);
+    assert_eq!(memory.data_size(), 65536);
+    assert_eq!(memory.data().len(), 65536);
+    assert!(memory.grow(1).is_err());
+
+    // misaligned
+    assert_eq!(memory.atomic_notify(1, 100), Err(Trap::HeapMisaligned));
+    assert_eq!(
+        memory.atomic_wait32(1, 100, None),
+        Err(Trap::HeapMisaligned)
+    );
+    assert_eq!(
+        memory.atomic_wait64(1, 100, None),
+        Err(Trap::HeapMisaligned)
+    );
+
+    // oob
+    assert_eq!(
+        memory.atomic_notify(1 << 20, 100),
+        Err(Trap::MemoryOutOfBounds)
+    );
+    assert_eq!(
+        memory.atomic_wait32(1 << 20, 100, None),
+        Err(Trap::MemoryOutOfBounds)
+    );
+    assert_eq!(
+        memory.atomic_wait64(1 << 20, 100, None),
+        Err(Trap::MemoryOutOfBounds)
+    );
+
+    // ok
+    assert_eq!(memory.atomic_notify(8, 100), Ok(0));
+    assert_eq!(memory.atomic_wait32(8, 1, None), Ok(WaitResult::Mismatch));
+    assert_eq!(memory.atomic_wait64(8, 1, None), Ok(WaitResult::Mismatch));
+
+    // timeout
+    let near_future = Instant::now() + Duration::new(0, 100);
+    assert_eq!(
+        memory.atomic_wait32(8, 0, Some(near_future)),
+        Ok(WaitResult::TimedOut)
+    );
+    assert_eq!(
+        memory.atomic_wait64(8, 0, Some(near_future)),
+        Ok(WaitResult::TimedOut)
+    );
+
+    Ok(())
+}
+
+#[test]
+fn shared_memory_wait_notify() -> Result<()> {
+    const THREADS: usize = 8;
+    const COUNT: usize = 100_000;
+
+    let engine = Engine::default();
+    let memory = SharedMemory::new(&engine, MemoryType::shared(1, 1))?;
+    let data = unsafe { &*(memory.data().as_ptr() as *const AtomicU32) };
+    let locked = unsafe { &*(memory.data().as_ptr().add(4) as *const AtomicU32) };
+
+    // Note that `SeqCst` is used here to not think much about the orderings
+    // here, and it also somewhat more closely mirrors what's happening in wasm.
+    let lock = || {
+        while locked.swap(1, SeqCst) == 1 {
+            memory.atomic_wait32(0, 1, None).unwrap();
+        }
+    };
+    let unlock = || {
+        locked.store(0, SeqCst);
+        memory.atomic_notify(0, 1).unwrap();
+    };
+
+    std::thread::scope(|s| {
+        for _ in 0..THREADS {
+            s.spawn(|| {
+                for _ in 0..COUNT {
+                    lock();
+                    let next = data.load(SeqCst) + 1;
+                    data.store(next, SeqCst);
+                    unlock();
+                }
+            });
+        }
+    });
+
+    assert_eq!(data.load(SeqCst), (THREADS * COUNT) as u32);
+
+    Ok(())
+}
diff --git a/tests/all/threads.rs b/tests/all/threads.rs
index fe497beec..30ca86bdd 100644
--- a/tests/all/threads.rs
+++ b/tests/all/threads.rs
@@ -107,7 +107,7 @@ fn test_probe_shared_memory_size() -> Result<()> {
     let mut store = Store::new(&engine, ());
     let instance = Instance::new(&mut store, &module, &[])?;
     let size_fn = instance.get_typed_func::<(), i32>(&mut store, "size")?;
-    let mut shared_memory = instance.get_shared_memory(&mut store, "memory").unwrap();
+    let shared_memory = instance.get_shared_memory(&mut store, "memory").unwrap();
 
     assert_eq!(size_fn.call(&mut store, ())?, 1);
     assert_eq!(shared_memory.size(), 1);
@@ -244,7 +244,7 @@ fn test_memory_size_accessibility() -> Result<()> {
     let shared_memory = SharedMemory::new(&engine, MemoryType::shared(1, NUM_GROW_OPS as u32))?;
     let done = Arc::new(AtomicBool::new(false));
 
-    let mut grow_memory = shared_memory.clone();
+    let grow_memory = shared_memory.clone();
     let grow_thread = std::thread::spawn(move || {
         for i in 0..NUM_GROW_OPS {
             if grow_memory.grow(1).is_err() {