Repository: tetratelabs/wazero
Pull request #1915 (2024-01-12 04:53:18)

threads: implement threads proposal in interpreter

mathetake: @achille-roussel @ncruces could you take a look?
anuraaga: Thanks for remembering @ncruces - yeah that doc will be on the feature flag that will be added after implementing compiler

ncruces: > Thanks for remembering @ncruces - yeah that doc will be on the feature flag that will be added after implementing compiler

I wonder if the answer here is to provide (a subset of?) atomic access methods in the memory API.

Can a memory be shared between an interpreter and a compiled instance?

Anyway, this is out-of-scope here.
anuraaga: @ncruces Thanks for the review! @mathetake just in case it's hard to follow, we got @ncruces's LGTM.

> I wonder if the answer here is to provide (a subset of?) atomic access methods in the memory API.

I think so, perhaps it could be a separate type `AtomicMemory` or something

> Can a memory be shared between an interpreter and a compiled instance?

Not sure I understand the question but because compiler or interpreter is a `Runtime` setting, I don't think so, all modules created by that runtime will be the same including memory.

Patch:
diff --git a/internal/engine/interpreter/interpreter.go b/internal/engine/interpreter/interpreter.go
index c60d0584..f67c0c72 100644
--- a/internal/engine/interpreter/interpreter.go
+++ b/internal/engine/interpreter/interpreter.go
@@ -3897,6 +3897,361 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
 			ce.pushValue(retLo)
 			ce.pushValue(retHi)
 			frame.pc++
+		case wazeroir.OperationKindAtomicMemoryWait:
+			timeout := int64(ce.popValue())
+			exp := ce.popValue()
+			offset := ce.popMemoryOffset(op)
+			// Runtime instead of validation error because the spec intends to allow binaries to include
+			// such instructions as long as they are not executed.
+			if !memoryInst.Shared {
+				panic(wasmruntime.ErrRuntimeExpectedSharedMemory)
+			}
+
+			switch wazeroir.UnsignedType(op.B1) {
+			case wazeroir.UnsignedTypeI32:
+				if offset%4 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				if int(offset) > len(memoryInst.Buffer)-4 {
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				ce.pushValue(memoryInst.Wait32(offset, uint32(exp), timeout, func(mem *wasm.MemoryInstance, offset uint32) uint32 {
+					mem.Mux.Lock()
+					defer mem.Mux.Unlock()
+					value, _ := mem.ReadUint32Le(offset)
+					return value
+				}))
+			case wazeroir.UnsignedTypeI64:
+				if offset%8 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				if int(offset) > len(memoryInst.Buffer)-8 {
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				ce.pushValue(memoryInst.Wait64(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint64 {
+					mem.Mux.Lock()
+					defer mem.Mux.Unlock()
+					value, _ := mem.ReadUint64Le(offset)
+					return value
+				}))
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicMemoryNotify:
+			count := ce.popValue()
+			offset := ce.popMemoryOffset(op)
+			if offset%4 != 0 {
+				panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+			}
+			// Just a bounds check
+			if offset >= memoryInst.Size() {
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			res := memoryInst.Notify(offset, uint32(count))
+			ce.pushValue(uint64(res))
+			frame.pc++
+		case wazeroir.OperationKindAtomicFence:
+			// Memory not required for fence only
+			if memoryInst != nil {
+				// An empty critical section can be used as a synchronization primitive, which is what
+				// fence is. Probably, there are no spectests or defined behavior to confirm this yet.
+				memoryInst.Mux.Lock()
+				memoryInst.Mux.Unlock() //nolint:staticcheck
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicLoad:
+			offset := ce.popMemoryOffset(op)
+			switch wazeroir.UnsignedType(op.B1) {
+			case wazeroir.UnsignedTypeI32:
+				if offset%4 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				val, ok := memoryInst.ReadUint32Le(offset)
+				memoryInst.Mux.Unlock()
+				if !ok {
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				ce.pushValue(uint64(val))
+			case wazeroir.UnsignedTypeI64:
+				if offset%8 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				val, ok := memoryInst.ReadUint64Le(offset)
+				memoryInst.Mux.Unlock()
+				if !ok {
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				ce.pushValue(val)
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicLoad8:
+			offset := ce.popMemoryOffset(op)
+			memoryInst.Mux.Lock()
+			val, ok := memoryInst.ReadByte(offset)
+			memoryInst.Mux.Unlock()
+			if !ok {
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			ce.pushValue(uint64(val))
+			frame.pc++
+		case wazeroir.OperationKindAtomicLoad16:
+			offset := ce.popMemoryOffset(op)
+			if offset%2 != 0 {
+				panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+			}
+			memoryInst.Mux.Lock()
+			val, ok := memoryInst.ReadUint16Le(offset)
+			memoryInst.Mux.Unlock()
+			if !ok {
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			ce.pushValue(uint64(val))
+			frame.pc++
+		case wazeroir.OperationKindAtomicStore:
+			val := ce.popValue()
+			offset := ce.popMemoryOffset(op)
+			switch wazeroir.UnsignedType(op.B1) {
+			case wazeroir.UnsignedTypeI32:
+				if offset%4 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				ok := memoryInst.WriteUint32Le(offset, uint32(val))
+				memoryInst.Mux.Unlock()
+				if !ok {
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+			case wazeroir.UnsignedTypeI64:
+				if offset%8 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				ok := memoryInst.WriteUint64Le(offset, val)
+				memoryInst.Mux.Unlock()
+				if !ok {
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicStore8:
+			val := byte(ce.popValue())
+			offset := ce.popMemoryOffset(op)
+			memoryInst.Mux.Lock()
+			ok := memoryInst.WriteByte(offset, val)
+			memoryInst.Mux.Unlock()
+			if !ok {
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicStore16:
+			val := uint16(ce.popValue())
+			offset := ce.popMemoryOffset(op)
+			if offset%2 != 0 {
+				panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+			}
+			memoryInst.Mux.Lock()
+			ok := memoryInst.WriteUint16Le(offset, val)
+			memoryInst.Mux.Unlock()
+			if !ok {
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicRMW:
+			val := ce.popValue()
+			offset := ce.popMemoryOffset(op)
+			switch wazeroir.UnsignedType(op.B1) {
+			case wazeroir.UnsignedTypeI32:
+				if offset%4 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				old, ok := memoryInst.ReadUint32Le(offset)
+				if !ok {
+					memoryInst.Mux.Unlock()
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				var newVal uint32
+				switch wazeroir.AtomicArithmeticOp(op.B2) {
+				case wazeroir.AtomicArithmeticOpAdd:
+					newVal = old + uint32(val)
+				case wazeroir.AtomicArithmeticOpSub:
+					newVal = old - uint32(val)
+				case wazeroir.AtomicArithmeticOpAnd:
+					newVal = old & uint32(val)
+				case wazeroir.AtomicArithmeticOpOr:
+					newVal = old | uint32(val)
+				case wazeroir.AtomicArithmeticOpXor:
+					newVal = old ^ uint32(val)
+				case wazeroir.AtomicArithmeticOpNop:
+					newVal = uint32(val)
+				}
+				memoryInst.WriteUint32Le(offset, newVal)
+				memoryInst.Mux.Unlock()
+				ce.pushValue(uint64(old))
+			case wazeroir.UnsignedTypeI64:
+				if offset%8 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				old, ok := memoryInst.ReadUint64Le(offset)
+				if !ok {
+					memoryInst.Mux.Unlock()
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				var newVal uint64
+				switch wazeroir.AtomicArithmeticOp(op.B2) {
+				case wazeroir.AtomicArithmeticOpAdd:
+					newVal = old + val
+				case wazeroir.AtomicArithmeticOpSub:
+					newVal = old - val
+				case wazeroir.AtomicArithmeticOpAnd:
+					newVal = old & val
+				case wazeroir.AtomicArithmeticOpOr:
+					newVal = old | val
+				case wazeroir.AtomicArithmeticOpXor:
+					newVal = old ^ val
+				case wazeroir.AtomicArithmeticOpNop:
+					newVal = val
+				}
+				memoryInst.WriteUint64Le(offset, newVal)
+				memoryInst.Mux.Unlock()
+				ce.pushValue(old)
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicRMW8:
+			val := ce.popValue()
+			offset := ce.popMemoryOffset(op)
+			memoryInst.Mux.Lock()
+			old, ok := memoryInst.ReadByte(offset)
+			if !ok {
+				memoryInst.Mux.Unlock()
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			arg := byte(val)
+			var newVal byte
+			switch wazeroir.AtomicArithmeticOp(op.B2) {
+			case wazeroir.AtomicArithmeticOpAdd:
+				newVal = old + arg
+			case wazeroir.AtomicArithmeticOpSub:
+				newVal = old - arg
+			case wazeroir.AtomicArithmeticOpAnd:
+				newVal = old & arg
+			case wazeroir.AtomicArithmeticOpOr:
+				newVal = old | arg
+			case wazeroir.AtomicArithmeticOpXor:
+				newVal = old ^ arg
+			case wazeroir.AtomicArithmeticOpNop:
+				newVal = arg
+			}
+			memoryInst.WriteByte(offset, newVal)
+			memoryInst.Mux.Unlock()
+			ce.pushValue(uint64(old))
+			frame.pc++
+		case wazeroir.OperationKindAtomicRMW16:
+			val := ce.popValue()
+			offset := ce.popMemoryOffset(op)
+			if offset%2 != 0 {
+				panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+			}
+			memoryInst.Mux.Lock()
+			old, ok := memoryInst.ReadUint16Le(offset)
+			if !ok {
+				memoryInst.Mux.Unlock()
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			arg := uint16(val)
+			var newVal uint16
+			switch wazeroir.AtomicArithmeticOp(op.B2) {
+			case wazeroir.AtomicArithmeticOpAdd:
+				newVal = old + arg
+			case wazeroir.AtomicArithmeticOpSub:
+				newVal = old - arg
+			case wazeroir.AtomicArithmeticOpAnd:
+				newVal = old & arg
+			case wazeroir.AtomicArithmeticOpOr:
+				newVal = old | arg
+			case wazeroir.AtomicArithmeticOpXor:
+				newVal = old ^ arg
+			case wazeroir.AtomicArithmeticOpNop:
+				newVal = arg
+			}
+			memoryInst.WriteUint16Le(offset, newVal)
+			memoryInst.Mux.Unlock()
+			ce.pushValue(uint64(old))
+			frame.pc++
+		case wazeroir.OperationKindAtomicRMWCmpxchg:
+			rep := ce.popValue()
+			exp := ce.popValue()
+			offset := ce.popMemoryOffset(op)
+			switch wazeroir.UnsignedType(op.B1) {
+			case wazeroir.UnsignedTypeI32:
+				if offset%4 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				old, ok := memoryInst.ReadUint32Le(offset)
+				if !ok {
+					memoryInst.Mux.Unlock()
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				if old == uint32(exp) {
+					memoryInst.WriteUint32Le(offset, uint32(rep))
+				}
+				memoryInst.Mux.Unlock()
+				ce.pushValue(uint64(old))
+			case wazeroir.UnsignedTypeI64:
+				if offset%8 != 0 {
+					panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+				}
+				memoryInst.Mux.Lock()
+				old, ok := memoryInst.ReadUint64Le(offset)
+				if !ok {
+					memoryInst.Mux.Unlock()
+					panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+				}
+				if old == exp {
+					memoryInst.WriteUint64Le(offset, rep)
+				}
+				memoryInst.Mux.Unlock()
+				ce.pushValue(old)
+			}
+			frame.pc++
+		case wazeroir.OperationKindAtomicRMW8Cmpxchg:
+			rep := byte(ce.popValue())
+			exp := byte(ce.popValue())
+			offset := ce.popMemoryOffset(op)
+			memoryInst.Mux.Lock()
+			old, ok := memoryInst.ReadByte(offset)
+			if !ok {
+				memoryInst.Mux.Unlock()
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			if old == exp {
+				memoryInst.WriteByte(offset, rep)
+			}
+			memoryInst.Mux.Unlock()
+			ce.pushValue(uint64(old))
+			frame.pc++
+		case wazeroir.OperationKindAtomicRMW16Cmpxchg:
+			rep := uint16(ce.popValue())
+			exp := uint16(ce.popValue())
+			offset := ce.popMemoryOffset(op)
+			if offset%2 != 0 {
+				panic(wasmruntime.ErrRuntimeUnalignedAtomic)
+			}
+			memoryInst.Mux.Lock()
+			old, ok := memoryInst.ReadUint16Le(offset)
+			if !ok {
+				memoryInst.Mux.Unlock()
+				panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
+			}
+			if old == exp {
+				memoryInst.WriteUint16Le(offset, rep)
+			}
+			memoryInst.Mux.Unlock()
+			ce.pushValue(uint64(old))
+			frame.pc++
 		default:
 			frame.pc++
 		}
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr.go b/internal/engine/wazevo/backend/isa/amd64/instr.go
index fcf62254..261142f0 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr.go
@@ -68,15 +68,45 @@ func (i *instruction) String() string {
 	case xmmUnaryRmR:
 		return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1))
 	case unaryRmR:
-		panic("TODO")
+		var suffix string
+		if i.b1 {
+			suffix = "q"
+		} else {
+			suffix = "l"
+		}
+		return fmt.Sprintf("%s%s %s, %s", unaryRmROpcode(i.u1), suffix, i.op1.format(i.b1), i.op2.format(i.b1))
 	case not:
-		panic("TODO")
+		var op string
+		if i.b1 {
+			op = "notq"
+		} else {
+			op = "notl"
+		}
+		return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
 	case neg:
-		panic("TODO")
+		var op string
+		if i.b1 {
+			op = "negq"
+		} else {
+			op = "negl"
+		}
+		return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
 	case div:
 		panic("TODO")
 	case mulHi:
-		panic("TODO")
+		signed, _64 := i.u1 != 0, i.b1
+		var op string
+		switch {
+		case signed && _64:
+			op = "imulq"
+		case !signed && _64:
+			op = "mulq"
+		case signed && !_64:
+			op = "imull"
+		case !signed && !_64:
+			op = "mull"
+		}
+		return fmt.Sprintf("%s %s", op, i.op1.format(i.b1))
 	case checkedDivOrRemSeq:
 		panic("TODO")
 	case signExtendData:
@@ -103,7 +133,13 @@ func (i *instruction) String() string {
 		}
 		return fmt.Sprintf("mov.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true))
 	case shiftR:
-		panic("TODO")
+		var suffix string
+		if i.b1 {
+			suffix = "q"
+		} else {
+			suffix = "l"
+		}
+		return fmt.Sprintf("%s%s %s, %s", shiftROp(i.u1), suffix, i.op1.format(false), i.op2.format(i.b1))
 	case xmmRmiReg:
 		panic("TODO")
 	case cmpRmiR:
@@ -693,6 +729,63 @@ func (i *instruction) asMovRR(rm, rd regalloc.VReg, _64 bool) *instruction {
 	return i
 }
 
+func (i *instruction) asNot(rm operand, _64 bool) *instruction {
+	if rm.kind != operandKindReg && rm.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = not
+	i.op1 = rm
+	i.b1 = _64
+	return i
+}
+
+func (i *instruction) asNeg(rm operand, _64 bool) *instruction {
+	if rm.kind != operandKindReg && rm.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = neg
+	i.op1 = rm
+	i.b1 = _64
+	return i
+}
+
+func (i *instruction) asMulHi(rm operand, signed, _64 bool) *instruction {
+	if rm.kind != operandKindReg && (rm.kind != operandKindMem) {
+		panic("BUG")
+	}
+	i.kind = mulHi
+	i.op1 = rm
+	i.b1 = _64
+	if signed {
+		i.u1 = 1
+	}
+	return i
+}
+
+func (i *instruction) asUnaryRmR(op unaryRmROpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction {
+	if rm.kind != operandKindReg && rm.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = unaryRmR
+	i.op1 = rm
+	i.op2 = newOperandReg(rd)
+	i.u1 = uint64(op)
+	i.b1 = _64
+	return i
+}
+
+func (i *instruction) asShiftR(op shiftROp, amount operand, rd regalloc.VReg, _64 bool) *instruction {
+	if amount.kind != operandKindReg && amount.kind != operandKindImm32 {
+		panic("BUG")
+	}
+	i.kind = shiftR
+	i.op1 = amount
+	i.op2 = newOperandReg(rd)
+	i.u1 = uint64(op)
+	i.b1 = _64
+	return i
+}
+
 func (i *instruction) asXmmUnaryRmR(op sseOpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction {
 	if rm.kind != operandKindReg && rm.kind != operandKindMem {
 		panic("BUG")
@@ -731,6 +824,60 @@ func (i *instruction) asPush64(op operand) *instruction {
 	return i
 }
 
+type unaryRmROpcode byte
+
+const (
+	unaryRmROpcodeBsr unaryRmROpcode = iota
+	unaryRmROpcodeBsf
+	unaryRmROpcodeLzcnt
+	unaryRmROpcodeTzcnt
+	unaryRmROpcodePopcnt
+)
+
+func (u unaryRmROpcode) String() string {
+	switch u {
+	case unaryRmROpcodeBsr:
+		return "bsr"
+	case unaryRmROpcodeBsf:
+		return "bsf"
+	case unaryRmROpcodeLzcnt:
+		return "lzcnt"
+	case unaryRmROpcodeTzcnt:
+		return "tzcnt"
+	case unaryRmROpcodePopcnt:
+		return "popcnt"
+	default:
+		panic("BUG")
+	}
+}
+
+type shiftROp byte
+
+const (
+	shiftROpRotateLeft           = 0
+	shiftROpRotateRight          = 1
+	shiftROpShiftLeft            = 4
+	shiftROpShiftRightLogical    = 5
+	shiftROpShiftRightArithmetic = 7
+)
+
+func (s shiftROp) String() string {
+	switch s {
+	case shiftROpRotateLeft:
+		return "rol"
+	case shiftROpRotateRight:
+		return "ror"
+	case shiftROpShiftLeft:
+		return "shl"
+	case shiftROpShiftRightLogical:
+		return "shr"
+	case shiftROpShiftRightArithmetic:
+		return "sar"
+	default:
+		panic("BUG")
+	}
+}
+
 type sseOpcode byte
 
 const (
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go b/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
index 8f0b76b2..bc429b5d 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
@@ -452,15 +452,102 @@ func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) {
 		}
 
 	case unaryRmR:
-		panic("TODO")
+		var prefix legacyPrefixes
+		var opcode uint32
+		var opcodeNum uint32
+		op := unaryRmROpcode(i.u1)
+		// We assume size is either 32 or 64.
+		switch op {
+		case unaryRmROpcodeBsr:
+			prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0fbd, 2
+		case unaryRmROpcodeBsf:
+			prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0fbc, 2
+		case unaryRmROpcodeLzcnt:
+			prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fbd, 2
+		case unaryRmROpcodeTzcnt:
+			prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fbc, 2
+		case unaryRmROpcodePopcnt:
+			prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fb8, 2
+		default:
+			panic(fmt.Sprintf("Unsupported unaryRmROpcode: %s", op))
+		}
+
+		dst := regEncodings[i.op2.r.RealReg()]
+
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+		op1 := i.op1
+		if op1.kind == operandKindReg {
+			src := regEncodings[op1.r.RealReg()]
+			encodeRegReg(c, prefix, opcode, opcodeNum, dst, src, rex)
+		} else if i.op1.kind == operandKindMem {
+			m := i.op1.amode
+			encodeRegMem(c, prefix, opcode, opcodeNum, dst, m, rex)
+		} else {
+			panic("BUG: invalid operand kind")
+		}
+
 	case not:
-		panic("TODO")
+		var prefix legacyPrefixes
+		src := regEncodings[i.op1.r.RealReg()]
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+		subopcode := uint8(2)
+		encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex)
+
 	case neg:
-		panic("TODO")
+		var prefix legacyPrefixes
+		src := regEncodings[i.op1.r.RealReg()]
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+		subopcode := uint8(3)
+		encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex)
+
 	case div:
 		panic("TODO")
 	case mulHi:
-		panic("TODO")
+		var prefix legacyPrefixes
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+
+		signed := i.u1 != 0
+		var subopcode uint8
+		if signed {
+			subopcode = 5
+		} else {
+			subopcode = 4
+		}
+
+		// src1 is implicitly rax,
+		// dst_lo is implicitly rax,
+		// dst_hi is implicitly rdx.
+		src2 := i.op1
+		if src2.kind == operandKindReg {
+			src := regEncodings[src2.r.RealReg()]
+			encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex)
+		} else if src2.kind == operandKindMem {
+			m := src2.amode
+			encodeEncMem(c, prefix, 0xf7, 1, subopcode, m, rex)
+		} else {
+			panic("BUG: invalid operand kind")
+		}
+
 	case checkedDivOrRemSeq:
 		panic("TODO")
 	case signExtendData:
@@ -558,7 +645,32 @@ func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) {
 		}
 
 	case shiftR:
-		panic("TODO")
+		src := regEncodings[i.op2.r.RealReg()]
+		amount := i.op1
+
+		var opcode uint32
+		var prefix legacyPrefixes
+		rex := rexInfo(0)
+		if i.b1 { // 64 bit.
+			rex = rexInfo(0).setW()
+		} else {
+			rex = rexInfo(0).clearW()
+		}
+
+		switch amount.kind {
+		case operandKindReg:
+			if amount.r != rcxVReg {
+				panic("BUG: invalid reg operand: must be rcx")
+			}
+			opcode, prefix = 0xd3, legacyPrefixesNone
+			encodeEncEnc(c, prefix, opcode, 1, uint8(i.u1), uint8(src), rex)
+		case operandKindImm32:
+			opcode, prefix = 0xc1, legacyPrefixesNone
+			encodeEncEnc(c, prefix, opcode, 1, uint8(i.u1), uint8(src), rex)
+			c.EmitByte(byte(amount.imm32))
+		default:
+			panic("BUG: invalid operand kind")
+		}
 	case xmmRmiReg:
 		panic("TODO")
 	case cmpRmiR:
@@ -741,23 +853,35 @@ func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) {
 	return
 }
 
-func encodeRegReg(
+func encodeEncEnc(
 	c backend.Compiler,
 	legPrefixes legacyPrefixes,
 	opcodes uint32,
 	opcodeNum uint32,
-	r regEnc,
-	rm regEnc,
+	r uint8,
+	rm uint8,
 	rex rexInfo,
 ) {
 	legPrefixes.encode(c)
-	rex.encode(c, r, rm)
+	rex.encode(c, r>>3, rm>>3)
 
 	for opcodeNum > 0 {
 		opcodeNum--
 		c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff))
 	}
-	c.EmitByte(encodeModRM(3, r.encoding(), rm.encoding()))
+	c.EmitByte(encodeModRM(3, r&0x7, rm&0x7))
+}
+
+func encodeRegReg(
+	c backend.Compiler,
+	legPrefixes legacyPrefixes,
+	opcodes uint32,
+	opcodeNum uint32,
+	r regEnc,
+	rm regEnc,
+	rex rexInfo,
+) {
+	encodeEncEnc(c, legPrefixes, opcodes, opcodeNum, uint8(r), uint8(rm), rex)
 }
 
 func encodeModRM(mod byte, reg byte, rm byte) byte {
@@ -770,6 +894,12 @@ func encodeSIB(shift byte, encIndex byte, encBase byte) byte {
 
 func encodeRegMem(
 	c backend.Compiler, legPrefixes legacyPrefixes, opcodes uint32, opcodeNum uint32, r regEnc, m amode, rex rexInfo,
+) {
+	encodeEncMem(c, legPrefixes, opcodes, opcodeNum, uint8(r), m, rex)
+}
+
+func encodeEncMem(
+	c backend.Compiler, legPrefixes legacyPrefixes, opcodes uint32, opcodeNum uint32, r uint8, m amode, rex rexInfo,
 ) {
 	legPrefixes.encode(c)
 
@@ -786,7 +916,7 @@ func encodeRegMem(
 		base := m.base.RealReg()
 		baseEnc := regEncodings[base]
 
-		rex.encode(c, r, baseEnc)
+		rex.encode(c, regRexBit(r), baseEnc.rexBit())
 
 		for opcodeNum > 0 {
 			opcodeNum--
@@ -801,18 +931,18 @@ func encodeRegMem(
 		rspOrR12 := base == rsp || base == r12
 
 		if immZero && !baseRbp && !baseR13 { // rbp or r13 can't be used as base for without displacement encoding.
-			c.EmitByte(encodeModRM(modNoDisplacement, r.encoding(), baseEnc.encoding()))
+			c.EmitByte(encodeModRM(modNoDisplacement, regEncoding(r), baseEnc.encoding()))
 			if rspOrR12 {
 				c.EmitByte(sibByte)
 			}
 		} else if short { // Note: this includes the case where m.imm32 == 0 && base == rbp || base == r13.
-			c.EmitByte(encodeModRM(modShortDisplacement, r.encoding(), baseEnc.encoding()))
+			c.EmitByte(encodeModRM(modShortDisplacement, regEncoding(r), baseEnc.encoding()))
 			if rspOrR12 {
 				c.EmitByte(sibByte)
 			}
 			c.EmitByte(byte(m.imm32))
 		} else {
-			c.EmitByte(encodeModRM(modLongDisplacement, r.encoding(), baseEnc.encoding()))
+			c.EmitByte(encodeModRM(modLongDisplacement, regEncoding(r), baseEnc.encoding()))
 			if rspOrR12 {
 				c.EmitByte(sibByte)
 			}
@@ -829,7 +959,7 @@ func encodeRegMem(
 			panic("BUG: rsp can't be used as index of addressing mode")
 		}
 
-		rex.encodeForIndex(c, r, indexEnc, baseEnc)
+		rex.encodeForIndex(c, regEnc(r), indexEnc, baseEnc)
 
 		for opcodeNum > 0 {
 			opcodeNum--
@@ -838,14 +968,14 @@ func encodeRegMem(
 
 		immZero, baseRbp, baseR13 := m.imm32 == 0, base == rbp, base == r13
 		if immZero && !baseRbp && !baseR13 { // rbp or r13 can't be used as base for without displacement encoding. (curious why? because it's interpreted as RIP relative addressing).
-			c.EmitByte(encodeModRM(modNoDisplacement, r.encoding(), useSBI))
+			c.EmitByte(encodeModRM(modNoDisplacement, regEncoding(r), useSBI))
 			c.EmitByte(encodeSIB(m.shift, indexEnc.encoding(), baseEnc.encoding()))
 		} else if lower8willSignExtendTo32(m.imm32) {
-			c.EmitByte(encodeModRM(modShortDisplacement, r.encoding(), useSBI))
+			c.EmitByte(encodeModRM(modShortDisplacement, regEncoding(r), useSBI))
 			c.EmitByte(encodeSIB(m.shift, indexEnc.encoding(), baseEnc.encoding()))
 			c.EmitByte(byte(m.imm32))
 		} else {
-			c.EmitByte(encodeModRM(modLongDisplacement, r.encoding(), useSBI))
+			c.EmitByte(encodeModRM(modLongDisplacement, regEncoding(r), useSBI))
 			c.EmitByte(encodeSIB(m.shift, indexEnc.encoding(), baseEnc.encoding()))
 			c.Emit4Bytes(m.imm32)
 		}
@@ -855,7 +985,7 @@ func encodeRegMem(
 			panic("BUG: label must be resolved for amodeRipRelative at this point")
 		}
 
-		rex.encode(c, r, 0)
+		rex.encode(c, regRexBit(r), 0)
 		for opcodeNum > 0 {
 			opcodeNum--
 			c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff))
@@ -863,7 +993,7 @@ func encodeRegMem(
 
 		// Indicate "LEAQ [RIP + 32bit displacement].
 		// https://wiki.osdev.org/X86-64_Instruction_Encoding#32.2F64-bit_addressing
-		c.EmitByte(encodeModRM(0b00, r.encoding(), 0b101))
+		c.EmitByte(encodeModRM(0b00, regEncoding(r), 0b101))
 		c.Emit4Bytes(m.imm32)
 	}
 }
@@ -895,13 +1025,11 @@ func (ri rexInfo) notAlways() rexInfo { //nolint
 	return ri & 0x01
 }
 
-func (ri rexInfo) encode(c backend.Compiler, encR regEnc, encRM regEnc) {
+func (ri rexInfo) encode(c backend.Compiler, r uint8, b uint8) {
 	var w byte = 0
 	if ri&0x01 != 0 {
 		w = 0x01
 	}
-	r := encR.rexBit()
-	b := encRM.rexBit()
 	rex := rexEncodingDefault | w<<3 | r<<2 | b
 	if rex != rexEncodingDefault || ri&0x02 == 1 {
 		c.EmitByte(rex)
@@ -925,11 +1053,19 @@ func (ri rexInfo) encodeForIndex(c backend.Compiler, encR regEnc, encIndex regEn
 type regEnc byte
 
 func (r regEnc) rexBit() byte {
-	return byte(r) >> 3
+	return regRexBit(byte(r))
 }
 
 func (r regEnc) encoding() byte {
-	return byte(r) & 0x07
+	return regEncoding(byte(r))
+}
+
+func regRexBit(r byte) byte {
+	return r >> 3
+}
+
+func regEncoding(r byte) byte {
+	return r & 0x07
 }
 
 var regEncodings = [...]regEnc{
diff --git a/internal/wasm/memory.go b/internal/wasm/memory.go
index 38781449..f0688e08 100644
--- a/internal/wasm/memory.go
+++ b/internal/wasm/memory.go
@@ -1,14 +1,18 @@
 package wasm
 
 import (
+	"container/list"
 	"encoding/binary"
 	"fmt"
 	"math"
 	"reflect"
+	"sync"
+	"time"
 	"unsafe"
 
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/internal/internalapi"
+	"github.com/tetratelabs/wazero/internal/wasmruntime"
 )
 
 const (
@@ -26,6 +30,11 @@ const (
 // compile-time check to ensure MemoryInstance implements api.Memory
 var _ api.Memory = &MemoryInstance{}
 
+type waiters struct {
+	mux sync.Mutex
+	l   *list.List
+}
+
 // MemoryInstance represents a memory instance in a store, and implements api.Memory.
 //
 // Note: In WebAssembly 1.0 (20191205), there may be up to one Memory per store, which means the precise memory is always
@@ -39,6 +48,14 @@ type MemoryInstance struct {
 	Shared        bool
 	// definition is known at compile time.
 	definition api.MemoryDefinition
+
+	// Mux is used in interpreter mode to prevent overlapping calls to atomic instructions,
+	// introduced with WebAssembly threads proposal.
+	Mux sync.Mutex
+
+	// waiters implements atomic wait and notify. It is implemented similarly to golang.org/x/sync/semaphore,
+	// with a fixed weight of 1 and no spurious notifications.
+	waiters sync.Map
 }
 
 // NewMemoryInstance creates a new instance based on the parameters in the SectionIDMemory.
@@ -279,3 +296,103 @@ func (m *MemoryInstance) writeUint64Le(offset uint32, v uint64) bool {
 	binary.LittleEndian.PutUint64(m.Buffer[offset:], v)
 	return true
 }
+
+// Wait32 suspends the caller until the offset is notified by a different agent.
+func (m *MemoryInstance) Wait32(offset uint32, exp uint32, timeout int64, reader func(mem *MemoryInstance, offset uint32) uint32) uint64 {
+	w := m.getWaiters(offset)
+	w.mux.Lock()
+
+	cur := reader(m, offset)
+	if cur != exp {
+		w.mux.Unlock()
+		return 1
+	}
+
+	return m.wait(w, timeout)
+}
+
+// Wait64 suspends the caller until the offset is notified by a different agent.
+func (m *MemoryInstance) Wait64(offset uint32, exp uint64, timeout int64, reader func(mem *MemoryInstance, offset uint32) uint64) uint64 {
+	w := m.getWaiters(offset)
+	w.mux.Lock()
+
+	cur := reader(m, offset)
+	if cur != exp {
+		w.mux.Unlock()
+		return 1
+	}
+
+	return m.wait(w, timeout)
+}
+
+func (m *MemoryInstance) wait(w *waiters, timeout int64) uint64 {
+	if w.l == nil {
+		w.l = list.New()
+	}
+
+	// The specification requires a trap if the number of existing waiters + 1 == 2^32, so we add a check here.
+	// In practice, it is unlikely the application would ever accumulate such a large number of waiters as it
+	// indicates several GB of RAM used just for the list of waiters.
+	// https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#wait
+	if uint64(w.l.Len()+1) == 1<<32 {
+		w.mux.Unlock()
+		panic(wasmruntime.ErrRuntimeTooManyWaiters)
+	}
+
+	ready := make(chan struct{})
+	elem := w.l.PushBack(ready)
+	w.mux.Unlock()
+
+	if timeout < 0 {
+		<-ready
+		return 0
+	} else {
+		select {
+		case <-ready:
+			return 0
+		case <-time.After(time.Duration(timeout)):
+			// While we could see if the channel completed by now and ignore the timeout, similar to x/sync/semaphore,
+			// the Wasm spec doesn't specify this behavior, so we keep things simple by prioritizing the timeout.
+			w.mux.Lock()
+			w.l.Remove(elem)
+			w.mux.Unlock()
+			return 2
+		}
+	}
+}
+
+func (m *MemoryInstance) getWaiters(offset uint32) *waiters {
+	wAny, ok := m.waiters.Load(offset)
+	if !ok {
+		// The first time an address is waited on, simultaneous waits will cause extra allocations.
+		// Further operations will be loaded above, which is also the general pattern of usage with
+		// mutexes.
+		wAny, _ = m.waiters.LoadOrStore(offset, &waiters{})
+	}
+
+	return wAny.(*waiters)
+}
+
+// Notify wakes up at most count waiters at the given offset.
+func (m *MemoryInstance) Notify(offset uint32, count uint32) uint32 {
+	wAny, ok := m.waiters.Load(offset)
+	if !ok {
+		return 0
+	}
+	w := wAny.(*waiters)
+
+	w.mux.Lock()
+	defer w.mux.Unlock()
+	if w.l == nil {
+		return 0
+	}
+
+	res := uint32(0)
+	for num := w.l.Len(); num > 0 && res < count; num = w.l.Len() {
+		w := w.l.Remove(w.l.Front()).(chan struct{})
+		close(w)
+		res++
+	}
+
+	return res
+}
diff --git a/internal/wasmruntime/errors.go b/internal/wasmruntime/errors.go
index 43f951e0..556e5de8 100644
--- a/internal/wasmruntime/errors.go
+++ b/internal/wasmruntime/errors.go
@@ -27,6 +27,12 @@ var (
 	ErrRuntimeInvalidTableAccess = New("invalid table access")
 	// ErrRuntimeIndirectCallTypeMismatch indicates that the type check failed during call_indirect.
 	ErrRuntimeIndirectCallTypeMismatch = New("indirect call type mismatch")
+	// ErrRuntimeUnalignedAtomic indicates that an atomic operation was made with incorrect memory alignment.
+	ErrRuntimeUnalignedAtomic = New("unaligned atomic")
+	// ErrRuntimeExpectedSharedMemory indicates that an operation was made against unshared memory when not allowed.
+	ErrRuntimeExpectedSharedMemory = New("expected shared memory")
+	// ErrRuntimeTooManyWaiters indicates that atomic.wait was called with too many waiters.
+	ErrRuntimeTooManyWaiters = New("too many waiters")
 )
 
 // Error is returned by a wasm.Engine during the execution of Wasm functions, and they indicate that the Wasm runtime


Test patch:
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go b/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
index 18a29e15..8a940ab2 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
@@ -100,6 +100,196 @@ func TestInstruction_format_encode(t *testing.T) {
 			want:       "4d89dc",
 			wantFormat: "movq %r11, %r12",
 		},
+		{
+			setup:      func(i *instruction) { i.asNot(newOperandReg(raxVReg), false) },
+			want:       "f7d0",
+			wantFormat: "notl %eax",
+		},
+		{
+			setup:      func(i *instruction) { i.asNot(newOperandReg(raxVReg), true) },
+			want:       "48f7d0",
+			wantFormat: "notq %rax",
+		},
+		{
+			setup:      func(i *instruction) { i.asNeg(newOperandReg(raxVReg), false) },
+			want:       "f7d8",
+			wantFormat: "negl %eax",
+		},
+		{
+			setup:      func(i *instruction) { i.asNeg(newOperandReg(raxVReg), true) },
+			want:       "48f7d8",
+			wantFormat: "negq %rax",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(rsiVReg), true, false) },
+			want:       "f7ee",
+			wantFormat: "imull %esi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(r14VReg), false, false) },
+			want:       "41f7e6",
+			wantFormat: "mull %r14d",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(r15VReg), true, true) },
+			want:       "49f7ef",
+			wantFormat: "imulq %r15",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandReg(rdiVReg), false, true) },
+			want:       "48f7e7",
+			wantFormat: "mulq %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), true, false) },
+			want:       "f7687b",
+			wantFormat: "imull 123(%rax)",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), false, false) },
+			want:       "f7607b",
+			wantFormat: "mull 123(%rax)",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), true, true) },
+			want:       "48f7687b",
+			wantFormat: "imulq 123(%rax)",
+		},
+		{
+			setup:      func(i *instruction) { i.asMulHi(newOperandMem(newAmodeImmReg(123, raxVReg)), false, true) },
+			want:       "48f7607b",
+			wantFormat: "mulq 123(%rax)",
+		},
+		// bsr
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsr, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "0fbdf8",
+			wantFormat: "bsrl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsr, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "0fbd787b",
+			wantFormat: "bsrl 123(%rax), %edi",
+		},
+		// bsf
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsf, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "0fbcf8",
+			wantFormat: "bsfl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsf, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "0fbc787b",
+			wantFormat: "bsfl 123(%rax), %edi",
+		},
+		// tzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "f30fbcf8",
+			wantFormat: "tzcntl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "f30fbc787b",
+			wantFormat: "tzcntl 123(%rax), %edi",
+		},
+		// lzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "f30fbdf8",
+			wantFormat: "lzcntl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "f30fbd787b",
+			wantFormat: "lzcntl 123(%rax), %edi",
+		},
+		// popcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandReg(raxVReg), rdiVReg, false) },
+			want:       "f30fb8f8",
+			wantFormat: "popcntl %eax, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, false)
+			},
+			want:       "f30fb8787b",
+			wantFormat: "popcntl 123(%rax), %edi",
+		},
+		// bsr
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsr, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "480fbdf8",
+			wantFormat: "bsrq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsr, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "480fbd787b",
+			wantFormat: "bsrq 123(%rax), %rdi",
+		},
+		// bsf
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeBsf, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "480fbcf8",
+			wantFormat: "bsfq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeBsf, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "480fbc787b",
+			wantFormat: "bsfq 123(%rax), %rdi",
+		},
+		// tzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "f3480fbcf8",
+			wantFormat: "tzcntq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeTzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "f3480fbc787b",
+			wantFormat: "tzcntq 123(%rax), %rdi",
+		},
+		// lzcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "f3480fbdf8",
+			wantFormat: "lzcntq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodeLzcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "f3480fbd787b",
+			wantFormat: "lzcntq 123(%rax), %rdi",
+		},
+		// popcnt
+		{
+			setup:      func(i *instruction) { i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandReg(raxVReg), rdiVReg, true) },
+			want:       "f3480fb8f8",
+			wantFormat: "popcntq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asUnaryRmR(unaryRmROpcodePopcnt, newOperandMem(newAmodeImmReg(123, raxVReg)), rdiVReg, true)
+			},
+			want:       "f3480fb8787b",
+			wantFormat: "popcntq 123(%rax), %rdi",
+		},
 		// addss
 		{
 			setup:      func(i *instruction) { i.asXmmRmR(sseOpcodeAddss, newOperandReg(xmm1VReg), xmm0VReg, true) },
@@ -1775,6 +1965,146 @@ func TestInstruction_format_encode(t *testing.T) {
 			want:       "66440f11797b",
 			wantFormat: "movupd %xmm15, 123(%rcx)",
 		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3c7",
+			wantFormat: "roll %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1c780",
+			wantFormat: "roll $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3c7",
+			wantFormat: "rolq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateLeft, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1c780",
+			wantFormat: "rolq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3cf",
+			wantFormat: "rorl %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1cf80",
+			wantFormat: "rorl $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3cf",
+			wantFormat: "rorq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpRotateRight, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1cf80",
+			wantFormat: "rorq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3e7",
+			wantFormat: "shll %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1e780",
+			wantFormat: "shll $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3e7",
+			wantFormat: "shlq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftLeft, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1e780",
+			wantFormat: "shlq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3ef",
+			wantFormat: "shrl %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1ef80",
+			wantFormat: "shrl $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3ef",
+			wantFormat: "shrq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightLogical, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1ef80",
+			wantFormat: "shrq $128, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), rdiVReg, false)
+			},
+			want:       "d3ff",
+			wantFormat: "sarl %ecx, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandImm32(128), rdiVReg, false)
+			},
+			want:       "c1ff80",
+			wantFormat: "sarl $128, %edi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), rdiVReg, true)
+			},
+			want:       "48d3ff",
+			wantFormat: "sarq %ecx, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asShiftR(shiftROpShiftRightArithmetic, newOperandImm32(128), rdiVReg, true)
+			},
+			want:       "48c1ff80",
+			wantFormat: "sarq $128, %rdi",
+		},
 	} {
 		tc := tc
 		t.Run(tc.wantFormat, func(t *testing.T) {
diff --git a/internal/integration_test/spectest/spectest.go b/internal/integration_test/spectest/spectest.go
index 77c16d59..87bc6fa1 100644
--- a/internal/integration_test/spectest/spectest.go
+++ b/internal/integration_test/spectest/spectest.go
@@ -287,6 +287,8 @@ func (c command) expectedError() (err error) {
 		panic("unreachable")
 	}
 	switch c.Text {
+	case "expected shared memory":
+		err = wasmruntime.ErrRuntimeExpectedSharedMemory
 	case "out of bounds memory access":
 		err = wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess
 	case "indirect call type mismatch", "indirect call":
@@ -299,6 +301,8 @@ func (c command) expectedError() (err error) {
 		err = wasmruntime.ErrRuntimeInvalidConversionToInteger
 	case "integer divide by zero":
 		err = wasmruntime.ErrRuntimeIntegerDivideByZero
+	case "unaligned atomic":
+		err = wasmruntime.ErrRuntimeUnalignedAtomic
 	case "unreachable":
 		err = wasmruntime.ErrRuntimeUnreachable
 	default:
@@ -336,7 +340,7 @@ func Run(t *testing.T, testDataFS embed.FS, ctx context.Context, config wazero.R
 
 	// If the go:embed path resolution was wrong, this fails.
 	// https://github.com/tetratelabs/wazero/issues/247
-	require.True(t, len(caseNames) > 1, "len(caseNames)=%d (not greater than one)", len(caseNames))
+	require.True(t, len(caseNames) > 0, "len(caseNames)=%d (not greater than zero)", len(caseNames))
 
 	for _, f := range caseNames {
 		RunCase(t, testDataFS, f, ctx, config, -1, 0, math.MaxInt)
diff --git a/internal/integration_test/spectest/threads/spec_test.go b/internal/integration_test/spectest/threads/spec_test.go
index 3bbbb042..966f7acb 100644
--- a/internal/integration_test/spectest/threads/spec_test.go
+++ b/internal/integration_test/spectest/threads/spec_test.go
@@ -26,6 +26,5 @@ func TestCompiler(t *testing.T) {
 }
 
 func TestInterpreter(t *testing.T) {
-	t.Skip() // TODO: Delete after implementing interpreter support
 	spectest.Run(t, testcases, context.Background(), wazero.NewRuntimeConfigInterpreter().WithCoreFeatures(enabledFeatures))
 }
diff --git a/internal/wasm/memory_test.go b/internal/wasm/memory_test.go
index 5957741d..e37c5ba4 100644
--- a/internal/wasm/memory_test.go
+++ b/internal/wasm/memory_test.go
@@ -5,6 +5,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 	"unsafe"
 
 	"github.com/tetratelabs/wazero/api"
@@ -797,3 +798,143 @@ func BenchmarkWriteString(b *testing.B) {
 		})
 	}
 }
+
+func TestMemoryInstance_WaitNotifyOnce(t *testing.T) {
+	reader := func(mem *MemoryInstance, offset uint32) uint32 {
+		val, _ := mem.ReadUint32Le(offset)
+		return val
+	}
+	t.Run("no waiters", func(t *testing.T) {
+		mem := &MemoryInstance{Buffer: []byte{0, 0, 0, 0, 16, 0, 0, 0}, Min: 1, Shared: true}
+
+		notifyWaiters(t, mem, 0, 1, 0)
+	})
+
+	t.Run("single wait, notify", func(t *testing.T) {
+		mem := &MemoryInstance{Buffer: []byte{0, 0, 0, 0, 16, 0, 0, 0}, Min: 1, Shared: true}
+
+		ch := make(chan string)
+		// Reuse same offset 3 times to verify reuse
+		for i := 0; i < 3; i++ {
+			go func() {
+				res := mem.Wait32(0, 0, -1, reader)
+				propagateWaitResult(t, ch, res)
+			}()
+
+			requireChannelEmpty(t, ch)
+			notifyWaiters(t, mem, 0, 1, 1)
+			require.Equal(t, "", <-ch)
+
+			notifyWaiters(t, mem, 0, 1, 0)
+		}
+	})
+
+	t.Run("multiple waiters, notify all", func(t *testing.T) {
+		mem := &MemoryInstance{Buffer: []byte{0, 0, 0, 0, 16, 0, 0, 0}, Min: 1, Shared: true}
+
+		ch := make(chan string)
+		go func() {
+			res := mem.Wait32(0, 0, -1, reader)
+			propagateWaitResult(t, ch, res)
+		}()
+		go func() {
+			res := mem.Wait32(0, 0, -1, reader)
+			propagateWaitResult(t, ch, res)
+		}()
+
+		requireChannelEmpty(t, ch)
+
+		notifyWaiters(t, mem, 0, 2, 2)
+		require.Equal(t, "", <-ch)
+		require.Equal(t, "", <-ch)
+	})
+
+	t.Run("multiple waiters, notify one", func(t *testing.T) {
+		mem := &MemoryInstance{Buffer: []byte{0, 0, 0, 0, 16, 0, 0, 0}, Min: 1, Shared: true}
+
+		ch := make(chan string)
+		go func() {
+			res := mem.Wait32(0, 0, -1, reader)
+			propagateWaitResult(t, ch, res)
+		}()
+		go func() {
+			res := mem.Wait32(0, 0, -1, reader)
+			propagateWaitResult(t, ch, res)
+		}()
+
+		requireChannelEmpty(t, ch)
+		notifyWaiters(t, mem, 0, 1, 1)
+		require.Equal(t, "", <-ch)
+		requireChannelEmpty(t, ch)
+		notifyWaiters(t, mem, 0, 1, 1)
+		require.Equal(t, "", <-ch)
+	})
+
+	t.Run("multiple offsets", func(t *testing.T) {
+		mem := &MemoryInstance{Buffer: []byte{0, 0, 0, 0, 16, 0, 0, 0}, Min: 1, Shared: true}
+
+		ch := make(chan string)
+		go func() {
+			res := mem.Wait32(0, 0, -1, reader)
+			propagateWaitResult(t, ch, res)
+		}()
+		go func() {
+			res := mem.Wait32(1, 268435456, -1, reader)
+			propagateWaitResult(t, ch, res)
+		}()
+
+		requireChannelEmpty(t, ch)
+		notifyWaiters(t, mem, 0, 2, 1)
+		require.Equal(t, "", <-ch)
+		requireChannelEmpty(t, ch)
+		notifyWaiters(t, mem, 1, 2, 1)
+		require.Equal(t, "", <-ch)
+	})
+
+	t.Run("timeout", func(t *testing.T) {
+		mem := &MemoryInstance{Buffer: []byte{0, 0, 0, 0, 16, 0, 0, 0}, Min: 1, Shared: true}
+
+		ch := make(chan string)
+		go func() {
+			res := mem.Wait32(0, 0, 10 /* ns */, reader)
+			propagateWaitResult(t, ch, res)
+		}()
+
+		require.Equal(t, "timeout", <-ch)
+	})
+}
+
+func notifyWaiters(t *testing.T, mem *MemoryInstance, offset, count, exp int) {
+	t.Helper()
+	cur := 0
+	tries := 0
+	for cur < exp {
+		if tries > 100 {
+			t.Fatal("too many tries waiting for wait and notify to converge")
+		}
+		n := mem.Notify(uint32(offset), uint32(count))
+		cur += int(n)
+		time.Sleep(1 * time.Millisecond)
+		tries++
+	}
+}
+
+func propagateWaitResult(t *testing.T, ch chan string, res uint64) {
+	t.Helper()
+	switch res {
+	case 2:
+		ch <- "timeout"
+	default:
+		ch <- ""
+	}
+}
+
+func requireChannelEmpty(t *testing.T, ch chan string) {
+	t.Helper()
+	select {
+	case <-ch:
+		t.Fatal("channel should be empty")
+	default:
+		// fallthrough
+	}
+}