Repository: tetratelabs/wazero
Pull request #1997 (2024-02-01 17:29:26)

wazevo(amd64): simd boolean, bitwise spec tests
implements anytrue, alltrue, bitmask, and, or, xor, andnot, bselect


Patch:
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go b/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
index dd84f430..89240251 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
@@ -954,6 +954,12 @@ func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) {
 		switch sseOpcode(i.u1) {
 		case sseOpcodeMovd, sseOpcodeMovq:
 			legPrefix, opcode, argSwap = legacyPrefixes0x66, 0x0f7e, false
+		case sseOpcodeMovmskps:
+			legPrefix, opcode, argSwap = legacyPrefixesNone, 0x0f50, true
+		case sseOpcodeMovmskpd:
+			legPrefix, opcode, argSwap = legacyPrefixes0x66, 0x0f50, true
+		case sseOpcodePmovmskb:
+			legPrefix, opcode, argSwap = legacyPrefixes0x66, 0x0fd7, true
 		case sseOpcodeCvttss2si:
 			legPrefix, opcode, argSwap = legacyPrefixes0xF3, 0x0f2c, true
 		case sseOpcodeCvttsd2si:
diff --git a/internal/engine/wazevo/backend/isa/amd64/machine.go b/internal/engine/wazevo/backend/isa/amd64/machine.go
index c04d1bd3..8e548596 100644
--- a/internal/engine/wazevo/backend/isa/amd64/machine.go
+++ b/internal/engine/wazevo/backend/isa/amd64/machine.go
@@ -380,6 +380,24 @@ func (m *machine) LowerInstr(instr *ssa.Instruction) {
 		rd := newOperandReg(m.c.VRegOf(instr.Return()))
 		m.lowerFcvtFromUint(rn, rd, x.Type() == ssa.TypeI64,
 			instr.Return().Type().Bits() == 64)
+	case ssa.OpcodeVanyTrue:
+		m.lowerVanyTrue(instr)
+	case ssa.OpcodeVallTrue:
+		m.lowerVallTrue(instr)
+	case ssa.OpcodeVhighBits:
+		m.lowerVhighBits(instr)
+	case ssa.OpcodeVbnot:
+		m.lowerVbnot(instr)
+	case ssa.OpcodeVband:
+		m.lowerVbBinOp(instr, sseOpcodePand)
+	case ssa.OpcodeVbor:
+		m.lowerVbBinOp(instr, sseOpcodePor)
+	case ssa.OpcodeVbxor:
+		m.lowerVbBinOp(instr, sseOpcodePxor)
+	case ssa.OpcodeVbandnot:
+		m.lowerVbandnot(instr, sseOpcodePandn)
+	case ssa.OpcodeVbitselect:
+		m.lowerVbitselect(instr)
 	case ssa.OpcodeUndefined:
 		m.insert(m.allocateInstr().asUD2())
 	case ssa.OpcodeExitWithCode:
@@ -2191,3 +2209,218 @@ func (m *machine) lowerFcvtFromUint(rn, rd operand, src64, dst64 bool) {
 	m.insert(endTarget)
 	m.copyTo(tmpXmm, rd.r)
 }
+
+func (m *machine) lowerVanyTrue(instr *ssa.Instruction) {
+	x := instr.Arg()
+	rm := m.getOperand_Reg(m.c.ValueDefinition(x))
+	rd := m.c.VRegOf(instr.Return())
+
+	tmp := m.c.AllocateVReg(ssa.TypeI32)
+
+	cmp := m.allocateInstr()
+	cmp.asXmmCmpRmR(sseOpcodePtest, rm, rm.r)
+	m.insert(cmp)
+
+	setcc := m.allocateInstr()
+	setcc.asSetcc(condNZ, tmp)
+	m.insert(setcc)
+
+	// Clear the irrelevant bits.
+	and := m.allocateInstr()
+	and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp, false)
+	m.insert(and)
+
+	m.copyTo(tmp, rd)
+}
+
+func (m *machine) lowerVallTrue(instr *ssa.Instruction) {
+	x, lane := instr.ArgWithLane()
+	var op sseOpcode
+	switch lane {
+	case ssa.VecLaneI8x16:
+		op = sseOpcodePcmpeqb
+	case ssa.VecLaneI16x8:
+		op = sseOpcodePcmpeqw
+	case ssa.VecLaneI32x4:
+		op = sseOpcodePcmpeqd
+	case ssa.VecLaneI64x2:
+		op = sseOpcodePcmpeqq
+	}
+	rm := m.getOperand_Reg(m.c.ValueDefinition(x))
+	rd := m.c.VRegOf(instr.Return())
+
+	tmp := m.c.AllocateVReg(ssa.TypeV128)
+
+	zeros := m.allocateInstr()
+	zeros.asZeros(tmp)
+	m.insert(zeros)
+
+	pcmp := m.allocateInstr()
+	pcmp.asXmmRmR(op, rm, tmp)
+	m.insert(pcmp)
+
+	test := m.allocateInstr()
+	test.asXmmCmpRmR(sseOpcodePtest, newOperandReg(tmp), tmp)
+	m.insert(test)
+
+	tmp2 := m.c.AllocateVReg(ssa.TypeI32)
+
+	setcc := m.allocateInstr()
+	setcc.asSetcc(condZ, tmp2)
+	m.insert(setcc)
+
+	// Clear the irrelevant bits.
+	and := m.allocateInstr()
+	and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, false)
+	m.insert(and)
+
+	m.copyTo(tmp2, rd)
+}
+
+func (m *machine) lowerVhighBits(instr *ssa.Instruction) {
+	x, lane := instr.ArgWithLane()
+	rm := m.getOperand_Reg(m.c.ValueDefinition(x))
+	rd := m.c.VRegOf(instr.Return())
+	switch lane {
+	case ssa.VecLaneI8x16:
+		mov := m.allocateInstr()
+		mov.asXmmToGpr(sseOpcodePmovmskb, rm.r, rd, false)
+		m.insert(mov)
+
+	case ssa.VecLaneI16x8:
+		// When we have:
+		// 	R1 = [R1(w1), R1(w2), R1(w3), R1(w4), R1(w5), R1(w6), R1(w7), R1(v8)]
+		// 	R2 = [R2(w1), R2(w2), R2(w3), R2(v4), R2(w5), R2(w6), R2(w7), R2(v8)]
+		//	where RX(wn) is n-th signed word (16-bit) of RX register,
+		//
+		// "PACKSSWB R1, R2" produces
+		//  R1 = [
+		// 		byte_sat(R1(w1)), byte_sat(R1(w2)), byte_sat(R1(w3)), byte_sat(R1(w4)),
+		// 		byte_sat(R1(w5)), byte_sat(R1(w6)), byte_sat(R1(w7)), byte_sat(R1(w8)),
+		// 		byte_sat(R2(w1)), byte_sat(R2(w2)), byte_sat(R2(w3)), byte_sat(R2(w4)),
+		// 		byte_sat(R2(w5)), byte_sat(R2(w6)), byte_sat(R2(w7)), byte_sat(R2(w8)),
+		//  ]
+		//  where R1 is the destination register, and
+		// 	byte_sat(w) = int8(w) if w fits as signed 8-bit,
+		//                0x80 if w is less than 0x80
+		//                0x7F if w is greater than 0x7f
+		//
+		// See https://www.felixcloutier.com/x86/packsswb:packssdw for detail.
+		//
+		// Therefore, v.register ends up having i-th and (i+8)-th bit set if i-th lane is negative (for i in 0..8).
+		tmp := m.copyToTmp(rm.r)
+		res := m.c.AllocateVReg(ssa.TypeI32)
+
+		pak := m.allocateInstr()
+		pak.asXmmRmR(sseOpcodePacksswb, rm, tmp)
+		m.insert(pak)
+
+		mov := m.allocateInstr()
+		mov.asXmmToGpr(sseOpcodePmovmskb, tmp, res, false)
+		m.insert(mov)
+
+		// Clear the higher bits than 8.
+		shr := m.allocateInstr()
+		shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(8), res, false)
+		m.insert(shr)
+
+		m.copyTo(res, rd)
+
+	case ssa.VecLaneI32x4:
+		mov := m.allocateInstr()
+		mov.asXmmToGpr(sseOpcodeMovmskps, rm.r, rd, true)
+		m.insert(mov)
+
+	case ssa.VecLaneI64x2:
+		mov := m.allocateInstr()
+		mov.asXmmToGpr(sseOpcodeMovmskpd, rm.r, rd, true)
+		m.insert(mov)
+	}
+}
+
+func (m *machine) lowerVbnot(instr *ssa.Instruction) {
+	x := instr.Arg()
+	xDef := m.c.ValueDefinition(x)
+	rm := m.getOperand_Reg(xDef)
+	rd := m.c.VRegOf(instr.Return())
+
+	tmp := m.copyToTmp(rm.r)
+	tmp2 := m.c.AllocateVReg(ssa.TypeV128)
+
+	// Ensure tmp2 is considered defined by regalloc.
+	m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
+
+	// Set all bits on tmp register.
+	pak := m.allocateInstr()
+	pak.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp2), tmp2)
+	m.insert(pak)
+
+	// Then XOR with tmp to reverse all bits on v.register.
+	xor := m.allocateInstr()
+	xor.asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)
+	m.insert(xor)
+
+	m.copyTo(tmp, rd)
+}
+
+func (m *machine) lowerVbBinOp(instr *ssa.Instruction, op sseOpcode) {
+	x, y := instr.Arg2()
+	xDef := m.c.ValueDefinition(x)
+	yDef := m.c.ValueDefinition(y)
+	rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
+	rd := m.c.VRegOf(instr.Return())
+
+	tmp := m.copyToTmp(rm.r)
+
+	// op between rn, rm.
+	binOp := m.allocateInstr()
+	binOp.asXmmRmR(op, rn, tmp)
+	m.insert(binOp)
+
+	m.copyTo(tmp, rd)
+}
+
+func (m *machine) lowerVbandnot(instr *ssa.Instruction, op sseOpcode) {
+	x, y := instr.Arg2()
+	xDef := m.c.ValueDefinition(x)
+	yDef := m.c.ValueDefinition(y)
+	rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
+	rd := m.c.VRegOf(instr.Return())
+
+	tmp := m.copyToTmp(rn.r)
+
+	// pandn between rn, rm.
+	pand := m.allocateInstr()
+	pand.asXmmRmR(sseOpcodePandn, rm, tmp)
+	m.insert(pand)
+
+	m.copyTo(tmp, rd)
+}
+
+func (m *machine) lowerVbitselect(instr *ssa.Instruction) {
+	c, x, y := instr.SelectData()
+	xDef := m.c.ValueDefinition(x)
+	yDef := m.c.ValueDefinition(y)
+	rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
+	creg := m.getOperand_Reg(m.c.ValueDefinition(c))
+	rd := m.c.VRegOf(instr.Return())
+
+	tmpC := m.copyToTmp(creg.r)
+	tmpX := m.copyToTmp(rm.r)
+
+	// And between c, x (overwrites x).
+	pand := m.allocateInstr()
+	pand.asXmmRmR(sseOpcodePand, creg, tmpX)
+	m.insert(pand)
+
+	// Andn between y, c (overwrites c).
+	pandn := m.allocateInstr()
+	pandn.asXmmRmR(sseOpcodePandn, rn, tmpC)
+	m.insert(pandn)
+
+	por := m.allocateInstr()
+	por.asXmmRmR(sseOpcodePor, newOperandReg(tmpC), tmpX)
+	m.insert(por)
+
+	m.copyTo(tmpX, rd)
+}
diff --git a/internal/engine/wazevo/backend/isa/arm64/instr.go b/internal/engine/wazevo/backend/isa/arm64/instr.go
index 7f9515fa..2eb74fa7 100644
--- a/internal/engine/wazevo/backend/isa/arm64/instr.go
+++ b/internal/engine/wazevo/backend/isa/arm64/instr.go
@@ -162,6 +162,7 @@ var defKinds = [numInstructionKinds]defKind{
 	movToFPSR:            defKindNone,
 	movFromFPSR:          defKindRD,
 	emitSourceOffsetInfo: defKindNone,
+	atomicRmw:            defKindRD,
 }
 
 // Defs returns the list of regalloc.VReg that are defined by the instruction.
@@ -285,6 +286,7 @@ var useKinds = [numInstructionKinds]useKind{
 	movFromFPSR:          useKindNone,
 	adr:                  useKindNone,
 	emitSourceOffsetInfo: useKindNone,
+	atomicRmw:            useKindRNRM,
 }
 
 // Uses returns the list of regalloc.VReg that are used by the instruction.
@@ -1460,6 +1462,18 @@ func (i *instruction) String() (str string) {
 		}
 	case exitSequence:
 		str = fmt.Sprintf("exit_sequence %s", formatVRegSized(i.rn.nr(), 64))
+	case atomicRmw:
+		m := atomicRmwOp(i.u1).String()
+		size := byte(32)
+		switch i.u2 {
+		case 8:
+			size = 64
+		case 2:
+			m = m + "h"
+		case 1:
+			m = m + "b"
+		}
+		str = fmt.Sprintf("%s %s, %s, %s", m, formatVRegSized(i.rm.nr(), size), formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), 64))
 	case udf:
 		str = "udf"
 	case emitSourceOffsetInfo:
@@ -1478,6 +1492,13 @@ func (i *instruction) asAdr(rd regalloc.VReg, offset int64) {
 	i.u1 = uint64(offset)
 }
 
+func (i *instruction) asAtomicRmw(op atomicRmwOp, rn, rs, rt operand, size uint64) {
+	i.kind = atomicRmw
+	i.rd, i.rn, i.rm = rt, rn, rs
+	i.u1 = uint64(op)
+	i.u2 = size
+}
+
 // TODO: delete unnecessary things.
 const (
 	// nop0 represents a no-op of zero size.
@@ -1644,6 +1665,8 @@ const (
 	// exitSequence consists of multiple instructions, and exits the execution immediately.
 	// See encodeExitSequence.
 	exitSequence
+	// atomicRmw represents an atomic read-modify-write operation with two register sources and a register destination.
+	atomicRmw
 	// UDF is the undefined instruction. For debugging only.
 	udf
 
@@ -1719,6 +1742,8 @@ func (a aluOp) String() string {
 		return "sub"
 	case aluOpOrr:
 		return "orr"
+	case aluOpOrn:
+		return "orn"
 	case aluOpAnd:
 		return "and"
 	case aluOpBic:
@@ -1760,6 +1785,8 @@ const (
 	aluOpSub
 	// 32/64-bit Bitwise OR.
 	aluOpOrr
+	// 32/64-bit Bitwise OR NOT.
+	aluOpOrn
 	// 32/64-bit Bitwise AND.
 	aluOpAnd
 	// 32/64-bit Bitwise AND NOT.
@@ -2388,3 +2415,36 @@ func ssaLaneToArrangement(lane ssa.VecLane) vecArrangement {
 		panic(lane)
 	}
 }
+
+// atomicRmwOp is the type of atomic read-modify-write operation.
+type atomicRmwOp byte
+
+const (
+	// atomicRmwOpAdd is an atomic add operation.
+	atomicRmwOpAdd atomicRmwOp = iota
+	// atomicRmwOpClr is an atomic clear operation, i.e. AND NOT.
+	atomicRmwOpClr
+	// atomicRmwOpSet is an atomic set operation, i.e. OR.
+	atomicRmwOpSet
+	// atomicRmwOpEor is an atomic exclusive OR operation.
+	atomicRmwOpEor
+	// atomicRmwOpSwp is an atomic swap operation.
+	atomicRmwOpSwp
+)
+
+// String implements fmt.Stringer
+func (a atomicRmwOp) String() string {
+	switch a {
+	case atomicRmwOpAdd:
+		return "ldaddal"
+	case atomicRmwOpClr:
+		return "ldclral"
+	case atomicRmwOpSet:
+		return "ldsetal"
+	case atomicRmwOpEor:
+		return "ldeoral"
+	case atomicRmwOpSwp:
+		return "swpal"
+	}
+	panic(fmt.Sprintf("unknown atomicRmwOp: %d", a))
+}
diff --git a/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go b/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
index cc69ad19..db35e035 100644
--- a/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
+++ b/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
@@ -386,6 +386,14 @@ func (i *instruction) encode(c backend.Compiler) {
 	case movToFPSR:
 		rt := regNumberInEncoding[i.rn.realReg()]
 		c.Emit4Bytes(encodeSystemRegisterMove(rt, false))
+	case atomicRmw:
+		c.Emit4Bytes(encodeAtomicRmw(
+			atomicRmwOp(i.u1),
+			regNumberInEncoding[i.rm.realReg()],
+			regNumberInEncoding[i.rd.realReg()],
+			regNumberInEncoding[i.rn.realReg()],
+			uint32(i.u2),
+		))
 	default:
 		panic(i.String())
 	}
@@ -1608,7 +1616,7 @@ func encodeAluRRR(op aluOp, rd, rn, rm uint32, _64bit, isRnSp bool) uint32 {
 		}
 		// "Shifted register" with shift = 0
 		_31to21 = 0b01101011_000
-	case aluOpAnd, aluOpOrr, aluOpEor:
+	case aluOpAnd, aluOpOrr, aluOpOrn, aluOpEor:
 		// "Logical (shifted register)".
 		var opc, n uint32
 		switch op {
@@ -1616,6 +1624,9 @@ func encodeAluRRR(op aluOp, rd, rn, rm uint32, _64bit, isRnSp bool) uint32 {
 			// all zeros
 		case aluOpOrr:
 			opc = 0b01
+		case aluOpOrn:
+			opc = 0b01
+			n = 1
 		case aluOpEor:
 			opc = 0b10
 		}
@@ -2218,3 +2229,35 @@ func encodeRet() uint32 {
 	// https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/RET--Return-from-subroutine-?lang=en
 	return 0b1101011001011111<<16 | regNumberInEncoding[lr]<<5
 }
+
+func encodeAtomicRmw(op atomicRmwOp, rs, rt, rn uint32, size uint32) uint32 {
+	var _31to21, _15to10, sz uint32
+
+	switch size {
+	case 8:
+		sz = 0b11
+	case 4:
+		sz = 0b10
+	case 2:
+		sz = 0b01
+	case 1:
+		sz = 0b00
+	}
+
+	_31to21 = 0b00111000_111 | sz<<9
+
+	switch op {
+	case atomicRmwOpAdd:
+		_15to10 = 0b000000
+	case atomicRmwOpClr:
+		_15to10 = 0b000100
+	case atomicRmwOpSet:
+		_15to10 = 0b001100
+	case atomicRmwOpEor:
+		_15to10 = 0b001000
+	case atomicRmwOpSwp:
+		_15to10 = 0b100000
+	}
+
+	return _31to21<<21 | rs<<16 | _15to10<<10 | rn<<5 | rt
+}
diff --git a/internal/engine/wazevo/backend/isa/arm64/lower_instr.go b/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
index dda56171..8ee83225 100644
--- a/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
+++ b/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
@@ -709,6 +709,10 @@ func (m *machine) LowerInstr(instr *ssa.Instruction) {
 	case ssa.OpcodeLoadSplat:
 		ptr, offset, lane := instr.LoadSplatData()
 		m.lowerLoadSplat(ptr, offset, lane, instr.Return())
+
+	case ssa.OpcodeAtomicRmw:
+		m.lowerAtomicRmw(instr)
+
 	default:
 		panic("TODO: lowering " + op.String())
 	}
@@ -1983,6 +1987,66 @@ func (m *machine) lowerSelectVec(rc, rn, rm, rd operand) {
 	m.insert(mov2)
 }
 
+func (m *machine) lowerAtomicRmw(si *ssa.Instruction) {
+	ssaOp, size := si.AtomicRmwData()
+
+	var op atomicRmwOp
+	var negateArg bool
+	var flipArg bool
+	switch ssaOp {
+	case ssa.AtomicRmwOpAdd:
+		op = atomicRmwOpAdd
+	case ssa.AtomicRmwOpSub:
+		op = atomicRmwOpAdd
+		negateArg = true
+	case ssa.AtomicRmwOpAnd:
+		op = atomicRmwOpClr
+		flipArg = true
+	case ssa.AtomicRmwOpOr:
+		op = atomicRmwOpSet
+	case ssa.AtomicRmwOpXor:
+		op = atomicRmwOpEor
+	case ssa.AtomicRmwOpXchg:
+		op = atomicRmwOpSwp
+	default:
+		panic(fmt.Sprintf("unknown ssa atomic rmw op: %s", ssaOp))
+	}
+
+	addr, val := si.Arg2()
+	addrDef, valDef := m.compiler.ValueDefinition(addr), m.compiler.ValueDefinition(val)
+	rn := m.getOperand_NR(addrDef, extModeNone)
+	rt := operandNR(m.compiler.VRegOf(si.Return()))
+	rs := m.getOperand_NR(valDef, extModeNone)
+
+	_64 := si.Return().Type().Bits() == 64
+	var tmp operand
+	if _64 {
+		tmp = operandNR(m.compiler.AllocateVReg(ssa.TypeI64))
+	} else {
+		tmp = operandNR(m.compiler.AllocateVReg(ssa.TypeI32))
+	}
+	m.lowerAtomicRmwImpl(op, rn, rs, rt, tmp, size, negateArg, flipArg, _64)
+}
+
+func (m *machine) lowerAtomicRmwImpl(op atomicRmwOp, rn, rs, rt, tmp operand, size uint64, negateArg, flipArg, dst64bit bool) {
+	switch {
+	case negateArg:
+		neg := m.allocateInstr()
+		neg.asALU(aluOpSub, tmp, operandNR(xzrVReg), rs, dst64bit)
+		m.insert(neg)
+	case flipArg:
+		flip := m.allocateInstr()
+		flip.asALU(aluOpOrn, tmp, operandNR(xzrVReg), rs, dst64bit)
+		m.insert(flip)
+	default:
+		tmp = rs
+	}
+
+	rmw := m.allocateInstr()
+	rmw.asAtomicRmw(op, rn, tmp, rt, size)
+	m.insert(rmw)
+}
+
 // copyToTmp copies the given regalloc.VReg to a temporary register. This is called before cbr to avoid the regalloc issue
 // e.g. reload happening in the middle of the exit sequence which is not the path the normal path executes
 func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg {
diff --git a/internal/engine/wazevo/frontend/lower.go b/internal/engine/wazevo/frontend/lower.go
index 60cc62a3..f09cc20e 100644
--- a/internal/engine/wazevo/frontend/lower.go
+++ b/internal/engine/wazevo/frontend/lower.go
@@ -3216,6 +3216,63 @@ func (c *Compiler) lowerCurrentOpcode() {
 				AsCallIndirect(memoryNotifyPtr, &c.memoryNotifySig, args).
 				Insert(builder).Return()
 			state.push(memoryNotifyRet)
+		case wasm.OpcodeAtomicI32Load, wasm.OpcodeAtomicI64Load, wasm.OpcodeAtomicI32Load8U, wasm.OpcodeAtomicI32Load16U, wasm.OpcodeAtomicI64Load8U, wasm.OpcodeAtomicI64Load16U, wasm.OpcodeAtomicI64Load32U:
+			_, offset := c.readMemArg()
+			if state.unreachable {
+				break
+			}
+
+			baseAddr := state.pop()
+
+			var size uint64
+			switch atomicOp {
+			case wasm.OpcodeAtomicI64Load:
+				size = 8
+			case wasm.OpcodeAtomicI32Load, wasm.OpcodeAtomicI64Load32U:
+				size = 4
+			case wasm.OpcodeAtomicI32Load16U, wasm.OpcodeAtomicI64Load16U:
+				size = 2
+			case wasm.OpcodeAtomicI32Load8U, wasm.OpcodeAtomicI64Load8U:
+				size = 1
+			}
+
+			var typ ssa.Type
+			switch atomicOp {
+			case wasm.OpcodeAtomicI64Load, wasm.OpcodeAtomicI64Load32U, wasm.OpcodeAtomicI64Load16U, wasm.OpcodeAtomicI64Load8U:
+				typ = ssa.TypeI64
+			case wasm.OpcodeAtomicI32Load, wasm.OpcodeAtomicI32Load16U, wasm.OpcodeAtomicI32Load8U:
+				typ = ssa.TypeI32
+			}
+
+			addr := c.memOpSetup(baseAddr, uint64(offset), size)
+			c.memAlignmentCheck(addr, size)
+			res := builder.AllocateInstruction().AsAtomicLoad(addr, size, typ).Insert(builder).Return()
+			state.push(res)
+		case wasm.OpcodeAtomicI32Store, wasm.OpcodeAtomicI64Store, wasm.OpcodeAtomicI32Store8, wasm.OpcodeAtomicI32Store16, wasm.OpcodeAtomicI64Store8, wasm.OpcodeAtomicI64Store16, wasm.OpcodeAtomicI64Store32:
+			_, offset := c.readMemArg()
+			if state.unreachable {
+				break
+			}
+
+			val := state.pop()
+			baseAddr := state.pop()
+
+			var size uint64
+			switch atomicOp {
+			case wasm.OpcodeAtomicI64Store:
+				size = 8
+			case wasm.OpcodeAtomicI32Store, wasm.OpcodeAtomicI64Store32:
+				size = 4
+			case wasm.OpcodeAtomicI32Store16, wasm.OpcodeAtomicI64Store16:
+				size = 2
+			case wasm.OpcodeAtomicI32Store8, wasm.OpcodeAtomicI64Store8:
+				size = 1
+			}
+
+			addr := c.memOpSetup(baseAddr, uint64(offset), size)
+			c.memAlignmentCheck(addr, size)
+			res := builder.AllocateInstruction().AsAtomicStore(addr, val, size).Insert(builder).Return()
+			state.push(res)
 		case wasm.OpcodeAtomicI32RmwAdd, wasm.OpcodeAtomicI64RmwAdd, wasm.OpcodeAtomicI32Rmw8AddU, wasm.OpcodeAtomicI32Rmw16AddU, wasm.OpcodeAtomicI64Rmw8AddU, wasm.OpcodeAtomicI64Rmw16AddU, wasm.OpcodeAtomicI64Rmw32AddU,
 			wasm.OpcodeAtomicI32RmwSub, wasm.OpcodeAtomicI64RmwSub, wasm.OpcodeAtomicI32Rmw8SubU, wasm.OpcodeAtomicI32Rmw16SubU, wasm.OpcodeAtomicI64Rmw8SubU, wasm.OpcodeAtomicI64Rmw16SubU, wasm.OpcodeAtomicI64Rmw32SubU,
 			wasm.OpcodeAtomicI32RmwAnd, wasm.OpcodeAtomicI64RmwAnd, wasm.OpcodeAtomicI32Rmw8AndU, wasm.OpcodeAtomicI32Rmw16AndU, wasm.OpcodeAtomicI64Rmw8AndU, wasm.OpcodeAtomicI64Rmw16AndU, wasm.OpcodeAtomicI64Rmw32AndU,
diff --git a/internal/engine/wazevo/ssa/instructions.go b/internal/engine/wazevo/ssa/instructions.go
index c28831ac..11ddd96a 100644
--- a/internal/engine/wazevo/ssa/instructions.go
+++ b/internal/engine/wazevo/ssa/instructions.go
@@ -854,6 +854,8 @@ var instructionSideEffects = [opcodeEnd]sideEffect{
 	OpcodeVFcvtToSintSat:     sideEffectNone,
 	OpcodeVZeroExtLoad:       sideEffectNone,
 	OpcodeAtomicRmw:          sideEffectStrict,
+	OpcodeAtomicLoad:         sideEffectStrict,
+	OpcodeAtomicStore:        sideEffectStrict,
 }
 
 // sideEffect returns true if this instruction has side effects.
@@ -1035,6 +1037,8 @@ var instructionReturnTypes = [opcodeEnd]returnTypesFn{
 	OpcodeVFcvtToUintSat:     returnTypesFnV128,
 	OpcodeVFcvtToSintSat:     returnTypesFnV128,
 	OpcodeAtomicRmw:          returnTypesFnSingle,
+	OpcodeAtomicLoad:         returnTypesFnSingle,
+	OpcodeAtomicStore:        returnTypesFnNoReturns,
 }
 
 // AsLoad initializes this instruction as a store instruction with OpcodeLoad.
@@ -1959,6 +1963,27 @@ func (i *Instruction) AsWiden(v Value, lane VecLane, signed, low bool) *Instruct
 	return i
 }
 
+// AsAtomicLoad initializes this instruction as an atomic load.
+// The size is in bytes and must be 1, 2, 4, or 8.
+func (i *Instruction) AsAtomicLoad(addr Value, size uint64, typ Type) *Instruction {
+	i.opcode = OpcodeAtomicLoad
+	i.u1 = size
+	i.v = addr
+	i.typ = typ
+	return i
+}
+
+// AsAtomicLoad initializes this instruction as an atomic store.
+// The size is in bytes and must be 1, 2, 4, or 8.
+func (i *Instruction) AsAtomicStore(addr, val Value, size uint64) *Instruction {
+	i.opcode = OpcodeAtomicStore
+	i.u1 = size
+	i.v = addr
+	i.v2 = val
+	i.typ = val.Type()
+	return i
+}
+
 // AsAtomicRmw initializes this instruction as an atomic read-modify-write.
 // The size is in bytes and must be 1, 2, 4, or 8.
 func (i *Instruction) AsAtomicRmw(op AtomicRmwOp, addr, val Value, size uint64) *Instruction {
@@ -1971,6 +1996,11 @@ func (i *Instruction) AsAtomicRmw(op AtomicRmwOp, addr, val Value, size uint64)
 	return i
 }
 
+// AtomicRmwData returns the data for this atomic read-modify-write instruction.
+func (i *Instruction) AtomicRmwData() (op AtomicRmwOp, size uint64) {
+	return AtomicRmwOp(i.u1), i.u2
+}
+
 // ReturnVals returns the return values of OpcodeReturn.
 func (i *Instruction) ReturnVals() []Value {
 	return i.vs
@@ -2529,6 +2559,10 @@ func (i *Instruction) Format(b Builder) string {
 		instSuffix = fmt.Sprintf(".%v %s, %s", lanes, i.v.Format(b), i.v2.Format(b))
 	case OpcodeAtomicRmw:
 		instSuffix = fmt.Sprintf(" %s_%d, %s, %s", AtomicRmwOp(i.u1), 8*i.u2, i.v.Format(b), i.v2.Format(b))
+	case OpcodeAtomicLoad:
+		instSuffix = fmt.Sprintf("_%d, %s", 8*i.u1, i.v.Format(b))
+	case OpcodeAtomicStore:
+		instSuffix = fmt.Sprintf("_%d, %s, %s", 8*i.u1, i.v.Format(b), i.v2.Format(b))
 	default:
 		panic(fmt.Sprintf("TODO: format for %s", i.opcode))
 	}


Test patch:
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go b/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
index 394cf00f..33822279 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
@@ -1433,6 +1433,36 @@ func TestInstruction_format_encode(t *testing.T) {
 			want:       "f20f2cf8",
 			wantFormat: "cvttsd2si %xmm0, %edi",
 		},
+		{
+			setup:      func(i *instruction) { i.asXmmToGpr(sseOpcodeMovmskps, xmm1VReg, xmm0VReg, true) },
+			want:       "480f50c1",
+			wantFormat: "movmskps %xmm1, %xmm0",
+		},
+		{
+			setup:      func(i *instruction) { i.asXmmToGpr(sseOpcodeMovmskpd, xmm1VReg, xmm0VReg, true) },
+			want:       "66480f50c1",
+			wantFormat: "movmskpd %xmm1, %xmm0",
+		},
+		{
+			setup:      func(i *instruction) { i.asXmmToGpr(sseOpcodePmovmskb, xmm1VReg, xmm0VReg, true) },
+			want:       "66480fd7c1",
+			wantFormat: "pmovmskb %xmm1, %xmm0",
+		},
+		{
+			setup:      func(i *instruction) { i.asXmmToGpr(sseOpcodeMovmskps, xmm1VReg, xmm0VReg, false) },
+			want:       "0f50c1",
+			wantFormat: "movmskps %xmm1, %xmm0",
+		},
+		{
+			setup:      func(i *instruction) { i.asXmmToGpr(sseOpcodeMovmskpd, xmm1VReg, xmm0VReg, false) },
+			want:       "660f50c1",
+			wantFormat: "movmskpd %xmm1, %xmm0",
+		},
+		{
+			setup:      func(i *instruction) { i.asXmmToGpr(sseOpcodePmovmskb, xmm1VReg, xmm0VReg, false) },
+			want:       "660fd7c1",
+			wantFormat: "pmovmskb %xmm1, %xmm0",
+		},
 		{
 			setup:      func(i *instruction) { i.asPop64(raxVReg) },
 			want:       "58",
diff --git a/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go b/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go
index 924a8119..8757ce1f 100644
--- a/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go
+++ b/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go
@@ -1062,6 +1062,12 @@ func TestInstruction_encode(t *testing.T) {
 		{want: "202cc29a", setup: func(i *instruction) {
 			i.asALU(aluOpRotR, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), true)
 		}},
+		{want: "2000222a", setup: func(i *instruction) {
+			i.asALU(aluOpOrn, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), false)
+		}},
+		{want: "200022aa", setup: func(i *instruction) {
+			i.asALU(aluOpOrn, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), true)
+		}},
 		{want: "30000010", setup: func(i *instruction) { i.asAdr(v16VReg, 4) }},
 		{want: "50050030", setup: func(i *instruction) { i.asAdr(v16VReg, 169) }},
 		{want: "101e302e", setup: func(i *instruction) { i.asLoadFpuConst32(v16VReg, uint64(math.Float32bits(0))) }},
@@ -1607,6 +1613,111 @@ func TestInstruction_encode(t *testing.T) {
 		{want: "41c4404d", setup: func(i *instruction) { i.asVecLoad1R(operandNR(v1VReg), operandNR(x2VReg), vecArrangement8H) }},
 		{want: "41c8404d", setup: func(i *instruction) { i.asVecLoad1R(operandNR(v1VReg), operandNR(x2VReg), vecArrangement4S) }},
 		{want: "41cc404d", setup: func(i *instruction) { i.asVecLoad1R(operandNR(v1VReg), operandNR(x2VReg), vecArrangement2D) }},
+		{want: "0200e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpAdd, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0200e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpAdd, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0200e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpAdd, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0200e1f8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpAdd, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 8)
+		}},
+		{want: "0200e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpAdd, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0200e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpAdd, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0200e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpAdd, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0210e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpClr, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0210e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpClr, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0210e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpClr, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0210e1f8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpClr, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 8)
+		}},
+		{want: "0210e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpClr, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0210e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpClr, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0210e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpClr, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0230e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSet, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0230e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSet, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0230e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSet, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0230e1f8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSet, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 8)
+		}},
+		{want: "0230e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSet, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0230e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSet, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0230e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSet, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0220e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpEor, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0220e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpEor, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0220e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpEor, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0220e1f8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpEor, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 8)
+		}},
+		{want: "0220e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpEor, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0220e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpEor, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0220e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpEor, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0280e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSwp, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0280e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSwp, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0280e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSwp, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
+		{want: "0280e1f8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSwp, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 8)
+		}},
+		{want: "0280e1b8", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSwp, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 4)
+		}},
+		{want: "0280e178", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSwp, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 2)
+		}},
+		{want: "0280e138", setup: func(i *instruction) {
+			i.asAtomicRmw(atomicRmwOpSwp, operandNR(x0VReg), operandNR(x1VReg), operandNR(x2VReg), 1)
+		}},
 		{want: "4201231e4201631e4201239e4201639e4201221e4201621e4201229e4201629e", setup: func(i *instruction) {
 			i.asNop0()
 			cur := i
diff --git a/internal/engine/wazevo/backend/isa/arm64/lower_instr_test.go b/internal/engine/wazevo/backend/isa/arm64/lower_instr_test.go
index 913162e7..8834d097 100644
--- a/internal/engine/wazevo/backend/isa/arm64/lower_instr_test.go
+++ b/internal/engine/wazevo/backend/isa/arm64/lower_instr_test.go
@@ -959,3 +959,429 @@ ror x4?, x2?, x1?
 		})
 	}
 }
+
+func TestMachine_lowerAtomicRmw(t *testing.T) {
+	tests := []struct {
+		name      string
+		op        atomicRmwOp
+		negateArg bool
+		flipArg   bool
+		_64bit    bool
+		size      uint64
+		exp       string
+	}{
+		{
+			name: "add 32",
+			op:   atomicRmwOpAdd,
+			size: 4,
+			exp: `
+ldaddal w3?, w4?, x2?
+`,
+		},
+		{
+			name: "add 32_16u",
+			op:   atomicRmwOpAdd,
+			size: 2,
+			exp: `
+ldaddalh w3?, w4?, x2?
+`,
+		},
+		{
+			name: "add 32_8u",
+			op:   atomicRmwOpAdd,
+			size: 1,
+			exp: `
+ldaddalb w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "add 64",
+			op:     atomicRmwOpAdd,
+			size:   8,
+			_64bit: true,
+			exp: `
+ldaddal x3?, x4?, x2?
+`,
+		},
+		{
+			name:   "add 64_32u",
+			op:     atomicRmwOpAdd,
+			size:   4,
+			_64bit: true,
+			exp: `
+ldaddal w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "add 64_16u",
+			op:     atomicRmwOpAdd,
+			size:   2,
+			_64bit: true,
+			exp: `
+ldaddalh w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "add 64_8u",
+			op:     atomicRmwOpAdd,
+			size:   1,
+			_64bit: true,
+			exp: `
+ldaddalb w3?, w4?, x2?
+`,
+		},
+		{
+			name:      "sub 32",
+			op:        atomicRmwOpAdd,
+			negateArg: true,
+			size:      4,
+			exp: `
+sub w1?, wzr, w3?
+ldaddal w1?, w4?, x2?
+`,
+		},
+		{
+			name:      "sub 32_16u",
+			op:        atomicRmwOpAdd,
+			negateArg: true,
+			size:      2,
+			exp: `
+sub w1?, wzr, w3?
+ldaddalh w1?, w4?, x2?
+`,
+		},
+		{
+			name:      "sub 32_8u",
+			op:        atomicRmwOpAdd,
+			negateArg: true,
+			size:      1,
+			exp: `
+sub w1?, wzr, w3?
+ldaddalb w1?, w4?, x2?
+`,
+		},
+		{
+			name:      "sub 64",
+			op:        atomicRmwOpAdd,
+			negateArg: true,
+			size:      8,
+			_64bit:    true,
+			exp: `
+sub x1?, xzr, x3?
+ldaddal x1?, x4?, x2?
+`,
+		},
+		{
+			name:      "sub 64_32u",
+			op:        atomicRmwOpAdd,
+			negateArg: true,
+			size:      4,
+			_64bit:    true,
+			exp: `
+sub x1?, xzr, x3?
+ldaddal w1?, w4?, x2?
+`,
+		},
+		{
+			name:      "sub 64_16u",
+			op:        atomicRmwOpAdd,
+			negateArg: true,
+			size:      2,
+			_64bit:    true,
+			exp: `
+sub x1?, xzr, x3?
+ldaddalh w1?, w4?, x2?
+`,
+		},
+		{
+			name:      "sub 64_8u",
+			op:        atomicRmwOpAdd,
+			negateArg: true,
+			size:      1,
+			_64bit:    true,
+			exp: `
+sub x1?, xzr, x3?
+ldaddalb w1?, w4?, x2?
+`,
+		},
+		{
+			name:    "and 32",
+			op:      atomicRmwOpClr,
+			flipArg: true,
+			size:    4,
+			exp: `
+orn w1?, wzr, w3?
+ldclral w1?, w4?, x2?
+`,
+		},
+		{
+			name:    "and 32_16u",
+			op:      atomicRmwOpClr,
+			flipArg: true,
+			size:    2,
+			exp: `
+orn w1?, wzr, w3?
+ldclralh w1?, w4?, x2?
+`,
+		},
+		{
+			name:    "and 32_8u",
+			op:      atomicRmwOpClr,
+			flipArg: true,
+			size:    1,
+			exp: `
+orn w1?, wzr, w3?
+ldclralb w1?, w4?, x2?
+`,
+		},
+		{
+			name:    "and 64",
+			op:      atomicRmwOpClr,
+			flipArg: true,
+			size:    8,
+			_64bit:  true,
+			exp: `
+orn x1?, xzr, x3?
+ldclral x1?, x4?, x2?
+`,
+		},
+		{
+			name:    "and 64_32u",
+			op:      atomicRmwOpClr,
+			flipArg: true,
+			size:    4,
+			_64bit:  true,
+			exp: `
+orn x1?, xzr, x3?
+ldclral w1?, w4?, x2?
+`,
+		},
+		{
+			name:    "and 64_16u",
+			op:      atomicRmwOpClr,
+			flipArg: true,
+			size:    2,
+			_64bit:  true,
+			exp: `
+orn x1?, xzr, x3?
+ldclralh w1?, w4?, x2?
+`,
+		},
+		{
+			name:    "and 64_8u",
+			op:      atomicRmwOpClr,
+			flipArg: true,
+			size:    1,
+			_64bit:  true,
+			exp: `
+orn x1?, xzr, x3?
+ldclralb w1?, w4?, x2?
+`,
+		},
+		{
+			name: "or 32",
+			op:   atomicRmwOpSet,
+			size: 4,
+			exp: `
+ldsetal w3?, w4?, x2?
+`,
+		},
+		{
+			name: "or 32_16u",
+			op:   atomicRmwOpSet,
+			size: 2,
+			exp: `
+ldsetalh w3?, w4?, x2?
+`,
+		},
+		{
+			name: "or 32_8u",
+			op:   atomicRmwOpSet,
+			size: 1,
+			exp: `
+ldsetalb w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "or 64",
+			op:     atomicRmwOpSet,
+			size:   8,
+			_64bit: true,
+			exp: `
+ldsetal x3?, x4?, x2?
+`,
+		},
+		{
+			name:   "or 64_32u",
+			op:     atomicRmwOpSet,
+			size:   4,
+			_64bit: true,
+			exp: `
+ldsetal w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "or 64_16u",
+			op:     atomicRmwOpSet,
+			size:   2,
+			_64bit: true,
+			exp: `
+ldsetalh w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "or 64_8u",
+			op:     atomicRmwOpSet,
+			size:   1,
+			_64bit: true,
+			exp: `
+ldsetalb w3?, w4?, x2?
+`,
+		},
+		{
+			name: "xor 32",
+			op:   atomicRmwOpEor,
+			size: 4,
+			exp: `
+ldeoral w3?, w4?, x2?
+`,
+		},
+		{
+			name: "xor 32_16u",
+			op:   atomicRmwOpEor,
+			size: 2,
+			exp: `
+ldeoralh w3?, w4?, x2?
+`,
+		},
+		{
+			name: "xor 32_8u",
+			op:   atomicRmwOpEor,
+			size: 1,
+			exp: `
+ldeoralb w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "xor 64",
+			op:     atomicRmwOpEor,
+			size:   8,
+			_64bit: true,
+			exp: `
+ldeoral x3?, x4?, x2?
+`,
+		},
+		{
+			name:   "xor 64_32u",
+			op:     atomicRmwOpEor,
+			size:   4,
+			_64bit: true,
+			exp: `
+ldeoral w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "xor 64_16u",
+			op:     atomicRmwOpEor,
+			size:   2,
+			_64bit: true,
+			exp: `
+ldeoralh w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "xor 64_8u",
+			op:     atomicRmwOpEor,
+			size:   1,
+			_64bit: true,
+			exp: `
+ldeoralb w3?, w4?, x2?
+`,
+		},
+		{
+			name: "xchg 32",
+			op:   atomicRmwOpSwp,
+			size: 4,
+			exp: `
+swpal w3?, w4?, x2?
+`,
+		},
+		{
+			name: "xchg 32_16u",
+			op:   atomicRmwOpSwp,
+			size: 2,
+			exp: `
+swpalh w3?, w4?, x2?
+`,
+		},
+		{
+			name: "xchg 32_8u",
+			op:   atomicRmwOpSwp,
+			size: 1,
+			exp: `
+swpalb w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "xchg 64",
+			op:     atomicRmwOpSwp,
+			size:   8,
+			_64bit: true,
+			exp: `
+swpal x3?, x4?, x2?
+`,
+		},
+		{
+			name:   "xchg 64_32u",
+			op:     atomicRmwOpSwp,
+			size:   4,
+			_64bit: true,
+			exp: `
+swpal w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "xchg 64_16u",
+			op:     atomicRmwOpSwp,
+			size:   2,
+			_64bit: true,
+			exp: `
+swpalh w3?, w4?, x2?
+`,
+		},
+		{
+			name:   "xchg 64_8u",
+			op:     atomicRmwOpSwp,
+			size:   1,
+			_64bit: true,
+			exp: `
+swpalb w3?, w4?, x2?
+`,
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			_, _, m := newSetupWithMockContext()
+			var typ ssa.Type
+			if tc._64bit {
+				typ = ssa.TypeI64
+			} else {
+				typ = ssa.TypeI32
+			}
+			tmp := operandNR(m.compiler.AllocateVReg(typ))
+			rn := operandNR(m.compiler.AllocateVReg(ssa.TypeI64))
+			rs := operandNR(m.compiler.AllocateVReg(typ))
+			rt := operandNR(m.compiler.AllocateVReg(typ))
+
+			require.Equal(t, 1, int(tmp.reg().ID()))
+			require.Equal(t, 2, int(rn.reg().ID()))
+			require.Equal(t, 3, int(rs.reg().ID()))
+			require.Equal(t, 4, int(rt.reg().ID()))
+
+			m.lowerAtomicRmwImpl(tc.op, rn, rs, rt, tmp, tc.size, tc.negateArg, tc.flipArg, tc._64bit)
+			require.Equal(t, tc.exp, "\n"+formatEmittedInstructionsInCurrentBlock(m)+"\n")
+		})
+	}
+}
diff --git a/internal/engine/wazevo/e2e_test.go b/internal/engine/wazevo/e2e_test.go
index 123e7ce1..95719731 100644
--- a/internal/engine/wazevo/e2e_test.go
+++ b/internal/engine/wazevo/e2e_test.go
@@ -52,6 +52,7 @@ func TestE2E(t *testing.T) {
 		calls       []callCase
 		features    api.CoreFeatures
 		skipAMD64   bool
+		setupMemory func(mem api.Memory)
 	}{
 		{
 			name: "empty", m: testcases.Empty.Module,
@@ -646,6 +647,143 @@ func TestE2E(t *testing.T) {
 				{params: []uint64{0x4, 0x1}, expResults: []uint64{0}}, // no waiters, returns 0
 			},
 		},
+		{
+			name:      "atomic_rmw_add",
+			m:         testcases.AtomicRmwAdd.Module,
+			features:  api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			skipAMD64: true,
+			calls: []callCase{
+				{params: []uint64{1, 2, 3, 4, 5, 6, 7}, expResults: []uint64{0, 0, 0, 0, 0, 0, 0}},
+				{params: []uint64{1, 2, 3, 4, 5, 6, 7}, expResults: []uint64{1, 2, 3, 4, 5, 6, 7}},
+				{params: []uint64{1, 2, 3, 4, 5, 6, 7}, expResults: []uint64{2, 4, 6, 8, 10, 12, 14}},
+			},
+		},
+		{
+			name:      "atomic_rmw_sub",
+			m:         testcases.AtomicRmwSub.Module,
+			features:  api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			skipAMD64: true,
+			calls: []callCase{
+				{params: []uint64{1, 2, 3, 4, 5, 6, 7}, expResults: []uint64{0, 0, 0, 0, 0, 0, 0}},
+				{
+					params: []uint64{1, 2, 3, 4, 5, 6, 7},
+					expResults: []uint64{
+						api.EncodeI32(-1) & 0xff,
+						api.EncodeI32(-2) & 0xffff,
+						api.EncodeI32(-3),
+						api.EncodeI64(-4) & 0xff,
+						api.EncodeI64(-5) & 0xffff,
+						api.EncodeI64(-6) & 0xffffffff,
+						api.EncodeI64(-7),
+					},
+				},
+				{
+					params: []uint64{1, 2, 3, 4, 5, 6, 7},
+					expResults: []uint64{
+						api.EncodeI32(-2) & 0xff,
+						api.EncodeI32(-4) & 0xffff,
+						api.EncodeI32(-6),
+						api.EncodeI64(-8) & 0xff,
+						api.EncodeI64(-10) & 0xffff,
+						api.EncodeI64(-12) & 0xffffffff,
+						api.EncodeI64(-14),
+					},
+				},
+			},
+		},
+		{
+			name:      "atomic_rmw_and",
+			m:         testcases.AtomicRmwAnd.Module,
+			features:  api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			skipAMD64: true,
+			setupMemory: func(mem api.Memory) {
+				mem.WriteUint32Le(0, 0xffffffff)
+				mem.WriteUint32Le(8, 0xffffffff)
+				mem.WriteUint32Le(16, 0xffffffff)
+				mem.WriteUint64Le(24, 0xffffffffffffffff)
+				mem.WriteUint64Le(32, 0xffffffffffffffff)
+				mem.WriteUint64Le(40, 0xffffffffffffffff)
+				mem.WriteUint64Le(48, 0xffffffffffffffff)
+			},
+			calls: []callCase{
+				{
+					params:     []uint64{0xfffffffe, 0xfffffffe, 0xfffffffe, 0xfffffffffffffffe, 0xfffffffffffffffe, 0xfffffffffffffffe, 0xfffffffffffffffe},
+					expResults: []uint64{0xff, 0xffff, 0xffffffff, 0xff, 0xffff, 0xffffffff, 0xffffffffffffffff},
+				},
+				{
+					params:     []uint64{0xffffffee, 0xffffffee, 0xffffffee, 0xffffffffffffffee, 0xffffffffffffffee, 0xffffffffffffffee, 0xffffffffffffffee},
+					expResults: []uint64{0xfe, 0xfffe, 0xfffffffe, 0xfe, 0xfffe, 0xfffffffe, 0xfffffffffffffffe},
+				},
+				{
+					params:     []uint64{0xffffffee, 0xffffffee, 0xffffffee, 0xffffffffffffffee, 0xffffffffffffffee, 0xffffffffffffffee, 0xffffffffffffffee},
+					expResults: []uint64{0xee, 0xffee, 0xffffffee, 0xee, 0xffee, 0xffffffee, 0xffffffffffffffee},
+				},
+			},
+		},
+		{
+			name:      "atomic_rmw_or",
+			m:         testcases.AtomicRmwOr.Module,
+			features:  api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			skipAMD64: true,
+			calls: []callCase{
+				{
+					params:     []uint64{0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f},
+					expResults: []uint64{0, 0, 0, 0, 0, 0, 0},
+				},
+				{
+					params:     []uint64{0xaf, 0xaf, 0xaf, 0xaf, 0xaf, 0xaf, 0xaf},
+					expResults: []uint64{0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f},
+				},
+				{
+					params:     []uint64{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+					expResults: []uint64{0xaf, 0xaf, 0xaf, 0xaf, 0xaf, 0xaf, 0xaf},
+				},
+			},
+		},
+		{
+			name:      "atomic_rmw_xor",
+			m:         testcases.AtomicRmwXor.Module,
+			features:  api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			skipAMD64: true,
+			calls: []callCase{
+				{
+					params:     []uint64{0, 0, 0, 0, 0, 0, 0},
+					expResults: []uint64{0, 0, 0, 0, 0, 0, 0},
+				},
+				{
+					params:     []uint64{0xffffffff, 0xffffffff, 0xffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff},
+					expResults: []uint64{0, 0, 0, 0, 0, 0, 0},
+				},
+				{
+					params:     []uint64{0xffffffff, 0xffffffff, 0xffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff},
+					expResults: []uint64{0xff, 0xffff, 0xffffffff, 0xff, 0xffff, 0xffffffff, 0xffffffffffffffff},
+				},
+				{
+					params:     []uint64{0xffffffff, 0xffffffff, 0xffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff},
+					expResults: []uint64{0, 0, 0, 0, 0, 0, 0},
+				},
+			},
+		},
+		{
+			name:      "atomic_rmw_xchg",
+			m:         testcases.AtomicRmwXchg.Module,
+			features:  api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			skipAMD64: true,
+			calls: []callCase{
+				{
+					params:     []uint64{1, 2, 3, 4, 5, 6, 7},
+					expResults: []uint64{0, 0, 0, 0, 0, 0, 0},
+				},
+				{
+					params:     []uint64{2, 3, 4, 5, 6, 7, 8},
+					expResults: []uint64{1, 2, 3, 4, 5, 6, 7},
+				},
+				{
+					params:     []uint64{2, 3, 4, 5, 6, 7, 8},
+					expResults: []uint64{2, 3, 4, 5, 6, 7, 8},
+				},
+			},
+		},
 		{
 			name: "float_le",
 			m:    testcases.FloatLe.Module,
@@ -697,6 +835,10 @@ func TestE2E(t *testing.T) {
 					inst, err := r.InstantiateModule(ctx, compiled, wazero.NewModuleConfig())
 					require.NoError(t, err)
 
+					if tc.setupMemory != nil {
+						tc.setupMemory(inst.Memory())
+					}
+
 					for _, cc := range tc.calls {
 						name := cc.funcName
 						if name == "" {
@@ -1303,9 +1445,9 @@ func TestSpectestV2(t *testing.T) {
 		{"table_set"},
 		{"table_size"},
 		{"select"},
-		//{"simd_boolean"},
+		{"simd_boolean"},
 		//{"simd_bit_shift"},
-		//{"simd_bitwise"},
+		{"simd_bitwise"},
 		//{"simd_const"},
 		//{"simd_conversions"},
 		//{"simd_i8x16_arith"},
diff --git a/internal/engine/wazevo/frontend/frontend_test.go b/internal/engine/wazevo/frontend/frontend_test.go
index 0eac56f2..95f8642d 100644
--- a/internal/engine/wazevo/frontend/frontend_test.go
+++ b/internal/engine/wazevo/frontend/frontend_test.go
@@ -2251,6 +2251,190 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i32, v4:i32, v5:i64, v6:i64, v7:
 	ExitIfTrue v78, exec_ctx, unaligned_atomic
 	v79:i64 = AtomicRmw xchg_64, v74, v8
 	Jump blk_ret, v17, v28, v39, v46, v57, v68, v79
+`,
+		},
+		{
+			name:     "AtomicLoad",
+			m:        testcases.AtomicLoad.Module,
+			features: api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			exp: `
+blk0: (exec_ctx:i64, module_ctx:i64)
+	v2:i32 = Iconst_32 0x0
+	v3:i64 = Iconst_64 0x1
+	v4:i64 = UExtend v2, 32->64
+	v5:i64 = Uload32 module_ctx, 0x10
+	v6:i64 = Iadd v4, v3
+	v7:i32 = Icmp lt_u, v5, v6
+	ExitIfTrue v7, exec_ctx, memory_out_of_bounds
+	v8:i64 = Load module_ctx, 0x8
+	v9:i64 = Iadd v8, v4
+	v10:i32 = AtomicLoad_8, v9
+	v11:i32 = Iconst_32 0x8
+	v12:i64 = Iconst_64 0x2
+	v13:i64 = UExtend v11, 32->64
+	v14:i64 = Iadd v13, v12
+	v15:i32 = Icmp lt_u, v5, v14
+	ExitIfTrue v15, exec_ctx, memory_out_of_bounds
+	v16:i64 = Iadd v8, v13
+	v17:i64 = Iconst_64 0x1
+	v18:i64 = Band v16, v17
+	v19:i64 = Iconst_64 0x0
+	v20:i32 = Icmp neq, v18, v19
+	ExitIfTrue v20, exec_ctx, unaligned_atomic
+	v21:i32 = AtomicLoad_16, v16
+	v22:i32 = Iconst_32 0x10
+	v23:i64 = Iconst_64 0x4
+	v24:i64 = UExtend v22, 32->64
+	v25:i64 = Iadd v24, v23
+	v26:i32 = Icmp lt_u, v5, v25
+	ExitIfTrue v26, exec_ctx, memory_out_of_bounds
+	v27:i64 = Iadd v8, v24
+	v28:i64 = Iconst_64 0x3
+	v29:i64 = Band v27, v28
+	v30:i64 = Iconst_64 0x0
+	v31:i32 = Icmp neq, v29, v30
+	ExitIfTrue v31, exec_ctx, unaligned_atomic
+	v32:i32 = AtomicLoad_32, v27
+	v33:i32 = Iconst_32 0x18
+	v34:i64 = Iconst_64 0x1
+	v35:i64 = UExtend v33, 32->64
+	v36:i64 = Iadd v35, v34
+	v37:i32 = Icmp lt_u, v5, v36
+	ExitIfTrue v37, exec_ctx, memory_out_of_bounds
+	v38:i64 = Iadd v8, v35
+	v39:i64 = AtomicLoad_8, v38
+	v40:i32 = Iconst_32 0x20
+	v41:i64 = Iconst_64 0x2
+	v42:i64 = UExtend v40, 32->64
+	v43:i64 = Iadd v42, v41
+	v44:i32 = Icmp lt_u, v5, v43
+	ExitIfTrue v44, exec_ctx, memory_out_of_bounds
+	v45:i64 = Iadd v8, v42
+	v46:i64 = Iconst_64 0x1
+	v47:i64 = Band v45, v46
+	v48:i64 = Iconst_64 0x0
+	v49:i32 = Icmp neq, v47, v48
+	ExitIfTrue v49, exec_ctx, unaligned_atomic
+	v50:i64 = AtomicLoad_16, v45
+	v51:i32 = Iconst_32 0x28
+	v52:i64 = Iconst_64 0x4
+	v53:i64 = UExtend v51, 32->64
+	v54:i64 = Iadd v53, v52
+	v55:i32 = Icmp lt_u, v5, v54
+	ExitIfTrue v55, exec_ctx, memory_out_of_bounds
+	v56:i64 = Iadd v8, v53
+	v57:i64 = Iconst_64 0x3
+	v58:i64 = Band v56, v57
+	v59:i64 = Iconst_64 0x0
+	v60:i32 = Icmp neq, v58, v59
+	ExitIfTrue v60, exec_ctx, unaligned_atomic
+	v61:i64 = AtomicLoad_32, v56
+	v62:i32 = Iconst_32 0x30
+	v63:i64 = Iconst_64 0x8
+	v64:i64 = UExtend v62, 32->64
+	v65:i64 = Iadd v64, v63
+	v66:i32 = Icmp lt_u, v5, v65
+	ExitIfTrue v66, exec_ctx, memory_out_of_bounds
+	v67:i64 = Iadd v8, v64
+	v68:i64 = Iconst_64 0x7
+	v69:i64 = Band v67, v68
+	v70:i64 = Iconst_64 0x0
+	v71:i32 = Icmp neq, v69, v70
+	ExitIfTrue v71, exec_ctx, unaligned_atomic
+	v72:i64 = AtomicLoad_64, v67
+	Jump blk_ret, v10, v21, v32, v39, v50, v61, v72
+`,
+		},
+		{
+			name:     "AtomicStore",
+			m:        testcases.AtomicStore.Module,
+			features: api.CoreFeaturesV2 | experimental.CoreFeaturesThreads,
+			exp: `
+blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i32, v4:i32, v5:i64, v6:i64, v7:i64, v8:i64)
+	v9:i32 = Iconst_32 0x0
+	v10:i64 = Iconst_64 0x1
+	v11:i64 = UExtend v9, 32->64
+	v12:i64 = Uload32 module_ctx, 0x10
+	v13:i64 = Iadd v11, v10
+	v14:i32 = Icmp lt_u, v12, v13
+	ExitIfTrue v14, exec_ctx, memory_out_of_bounds
+	v15:i64 = Load module_ctx, 0x8
+	v16:i64 = Iadd v15, v11
+	AtomicStore_8, v16, v2
+	v17:i32 = Iconst_32 0x8
+	v18:i64 = Iconst_64 0x2
+	v19:i64 = UExtend v17, 32->64
+	v20:i64 = Iadd v19, v18
+	v21:i32 = Icmp lt_u, v12, v20
+	ExitIfTrue v21, exec_ctx, memory_out_of_bounds
+	v22:i64 = Iadd v15, v19
+	v23:i64 = Iconst_64 0x1
+	v24:i64 = Band v22, v23
+	v25:i64 = Iconst_64 0x0
+	v26:i32 = Icmp neq, v24, v25
+	ExitIfTrue v26, exec_ctx, unaligned_atomic
+	AtomicStore_16, v22, v3
+	v27:i32 = Iconst_32 0x10
+	v28:i64 = Iconst_64 0x4
+	v29:i64 = UExtend v27, 32->64
+	v30:i64 = Iadd v29, v28
+	v31:i32 = Icmp lt_u, v12, v30
+	ExitIfTrue v31, exec_ctx, memory_out_of_bounds
+	v32:i64 = Iadd v15, v29
+	v33:i64 = Iconst_64 0x3
+	v34:i64 = Band v32, v33
+	v35:i64 = Iconst_64 0x0
+	v36:i32 = Icmp neq, v34, v35
+	ExitIfTrue v36, exec_ctx, unaligned_atomic
+	AtomicStore_32, v32, v4
+	v37:i32 = Iconst_32 0x18
+	v38:i64 = Iconst_64 0x1
+	v39:i64 = UExtend v37, 32->64
+	v40:i64 = Iadd v39, v38
+	v41:i32 = Icmp lt_u, v12, v40
+	ExitIfTrue v41, exec_ctx, memory_out_of_bounds
+	v42:i64 = Iadd v15, v39
+	AtomicStore_8, v42, v5
+	v43:i32 = Iconst_32 0x20
+	v44:i64 = Iconst_64 0x2
+	v45:i64 = UExtend v43, 32->64
+	v46:i64 = Iadd v45, v44
+	v47:i32 = Icmp lt_u, v12, v46
+	ExitIfTrue v47, exec_ctx, memory_out_of_bounds
+	v48:i64 = Iadd v15, v45
+	v49:i64 = Iconst_64 0x1
+	v50:i64 = Band v48, v49
+	v51:i64 = Iconst_64 0x0
+	v52:i32 = Icmp neq, v50, v51
+	ExitIfTrue v52, exec_ctx, unaligned_atomic
+	AtomicStore_16, v48, v6
+	v53:i32 = Iconst_32 0x28
+	v54:i64 = Iconst_64 0x4
+	v55:i64 = UExtend v53, 32->64
+	v56:i64 = Iadd v55, v54
+	v57:i32 = Icmp lt_u, v12, v56
+	ExitIfTrue v57, exec_ctx, memory_out_of_bounds
+	v58:i64 = Iadd v15, v55
+	v59:i64 = Iconst_64 0x3
+	v60:i64 = Band v58, v59
+	v61:i64 = Iconst_64 0x0
+	v62:i32 = Icmp neq, v60, v61
+	ExitIfTrue v62, exec_ctx, unaligned_atomic
+	AtomicStore_32, v58, v7
+	v63:i32 = Iconst_32 0x30
+	v64:i64 = Iconst_64 0x8
+	v65:i64 = UExtend v63, 32->64
+	v66:i64 = Iadd v65, v64
+	v67:i32 = Icmp lt_u, v12, v66
+	ExitIfTrue v67, exec_ctx, memory_out_of_bounds
+	v68:i64 = Iadd v15, v65
+	v69:i64 = Iconst_64 0x7
+	v70:i64 = Band v68, v69
+	v71:i64 = Iconst_64 0x0
+	v72:i32 = Icmp neq, v70, v71
+	ExitIfTrue v72, exec_ctx, unaligned_atomic
+	AtomicStore_64, v68, v8
+	Jump blk_ret
 `,
 		},
 	} {
diff --git a/internal/engine/wazevo/testcases/testcases.go b/internal/engine/wazevo/testcases/testcases.go
index 488ad1c4..5af03b90 100644
--- a/internal/engine/wazevo/testcases/testcases.go
+++ b/internal/engine/wazevo/testcases/testcases.go
@@ -2207,6 +2207,73 @@ var (
 			}}},
 		},
 	}
+
+	AtomicLoad = TestCase{
+		Name: "atomic_load",
+		Module: &wasm.Module{
+			TypeSection: []wasm.FunctionType{{
+				Params:  []wasm.ValueType{},
+				Results: []wasm.ValueType{i32, i32, i32, i64, i64, i64, i64},
+			}},
+			ExportSection:   []wasm.Export{{Name: ExportedFunctionName, Type: wasm.ExternTypeFunc, Index: 0}},
+			MemorySection:   &wasm.Memory{Min: 1, Max: 1, IsMaxEncoded: true, IsShared: true},
+			FunctionSection: []wasm.Index{0},
+			CodeSection: []wasm.Code{{Body: []byte{
+				wasm.OpcodeI32Const, 0,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI32Load8U, 0x0, 0,
+				wasm.OpcodeI32Const, 8,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI32Load16U, 0x1, 0,
+				wasm.OpcodeI32Const, 16,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI32Load, 0x2, 0,
+				wasm.OpcodeI32Const, 24,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Load8U, 0x0, 0,
+				wasm.OpcodeI32Const, 32,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Load16U, 0x1, 0,
+				wasm.OpcodeI32Const, 40,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Load32U, 0x2, 0,
+				wasm.OpcodeI32Const, 48,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Load, 0x3, 0,
+				wasm.OpcodeEnd,
+			}}},
+		},
+	}
+
+	AtomicStore = TestCase{
+		Name: "atomic_store",
+		Module: &wasm.Module{
+			TypeSection: []wasm.FunctionType{{
+				Params:  []wasm.ValueType{i32, i32, i32, i64, i64, i64, i64},
+				Results: []wasm.ValueType{},
+			}},
+			ExportSection:   []wasm.Export{{Name: ExportedFunctionName, Type: wasm.ExternTypeFunc, Index: 0}},
+			MemorySection:   &wasm.Memory{Min: 1, Max: 1, IsMaxEncoded: true, IsShared: true},
+			FunctionSection: []wasm.Index{0},
+			CodeSection: []wasm.Code{{Body: []byte{
+				wasm.OpcodeI32Const, 0,
+				wasm.OpcodeLocalGet, 0,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI32Store8, 0x0, 0,
+				wasm.OpcodeI32Const, 8,
+				wasm.OpcodeLocalGet, 1,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI32Store16, 0x1, 0,
+				wasm.OpcodeI32Const, 16,
+				wasm.OpcodeLocalGet, 2,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI32Store, 0x2, 0,
+				wasm.OpcodeI32Const, 24,
+				wasm.OpcodeLocalGet, 3,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Store8, 0x0, 0,
+				wasm.OpcodeI32Const, 32,
+				wasm.OpcodeLocalGet, 4,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Store16, 0x1, 0,
+				wasm.OpcodeI32Const, 40,
+				wasm.OpcodeLocalGet, 5,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Store32, 0x2, 0,
+				wasm.OpcodeI32Const, 48,
+				wasm.OpcodeLocalGet, 6,
+				wasm.OpcodeAtomicPrefix, wasm.OpcodeAtomicI64Store, 0x3, 0,
+				wasm.OpcodeEnd,
+			}}},
+		},
+	}
 )
 
 // VecShuffleWithLane returns a VecShuffle test with a custom 16-bytes immediate (lane indexes).