Repository: tetratelabs/wazero
Pull request #1756 (2023-10-05 00:23:31)

wazevo: completes the bulk memory and reference types spec
This implements {data,element}.drop and {memory,table}.init instructions,
and hence completes the implementation of bulk memory and reference
types specification in v2. Consequently, this leaves SIMD is the only
spec as the last missing piece to #1496 .


Patch:
diff --git a/internal/engine/wazevo/frontend/lower.go b/internal/engine/wazevo/frontend/lower.go
index b288806e..dd7ff577 100644
--- a/internal/engine/wazevo/frontend/lower.go
+++ b/internal/engine/wazevo/frontend/lower.go
@@ -592,91 +592,29 @@ func (c *Compiler) lowerCurrentOpcode() {
 				break
 			}
 
-			copySize := state.pop()
-			srcOffset := state.pop()
-			dstOffset := state.pop()
-			copySizeExt := builder.
-				AllocateInstruction().AsUExtend(copySize, 32, 64).Insert(builder).Return()
-			srcOffsetExt := builder.
-				AllocateInstruction().AsUExtend(srcOffset, 32, 64).Insert(builder).Return()
-			dstOffsetExt := builder.
-				AllocateInstruction().AsUExtend(dstOffset, 32, 64).Insert(builder).Return()
+			copySize := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			srcOffset := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			dstOffset := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
 
 			// Out of bounds check.
-			var dstTableInstancePtr, srcTableInstancePtr ssa.Value
-			{
-				dstCeil := builder.AllocateInstruction().AsIadd(dstOffsetExt, copySizeExt).Insert(builder).Return()
-
-				// Load the table.
-				dstTableInstancePtr = builder.AllocateInstruction().
-					AsLoad(c.moduleCtxPtrValue, c.offset.TableOffset(int(dstTableIndex)).U32(), ssa.TypeI64).
-					Insert(builder).Return()
-
-				// Load the table's length.
-				tableLen := builder.AllocateInstruction().
-					AsLoad(dstTableInstancePtr, tableInstanceLenOffset, ssa.TypeI32).Insert(builder).Return()
-				tableLenExt := builder.AllocateInstruction().AsUExtend(tableLen, 32, 64).Insert(builder).Return()
-
-				// Compare the length and the target, and trap if out of bounds.
-				checkOOB := builder.AllocateInstruction()
-				checkOOB.AsIcmp(tableLenExt, dstCeil, ssa.IntegerCmpCondUnsignedLessThan)
-				builder.InsertInstruction(checkOOB)
-				exitIfOOB := builder.AllocateInstruction()
-				exitIfOOB.AsExitIfTrueWithCode(c.execCtxPtrValue, checkOOB.Return(), wazevoapi.ExitCodeTableOutOfBounds)
-				builder.InsertInstruction(exitIfOOB)
-			}
-			{
-				srcCeil := builder.AllocateInstruction().AsIadd(srcOffsetExt, copySizeExt).Insert(builder).Return()
-
-				// Load the table.
-				srcTableInstancePtr = builder.AllocateInstruction().
-					AsLoad(c.moduleCtxPtrValue, c.offset.TableOffset(int(srcTableIndex)).U32(), ssa.TypeI64).
-					Insert(builder).Return()
+			dstTableInstancePtr := c.boundsCheckInTable(dstTableIndex, dstOffset, copySize)
+			srcTableInstancePtr := c.boundsCheckInTable(srcTableIndex, srcOffset, copySize)
 
-				// Load the table's length.
-				tableLen := builder.AllocateInstruction().
-					AsLoad(srcTableInstancePtr, tableInstanceLenOffset, ssa.TypeI32).Insert(builder).Return()
-				tableLenExt := builder.AllocateInstruction().AsUExtend(tableLen, 32, 64).Insert(builder).Return()
+			dstTableBaseAddr := c.loadTableBaseAddr(dstTableInstancePtr)
+			srcTableBaseAddr := c.loadTableBaseAddr(srcTableInstancePtr)
 
-				// Compare the length and the target, and trap if out of bounds.
-				checkOOB := builder.AllocateInstruction()
-				checkOOB.AsIcmp(tableLenExt, srcCeil, ssa.IntegerCmpCondUnsignedLessThan)
-				builder.InsertInstruction(checkOOB)
-				exitIfOOB := builder.AllocateInstruction()
-				exitIfOOB.AsExitIfTrueWithCode(c.execCtxPtrValue, checkOOB.Return(), wazevoapi.ExitCodeTableOutOfBounds)
-				builder.InsertInstruction(exitIfOOB)
-			}
-
-			var dstTableBaseAddr, srcTableBaseAddr ssa.Value
-			{
-				loadTableBaseAddress := builder.AllocateInstruction()
-				loadTableBaseAddress.AsLoad(dstTableInstancePtr, tableInstanceBaseAddressOffset, ssa.TypeI64)
-				builder.InsertInstruction(loadTableBaseAddress)
-				dstTableBaseAddr = loadTableBaseAddress.Return()
-			}
-			{
-				loadTableBaseAddress := builder.AllocateInstruction()
-				loadTableBaseAddress.AsLoad(srcTableInstancePtr, tableInstanceBaseAddressOffset, ssa.TypeI64)
-				builder.InsertInstruction(loadTableBaseAddress)
-				srcTableBaseAddr = loadTableBaseAddress.Return()
-			}
 			three := builder.AllocateInstruction().AsIconst64(3).Insert(builder).Return()
 
-			dstOffsetInBytes := builder.AllocateInstruction().AsIshl(dstOffsetExt, three).Insert(builder).Return()
+			dstOffsetInBytes := builder.AllocateInstruction().AsIshl(dstOffset, three).Insert(builder).Return()
 			dstAddr := builder.AllocateInstruction().AsIadd(dstTableBaseAddr, dstOffsetInBytes).Insert(builder).Return()
-			srcOffsetInBytes := builder.AllocateInstruction().AsIshl(srcOffsetExt, three).Insert(builder).Return()
+			srcOffsetInBytes := builder.AllocateInstruction().AsIshl(srcOffset, three).Insert(builder).Return()
 			srcAddr := builder.AllocateInstruction().AsIadd(srcTableBaseAddr, srcOffsetInBytes).Insert(builder).Return()
 
-			memmovePtr := c.loadMemmoveAddr()
-
-			copySizeExtInBytes := builder.AllocateInstruction().AsIshl(copySizeExt, three).Insert(builder).Return()
-
-			// TODO: reuse the slice.
-			args := []ssa.Value{dstAddr, srcAddr, copySizeExtInBytes}
-			builder.
-				AllocateInstruction().
-				AsCallIndirect(memmovePtr, &c.memmoveSig, args).
-				Insert(builder)
+			copySizeInBytes := builder.AllocateInstruction().AsIshl(copySize, three).Insert(builder).Return()
+			c.callMemmove(dstAddr, srcAddr, copySizeInBytes)
 
 		case wasm.OpcodeMiscMemoryCopy:
 			state.pc += 2 // +2 to skip two memory indexes which are fixed to zero.
@@ -684,53 +622,23 @@ func (c *Compiler) lowerCurrentOpcode() {
 				break
 			}
 
-			copySize := state.pop()
-			srcOffset := state.pop()
-			dstOffset := state.pop()
-
-			copySizeExt := builder.
-				AllocateInstruction().AsUExtend(copySize, 32, 64).Insert(builder).Return()
-			srcOffsetExt := builder.
-				AllocateInstruction().AsUExtend(srcOffset, 32, 64).Insert(builder).Return()
-			dstOffsetExt := builder.
-				AllocateInstruction().AsUExtend(dstOffset, 32, 64).Insert(builder).Return()
+			copySize := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			srcOffset := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			dstOffset := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
 
 			// Out of bounds check.
 			memLen := c.getMemoryLenValue(false)
-
-			{
-				srcCeil := builder.AllocateInstruction().AsIadd(srcOffsetExt, copySizeExt).Insert(builder).Return()
-				cmp := builder.AllocateInstruction().
-					AsIcmp(memLen, srcCeil, ssa.IntegerCmpCondUnsignedLessThan).
-					Insert(builder).
-					Return()
-				builder.AllocateInstruction().
-					AsExitIfTrueWithCode(c.execCtxPtrValue, cmp, wazevoapi.ExitCodeMemoryOutOfBounds).
-					Insert(builder)
-			}
-			{
-				dstCeil := builder.AllocateInstruction().AsIadd(dstOffsetExt, copySizeExt).Insert(builder).Return()
-				cmp := builder.AllocateInstruction().
-					AsIcmp(memLen, dstCeil, ssa.IntegerCmpCondUnsignedLessThan).
-					Insert(builder).
-					Return()
-				builder.AllocateInstruction().
-					AsExitIfTrueWithCode(c.execCtxPtrValue, cmp, wazevoapi.ExitCodeMemoryOutOfBounds).
-					Insert(builder)
-			}
+			c.boundsCheckInMemory(memLen, dstOffset, copySize)
+			c.boundsCheckInMemory(memLen, srcOffset, copySize)
 
 			memBase := c.getMemoryBaseValue(false)
-			dstAddr := builder.AllocateInstruction().AsIadd(memBase, dstOffsetExt).Insert(builder).Return()
-			srcAddr := builder.AllocateInstruction().AsIadd(memBase, srcOffsetExt).Insert(builder).Return()
+			dstAddr := builder.AllocateInstruction().AsIadd(memBase, dstOffset).Insert(builder).Return()
+			srcAddr := builder.AllocateInstruction().AsIadd(memBase, srcOffset).Insert(builder).Return()
 
-			memmovePtr := c.loadMemmoveAddr()
-
-			// TODO: reuse the slice.
-			args := []ssa.Value{dstAddr, srcAddr, copySize}
-			builder.
-				AllocateInstruction().
-				AsCallIndirect(memmovePtr, &c.memmoveSig, args).
-				Insert(builder)
+			c.callMemmove(dstAddr, srcAddr, copySize)
 
 		case wasm.OpcodeMiscTableFill:
 			tableIndex := c.readI32u()
@@ -745,36 +653,14 @@ func (c *Compiler) lowerCurrentOpcode() {
 				AllocateInstruction().AsUExtend(fillSize, 32, 64).Insert(builder).Return()
 			offsetExt := builder.
 				AllocateInstruction().AsUExtend(offset, 32, 64).Insert(builder).Return()
-			var tableInstancePtr ssa.Value
-			{
-				dstCeil := builder.AllocateInstruction().AsIadd(offsetExt, fillSizeExt).Insert(builder).Return()
-
-				// Load the table.
-				tableInstancePtr = builder.AllocateInstruction().
-					AsLoad(c.moduleCtxPtrValue, c.offset.TableOffset(int(tableIndex)).U32(), ssa.TypeI64).
-					Insert(builder).Return()
-
-				// Load the table's length.
-				tableLen := builder.AllocateInstruction().
-					AsLoad(tableInstancePtr, tableInstanceLenOffset, ssa.TypeI32).Insert(builder).Return()
-				tableLenExt := builder.AllocateInstruction().AsUExtend(tableLen, 32, 64).Insert(builder).Return()
+			tableInstancePtr := c.boundsCheckInTable(tableIndex, offsetExt, fillSizeExt)
 
-				// Compare the length and the target, and trap if out of bounds.
-				checkOOB := builder.AllocateInstruction()
-				checkOOB.AsIcmp(tableLenExt, dstCeil, ssa.IntegerCmpCondUnsignedLessThan)
-				builder.InsertInstruction(checkOOB)
-				exitIfOOB := builder.AllocateInstruction()
-				exitIfOOB.AsExitIfTrueWithCode(c.execCtxPtrValue, checkOOB.Return(), wazevoapi.ExitCodeTableOutOfBounds)
-				builder.InsertInstruction(exitIfOOB)
-			}
 			three := builder.AllocateInstruction().AsIconst64(3).Insert(builder).Return()
 			offsetInBytes := builder.AllocateInstruction().AsIshl(offsetExt, three).Insert(builder).Return()
 			fillSizeInBytes := builder.AllocateInstruction().AsIshl(fillSizeExt, three).Insert(builder).Return()
 
 			// Calculate the base address of the table.
-			tableBaseAddr := builder.AllocateInstruction().
-				AsLoad(tableInstancePtr, tableInstanceBaseAddressOffset, ssa.TypeI64).
-				Insert(builder).Return()
+			tableBaseAddr := c.loadTableBaseAddr(tableInstancePtr)
 			addr := builder.AllocateInstruction().AsIadd(tableBaseAddr, offsetInBytes).Insert(builder).Return()
 
 			// Prepare the loop and following block.
@@ -817,11 +703,7 @@ func (c *Compiler) lowerCurrentOpcode() {
 				count = builder.AllocateInstruction().AsSelect(loopVarDoubledLargerThanFillSize, diff, loopVar).Insert(builder).Return()
 			}
 
-			memmovePtr := c.loadMemmoveAddr()
-			builder.
-				AllocateInstruction().
-				AsCallIndirect(memmovePtr, &c.memmoveSig, []ssa.Value{dstAddr, addr, count}). // TODO: reuse the slice.
-				Insert(builder)
+			c.callMemmove(dstAddr, addr, count)
 
 			shiftAmount := builder.AllocateInstruction().AsIconst64(1).Insert(builder).Return()
 			newLoopVar := builder.AllocateInstruction().AsIshl(loopVar, shiftAmount).Insert(builder).Return()
@@ -844,28 +726,18 @@ func (c *Compiler) lowerCurrentOpcode() {
 			if state.unreachable {
 				break
 			}
-			fillSize := state.pop()
-			value := state.pop()
-			offset := state.pop()
 
-			fillSizeExt := builder.
-				AllocateInstruction().AsUExtend(fillSize, 32, 64).Insert(builder).Return()
-			offsetExt := builder.
-				AllocateInstruction().AsUExtend(offset, 32, 64).Insert(builder).Return()
+			fillSize := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			value := state.pop()
+			offset := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
 
 			// Out of bounds check.
-			memLen := c.getMemoryLenValue(false)
-			ceil := builder.AllocateInstruction().AsIadd(fillSizeExt, offsetExt).Insert(builder).Return()
-			boundsCheckFailed := builder.AllocateInstruction().
-				AsIcmp(memLen, ceil, ssa.IntegerCmpCondUnsignedLessThan).
-				Insert(builder).
-				Return()
-			builder.AllocateInstruction().
-				AsExitIfTrueWithCode(c.execCtxPtrValue, boundsCheckFailed, wazevoapi.ExitCodeMemoryOutOfBounds).
-				Insert(builder)
+			c.boundsCheckInMemory(c.getMemoryLenValue(false), offset, fillSize)
 
 			// Calculate the base address:
-			addr := builder.AllocateInstruction().AsIadd(c.getMemoryBaseValue(false), offsetExt).Insert(builder).Return()
+			addr := builder.AllocateInstruction().AsIadd(c.getMemoryBaseValue(false), offset).Insert(builder).Return()
 
 			// Uses the copy trick for faster filling buffer: https://gist.github.com/taylorza/df2f89d5f9ab3ffd06865062a4cf015d
 			// 	buf := memoryInst.Buffer[offset : offset+fillSize]
@@ -882,7 +754,7 @@ func (c *Compiler) lowerCurrentOpcode() {
 
 			// Insert the jump to the beforeLoop block; If the fillSize is zero, then jump to the following block to skip entire logics.
 			zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return()
-			ifFillSizeZero := builder.AllocateInstruction().AsIcmp(fillSizeExt, zero, ssa.IntegerCmpCondEqual).
+			ifFillSizeZero := builder.AllocateInstruction().AsIcmp(fillSize, zero, ssa.IntegerCmpCondEqual).
 				Insert(builder).Return()
 			builder.AllocateInstruction().AsBrnz(ifFillSizeZero, nil, followingBlk).Insert(builder)
 			c.insertJumpToBlock(nil, beforeLoop)
@@ -901,22 +773,18 @@ func (c *Compiler) lowerCurrentOpcode() {
 			{
 				loopVarDoubled := builder.AllocateInstruction().AsIadd(loopVar, loopVar).Insert(builder).Return()
 				loopVarDoubledLargerThanFillSize := builder.
-					AllocateInstruction().AsIcmp(loopVarDoubled, fillSizeExt, ssa.IntegerCmpCondUnsignedGreaterThanOrEqual).
+					AllocateInstruction().AsIcmp(loopVarDoubled, fillSize, ssa.IntegerCmpCondUnsignedGreaterThanOrEqual).
 					Insert(builder).Return()
-				diff := builder.AllocateInstruction().AsIsub(fillSizeExt, loopVar).Insert(builder).Return()
+				diff := builder.AllocateInstruction().AsIsub(fillSize, loopVar).Insert(builder).Return()
 				count = builder.AllocateInstruction().AsSelect(loopVarDoubledLargerThanFillSize, diff, loopVar).Insert(builder).Return()
 			}
 
-			memmovePtr := c.loadMemmoveAddr()
-			builder.
-				AllocateInstruction().
-				AsCallIndirect(memmovePtr, &c.memmoveSig, []ssa.Value{dstAddr, addr, count}). // TODO: reuse the slice.
-				Insert(builder)
+			c.callMemmove(dstAddr, addr, count)
 
 			shiftAmount := builder.AllocateInstruction().AsIconst64(1).Insert(builder).Return()
 			newLoopVar := builder.AllocateInstruction().AsIshl(loopVar, shiftAmount).Insert(builder).Return()
 			loopVarLessThanFillSize := builder.AllocateInstruction().
-				AsIcmp(newLoopVar, fillSizeExt, ssa.IntegerCmpCondUnsignedLessThan).Insert(builder).Return()
+				AsIcmp(newLoopVar, fillSize, ssa.IntegerCmpCondUnsignedLessThan).Insert(builder).Return()
 
 			builder.AllocateInstruction().
 				AsBrnz(loopVarLessThanFillSize, []ssa.Value{newLoopVar}, loopBlk). // TODO: reuse the slice.
@@ -929,6 +797,83 @@ func (c *Compiler) lowerCurrentOpcode() {
 			builder.Seal(loopBlk)
 			builder.Seal(followingBlk)
 
+		case wasm.OpcodeMiscMemoryInit:
+			index := c.readI32u()
+			state.pc++ // Skip the memory index which is fixed to zero.
+			if state.unreachable {
+				break
+			}
+
+			copySize := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			offsetInDataInstance := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			offsetInMemory := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+
+			dataInstPtr := c.dataOrElementInstanceAddr(index, c.offset.DataInstances1stElement)
+
+			// Bounds check.
+			c.boundsCheckInMemory(c.getMemoryLenValue(false), offsetInMemory, copySize)
+			c.boundsCheckInDataOrElementInstance(dataInstPtr, offsetInDataInstance, copySize, wazevoapi.ExitCodeMemoryOutOfBounds)
+
+			dataInstBaseAddr := builder.AllocateInstruction().AsLoad(dataInstPtr, 0, ssa.TypeI64).Insert(builder).Return()
+			srcAddr := builder.AllocateInstruction().AsIadd(dataInstBaseAddr, offsetInDataInstance).Insert(builder).Return()
+
+			memBase := c.getMemoryBaseValue(false)
+			dstAddr := builder.AllocateInstruction().AsIadd(memBase, offsetInMemory).Insert(builder).Return()
+
+			c.callMemmove(dstAddr, srcAddr, copySize)
+
+		case wasm.OpcodeMiscTableInit:
+			elemIndex := c.readI32u()
+			tableIndex := c.readI32u()
+			if state.unreachable {
+				break
+			}
+
+			copySize := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			offsetInElementInstance := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+			offsetInTable := builder.
+				AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return()
+
+			elemInstPtr := c.dataOrElementInstanceAddr(elemIndex, c.offset.ElementInstances1stElement)
+
+			// Bounds check.
+			tableInstancePtr := c.boundsCheckInTable(tableIndex, offsetInTable, copySize)
+			c.boundsCheckInDataOrElementInstance(elemInstPtr, offsetInElementInstance, copySize, wazevoapi.ExitCodeTableOutOfBounds)
+
+			three := builder.AllocateInstruction().AsIconst64(3).Insert(builder).Return()
+			// Calculates the destination address in the table.
+			tableOffsetInBytes := builder.AllocateInstruction().AsIshl(offsetInTable, three).Insert(builder).Return()
+			tableBaseAddr := c.loadTableBaseAddr(tableInstancePtr)
+			dstAddr := builder.AllocateInstruction().AsIadd(tableBaseAddr, tableOffsetInBytes).Insert(builder).Return()
+
+			// Calculates the source address in the element instance.
+			srcOffsetInBytes := builder.AllocateInstruction().AsIshl(offsetInElementInstance, three).Insert(builder).Return()
+			elemInstBaseAddr := builder.AllocateInstruction().AsLoad(elemInstPtr, 0, ssa.TypeI64).Insert(builder).Return()
+			srcAddr := builder.AllocateInstruction().AsIadd(elemInstBaseAddr, srcOffsetInBytes).Insert(builder).Return()
+
+			copySizeInBytes := builder.AllocateInstruction().AsIshl(copySize, three).Insert(builder).Return()
+			c.callMemmove(dstAddr, srcAddr, copySizeInBytes)
+
+		case wasm.OpcodeMiscElemDrop:
+			index := c.readI32u()
+			if state.unreachable {
+				break
+			}
+
+			c.dropDataOrElementInstance(index, c.offset.ElementInstances1stElement)
+
+		case wasm.OpcodeMiscDataDrop:
+			index := c.readI32u()
+			if state.unreachable {
+				break
+			}
+			c.dropDataOrElementInstance(index, c.offset.DataInstances1stElement)
+
 		default:
 			panic("Unknown MiscOp " + wasm.MiscInstructionName(miscOp))
 		}
@@ -3038,14 +2983,16 @@ func (c *Compiler) memOpSetup(baseAddr ssa.Value, constOffset, operationSizeInBy
 	return addrCalc.Return()
 }
 
-func (c *Compiler) loadMemmoveAddr() ssa.Value {
+func (c *Compiler) callMemmove(dst, src, size ssa.Value) {
+	args := []ssa.Value{dst, src, size} // TODO: reuse the slice.
+
 	builder := c.ssaBuilder
 	memmovePtr := builder.AllocateInstruction().
 		AsLoad(c.execCtxPtrValue,
 			wazevoapi.ExecutionContextOffsetMemmoveAddress.U32(),
 			ssa.TypeI64,
 		).Insert(builder).Return()
-	return memmovePtr
+	builder.AllocateInstruction().AsCallIndirect(memmovePtr, &c.memmoveSig, args).Insert(builder)
 }
 
 func (c *Compiler) reloadAfterCall() {
@@ -3449,3 +3396,97 @@ func (c *Compiler) callListenerAfter() {
 		AsCallIndirect(afterListenerPtr, afterSig, args).
 		Insert(builder)
 }
+
+const (
+	elementOrDataInstanceLenOffset = 8
+	elementOrDataInstanceSize      = 24
+)
+
+// dropInstance inserts instructions to drop the element/data instance specified by the given index.
+func (c *Compiler) dropDataOrElementInstance(index uint32, firstItemOffset wazevoapi.Offset) {
+	builder := c.ssaBuilder
+	instPtr := c.dataOrElementInstanceAddr(index, firstItemOffset)
+
+	zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return()
+
+	// Clear the instance.
+	builder.AllocateInstruction().AsStore(ssa.OpcodeStore, zero, instPtr, 0).Insert(builder)
+	builder.AllocateInstruction().AsStore(ssa.OpcodeStore, zero, instPtr, elementOrDataInstanceLenOffset).Insert(builder)
+	builder.AllocateInstruction().AsStore(ssa.OpcodeStore, zero, instPtr, elementOrDataInstanceLenOffset+8).Insert(builder)
+}
+
+func (c *Compiler) dataOrElementInstanceAddr(index uint32, firstItemOffset wazevoapi.Offset) ssa.Value {
+	builder := c.ssaBuilder
+
+	_1stItemPtr := builder.
+		AllocateInstruction().
+		AsLoad(c.moduleCtxPtrValue, firstItemOffset.U32(), ssa.TypeI64).
+		Insert(builder).Return()
+
+	// Each data/element instance is a slice, so we need to multiply index by 16 to get the offset of the target instance.
+	index = index * elementOrDataInstanceSize
+	indexExt := builder.AllocateInstruction().AsIconst64(uint64(index)).Insert(builder).Return()
+	// Then, add the offset to the address of the instance.
+	instPtr := builder.AllocateInstruction().AsIadd(_1stItemPtr, indexExt).Insert(builder).Return()
+	return instPtr
+}
+
+func (c *Compiler) boundsCheckInDataOrElementInstance(instPtr, offsetInInstance, copySize ssa.Value, exitCode wazevoapi.ExitCode) {
+	builder := c.ssaBuilder
+	dataInstLen := builder.AllocateInstruction().
+		AsLoad(instPtr, elementOrDataInstanceLenOffset, ssa.TypeI64).
+		Insert(builder).Return()
+	ceil := builder.AllocateInstruction().AsIadd(offsetInInstance, copySize).Insert(builder).Return()
+	cmp := builder.AllocateInstruction().
+		AsIcmp(dataInstLen, ceil, ssa.IntegerCmpCondUnsignedLessThan).
+		Insert(builder).
+		Return()
+	builder.AllocateInstruction().
+		AsExitIfTrueWithCode(c.execCtxPtrValue, cmp, exitCode).
+		Insert(builder)
+}
+
+func (c *Compiler) boundsCheckInTable(tableIndex uint32, offset, size ssa.Value) (tableInstancePtr ssa.Value) {
+	builder := c.ssaBuilder
+	dstCeil := builder.AllocateInstruction().AsIadd(offset, size).Insert(builder).Return()
+
+	// Load the table.
+	tableInstancePtr = builder.AllocateInstruction().
+		AsLoad(c.moduleCtxPtrValue, c.offset.TableOffset(int(tableIndex)).U32(), ssa.TypeI64).
+		Insert(builder).Return()
+
+	// Load the table's length.
+	tableLen := builder.AllocateInstruction().
+		AsLoad(tableInstancePtr, tableInstanceLenOffset, ssa.TypeI32).Insert(builder).Return()
+	tableLenExt := builder.AllocateInstruction().AsUExtend(tableLen, 32, 64).Insert(builder).Return()
+
+	// Compare the length and the target, and trap if out of bounds.
+	checkOOB := builder.AllocateInstruction()
+	checkOOB.AsIcmp(tableLenExt, dstCeil, ssa.IntegerCmpCondUnsignedLessThan)
+	builder.InsertInstruction(checkOOB)
+	exitIfOOB := builder.AllocateInstruction()
+	exitIfOOB.AsExitIfTrueWithCode(c.execCtxPtrValue, checkOOB.Return(), wazevoapi.ExitCodeTableOutOfBounds)
+	builder.InsertInstruction(exitIfOOB)
+	return
+}
+
+func (c *Compiler) loadTableBaseAddr(tableInstancePtr ssa.Value) ssa.Value {
+	builder := c.ssaBuilder
+	loadTableBaseAddress := builder.
+		AllocateInstruction().
+		AsLoad(tableInstancePtr, tableInstanceBaseAddressOffset, ssa.TypeI64).
+		Insert(builder)
+	return loadTableBaseAddress.Return()
+}
+
+func (c *Compiler) boundsCheckInMemory(memLen, offset, size ssa.Value) {
+	builder := c.ssaBuilder
+	ceil := builder.AllocateInstruction().AsIadd(offset, size).Insert(builder).Return()
+	cmp := builder.AllocateInstruction().
+		AsIcmp(memLen, ceil, ssa.IntegerCmpCondUnsignedLessThan).
+		Insert(builder).
+		Return()
+	builder.AllocateInstruction().
+		AsExitIfTrueWithCode(c.execCtxPtrValue, cmp, wazevoapi.ExitCodeMemoryOutOfBounds).
+		Insert(builder)
+}
diff --git a/internal/engine/wazevo/module_engine.go b/internal/engine/wazevo/module_engine.go
index 93e55273..ae507d24 100644
--- a/internal/engine/wazevo/module_engine.go
+++ b/internal/engine/wazevo/module_engine.go
@@ -52,7 +52,10 @@ type (
 	//      globals                                   []*wasm.GlobalInstance (optional)
 	//      typeIDsBegin                              &wasm.ModuleInstance.TypeIDs[0]  (optional)
 	//      tables                                    []*wasm.TableInstance  (optional)
-	// 	    TODO: add more fields, like tables, etc.
+	// 	    beforeListenerTrampolines1stElement       **byte                 (optional)
+	// 	    afterListenerTrampolines1stElement        **byte                 (optional)
+	//      dataInstances1stElement                   []wasm.DataInstance    (optional)
+	//      elementInstances1stElement                []wasm.ElementInstance (optional)
 	// 	}
 	//
 	// See wazevoapi.NewModuleContextOffsetData for the details of the offsets.
@@ -114,6 +117,12 @@ func (m *moduleEngine) setupOpaque() {
 	if afterListenerOffset := offsets.AfterListenerTrampolines1stElement; afterListenerOffset >= 0 {
 		binary.LittleEndian.PutUint64(opaque[afterListenerOffset:], uint64(uintptr(unsafe.Pointer(&m.parent.listenerAfterTrampolines[0]))))
 	}
+	if len(inst.DataInstances) > 0 {
+		binary.LittleEndian.PutUint64(opaque[offsets.DataInstances1stElement:], uint64(uintptr(unsafe.Pointer(&inst.DataInstances[0]))))
+	}
+	if len(inst.ElementInstances) > 0 {
+		binary.LittleEndian.PutUint64(opaque[offsets.ElementInstances1stElement:], uint64(uintptr(unsafe.Pointer(&inst.ElementInstances[0]))))
+	}
 }
 
 // NewFunction implements wasm.ModuleEngine.
diff --git a/internal/engine/wazevo/wazevoapi/offsetdata.go b/internal/engine/wazevo/wazevoapi/offsetdata.go
index 1abaf4fb..d21489b1 100644
--- a/internal/engine/wazevo/wazevoapi/offsetdata.go
+++ b/internal/engine/wazevo/wazevoapi/offsetdata.go
@@ -63,7 +63,9 @@ type ModuleContextOffsetData struct {
 	TypeIDs1stElement,
 	TablesBegin,
 	BeforeListenerTrampolines1stElement,
-	AfterListenerTrampolines1stElement Offset
+	AfterListenerTrampolines1stElement,
+	DataInstances1stElement,
+	ElementInstances1stElement Offset
 }
 
 // ImportedFunctionOffset returns an offset of the i-th imported function.
@@ -185,6 +187,12 @@ func NewModuleContextOffsetData(m *wasm.Module, withListener bool) ModuleContext
 		ret.AfterListenerTrampolines1stElement = -1
 	}
 
+	ret.DataInstances1stElement = offset
+	offset += 8 // First element of DataInstances.
+
+	ret.ElementInstances1stElement = offset
+	offset += 8 // First element of ElementInstances.
+
 	ret.TotalSize = int(offset)
 	return ret
 }


Test patch:
diff --git a/internal/engine/wazevo/e2e_test.go b/internal/engine/wazevo/e2e_test.go
index e258f91f..5586bcd1 100644
--- a/internal/engine/wazevo/e2e_test.go
+++ b/internal/engine/wazevo/e2e_test.go
@@ -136,24 +136,31 @@ func TestSpectestV2(t *testing.T) {
 		name string
 	}{
 		{"block"},
+		{"bulk"},
 		{"i32"},
 		{"i64"},
 		{"br"},
 		{"call"},
 		{"call_indirect"},
 		{"conversions"},
+		{"global"},
 		{"if"},
+		{"linking"},
 		{"loop"},
 		{"memory_copy"},
 		{"memory_fill"},
+		{"memory_init"},
+		{"memory_trap"},
 		{"ref_null"},
 		{"ref_is_null"},
 		{"ref_func"},
 		{"table_copy"},
 		{"table_fill"},
 		{"table_get"},
+		{"table_init"},
 		{"table_set"},
 		{"table_size"},
+		{"select"},
 		{"simd_boolean"},
 		{"simd_bit_shift"},
 		{"simd_bitwise"},
diff --git a/internal/engine/wazevo/frontend/lower_test.go b/internal/engine/wazevo/frontend/lower_test.go
index 268915ff..0bb8fe9a 100644
--- a/internal/engine/wazevo/frontend/lower_test.go
+++ b/internal/engine/wazevo/frontend/lower_test.go
@@ -15,4 +15,10 @@ func Test_Offsets(t *testing.T) {
 	require.Equal(t, int(unsafe.Offsetof(memInstance.Buffer)), memoryInstanceBufOffset)
 	var tableInstance wasm.TableInstance
 	require.Equal(t, int(unsafe.Offsetof(tableInstance.References)), tableInstanceBaseAddressOffset)
+
+	var dataInstance wasm.DataInstance
+	var elementInstance wasm.ElementInstance
+
+	require.Equal(t, int(unsafe.Sizeof(dataInstance)), elementOrDataInstanceSize)
+	require.Equal(t, int(unsafe.Sizeof(elementInstance)), elementOrDataInstanceSize)
 }
diff --git a/internal/engine/wazevo/wazevoapi/offsetdata_test.go b/internal/engine/wazevo/wazevoapi/offsetdata_test.go
index c4028634..ea64a985 100644
--- a/internal/engine/wazevo/wazevoapi/offsetdata_test.go
+++ b/internal/engine/wazevo/wazevoapi/offsetdata_test.go
@@ -26,7 +26,9 @@ func TestNewModuleContextOffsetData(t *testing.T) {
 				TablesBegin:                         -1,
 				BeforeListenerTrampolines1stElement: -1,
 				AfterListenerTrampolines1stElement:  -1,
-				TotalSize:                           8,
+				DataInstances1stElement:             8,
+				ElementInstances1stElement:          16,
+				TotalSize:                           24,
 			},
 		},
 		{
@@ -41,7 +43,9 @@ func TestNewModuleContextOffsetData(t *testing.T) {
 				TablesBegin:                         -1,
 				BeforeListenerTrampolines1stElement: -1,
 				AfterListenerTrampolines1stElement:  -1,
-				TotalSize:                           24,
+				DataInstances1stElement:             24,
+				ElementInstances1stElement:          32,
+				TotalSize:                           40,
 			},
 		},
 		{
@@ -56,7 +60,9 @@ func TestNewModuleContextOffsetData(t *testing.T) {
 				TablesBegin:                         -1,
 				BeforeListenerTrampolines1stElement: -1,
 				AfterListenerTrampolines1stElement:  -1,
-				TotalSize:                           24,
+				DataInstances1stElement:             24,
+				ElementInstances1stElement:          32,
+				TotalSize:                           40,
 			},
 		},
 		{
@@ -71,7 +77,9 @@ func TestNewModuleContextOffsetData(t *testing.T) {
 				TablesBegin:                         -1,
 				BeforeListenerTrampolines1stElement: -1,
 				AfterListenerTrampolines1stElement:  -1,
-				TotalSize:                           10*FunctionInstanceSize + 8,
+				DataInstances1stElement:             10*FunctionInstanceSize + 8,
+				ElementInstances1stElement:          10*FunctionInstanceSize + 16,
+				TotalSize:                           10*FunctionInstanceSize + 24,
 			},
 		},
 		{
@@ -86,7 +94,9 @@ func TestNewModuleContextOffsetData(t *testing.T) {
 				TablesBegin:                         -1,
 				BeforeListenerTrampolines1stElement: -1,
 				AfterListenerTrampolines1stElement:  -1,
-				TotalSize:                           10*FunctionInstanceSize + 24,
+				DataInstances1stElement:             10*FunctionInstanceSize + 24,
+				ElementInstances1stElement:          10*FunctionInstanceSize + 32,
+				TotalSize:                           10*FunctionInstanceSize + 40,
 			},
 		},
 		{
@@ -108,7 +118,9 @@ func TestNewModuleContextOffsetData(t *testing.T) {
 				TablesBegin:                         24 + 10*FunctionInstanceSize + 8*30 + 8,
 				BeforeListenerTrampolines1stElement: -1,
 				AfterListenerTrampolines1stElement:  -1,
-				TotalSize:                           24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15,
+				DataInstances1stElement:             24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15,
+				ElementInstances1stElement:          24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15 + 8,
+				TotalSize:                           24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15 + 16,
 			},
 		},
 		{
@@ -131,7 +143,9 @@ func TestNewModuleContextOffsetData(t *testing.T) {
 				TablesBegin:                         24 + 10*FunctionInstanceSize + 8*30 + 8,
 				BeforeListenerTrampolines1stElement: 24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15,
 				AfterListenerTrampolines1stElement:  24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15 + 8,
-				TotalSize:                           24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15 + 16,
+				DataInstances1stElement:             24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15 + 16,
+				ElementInstances1stElement:          24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15 + 24,
+				TotalSize:                           24 + 10*FunctionInstanceSize + 8*30 + 8 + 8*15 + 32,
 			},
 		},
 	} {