Repository: tetratelabs/wazero
Pull request #1691 (2023-09-08 01:00:34)

wazevo(arm64): fixes Go entry ABI for arbitrary # of params/results
#1496 


Patch:
diff --git a/internal/engine/wazevo/backend/abi.go b/internal/engine/wazevo/backend/abi.go
index cea5c6ff..22ef6a2e 100644
--- a/internal/engine/wazevo/backend/abi.go
+++ b/internal/engine/wazevo/backend/abi.go
@@ -28,6 +28,7 @@ type (
 		// This VReg must be based on RealReg.
 		Reg regalloc.VReg
 		// Offset is valid if Kind == ABIArgKindStack.
+		// This is the offset from the stack pointer.
 		Offset int64
 		// Type is the type of the argument.
 		Type ssa.Type
diff --git a/internal/engine/wazevo/backend/isa/arm64/abi.go b/internal/engine/wazevo/backend/isa/arm64/abi.go
index 001c1f35..37a2a42b 100644
--- a/internal/engine/wazevo/backend/isa/arm64/abi.go
+++ b/internal/engine/wazevo/backend/isa/arm64/abi.go
@@ -300,6 +300,15 @@ func (a *abiImpl) callerGenFunctionReturnVReg(retIndex int, reg regalloc.VReg) {
 	}
 }
 
+func (m *machine) resolveAddressModeForOffsetAndInsert(cur *instruction, offset int64, dstBits byte, rn regalloc.VReg) (*instruction, addressMode) {
+	m.pendingInstructions = m.pendingInstructions[:0]
+	mode := m.resolveAddressModeForOffset(offset, dstBits, rn)
+	for _, instr := range m.pendingInstructions {
+		cur = linkInstr(cur, instr)
+	}
+	return cur, mode
+}
+
 func (m *machine) resolveAddressModeForOffset(offset int64, dstBits byte, rn regalloc.VReg) addressMode {
 	if rn.RegType() != regalloc.RegTypeInt {
 		panic("BUG: rn should be a pointer: " + formatVRegSized(rn, 64))
@@ -316,7 +325,7 @@ func (m *machine) resolveAddressModeForOffset(offset int64, dstBits byte, rn reg
 	return amode
 }
 
-func (a *abiImpl) alignedStackSlotSize() int64 {
+func (a *abiImpl) alignedArgResultStackSlotSize() int64 {
 	stackSlotSize := a.retStackSize + a.argStackSize
 	// Align stackSlotSize to 16 bytes.
 	stackSlotSize = (stackSlotSize + 15) &^ 15
@@ -336,7 +345,7 @@ func (m *machine) lowerCall(si *ssa.Instruction) {
 	}
 	calleeABI := m.getOrCreateABIImpl(m.compiler.ResolveSignature(sigID))
 
-	stackSlotSize := calleeABI.alignedStackSlotSize()
+	stackSlotSize := calleeABI.alignedArgResultStackSlotSize()
 	if stackSlotSize > 0 {
 		m.insertAddOrSubStackPointer(spVReg, stackSlotSize, false /* == sub */, false)
 	}
diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go b/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go
index 579c8129..dfb335f3 100644
--- a/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go
+++ b/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go
@@ -38,9 +38,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *
 		store := m.allocateInstr()
 		amode := addressMode{kind: addressModeKindRegUnsignedImm12, rn: execCtrPtr, imm: offset}
 		store.asStore(operandNR(moduleCtrPtr), amode, 64)
-		store.prev = cur
-		cur.next = store
-		cur = store
+		cur = linkInstr(cur, store)
 
 		argBegin++
 	}
@@ -53,9 +51,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *
 
 	goCallStackPtrCalc := m.allocateInstr()
 	goCallStackPtrCalc.asALU(aluOpAdd, operandNR(stackPtrReg), operandNR(execCtrPtr), imm12Op, true)
-	goCallStackPtrCalc.prev = cur
-	cur.next = goCallStackPtrCalc
-	cur = goCallStackPtrCalc
+	cur = linkInstr(cur, goCallStackPtrCalc)
 
 	abi := m.getOrCreateABIImpl(sig)
 	for _, arg := range abi.args[argBegin:] {
@@ -67,16 +63,14 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *
 			case ssa.TypeI32, ssa.TypeF32, ssa.TypeI64, ssa.TypeF64:
 				sizeInBits = 64 // We use uint64 for all basic types, except SIMD v128.
 			default:
-				panic("TODO")
+				panic("TODO? do you really need to pass v128 to host function?")
 			}
 			store.asStore(operandNR(v),
 				addressMode{
 					kind: addressModeKindPostIndex,
 					rn:   stackPtrReg, imm: int64(sizeInBits / 8),
 				}, sizeInBits)
-			store.prev = cur
-			cur.next = store
-			cur = store
+			cur = linkInstr(cur, store)
 		} else {
 			// We can temporarily load the argument to the tmp register, and then store it to the stack.
 			panic("TODO: many params for Go function call")
@@ -127,14 +121,11 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *
 				mode.imm = 8 // We use uint64 for all basic types, except SIMD v128.
 				loadIntoTmp.asFpuLoad(operandNR(tmpRegVRegVec), mode, 64)
 				movTmpToReg.asFpuMov64(r.Reg, tmpRegVRegVec)
+			default:
+				panic("Do you really need to return v128 from host function?")
 			}
-			loadIntoTmp.prev = cur
-			cur.next = loadIntoTmp
-			cur = loadIntoTmp
-
-			movTmpToReg.prev = cur
-			cur.next = movTmpToReg
-			cur = movTmpToReg
+			cur = linkInstr(cur, loadIntoTmp)
+			cur = linkInstr(cur, movTmpToReg)
 		} else {
 			// We can temporarily load the argument to the tmp register, and then store it to the stack.
 			panic("TODO: many params for Go function call")
@@ -143,8 +134,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *
 
 	ret := m.allocateInstrAfterLowering()
 	ret.asRet(nil)
-	ret.prev = cur
-	cur.next = ret
+	linkInstr(cur, ret)
 
 	m.encode(m.rootInstr)
 	return m.compiler.Buf()
@@ -195,24 +185,33 @@ func (m *machine) restoreRegistersInExecutionContext(cur *instruction, regs []re
 				// Execution context is always the first argument.
 				rn: x0VReg, imm: offset,
 			}, sizeInBits)
-		load.prev = cur
-		cur.next = load
-		cur = load
+		cur = linkInstr(cur, load)
 		offset += 16 // Imm12 must be aligned 16 for vector regs, so we unconditionally load regs at the offset of multiple of 16.
 	}
 	return cur
 }
 
-func (m *machine) setExitCode(cur *instruction, execCtr regalloc.VReg, exitCode wazevoapi.ExitCode) *instruction {
-	constReg := x17VReg // caller-saved, so we can use it.
+func (m *machine) lowerConstantI64AndInsert(cur *instruction, dst regalloc.VReg, v int64) *instruction {
+	m.pendingInstructions = m.pendingInstructions[:0]
+	m.lowerConstantI64(dst, v)
+	for _, instr := range m.pendingInstructions {
+		cur = linkInstr(cur, instr)
+	}
+	return cur
+}
 
+func (m *machine) lowerConstantI32AndInsert(cur *instruction, dst regalloc.VReg, v int32) *instruction {
 	m.pendingInstructions = m.pendingInstructions[:0]
-	m.lowerConstantI32(x17VReg, int32(exitCode))
+	m.lowerConstantI32(dst, v)
 	for _, instr := range m.pendingInstructions {
-		instr.prev = cur
-		cur.next = instr
-		cur = instr
+		cur = linkInstr(cur, instr)
 	}
+	return cur
+}
+
+func (m *machine) setExitCode(cur *instruction, execCtr regalloc.VReg, exitCode wazevoapi.ExitCode) *instruction {
+	constReg := x17VReg // caller-saved, so we can use it.
+	cur = m.lowerConstantI32AndInsert(cur, constReg, int32(exitCode))
 
 	// Set the exit status on the execution context.
 	setExistStatus := m.allocateInstrAfterLowering()
@@ -221,9 +220,7 @@ func (m *machine) setExitCode(cur *instruction, execCtr regalloc.VReg, exitCode
 			kind: addressModeKindRegUnsignedImm12,
 			rn:   execCtr, imm: wazevoapi.ExecutionContextOffsets.ExitCodeOffset.I64(),
 		}, 32)
-	setExistStatus.prev = cur
-	cur.next = setExistStatus
-	cur = setExistStatus
+	cur = linkInstr(cur, setExistStatus)
 	return cur
 }
 
@@ -231,9 +228,8 @@ func (m *machine) storeReturnAddressAndExit(cur *instruction) *instruction {
 	// Read the return address into tmp, and store it in the execution context.
 	adr := m.allocateInstrAfterLowering()
 	adr.asAdr(tmpRegVReg, exitSequenceSize+8)
-	adr.prev = cur
-	cur.next = adr
-	cur = adr
+	cur = linkInstr(cur, adr)
+
 	storeReturnAddr := m.allocateInstrAfterLowering()
 	storeReturnAddr.asStore(operandNR(tmpRegVReg),
 		addressMode{
@@ -241,16 +237,12 @@ func (m *machine) storeReturnAddressAndExit(cur *instruction) *instruction {
 			// Execution context is always the first argument.
 			rn: x0VReg, imm: wazevoapi.ExecutionContextOffsets.GoCallReturnAddress.I64(),
 		}, 64)
-	storeReturnAddr.prev = cur
-	cur.next = storeReturnAddr
-	cur = storeReturnAddr
+	cur = linkInstr(cur, storeReturnAddr)
 
 	// Exit the execution.
 	trapSeq := m.allocateInstrAfterLowering()
 	trapSeq.asExitSequence(x0VReg)
-	trapSeq.prev = cur
-	cur.next = trapSeq
-	cur = trapSeq
+	cur = linkInstr(cur, trapSeq)
 	return cur
 }
 
@@ -260,17 +252,14 @@ func (m *machine) saveCurrentStackPointer(cur *instruction, execCtr regalloc.VRe
 	// 	str tmp, [exec_ctx, #stackPointerBeforeGoCall]
 	movSp := m.allocateInstrAfterLowering()
 	movSp.asMove64(tmpRegVReg, spVReg)
-	movSp.prev = cur
-	cur.next = movSp
-	cur = movSp
+	cur = linkInstr(cur, movSp)
+
 	strSp := m.allocateInstrAfterLowering()
 	strSp.asStore(operandNR(tmpRegVReg),
 		addressMode{
 			kind: addressModeKindRegUnsignedImm12,
 			rn:   execCtr, imm: wazevoapi.ExecutionContextOffsets.StackPointerBeforeGrow.I64(),
 		}, 64)
-	strSp.prev = cur
-	cur.next = strSp
-	cur = strSp
+	cur = linkInstr(cur, strSp)
 	return cur
 }
diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_go_entry.go b/internal/engine/wazevo/backend/isa/arm64/abi_go_entry.go
index 355dde62..b8a0170f 100644
--- a/internal/engine/wazevo/backend/isa/arm64/abi_go_entry.go
+++ b/internal/engine/wazevo/backend/isa/arm64/abi_go_entry.go
@@ -26,8 +26,108 @@ var (
 	savedExecutionContextPtr = x20VReg
 	// goAllocatedStackPtr is not used in the epilogue.
 	goAllocatedStackPtr = x26VReg
+	// paramResultSliceCopied is not used in the epilogue.
+	paramResultSliceCopied = x25VReg
 )
 
+func (m *machine) goEntryPreamblePassArg(cur *instruction, paramSlicePtr regalloc.VReg, arg *backend.ABIArg) *instruction {
+	typ := arg.Type
+	bits := typ.Bits()
+	isStackArg := arg.Kind == backend.ABIArgKindStack
+
+	var loadTargetReg operand
+	if !isStackArg {
+		loadTargetReg = operandNR(arg.Reg)
+	} else {
+		switch typ {
+		case ssa.TypeI32, ssa.TypeI64:
+			loadTargetReg = operandNR(tmpRegVReg)
+		case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
+			loadTargetReg = operandNR(v15VReg)
+		default:
+			panic("TODO?")
+		}
+	}
+
+	var postIndexImm int64
+	if typ == ssa.TypeV128 {
+		postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice.
+	} else {
+		postIndexImm = 8
+	}
+	loadMode := addressMode{kind: addressModeKindPostIndex, rn: paramSlicePtr, imm: postIndexImm}
+
+	instr := m.allocateInstr()
+	switch typ {
+	case ssa.TypeI32:
+		instr.asULoad(loadTargetReg, loadMode, 32)
+	case ssa.TypeI64:
+		instr.asULoad(loadTargetReg, loadMode, 64)
+	case ssa.TypeF32:
+		instr.asFpuLoad(loadTargetReg, loadMode, 32)
+	case ssa.TypeF64:
+		instr.asFpuLoad(loadTargetReg, loadMode, 64)
+	case ssa.TypeV128:
+		instr.asFpuLoad(loadTargetReg, loadMode, 128)
+	}
+	cur = linkInstr(cur, instr)
+
+	if isStackArg {
+		var storeMode addressMode
+		cur, storeMode = m.resolveAddressModeForOffsetAndInsert(cur, arg.Offset, bits, spVReg)
+		toStack := m.allocateInstr()
+		toStack.asStore(loadTargetReg, storeMode, bits)
+		cur = linkInstr(cur, toStack)
+	}
+	return cur
+}
+
+func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr regalloc.VReg, result *backend.ABIArg) *instruction {
+	isStackArg := result.Kind == backend.ABIArgKindStack
+	typ := result.Type
+	bits := typ.Bits()
+
+	var storeTargetReg operand
+	if !isStackArg {
+		storeTargetReg = operandNR(result.Reg)
+	} else {
+		switch typ {
+		case ssa.TypeI32, ssa.TypeI64:
+			storeTargetReg = operandNR(tmpRegVReg)
+		case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
+			storeTargetReg = operandNR(v15VReg)
+		default:
+			panic("TODO?")
+		}
+	}
+
+	var postIndexImm int64
+	if typ == ssa.TypeV128 {
+		postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice.
+	} else {
+		postIndexImm = 8
+	}
+
+	if isStackArg {
+		var loadMode addressMode
+		cur, loadMode = m.resolveAddressModeForOffsetAndInsert(cur, result.Offset, bits, spVReg)
+		toReg := m.allocateInstr()
+		switch typ {
+		case ssa.TypeI32, ssa.TypeI64:
+			toReg.asULoad(storeTargetReg, loadMode, bits)
+		case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
+			toReg.asFpuLoad(storeTargetReg, loadMode, bits)
+		}
+		cur = linkInstr(cur, toReg)
+	}
+
+	mode := addressMode{kind: addressModeKindPostIndex, rn: resultSlicePtr, imm: postIndexImm}
+	instr := m.allocateInstr()
+	instr.asStore(storeTargetReg, mode, bits)
+	cur = linkInstr(cur, instr)
+	return cur
+}
+
 func (a *abiImpl) constructGoEntryPreamble() (root *instruction) {
 	m := a.m
 	root = m.allocateNop()
@@ -50,7 +150,7 @@ func (a *abiImpl) constructGoEntryPreamble() (root *instruction) {
 
 	// Next, adjust the Go-allocated stack pointer to reserve the arg/result spaces.
 	// 		sub x28, x28, #stackSlotSize
-	if stackSlotSize := a.alignedStackSlotSize(); stackSlotSize > 0 {
+	if stackSlotSize := a.alignedArgResultStackSlotSize(); stackSlotSize > 0 {
 		if imm12Operand, ok := asImm12Operand(uint64(stackSlotSize)); ok {
 			instr := m.allocateInstr()
 			rd := operandNR(goAllocatedStackPtr)
@@ -65,44 +165,20 @@ func (a *abiImpl) constructGoEntryPreamble() (root *instruction) {
 	// 		mov sp, x28
 	cur = a.move64(spVReg, goAllocatedStackPtr, cur)
 
-	var offset int64
+	prReg := paramResultSlicePtr
+	if len(a.args) > 2 && len(a.rets) > 0 {
+		// paramResultSlicePtr is modified during the execution of goEntryPreamblePassArg,
+		// so copy it to another reg.
+		cur = a.move64(paramResultSliceCopied, paramResultSlicePtr, cur)
+		prReg = paramResultSliceCopied
+	}
 	for i := range a.args {
 		if i < 2 {
 			// module context ptr and execution context ptr are passed in x0 and x1 by the Go assembly function.
 			continue
 		}
 		arg := &a.args[i]
-		switch arg.Kind {
-		case backend.ABIArgKindReg:
-			r := arg.Reg
-			rd, mode := operandNR(r), addressMode{
-				kind: addressModeKindRegUnsignedImm12,
-				rn:   paramResultSlicePtr,
-				imm:  offset,
-			}
-
-			instr := m.allocateInstr()
-			switch arg.Type {
-			case ssa.TypeI32:
-				instr.asULoad(rd, mode, 32)
-			case ssa.TypeI64:
-				instr.asULoad(rd, mode, 64)
-			case ssa.TypeF32:
-				instr.asFpuLoad(rd, mode, 32)
-			case ssa.TypeF64:
-				instr.asFpuLoad(rd, mode, 64)
-			case ssa.TypeV128:
-				instr.asFpuLoad(rd, mode, 128)
-			}
-			cur = linkInstr(cur, instr)
-		case backend.ABIArgKindStack:
-			panic("TODO")
-		}
-		if arg.Type == ssa.TypeV128 {
-			offset += 16
-		} else {
-			offset += 8
-		}
+		cur = m.goEntryPreamblePassArg(cur, prReg, arg)
 	}
 
 	// Call the real function coming after epilogue:
@@ -115,43 +191,9 @@ func (a *abiImpl) constructGoEntryPreamble() (root *instruction) {
 
 	// Store the register results into paramResultSlicePtr.
 	for i := range a.rets {
-		ret := &a.rets[i]
-		switch ret.Kind {
-		case backend.ABIArgKindReg:
-			rd := operandNR(ret.Reg)
-			mode := addressMode{kind: addressModeKindPostIndex, rn: paramResultSlicePtr, imm: 8}
-			instr := m.allocateInstr()
-			instr.asStore(rd, mode, ret.Type.Bits())
-			cur = linkInstr(cur, instr)
-		case backend.ABIArgKindStack:
-			offset, typ := ret.Offset, ret.Type
-			if offset != 0 && !offsetFitsInAddressModeKindRegUnsignedImm12(typ.Bits(), ret.Offset) {
-				// Do we really want to support?
-				panic("TODO: too many parameters")
-			}
-
-			tmpOperand := operandNR(tmpRegVReg)
-
-			// First load the value from the Go-allocated stack into temporary.
-			mode := addressMode{kind: addressModeKindRegUnsignedImm12, rn: spVReg, imm: offset}
-			toTmp := m.allocateInstr()
-			switch ret.Type {
-			case ssa.TypeI32, ssa.TypeI64:
-				toTmp.asULoad(tmpOperand, mode, typ.Bits())
-			case ssa.TypeF32, ssa.TypeF64:
-				toTmp.asFpuLoad(tmpOperand, mode, typ.Bits())
-			default:
-				panic("TODO")
-			}
-			cur = linkInstr(cur, toTmp)
-
-			// Then write it back to the paramResultSlicePtr.
-			mode = addressMode{kind: addressModeKindPostIndex, rn: paramResultSlicePtr, imm: 8}
-			storeTmp := m.allocateInstr()
-			storeTmp.asStore(tmpOperand, mode, ret.Type.Bits())
-			cur = linkInstr(cur, storeTmp)
-		}
+		cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, &a.rets[i])
 	}
+
 	// Finally, restore the FP, SP and LR, and return to the Go code.
 	// 		ldr fp, [savedExecutionContextPtr, #OriginalFramePointer]
 	// 		ldr tmp, [savedExecutionContextPtr, #OriginalStackPointer]
diff --git a/internal/engine/wazevo/backend/isa/arm64/instr.go b/internal/engine/wazevo/backend/isa/arm64/instr.go
index 2350b480..34bb60f7 100644
--- a/internal/engine/wazevo/backend/isa/arm64/instr.go
+++ b/internal/engine/wazevo/backend/isa/arm64/instr.go
@@ -399,8 +399,9 @@ func (i *instruction) asMOVN(dst regalloc.VReg, imm uint64, shift uint64, dst64b
 	}
 }
 
-func (i *instruction) asNop0() {
+func (i *instruction) asNop0() *instruction {
 	i.kind = nop0
+	return i
 }
 
 func (i *instruction) asNop0WithLabel(l label) {
@@ -1399,9 +1400,10 @@ func (i *instruction) asIntToFpu(rd, rn operand, rnSigned, src64bit, dst64bit bo
 	}
 }
 
-func (i *instruction) asExitSequence(ctx regalloc.VReg) {
+func (i *instruction) asExitSequence(ctx regalloc.VReg) *instruction {
 	i.kind = exitSequence
 	i.rn = operandNR(ctx)
+	return i
 }
 
 // aluOp determines the type of ALU operation. Instructions whose kind is one of
diff --git a/internal/engine/wazevo/backend/isa/arm64/machine.go b/internal/engine/wazevo/backend/isa/arm64/machine.go
index 11d276b1..add5f38a 100644
--- a/internal/engine/wazevo/backend/isa/arm64/machine.go
+++ b/internal/engine/wazevo/backend/isa/arm64/machine.go
@@ -322,18 +322,8 @@ func (m *machine) resolveAddressingMode(arg0offset, ret0offset int64, i *instruc
 	} else {
 		// This case, we load the offset into the temporary register,
 		// and then use it as the index register.
-		m.lowerConstantI64(tmpRegVReg, amode.imm)
-		// lowerConstantI64 adds instructions into m.pendingInstructions,
-		// so we manually link them together.
-		cur := i.prev
-		for _, inserted := range m.pendingInstructions {
-			cur.next = inserted
-			inserted.prev = cur
-			cur = inserted
-		}
-		cur.next = i
-		i.prev = cur
-		m.pendingInstructions = m.pendingInstructions[:0]
+		newPrev := m.lowerConstantI64AndInsert(i.prev, tmpRegVReg, amode.imm)
+		linkInstr(newPrev, i)
 		*amode = addressMode{kind: addressModeKindRegReg, rn: amode.rn, rm: tmpRegVReg, extOp: extendOpUXTX /* indicates rm reg is 64-bit */}
 	}
 }
diff --git a/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go b/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go
index a369b256..da35792c 100644
--- a/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go
+++ b/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go
@@ -268,26 +268,14 @@ func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instructi
 		// sub tmp, sp, #requiredStackSize
 		sub := m.allocateInstrAfterLowering()
 		sub.asALU(aluOpSub, operandNR(tmpRegVReg), operandNR(spVReg), immm12op, true)
-		sub.prev = cur
-		cur.next = sub
-		cur = sub
+		cur = linkInstr(cur, sub)
 	} else {
 		// This case, we first load the requiredStackSize into the temporary register,
-		m.lowerConstantI64(tmpRegVReg, requiredStackSize)
-		// lowerConstantI64 adds instructions into m.pendingInstructions,
-		// so we manually link them together.
-		for _, inserted := range m.pendingInstructions {
-			cur.next = inserted
-			inserted.prev = cur
-			cur = inserted
-		}
-		m.pendingInstructions = m.pendingInstructions[:0]
+		cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, requiredStackSize)
 		// Then subtract it.
 		sub := m.allocateInstrAfterLowering()
 		sub.asALU(aluOpSub, operandNR(tmpRegVReg), operandNR(spVReg), operandNR(tmpRegVReg), true)
-		sub.prev = cur
-		cur.next = sub
-		cur = sub
+		cur = linkInstr(cur, sub)
 	}
 
 	tmp2 := x11VReg // Callee save, so it is safe to use it here in the prologue.
@@ -320,15 +308,7 @@ func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instructi
 	// Set the required stack size and set it to the exec context.
 	{
 		// First load the requiredStackSize into the temporary register,
-		m.pendingInstructions = m.pendingInstructions[:0]
-		m.lowerConstantI64(tmpRegVReg, requiredStackSize)
-		// lowerConstantI64 adds instructions into m.pendingInstructions,
-		// so we manually link them together.
-		for _, inserted := range m.pendingInstructions {
-			cur.next = inserted
-			inserted.prev = cur
-			cur = inserted
-		}
+		cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, requiredStackSize)
 		setRequiredStackSize := m.allocateInstrAfterLowering()
 		setRequiredStackSize.asStore(operandNR(tmpRegVReg),
 			addressMode{
@@ -408,10 +388,7 @@ func (m *machine) addsAddOrSubStackPointer(cur *instruction, rd regalloc.VReg, d
 	m.pendingInstructions = m.pendingInstructions[:0]
 	m.insertAddOrSubStackPointer(rd, diff, add, afterLowering)
 	for _, inserted := range m.pendingInstructions {
-		cur.next = inserted
-		inserted.prev = cur
-		cur = inserted
+		cur = linkInstr(cur, inserted)
 	}
-	m.pendingInstructions = m.pendingInstructions[:0]
 	return cur
 }
diff --git a/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go b/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go
index 53179deb..68993bf2 100644
--- a/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go
+++ b/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go
@@ -116,13 +116,13 @@ func (f *regAllocFunctionImpl) StoreRegisterAfter(v regalloc.VReg, instr regallo
 // ReloadRegisterBefore implements regalloc.Function ReloadRegisterBefore.
 func (f *regAllocFunctionImpl) ReloadRegisterBefore(v regalloc.VReg, instr regalloc.Instr) {
 	m := f.m
-	m.reloadRegister(v, instr.(*regAllocInstrImpl).i, false)
+	m.insertReloadRegisterAt(v, instr.(*regAllocInstrImpl).i, false)
 }
 
 // ReloadRegisterAfter implements regalloc.Function ReloadRegisterAfter.
 func (f *regAllocFunctionImpl) ReloadRegisterAfter(v regalloc.VReg, instr regalloc.Instr) {
 	m := f.m
-	m.reloadRegister(v, instr.(*regAllocInstrImpl).i, true)
+	m.insertReloadRegisterAt(v, instr.(*regAllocInstrImpl).i, true)
 }
 
 // Done implements regalloc.Function Done.
@@ -278,12 +278,6 @@ func (m *machine) insertStoreRegisterAt(v regalloc.VReg, instr *instruction, aft
 
 	typ := m.compiler.TypeOf(v)
 
-	offsetFromSP := m.getVRegSpillSlotOffset(v.ID(), typ.Size()) + m.clobberedRegSlotSize()
-	m.pendingInstructions = m.pendingInstructions[:0]
-	admode := m.resolveAddressModeForOffset(offsetFromSP, typ.Bits(), spVReg)
-	store := m.allocateInstrAfterLowering()
-	store.asStore(operandNR(v), admode, typ.Bits())
-
 	var prevNext, cur *instruction
 	if after {
 		cur, prevNext = instr, instr.next
@@ -291,61 +285,47 @@ func (m *machine) insertStoreRegisterAt(v regalloc.VReg, instr *instruction, aft
 		cur, prevNext = instr.prev, instr
 	}
 
-	// If the offset is large, we might end up with having multiple instructions inserted in resolveAddressModeForOffset.
-	for _, instr := range m.pendingInstructions {
-		instr.addedAfterLowering = true
-		cur.next = instr
-		instr.prev = cur
-		cur = instr
-	}
-
-	cur.next = store
-	store.prev = cur
+	offsetFromSP := m.getVRegSpillSlotOffset(v.ID(), typ.Size()) + m.clobberedRegSlotSize()
+	var amode addressMode
+	cur, amode = m.resolveAddressModeForOffsetAndInsert(cur, offsetFromSP, typ.Bits(), spVReg)
+	store := m.allocateInstrAfterLowering()
+	store.asStore(operandNR(v), amode, typ.Bits())
 
-	store.next = prevNext
-	prevNext.prev = store
+	cur = linkInstr(cur, store)
+	linkInstr(cur, prevNext)
 }
 
-func (m *machine) reloadRegister(v regalloc.VReg, instr *instruction, after bool) {
+func (m *machine) insertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) {
 	if !v.IsRealReg() {
 		panic("BUG: VReg must be backed by real reg to be stored")
 	}
 
 	typ := m.compiler.TypeOf(v)
 
+	var prevNext, cur *instruction
+	if after {
+		cur, prevNext = instr, instr.next
+	} else {
+		cur, prevNext = instr.prev, instr
+	}
+
 	offsetFromSP := m.getVRegSpillSlotOffset(v.ID(), typ.Size()) + m.clobberedRegSlotSize()
-	m.pendingInstructions = m.pendingInstructions[:0]
-	admode := m.resolveAddressModeForOffset(offsetFromSP, typ.Bits(), spVReg)
+	var amode addressMode
+	cur, amode = m.resolveAddressModeForOffsetAndInsert(cur, offsetFromSP, typ.Bits(), spVReg)
 	load := m.allocateInstrAfterLowering()
 	switch typ {
 	case ssa.TypeI32, ssa.TypeI64:
-		load.asULoad(operandNR(v), admode, typ.Bits())
+		load.asULoad(operandNR(v), amode, typ.Bits())
 	case ssa.TypeF32, ssa.TypeF64:
-		load.asFpuLoad(operandNR(v), admode, typ.Bits())
+		load.asFpuLoad(operandNR(v), amode, typ.Bits())
 	case ssa.TypeV128:
-		load.asFpuLoad(operandNR(v), admode, 128)
+		load.asFpuLoad(operandNR(v), amode, 128)
 	default:
 		panic("TODO")
 	}
 
-	var prevNext, cur *instruction
-	if after {
-		cur, prevNext = instr, instr.next
-	} else {
-		cur, prevNext = instr.prev, instr
-	}
-
-	// If the offset is large, we might end up with having multiple instructions inserted in resolveAddressModeForOffset.
-	for _, instr := range m.pendingInstructions {
-		instr.addedAfterLowering = true
-		cur.next = instr
-		instr.prev = cur
-		cur = instr
-	}
-
 	cur.next = load
 	load.prev = cur
-
-	load.next = prevNext
-	prevNext.prev = load
+	cur = linkInstr(cur, load)
+	linkInstr(cur, prevNext)
 }


Test patch:
diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_go_entry_test.go b/internal/engine/wazevo/backend/isa/arm64/abi_go_entry_test.go
index 5b41d9ed..9f576ee2 100644
--- a/internal/engine/wazevo/backend/isa/arm64/abi_go_entry_test.go
+++ b/internal/engine/wazevo/backend/isa/arm64/abi_go_entry_test.go
@@ -4,12 +4,13 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
 	"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
 	"github.com/tetratelabs/wazero/internal/testing/require"
 )
 
 func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
-	const i32, f32, i64, f64 = ssa.TypeI32, ssa.TypeF32, ssa.TypeI64, ssa.TypeF64
+	const i32, f32, i64, f64, v128 = ssa.TypeI32, ssa.TypeF32, ssa.TypeI64, ssa.TypeF64, ssa.TypeV128
 
 	for _, tc := range []struct {
 		name string
@@ -49,11 +50,11 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 	str x27, [x20, #0x18]
 	str x30, [x20, #0x20]
 	mov sp, x26
-	ldr s0, [x19]
-	ldr s1, [x19, #0x8]
-	ldr s2, [x19, #0x10]
-	ldr s3, [x19, #0x18]
-	ldr d4, [x19, #0x20]
+	ldr s0, [x19], #0x8
+	ldr s1, [x19], #0x8
+	ldr s2, [x19], #0x8
+	ldr s3, [x19], #0x8
+	ldr d4, [x19], #0x8
 	bl #0x18
 	ldr x29, [x20, #0x10]
 	ldr x27, [x20, #0x18]
@@ -77,11 +78,11 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 	str x27, [x20, #0x18]
 	str x30, [x20, #0x20]
 	mov sp, x26
-	ldr w2, [x19]
-	ldr w3, [x19, #0x8]
-	ldr w4, [x19, #0x10]
-	ldr x5, [x19, #0x18]
-	ldr w6, [x19, #0x20]
+	ldr w2, [x19], #0x8
+	ldr w3, [x19], #0x8
+	ldr w4, [x19], #0x8
+	ldr x5, [x19], #0x8
+	ldr w6, [x19], #0x8
 	bl #0x18
 	ldr x29, [x20, #0x10]
 	ldr x27, [x20, #0x18]
@@ -95,7 +96,7 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 			sig: &ssa.Signature{
 				Params: []ssa.Type{
 					i64, i64, // first and second will be skipped.
-					i32, f64, i32, f32, i64, i32, i64, f64, i32, f32,
+					i32, f64, i32, f32, i64, i32, i64, f64, i32, f32, v128, f32,
 				},
 			},
 			exp: `
@@ -105,16 +106,18 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 	str x27, [x20, #0x18]
 	str x30, [x20, #0x20]
 	mov sp, x26
-	ldr w2, [x19]
-	ldr d0, [x19, #0x8]
-	ldr w3, [x19, #0x10]
-	ldr s1, [x19, #0x18]
-	ldr x4, [x19, #0x20]
-	ldr w5, [x19, #0x28]
-	ldr x6, [x19, #0x30]
-	ldr d2, [x19, #0x38]
-	ldr w7, [x19, #0x40]
-	ldr s3, [x19, #0x48]
+	ldr w2, [x19], #0x8
+	ldr d0, [x19], #0x8
+	ldr w3, [x19], #0x8
+	ldr s1, [x19], #0x8
+	ldr x4, [x19], #0x8
+	ldr w5, [x19], #0x8
+	ldr x6, [x19], #0x8
+	ldr d2, [x19], #0x8
+	ldr w7, [x19], #0x8
+	ldr s3, [x19], #0x8
+	ldr q4, [x19], #0x10
+	ldr s5, [x19], #0x8
 	bl #0x18
 	ldr x29, [x20, #0x10]
 	ldr x27, [x20, #0x18]
@@ -139,11 +142,12 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 	str x27, [x20, #0x18]
 	str x30, [x20, #0x20]
 	mov sp, x26
-	ldr w2, [x19]
-	ldr d0, [x19, #0x8]
-	ldr w3, [x19, #0x10]
-	ldr s1, [x19, #0x18]
-	ldr x4, [x19, #0x20]
+	mov x25, x19
+	ldr w2, [x25], #0x8
+	ldr d0, [x25], #0x8
+	ldr w3, [x25], #0x8
+	ldr s1, [x25], #0x8
+	ldr x4, [x25], #0x8
 	bl #0x34
 	str s0, [x19], #0x8
 	str d1, [x19], #0x8
@@ -165,6 +169,7 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 				Results: []ssa.Type{
 					f32, f64, i32, f32, i64, i32, i32, i64, i32, i64,
 					f32, f64, i32, f32, i64, i32, i32, i64, i32, i64,
+					f32, f64, f64, f32, f64, v128, v128,
 				},
 			},
 			exp: `
@@ -173,9 +178,9 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 	mov x27, sp
 	str x27, [x20, #0x18]
 	str x30, [x20, #0x20]
-	sub x26, x26, #0x30
+	sub x26, x26, #0x70
 	mov sp, x26
-	bl #0x80
+	bl #0xb0
 	str s0, [x19], #0x8
 	str d1, [x19], #0x8
 	str w0, [x19], #0x8
@@ -202,6 +207,189 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 	str w27, [x19], #0x8
 	ldr x27, [sp, #0x28]
 	str x27, [x19], #0x8
+	str s6, [x19], #0x8
+	str d7, [x19], #0x8
+	ldr d15, [sp, #0x30]
+	str d15, [x19], #0x8
+	ldr s15, [sp, #0x38]
+	str s15, [x19], #0x8
+	ldr d15, [sp, #0x40]
+	str d15, [x19], #0x8
+	ldr q15, [sp, #0x50]
+	str q15, [x19], #0x10
+	ldr q15, [sp, #0x60]
+	str q15, [x19], #0x10
+	ldr x29, [x20, #0x10]
+	ldr x27, [x20, #0x18]
+	mov sp, x27
+	ldr x30, [x20, #0x20]
+	ret
+`,
+		},
+		{
+			name: "many params",
+			sig: &ssa.Signature{
+				Params: []ssa.Type{
+					i32, i32, v128, v128, v128, i64, i32, i32, i64, i32, i64,
+					f32, f64, i32, f32, i64, i32, i32, i64, i32, i64,
+					f32, f64, f64, f32, f64, v128, v128, v128, v128, v128,
+				},
+			},
+			exp: `
+	mov x20, x0
+	str x29, [x20, #0x10]
+	mov x27, sp
+	str x27, [x20, #0x18]
+	str x30, [x20, #0x20]
+	sub x26, x26, #0xa0
+	mov sp, x26
+	ldr q0, [x19], #0x10
+	ldr q1, [x19], #0x10
+	ldr q2, [x19], #0x10
+	ldr x2, [x19], #0x8
+	ldr w3, [x19], #0x8
+	ldr w4, [x19], #0x8
+	ldr x5, [x19], #0x8
+	ldr w6, [x19], #0x8
+	ldr x7, [x19], #0x8
+	ldr s3, [x19], #0x8
+	ldr d4, [x19], #0x8
+	ldr w27, [x19], #0x8
+	str w27, [sp]
+	ldr s5, [x19], #0x8
+	ldr x27, [x19], #0x8
+	str x27, [sp, #0x8]
+	ldr w27, [x19], #0x8
+	str w27, [sp, #0x10]
+	ldr w27, [x19], #0x8
+	str w27, [sp, #0x18]
+	ldr x27, [x19], #0x8
+	str x27, [sp, #0x20]
+	ldr w27, [x19], #0x8
+	str w27, [sp, #0x28]
+	ldr x27, [x19], #0x8
+	str x27, [sp, #0x30]
+	ldr s6, [x19], #0x8
+	ldr d7, [x19], #0x8
+	ldr d15, [x19], #0x8
+	str d15, [sp, #0x38]
+	ldr s15, [x19], #0x8
+	str s15, [sp, #0x40]
+	ldr d15, [x19], #0x8
+	str d15, [sp, #0x48]
+	ldr q15, [x19], #0x10
+	str q15, [sp, #0x50]
+	ldr q15, [x19], #0x10
+	str q15, [sp, #0x60]
+	ldr q15, [x19], #0x10
+	str q15, [sp, #0x70]
+	ldr q15, [x19], #0x10
+	str q15, [sp, #0x80]
+	ldr q15, [x19], #0x10
+	str q15, [sp, #0x90]
+	bl #0x18
+	ldr x29, [x20, #0x10]
+	ldr x27, [x20, #0x18]
+	mov sp, x27
+	ldr x30, [x20, #0x20]
+	ret
+`,
+		},
+		{
+			name: "many params/results",
+			sig: &ssa.Signature{
+				Params: []ssa.Type{
+					i32, i32, v128, v128, v128, i64, i32, i32, i64, i32, i64,
+					f32, f64, i32, f32, i64, i32, i32, i64, i32, i64,
+					f32, f64, f64, f32, f64, v128, v128, v128, v128, v128,
+				},
+				Results: []ssa.Type{
+					f32, f64, i32, f32, i64, i32, i32, i64, i32, i64,
+					i32, i32, v128, v128, v128, i64, i32, i32, i64, i32, i64,
+				},
+			},
+			exp: `
+	mov x20, x0
+	str x29, [x20, #0x10]
+	mov x27, sp
+	str x27, [x20, #0x18]
+	str x30, [x20, #0x20]
+	sub x26, x26, #0xe0
+	mov sp, x26
+	mov x25, x19
+	ldr q0, [x25], #0x10
+	ldr q1, [x25], #0x10
+	ldr q2, [x25], #0x10
+	ldr x2, [x25], #0x8
+	ldr w3, [x25], #0x8
+	ldr w4, [x25], #0x8
+	ldr x5, [x25], #0x8
+	ldr w6, [x25], #0x8
+	ldr x7, [x25], #0x8
+	ldr s3, [x25], #0x8
+	ldr d4, [x25], #0x8
+	ldr w27, [x25], #0x8
+	str w27, [sp]
+	ldr s5, [x25], #0x8
+	ldr x27, [x25], #0x8
+	str x27, [sp, #0x8]
+	ldr w27, [x25], #0x8
+	str w27, [sp, #0x10]
+	ldr w27, [x25], #0x8
+	str w27, [sp, #0x18]
+	ldr x27, [x25], #0x8
+	str x27, [sp, #0x20]
+	ldr w27, [x25], #0x8
+	str w27, [sp, #0x28]
+	ldr x27, [x25], #0x8
+	str x27, [sp, #0x30]
+	ldr s6, [x25], #0x8
+	ldr d7, [x25], #0x8
+	ldr d15, [x25], #0x8
+	str d15, [sp, #0x38]
+	ldr s15, [x25], #0x8
+	str s15, [sp, #0x40]
+	ldr d15, [x25], #0x8
+	str d15, [sp, #0x48]
+	ldr q15, [x25], #0x10
+	str q15, [sp, #0x50]
+	ldr q15, [x25], #0x10
+	str q15, [sp, #0x60]
+	ldr q15, [x25], #0x10
+	str q15, [sp, #0x70]
+	ldr q15, [x25], #0x10
+	str q15, [sp, #0x80]
+	ldr q15, [x25], #0x10
+	str q15, [sp, #0x90]
+	bl #0x88
+	str s0, [x19], #0x8
+	str d1, [x19], #0x8
+	str w0, [x19], #0x8
+	str s2, [x19], #0x8
+	str x1, [x19], #0x8
+	str w2, [x19], #0x8
+	str w3, [x19], #0x8
+	str x4, [x19], #0x8
+	str w5, [x19], #0x8
+	str x6, [x19], #0x8
+	str w7, [x19], #0x8
+	ldr w27, [sp]
+	str w27, [x19], #0x8
+	str q3, [x19], #0x10
+	str q4, [x19], #0x10
+	str q5, [x19], #0x10
+	ldr x27, [sp, #0x8]
+	str x27, [x19], #0x8
+	ldr w27, [sp, #0x10]
+	str w27, [x19], #0x8
+	ldr w27, [sp, #0x18]
+	str w27, [x19], #0x8
+	ldr x27, [sp, #0x20]
+	str x27, [x19], #0x8
+	ldr w27, [sp, #0x28]
+	str w27, [x19], #0x8
+	ldr x27, [sp, #0x30]
+	str x27, [x19], #0x8
 	ldr x29, [x20, #0x10]
 	ldr x27, [x20, #0x18]
 	mov sp, x27
@@ -219,3 +407,257 @@ func TestAbiImpl_constructGoEntryPreamble(t *testing.T) {
 		})
 	}
 }
+
+func TestMachine_goEntryPreamblePassArg(t *testing.T) {
+	paramSlicePtr := x16VReg
+	for _, tc := range []struct {
+		arg backend.ABIArg
+		exp string
+	}{
+		// Reg kinds.
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI32, Reg: x0VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	ldr w0, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI64, Reg: x0VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	ldr x0, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF32, Reg: v11VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	ldr s11, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF64, Reg: v12VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	ldr d12, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Reg: v12VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	ldr q12, [x16], #0x10
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Reg: v12VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	ldr q12, [x16], #0x10
+`,
+		},
+		// Stack kinds.
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI32, Offset: 0, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr w27, [x16], #0x8
+	str w27, [sp]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI32, Offset: 8, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr w27, [x16], #0x8
+	str w27, [sp, #0x8]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI64, Offset: 0, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr x27, [x16], #0x8
+	str x27, [sp]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI64, Offset: 128, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr x27, [x16], #0x8
+	str x27, [sp, #0x80]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF32, Offset: 64, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr s15, [x16], #0x8
+	str s15, [sp, #0x40]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF32, Offset: 2056, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr s15, [x16], #0x8
+	str s15, [sp, #0x808]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF64, Offset: 64, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr d15, [x16], #0x8
+	str d15, [sp, #0x40]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF64, Offset: 2056, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr d15, [x16], #0x8
+	str d15, [sp, #0x808]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Offset: 64, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr q15, [x16], #0x10
+	str q15, [sp, #0x40]
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Offset: 2056, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr q15, [x16], #0x10
+	movz x27, #0x808, lsl 0
+	str q15, [sp, x27]
+`,
+		},
+	} {
+		t.Run(tc.exp, func(t *testing.T) {
+			_, _, m := newSetupWithMockContext()
+			cur := m.allocateNop()
+			m.rootInstr = cur
+			m.goEntryPreamblePassArg(cur, paramSlicePtr, &tc.arg)
+			fmt.Println(m.Format())
+			require.Equal(t, tc.exp, m.Format())
+		})
+	}
+}
+
+func TestMachine_goEntryPreamblePassResult(t *testing.T) {
+	paramSlicePtr := x16VReg
+	for _, tc := range []struct {
+		arg backend.ABIArg
+		exp string
+	}{
+		// Reg kinds.
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI32, Reg: x0VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	str w0, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI64, Reg: x0VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	str x0, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF32, Reg: v11VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	str s11, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF64, Reg: v12VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	str d12, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Reg: v12VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	str q12, [x16], #0x10
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Reg: v12VReg, Kind: backend.ABIArgKindReg},
+			exp: `
+	str q12, [x16], #0x10
+`,
+		},
+		// Stack kinds.
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI32, Offset: 0, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr w27, [sp]
+	str w27, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI32, Offset: 8, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr w27, [sp, #0x8]
+	str w27, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI64, Offset: 0, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr x27, [sp]
+	str x27, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeI64, Offset: 128, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr x27, [sp, #0x80]
+	str x27, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF32, Offset: 64, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr s15, [sp, #0x40]
+	str s15, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF32, Offset: 2056, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr s15, [sp, #0x808]
+	str s15, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF64, Offset: 64, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr d15, [sp, #0x40]
+	str d15, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeF64, Offset: 2056, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr d15, [sp, #0x808]
+	str d15, [x16], #0x8
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Offset: 64, Kind: backend.ABIArgKindStack},
+			exp: `
+	ldr q15, [sp, #0x40]
+	str q15, [x16], #0x10
+`,
+		},
+		{
+			arg: backend.ABIArg{Type: ssa.TypeV128, Offset: 2056, Kind: backend.ABIArgKindStack},
+			exp: `
+	movz x27, #0x808, lsl 0
+	ldr q15, [sp, x27]
+	str q15, [x16], #0x10
+`,
+		},
+	} {
+		t.Run(tc.exp, func(t *testing.T) {
+			_, _, m := newSetupWithMockContext()
+			cur := m.allocateNop()
+			m.rootInstr = cur
+			m.goEntryPreamblePassResult(cur, paramSlicePtr, &tc.arg)
+			fmt.Println(m.Format())
+			require.Equal(t, tc.exp, m.Format())
+		})
+	}
+}
diff --git a/internal/engine/wazevo/backend/isa/arm64/machine_regalloc_test.go b/internal/engine/wazevo/backend/isa/arm64/machine_regalloc_test.go
index 2dcde7b5..293176d1 100644
--- a/internal/engine/wazevo/backend/isa/arm64/machine_regalloc_test.go
+++ b/internal/engine/wazevo/backend/isa/arm64/machine_regalloc_test.go
@@ -122,54 +122,146 @@ func TestRegAllocFunctionImpl_StoreRegisterBefore(t *testing.T) {
 `, m.Format())
 }
 
-func TestMachine_StoreRegisterAfter(t *testing.T) {
-	ctx, _, m := newSetupWithMockContext()
-	m.spillSlotSize = 0xffff
-
-	ctx.typeOf = map[regalloc.VReg]ssa.Type{x1VReg: ssa.TypeI64, v1VReg: ssa.TypeF64}
-	i1, i2 := m.allocateInstr().asUDF(), m.allocateInstr().asUDF()
-	i1.next = i2
-	i2.prev = i1
-
-	f := &regAllocFunctionImpl{m: m}
-	f.StoreRegisterAfter(x1VReg, &regAllocInstrImpl{i: i1})
-	f.StoreRegisterAfter(v1VReg, &regAllocInstrImpl{i: i1})
-
-	m.rootInstr = i1
-	require.Equal(t, `
+func TestMachine_insertStoreRegisterAt(t *testing.T) {
+	for _, tc := range []struct {
+		spillSlotSize int64
+		expected      string
+	}{
+		{
+			spillSlotSize: 0,
+			expected: `
 	udf
+	str x1, [sp]
+	str d1, [sp, #0x8]
+	exit_sequence x30
+`,
+		},
+		{
+			spillSlotSize: 0xffff,
+			expected: `
+	udf
+	movz x27, #0xffff, lsl 0
+	str x1, [sp, x27]
 	movz x27, #0x7, lsl 0
 	movk x27, #0x1, lsl 16
 	str d1, [sp, x27]
-	movz x27, #0xffff, lsl 0
-	str x1, [sp, x27]
+	exit_sequence x30
+`,
+		},
+		{
+			spillSlotSize: 0xffff_00,
+			expected: `
 	udf
-`, m.Format())
+	orr x27, xzr, #0xffff00
+	str x1, [sp, x27]
+	movz x27, #0xff08, lsl 0
+	movk x27, #0xff, lsl 16
+	str d1, [sp, x27]
+	exit_sequence x30
+`,
+		},
+	} {
+		t.Run(tc.expected, func(t *testing.T) {
+			ctx, _, m := newSetupWithMockContext()
+			m.spillSlotSize = tc.spillSlotSize
+
+			for _, after := range []bool{false, true} {
+				var name string
+				if after {
+					name = "after"
+				} else {
+					name = "before"
+				}
+				t.Run(name, func(t *testing.T) {
+					ctx.typeOf = map[regalloc.VReg]ssa.Type{x1VReg: ssa.TypeI64, v1VReg: ssa.TypeF64}
+					i1, i2 := m.allocateInstr().asUDF(), m.allocateInstr().asExitSequence(x30VReg)
+					i1.next = i2
+					i2.prev = i1
+
+					if after {
+						m.insertStoreRegisterAt(v1VReg, i1, after)
+						m.insertStoreRegisterAt(x1VReg, i1, after)
+					} else {
+						m.insertStoreRegisterAt(x1VReg, i2, after)
+						m.insertStoreRegisterAt(v1VReg, i2, after)
+					}
+					m.rootInstr = i1
+					require.Equal(t, tc.expected, m.Format())
+				})
+			}
+		})
+	}
 }
 
-func TestMachine_ReloadRegisterBefore(t *testing.T) {
-	ctx, _, m := newSetupWithMockContext()
-	m.spillSlotSize = 0xffff
-
-	ctx.typeOf = map[regalloc.VReg]ssa.Type{x1VReg: ssa.TypeI64, v1VReg: ssa.TypeF64}
-	i1, i2 := m.allocateInstr().asUDF(), m.allocateInstr().asUDF()
-	i1.next = i2
-	i2.prev = i1
-
-	f := &regAllocFunctionImpl{m: m}
-	f.ReloadRegisterBefore(x1VReg, &regAllocInstrImpl{i: i2})
-	f.ReloadRegisterBefore(v1VReg, &regAllocInstrImpl{i: i2})
-
-	m.rootInstr = i1
-	require.Equal(t, `
+func TestMachine_insertReloadRegisterAt(t *testing.T) {
+	for _, tc := range []struct {
+		spillSlotSize int64
+		expected      string
+	}{
+		{
+			spillSlotSize: 0,
+			expected: `
+	udf
+	ldr x1, [sp]
+	ldr d1, [sp, #0x8]
+	exit_sequence x30
+`,
+		},
+		{
+			spillSlotSize: 0xffff,
+			expected: `
 	udf
 	movz x27, #0xffff, lsl 0
 	ldr x1, [sp, x27]
 	movz x27, #0x7, lsl 0
 	movk x27, #0x1, lsl 16
 	ldr d1, [sp, x27]
+	exit_sequence x30
+`,
+		},
+		{
+			spillSlotSize: 0xffff_00,
+			expected: `
 	udf
-`, m.Format())
+	orr x27, xzr, #0xffff00
+	ldr x1, [sp, x27]
+	movz x27, #0xff08, lsl 0
+	movk x27, #0xff, lsl 16
+	ldr d1, [sp, x27]
+	exit_sequence x30
+`,
+		},
+	} {
+		t.Run(tc.expected, func(t *testing.T) {
+			ctx, _, m := newSetupWithMockContext()
+			m.spillSlotSize = tc.spillSlotSize
+
+			for _, after := range []bool{false, true} {
+				var name string
+				if after {
+					name = "after"
+				} else {
+					name = "before"
+				}
+				t.Run(name, func(t *testing.T) {
+					ctx.typeOf = map[regalloc.VReg]ssa.Type{x1VReg: ssa.TypeI64, v1VReg: ssa.TypeF64}
+					i1, i2 := m.allocateInstr().asUDF(), m.allocateInstr().asExitSequence(x30VReg)
+					i1.next = i2
+					i2.prev = i1
+
+					if after {
+						m.insertReloadRegisterAt(v1VReg, i1, after)
+						m.insertReloadRegisterAt(x1VReg, i1, after)
+					} else {
+						m.insertReloadRegisterAt(x1VReg, i2, after)
+						m.insertReloadRegisterAt(v1VReg, i2, after)
+					}
+					m.rootInstr = i1
+					require.Equal(t, tc.expected, m.Format())
+				})
+			}
+		})
+	}
 }
 
 func TestRegAllocFunctionImpl_ClobberedRegisters(t *testing.T) {
diff --git a/internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs b/internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs
index 27085809..07f14dbc 100644
--- a/internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs
+++ b/internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs
@@ -18,6 +18,7 @@ extern "C" {
     pub fn validate(binary_ptr: *const u8, binary_size: usize);
 }
 
+#[allow(dead_code)]
 pub fn maybe_disable_v2(config: &mut SwarmConfig) {
     if std::env::var("WAZERO_FUZZ_WAZEVO").is_ok() {
         config.simd_enabled = false;