Repository: tetratelabs/wazero
Pull request #2056 (2024-02-13 21:26:07)

wazevo: redo SIMD dot translation for both arm/amd


Patch:
diff --git a/internal/engine/wazevo/backend/isa/amd64/machine.go b/internal/engine/wazevo/backend/isa/amd64/machine.go
index c766d440..11d34129 100644
--- a/internal/engine/wazevo/backend/isa/amd64/machine.go
+++ b/internal/engine/wazevo/backend/isa/amd64/machine.go
@@ -867,6 +867,10 @@ func (m *machine) LowerInstr(instr *ssa.Instruction) {
 		dst := m.c.VRegOf(instr.Return())
 		m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtpd2ps, src, dst))
 
+	case ssa.OpcodeWideningPairwiseDotProductS:
+		x, y := instr.Arg2()
+		m.lowerWideningPairwiseDotProductS(x, y, instr.Return())
+
 	case ssa.OpcodeVIabs:
 		m.lowerVIabs(instr)
 	case ssa.OpcodeVIpopcnt:
diff --git a/internal/engine/wazevo/backend/isa/amd64/machine_vec.go b/internal/engine/wazevo/backend/isa/amd64/machine_vec.go
index 7acb9810..85bdd7e7 100644
--- a/internal/engine/wazevo/backend/isa/amd64/machine_vec.go
+++ b/internal/engine/wazevo/backend/isa/amd64/machine_vec.go
@@ -764,3 +764,11 @@ func (m *machine) lowerNarrow(x, y, ret ssa.Value, lane ssa.VecLane, signed bool
 	m.insert(m.allocateInstr().asXmmRmR(sseOp, yy, xx))
 	m.copyTo(xx, m.c.VRegOf(ret))
 }
+
+func (m *machine) lowerWideningPairwiseDotProductS(x, y, ret ssa.Value) {
+	_xx := m.getOperand_Reg(m.c.ValueDefinition(x))
+	xx := m.copyToTmp(_xx.reg())
+	yy := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
+	m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddwd, yy, xx))
+	m.copyTo(xx, m.c.VRegOf(ret))
+}
diff --git a/internal/engine/wazevo/backend/isa/arm64/instr.go b/internal/engine/wazevo/backend/isa/arm64/instr.go
index dd5a0b77..62deac3f 100644
--- a/internal/engine/wazevo/backend/isa/arm64/instr.go
+++ b/internal/engine/wazevo/backend/isa/arm64/instr.go
@@ -1002,11 +1002,12 @@ func (i *instruction) asVecPermute(op vecOp, rd, rn, rm operand, arr vecArrangem
 	i.u2 = uint64(arr)
 }
 
-func (i *instruction) asVecRRR(op vecOp, rd, rn, rm operand, arr vecArrangement) {
+func (i *instruction) asVecRRR(op vecOp, rd, rn, rm operand, arr vecArrangement) *instruction {
 	i.kind = vecRRR
 	i.u1 = uint64(op)
 	i.rn, i.rd, i.rm = rn, rd, rm
 	i.u2 = uint64(arr)
+	return i
 }
 
 // asVecRRRRewrite encodes a vector instruction that rewrites the destination register.
@@ -2011,6 +2012,10 @@ func (b vecOp) String() string {
 		return "fmin"
 	case vecOpFmax:
 		return "fmax"
+	case vecOpSmull:
+		return "smull"
+	case vecOpSmull2:
+		return "smull2"
 	}
 	panic(int(b))
 }
@@ -2085,6 +2090,8 @@ const (
 	vecOpUshll
 	vecOpSshr
 	vecOpZip1
+	vecOpSmull
+	vecOpSmull2
 )
 
 // bitOp determines the type of bitwise operation. Instructions whose kind is one of
diff --git a/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go b/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
index f2efb5e9..2b593f21 100644
--- a/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
+++ b/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go
@@ -691,6 +691,20 @@ func encodeVecRRR(op vecOp, rd, rn, rm uint32, arr vecArrangement) uint32 {
 		size, q := arrToSizeQEncoded(arr)
 		return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01000, size, 0b1, q)
 
+	case vecOpSmull:
+		if arr > vecArrangement4S {
+			panic("unsupported arrangement: " + arr.String())
+		}
+		size, _ := arrToSizeQEncoded(arr)
+		return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b0)
+
+	case vecOpSmull2:
+		if arr > vecArrangement4S {
+			panic("unsupported arrangement: " + arr.String())
+		}
+		size, _ := arrToSizeQEncoded(arr)
+		return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b1)
+
 	default:
 		panic("TODO: " + op.String())
 	}
diff --git a/internal/engine/wazevo/backend/isa/arm64/lower_instr.go b/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
index 9f902632..c2858f4e 100644
--- a/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
+++ b/internal/engine/wazevo/backend/isa/arm64/lower_instr.go
@@ -701,6 +701,18 @@ func (m *machine) LowerInstr(instr *ssa.Instruction) {
 		}
 		m.insert(dup)
 
+	case ssa.OpcodeWideningPairwiseDotProductS:
+		x, y := instr.Arg2()
+		xx, yy := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone),
+			m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone)
+		tmp, tmp2 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)), operandNR(m.compiler.AllocateVReg(ssa.TypeV128))
+		m.insert(m.allocateInstr().asVecRRR(vecOpSmull, tmp, xx, yy, vecArrangement8H))
+		m.insert(m.allocateInstr().asVecRRR(vecOpSmull2, tmp2, xx, yy, vecArrangement8H))
+		m.insert(m.allocateInstr().asVecRRR(vecOpAddp, tmp, tmp, tmp2, vecArrangement4S))
+
+		rd := operandNR(m.compiler.VRegOf(instr.Return()))
+		m.insert(m.allocateInstr().asFpuMov128(rd.nr(), tmp.nr()))
+
 	case ssa.OpcodeLoadSplat:
 		ptr, offset, lane := instr.LoadSplatData()
 		m.lowerLoadSplat(ptr, offset, lane, instr.Return())
diff --git a/internal/engine/wazevo/frontend/lower.go b/internal/engine/wazevo/frontend/lower.go
index aeea2c7e..e235c868 100644
--- a/internal/engine/wazevo/frontend/lower.go
+++ b/internal/engine/wazevo/frontend/lower.go
@@ -2647,16 +2647,7 @@ func (c *Compiler) lowerCurrentOpcode() {
 			v2 := state.pop()
 			v1 := state.pop()
 
-			// TODO: The sequence `Widen; Widen; VIMul` can be substituted for a single instruction on some ISAs.
-			v1lo := builder.AllocateInstruction().AsWiden(v1, ssa.VecLaneI16x8, true, true).Insert(builder).Return()
-			v2lo := builder.AllocateInstruction().AsWiden(v2, ssa.VecLaneI16x8, true, true).Insert(builder).Return()
-			low := builder.AllocateInstruction().AsVImul(v1lo, v2lo, ssa.VecLaneI32x4).Insert(builder).Return()
-
-			v1hi := builder.AllocateInstruction().AsWiden(v1, ssa.VecLaneI16x8, true, false).Insert(builder).Return()
-			v2hi := builder.AllocateInstruction().AsWiden(v2, ssa.VecLaneI16x8, true, false).Insert(builder).Return()
-			high := builder.AllocateInstruction().AsVImul(v1hi, v2hi, ssa.VecLaneI32x4).Insert(builder).Return()
-
-			ret := builder.AllocateInstruction().AsIaddPairwise(low, high, ssa.VecLaneI32x4).Insert(builder).Return()
+			ret := builder.AllocateInstruction().AsWideningPairwiseDotProductS(v1, v2).Insert(builder).Return()
 			state.push(ret)
 
 		case wasm.OpcodeVecF32x4Eq, wasm.OpcodeVecF64x2Eq:
diff --git a/internal/engine/wazevo/ssa/instructions.go b/internal/engine/wazevo/ssa/instructions.go
index 0fc4288a..1d5117ac 100644
--- a/internal/engine/wazevo/ssa/instructions.go
+++ b/internal/engine/wazevo/ssa/instructions.go
@@ -575,8 +575,8 @@ const (
 	// OpcodeIaddPairwise is a lane-wise integer extended pairwise addition producing extended results (twice wider results than the inputs): `v = iadd_pairwise x, y` on vector.
 	OpcodeIaddPairwise
 
-	// OpcodeWideningPairwiseDotProductS ...
-	// `v = widening_pairwise_dot_product_s x, y`.
+	// OpcodeWideningPairwiseDotProductS is a lane-wise widening pairwise dot product with signed saturation: `v = WideningPairwiseDotProductS x, y` on vector.
+	// Currently, the only lane is i16, and the result is i32.
 	OpcodeWideningPairwiseDotProductS
 
 	// OpcodeUExtend zero-extends the given integer: `v = UExtend x, from->to`.
@@ -716,148 +716,149 @@ const (
 // instructionSideEffects provides the info to determine if an instruction has side effects.
 // Instructions with side effects must not be eliminated regardless whether the result is used or not.
 var instructionSideEffects = [opcodeEnd]sideEffect{
-	OpcodeUndefined:          sideEffectStrict,
-	OpcodeJump:               sideEffectStrict,
-	OpcodeIconst:             sideEffectNone,
-	OpcodeCall:               sideEffectStrict,
-	OpcodeCallIndirect:       sideEffectStrict,
-	OpcodeIadd:               sideEffectNone,
-	OpcodeImul:               sideEffectNone,
-	OpcodeIsub:               sideEffectNone,
-	OpcodeIcmp:               sideEffectNone,
-	OpcodeExtractlane:        sideEffectNone,
-	OpcodeInsertlane:         sideEffectNone,
-	OpcodeBand:               sideEffectNone,
-	OpcodeBor:                sideEffectNone,
-	OpcodeBxor:               sideEffectNone,
-	OpcodeRotl:               sideEffectNone,
-	OpcodeRotr:               sideEffectNone,
-	OpcodeFcmp:               sideEffectNone,
-	OpcodeFadd:               sideEffectNone,
-	OpcodeClz:                sideEffectNone,
-	OpcodeCtz:                sideEffectNone,
-	OpcodePopcnt:             sideEffectNone,
-	OpcodeLoad:               sideEffectNone,
-	OpcodeLoadSplat:          sideEffectNone,
-	OpcodeUload8:             sideEffectNone,
-	OpcodeUload16:            sideEffectNone,
-	OpcodeUload32:            sideEffectNone,
-	OpcodeSload8:             sideEffectNone,
-	OpcodeSload16:            sideEffectNone,
-	OpcodeSload32:            sideEffectNone,
-	OpcodeSExtend:            sideEffectNone,
-	OpcodeUExtend:            sideEffectNone,
-	OpcodeSwidenLow:          sideEffectNone,
-	OpcodeUwidenLow:          sideEffectNone,
-	OpcodeSwidenHigh:         sideEffectNone,
-	OpcodeUwidenHigh:         sideEffectNone,
-	OpcodeSnarrow:            sideEffectNone,
-	OpcodeUnarrow:            sideEffectNone,
-	OpcodeSwizzle:            sideEffectNone,
-	OpcodeShuffle:            sideEffectNone,
-	OpcodeSplat:              sideEffectNone,
-	OpcodeFsub:               sideEffectNone,
-	OpcodeF32const:           sideEffectNone,
-	OpcodeF64const:           sideEffectNone,
-	OpcodeIshl:               sideEffectNone,
-	OpcodeSshr:               sideEffectNone,
-	OpcodeUshr:               sideEffectNone,
-	OpcodeStore:              sideEffectStrict,
-	OpcodeIstore8:            sideEffectStrict,
-	OpcodeIstore16:           sideEffectStrict,
-	OpcodeIstore32:           sideEffectStrict,
-	OpcodeExitWithCode:       sideEffectStrict,
-	OpcodeExitIfTrueWithCode: sideEffectStrict,
-	OpcodeReturn:             sideEffectStrict,
-	OpcodeBrz:                sideEffectStrict,
-	OpcodeBrnz:               sideEffectStrict,
-	OpcodeBrTable:            sideEffectStrict,
-	OpcodeFdiv:               sideEffectNone,
-	OpcodeFmul:               sideEffectNone,
-	OpcodeFmax:               sideEffectNone,
-	OpcodeSqmulRoundSat:      sideEffectNone,
-	OpcodeSelect:             sideEffectNone,
-	OpcodeFmin:               sideEffectNone,
-	OpcodeFneg:               sideEffectNone,
-	OpcodeFcvtToSint:         sideEffectTraps,
-	OpcodeFcvtToUint:         sideEffectTraps,
-	OpcodeFcvtFromSint:       sideEffectNone,
-	OpcodeFcvtFromUint:       sideEffectNone,
-	OpcodeFcvtToSintSat:      sideEffectNone,
-	OpcodeFcvtToUintSat:      sideEffectNone,
-	OpcodeVFcvtFromUint:      sideEffectNone,
-	OpcodeVFcvtFromSint:      sideEffectNone,
-	OpcodeFdemote:            sideEffectNone,
-	OpcodeFvpromoteLow:       sideEffectNone,
-	OpcodeFvdemote:           sideEffectNone,
-	OpcodeFpromote:           sideEffectNone,
-	OpcodeBitcast:            sideEffectNone,
-	OpcodeIreduce:            sideEffectNone,
-	OpcodeSqrt:               sideEffectNone,
-	OpcodeCeil:               sideEffectNone,
-	OpcodeFloor:              sideEffectNone,
-	OpcodeTrunc:              sideEffectNone,
-	OpcodeNearest:            sideEffectNone,
-	OpcodeSdiv:               sideEffectTraps,
-	OpcodeSrem:               sideEffectTraps,
-	OpcodeUdiv:               sideEffectTraps,
-	OpcodeUrem:               sideEffectTraps,
-	OpcodeFabs:               sideEffectNone,
-	OpcodeFcopysign:          sideEffectNone,
-	OpcodeIaddPairwise:       sideEffectNone,
-	OpcodeVconst:             sideEffectNone,
-	OpcodeVbor:               sideEffectNone,
-	OpcodeVbxor:              sideEffectNone,
-	OpcodeVband:              sideEffectNone,
-	OpcodeVbandnot:           sideEffectNone,
-	OpcodeVbnot:              sideEffectNone,
-	OpcodeVbitselect:         sideEffectNone,
-	OpcodeVanyTrue:           sideEffectNone,
-	OpcodeVallTrue:           sideEffectNone,
-	OpcodeVhighBits:          sideEffectNone,
-	OpcodeVIadd:              sideEffectNone,
-	OpcodeVSaddSat:           sideEffectNone,
-	OpcodeVUaddSat:           sideEffectNone,
-	OpcodeVIsub:              sideEffectNone,
-	OpcodeVSsubSat:           sideEffectNone,
-	OpcodeVUsubSat:           sideEffectNone,
-	OpcodeVIcmp:              sideEffectNone,
-	OpcodeVImin:              sideEffectNone,
-	OpcodeVUmin:              sideEffectNone,
-	OpcodeVImax:              sideEffectNone,
-	OpcodeVUmax:              sideEffectNone,
-	OpcodeVAvgRound:          sideEffectNone,
-	OpcodeVImul:              sideEffectNone,
-	OpcodeVIabs:              sideEffectNone,
-	OpcodeVIneg:              sideEffectNone,
-	OpcodeVIpopcnt:           sideEffectNone,
-	OpcodeVIshl:              sideEffectNone,
-	OpcodeVSshr:              sideEffectNone,
-	OpcodeVUshr:              sideEffectNone,
-	OpcodeVSqrt:              sideEffectNone,
-	OpcodeVFabs:              sideEffectNone,
-	OpcodeVFmin:              sideEffectNone,
-	OpcodeVFmax:              sideEffectNone,
-	OpcodeVFneg:              sideEffectNone,
-	OpcodeVFadd:              sideEffectNone,
-	OpcodeVFsub:              sideEffectNone,
-	OpcodeVFmul:              sideEffectNone,
-	OpcodeVFdiv:              sideEffectNone,
-	OpcodeVFcmp:              sideEffectNone,
-	OpcodeVCeil:              sideEffectNone,
-	OpcodeVFloor:             sideEffectNone,
-	OpcodeVTrunc:             sideEffectNone,
-	OpcodeVNearest:           sideEffectNone,
-	OpcodeVMaxPseudo:         sideEffectNone,
-	OpcodeVMinPseudo:         sideEffectNone,
-	OpcodeVFcvtToUintSat:     sideEffectNone,
-	OpcodeVFcvtToSintSat:     sideEffectNone,
-	OpcodeVZeroExtLoad:       sideEffectNone,
-	OpcodeAtomicRmw:          sideEffectStrict,
-	OpcodeAtomicLoad:         sideEffectStrict,
-	OpcodeAtomicStore:        sideEffectStrict,
-	OpcodeAtomicCas:          sideEffectStrict,
-	OpcodeFence:              sideEffectStrict,
+	OpcodeUndefined:                   sideEffectStrict,
+	OpcodeJump:                        sideEffectStrict,
+	OpcodeIconst:                      sideEffectNone,
+	OpcodeCall:                        sideEffectStrict,
+	OpcodeCallIndirect:                sideEffectStrict,
+	OpcodeIadd:                        sideEffectNone,
+	OpcodeImul:                        sideEffectNone,
+	OpcodeIsub:                        sideEffectNone,
+	OpcodeIcmp:                        sideEffectNone,
+	OpcodeExtractlane:                 sideEffectNone,
+	OpcodeInsertlane:                  sideEffectNone,
+	OpcodeBand:                        sideEffectNone,
+	OpcodeBor:                         sideEffectNone,
+	OpcodeBxor:                        sideEffectNone,
+	OpcodeRotl:                        sideEffectNone,
+	OpcodeRotr:                        sideEffectNone,
+	OpcodeFcmp:                        sideEffectNone,
+	OpcodeFadd:                        sideEffectNone,
+	OpcodeClz:                         sideEffectNone,
+	OpcodeCtz:                         sideEffectNone,
+	OpcodePopcnt:                      sideEffectNone,
+	OpcodeLoad:                        sideEffectNone,
+	OpcodeLoadSplat:                   sideEffectNone,
+	OpcodeUload8:                      sideEffectNone,
+	OpcodeUload16:                     sideEffectNone,
+	OpcodeUload32:                     sideEffectNone,
+	OpcodeSload8:                      sideEffectNone,
+	OpcodeSload16:                     sideEffectNone,
+	OpcodeSload32:                     sideEffectNone,
+	OpcodeSExtend:                     sideEffectNone,
+	OpcodeUExtend:                     sideEffectNone,
+	OpcodeSwidenLow:                   sideEffectNone,
+	OpcodeUwidenLow:                   sideEffectNone,
+	OpcodeSwidenHigh:                  sideEffectNone,
+	OpcodeUwidenHigh:                  sideEffectNone,
+	OpcodeSnarrow:                     sideEffectNone,
+	OpcodeUnarrow:                     sideEffectNone,
+	OpcodeSwizzle:                     sideEffectNone,
+	OpcodeShuffle:                     sideEffectNone,
+	OpcodeSplat:                       sideEffectNone,
+	OpcodeFsub:                        sideEffectNone,
+	OpcodeF32const:                    sideEffectNone,
+	OpcodeF64const:                    sideEffectNone,
+	OpcodeIshl:                        sideEffectNone,
+	OpcodeSshr:                        sideEffectNone,
+	OpcodeUshr:                        sideEffectNone,
+	OpcodeStore:                       sideEffectStrict,
+	OpcodeIstore8:                     sideEffectStrict,
+	OpcodeIstore16:                    sideEffectStrict,
+	OpcodeIstore32:                    sideEffectStrict,
+	OpcodeExitWithCode:                sideEffectStrict,
+	OpcodeExitIfTrueWithCode:          sideEffectStrict,
+	OpcodeReturn:                      sideEffectStrict,
+	OpcodeBrz:                         sideEffectStrict,
+	OpcodeBrnz:                        sideEffectStrict,
+	OpcodeBrTable:                     sideEffectStrict,
+	OpcodeFdiv:                        sideEffectNone,
+	OpcodeFmul:                        sideEffectNone,
+	OpcodeFmax:                        sideEffectNone,
+	OpcodeSqmulRoundSat:               sideEffectNone,
+	OpcodeSelect:                      sideEffectNone,
+	OpcodeFmin:                        sideEffectNone,
+	OpcodeFneg:                        sideEffectNone,
+	OpcodeFcvtToSint:                  sideEffectTraps,
+	OpcodeFcvtToUint:                  sideEffectTraps,
+	OpcodeFcvtFromSint:                sideEffectNone,
+	OpcodeFcvtFromUint:                sideEffectNone,
+	OpcodeFcvtToSintSat:               sideEffectNone,
+	OpcodeFcvtToUintSat:               sideEffectNone,
+	OpcodeVFcvtFromUint:               sideEffectNone,
+	OpcodeVFcvtFromSint:               sideEffectNone,
+	OpcodeFdemote:                     sideEffectNone,
+	OpcodeFvpromoteLow:                sideEffectNone,
+	OpcodeFvdemote:                    sideEffectNone,
+	OpcodeFpromote:                    sideEffectNone,
+	OpcodeBitcast:                     sideEffectNone,
+	OpcodeIreduce:                     sideEffectNone,
+	OpcodeSqrt:                        sideEffectNone,
+	OpcodeCeil:                        sideEffectNone,
+	OpcodeFloor:                       sideEffectNone,
+	OpcodeTrunc:                       sideEffectNone,
+	OpcodeNearest:                     sideEffectNone,
+	OpcodeSdiv:                        sideEffectTraps,
+	OpcodeSrem:                        sideEffectTraps,
+	OpcodeUdiv:                        sideEffectTraps,
+	OpcodeUrem:                        sideEffectTraps,
+	OpcodeFabs:                        sideEffectNone,
+	OpcodeFcopysign:                   sideEffectNone,
+	OpcodeIaddPairwise:                sideEffectNone,
+	OpcodeVconst:                      sideEffectNone,
+	OpcodeVbor:                        sideEffectNone,
+	OpcodeVbxor:                       sideEffectNone,
+	OpcodeVband:                       sideEffectNone,
+	OpcodeVbandnot:                    sideEffectNone,
+	OpcodeVbnot:                       sideEffectNone,
+	OpcodeVbitselect:                  sideEffectNone,
+	OpcodeVanyTrue:                    sideEffectNone,
+	OpcodeVallTrue:                    sideEffectNone,
+	OpcodeVhighBits:                   sideEffectNone,
+	OpcodeVIadd:                       sideEffectNone,
+	OpcodeVSaddSat:                    sideEffectNone,
+	OpcodeVUaddSat:                    sideEffectNone,
+	OpcodeVIsub:                       sideEffectNone,
+	OpcodeVSsubSat:                    sideEffectNone,
+	OpcodeVUsubSat:                    sideEffectNone,
+	OpcodeVIcmp:                       sideEffectNone,
+	OpcodeVImin:                       sideEffectNone,
+	OpcodeVUmin:                       sideEffectNone,
+	OpcodeVImax:                       sideEffectNone,
+	OpcodeVUmax:                       sideEffectNone,
+	OpcodeVAvgRound:                   sideEffectNone,
+	OpcodeVImul:                       sideEffectNone,
+	OpcodeVIabs:                       sideEffectNone,
+	OpcodeVIneg:                       sideEffectNone,
+	OpcodeVIpopcnt:                    sideEffectNone,
+	OpcodeVIshl:                       sideEffectNone,
+	OpcodeVSshr:                       sideEffectNone,
+	OpcodeVUshr:                       sideEffectNone,
+	OpcodeVSqrt:                       sideEffectNone,
+	OpcodeVFabs:                       sideEffectNone,
+	OpcodeVFmin:                       sideEffectNone,
+	OpcodeVFmax:                       sideEffectNone,
+	OpcodeVFneg:                       sideEffectNone,
+	OpcodeVFadd:                       sideEffectNone,
+	OpcodeVFsub:                       sideEffectNone,
+	OpcodeVFmul:                       sideEffectNone,
+	OpcodeVFdiv:                       sideEffectNone,
+	OpcodeVFcmp:                       sideEffectNone,
+	OpcodeVCeil:                       sideEffectNone,
+	OpcodeVFloor:                      sideEffectNone,
+	OpcodeVTrunc:                      sideEffectNone,
+	OpcodeVNearest:                    sideEffectNone,
+	OpcodeVMaxPseudo:                  sideEffectNone,
+	OpcodeVMinPseudo:                  sideEffectNone,
+	OpcodeVFcvtToUintSat:              sideEffectNone,
+	OpcodeVFcvtToSintSat:              sideEffectNone,
+	OpcodeVZeroExtLoad:                sideEffectNone,
+	OpcodeAtomicRmw:                   sideEffectStrict,
+	OpcodeAtomicLoad:                  sideEffectStrict,
+	OpcodeAtomicStore:                 sideEffectStrict,
+	OpcodeAtomicCas:                   sideEffectStrict,
+	OpcodeFence:                       sideEffectStrict,
+	OpcodeWideningPairwiseDotProductS: sideEffectNone,
 }
 
 // sideEffect returns true if this instruction has side effects.
@@ -970,79 +971,80 @@ var instructionReturnTypes = [opcodeEnd]returnTypesFn{
 		}
 		return
 	},
-	OpcodeLoad:               returnTypesFnSingle,
-	OpcodeVZeroExtLoad:       returnTypesFnV128,
-	OpcodeLoadSplat:          returnTypesFnV128,
-	OpcodeIadd:               returnTypesFnSingle,
-	OpcodeIsub:               returnTypesFnSingle,
-	OpcodeImul:               returnTypesFnSingle,
-	OpcodeIcmp:               returnTypesFnI32,
-	OpcodeFcmp:               returnTypesFnI32,
-	OpcodeFadd:               returnTypesFnSingle,
-	OpcodeFsub:               returnTypesFnSingle,
-	OpcodeFdiv:               returnTypesFnSingle,
-	OpcodeFmul:               returnTypesFnSingle,
-	OpcodeFmax:               returnTypesFnSingle,
-	OpcodeFmin:               returnTypesFnSingle,
-	OpcodeSqmulRoundSat:      returnTypesFnV128,
-	OpcodeF32const:           returnTypesFnF32,
-	OpcodeF64const:           returnTypesFnF64,
-	OpcodeClz:                returnTypesFnSingle,
-	OpcodeCtz:                returnTypesFnSingle,
-	OpcodePopcnt:             returnTypesFnSingle,
-	OpcodeStore:              returnTypesFnNoReturns,
-	OpcodeIstore8:            returnTypesFnNoReturns,
-	OpcodeIstore16:           returnTypesFnNoReturns,
-	OpcodeIstore32:           returnTypesFnNoReturns,
-	OpcodeExitWithCode:       returnTypesFnNoReturns,
-	OpcodeExitIfTrueWithCode: returnTypesFnNoReturns,
-	OpcodeReturn:             returnTypesFnNoReturns,
-	OpcodeBrz:                returnTypesFnNoReturns,
-	OpcodeBrnz:               returnTypesFnNoReturns,
-	OpcodeBrTable:            returnTypesFnNoReturns,
-	OpcodeUload8:             returnTypesFnSingle,
-	OpcodeUload16:            returnTypesFnSingle,
-	OpcodeUload32:            returnTypesFnSingle,
-	OpcodeSload8:             returnTypesFnSingle,
-	OpcodeSload16:            returnTypesFnSingle,
-	OpcodeSload32:            returnTypesFnSingle,
-	OpcodeFcvtToSint:         returnTypesFnSingle,
-	OpcodeFcvtToUint:         returnTypesFnSingle,
-	OpcodeFcvtFromSint:       returnTypesFnSingle,
-	OpcodeFcvtFromUint:       returnTypesFnSingle,
-	OpcodeFcvtToSintSat:      returnTypesFnSingle,
-	OpcodeFcvtToUintSat:      returnTypesFnSingle,
-	OpcodeVFcvtFromUint:      returnTypesFnV128,
-	OpcodeVFcvtFromSint:      returnTypesFnV128,
-	OpcodeFneg:               returnTypesFnSingle,
-	OpcodeFdemote:            returnTypesFnF32,
-	OpcodeFvdemote:           returnTypesFnV128,
-	OpcodeFvpromoteLow:       returnTypesFnV128,
-	OpcodeFpromote:           returnTypesFnF64,
-	OpcodeVconst:             returnTypesFnV128,
-	OpcodeVFabs:              returnTypesFnV128,
-	OpcodeVSqrt:              returnTypesFnV128,
-	OpcodeVFmax:              returnTypesFnV128,
-	OpcodeVFmin:              returnTypesFnV128,
-	OpcodeVFneg:              returnTypesFnV128,
-	OpcodeVFadd:              returnTypesFnV128,
-	OpcodeVFsub:              returnTypesFnV128,
-	OpcodeVFmul:              returnTypesFnV128,
-	OpcodeVFdiv:              returnTypesFnV128,
-	OpcodeVFcmp:              returnTypesFnV128,
-	OpcodeVCeil:              returnTypesFnV128,
-	OpcodeVFloor:             returnTypesFnV128,
-	OpcodeVTrunc:             returnTypesFnV128,
-	OpcodeVNearest:           returnTypesFnV128,
-	OpcodeVMaxPseudo:         returnTypesFnV128,
-	OpcodeVMinPseudo:         returnTypesFnV128,
-	OpcodeVFcvtToUintSat:     returnTypesFnV128,
-	OpcodeVFcvtToSintSat:     returnTypesFnV128,
-	OpcodeAtomicRmw:          returnTypesFnSingle,
-	OpcodeAtomicLoad:         returnTypesFnSingle,
-	OpcodeAtomicStore:        returnTypesFnNoReturns,
-	OpcodeAtomicCas:          returnTypesFnSingle,
-	OpcodeFence:              returnTypesFnNoReturns,
+	OpcodeLoad:                        returnTypesFnSingle,
+	OpcodeVZeroExtLoad:                returnTypesFnV128,
+	OpcodeLoadSplat:                   returnTypesFnV128,
+	OpcodeIadd:                        returnTypesFnSingle,
+	OpcodeIsub:                        returnTypesFnSingle,
+	OpcodeImul:                        returnTypesFnSingle,
+	OpcodeIcmp:                        returnTypesFnI32,
+	OpcodeFcmp:                        returnTypesFnI32,
+	OpcodeFadd:                        returnTypesFnSingle,
+	OpcodeFsub:                        returnTypesFnSingle,
+	OpcodeFdiv:                        returnTypesFnSingle,
+	OpcodeFmul:                        returnTypesFnSingle,
+	OpcodeFmax:                        returnTypesFnSingle,
+	OpcodeFmin:                        returnTypesFnSingle,
+	OpcodeSqmulRoundSat:               returnTypesFnV128,
+	OpcodeF32const:                    returnTypesFnF32,
+	OpcodeF64const:                    returnTypesFnF64,
+	OpcodeClz:                         returnTypesFnSingle,
+	OpcodeCtz:                         returnTypesFnSingle,
+	OpcodePopcnt:                      returnTypesFnSingle,
+	OpcodeStore:                       returnTypesFnNoReturns,
+	OpcodeIstore8:                     returnTypesFnNoReturns,
+	OpcodeIstore16:                    returnTypesFnNoReturns,
+	OpcodeIstore32:                    returnTypesFnNoReturns,
+	OpcodeExitWithCode:                returnTypesFnNoReturns,
+	OpcodeExitIfTrueWithCode:          returnTypesFnNoReturns,
+	OpcodeReturn:                      returnTypesFnNoReturns,
+	OpcodeBrz:                         returnTypesFnNoReturns,
+	OpcodeBrnz:                        returnTypesFnNoReturns,
+	OpcodeBrTable:                     returnTypesFnNoReturns,
+	OpcodeUload8:                      returnTypesFnSingle,
+	OpcodeUload16:                     returnTypesFnSingle,
+	OpcodeUload32:                     returnTypesFnSingle,
+	OpcodeSload8:                      returnTypesFnSingle,
+	OpcodeSload16:                     returnTypesFnSingle,
+	OpcodeSload32:                     returnTypesFnSingle,
+	OpcodeFcvtToSint:                  returnTypesFnSingle,
+	OpcodeFcvtToUint:                  returnTypesFnSingle,
+	OpcodeFcvtFromSint:                returnTypesFnSingle,
+	OpcodeFcvtFromUint:                returnTypesFnSingle,
+	OpcodeFcvtToSintSat:               returnTypesFnSingle,
+	OpcodeFcvtToUintSat:               returnTypesFnSingle,
+	OpcodeVFcvtFromUint:               returnTypesFnV128,
+	OpcodeVFcvtFromSint:               returnTypesFnV128,
+	OpcodeFneg:                        returnTypesFnSingle,
+	OpcodeFdemote:                     returnTypesFnF32,
+	OpcodeFvdemote:                    returnTypesFnV128,
+	OpcodeFvpromoteLow:                returnTypesFnV128,
+	OpcodeFpromote:                    returnTypesFnF64,
+	OpcodeVconst:                      returnTypesFnV128,
+	OpcodeVFabs:                       returnTypesFnV128,
+	OpcodeVSqrt:                       returnTypesFnV128,
+	OpcodeVFmax:                       returnTypesFnV128,
+	OpcodeVFmin:                       returnTypesFnV128,
+	OpcodeVFneg:                       returnTypesFnV128,
+	OpcodeVFadd:                       returnTypesFnV128,
+	OpcodeVFsub:                       returnTypesFnV128,
+	OpcodeVFmul:                       returnTypesFnV128,
+	OpcodeVFdiv:                       returnTypesFnV128,
+	OpcodeVFcmp:                       returnTypesFnV128,
+	OpcodeVCeil:                       returnTypesFnV128,
+	OpcodeVFloor:                      returnTypesFnV128,
+	OpcodeVTrunc:                      returnTypesFnV128,
+	OpcodeVNearest:                    returnTypesFnV128,
+	OpcodeVMaxPseudo:                  returnTypesFnV128,
+	OpcodeVMinPseudo:                  returnTypesFnV128,
+	OpcodeVFcvtToUintSat:              returnTypesFnV128,
+	OpcodeVFcvtToSintSat:              returnTypesFnV128,
+	OpcodeAtomicRmw:                   returnTypesFnSingle,
+	OpcodeAtomicLoad:                  returnTypesFnSingle,
+	OpcodeAtomicStore:                 returnTypesFnNoReturns,
+	OpcodeAtomicCas:                   returnTypesFnSingle,
+	OpcodeFence:                       returnTypesFnNoReturns,
+	OpcodeWideningPairwiseDotProductS: returnTypesFnV128,
 }
 
 // AsLoad initializes this instruction as a store instruction with OpcodeLoad.
@@ -1165,6 +1167,16 @@ func (i *Instruction) AsVIadd(x, y Value, lane VecLane) *Instruction {
 	return i
 }
 
+// AsWideningPairwiseDotProductS initializes this instruction as a lane-wise integer extended pairwise addition instruction
+// with OpcodeIaddPairwise on a vector.
+func (i *Instruction) AsWideningPairwiseDotProductS(x, y Value) *Instruction {
+	i.opcode = OpcodeWideningPairwiseDotProductS
+	i.v = x
+	i.v2 = y
+	i.typ = TypeV128
+	return i
+}
+
 // AsIaddPairwise initializes this instruction as a lane-wise integer extended pairwise addition instruction
 // with OpcodeIaddPairwise on a vector.
 func (i *Instruction) AsIaddPairwise(x, y Value, lane VecLane) *Instruction {
@@ -2601,6 +2613,8 @@ func (i *Instruction) Format(b Builder) string {
 		instSuffix = fmt.Sprintf("_%d, %s, %s, %s", 8*i.u1, i.v.Format(b), i.v2.Format(b), i.v3.Format(b))
 	case OpcodeFence:
 		instSuffix = fmt.Sprintf(" %d", i.u1)
+	case OpcodeWideningPairwiseDotProductS:
+		instSuffix = fmt.Sprintf(" %s, %s", i.v.Format(b), i.v2.Format(b))
 	default:
 		panic(fmt.Sprintf("TODO: format for %s", i.opcode))
 	}


Test patch:
diff --git a/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go b/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go
index df07085c..5b701a6c 100644
--- a/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go
+++ b/internal/engine/wazevo/backend/isa/arm64/instr_encoding_test.go
@@ -1494,6 +1494,24 @@ func TestInstruction_encode(t *testing.T) {
 		{want: "4138b06e", setup: func(i *instruction) {
 			i.asVecLanes(vecOpUaddlv, operandNR(v1VReg), operandNR(v2VReg), vecArrangement4S)
 		}},
+		{want: "41c0230e", setup: func(i *instruction) {
+			i.asVecRRR(vecOpSmull, operandNR(v1VReg), operandNR(v2VReg), operandNR(v3VReg), vecArrangement8B)
+		}},
+		{want: "41c0630e", setup: func(i *instruction) {
+			i.asVecRRR(vecOpSmull, operandNR(v1VReg), operandNR(v2VReg), operandNR(v3VReg), vecArrangement8H)
+		}},
+		{want: "41c0a30e", setup: func(i *instruction) {
+			i.asVecRRR(vecOpSmull, operandNR(v1VReg), operandNR(v2VReg), operandNR(v3VReg), vecArrangement4S)
+		}},
+		{want: "41c0234e", setup: func(i *instruction) {
+			i.asVecRRR(vecOpSmull2, operandNR(v1VReg), operandNR(v2VReg), operandNR(v3VReg), vecArrangement8B)
+		}},
+		{want: "41c0634e", setup: func(i *instruction) {
+			i.asVecRRR(vecOpSmull2, operandNR(v1VReg), operandNR(v2VReg), operandNR(v3VReg), vecArrangement8H)
+		}},
+		{want: "41c0a34e", setup: func(i *instruction) {
+			i.asVecRRR(vecOpSmull2, operandNR(v1VReg), operandNR(v2VReg), operandNR(v3VReg), vecArrangement4S)
+		}},
 		{want: "411c630e", setup: func(i *instruction) {
 			i.asVecRRR(vecOpBic, operandNR(v1VReg), operandNR(v2VReg), operandNR(v3VReg), vecArrangement8B)
 		}},