Repository: tetratelabs/wazero
Pull request #1911 (2024-01-11 17:00:51)

wazevo(amd64): load, store move encodings


Patch:
diff --git a/internal/engine/wazevo/backend/isa/amd64/ext.go b/internal/engine/wazevo/backend/isa/amd64/ext.go
new file mode 100644
index 00000000..5e731e82
--- /dev/null
+++ b/internal/engine/wazevo/backend/isa/amd64/ext.go
@@ -0,0 +1,35 @@
+package amd64
+
+// extMode represents the mode of extension in movzx/movsx.
+type extMode byte
+
+const (
+	// extModeBL represents Byte -> Longword.
+	extModeBL extMode = iota
+	// extModeBQ represents Byte -> Quadword.
+	extModeBQ
+	// extModeWL represents Word -> Longword.
+	extModeWL
+	// extModeWQ represents Word -> Quadword.
+	extModeWQ
+	// extModeLQ represents Longword -> Quadword.
+	extModeLQ
+)
+
+// String implements fmt.Stringer.
+func (e extMode) String() string {
+	switch e {
+	case extModeBL:
+		return "bl"
+	case extModeBQ:
+		return "bq"
+	case extModeWL:
+		return "wl"
+	case extModeWQ:
+		return "wq"
+	case extModeLQ:
+		return "lq"
+	default:
+		panic("BUG: invalid ext mode")
+	}
+}
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr.go b/internal/engine/wazevo/backend/isa/amd64/instr.go
index fb280562..aff8803c 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr.go
@@ -9,13 +9,13 @@ import (
 )
 
 type instruction struct {
-	kind                instructionKind
 	prev, next          *instruction
 	abi                 *backend.FunctionABI
 	op1, op2            operand
 	u1, u2              uint64
 	b1                  bool
 	addedBeforeRegAlloc bool
+	kind                instructionKind
 }
 
 // Next implements regalloc.Instr.
@@ -82,15 +82,26 @@ func (i *instruction) String() string {
 	case signExtendData:
 		panic("TODO")
 	case movzxRmR:
-		panic("TODO")
+		return fmt.Sprintf("movzx.%s %s, %s", extMode(i.u1), i.op1.format(true), i.op2.format(true))
 	case mov64MR:
-		panic("TODO")
+		return fmt.Sprintf("movq %s, %s", i.op1.format(true), i.op2.format(true))
 	case lea:
 		return fmt.Sprintf("lea %s, %s", i.op1.format(true), i.op2.format(true))
 	case movsxRmR:
-		panic("TODO")
+		return fmt.Sprintf("movsx.%s %s, %s", extMode(i.u1), i.op1.format(true), i.op2.format(true))
 	case movRM:
-		panic("TODO")
+		var suffix string
+		switch i.u1 {
+		case 1:
+			suffix = "b"
+		case 2:
+			suffix = "w"
+		case 4:
+			suffix = "l"
+		case 8:
+			suffix = "q"
+		}
+		return fmt.Sprintf("mov.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true))
 	case shiftR:
 		panic("TODO")
 	case xmmRmiReg:
@@ -241,12 +252,12 @@ const (
 	// GPR to GPR move: mov (64 32) reg reg.
 	movRR
 
-	// Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr reg.
+	// movzxRmR is zero-extended loads or move (R to R), except for 64 bits: movz (bl bq wl wq lq) addr reg.
 	// Note that the lq variant doesn't really exist since the default zero-extend rule makes it
 	// unnecessary. For that case we emit the equivalent "movl AM, reg32".
 	movzxRmR
 
-	// A plain 64-bit integer load, since MovZX_RM_R can't represent that.
+	// mov64MR is a plain 64-bit integer load, since movzxRmR can't represent that.
 	mov64MR
 
 	// Loads the memory address of addr into dst.
@@ -561,6 +572,49 @@ func (i *instruction) asGprToXmm(op sseOpcode, rm operand, rd regalloc.VReg, _64
 	return i
 }
 
+func (i *instruction) movRM(rm regalloc.VReg, rd operand, size byte) *instruction {
+	if rd.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = movRM
+	i.op1 = newOperandReg(rm)
+	i.op2 = rd
+	i.u1 = uint64(size)
+	return i
+}
+
+func (i *instruction) asMovsxRmR(ext extMode, src operand, rd regalloc.VReg) *instruction {
+	if src.kind != operandKindReg && src.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = movsxRmR
+	i.op1 = src
+	i.op2 = newOperandReg(rd)
+	i.u1 = uint64(ext)
+	return i
+}
+
+func (i *instruction) asMovzxRmR(ext extMode, src operand, rd regalloc.VReg) *instruction {
+	if src.kind != operandKindReg && src.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = movzxRmR
+	i.op1 = src
+	i.op2 = newOperandReg(rd)
+	i.u1 = uint64(ext)
+	return i
+}
+
+func (i *instruction) asMov64MR(rm operand, rd regalloc.VReg) *instruction {
+	if rm.kind != operandKindMem {
+		panic("BUG")
+	}
+	i.kind = mov64MR
+	i.op1 = rm
+	i.op2 = newOperandReg(rd)
+	return i
+}
+
 func (i *instruction) asMovRR(rm, rd regalloc.VReg, _64 bool) *instruction {
 	i.kind = movRR
 	i.op1 = newOperandReg(rm)
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go b/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
index 648fe573..6959fba4 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go
@@ -476,19 +476,98 @@ func (i *instruction) encode(c backend.Compiler) {
 		panic("TODO")
 	case signExtendData:
 		panic("TODO")
-	case movzxRmR:
-		panic("TODO")
+	case movzxRmR, movsxRmR:
+		signed := i.kind == movsxRmR
+
+		ext := extMode(i.u1)
+		var opcode uint32
+		var opcodeNum uint32
+		var rex rexInfo
+		switch ext {
+		case extModeBL:
+			if signed {
+				opcode, opcodeNum, rex = 0x0fbe, 2, rex.clearW()
+			} else {
+				opcode, opcodeNum, rex = 0x0fb6, 2, rex.clearW()
+			}
+		case extModeBQ:
+			if signed {
+				opcode, opcodeNum, rex = 0x0fbe, 2, rex.setW()
+			} else {
+				opcode, opcodeNum, rex = 0x0fb6, 2, rex.setW()
+			}
+		case extModeWL:
+			if signed {
+				opcode, opcodeNum, rex = 0x0fbf, 2, rex.clearW()
+			} else {
+				opcode, opcodeNum, rex = 0x0fb7, 2, rex.clearW()
+			}
+		case extModeWQ:
+			if signed {
+				opcode, opcodeNum, rex = 0x0fbf, 2, rex.setW()
+			} else {
+				opcode, opcodeNum, rex = 0x0fb7, 2, rex.setW()
+			}
+		case extModeLQ:
+			if signed {
+				opcode, opcodeNum, rex = 0x63, 1, rex.setW()
+			} else {
+				opcode, opcodeNum, rex = 0x8b, 1, rex.clearW()
+			}
+		default:
+			panic("BUG: invalid extMode")
+		}
+
+		op := i.op1
+		dst := regEncodings[i.op2.r.RealReg()]
+		switch op.kind {
+		case operandKindReg:
+			src := regEncodings[op.r.RealReg()]
+			if ext == extModeBL || ext == extModeBQ {
+				// Some destinations must be encoded with REX.R = 1.
+				if e := src.encoding(); e >= 4 && e <= 7 {
+					rex = rex.always()
+				}
+			}
+			encodeRegReg(c, legacyPrefixesNone, opcode, opcodeNum, dst, src, rex)
+		case operandKindMem:
+			m := op.amode
+			encodeRegMem(c, legacyPrefixesNone, opcode, opcodeNum, dst, m, rex)
+		default:
+			panic("BUG: invalid operand kind")
+		}
+
 	case mov64MR:
-		panic("TODO")
+		m := i.op1.amode
+		dst := regEncodings[i.op2.r.RealReg()]
+		encodeRegMem(c, legacyPrefixesNone, 0x8b, 1, dst, m, rexInfo(0).setW())
+
 	case lea:
 		a := i.op1.amode
 		dst := regEncodings[i.op2.r.RealReg()]
 		encodeRegMem(c, legacyPrefixesNone, 0x8d, 1, dst, a, rexInfo(0).setW())
 
-	case movsxRmR:
-		panic("TODO")
 	case movRM:
-		panic("TODO")
+		m := i.op2.amode
+		src := regEncodings[i.op1.r.RealReg()]
+
+		var rex rexInfo
+		switch i.u1 {
+		case 1:
+			if e := src.encoding(); e >= 4 && e <= 7 {
+				rex = rex.always()
+			}
+			encodeRegMem(c, legacyPrefixesNone, 0x88, 1, src, m, rex.clearW())
+		case 2:
+			encodeRegMem(c, legacyPrefixes0x66, 0x89, 1, src, m, rex.clearW())
+		case 4:
+			encodeRegMem(c, legacyPrefixesNone, 0x89, 1, src, m, rex.clearW())
+		case 8:
+			encodeRegMem(c, legacyPrefixesNone, 0x89, 1, src, m, rex.setW())
+		default:
+			panic("BUG: invalid size")
+		}
+
 	case shiftR:
 		panic("TODO")
 	case xmmRmiReg:
@@ -792,7 +871,7 @@ func (ri rexInfo) clearW() rexInfo {
 	return ri & 0x02
 }
 
-func (ri rexInfo) always() rexInfo { //nolint
+func (ri rexInfo) always() rexInfo {
 	return ri | 0x02
 }
 


Test patch:
diff --git a/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go b/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
index f3513e19..c1e560ba 100644
--- a/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
+++ b/internal/engine/wazevo/backend/isa/amd64/instr_encoding_test.go
@@ -1499,6 +1499,210 @@ func TestInstruction_format_encode(t *testing.T) {
 			want:       "0f8b04000000",
 			wantFormat: "jnp $4",
 		},
+		{
+			setup: func(i *instruction) {
+				i.asMov64MR(newOperandMem(
+					newAmodeImmReg(0, raxVReg),
+				), r15VReg)
+			},
+			want:       "4c8b38",
+			wantFormat: "movq (%rax), %r15",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asMov64MR(newOperandMem(
+					newAmodeImmReg(0, r12VReg),
+				), r15VReg)
+			},
+			want:       "4d8b3c24",
+			wantFormat: "movq (%r12), %r15",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asMov64MR(newOperandMem(
+					newAmodeImmReg(1, r12VReg),
+				), r15VReg)
+			},
+			want:       "4d8b7c2401",
+			wantFormat: "movq 1(%r12), %r15",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asMov64MR(newOperandMem(
+					newAmodeImmReg(1<<20, r12VReg),
+				), r15VReg)
+			},
+			want:       "4d8bbc2400001000",
+			wantFormat: "movq 1048576(%r12), %r15",
+		},
+		{
+			setup: func(i *instruction) {
+				i.asMov64MR(newOperandMem(
+					newAmodeImmReg(1<<20, raxVReg),
+				), r15VReg)
+			},
+			want:       "4c8bb800001000",
+			wantFormat: "movq 1048576(%rax), %r15",
+		},
+		//
+		{
+			setup:      func(i *instruction) { i.asMovsxRmR(extModeBL, newOperandReg(raxVReg), rdiVReg) },
+			want:       "0fbef8",
+			wantFormat: "movsx.bl %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovsxRmR(extModeBL, newOperandReg(rbxVReg), rdiVReg) },
+			want:       "0fbefb",
+			wantFormat: "movsx.bl %rbx, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovsxRmR(extModeBQ, newOperandReg(raxVReg), rdiVReg) },
+			want:       "480fbef8",
+			wantFormat: "movsx.bq %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovsxRmR(extModeBQ, newOperandReg(r15VReg), rdiVReg) },
+			want:       "490fbeff",
+			wantFormat: "movsx.bq %r15, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovsxRmR(extModeWL, newOperandReg(raxVReg), rdiVReg) },
+			want:       "0fbff8",
+			wantFormat: "movsx.wl %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovsxRmR(extModeWQ, newOperandReg(raxVReg), rdiVReg) },
+			want:       "480fbff8",
+			wantFormat: "movsx.wq %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovsxRmR(extModeLQ, newOperandReg(raxVReg), rdiVReg) },
+			want:       "4863f8",
+			wantFormat: "movsx.lq %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovzxRmR(extModeBL, newOperandReg(raxVReg), rdiVReg) },
+			want:       "0fb6f8",
+			wantFormat: "movzx.bl %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovzxRmR(extModeBL, newOperandReg(rbxVReg), rdiVReg) },
+			want:       "0fb6fb",
+			wantFormat: "movzx.bl %rbx, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovzxRmR(extModeBQ, newOperandReg(raxVReg), rdiVReg) },
+			want:       "480fb6f8",
+			wantFormat: "movzx.bq %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovzxRmR(extModeBQ, newOperandReg(r15VReg), rdiVReg) },
+			want:       "490fb6ff",
+			wantFormat: "movzx.bq %r15, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovzxRmR(extModeWL, newOperandReg(raxVReg), rdiVReg) },
+			want:       "0fb7f8",
+			wantFormat: "movzx.wl %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovzxRmR(extModeWQ, newOperandReg(raxVReg), rdiVReg) },
+			want:       "480fb7f8",
+			wantFormat: "movzx.wq %rax, %rdi",
+		},
+		{
+			setup:      func(i *instruction) { i.asMovzxRmR(extModeLQ, newOperandReg(raxVReg), rdiVReg) },
+			want:       "8bf8",
+			wantFormat: "movzx.lq %rax, %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.asMovsxRmR(extModeBQ, a, rdiVReg)
+			},
+			want:       "480fbeb800001000",
+			wantFormat: "movsx.bq 1048576(%rax), %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.asMovsxRmR(extModeBL, a, rdiVReg)
+			},
+			want:       "0fbeb800001000",
+			wantFormat: "movsx.bl 1048576(%rax), %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.asMovsxRmR(extModeWL, a, rdiVReg)
+			},
+			want:       "0fbfb800001000",
+			wantFormat: "movsx.wl 1048576(%rax), %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.asMovsxRmR(extModeWQ, a, rdiVReg)
+			},
+			want:       "480fbfb800001000",
+			wantFormat: "movsx.wq 1048576(%rax), %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.asMovsxRmR(extModeLQ, a, rdiVReg)
+			},
+			want:       "4863b800001000",
+			wantFormat: "movsx.lq 1048576(%rax), %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.asMovzxRmR(extModeLQ, a, rdiVReg)
+			},
+			want:       "8bb800001000",
+			wantFormat: "movzx.lq 1048576(%rax), %rdi",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.movRM(rcxVReg, a, 1)
+			},
+			want:       "888800001000",
+			wantFormat: "mov.b %rcx, 1048576(%rax)",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeImmReg(1<<20, raxVReg))
+				i.movRM(rdiVReg, a, 1)
+			},
+			want:       "88b800001000",
+			wantFormat: "mov.b %rdi, 1048576(%rax)",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeRegRegShit(1<<20, raxVReg, rcxVReg, 3))
+				i.movRM(rdiVReg, a, 2)
+			},
+			want:       "6689bcc800001000",
+			wantFormat: "mov.w %rdi, 1048576(%rax,%rcx,8)",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeRegRegShit(123, raxVReg, rcxVReg, 0))
+				i.movRM(rdiVReg, a, 4)
+			},
+			want:       "897c087b",
+			wantFormat: "mov.l %rdi, 123(%rax,%rcx,1)",
+		},
+		{
+			setup: func(i *instruction) {
+				a := newOperandMem(newAmodeRegRegShit(123, raxVReg, rcxVReg, 0))
+				i.movRM(rdiVReg, a, 8)
+			},
+			want:       "48897c087b",
+			wantFormat: "mov.q %rdi, 123(%rax,%rcx,1)",
+		},
 	} {
 		tc := tc
 		t.Run(tc.wantFormat, func(t *testing.T) {