Repository: bytecodealliance/wasmtime
Pull request #6045 (2023-03-17 15:53:52)

x64: Elide more uextend with extractlane
I've confirmed locally now that `pextr{b,w,d}` all zero the upper bits of the full 64-bit register size which means that the `extractlane` operation with a zero-extend can be elided for more cases, including 8-to-64-bit casts as well as 32-to-64.

This helps elide a few extra `mov`s in a loop I was looking at and had a modest corresponding increase in performance (my guess was due to the slightly decreased code size mostly as opposed to the removed `mov`s).

<!--

Please ensure that the following steps are all taken care of before submitting
the PR.

- [ ] This has been discussed in issue #..., or if not, please tell us why
  here.
- [ ] A short description of what this does, why it is needed; if the
  description becomes long, the matter should probably be discussed in an issue
  first.
- [ ] This PR contains test cases, if meaningful.
- [ ] A reviewer from the core maintainer team has been assigned for this PR.
  If you don't know who could review this, please indicate so. The list of
  suggested reviewers on the right can help you.

Please ensure all communication adheres to the [code of
conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
-->



Patch:
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index fbaac1da3..e51c9e891 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -1715,14 +1715,11 @@
         (if-let $true (value32_zeros_upper32 src))
         src)
 
-;; The `extractlane` instruction, extended to `$I32`, means that either an
-;; i8x16 or an i16x8 is being extracted. These are implemented with
-;; the `pextr{b,w}` instruction which automatically zero the upper bits of the
-;; destination register so the `uextend` in these cases can be elided.
-;;
-;; TODO: the documentation for `pextr{b,w}` seems to indicate it zero extends
-;; to not only 32-bits but probably the whole 64-bit register. If that's the
-;; case then this should match a zero-extend to any size instead of just `$I32`.
+;; The `extractlane` instruction, when paired with a zero-extension, means
+;; that one of `i{8x16,16x8,32x4}` is being extracted and extend. These
+;; extractions are implemented with the `pextr{b,w,d}` instructions which
+;; automatically zero the upper bits of the destination register so the
+;; `uextend` in these cases can be elided.
 ;;
 ;; TODO: the interaction here between this rule and the "it's written far away"
 ;; rule to lower `extractlane` isn't great. Ideally this rule (and the other
@@ -1730,7 +1727,7 @@
 ;; be connected to the extractlane rules. There's some discussion of this on
 ;; #6022 but the gist is that there's not a lot of great options at this time,
 ;; so this doc block is what's here for now.
-(rule 1 (extend_to_gpr src @ (extractlane _ _) $I32 (ExtendKind.Zero))
+(rule 1 (extend_to_gpr src @ (extractlane _ _) _ (ExtendKind.Zero))
         src)
 
 (rule (extend_to_gpr (and val (value_type from_ty))
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index f42270f12..98f1b1375 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -3808,6 +3808,8 @@
 (rule 0 (lower (extractlane val @ (value_type ty @ (multi_lane 16 8)) (u8_from_uimm8 lane)))
       (x64_pextrw val lane))
 
+;; See the note in the 8x16 case above for how this rule is connected to
+;; `extend_to_gpr`.
 (rule 0 (lower (extractlane val @ (value_type ty @ (multi_lane 32 4)) (u8_from_uimm8 lane)))
       (x64_pextrd val lane))
 
diff --git a/cranelift/codegen/src/opts/algebraic.isle b/cranelift/codegen/src/opts/algebraic.isle
index ce04c2716..3efa05dfe 100644
--- a/cranelift/codegen/src/opts/algebraic.isle
+++ b/cranelift/codegen/src/opts/algebraic.isle
@@ -35,6 +35,9 @@
 ;; ineg(ineg(x)) == x.
 (rule (simplify (ineg ty (ineg ty x))) (subsume x))
 
+;; x-x == 0.
+(rule (simplify (isub (fits_in_64 (ty_int ty)) x x)) (subsume (iconst ty (imm64 0))))
+
 ;; x*1 == 1*x == x.
 (rule (simplify (imul ty
                       x
diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs
index e48301b27..d51f008f9 100644
--- a/cranelift/fuzzgen/src/function_generator.rs
+++ b/cranelift/fuzzgen/src/function_generator.rs
@@ -415,6 +415,7 @@ fn valid_for_target(triple: &Triple, op: Opcode, args: &[Type], rets: &[Type]) -
         Architecture::X86_64 => {
             exceptions!(
                 (Opcode::IaddCout, &([I8, I8] | [I16, I16] | [I128, I128])),
+                (Opcode::Imul, &[I8X16, I8X16]),
                 // https://github.com/bytecodealliance/wasmtime/issues/5468
                 (Opcode::Smulhi | Opcode::Umulhi, &[I8, I8]),
                 // https://github.com/bytecodealliance/wasmtime/issues/4756
@@ -680,6 +681,10 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Iadd, &[I32, I32], &[I32]),
     (Opcode::Iadd, &[I64, I64], &[I64]),
     (Opcode::Iadd, &[I128, I128], &[I128]),
+    (Opcode::Iadd, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::Iadd, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::Iadd, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::Iadd, &[I64X2, I64X2], &[I64X2]),
     // IaddCout
     (Opcode::IaddCout, &[I8, I8], &[I8, I8]),
     (Opcode::IaddCout, &[I16, I16], &[I16, I8]),
@@ -702,6 +707,10 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Isub, &[I32, I32], &[I32]),
     (Opcode::Isub, &[I64, I64], &[I64]),
     (Opcode::Isub, &[I128, I128], &[I128]),
+    (Opcode::Isub, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::Isub, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::Isub, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::Isub, &[I64X2, I64X2], &[I64X2]),
     // UsubSat
     (Opcode::UsubSat, &[I8X16, I8X16], &[I8X16]),
     (Opcode::UsubSat, &[I16X8, I16X8], &[I16X8]),
@@ -718,6 +727,10 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Imul, &[I32, I32], &[I32]),
     (Opcode::Imul, &[I64, I64], &[I64]),
     (Opcode::Imul, &[I128, I128], &[I128]),
+    (Opcode::Imul, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::Imul, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::Imul, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::Imul, &[I64X2, I64X2], &[I64X2]),
     // Smulhi
     (Opcode::Smulhi, &[I8, I8], &[I8]),
     (Opcode::Smulhi, &[I16, I16], &[I16]),
@@ -991,6 +1004,12 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Band, &[I128, I128], &[I128]),
     (Opcode::Band, &[F32, F32], &[F32]),
     (Opcode::Band, &[F64, F64], &[F64]),
+    (Opcode::Band, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::Band, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::Band, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::Band, &[I64X2, I64X2], &[I64X2]),
+    (Opcode::Band, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::Band, &[F64X2, F64X2], &[F64X2]),
     // Bor
     (Opcode::Bor, &[I8, I8], &[I8]),
     (Opcode::Bor, &[I16, I16], &[I16]),
@@ -999,6 +1018,12 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Bor, &[I128, I128], &[I128]),
     (Opcode::Bor, &[F32, F32], &[F32]),
     (Opcode::Bor, &[F64, F64], &[F64]),
+    (Opcode::Bor, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::Bor, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::Bor, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::Bor, &[I64X2, I64X2], &[I64X2]),
+    (Opcode::Bor, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::Bor, &[F64X2, F64X2], &[F64X2]),
     // Bxor
     (Opcode::Bxor, &[I8, I8], &[I8]),
     (Opcode::Bxor, &[I16, I16], &[I16]),
@@ -1007,14 +1032,26 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Bxor, &[I128, I128], &[I128]),
     (Opcode::Bxor, &[F32, F32], &[F32]),
     (Opcode::Bxor, &[F64, F64], &[F64]),
+    (Opcode::Bxor, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::Bxor, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::Bxor, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::Bxor, &[I64X2, I64X2], &[I64X2]),
+    (Opcode::Bxor, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::Bxor, &[F64X2, F64X2], &[F64X2]),
     // Bnot
-    (Opcode::Bnot, &[I8, I8], &[I8]),
-    (Opcode::Bnot, &[I16, I16], &[I16]),
-    (Opcode::Bnot, &[I32, I32], &[I32]),
-    (Opcode::Bnot, &[I64, I64], &[I64]),
-    (Opcode::Bnot, &[I128, I128], &[I128]),
-    (Opcode::Bnot, &[F32, F32], &[F32]),
-    (Opcode::Bnot, &[F64, F64], &[F64]),
+    (Opcode::Bnot, &[I8], &[I8]),
+    (Opcode::Bnot, &[I16], &[I16]),
+    (Opcode::Bnot, &[I32], &[I32]),
+    (Opcode::Bnot, &[I64], &[I64]),
+    (Opcode::Bnot, &[I128], &[I128]),
+    (Opcode::Bnot, &[F32], &[F32]),
+    (Opcode::Bnot, &[F64], &[F64]),
+    (Opcode::Bnot, &[I8X16], &[I8X16]),
+    (Opcode::Bnot, &[I16X8], &[I16X8]),
+    (Opcode::Bnot, &[I32X4], &[I32X4]),
+    (Opcode::Bnot, &[I64X2], &[I64X2]),
+    (Opcode::Bnot, &[F32X4], &[F32X4]),
+    (Opcode::Bnot, &[F64X2], &[F64X2]),
     // BandNot
     (Opcode::BandNot, &[I8, I8], &[I8]),
     (Opcode::BandNot, &[I16, I16], &[I16]),
@@ -1023,6 +1060,12 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::BandNot, &[I128, I128], &[I128]),
     (Opcode::BandNot, &[F32, F32], &[F32]),
     (Opcode::BandNot, &[F64, F64], &[F64]),
+    (Opcode::BandNot, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::BandNot, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::BandNot, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::BandNot, &[I64X2, I64X2], &[I64X2]),
+    (Opcode::BandNot, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::BandNot, &[F64X2, F64X2], &[F64X2]),
     // BorNot
     (Opcode::BorNot, &[I8, I8], &[I8]),
     (Opcode::BorNot, &[I16, I16], &[I16]),
@@ -1031,6 +1074,12 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::BorNot, &[I128, I128], &[I128]),
     (Opcode::BorNot, &[F32, F32], &[F32]),
     (Opcode::BorNot, &[F64, F64], &[F64]),
+    (Opcode::BorNot, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::BorNot, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::BorNot, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::BorNot, &[I64X2, I64X2], &[I64X2]),
+    (Opcode::BorNot, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::BorNot, &[F64X2, F64X2], &[F64X2]),
     // BxorNot
     (Opcode::BxorNot, &[I8, I8], &[I8]),
     (Opcode::BxorNot, &[I16, I16], &[I16]),
@@ -1039,6 +1088,12 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::BxorNot, &[I128, I128], &[I128]),
     (Opcode::BxorNot, &[F32, F32], &[F32]),
     (Opcode::BxorNot, &[F64, F64], &[F64]),
+    (Opcode::BxorNot, &[I8X16, I8X16], &[I8X16]),
+    (Opcode::BxorNot, &[I16X8, I16X8], &[I16X8]),
+    (Opcode::BxorNot, &[I32X4, I32X4], &[I32X4]),
+    (Opcode::BxorNot, &[I64X2, I64X2], &[I64X2]),
+    (Opcode::BxorNot, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::BxorNot, &[F64X2, F64X2], &[F64X2]),
     // Bitrev
     (Opcode::Bitrev, &[I8], &[I8]),
     (Opcode::Bitrev, &[I16], &[I16]),
@@ -1114,6 +1169,10 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     (Opcode::Bitselect, &[I32, I32, I32], &[I32]),
     (Opcode::Bitselect, &[I64, I64, I64], &[I64]),
     (Opcode::Bitselect, &[I128, I128, I128], &[I128]),
+    (Opcode::Bitselect, &[I8X16, I8X16, I8X16], &[I8X16]),
+    (Opcode::Bitselect, &[I16X8, I16X8, I16X8], &[I16X8]),
+    (Opcode::Bitselect, &[I32X4, I32X4, I32X4], &[I32X4]),
+    (Opcode::Bitselect, &[I64X2, I64X2, I64X2], &[I64X2]),
     // Select
     (Opcode::Select, &[I8, I8, I8], &[I8]),
     (Opcode::Select, &[I8, I16, I16], &[I16]),
@@ -1169,15 +1228,23 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     // Fadd
     (Opcode::Fadd, &[F32, F32], &[F32]),
     (Opcode::Fadd, &[F64, F64], &[F64]),
+    (Opcode::Fadd, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::Fadd, &[F64X2, F64X2], &[F64X2]),
     // Fmul
     (Opcode::Fmul, &[F32, F32], &[F32]),
     (Opcode::Fmul, &[F64, F64], &[F64]),
+    (Opcode::Fmul, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::Fmul, &[F64X2, F64X2], &[F64X2]),
     // Fsub
     (Opcode::Fsub, &[F32, F32], &[F32]),
     (Opcode::Fsub, &[F64, F64], &[F64]),
+    (Opcode::Fsub, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::Fsub, &[F64X2, F64X2], &[F64X2]),
     // Fdiv
     (Opcode::Fdiv, &[F32, F32], &[F32]),
     (Opcode::Fdiv, &[F64, F64], &[F64]),
+    (Opcode::Fdiv, &[F32X4, F32X4], &[F32X4]),
+    (Opcode::Fdiv, &[F64X2, F64X2], &[F64X2]),
     // Fmin
     (Opcode::Fmin, &[F32, F32], &[F32]),
     (Opcode::Fmin, &[F64, F64], &[F64]),
@@ -1198,27 +1265,43 @@ const OPCODE_SIGNATURES: &[OpcodeSignature] = &[
     // Fma
     (Opcode::Fma, &[F32, F32, F32], &[F32]),
     (Opcode::Fma, &[F64, F64, F64], &[F64]),
+    (Opcode::Fma, &[F32X4, F32X4, F32X4], &[F32X4]),
+    (Opcode::Fma, &[F64X2, F64X2, F64X4], &[F64X2]),
     // Fabs
     (Opcode::Fabs, &[F32], &[F32]),
     (Opcode::Fabs, &[F64], &[F64]),
+    (Opcode::Fabs, &[F32X4], &[F32X4]),
+    (Opcode::Fabs, &[F64X2], &[F64X2]),
     // Fneg
     (Opcode::Fneg, &[F32], &[F32]),
     (Opcode::Fneg, &[F64], &[F64]),
+    (Opcode::Fneg, &[F32X4], &[F32X4]),
+    (Opcode::Fneg, &[F64X2], &[F64X2]),
     // Sqrt
     (Opcode::Sqrt, &[F32], &[F32]),
     (Opcode::Sqrt, &[F64], &[F64]),
+    (Opcode::Sqrt, &[F32X4], &[F32X4]),
+    (Opcode::Sqrt, &[F64X2], &[F64X2]),
     // Ceil
     (Opcode::Ceil, &[F32], &[F32]),
     (Opcode::Ceil, &[F64], &[F64]),
+    (Opcode::Ceil, &[F32X4], &[F32X4]),
+    (Opcode::Ceil, &[F64X2], &[F64X2]),
     // Floor
     (Opcode::Floor, &[F32], &[F32]),
     (Opcode::Floor, &[F64], &[F64]),
+    (Opcode::Floor, &[F32X4], &[F32X4]),
+    (Opcode::Floor, &[F64X2], &[F64X2]),
     // Trunc
     (Opcode::Trunc, &[F32], &[F32]),
     (Opcode::Trunc, &[F64], &[F64]),
+    (Opcode::Trunc, &[F32X4], &[F32X4]),
+    (Opcode::Trunc, &[F64X2], &[F64X2]),
     // Nearest
     (Opcode::Nearest, &[F32], &[F32]),
     (Opcode::Nearest, &[F64], &[F64]),
+    (Opcode::Nearest, &[F32X4], &[F32X4]),
+    (Opcode::Nearest, &[F64X2], &[F64X2]),
     // Fpromote
     (Opcode::Fpromote, &[F32], &[F64]),
     // Fdemote


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/algebraic.clif b/cranelift/filetests/filetests/egraph/algebraic.clif
index 02d38cdd9..faae822fe 100644
--- a/cranelift/filetests/filetests/egraph/algebraic.clif
+++ b/cranelift/filetests/filetests/egraph/algebraic.clif
@@ -232,6 +232,14 @@ block0(v0: i32):
     ; check: return v0
 }
 
+function %isub_self(i32) -> i32 {
+block0(v0: i32):
+    v1 = isub v0, v0
+    return v1
+    ; check: v2 = iconst.i32 0
+    ; check: return v2
+}
+
 function %or_and_y_with_not_y_i8(i8, i8) -> i8 {
 block0(v0: i8, v1: i8):
     v2 = band v0, v1
diff --git a/cranelift/filetests/filetests/isa/x64/uextend-elision.clif b/cranelift/filetests/filetests/isa/x64/uextend-elision.clif
index cc277ff60..76689c768 100644
--- a/cranelift/filetests/filetests/isa/x64/uextend-elision.clif
+++ b/cranelift/filetests/filetests/isa/x64/uextend-elision.clif
@@ -79,6 +79,32 @@ block0(v0: i8x16):
 ;   popq %rbp
 ;   retq
 
+function %extractlane_i8x16_i64(i8x16) -> i64 {
+block0(v0: i8x16):
+    v1 = extractlane v0, 1
+    v2 = uextend.i64 v1
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pextrb  $1, %xmm0, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pextrb $1, %xmm0, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
 function %extractlane_i16x8_i32(i16x8) -> i32 {
 block0(v0: i16x8):
     v1 = extractlane v0, 1
@@ -105,3 +131,55 @@ block0(v0: i16x8):
 ;   popq %rbp
 ;   retq
 
+function %extractlane_i16x8_i64(i16x8) -> i64 {
+block0(v0: i16x8):
+    v1 = extractlane v0, 1
+    v2 = uextend.i64 v1
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pextrw  $1, %xmm0, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pextrw $1, %xmm0, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %extractlane_i32x4_i64(i32x4) -> i64 {
+block0(v0: i32x4):
+    v1 = extractlane v0, 1
+    v2 = uextend.i64 v1
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   pextrd  $1, %xmm0, %rax
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pextrd $1, %xmm0, %eax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+