Repository: bytecodealliance/wasmtime
Pull request #5999 (2023-03-12 19:05:05)

x64: Add `shuffle` specialization for `palignr`
This commit adds specializations for the `palignr` instruction to the x64 backend to specialize some more patterns of byte shuffles.

<!--

Please ensure that the following steps are all taken care of before submitting
the PR.

- [ ] This has been discussed in issue #..., or if not, please tell us why
  here.
- [ ] A short description of what this does, why it is needed; if the
  description becomes long, the matter should probably be discussed in an issue
  first.
- [ ] This PR contains test cases, if meaningful.
- [ ] A reviewer from the core maintainer team has been assigned for this PR.
  If you don't know who could review this, please indicate so. The list of
  suggested reviewers on the right can help you.

Please ensure all communication adheres to the [code of
conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
-->


github-actions[bot]: #### Subscribe to Label Action

cc @cfallin, @fitzgen

<details>
This issue or pull request has been labeled: "cranelift", "cranelift:area:x64", "isle"

Thus the following users have been cc'd because of the following labels:

* cfallin: isle
* fitzgen: isle

To subscribe or unsubscribe from this label, edit the <code>.github/subscribe-to-label.json</code> configuration file.

[Learn more.](https://github.com/bytecodealliance/subscribe-to-label-action)
</details>

Patch:
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index bcc63bfdb..9b1b6ad8b 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -3231,14 +3231,14 @@
         dst))
 
 ;; Helper for creating `palignr` instructions.
-(decl x64_palignr (Xmm XmmMem u8 OperandSize) Xmm)
-(rule 0 (x64_palignr src1 src2 imm size)
+(decl x64_palignr (Xmm XmmMem u8) Xmm)
+(rule 0 (x64_palignr src1 src2 imm)
       (xmm_rm_r_imm (SseOpcode.Palignr)
                     src1
                     src2
                     imm
-                    size))
-(rule 1 (x64_palignr src1 src2 imm size)
+                    (OperandSize.Size32)))
+(rule 1 (x64_palignr src1 src2 imm)
       (if-let $true (has_avx))
       (xmm_rmr_imm_vex (AvxOpcode.Vpalignr) src1 src2 imm))
 
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 5b4773f6c..6a4f73620 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -894,10 +894,10 @@
                              (swiden_high (and (value_type (multi_lane 8 16))
                                                y)))))
       (let ((x1 Xmm x)
-            (x2 Xmm (x64_palignr x1 x1 8 (OperandSize.Size32)))
+            (x2 Xmm (x64_palignr x1 x1 8))
             (x3 Xmm (x64_pmovsxbw x2))
             (y1 Xmm y)
-            (y2 Xmm (x64_palignr y1 y1 8 (OperandSize.Size32)))
+            (y2 Xmm (x64_palignr y1 y1 8))
             (y3 Xmm (x64_pmovsxbw y2)))
         (x64_pmullw x3 y3)))
 
@@ -962,10 +962,10 @@
                              (uwiden_high (and (value_type (multi_lane 8 16))
                                                y)))))
       (let ((x1 Xmm x)
-            (x2 Xmm (x64_palignr x1 x1 8 (OperandSize.Size32)))
+            (x2 Xmm (x64_palignr x1 x1 8))
             (x3 Xmm (x64_pmovzxbw x2))
             (y1 Xmm y)
-            (y2 Xmm (x64_palignr y1 y1 8 (OperandSize.Size32)))
+            (y2 Xmm (x64_palignr y1 y1 8))
             (y3 Xmm (x64_pmovzxbw y2)))
         (x64_pmullw x3 y3)))
 
@@ -3284,11 +3284,11 @@
 
 (rule (lower (has_type $I16X8 (swiden_high val @ (value_type $I8X16))))
       (let ((x Xmm val))
-        (x64_pmovsxbw (x64_palignr x x 8 (OperandSize.Size32)))))
+        (x64_pmovsxbw (x64_palignr x x 8))))
 
 (rule (lower (has_type $I32X4 (swiden_high val @ (value_type $I16X8))))
       (let ((x Xmm val))
-        (x64_pmovsxwd (x64_palignr x x 8 (OperandSize.Size32)))))
+        (x64_pmovsxwd (x64_palignr x x 8))))
 
 (rule (lower (has_type $I64X2 (swiden_high val @ (value_type $I32X4))))
       (x64_pmovsxdq (x64_pshufd val 0xEE)))
@@ -3308,11 +3308,11 @@
 
 (rule (lower (has_type $I16X8 (uwiden_high val @ (value_type $I8X16))))
       (let ((x Xmm val))
-        (x64_pmovzxbw (x64_palignr x x 8 (OperandSize.Size32)))))
+        (x64_pmovzxbw (x64_palignr x x 8))))
 
 (rule (lower (has_type $I32X4 (uwiden_high val @ (value_type $I16X8))))
       (let ((x Xmm val))
-        (x64_pmovzxwd (x64_palignr x x 8 (OperandSize.Size32)))))
+        (x64_pmovzxwd (x64_palignr x x 8))))
 
 (rule (lower (has_type $I64X2 (uwiden_high val @ (value_type $I32X4))))
       (x64_pmovzxdq (x64_pshufd val 0xEE)))
@@ -3561,6 +3561,16 @@
 
 ;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; When the shuffle looks like "concatenate `a` and `b` and shift right by n*8
+;; bytes", that's a `palignr` instruction. Note that the order of operands are
+;; swapped in the instruction here. The `palignr` instruction uses the second
+;; operand as the low-order bytes and the first operand as high-order bytes,
+;; so put `a` second.
+(rule 13 (lower (shuffle a b (palignr_imm_from_immediate n)))
+         (x64_palignr b a n))
+(decl palignr_imm_from_immediate (u8) Immediate)
+(extern extractor palignr_imm_from_immediate palignr_imm_from_immediate)
+
 ;; Special case the `pshuf{l,h}w` instruction which shuffles four 16-bit
 ;; integers within one value, preserving the other four 16-bit integers in that
 ;; value (either the high or low half). The complicated logic is in the
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 815e40e35..bff7c4280 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -1117,6 +1117,16 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
             None
         }
     }
+
+    fn palignr_imm_from_immediate(&mut self, imm: Immediate) -> Option<u8> {
+        let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
+
+        if bytes.windows(2).all(|a| a[0] + 1 == a[1]) {
+            Some(bytes[0])
+        } else {
+            None
+        }
+    }
 }
 
 impl IsleContext<'_, '_, MInst, X64Backend> {


Test patch:
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle.clif b/cranelift/filetests/filetests/isa/x64/shuffle.clif
index ce1d1b484..f8f9b613e 100644
--- a/cranelift/filetests/filetests/isa/x64/shuffle.clif
+++ b/cranelift/filetests/filetests/isa/x64/shuffle.clif
@@ -196,7 +196,7 @@ function %not_single_pshufd(i32x4, i32x4) -> i32x4 {
 block0(v0: i32x4, v1: i32x4):
     v2 = bitcast.i8x16 little v0
     v3 = bitcast.i8x16 little v1
-    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23]
+    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 20 21 22 23 20 21 22 23]
     v5 = bitcast.i32x4 little v4
     return v5
 }
@@ -205,7 +205,7 @@ block0(v0: i32x4, v1: i32x4):
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   shufps  $78, %xmm0, %xmm1, %xmm0
+;   shufps  $94, %xmm0, %xmm1, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -215,7 +215,7 @@ block0(v0: i32x4, v1: i32x4):
 ;   pushq %rbp
 ;   movq %rsp, %rbp
 ; block1: ; offset 0x4
-;   shufps $0x4e, %xmm1, %xmm0
+;   shufps $0x5e, %xmm1, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq
@@ -644,3 +644,148 @@ block0(v0: i8x16, v1: i8x16):
 ;   popq %rbp
 ;   retq
 
+function %palignr_0(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15]
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movdqa  %xmm0, %xmm4
+;   movdqa  %xmm1, %xmm0
+;   palignr $0, %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movdqa %xmm0, %xmm4
+;   movdqa %xmm1, %xmm0
+;   palignr $0, %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %palignr_1(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16]
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movdqa  %xmm0, %xmm4
+;   movdqa  %xmm1, %xmm0
+;   palignr $1, %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movdqa %xmm0, %xmm4
+;   movdqa %xmm1, %xmm0
+;   palignr $1, %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %palignr_5(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20]
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movdqa  %xmm0, %xmm4
+;   movdqa  %xmm1, %xmm0
+;   palignr $5, %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movdqa %xmm0, %xmm4
+;   movdqa %xmm1, %xmm0
+;   palignr $5, %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %palignr_11(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26]
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movdqa  %xmm0, %xmm4
+;   movdqa  %xmm1, %xmm0
+;   palignr $11, %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movdqa %xmm0, %xmm4
+;   movdqa %xmm1, %xmm0
+;   palignr $0xb, %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %palignr_16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31]
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   movdqa  %xmm0, %xmm4
+;   movdqa  %xmm1, %xmm0
+;   palignr $16, %xmm0, %xmm4, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   movdqa %xmm0, %xmm4
+;   movdqa %xmm1, %xmm0
+;   palignr $0x10, %xmm4, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
index ac0421604..c33650d1a 100644
--- a/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
+++ b/cranelift/filetests/filetests/isa/x64/simd-arith-avx.clif
@@ -1948,3 +1948,28 @@ block0(v0: i32x4):
 ;   popq %rbp
 ;   retq
 
+function %palignr_11(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+    v2 = shuffle v0, v1, [11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26]
+    return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   vpalignr $11, %xmm1, %xmm0, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+; 
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   vpalignr $0xb, %xmm0, %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/runtests/simd-shuffle.clif b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
index f236a0205..2ef1671e2 100644
--- a/cranelift/filetests/filetests/runtests/simd-shuffle.clif
+++ b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
@@ -73,7 +73,7 @@ block0(v0: i32x4, v1: i32x4):
     v5 = bitcast.i32x4 little v4
     return v5
 }
-; run: %pshufd_0022([1 2 3 4], [5 6 7 8]) == [4 2 3 1]
+; run: %pshufd_3120([1 2 3 4], [5 6 7 8]) == [4 2 3 1]
 
 function %pshufd_7546(i32x4, i32x4) -> i32x4 {
 block0(v0: i32x4, v1: i32x4):
@@ -83,7 +83,7 @@ block0(v0: i32x4, v1: i32x4):
     v5 = bitcast.i32x4 little v4
     return v5
 }
-; run: %pshufd_0022([1 2 3 4], [5 6 7 8]) == [8 6 5 7]
+; run: %pshufd_7546([1 2 3 4], [5 6 7 8]) == [8 6 5 7]
 
 function %not_pshufd(i32x4, i32x4) -> i32x4 {
 block0(v0: i32x4, v1: i32x4):
@@ -93,7 +93,17 @@ block0(v0: i32x4, v1: i32x4):
     v5 = bitcast.i32x4 little v4
     return v5
 }
-; run: %pshufd_0022([1 2 3 4], [5 6 7 8]) == [3 4 5 6]
+; run: %not_pshufd([1 2 3 4], [5 6 7 8]) == [3 4 5 6]
+
+function %not_pshufd2(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+    v2 = bitcast.i8x16 little v0
+    v3 = bitcast.i8x16 little v1
+    v4 = shuffle v2, v3, [8 9 10 11 12 13 14 15 20 21 22 23 20 21 22 23]
+    v5 = bitcast.i32x4 little v4
+    return v5
+}
+; run: %not_pshufd2([1 2 3 4], [5 6 7 8]) == [3 4 6 6]
 
 function %punpckldq(i32x4, i32x4) -> i32x4 {
 block0(v0: i32x4, v1: i32x4):
diff --git a/tests/all/wast.rs b/tests/all/wast.rs
index 260e3bc00..994d27269 100644
--- a/tests/all/wast.rs
+++ b/tests/all/wast.rs
@@ -97,6 +97,23 @@ fn run_wast(wast: &str, strategy: Strategy, pooling: bool) -> anyhow::Result<()>
             return Ok(());
         }
 
+        // Reduce the virtual memory required to run multi-memory-based tests.
+        //
+        // The configuration parameters below require that a bare minimum
+        // virtual address space reservation of 450*9*805*65536 == 200G be made
+        // to support each test. If 6G reservations are made for each linear
+        // memory then not that many tests can run concurrently with much else.
+        //
+        // When multiple memories are used and are configured in the pool then
+        // force the usage of static memories without guards to reduce the VM
+        // impact.
+        if multi_memory {
+            cfg.static_memory_maximum_size(0);
+            cfg.dynamic_memory_reserved_for_growth(0);
+            cfg.static_memory_guard_size(0);
+            cfg.dynamic_memory_guard_size(0);
+        }
+
         // The limits here are crafted such that the wast tests should pass.
         // However, these limits may become insufficient in the future as the wast tests change.
         // If a wast test fails because of a limit being "exceeded" or if memory/table