Repository: bytecodealliance/wasmtime
Pull request #9140 (2024-08-19 16:17:35)

cranelift/aarch64: support fnmsub and fnmadd
issue #8603.


Patch:
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 672ebf4d8..d7aaaed0d 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -1355,6 +1355,10 @@
     (MAdd)
     ;; Multiply-sub
     (MSub)
+    ;; Negated fused Multiply-add
+    (NMAdd)
+    ;; Negated fused Multiply-sub
+    (NMSub)
 ))
 
 ;; A conversion from an FP to an integer value.
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 9da0b932c..d458f0d18 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -1878,6 +1878,8 @@ impl MachInstEmit for Inst {
                 let top17 = match fpu_op {
                     FPUOp3::MAdd => 0b000_11111_00_0_00000_0,
                     FPUOp3::MSub => 0b000_11111_00_0_00000_1,
+                    FPUOp3::NMAdd => 0b000_11111_00_1_00000_0,
+                    FPUOp3::NMSub => 0b000_11111_00_1_00000_1,
                 };
                 let top17 = top17 | size.ftype() << 7;
                 sink.put4(enc_fpurrrr(top17, rd, rn, rm, ra));
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index cd51aa77e..a7f3ba714 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -1823,6 +1823,8 @@ impl Inst {
                 let op = match fpu_op {
                     FPUOp3::MAdd => "fmadd",
                     FPUOp3::MSub => "fmsub",
+                    FPUOp3::NMAdd => "fnmadd",
+                    FPUOp3::NMSub => "fnmsub",
                 };
                 let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
                 let rn = pretty_print_vreg_scalar(rn, size);
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 6267de3b5..a36d0d6fc 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -535,15 +535,22 @@
 (decl pure get_fneg_value (IsFneg) Value)
 (rule (get_fneg_value (IsFneg.Result _ v)) v)
 
-(decl fmadd_series (Type u64 Value Value Value) InstOutput)
-(rule 0 (fmadd_series (ty_scalar_float ty) 0 x y z) (fpu_rrrr (FPUOp3.MAdd) (scalar_size ty) x y z))
-(rule 0 (fmadd_series (ty_scalar_float ty) 1 x y z) (fpu_rrrr (FPUOp3.MSub) (scalar_size ty) x y z))
-
-(rule (lower (has_type (ty_scalar_float ty) (fma x_src y_src z)))
+(decl fmadd_series (Type u64 u64 Value Value Value) InstOutput)
+;; MAdd: + x * y + z
+(rule 0 (fmadd_series (ty_scalar_float ty) 0 0 x y z) (fpu_rrrr (FPUOp3.MAdd) (scalar_size ty) x y z))
+;; MSub: - x * y + z
+(rule 0 (fmadd_series (ty_scalar_float ty) 1 0 x y z) (fpu_rrrr (FPUOp3.MSub) (scalar_size ty) x y z))
+;; NMAdd: - x * y - z
+(rule 0 (fmadd_series (ty_scalar_float ty) 1 1 x y z) (fpu_rrrr (FPUOp3.NMAdd) (scalar_size ty) x y z))
+;; NMSub: + x * y - z
+(rule 0 (fmadd_series (ty_scalar_float ty) 0 1 x y z) (fpu_rrrr (FPUOp3.NMSub) (scalar_size ty) x y z))
+
+(rule (lower (has_type (ty_scalar_float ty) (fma x_src y_src z_src)))
   (let
       ((x_res IsFneg (is_fneg x_src))
-       (y_res IsFneg (is_fneg y_src)))
-      (fmadd_series ty (u64_xor (is_fneg_neg x_res) (is_fneg_neg y_res)) (get_fneg_value x_res) (get_fneg_value y_res) z)))
+       (y_res IsFneg (is_fneg y_src))
+       (z_res IsFneg (is_fneg z_src)))
+      (fmadd_series ty (u64_xor (is_fneg_neg x_res) (is_fneg_neg y_res)) (is_fneg_neg z_res) (get_fneg_value x_res) (get_fneg_value y_res) (get_fneg_value z_res))))
 
 ;; Delegate vector-based lowerings to helpers below
 (rule 1 (lower (has_type ty @ (multi_lane _ _) (fma x y z)))


Test patch:
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
index e6f2b4c22..0b50795ec 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
@@ -6477,6 +6477,32 @@ fn test_aarch64_binemit() {
         "fmsub d15, d30, d31, d1",
     ));
 
+    insns.push((
+        Inst::FpuRRRR {
+            fpu_op: FPUOp3::NMAdd,
+            size: ScalarSize::Size64,
+            rd: writable_vreg(15),
+            rn: vreg(30),
+            rm: vreg(31),
+            ra: vreg(1),
+        },
+        "CF077F1F",
+        "fnmadd d15, d30, d31, d1",
+    ));
+
+    insns.push((
+        Inst::FpuRRRR {
+            fpu_op: FPUOp3::NMSub,
+            size: ScalarSize::Size64,
+            rd: writable_vreg(15),
+            rn: vreg(30),
+            rm: vreg(31),
+            ra: vreg(1),
+        },
+        "CF877F1F",
+        "fnmsub d15, d30, d31, d1",
+    ));
+
     insns.push((
         Inst::FpuRRI {
             fpu_op: FPUOpRI::UShr32(FPURightShiftImm::maybe_from_u8(32, 32).unwrap()),
diff --git a/cranelift/filetests/filetests/isa/aarch64/fma.clif b/cranelift/filetests/filetests/isa/aarch64/fma.clif
index 12e0f081a..d44099cec 100644
--- a/cranelift/filetests/filetests/isa/aarch64/fma.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/fma.clif
@@ -67,6 +67,72 @@ block0(v0: f64, v1: f64, v2: f64):
 ;   fmsub d0, d0, d1, d2
 ;   ret
 
+function %fnmadd_f32(f32, f32, f32) -> f32 {
+block0(v0: f32, v1: f32, v2: f32):
+    v3 = fneg v1
+    v4 = fneg v2
+    v5 = fma v0, v3, v4
+    return v5
+}
+; VCode:
+; block0:
+;   fnmadd s0, s0, s1, s2
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fnmadd s0, s0, s1, s2
+;   ret
+
+function %fnmadd_f64(f64, f64, f64) -> f64 {
+block0(v0: f64, v1: f64, v2: f64):
+    v3 = fneg v1
+    v4 = fneg v2
+    v5 = fma v0, v3, v4
+    return v5
+}
+; VCode:
+; block0:
+;   fnmadd d0, d0, d1, d2
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fnmadd d0, d0, d1, d2
+;   ret
+
+function %fnmadd_f32(f32, f32, f32) -> f32 {
+block0(v0: f32, v1: f32, v2: f32):
+    v3 = fneg v2
+    v4 = fma v0, v1, v3
+    return v4
+}
+; VCode:
+; block0:
+;   fnmsub s0, s0, s1, s2
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fnmsub s0, s0, s1, s2
+;   ret
+
+function %fnmadd_f64(f64, f64, f64) -> f64 {
+block0(v0: f64, v1: f64, v2: f64):
+    v3 = fneg v2
+    v4 = fma v0, v1, v3
+    return v4
+}
+; VCode:
+; block0:
+;   fnmsub d0, d0, d1, d2
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fnmsub d0, d0, d1, d2
+;   ret
+
 function %fma_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
 block0(v0: f32x4, v1: f32x4, v2: f32x4):
     v3 = fma v0, v1, v2