Repository: RustCrypto/hashes (language: rust)

Issue:
sha1: ARM acceleration using `stdarch` intrinsics
There are a number of issues and comments floating around about this, so I thought I'd make a single tracking issue for this.

Right now we have a separate [asm-hashes](https://github.com/RustCrypto/asm-hashes/) repo containing assembly implementations of various hash functions. 

Longer-term, it would be nice to have ARM implementations of hash functions using [stdarch intrinsics](https://github.com/rust-lang/stdarch/), particularly if they provide similar performance via use of e.g. ARMv8 Cryptography Extensions. I was [almost able prototype](https://github.com/RustCrypto/asm-hashes/issues/5#issuecomment-812890980) a SHA-256 implementation this way:

https://gist.github.com/tarcieri/414a3300072160f372b5d93ccfce280b

There are a few notable blockers though:

- [x] some of the intrinsics we'd need are missing, e.g. `vst1q_u32` and ~~`vreinterpretq_u8_u32`~~ (added)
- [x] ~~the `stdarch` intrinsics for ARM are unstable/nightly-only~~
- [x] ~~there is presently no CI solution for e.g. Apple M1 (we can and already do use `cross` for more generic `aarch64`)~~
- [x] some way of gating the use of these intrinsics, e.g. via `target_feature`/`RUSTFLAGS` and/or runtime detection. see also https://github.com/RustCrypto/utils/issues/378

I think we could move forward prototyping things like SHA-1 and SHA-256 using the `stdarch` intrinsics for the ARMv8 Cryptography Extensions, but we may just need to leave those as draft PRs for now, or if we do merge them potentially add something like a `nightly` feature to gate them under.


Discussion / hints:
Here is a public domain implementation of SHA-256 using the ARMv8 intrinsics:

https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-arm.c

I translated it in the aforementioned gist:

https://gist.github.com/tarcieri/414a3300072160f372b5d93ccfce280b
as a note: all of the intrinsics I've found used are stable for aarch64 as of 1.72, I went on a quest to figure that out, so I felt like sharing the results of that. (https://doc.rust-lang.org/stable/core/arch/aarch64/fn.vsha256hq_u32.html being 1 of them, the rest I checked are _also_ there, note that it says 1.72.1, it was actually stabilized in 1.72.0 but the version tagging and changelogging got missed, and it currently just says whatever version stable is)
Yeah, we managed to make both `aes` and `polyval` support `stable` when using ARMv8 intrinsics.

~~It seems my little port of the SHA-256 intrinsics went by the wayside, but we should do something similar for that.~~

Oh nevermind, it's there, it's just using ASM "polyfills". We can bump to MSRV 1.72 in the next breaking release:

https://github.com/RustCrypto/hashes/blob/master/sha2/src/sha256/aarch64.rs
I removed `sha2` from the issue title as it's effectively complete aside from removing the ASM polyfills.

`sha1` is still using an ASM backend rather than intrinsics: https://github.com/RustCrypto/hashes/blob/1b4a55c/sha1/src/compress/aarch64.rs#L14
Dependency on asm-hashes was removed recently in preparation for v0.11 releases, so it's worth to port the assembly to `asm!`.

Fix patch:
diff --git a/sha1/src/compress.rs b/sha1/src/compress.rs
index d1032bbf2..72d0a3937 100644
--- a/sha1/src/compress.rs
+++ b/sha1/src/compress.rs
@@ -4,7 +4,11 @@ cfg_if::cfg_if! {
     if #[cfg(feature = "force-soft")] {
         mod soft;
         use soft::compress as compress_inner;
-   } else if #[cfg(target_arch = "loongarch64")] {
+    } else if #[cfg(all(target_arch = "aarch64"))] {
+        mod soft;
+        mod aarch64;
+        use aarch64::compress as compress_inner;
+    } else if #[cfg(target_arch = "loongarch64")] {
         mod loongarch64_asm;
         use loongarch64_asm::compress as compress_inner;
     } else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] {
diff --git a/sha1/src/compress/aarch64.rs b/sha1/src/compress/aarch64.rs
index 5952d1f62..f757f7f2d 100644
--- a/sha1/src/compress/aarch64.rs
+++ b/sha1/src/compress/aarch64.rs
@@ -1,5 +1,7 @@
 //! SHA-1 `aarch64` backend.
 
+use crate::K;
+
 // Per rustc target feature docs for `aarch64-unknown-linux-gnu` and
 // `aarch64-apple-darwin` platforms, the `sha2` target feature enables
 // SHA-1 as well:
@@ -7,11 +9,175 @@
 // > Enable SHA1 and SHA256 support.
 cpufeatures::new!(sha1_hwcap, "sha2");
 
+// note that `sha2` implicitly enables `neon`
+#[target_feature(enable = "sha2")]
+unsafe fn compress_sha1_neon(state: &mut [u32; 5], blocks: &[[u8; 64]]) {
+    use core::arch::aarch64::*;
+
+    let mut abcd = vld1q_u32(state.as_ptr());
+    let mut e0 = state[4];
+    let [k0, k1, k2, k3] = K.map(|k| vdupq_n_u32(k));
+    let (mut e1, mut tmp0, mut tmp1);
+
+    for block in blocks {
+        let abcd_cpy = abcd;
+        let e0_cpy = e0;
+
+        // Load and reverse byte order
+        let [mut msg0, mut msg1, mut msg2, mut msg3] = [0, 1, 2, 3].map(|i| {
+            let p = block.as_ptr().add(16 * i);
+            vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(p)))
+        });
+
+        tmp0 = vaddq_u32(msg0, k0);
+        tmp1 = vaddq_u32(msg1, k0);
+
+        // Rounds 0-3
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1cq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg2, k0);
+        msg0 = vsha1su0q_u32(msg0, msg1, msg2);
+
+        // Rounds 4-7
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1cq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg3, k0);
+        msg0 = vsha1su1q_u32(msg0, msg3);
+        msg1 = vsha1su0q_u32(msg1, msg2, msg3);
+
+        // Rounds 8-11
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1cq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg0, k0);
+        msg1 = vsha1su1q_u32(msg1, msg0);
+        msg2 = vsha1su0q_u32(msg2, msg3, msg0);
+
+        // Rounds 12-15
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1cq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg1, k1);
+        msg2 = vsha1su1q_u32(msg2, msg1);
+        msg3 = vsha1su0q_u32(msg3, msg0, msg1);
+
+        // Rounds 16-19
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1cq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg2, k1);
+        msg3 = vsha1su1q_u32(msg3, msg2);
+        msg0 = vsha1su0q_u32(msg0, msg1, msg2);
+
+        // Rounds 20-23
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg3, k1);
+        msg0 = vsha1su1q_u32(msg0, msg3);
+        msg1 = vsha1su0q_u32(msg1, msg2, msg3);
+
+        // Rounds 24-27
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg0, k1);
+        msg1 = vsha1su1q_u32(msg1, msg0);
+        msg2 = vsha1su0q_u32(msg2, msg3, msg0);
+
+        // Rounds 28-31
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg1, k1);
+        msg2 = vsha1su1q_u32(msg2, msg1);
+        msg3 = vsha1su0q_u32(msg3, msg0, msg1);
+
+        // Rounds 32-35
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg2, k2);
+        msg3 = vsha1su1q_u32(msg3, msg2);
+        msg0 = vsha1su0q_u32(msg0, msg1, msg2);
+
+        // Rounds 36-39
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg3, k2);
+        msg0 = vsha1su1q_u32(msg0, msg3);
+        msg1 = vsha1su0q_u32(msg1, msg2, msg3);
+
+        // Rounds 40-43
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1mq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg0, k2);
+        msg1 = vsha1su1q_u32(msg1, msg0);
+        msg2 = vsha1su0q_u32(msg2, msg3, msg0);
+
+        // Rounds 44-47
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1mq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg1, k2);
+        msg2 = vsha1su1q_u32(msg2, msg1);
+        msg3 = vsha1su0q_u32(msg3, msg0, msg1);
+
+        // Rounds 48-51
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1mq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg2, k2);
+        msg3 = vsha1su1q_u32(msg3, msg2);
+        msg0 = vsha1su0q_u32(msg0, msg1, msg2);
+
+        // Rounds 52-55
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1mq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg3, k3);
+        msg0 = vsha1su1q_u32(msg0, msg3);
+        msg1 = vsha1su0q_u32(msg1, msg2, msg3);
+
+        // Rounds 56-59
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1mq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg0, k3);
+        msg1 = vsha1su1q_u32(msg1, msg0);
+        msg2 = vsha1su0q_u32(msg2, msg3, msg0);
+
+        // Rounds 60-63
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg1, k3);
+        msg2 = vsha1su1q_u32(msg2, msg1);
+        msg3 = vsha1su0q_u32(msg3, msg0, msg1);
+
+        // Rounds 64-67
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e0, tmp0);
+        tmp0 = vaddq_u32(msg2, k3);
+        msg3 = vsha1su1q_u32(msg3, msg2);
+
+        // Rounds 68-71
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e1, tmp1);
+        tmp1 = vaddq_u32(msg3, k3);
+
+        // Rounds 72-75
+        e1 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e0, tmp0);
+
+        // Rounds 76-79
+        e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0));
+        abcd = vsha1pq_u32(abcd, e1, tmp1);
+
+        // Update state
+        abcd = vaddq_u32(abcd_cpy, abcd);
+        e0 = e0.wrapping_add(e0_cpy);
+    }
+
+    // Save state
+    vst1q_u32(state.as_mut_ptr(), abcd);
+    state[4] = e0;
+}
+
 pub fn compress(state: &mut [u32; 5], blocks: &[[u8; 64]]) {
-    // TODO: Replace with https://github.com/rust-lang/rfcs/pull/2725
-    // after stabilization
+    // TODO: Replace with https://github.com/rust-lang/rfcs/pull/2725 after stabilization
     if sha1_hwcap::get() {
-        sha1_asm::compress(state, blocks);
+        unsafe {
+            compress_sha1_neon(state, blocks);
+        }
     } else {
         super::soft::compress(state, blocks);
     }
diff --git a/sha1/src/compress/loongarch64_asm.rs b/sha1/src/compress/loongarch64_asm.rs
index facef1b19..69133e0d4 100644
--- a/sha1/src/compress/loongarch64_asm.rs
+++ b/sha1/src/compress/loongarch64_asm.rs
@@ -1,9 +1,8 @@
 //! LoongArch64 assembly backend
 
+use crate::K;
 use core::arch::asm;
 
-const K: [u32; 4] = [0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6];
-
 macro_rules! c {
     ($($l:expr)*) => {
         concat!($($l ,)*)
diff --git a/sha1/src/compress/soft.rs b/sha1/src/compress/soft.rs
index 0e28e5e2c..b2a1be7b9 100644
--- a/sha1/src/compress/soft.rs
+++ b/sha1/src/compress/soft.rs
@@ -1,7 +1,6 @@
 #![allow(clippy::many_single_char_names)]
 use super::BLOCK_SIZE;
-
-const K: [u32; 4] = [0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6];
+use crate::K;
 
 #[inline(always)]
 fn add(a: [u32; 4], b: [u32; 4]) -> [u32; 4] {
diff --git a/sha1/src/lib.rs b/sha1/src/lib.rs
index e77e9bfd7..5361f6042 100644
--- a/sha1/src/lib.rs
+++ b/sha1/src/lib.rs
@@ -33,6 +33,9 @@ pub use compress::compress;
 
 const STATE_LEN: usize = 5;
 const BLOCK_SIZE: usize = <Sha1Core as BlockSizeUser>::BlockSize::USIZE;
+const H0: [u32; STATE_LEN] = [0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0];
+#[allow(dead_code)]
+const K: [u32; 4] = [0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6];
 
 /// Core SHA-1 hasher state.
 #[derive(Clone)]
@@ -85,7 +88,7 @@ impl Default for Sha1Core {
     #[inline]
     fn default() -> Self {
         Self {
-            h: [0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0],
+            h: H0,
             block_len: 0,
         }
     }


Test patch: