#!/usr/bin/env perl
# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for ARMv4.
#
# June 2017.
#
# Non-NEON code is KECCAK_1X variant (see sha/keccak1600.c) with bit
# interleaving. How does it compare to Keccak Code Package? It's as
# fast, but several times smaller, and is endian- and ISA-neutral. ISA
# neutrality means that minimum ISA requirement is ARMv4, yet it can
# be assembled even as Thumb-2. NEON code path is KECCAK_1X_ALT with
# register layout taken from Keccak Code Package. It's also as fast,
# in fact faster by 10-15% on some processors, and endian-neutral.
#
# August 2017.
#
# Switch to KECCAK_2X variant for non-NEON code and merge almost 1/2
# of rotate instructions with logical ones. This resulted in ~10%
# improvement on most processors. Switch to KECCAK_2X effectively
# minimizes re-loads from temporary storage, and merged rotates just
# eliminate corresponding instructions. As for latter. When examining
# code you'll notice commented ror instructions. These are eliminated
# ones, and you should trace destination register below to see what's
# going on. Just in case, why not all rotates are eliminated. Trouble
# is that you have operations that require both inputs to be rotated,
# e.g. 'eor a,b>>>x,c>>>y'. This conundrum is resolved by using
# 'eor a,b,c>>>(x-y)' and then merge-rotating 'a' in next operation
# that takes 'a' as input. And thing is that this next operation can
# be in next round. It's totally possible to "carry" rotate "factors"
# to the next round, but it makes code more complex. And the last word
# is the keyword, i.e. "almost 1/2" is kind of complexity cap [for the
# time being]...
#
# Reduce per-round instruction count in Thumb-2 case by 16%. This is
# achieved by folding ldr/str pairs to their double-word counterparts.
# Theoretically this should have improved performance on single-issue
# cores, such as Cortex-A5/A7, by 19%. Reality is a bit different, as
# usual...
#
########################################################################
# Numbers are cycles per processed byte. Non-NEON results account even
# for input bit interleaving.
#
#		r=1088(*)   Thumb-2(**) NEON
#
# ARM11xx	82/+150%
# Cortex-A5	88/+160%,   86,         36
# Cortex-A7	78/+160%,   68,         34
# Cortex-A8	51/+230%,   57,         30
# Cortex-A9	53/+210%,   51,         26
# Cortex-A15	42/+160%,   38,         18
# Snapdragon S4	43/+210%,   38,         24
#
# (*)	Corresponds to SHA3-256. Percentage after slash is improvement
#	over compiler-generated KECCAK_2X reference code.
# (**)	Thumb-2 results for Cortex-A5/A7 are likely to apply even to
#	Cortex-Mx, x>=3. Otherwise, non-NEON results for NEON-capable
#	processors are presented mostly for reference purposes.

$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
    open STDOUT,">$output";
}

my @C = map("r$_",(0..9));
my @E = map("r$_",(10..12,14));

########################################################################
# Stack layout
# ----->+-----------------------+
#       | uint64_t A[5][5]      |
#       | ...                   |
# +200->+-----------------------+
#       | uint64_t D[5]         |
#       | ...                   |
# +240->+-----------------------+
#       | uint64_t T[5][5]      |
#       | ...                   |
# +440->+-----------------------+
#       | saved lr              |
# +444->+-----------------------+
#       | loop counter          |
# +448->+-----------------------+
#       | ...

my @A = map([ 8*$_, 8*($_+1), 8*($_+2), 8*($_+3), 8*($_+4) ], (0,5,10,15,20));
my @D = map(8*$_, (25..29));
my @T = map([ 8*$_, 8*($_+1), 8*($_+2), 8*($_+3), 8*($_+4) ], (30,35,40,45,50));

$code.=<<___;
#include "arm_arch.h"

.text

#if defined(__thumb2__)
.syntax	unified
.thumb
#else
.code	32
#endif

.type	iotas32, %object
.align	5
iotas32:
	.long	0x00000001, 0x00000000
	.long	0x00000000, 0x00000089
	.long	0x00000000, 0x8000008b
	.long	0x00000000, 0x80008080
	.long	0x00000001, 0x0000008b
	.long	0x00000001, 0x00008000
	.long	0x00000001, 0x80008088
	.long	0x00000001, 0x80000082
	.long	0x00000000, 0x0000000b
	.long	0x00000000, 0x0000000a
	.long	0x00000001, 0x00008082
	.long	0x00000000, 0x00008003
	.long	0x00000001, 0x0000808b
	.long	0x00000001, 0x8000000b
	.long	0x00000001, 0x8000008a
	.long	0x00000001, 0x80000081
	.long	0x00000000, 0x80000081
	.long	0x00000000, 0x80000008
	.long	0x00000000, 0x00000083
	.long	0x00000000, 0x80008003
	.long	0x00000001, 0x80008088
	.long	0x00000000, 0x80000088
	.long	0x00000001, 0x00008000
	.long	0x00000000, 0x80008082
.size	iotas32,.-iotas32

.type	KeccakF1600_int, %function
.align	5
KeccakF1600_int:
	add	@C[9],sp,#$A[4][2]
	add	@E[2],sp,#$A[0][0]
	add	@E[0],sp,#$A[1][0]
	ldmia	@C[9],{@C[4]-@C[9]}		@ A[4][2..4]
KeccakF1600_enter:
	str	lr,[sp,#440]
	eor	@E[1],@E[1],@E[1]
	str	@E[1],[sp,#444]
	b	.Lround2x

.align	4
.Lround2x:
___
sub Round {
my (@A,@R); (@A[0..4],@R) = @_;

$code.=<<___;
	ldmia	@E[2],{@C[0]-@C[3]}		@ A[0][0..1]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[1][0..1]
#ifdef	__thumb2__
	eor	@C[0],@C[0],@E[0]
	eor	@C[1],@C[1],@E[1]
	eor	@C[2],@C[2],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[1][2]]
	eor	@C[3],@C[3],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[1][3]]
	eor	@C[4],@C[4],@E[0]
	eor	@C[5],@C[5],@E[1]
	eor	@C[6],@C[6],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[1][4]]
	eor	@C[7],@C[7],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[2][0]]
	eor	@C[8],@C[8],@E[0]
	eor	@C[9],@C[9],@E[1]
	eor	@C[0],@C[0],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[2][1]]
	eor	@C[1],@C[1],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[2][2]]
	eor	@C[2],@C[2],@E[0]
	eor	@C[3],@C[3],@E[1]
	eor	@C[4],@C[4],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[2][3]]
	eor	@C[5],@C[5],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[2][4]]
	eor	@C[6],@C[6],@E[0]
	eor	@C[7],@C[7],@E[1]
	eor	@C[8],@C[8],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[3][0]]
	eor	@C[9],@C[9],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[3][1]]
	eor	@C[0],@C[0],@E[0]
	eor	@C[1],@C[1],@E[1]
	eor	@C[2],@C[2],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[3][2]]
	eor	@C[3],@C[3],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[3][3]]
	eor	@C[4],@C[4],@E[0]
	eor	@C[5],@C[5],@E[1]
	eor	@C[6],@C[6],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[3][4]]
	eor	@C[7],@C[7],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[4][0]]
	eor	@C[8],@C[8],@E[0]
	eor	@C[9],@C[9],@E[1]
	eor	@C[0],@C[0],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[4][1]]
	eor	@C[1],@C[1],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[0][2]]
	eor	@C[2],@C[2],@E[0]
	eor	@C[3],@C[3],@E[1]
	eor	@C[4],@C[4],@E[2]
	ldrd	@E[0],@E[1],[sp,#$A[0][3]]
	eor	@C[5],@C[5],@E[3]
	ldrd	@E[2],@E[3],[sp,#$A[0][4]]
#else
	eor	@C[0],@C[0],@E[0]
	 add	@E[0],sp,#$A[1][2]
	eor	@C[1],@C[1],@E[1]
	eor	@C[2],@C[2],@E[2]
	eor	@C[3],@C[3],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[1][2..3]
	eor	@C[4],@C[4],@E[0]
	 add	@E[0],sp,#$A[1][4]
	eor	@C[5],@C[5],@E[1]
	eor	@C[6],@C[6],@E[2]
	eor	@C[7],@C[7],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[1][4]..A[2][0]
	eor	@C[8],@C[8],@E[0]
	 add	@E[0],sp,#$A[2][1]
	eor	@C[9],@C[9],@E[1]
	eor	@C[0],@C[0],@E[2]
	eor	@C[1],@C[1],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[2][1..2]
	eor	@C[2],@C[2],@E[0]
	 add	@E[0],sp,#$A[2][3]
	eor	@C[3],@C[3],@E[1]
	eor	@C[4],@C[4],@E[2]
	eor	@C[5],@C[5],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[2][3..4]
	eor	@C[6],@C[6],@E[0]
	 add	@E[0],sp,#$A[3][0]
	eor	@C[7],@C[7],@E[1]
	eor	@C[8],@C[8],@E[2]
	eor	@C[9],@C[9],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[3][0..1]
	eor	@C[0],@C[0],@E[0]
	 add	@E[0],sp,#$A[3][2]
	eor	@C[1],@C[1],@E[1]
	eor	@C[2],@C[2],@E[2]
	eor	@C[3],@C[3],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[3][2..3]
	eor	@C[4],@C[4],@E[0]
	 add	@E[0],sp,#$A[3][4]
	eor	@C[5],@C[5],@E[1]
	eor	@C[6],@C[6],@E[2]
	eor	@C[7],@C[7],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[3][4]..A[4][0]
	eor	@C[8],@C[8],@E[0]
	ldr	@E[0],[sp,#$A[4][1]]		@ A[4][1]
	eor	@C[9],@C[9],@E[1]
	ldr	@E[1],[sp,#$A[4][1]+4]
	eor	@C[0],@C[0],@E[2]
	ldr	@E[2],[sp,#$A[0][2]]		@ A[0][2]
	eor	@C[1],@C[1],@E[3]
	ldr	@E[3],[sp,#$A[0][2]+4]
	eor	@C[2],@C[2],@E[0]
	 add	@E[0],sp,#$A[0][3]
	eor	@C[3],@C[3],@E[1]
	eor	@C[4],@C[4],@E[2]
	eor	@C[5],@C[5],@E[3]
	ldmia	@E[0],{@E[0]-@E[2],@E[3]}	@ A[0][3..4]
#endif
	eor	@C[6],@C[6],@E[0]
	eor	@C[7],@C[7],@E[1]
	eor	@C[8],@C[8],@E[2]
	eor	@C[9],@C[9],@E[3]

	eor	@E[0],@C[0],@C[5],ror#32-1	@ E[0] = ROL64(C[2], 1) ^ C[0];
	str.l	@E[0],[sp,#$D[1]]		@ D[1] = E[0]
	eor	@E[1],@C[1],@C[4]
	str.h	@E[1],[sp,#$D[1]+4]
	eor	@E[2],@C[6],@C[1],ror#32-1	@ E[1] = ROL64(C[0], 1) ^ C[3];
	eor	@E[3],@C[7],@C[0]
	str.l	@E[2],[sp,#$D[4]]		@ D[4] = E[1]
	eor	@C[0],@C[8],@C[3],ror#32-1	@ C[0] = ROL64(C[1], 1) ^ C[4];
	str.h	@E[3],[sp,#$D[4]+4]
	eor	@C[1],@C[9],@C[2]
	str.l	@C[0],[sp,#$D[0]]		@ D[0] = C[0]
	eor	@C[2],@C[2],@C[7],ror#32-1	@ C[1] = ROL64(C[3], 1) ^ C[1];
	 ldr.l	@C[7],[sp,#$A[3][3]]
	eor	@C[3],@C[3],@C[6]
	str.h	@C[1],[sp,#$D[0]+4]
	 ldr.h	@C[6],[sp,#$A[3][3]+4]
	str.l	@C[2],[sp,#$D[2]]		@ D[2] = C[1]
	eor	@C[4],@C[4],@C[9],ror#32-1	@ C[2] = ROL64(C[4], 1) ^ C[2];
	str.h	@C[3],[sp,#$D[2]+4]
	eor	@C[5],@C[5],@C[8]

	ldr.l	@C[8],[sp,#$A[4][4]]
	ldr.h	@C[9],[sp,#$A[4][4]+4]
	 str.l	@C[4],[sp,#$D[3]]		@ D[3] = C[2]
	eor	@C[7],@C[7],@C[4]
	 str.h	@C[5],[sp,#$D[3]+4]
	eor	@C[6],@C[6],@C[5]
	ldr.l	@C[4],[sp,#$A[0][0]]
	@ ror	@C[7],@C[7],#32-10		@ C[3] = ROL64(A[3][3] ^ C[2], rhotates[3][3]);   /* D[3] */
	@ ror	@C[6],@C[6],#32-11
	ldr.h	@C[5],[sp,#$A[0][0]+4]
	eor	@C[8],@C[8],@E[2]
	eor	@C[9],@C[9],@E[3]
	ldr.l	@E[2],[sp,#$A[2][2]]
	eor	@C[0],@C[0],@C[4]
	ldr.h	@E[3],[sp,#$A[2][2]+4]
	@ ror	@C[8],@C[8],#32-7		@ C[4] = ROL64(A[4][4] ^ E[1], rhotates[4][4]);   /* D[4] */
	@ ror	@C[9],@C[9],#32-7
	eor	@C[1],@C[1],@C[5]		@ C[0] =       A[0][0] ^ C[0]; /* rotate by 0 */  /* D[0] */
	eor	@E[2],@E[2],@C[2]
	ldr.l	@C[2],[sp,#$A[1][1]]
	eor	@E[3],@E[3],@C[3]
	ldr.h	@C[3],[sp,#$A[1][1]+4]
	ror	@C[5],@E[2],#32-21		@ C[2] = ROL64(A[2][2] ^ C[1], rhotates[2][2]);   /* D[2] */
	 ldr	@E[2],[sp,#444]			@ load counter
	eor	@C[2],@C[2],@E[0]
	 adr	@E[0],iotas32
	ror	@C[4],@E[3],#32-22
	 add	@E[3],@E[0],@E[2]
	eor	@C[3],@C[3],@E[1]
___
$code.=<<___	if ($A[0][0] != $T[0][0]);
	ldmia	@E[3],{@E[0],@E[1]}		@ iotas[i]
___
$code.=<<___	if ($A[0][0] == $T[0][0]);
	ldr.l	@E[0],[@E[3],#8]		@ iotas[i].lo
	add	@E[2],@E[2],#16
	ldr.h	@E[1],[@E[3],#12]		@ iotas[i].hi
	cmp	@E[2],#192
	str	@E[2],[sp,#444]			@ store counter
___
$code.=<<___;
	bic	@E[2],@C[4],@C[2],ror#32-22
	bic	@E[3],@C[5],@C[3],ror#32-22
	 ror	@C[2],@C[2],#32-22		@ C[1] = ROL64(A[1][1] ^ E[0], rhotates[1][1]);   /* D[1] */
	 ror	@C[3],@C[3],#32-22
	eor	@E[2],@E[2],@C[0]
	eor	@E[3],@E[3],@C[1]
	eor	@E[0],@E[0],@E[2]
	eor	@E[1],@E[1],@E[3]
	str.l	@E[0],[sp,#$R[0][0]]		@ R[0][0] = C[0] ^ (~C[1] & C[2]) ^ iotas[i];
	bic	@E[2],@C[6],@C[4],ror#11
	str.h	@E[1],[sp,#$R[0][0]+4]
	bic	@E[3],@C[7],@C[5],ror#10
	bic	@E[0],@C[8],@C[6],ror#32-(11-7)
	bic	@E[1],@C[9],@C[7],ror#32-(10-7)
	eor	@E[2],@C[2],@E[2],ror#32-11
	str.l	@E[2],[sp,#$R[0][1]]		@ R[0][1] = C[1] ^ (~C[2] & C[3]);
	eor	@E[3],@C[3],@E[3],ror#32-10
	str.h	@E[3],[sp,#$R[0][1]+4]
	eor	@E[0],@C[4],@E[0],ror#32-7
	eor	@E[1],@C[5],@E[1],ror#32-7
	str.l	@E[0],[sp,#$R[0][2]]		@ R[0][2] = C[2] ^ (~C[3] & C[4]);
	bic	@E[2],@C[0],@C[8],ror#32-7
	str.h	@E[1],[sp,#$R[0][2]+4]
	bic	@E[3],@C[1],@C[9],ror#32-7
	eor	@E[2],@E[2],@C[6],ror#32-11
	str.l	@E[2],[sp,#$R[0][3]]		@ R[0][3] = C[3] ^ (~C[4] & C[0]);
	eor	@E[3],@E[3],@C[7],ror#32-10
	str.h	@E[3],[sp,#$R[0][3]+4]
	bic	@E[0],@C[2],@C[0]
	 add	@E[3],sp,#$D[3]
	 ldr.l	@C[0],[sp,#$A[0][3]]		@ A[0][3]
	bic	@E[1],@C[3],@C[1]
	 ldr.h	@C[1],[sp,#$A[0][3]+4]
	eor	@E[0],@E[0],@C[8],ror#32-7
	eor	@E[1],@E[1],@C[9],ror#32-7
	str.l	@E[0],[sp,#$R[0][4]]		@ R[0][4] = C[4] ^ (~C[0] & C[1]);
	 add	@C[9],sp,#$D[0]
	str.h	@E[1],[sp,#$R[0][4]+4]

	ldmia	@E[3],{@E[0]-@E[2],@E[3]}	@ D[3..4]
	ldmia	@C[9],{@C[6]-@C[9]}		@ D[0..1]

	ldr.l	@C[2],[sp,#$A[1][4]]		@ A[1][4]
	eor	@C[0],@C[0],@E[0]
	ldr.h	@C[3],[sp,#$A[1][4]+4]
	eor	@C[1],@C[1],@E[1]
	@ ror	@C[0],@C[0],#32-14		@ C[0] = ROL64(A[0][3] ^ D[3], rhotates[0][3]);
	ldr.l	@E[0],[sp,#$A[3][1]]		@ A[3][1]
	@ ror	@C[1],@C[1],#32-14
	ldr.h	@E[1],[sp,#$A[3][1]+4]

	eor	@C[2],@C[2],@E[2]
	ldr.l	@C[4],[sp,#$A[2][0]]		@ A[2][0]
	eor	@C[3],@C[3],@E[3]
	ldr.h	@C[5],[sp,#$A[2][0]+4]
	@ ror	@C[2],@C[2],#32-10		@ C[1] = ROL64(A[1][4] ^ D[4], rhotates[1][4]);
	@ ror	@C[3],@C[3],#32-10

	eor	@C[6],@C[6],@C[4]
	ldr.l	@E[2],[sp,#$D[2]]		@ D[2]
	eor	@C[7],@C[7],@C[5]
	ldr.h	@E[3],[sp,#$D[2]+4]
	ror	@C[5],@C[6],#32-1		@ C[2] = ROL64(A[2][0] ^ D[0], rhotates[2][0]);
	ror	@C[4],@C[7],#32-2

	eor	@E[0],@E[0],@C[8]
	ldr.l	@C[8],[sp,#$A[4][2]]		@ A[4][2]
	eor	@E[1],@E[1],@C[9]
	ldr.h	@C[9],[sp,#$A[4][2]+4]
	ror	@C[7],@E[0],#32-22		@ C[3] = ROL64(A[3][1] ^ D[1], rhotates[3][1]);
	ror	@C[6],@E[1],#32-23

	bic	@E[0],@C[4],@C[2],ror#32-10
	bic	@E[1],@C[5],@C[3],ror#32-10
	 eor	@E[2],@E[2],@C[8]
	 eor	@E[3],@E[3],@C[9]
	 ror	@C[9],@E[2],#32-30		@ C[4] = ROL64(A[4][2] ^ D[2], rhotates[4][2]);
	 ror	@C[8],@E[3],#32-31
	eor	@E[0],@E[0],@C[0],ror#32-14
	eor	@E[1],@E[1],@C[1],ror#32-14
	str.l	@E[0],[sp,#$R[1][0]]		@ R[1][0] = C[0] ^ (~C[1] & C[2])
	bic	@E[2],@C[6],@C[4]
	str.h	@E[1],[sp,#$R[1][0]+4]
	bic	@E[3],@C[7],@C[5]
	eor	@E[2],@E[2],@C[2],ror#32-10
	str.l	@E[2],[sp,#$R[1][1]]		@ R[1][1] = C[1] ^ (~C[2] & C[3]);
	eor	@E[3],@E[3],@C[3],ror#32-10
	str.h	@E[3],[sp,#$R[1][1]+4]
	bic	@E[0],@C[8],@C[6]
	bic	@E[1],@C[9],@C[7]
	bic	@E[2],@C[0],@C[8],ror#14
	bic	@E[3],@C[1],@C[9],ror#14
	eor	@E[0],@E[0],@C[4]
	eor	@E[1],@E[1],@C[5]
	str.l	@E[0],[sp,#$R[1][2]]		@ R[1][2] = C[2] ^ (~C[3] & C[4]);
	bic	@C[2],@C[2],@C[0],ror#32-(14-10)
	str.h	@E[1],[sp,#$R[1][2]+4]
	eor	@E[2],@C[6],@E[2],ror#32-14
	bic	@E[1],@C[3],@C[1],ror#32-(14-10)
	str.l	@E[2],[sp,#$R[1][3]]		@ R[1][3] = C[3] ^ (~C[4] & C[0]);
	eor	@E[3],@C[7],@E[3],ror#32-14
	str.h	@E[3],[sp,#$R[1][3]+4]
	 add	@E[2],sp,#$D[1]
	 ldr.l	@C[1],[sp,#$A[0][1]]		@ A[0][1]
	eor	@E[0],@C[8],@C[2],ror#32-10
	 ldr.h	@C[0],[sp,#$A[0][1]+4]
	eor	@E[1],@C[9],@E[1],ror#32-10
	str.l	@E[0],[sp,#$R[1][4]]		@ R[1][4] = C[4] ^ (~C[0] & C[1]);
	str.h	@E[1],[sp,#$R[1][4]+4]

	add	@C[9],sp,#$D[3]
	ldmia	@E[2],{@E[0]-@E[2],@E[3]}	@ D[1..2]
	ldr.l	@C[2],[sp,#$A[1][2]]		@ A[1][2]
	ldr.h	@C[3],[sp,#$A[1][2]+4]
	ldmia	@C[9],{@C[6]-@C[9]}		@ D[3..4]

	eor	@C[1],@C[1],@E[0]
	ldr.l	@C[4],[sp,#$A[2][3]]		@ A[2][3]
	eor	@C[0],@C[0],@E[1]
	ldr.h	@C[5],[sp,#$A[2][3]+4]
	ror	@C[0],@C[0],#32-1		@ C[0] = ROL64(A[0][1] ^ D[1], rhotates[0][1]);

	eor	@C[2],@C[2],@E[2]
	ldr.l	@E[0],[sp,#$A[3][4]]		@ A[3][4]
	eor	@C[3],@C[3],@E[3]
	ldr.h	@E[1],[sp,#$A[3][4]+4]
	@ ror	@C[2],@C[2],#32-3		@ C[1] = ROL64(A[1][2] ^ D[2], rhotates[1][2]);
	ldr.l	@E[2],[sp,#$D[0]]		@ D[0]
	@ ror	@C[3],@C[3],#32-3
	ldr.h	@E[3],[sp,#$D[0]+4]

	eor	@C[4],@C[4],@C[6]
	eor	@C[5],@C[5],@C[7]
	@ ror	@C[5],@C[6],#32-12		@ C[2] = ROL64(A[2][3] ^ D[3], rhotates[2][3]);
	@ ror	@C[4],@C[7],#32-13		@ [track reverse order below]

	eor	@E[0],@E[0],@C[8]
	ldr.l	@C[8],[sp,#$A[4][0]]		@ A[4][0]
	eor	@E[1],@E[1],@C[9]
	ldr.h	@C[9],[sp,#$A[4][0]+4]
	ror	@C[6],@E[0],#32-4		@ C[3] = ROL64(A[3][4] ^ D[4], rhotates[3][4]);
	ror	@C[7],@E[1],#32-4

	eor	@E[2],@E[2],@C[8]
	eor	@E[3],@E[3],@C[9]
	ror	@C[8],@E[2],#32-9		@ C[4] = ROL64(A[4][0] ^ D[0], rhotates[4][0]);
	ror	@C[9],@E[3],#32-9

	bic	@E[0],@C[5],@C[2],ror#13-3
	bic	@E[1],@C[4],@C[3],ror#12-3
	bic	@E[2],@C[6],@C[5],ror#32-13
	bic	@E[3],@C[7],@C[4],ror#32-12
	eor	@E[0],@C[0],@E[0],ror#32-13
	eor	@E[1],@C[1],@E[1],ror#32-12
	str.l	@E[0],[sp,#$R[2][0]]		@ R[2][0] = C[0] ^ (~C[1] & C[2])
	eor	@E[2],@E[2],@C[2],ror#32-3
	str.h	@E[1],[sp,#$R[2][0]+4]
	eor	@E[3],@E[3],@C[3],ror#32-3
	str.l	@E[2],[sp,#$R[2][1]]		@ R[2][1] = C[1] ^ (~C[2] & C[3]);
	bic	@E[0],@C[8],@C[6]
	bic	@E[1],@C[9],@C[7]
	str.h	@E[3],[sp,#$R[2][1]+4]
	eor	@E[0],@E[0],@C[5],ror#32-13
	eor	@E[1],@E[1],@C[4],ror#32-12
	str.l	@E[0],[sp,#$R[2][2]]		@ R[2][2] = C[2] ^ (~C[3] & C[4]);
	bic	@E[2],@C[0],@C[8]
	str.h	@E[1],[sp,#$R[2][2]+4]
	bic	@E[3],@C[1],@C[9]
	eor	@E[2],@E[2],@C[6]
	eor	@E[3],@E[3],@C[7]
	str.l	@E[2],[sp,#$R[2][3]]		@ R[2][3] = C[3] ^ (~C[4] & C[0]);
	bic	@E[0],@C[2],@C[0],ror#3
	str.h	@E[3],[sp,#$R[2][3]+4]
	bic	@E[1],@C[3],@C[1],ror#3
	 ldr.l	@C[1],[sp,#$A[0][4]]		@ A[0][4] [in reverse order]
	eor	@E[0],@C[8],@E[0],ror#32-3
	 ldr.h	@C[0],[sp,#$A[0][4]+4]
	eor	@E[1],@C[9],@E[1],ror#32-3
	str.l	@E[0],[sp,#$R[2][4]]		@ R[2][4] = C[4] ^ (~C[0] & C[1]);
	 add	@C[9],sp,#$D[1]
	str.h	@E[1],[sp,#$R[2][4]+4]

	ldr.l	@E[0],[sp,#$D[4]]		@ D[4]
	ldr.h	@E[1],[sp,#$D[4]+4]
	ldr.l	@E[2],[sp,#$D[0]]		@ D[0]
	ldr.h	@E[3],[sp,#$D[0]+4]

	ldmia	@C[9],{@C[6]-@C[9]}		@ D[1..2]

	eor	@C[1],@C[1],@E[0]
	ldr.l	@C[2],[sp,#$A[1][0]]		@ A[1][0]
	eor	@C[0],@C[0],@E[1]
	ldr.h	@C[3],[sp,#$A[1][0]+4]
	@ ror	@C[1],@E[0],#32-13		@ C[0] = ROL64(A[0][4] ^ D[4], rhotates[0][4]);
	ldr.l	@C[4],[sp,#$A[2][1]]		@ A[2][1]
	@ ror	@C[0],@E[1],#32-14		@ [was loaded in reverse order]
	ldr.h	@C[5],[sp,#$A[2][1]+4]

	eor	@C[2],@C[2],@E[2]
	ldr.l	@E[0],[sp,#$A[3][2]]		@ A[3][2]
	eor	@C[3],@C[3],@E[3]
	ldr.h	@E[1],[sp,#$A[3][2]+4]
	@ ror	@C[2],@C[2],#32-18		@ C[1] = ROL64(A[1][0] ^ D[0], rhotates[1][0]);
	ldr.l	@E[2],[sp,#$D[3]]		@ D[3]
	@ ror	@C[3],@C[3],#32-18
	ldr.h	@E[3],[sp,#$D[3]+4]

	eor	@C[6],@C[6],@C[4]
	eor	@C[7],@C[7],@C[5]
	ror	@C[4],@C[6],#32-5		@ C[2] = ROL64(A[2][1] ^ D[1], rhotates[2][1]);
	ror	@C[5],@C[7],#32-5

	eor	@E[0],@E[0],@C[8]
	ldr.l	@C[8],[sp,#$A[4][3]]		@ A[4][3]
	eor	@E[1],@E[1],@C[9]
	ldr.h	@C[9],[sp,#$A[4][3]+4]
	ror	@C[7],@E[0],#32-7		@ C[3] = ROL64(A[3][2] ^ D[2], rhotates[3][2]);
	ror	@C[6],@E[1],#32-8

	eor	@E[2],@E[2],@C[8]
	eor	@E[3],@E[3],@C[9]
	ror	@C[8],@E[2],#32-28		@ C[4] = ROL64(A[4][3] ^ D[3], rhotates[4][3]);
	ror	@C[9],@E[3],#32-28

	bic	@E[0],@C[4],@C[2],ror#32-18
	bic	@E[1],@C[5],@C[3],ror#32-18
	eor	@E[0],@E[0],@C[0],ror#32-14
	eor	@E[1],@E[1],@C[1],ror#32-13
	str.l	@E[0],[sp,#$R[3][0]]		@ R[3][0] = C[0] ^ (~C[1] & C[2])
	bic	@E[2],@C[6],@C[4]
	str.h	@E[1],[sp,#$R[3][0]+4]
	bic	@E[3],@C[7],@C[5]
	eor	@E[2],@E[2],@C[2],ror#32-18
	str.l	@E[2],[sp,#$R[3][1]]		@ R[3][1] = C[1] ^ (~C[2] & C[3]);
	eor	@E[3],@E[3],@C[3],ror#32-18
	str.h	@E[3],[sp,#$R[3][1]+4]
	bic	@E[0],@C[8],@C[6]
	bic	@E[1],@C[9],@C[7]
	bic	@E[2],@C[0],@C[8],ror#14
	bic	@E[3],@C[1],@C[9],ror#13
	eor	@E[0],@E[0],@C[4]
	eor	@E[1],@E[1],@C[5]
	str.l	@E[0],[sp,#$R[3][2]]		@ R[3][2] = C[2] ^ (~C[3] & C[4]);
	bic	@C[2],@C[2],@C[0],ror#18-14
	str.h	@E[1],[sp,#$R[3][2]+4]
	eor	@E[2],@C[6],@E[2],ror#32-14
	bic	@E[1],@C[3],@C[1],ror#18-13
	eor	@E[3],@C[7],@E[3],ror#32-13
	str.l	@E[2],[sp,#$R[3][3]]		@ R[3][3] = C[3] ^ (~C[4] & C[0]);
	str.h	@E[3],[sp,#$R[3][3]+4]
	 add	@E[3],sp,#$D[2]
	 ldr.l	@C[0],[sp,#$A[0][2]]		@ A[0][2]
	eor	@E[0],@C[8],@C[2],ror#32-18
	 ldr.h	@C[1],[sp,#$A[0][2]+4]
	eor	@E[1],@C[9],@E[1],ror#32-18
	str.l	@E[0],[sp,#$R[3][4]]		@ R[3][4] = C[4] ^ (~C[0] & C[1]);
	str.h	@E[1],[sp,#$R[3][4]+4]

	ldmia	@E[3],{@E[0]-@E[2],@E[3]}	@ D[2..3]
	ldr.l	@C[2],[sp,#$A[1][3]]		@ A[1][3]
	ldr.h	@C[3],[sp,#$A[1][3]+4]
	ldr.l	@C[6],[sp,#$D[4]]		@ D[4]
	ldr.h	@C[7],[sp,#$D[4]+4]

	eor	@C[0],@C[0],@E[0]
	ldr.l	@C[4],[sp,#$A[2][4]]		@ A[2][4]
	eor	@C[1],@C[1],@E[1]
	ldr.h	@C[5],[sp,#$A[2][4]+4]
	@ ror	@C[0],@C[0],#32-31		@ C[0] = ROL64(A[0][2] ^ D[2], rhotates[0][2]);
	ldr.l	@C[8],[sp,#$D[0]]		@ D[0]
	@ ror	@C[1],@C[1],#32-31
	ldr.h	@C[9],[sp,#$D[0]+4]

	eor	@E[2],@E[2],@C[2]
	ldr.l	@E[0],[sp,#$A[3][0]]		@ A[3][0]
	eor	@E[3],@E[3],@C[3]
	ldr.h	@E[1],[sp,#$A[3][0]+4]
	ror	@C[3],@E[2],#32-27		@ C[1] = ROL64(A[1][3] ^ D[3], rhotates[1][3]);
	ldr.l	@E[2],[sp,#$D[1]]		@ D[1]
	ror	@C[2],@E[3],#32-28
	ldr.h	@E[3],[sp,#$D[1]+4]

	eor	@C[6],@C[6],@C[4]
	eor	@C[7],@C[7],@C[5]
	ror	@C[5],@C[6],#32-19		@ C[2] = ROL64(A[2][4] ^ D[4], rhotates[2][4]);
	ror	@C[4],@C[7],#32-20

	eor	@E[0],@E[0],@C[8]
	ldr.l	@C[8],[sp,#$A[4][1]]		@ A[4][1]
	eor	@E[1],@E[1],@C[9]
	ldr.h	@C[9],[sp,#$A[4][1]+4]
	ror	@C[7],@E[0],#32-20		@ C[3] = ROL64(A[3][0] ^ D[0], rhotates[3][0]);
	ror	@C[6],@E[1],#32-21

	eor	@C[8],@C[8],@E[2]
	eor	@C[9],@C[9],@E[3]
	@ ror	@C[8],@C[2],#32-1		@ C[4] = ROL64(A[4][1] ^ D[1], rhotates[4][1]);
	@ ror	@C[9],@C[3],#32-1

	bic	@E[0],@C[4],@C[2]
	bic	@E[1],@C[5],@C[3]
	eor	@E[0],@E[0],@C[0],ror#32-31
	str.l	@E[0],[sp,#$R[4][0]]		@ R[4][0] = C[0] ^ (~C[1] & C[2])
	eor	@E[1],@E[1],@C[1],ror#32-31
	str.h	@E[1],[sp,#$R[4][0]+4]
	bic	@E[2],@C[6],@C[4]
	bic	@E[3],@C[7],@C[5]
	eor	@E[2],@E[2],@C[2]
	eor	@E[3],@E[3],@C[3]
	str.l	@E[2],[sp,#$R[4][1]]		@ R[4][1] = C[1] ^ (~C[2] & C[3]);
	bic	@E[0],@C[8],@C[6],ror#1
	str.h	@E[3],[sp,#$R[4][1]+4]
	bic	@E[1],@C[9],@C[7],ror#1
	bic	@E[2],@C[0],@C[8],ror#31-1
	bic	@E[3],@C[1],@C[9],ror#31-1
	eor	@C[4],@C[4],@E[0],ror#32-1
	str.l	@C[4],[sp,#$R[4][2]]		@ R[4][2] = C[2] ^= (~C[3] & C[4]);
	eor	@C[5],@C[5],@E[1],ror#32-1
	str.h	@C[5],[sp,#$R[4][2]+4]
	eor	@C[6],@C[6],@E[2],ror#32-31
	eor	@C[7],@C[7],@E[3],ror#32-31
	str.l	@C[6],[sp,#$R[4][3]]		@ R[4][3] = C[3] ^= (~C[4] & C[0]);
	bic	@E[0],@C[2],@C[0],ror#32-31
	str.h	@C[7],[sp,#$R[4][3]+4]
	bic	@E[1],@C[3],@C[1],ror#32-31
	 add	@E[2],sp,#$R[0][0]
	eor	@C[8],@E[0],@C[8],ror#32-1
	 add	@E[0],sp,#$R[1][0]
	eor	@C[9],@E[1],@C[9],ror#32-1
	str.l	@C[8],[sp,#$R[4][4]]		@ R[4][4] = C[4] ^= (~C[0] & C[1]);
	str.h	@C[9],[sp,#$R[4][4]+4]
___
}
	Round(@A,@T);
	Round(@T,@A);
$code.=<<___;
	blo	.Lround2x

	ldr	pc,[sp,#440]
.size	KeccakF1600_int,.-KeccakF1600_int

.type	KeccakF1600, %function
.align	5
KeccakF1600:
	stmdb	sp!,{r0,r4-r11,lr}
	sub	sp,sp,#440+16			@ space for A[5][5],D[5],T[5][5],...

	add	@E[0],r0,#$A[1][0]
	add	@E[1],sp,#$A[1][0]
	ldmia	r0,    {@C[0]-@C[9]}		@ copy A[5][5] to stack
	stmia	sp,    {@C[0]-@C[9]}
	ldmia	@E[0]!,{@C[0]-@C[9]}
	stmia	@E[1]!,{@C[0]-@C[9]}
	ldmia	@E[0]!,{@C[0]-@C[9]}
	stmia	@E[1]!,{@C[0]-@C[9]}
	ldmia	@E[0]!,{@C[0]-@C[9]}
	stmia	@E[1]!,{@C[0]-@C[9]}
	ldmia	@E[0], {@C[0]-@C[9]}
	add	@E[2],sp,#$A[0][0]
	add	@E[0],sp,#$A[1][0]
	stmia	@E[1], {@C[0]-@C[9]}

	bl	KeccakF1600_enter

	ldr	@E[1], [sp,#440+16]		@ restore pointer to A
	ldmia	sp,    {@C[0]-@C[9]}
	stmia	@E[1]!,{@C[0]-@C[9]}		@ return A[5][5]
	ldmia	@E[0]!,{@C[0]-@C[9]}
	stmia	@E[1]!,{@C[0]-@C[9]}
	ldmia	@E[0]!,{@C[0]-@C[9]}
	stmia	@E[1]!,{@C[0]-@C[9]}
	ldmia	@E[0]!,{@C[0]-@C[9]}
	stmia	@E[1]!,{@C[0]-@C[9]}
	ldmia	@E[0], {@C[0]-@C[9]}
	stmia	@E[1], {@C[0]-@C[9]}

	add	sp,sp,#440+20
	ldmia	sp!,{r4-r11,pc}
.size	KeccakF1600,.-KeccakF1600
___
{ my ($A_flat,$inp,$len,$bsz) = map("r$_",(10..12,14));

########################################################################
# Stack layout
# ----->+-----------------------+
#       | uint64_t A[5][5]      |
#       | ...                   |
#       | ...                   |
# +456->+-----------------------+
#       | 0x55555555            |
# +460->+-----------------------+
#       | 0x33333333            |
# +464->+-----------------------+
#       | 0x0f0f0f0f            |
# +468->+-----------------------+
#       | 0x00ff00ff            |
# +472->+-----------------------+
#       | uint64_t *A           |
# +476->+-----------------------+
#       | const void *inp       |
# +480->+-----------------------+
#       | size_t len            |
# +484->+-----------------------+
#       | size_t bs             |
# +488->+-----------------------+
#       | ....

$code.=<<___;
.global	SHA3_absorb
.type	SHA3_absorb,%function
.align	5
SHA3_absorb:
	stmdb	sp!,{r0-r12,lr}
	sub	sp,sp,#456+16

	add	$A_flat,r0,#$A[1][0]
	@ mov	$inp,r1
	mov	$len,r2
	mov	$bsz,r3
	cmp	r2,r3
	blo	.Labsorb_abort

	add	$inp,sp,#0
	ldmia	r0,      {@C[0]-@C[9]}	@ copy A[5][5] to stack
	stmia	$inp!,   {@C[0]-@C[9]}
	ldmia	$A_flat!,{@C[0]-@C[9]}
	stmia	$inp!,   {@C[0]-@C[9]}
	ldmia	$A_flat!,{@C[0]-@C[9]}
	stmia	$inp!,   {@C[0]-@C[9]}
	ldmia	$A_flat!,{@C[0]-@C[9]}
	stmia	$inp!,   {@C[0]-@C[9]}
	ldmia	$A_flat!,{@C[0]-@C[9]}
	stmia	$inp,    {@C[0]-@C[9]}

	ldr	$inp,[sp,#476]		@ restore $inp
#ifdef	__thumb2__
	mov	r9,#0x00ff00ff
	mov	r8,#0x0f0f0f0f
	mov	r7,#0x33333333
	mov	r6,#0x55555555
#else
	mov	r6,#0x11		@ compose constants
	mov	r8,#0x0f
	mov	r9,#0xff
	orr	r6,r6,r6,lsl#8
	orr	r8,r8,r8,lsl#8
	orr	r6,r6,r6,lsl#16		@ 0x11111111
	orr	r9,r9,r9,lsl#16		@ 0x00ff00ff
	orr	r8,r8,r8,lsl#16		@ 0x0f0f0f0f
	orr	r7,r6,r6,lsl#1		@ 0x33333333
	orr	r6,r6,r6,lsl#2		@ 0x55555555
#endif
	str	r9,[sp,#468]
	str	r8,[sp,#464]
	str	r7,[sp,#460]
	str	r6,[sp,#456]
	b	.Loop_absorb

.align	4
.Loop_absorb:
	subs	r0,$len,$bsz
	blo	.Labsorbed
	add	$A_flat,sp,#0
	str	r0,[sp,#480]		@ save len - bsz

.align	4
.Loop_block:
	ldrb	r0,[$inp],#1
	ldrb	r1,[$inp],#1
	ldrb	r2,[$inp],#1
	ldrb	r3,[$inp],#1
	ldrb	r4,[$inp],#1
	orr	r0,r0,r1,lsl#8
	ldrb	r1,[$inp],#1
	orr	r0,r0,r2,lsl#16
	ldrb	r2,[$inp],#1
	orr	r0,r0,r3,lsl#24		@ lo
	ldrb	r3,[$inp],#1
	orr	r1,r4,r1,lsl#8
	orr	r1,r1,r2,lsl#16
	orr	r1,r1,r3,lsl#24		@ hi

	and	r2,r0,r6		@ &=0x55555555
	and	r0,r0,r6,lsl#1		@ &=0xaaaaaaaa
	and	r3,r1,r6		@ &=0x55555555
	and	r1,r1,r6,lsl#1		@ &=0xaaaaaaaa
	orr	r2,r2,r2,lsr#1
	orr	r0,r0,r0,lsl#1
	orr	r3,r3,r3,lsr#1
	orr	r1,r1,r1,lsl#1
	and	r2,r2,r7		@ &=0x33333333
	and	r0,r0,r7,lsl#2		@ &=0xcccccccc
	and	r3,r3,r7		@ &=0x33333333
	and	r1,r1,r7,lsl#2		@ &=0xcccccccc
	orr	r2,r2,r2,lsr#2
	orr	r0,r0,r0,lsl#2
	orr	r3,r3,r3,lsr#2
	orr	r1,r1,r1,lsl#2
	and	r2,r2,r8		@ &=0x0f0f0f0f
	and	r0,r0,r8,lsl#4		@ &=0xf0f0f0f0
	and	r3,r3,r8		@ &=0x0f0f0f0f
	and	r1,r1,r8,lsl#4		@ &=0xf0f0f0f0
	ldmia	$A_flat,{r4-r5}		@ A_flat[i]
	orr	r2,r2,r2,lsr#4
	orr	r0,r0,r0,lsl#4
	orr	r3,r3,r3,lsr#4
	orr	r1,r1,r1,lsl#4
	and	r2,r2,r9		@ &=0x00ff00ff
	and	r0,r0,r9,lsl#8		@ &=0xff00ff00
	and	r3,r3,r9		@ &=0x00ff00ff
	and	r1,r1,r9,lsl#8		@ &=0xff00ff00
	orr	r2,r2,r2,lsr#8
	orr	r0,r0,r0,lsl#8
	orr	r3,r3,r3,lsr#8
	orr	r1,r1,r1,lsl#8

	lsl	r2,r2,#16
	lsr	r1,r1,#16
	eor	r4,r4,r3,lsl#16
	eor	r5,r5,r0,lsr#16
	eor	r4,r4,r2,lsr#16
	eor	r5,r5,r1,lsl#16
	stmia	$A_flat!,{r4-r5}	@ A_flat[i++] ^= BitInterleave(inp[0..7])

	subs	$bsz,$bsz,#8
	bhi	.Loop_block

	str	$inp,[sp,#476]

	bl	KeccakF1600_int

	add	r14,sp,#456
	ldmia	r14,{r6-r12,r14}	@ restore constants and variables
	b	.Loop_absorb

.align	4
.Labsorbed:
	add	$inp,sp,#$A[1][0]
	ldmia	sp,      {@C[0]-@C[9]}
	stmia	$A_flat!,{@C[0]-@C[9]}	@ return A[5][5]
	ldmia	$inp!,   {@C[0]-@C[9]}
	stmia	$A_flat!,{@C[0]-@C[9]}
	ldmia	$inp!,   {@C[0]-@C[9]}
	stmia	$A_flat!,{@C[0]-@C[9]}
	ldmia	$inp!,   {@C[0]-@C[9]}
	stmia	$A_flat!,{@C[0]-@C[9]}
	ldmia	$inp,    {@C[0]-@C[9]}
	stmia	$A_flat, {@C[0]-@C[9]}

.Labsorb_abort:
	add	sp,sp,#456+32
	mov	r0,$len			@ return value
	ldmia	sp!,{r4-r12,pc}
.size	SHA3_absorb,.-SHA3_absorb
___
}
{ my ($out,$len,$A_flat,$bsz) = map("r$_", (4,5,10,12));

$code.=<<___;
.global	SHA3_squeeze
.type	SHA3_squeeze,%function
.align	5
SHA3_squeeze:
	stmdb	sp!,{r0,r3-r10,lr}

	mov	$A_flat,r0
	mov	$out,r1
	mov	$len,r2
	mov	$bsz,r3

#ifdef	__thumb2__
	mov	r9,#0x00ff00ff
	mov	r8,#0x0f0f0f0f
	mov	r7,#0x33333333
	mov	r6,#0x55555555
#else
	mov	r6,#0x11		@ compose constants
	mov	r8,#0x0f
	mov	r9,#0xff
	orr	r6,r6,r6,lsl#8
	orr	r8,r8,r8,lsl#8
	orr	r6,r6,r6,lsl#16		@ 0x11111111
	orr	r9,r9,r9,lsl#16		@ 0x00ff00ff
	orr	r8,r8,r8,lsl#16		@ 0x0f0f0f0f
	orr	r7,r6,r6,lsl#1		@ 0x33333333
	orr	r6,r6,r6,lsl#2		@ 0x55555555
#endif
	stmdb	sp!,{r6-r9}

	mov	r14,$A_flat
	b	.Loop_squeeze

.align	4
.Loop_squeeze:
	ldmia	$A_flat!,{r0,r1}	@ A_flat[i++]

	lsl	r2,r0,#16
	lsl	r3,r1,#16		@ r3 = r1 << 16
	lsr	r2,r2,#16		@ r2 = r0 & 0x0000ffff
	lsr	r1,r1,#16
	lsr	r0,r0,#16		@ r0 = r0 >> 16
	lsl	r1,r1,#16		@ r1 = r1 & 0xffff0000

	orr	r2,r2,r2,lsl#8
	orr	r3,r3,r3,lsr#8
	orr	r0,r0,r0,lsl#8
	orr	r1,r1,r1,lsr#8
	and	r2,r2,r9		@ &=0x00ff00ff
	and	r3,r3,r9,lsl#8		@ &=0xff00ff00
	and	r0,r0,r9		@ &=0x00ff00ff
	and	r1,r1,r9,lsl#8		@ &=0xff00ff00
	orr	r2,r2,r2,lsl#4
	orr	r3,r3,r3,lsr#4
	orr	r0,r0,r0,lsl#4
	orr	r1,r1,r1,lsr#4
	and	r2,r2,r8		@ &=0x0f0f0f0f
	and	r3,r3,r8,lsl#4		@ &=0xf0f0f0f0
	and	r0,r0,r8		@ &=0x0f0f0f0f
	and	r1,r1,r8,lsl#4		@ &=0xf0f0f0f0
	orr	r2,r2,r2,lsl#2
	orr	r3,r3,r3,lsr#2
	orr	r0,r0,r0,lsl#2
	orr	r1,r1,r1,lsr#2
	and	r2,r2,r7		@ &=0x33333333
	and	r3,r3,r7,lsl#2		@ &=0xcccccccc
	and	r0,r0,r7		@ &=0x33333333
	and	r1,r1,r7,lsl#2		@ &=0xcccccccc
	orr	r2,r2,r2,lsl#1
	orr	r3,r3,r3,lsr#1
	orr	r0,r0,r0,lsl#1
	orr	r1,r1,r1,lsr#1
	and	r2,r2,r6		@ &=0x55555555
	and	r3,r3,r6,lsl#1		@ &=0xaaaaaaaa
	and	r0,r0,r6		@ &=0x55555555
	and	r1,r1,r6,lsl#1		@ &=0xaaaaaaaa

	orr	r2,r2,r3
	orr	r0,r0,r1

	cmp	$len,#8
	blo	.Lsqueeze_tail
	lsr	r1,r2,#8
	strb	r2,[$out],#1
	lsr	r3,r2,#16
	strb	r1,[$out],#1
	lsr	r2,r2,#24
	strb	r3,[$out],#1
	strb	r2,[$out],#1

	lsr	r1,r0,#8
	strb	r0,[$out],#1
	lsr	r3,r0,#16
	strb	r1,[$out],#1
	lsr	r0,r0,#24
	strb	r3,[$out],#1
	strb	r0,[$out],#1
	subs	$len,$len,#8
	beq	.Lsqueeze_done

	subs	$bsz,$bsz,#8		@ bsz -= 8
	bhi	.Loop_squeeze

	mov	r0,r14			@ original $A_flat

	bl	KeccakF1600

	ldmia	sp,{r6-r10,r12}		@ restore constants and variables
	mov	r14,$A_flat
	b	.Loop_squeeze

.align	4
.Lsqueeze_tail:
	strb	r2,[$out],#1
	lsr	r2,r2,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	r2,[$out],#1
	lsr	r2,r2,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	r2,[$out],#1
	lsr	r2,r2,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	r2,[$out],#1
	subs	$len,$len,#1
	beq	.Lsqueeze_done

	strb	r0,[$out],#1
	lsr	r0,r0,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	r0,[$out],#1
	lsr	r0,r0,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	r0,[$out]
	b	.Lsqueeze_done

.align	4
.Lsqueeze_done:
	add	sp,sp,#24
	ldmia	sp!,{r4-r10,pc}
.size	SHA3_squeeze,.-SHA3_squeeze
___
}

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.fpu	neon

.type	iotas64, %object
.align 5
iotas64:
	.quad	0x0000000000000001
	.quad	0x0000000000008082
	.quad	0x800000000000808a
	.quad	0x8000000080008000
	.quad	0x000000000000808b
	.quad	0x0000000080000001
	.quad	0x8000000080008081
	.quad	0x8000000000008009
	.quad	0x000000000000008a
	.quad	0x0000000000000088
	.quad	0x0000000080008009
	.quad	0x000000008000000a
	.quad	0x000000008000808b
	.quad	0x800000000000008b
	.quad	0x8000000000008089
	.quad	0x8000000000008003
	.quad	0x8000000000008002
	.quad	0x8000000000000080
	.quad	0x000000000000800a
	.quad	0x800000008000000a
	.quad	0x8000000080008081
	.quad	0x8000000000008080
	.quad	0x0000000080000001
	.quad	0x8000000080008008
.size	iotas64,.-iotas64

.type	KeccakF1600_neon, %function
.align	5
KeccakF1600_neon:
	add	r1, r0, #16
	adr	r2, iotas64
	mov	r3, #24			@ loop counter
	b	.Loop_neon

.align	4
.Loop_neon:
	@ Theta
	vst1.64		{q4},  [r0,:64]		@ offload A[0..1][4]
	veor		q13, q0,  q5		@ A[0..1][0]^A[2..3][0]
	vst1.64		{d18}, [r1,:64]		@ offload A[2][4]
	veor		q14, q1,  q6		@ A[0..1][1]^A[2..3][1]
	veor		q15, q2,  q7		@ A[0..1][2]^A[2..3][2]
	veor		d26, d26, d27		@ C[0]=A[0][0]^A[1][0]^A[2][0]^A[3][0]
	veor		d27, d28, d29		@ C[1]=A[0][1]^A[1][1]^A[2][1]^A[3][1]
	veor		q14, q3,  q8		@ A[0..1][3]^A[2..3][3]
	veor		q4,  q4,  q9		@ A[0..1][4]^A[2..3][4]
	veor		d30, d30, d31		@ C[2]=A[0][2]^A[1][2]^A[2][2]^A[3][2]
	veor		d31, d28, d29		@ C[3]=A[0][3]^A[1][3]^A[2][3]^A[3][3]
	veor		d25, d8,  d9		@ C[4]=A[0][4]^A[1][4]^A[2][4]^A[3][4]
	veor		q13, q13, q10		@ C[0..1]^=A[4][0..1]
	veor		q14, q15, q11		@ C[2..3]^=A[4][2..3]
	veor		d25, d25, d24		@ C[4]^=A[4][4]

	vadd.u64	q4,  q13, q13		@ C[0..1]<<1
	vadd.u64	q15, q14, q14		@ C[2..3]<<1
	vadd.u64	d18, d25, d25		@ C[4]<<1
	vsri.u64	q4,  q13, #63		@ ROL64(C[0..1],1)
	vsri.u64	q15, q14, #63		@ ROL64(C[2..3],1)
	vsri.u64	d18, d25, #63		@ ROL64(C[4],1)
	veor		d25, d25, d9		@ D[0] = C[4] ^= ROL64(C[1],1)
	veor		q13, q13, q15		@ D[1..2] = C[0..1] ^ ROL64(C[2..3],1)
	veor		d28, d28, d18		@ D[3] = C[2] ^= ROL64(C[4],1)
	veor		d29, d29, d8		@ D[4] = C[3] ^= ROL64(C[0],1)

	veor		d0,  d0,  d25		@ A[0][0] ^= C[4]
	veor		d1,  d1,  d25		@ A[1][0] ^= C[4]
	veor		d10, d10, d25		@ A[2][0] ^= C[4]
	veor		d11, d11, d25		@ A[3][0] ^= C[4]
	veor		d20, d20, d25		@ A[4][0] ^= C[4]

	veor		d2,  d2,  d26		@ A[0][1] ^= D[1]
	veor		d3,  d3,  d26		@ A[1][1] ^= D[1]
	veor		d12, d12, d26		@ A[2][1] ^= D[1]
	veor		d13, d13, d26		@ A[3][1] ^= D[1]
	veor		d21, d21, d26		@ A[4][1] ^= D[1]
	vmov		d26, d27

	veor		d6,  d6,  d28		@ A[0][3] ^= C[2]
	veor		d7,  d7,  d28		@ A[1][3] ^= C[2]
	veor		d16, d16, d28		@ A[2][3] ^= C[2]
	veor		d17, d17, d28		@ A[3][3] ^= C[2]
	veor		d23, d23, d28		@ A[4][3] ^= C[2]
	vld1.64		{q4},  [r0,:64]		@ restore A[0..1][4]
	vmov		d28, d29

	vld1.64		{d18}, [r1,:64]		@ restore A[2][4]
	veor		q2,  q2,  q13		@ A[0..1][2] ^= D[2]
	veor		q7,  q7,  q13		@ A[2..3][2] ^= D[2]
	veor		d22, d22, d27		@ A[4][2]    ^= D[2]

	veor		q4,  q4,  q14		@ A[0..1][4] ^= C[3]
	veor		q9,  q9,  q14		@ A[2..3][4] ^= C[3]
	veor		d24, d24, d29		@ A[4][4]    ^= C[3]

	@ Rho + Pi
	vmov		d26, d2			@ C[1] = A[0][1]
	vshl.u64	d2,  d3,  #44
	vmov		d27, d4			@ C[2] = A[0][2]
	vshl.u64	d4,  d14, #43
	vmov		d28, d6			@ C[3] = A[0][3]
	vshl.u64	d6,  d17, #21
	vmov		d29, d8			@ C[4] = A[0][4]
	vshl.u64	d8,  d24, #14
	vsri.u64	d2,  d3,  #64-44	@ A[0][1] = ROL64(A[1][1], rhotates[1][1])
	vsri.u64	d4,  d14, #64-43	@ A[0][2] = ROL64(A[2][2], rhotates[2][2])
	vsri.u64	d6,  d17, #64-21	@ A[0][3] = ROL64(A[3][3], rhotates[3][3])
	vsri.u64	d8,  d24, #64-14	@ A[0][4] = ROL64(A[4][4], rhotates[4][4])

	vshl.u64	d3,  d9,  #20
	vshl.u64	d14, d16, #25
	vshl.u64	d17, d15, #15
	vshl.u64	d24, d21, #2
	vsri.u64	d3,  d9,  #64-20	@ A[1][1] = ROL64(A[1][4], rhotates[1][4])
	vsri.u64	d14, d16, #64-25	@ A[2][2] = ROL64(A[2][3], rhotates[2][3])
	vsri.u64	d17, d15, #64-15	@ A[3][3] = ROL64(A[3][2], rhotates[3][2])
	vsri.u64	d24, d21, #64-2		@ A[4][4] = ROL64(A[4][1], rhotates[4][1])

	vshl.u64	d9,  d22, #61
	@ vshl.u64	d16, d19, #8
	vshl.u64	d15, d12, #10
	vshl.u64	d21, d7,  #55
	vsri.u64	d9,  d22, #64-61	@ A[1][4] = ROL64(A[4][2], rhotates[4][2])
	vext.8		d16, d19, d19, #8-1	@ A[2][3] = ROL64(A[3][4], rhotates[3][4])
	vsri.u64	d15, d12, #64-10	@ A[3][2] = ROL64(A[2][1], rhotates[2][1])
	vsri.u64	d21, d7,  #64-55	@ A[4][1] = ROL64(A[1][3], rhotates[1][3])

	vshl.u64	d22, d18, #39
	@ vshl.u64	d19, d23, #56
	vshl.u64	d12, d5,  #6
	vshl.u64	d7,  d13, #45
	vsri.u64	d22, d18, #64-39	@ A[4][2] = ROL64(A[2][4], rhotates[2][4])
	vext.8		d19, d23, d23, #8-7	@ A[3][4] = ROL64(A[4][3], rhotates[4][3])
	vsri.u64	d12, d5,  #64-6		@ A[2][1] = ROL64(A[1][2], rhotates[1][2])
	vsri.u64	d7,  d13, #64-45	@ A[1][3] = ROL64(A[3][1], rhotates[3][1])

	vshl.u64	d18, d20, #18
	vshl.u64	d23, d11, #41
	vshl.u64	d5,  d10, #3
	vshl.u64	d13, d1,  #36
	vsri.u64	d18, d20, #64-18	@ A[2][4] = ROL64(A[4][0], rhotates[4][0])
	vsri.u64	d23, d11, #64-41	@ A[4][3] = ROL64(A[3][0], rhotates[3][0])
	vsri.u64	d5,  d10, #64-3		@ A[1][2] = ROL64(A[2][0], rhotates[2][0])
	vsri.u64	d13, d1,  #64-36	@ A[3][1] = ROL64(A[1][0], rhotates[1][0])

	vshl.u64	d1,  d28, #28
	vshl.u64	d10, d26, #1
	vshl.u64	d11, d29, #27
	vshl.u64	d20, d27, #62
	vsri.u64	d1,  d28, #64-28	@ A[1][0] = ROL64(C[3],    rhotates[0][3])
	vsri.u64	d10, d26, #64-1		@ A[2][0] = ROL64(C[1],    rhotates[0][1])
	vsri.u64	d11, d29, #64-27	@ A[3][0] = ROL64(C[4],    rhotates[0][4])
	vsri.u64	d20, d27, #64-62	@ A[4][0] = ROL64(C[2],    rhotates[0][2])

	@ Chi + Iota
	vbic		q13, q2,  q1
	vbic		q14, q3,  q2
	vbic		q15, q4,  q3
	veor		q13, q13, q0		@ A[0..1][0] ^ (~A[0..1][1] & A[0..1][2])
	veor		q14, q14, q1		@ A[0..1][1] ^ (~A[0..1][2] & A[0..1][3])
	veor		q2,  q2,  q15		@ A[0..1][2] ^= (~A[0..1][3] & A[0..1][4])
	vst1.64		{q13}, [r0,:64]		@ offload A[0..1][0]
	vbic		q13, q0,  q4
	vbic		q15, q1,  q0
	vmov		q1,  q14		@ A[0..1][1]
	veor		q3,  q3,  q13		@ A[0..1][3] ^= (~A[0..1][4] & A[0..1][0])
	veor		q4,  q4,  q15		@ A[0..1][4] ^= (~A[0..1][0] & A[0..1][1])

	vbic		q13, q7,  q6
	vmov		q0,  q5			@ A[2..3][0]
	vbic		q14, q8,  q7
	vmov		q15, q6			@ A[2..3][1]
	veor		q5,  q5,  q13		@ A[2..3][0] ^= (~A[2..3][1] & A[2..3][2])
	vbic		q13, q9,  q8
	veor		q6,  q6,  q14		@ A[2..3][1] ^= (~A[2..3][2] & A[2..3][3])
	vbic		q14, q0,  q9
	veor		q7,  q7,  q13		@ A[2..3][2] ^= (~A[2..3][3] & A[2..3][4])
	vbic		q13, q15, q0
	veor		q8,  q8,  q14		@ A[2..3][3] ^= (~A[2..3][4] & A[2..3][0])
	vmov		q14, q10		@ A[4][0..1]
	veor		q9,  q9,  q13		@ A[2..3][4] ^= (~A[2..3][0] & A[2..3][1])

	vld1.64		d25, [r2,:64]!		@ Iota[i++]
	vbic		d26, d22, d21
	vbic		d27, d23, d22
	vld1.64		{q0}, [r0,:64]		@ restore A[0..1][0]
	veor		d20, d20, d26		@ A[4][0] ^= (~A[4][1] & A[4][2])
	vbic		d26, d24, d23
	veor		d21, d21, d27		@ A[4][1] ^= (~A[4][2] & A[4][3])
	vbic		d27, d28, d24
	veor		d22, d22, d26		@ A[4][2] ^= (~A[4][3] & A[4][4])
	vbic		d26, d29, d28
	veor		d23, d23, d27		@ A[4][3] ^= (~A[4][4] & A[4][0])
	veor		d0,  d0,  d25		@ A[0][0] ^= Iota[i]
	veor		d24, d24, d26		@ A[4][4] ^= (~A[4][0] & A[4][1])

	subs	r3, r3, #1
	bne	.Loop_neon

	bx	lr
.size	KeccakF1600_neon,.-KeccakF1600_neon

.global	SHA3_absorb_neon
.type	SHA3_absorb_neon, %function
.align	5
SHA3_absorb_neon:
	stmdb	sp!, {r4-r6,lr}
	vstmdb	sp!, {d8-d15}

	mov	r4, r1			@ inp
	mov	r5, r2			@ len
	mov	r6, r3			@ bsz

	vld1.32	{d0}, [r0,:64]!		@ A[0][0]
	vld1.32	{d2}, [r0,:64]!		@ A[0][1]
	vld1.32	{d4}, [r0,:64]!		@ A[0][2]
	vld1.32	{d6}, [r0,:64]!		@ A[0][3]
	vld1.32	{d8}, [r0,:64]!		@ A[0][4]

	vld1.32	{d1}, [r0,:64]!		@ A[1][0]
	vld1.32	{d3}, [r0,:64]!		@ A[1][1]
	vld1.32	{d5}, [r0,:64]!		@ A[1][2]
	vld1.32	{d7}, [r0,:64]!		@ A[1][3]
	vld1.32	{d9}, [r0,:64]!		@ A[1][4]

	vld1.32	{d10}, [r0,:64]!		@ A[2][0]
	vld1.32	{d12}, [r0,:64]!		@ A[2][1]
	vld1.32	{d14}, [r0,:64]!		@ A[2][2]
	vld1.32	{d16}, [r0,:64]!		@ A[2][3]
	vld1.32	{d18}, [r0,:64]!		@ A[2][4]

	vld1.32	{d11}, [r0,:64]!		@ A[3][0]
	vld1.32	{d13}, [r0,:64]!		@ A[3][1]
	vld1.32	{d15}, [r0,:64]!		@ A[3][2]
	vld1.32	{d17}, [r0,:64]!		@ A[3][3]
	vld1.32	{d19}, [r0,:64]!		@ A[3][4]

	vld1.32	{d20-d23}, [r0,:64]!	@ A[4][0..3]
	vld1.32	{d24}, [r0,:64]		@ A[4][4]
	sub	r0, r0, #24*8		@ rewind
	b	.Loop_absorb_neon

.align	4
.Loop_absorb_neon:
	subs	r12, r5, r6		@ len - bsz
	blo	.Labsorbed_neon
	mov	r5, r12

	vld1.8	{d31}, [r4]!		@ endian-neutral loads...
	cmp	r6, #8*2
	veor	d0, d0, d31		@ A[0][0] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d2, d2, d31		@ A[0][1] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*4
	veor	d4, d4, d31		@ A[0][2] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d6, d6, d31		@ A[0][3] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31},[r4]!
	cmp	r6, #8*6
	veor	d8, d8, d31		@ A[0][4] ^= *inp++
	blo	.Lprocess_neon

	vld1.8	{d31}, [r4]!
	veor	d1, d1, d31		@ A[1][0] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*8
	veor	d3, d3, d31		@ A[1][1] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d5, d5, d31		@ A[1][2] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*10
	veor	d7, d7, d31		@ A[1][3] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d9, d9, d31		@ A[1][4] ^= *inp++
	beq	.Lprocess_neon

	vld1.8	{d31}, [r4]!
	cmp	r6, #8*12
	veor	d10, d10, d31		@ A[2][0] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d12, d12, d31		@ A[2][1] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*14
	veor	d14, d14, d31		@ A[2][2] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d16, d16, d31		@ A[2][3] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*16
	veor	d18, d18, d31		@ A[2][4] ^= *inp++
	blo	.Lprocess_neon

	vld1.8	{d31}, [r4]!
	veor	d11, d11, d31		@ A[3][0] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*18
	veor	d13, d13, d31		@ A[3][1] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d15, d15, d31		@ A[3][2] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*20
	veor	d17, d17, d31		@ A[3][3] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d19, d19, d31		@ A[3][4] ^= *inp++
	beq	.Lprocess_neon

	vld1.8	{d31}, [r4]!
	cmp	r6, #8*22
	veor	d20, d20, d31		@ A[4][0] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d21, d21, d31		@ A[4][1] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	cmp	r6, #8*24
	veor	d22, d22, d31		@ A[4][2] ^= *inp++
	blo	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d23, d23, d31		@ A[4][3] ^= *inp++
	beq	.Lprocess_neon
	vld1.8	{d31}, [r4]!
	veor	d24, d24, d31		@ A[4][4] ^= *inp++

.Lprocess_neon:
	bl	KeccakF1600_neon
	b 	.Loop_absorb_neon

.align	4
.Labsorbed_neon:
	vst1.32	{d0}, [r0,:64]!		@ A[0][0..4]
	vst1.32	{d2}, [r0,:64]!
	vst1.32	{d4}, [r0,:64]!
	vst1.32	{d6}, [r0,:64]!
	vst1.32	{d8}, [r0,:64]!

	vst1.32	{d1}, [r0,:64]!		@ A[1][0..4]
	vst1.32	{d3}, [r0,:64]!
	vst1.32	{d5}, [r0,:64]!
	vst1.32	{d7}, [r0,:64]!
	vst1.32	{d9}, [r0,:64]!

	vst1.32	{d10}, [r0,:64]!		@ A[2][0..4]
	vst1.32	{d12}, [r0,:64]!
	vst1.32	{d14}, [r0,:64]!
	vst1.32	{d16}, [r0,:64]!
	vst1.32	{d18}, [r0,:64]!

	vst1.32	{d11}, [r0,:64]!		@ A[3][0..4]
	vst1.32	{d13}, [r0,:64]!
	vst1.32	{d15}, [r0,:64]!
	vst1.32	{d17}, [r0,:64]!
	vst1.32	{d19}, [r0,:64]!

	vst1.32	{d20-d23}, [r0,:64]!	@ A[4][0..4]
	vst1.32	{d24}, [r0,:64]

	mov	r0, r5			@ return value
	vldmia	sp!, {d8-d15}
	ldmia	sp!, {r4-r6,pc}
.size	SHA3_absorb_neon,.-SHA3_absorb_neon

.global	SHA3_squeeze_neon
.type	SHA3_squeeze_neon, %function
.align	5
SHA3_squeeze_neon:
	stmdb	sp!, {r4-r6,lr}

	mov	r4, r1			@ out
	mov	r5, r2			@ len
	mov	r6, r3			@ bsz
	mov	r12, r0			@ A_flat
	mov	r14, r3			@ bsz
	b	.Loop_squeeze_neon

.align	4
.Loop_squeeze_neon:
	cmp	r5, #8
	blo	.Lsqueeze_neon_tail
	vld1.32	{d0}, [r12]!
	vst1.8	{d0}, [r4]!		@ endian-neutral store

	subs	r5, r5, #8		@ len -= 8
	beq	.Lsqueeze_neon_done

	subs	r14, r14, #8		@ bsz -= 8
	bhi	.Loop_squeeze_neon

	vstmdb	sp!,  {d8-d15}

	vld1.32	{d0}, [r0,:64]!		@ A[0][0..4]
	vld1.32	{d2}, [r0,:64]!
	vld1.32	{d4}, [r0,:64]!
	vld1.32	{d6}, [r0,:64]!
	vld1.32	{d8}, [r0,:64]!

	vld1.32	{d1}, [r0,:64]!		@ A[1][0..4]
	vld1.32	{d3}, [r0,:64]!
	vld1.32	{d5}, [r0,:64]!
	vld1.32	{d7}, [r0,:64]!
	vld1.32	{d9}, [r0,:64]!

	vld1.32	{d10}, [r0,:64]!		@ A[2][0..4]
	vld1.32	{d12}, [r0,:64]!
	vld1.32	{d14}, [r0,:64]!
	vld1.32	{d16}, [r0,:64]!
	vld1.32	{d18}, [r0,:64]!

	vld1.32	{d11}, [r0,:64]!		@ A[3][0..4]
	vld1.32	{d13}, [r0,:64]!
	vld1.32	{d15}, [r0,:64]!
	vld1.32	{d17}, [r0,:64]!
	vld1.32	{d19}, [r0,:64]!

	vld1.32	{d20-d23}, [r0,:64]!	@ A[4][0..4]
	vld1.32	{d24}, [r0,:64]
	sub	r0, r0, #24*8		@ rewind

	bl	KeccakF1600_neon

	mov	r12, r0			@ A_flat
	vst1.32	{d0}, [r0,:64]!		@ A[0][0..4]
	vst1.32	{d2}, [r0,:64]!
	vst1.32	{d4}, [r0,:64]!
	vst1.32	{d6}, [r0,:64]!
	vst1.32	{d8}, [r0,:64]!

	vst1.32	{d1}, [r0,:64]!		@ A[1][0..4]
	vst1.32	{d3}, [r0,:64]!
	vst1.32	{d5}, [r0,:64]!
	vst1.32	{d7}, [r0,:64]!
	vst1.32	{d9}, [r0,:64]!

	vst1.32	{d10}, [r0,:64]!		@ A[2][0..4]
	vst1.32	{d12}, [r0,:64]!
	vst1.32	{d14}, [r0,:64]!
	vst1.32	{d16}, [r0,:64]!
	vst1.32	{d18}, [r0,:64]!

	vst1.32	{d11}, [r0,:64]!		@ A[3][0..4]
	vst1.32	{d13}, [r0,:64]!
	vst1.32	{d15}, [r0,:64]!
	vst1.32	{d17}, [r0,:64]!
	vst1.32	{d19}, [r0,:64]!

	vst1.32	{d20-d23}, [r0,:64]!	@ A[4][0..4]
	mov	r14, r6			@ bsz
	vst1.32	{d24}, [r0,:64]
	mov	r0,  r12		@ rewind

	vldmia	sp!, {d8-d15}
	b	.Loop_squeeze_neon

.align	4
.Lsqueeze_neon_tail:
	ldmia	r12, {r2,r3}
	cmp	r5, #2
	strb	r2, [r4],#1		@ endian-neutral store
	lsr	r2, r2, #8
	blo	.Lsqueeze_neon_done
	strb	r2, [r4], #1
	lsr	r2, r2, #8
	beq	.Lsqueeze_neon_done
	strb	r2, [r4], #1
	lsr	r2, r2, #8
	cmp	r5, #4
	blo	.Lsqueeze_neon_done
	strb	r2, [r4], #1
	beq	.Lsqueeze_neon_done

	strb	r3, [r4], #1
	lsr	r3, r3, #8
	cmp	r5, #6
	blo	.Lsqueeze_neon_done
	strb	r3, [r4], #1
	lsr	r3, r3, #8
	beq	.Lsqueeze_neon_done
	strb	r3, [r4], #1

.Lsqueeze_neon_done:
	ldmia	sp!, {r4-r6,pc}
.size	SHA3_squeeze_neon,.-SHA3_squeeze_neon
#endif
.asciz	"Keccak-1600 absorb and squeeze for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
___

{
    my %ldr, %str;

    sub ldrd {
	my ($mnemonic,$half,$reg,$ea) = @_;
	my $op = $mnemonic eq "ldr" ? \%ldr : \%str;

	if ($half eq "l") {
	    $$op{reg} = $reg;
	    $$op{ea}  = $ea;
	    sprintf "#ifndef	__thumb2__\n"	.
		    "	%s\t%s,%s\n"		.
		    "#endif", $mnemonic,$reg,$ea;
	} else {
	    sprintf "#ifndef	__thumb2__\n"	.
		    "	%s\t%s,%s\n"		.
		    "#else\n"			.
		    "	%sd\t%s,%s,%s\n"	.
		    "#endif",	$mnemonic,$reg,$ea,
				$mnemonic,$$op{reg},$reg,$$op{ea};
	}
    }
}

foreach (split($/,$code)) {
	s/\`([^\`]*)\`/eval $1/ge;

	s/^\s+(ldr|str)\.([lh])\s+(r[0-9]+),\s*(\[.*)/ldrd($1,$2,$3,$4)/ge or
	s/\b(ror|ls[rl])\s+(r[0-9]+.*)#/mov	$2$1#/g or
	s/\bret\b/bx	lr/g		or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/g;	# make it possible to compile with -march=armv4

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#!/usr/bin/env perl
# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for ARMv8.
#
# June 2017.
#
# This is straightforward KECCAK_1X_ALT implementation. It makes no
# sense to attempt SIMD/NEON implementation for following reason.
# 64-bit lanes of vector registers can't be addressed as easily as in
# 32-bit mode. This means that 64-bit NEON is bound to be slower than
# 32-bit NEON, and this implementation is faster than 32-bit NEON on
# same processor. Even though it takes more scalar xor's and andn's,
# it gets compensated by availability of rotate. Not to forget that
# most processors achieve higher issue rate with scalar instructions.
#
# February 2018.
#
# Add hardware-assisted ARMv8.2 implementation. It's KECCAK_1X_ALT
# variant with register permutation/rotation twist that allows to
# eliminate copies to temporary registers. If you look closely you'll
# notice that it uses only one lane of vector registers. The new
# instructions effectively facilitate parallel hashing, which we don't
# support [yet?]. But lowest-level core procedure is prepared for it.
# The inner round is 67 [vector] instructions, so it's not actually
# obvious that it will provide performance improvement [in serial
# hash] as long as vector instructions issue rate is limited to 1 per
# cycle...
#
######################################################################
# Numbers are cycles per processed byte.
#
#		r=1088(*)
#
# Cortex-A53	13
# Cortex-A57	12
# X-Gene	14
# Mongoose	10
# Kryo		12
# Denver	7.8
# Apple A7	7.2
#
# (*)	Corresponds to SHA3-256. No improvement coefficients are listed
#	because they vary too much from compiler to compiler. Newer
#	compiler does much better and improvement varies from 5% on
#	Cortex-A57 to 25% on Cortex-A53. While in comparison to older
#	compiler this code is at least 2x faster...

$flavour = shift;
$output  = shift;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;

my @rhotates = ([  0,  1, 62, 28, 27 ],
                [ 36, 44,  6, 55, 20 ],
                [  3, 10, 43, 25, 39 ],
                [ 41, 45, 15, 21,  8 ],
                [ 18,  2, 61, 56, 14 ]);

$code.=<<___;
.text

.align 8	// strategic alignment and padding that allows to use
		// address value as loop termination condition...
	.quad	0,0,0,0,0,0,0,0
.type	iotas,%object
iotas:
	.quad	0x0000000000000001
	.quad	0x0000000000008082
	.quad	0x800000000000808a
	.quad	0x8000000080008000
	.quad	0x000000000000808b
	.quad	0x0000000080000001
	.quad	0x8000000080008081
	.quad	0x8000000000008009
	.quad	0x000000000000008a
	.quad	0x0000000000000088
	.quad	0x0000000080008009
	.quad	0x000000008000000a
	.quad	0x000000008000808b
	.quad	0x800000000000008b
	.quad	0x8000000000008089
	.quad	0x8000000000008003
	.quad	0x8000000000008002
	.quad	0x8000000000000080
	.quad	0x000000000000800a
	.quad	0x800000008000000a
	.quad	0x8000000080008081
	.quad	0x8000000000008080
	.quad	0x0000000080000001
	.quad	0x8000000080008008
.size	iotas,.-iotas
___
								{{{
my @A = map([ "x$_", "x".($_+1), "x".($_+2), "x".($_+3), "x".($_+4) ],
            (0, 5, 10, 15, 20));
   $A[3][3] = "x25"; # x18 is reserved

my @C = map("x$_", (26,27,28,30));

$code.=<<___;
.type	KeccakF1600_int,%function
.align	5
KeccakF1600_int:
	adr	$C[2],iotas
	.inst	0xd503233f			// paciasp
	stp	$C[2],x30,[sp,#16]		// 32 bytes on top are mine
	b	.Loop
.align	4
.Loop:
	////////////////////////////////////////// Theta
	eor	$C[0],$A[0][0],$A[1][0]
	stp	$A[0][4],$A[1][4],[sp,#0]	// offload pair...
	eor	$C[1],$A[0][1],$A[1][1]
	eor	$C[2],$A[0][2],$A[1][2]
	eor	$C[3],$A[0][3],$A[1][3]
___
	$C[4]=$A[0][4];
	$C[5]=$A[1][4];
$code.=<<___;
	eor	$C[4],$A[0][4],$A[1][4]
	eor	$C[0],$C[0],$A[2][0]
	eor	$C[1],$C[1],$A[2][1]
	eor	$C[2],$C[2],$A[2][2]
	eor	$C[3],$C[3],$A[2][3]
	eor	$C[4],$C[4],$A[2][4]
	eor	$C[0],$C[0],$A[3][0]
	eor	$C[1],$C[1],$A[3][1]
	eor	$C[2],$C[2],$A[3][2]
	eor	$C[3],$C[3],$A[3][3]
	eor	$C[4],$C[4],$A[3][4]
	eor	$C[0],$C[0],$A[4][0]
	eor	$C[2],$C[2],$A[4][2]
	eor	$C[1],$C[1],$A[4][1]
	eor	$C[3],$C[3],$A[4][3]
	eor	$C[4],$C[4],$A[4][4]

	eor	$C[5],$C[0],$C[2],ror#63

	eor	$A[0][1],$A[0][1],$C[5]
	eor	$A[1][1],$A[1][1],$C[5]
	eor	$A[2][1],$A[2][1],$C[5]
	eor	$A[3][1],$A[3][1],$C[5]
	eor	$A[4][1],$A[4][1],$C[5]

	eor	$C[5],$C[1],$C[3],ror#63
	eor	$C[2],$C[2],$C[4],ror#63
	eor	$C[3],$C[3],$C[0],ror#63
	eor	$C[4],$C[4],$C[1],ror#63

	eor	$C[1],   $A[0][2],$C[5]		// mov	$C[1],$A[0][2]
	eor	$A[1][2],$A[1][2],$C[5]
	eor	$A[2][2],$A[2][2],$C[5]
	eor	$A[3][2],$A[3][2],$C[5]
	eor	$A[4][2],$A[4][2],$C[5]

	eor	$A[0][0],$A[0][0],$C[4]
	eor	$A[1][0],$A[1][0],$C[4]
	eor	$A[2][0],$A[2][0],$C[4]
	eor	$A[3][0],$A[3][0],$C[4]
	eor	$A[4][0],$A[4][0],$C[4]
___
	$C[4]=undef;
	$C[5]=undef;
$code.=<<___;
	ldp	$A[0][4],$A[1][4],[sp,#0]	// re-load offloaded data
	eor	$C[0],   $A[0][3],$C[2]		// mov	$C[0],$A[0][3]
	eor	$A[1][3],$A[1][3],$C[2]
	eor	$A[2][3],$A[2][3],$C[2]
	eor	$A[3][3],$A[3][3],$C[2]
	eor	$A[4][3],$A[4][3],$C[2]

	eor	$C[2],   $A[0][4],$C[3]		// mov	$C[2],$A[0][4]
	eor	$A[1][4],$A[1][4],$C[3]
	eor	$A[2][4],$A[2][4],$C[3]
	eor	$A[3][4],$A[3][4],$C[3]
	eor	$A[4][4],$A[4][4],$C[3]

	////////////////////////////////////////// Rho+Pi
	mov	$C[3],$A[0][1]
	ror	$A[0][1],$A[1][1],#64-$rhotates[1][1]
	//mov	$C[1],$A[0][2]
	ror	$A[0][2],$A[2][2],#64-$rhotates[2][2]
	//mov	$C[0],$A[0][3]
	ror	$A[0][3],$A[3][3],#64-$rhotates[3][3]
	//mov	$C[2],$A[0][4]
	ror	$A[0][4],$A[4][4],#64-$rhotates[4][4]

	ror	$A[1][1],$A[1][4],#64-$rhotates[1][4]
	ror	$A[2][2],$A[2][3],#64-$rhotates[2][3]
	ror	$A[3][3],$A[3][2],#64-$rhotates[3][2]
	ror	$A[4][4],$A[4][1],#64-$rhotates[4][1]

	ror	$A[1][4],$A[4][2],#64-$rhotates[4][2]
	ror	$A[2][3],$A[3][4],#64-$rhotates[3][4]
	ror	$A[3][2],$A[2][1],#64-$rhotates[2][1]
	ror	$A[4][1],$A[1][3],#64-$rhotates[1][3]

	ror	$A[4][2],$A[2][4],#64-$rhotates[2][4]
	ror	$A[3][4],$A[4][3],#64-$rhotates[4][3]
	ror	$A[2][1],$A[1][2],#64-$rhotates[1][2]
	ror	$A[1][3],$A[3][1],#64-$rhotates[3][1]

	ror	$A[2][4],$A[4][0],#64-$rhotates[4][0]
	ror	$A[4][3],$A[3][0],#64-$rhotates[3][0]
	ror	$A[1][2],$A[2][0],#64-$rhotates[2][0]
	ror	$A[3][1],$A[1][0],#64-$rhotates[1][0]

	ror	$A[1][0],$C[0],#64-$rhotates[0][3]
	ror	$A[2][0],$C[3],#64-$rhotates[0][1]
	ror	$A[3][0],$C[2],#64-$rhotates[0][4]
	ror	$A[4][0],$C[1],#64-$rhotates[0][2]

	////////////////////////////////////////// Chi+Iota
	bic	$C[0],$A[0][2],$A[0][1]
	bic	$C[1],$A[0][3],$A[0][2]
	bic	$C[2],$A[0][0],$A[0][4]
	bic	$C[3],$A[0][1],$A[0][0]
	eor	$A[0][0],$A[0][0],$C[0]
	bic	$C[0],$A[0][4],$A[0][3]
	eor	$A[0][1],$A[0][1],$C[1]
	 ldr	$C[1],[sp,#16]
	eor	$A[0][3],$A[0][3],$C[2]
	eor	$A[0][4],$A[0][4],$C[3]
	eor	$A[0][2],$A[0][2],$C[0]
	 ldr	$C[3],[$C[1]],#8		// Iota[i++]

	bic	$C[0],$A[1][2],$A[1][1]
	 tst	$C[1],#255			// are we done?
	 str	$C[1],[sp,#16]
	bic	$C[1],$A[1][3],$A[1][2]
	bic	$C[2],$A[1][0],$A[1][4]
	 eor	$A[0][0],$A[0][0],$C[3]		// A[0][0] ^= Iota
	bic	$C[3],$A[1][1],$A[1][0]
	eor	$A[1][0],$A[1][0],$C[0]
	bic	$C[0],$A[1][4],$A[1][3]
	eor	$A[1][1],$A[1][1],$C[1]
	eor	$A[1][3],$A[1][3],$C[2]
	eor	$A[1][4],$A[1][4],$C[3]
	eor	$A[1][2],$A[1][2],$C[0]

	bic	$C[0],$A[2][2],$A[2][1]
	bic	$C[1],$A[2][3],$A[2][2]
	bic	$C[2],$A[2][0],$A[2][4]
	bic	$C[3],$A[2][1],$A[2][0]
	eor	$A[2][0],$A[2][0],$C[0]
	bic	$C[0],$A[2][4],$A[2][3]
	eor	$A[2][1],$A[2][1],$C[1]
	eor	$A[2][3],$A[2][3],$C[2]
	eor	$A[2][4],$A[2][4],$C[3]
	eor	$A[2][2],$A[2][2],$C[0]

	bic	$C[0],$A[3][2],$A[3][1]
	bic	$C[1],$A[3][3],$A[3][2]
	bic	$C[2],$A[3][0],$A[3][4]
	bic	$C[3],$A[3][1],$A[3][0]
	eor	$A[3][0],$A[3][0],$C[0]
	bic	$C[0],$A[3][4],$A[3][3]
	eor	$A[3][1],$A[3][1],$C[1]
	eor	$A[3][3],$A[3][3],$C[2]
	eor	$A[3][4],$A[3][4],$C[3]
	eor	$A[3][2],$A[3][2],$C[0]

	bic	$C[0],$A[4][2],$A[4][1]
	bic	$C[1],$A[4][3],$A[4][2]
	bic	$C[2],$A[4][0],$A[4][4]
	bic	$C[3],$A[4][1],$A[4][0]
	eor	$A[4][0],$A[4][0],$C[0]
	bic	$C[0],$A[4][4],$A[4][3]
	eor	$A[4][1],$A[4][1],$C[1]
	eor	$A[4][3],$A[4][3],$C[2]
	eor	$A[4][4],$A[4][4],$C[3]
	eor	$A[4][2],$A[4][2],$C[0]

	bne	.Loop

	ldr	x30,[sp,#24]
	.inst	0xd50323bf			// autiasp
	ret
.size	KeccakF1600_int,.-KeccakF1600_int

.type	KeccakF1600,%function
.align	5
KeccakF1600:
	.inst	0xd503233f			// paciasp
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]
	sub	sp,sp,#48

	str	x0,[sp,#32]			// offload argument
	mov	$C[0],x0
	ldp	$A[0][0],$A[0][1],[x0,#16*0]
	ldp	$A[0][2],$A[0][3],[$C[0],#16*1]
	ldp	$A[0][4],$A[1][0],[$C[0],#16*2]
	ldp	$A[1][1],$A[1][2],[$C[0],#16*3]
	ldp	$A[1][3],$A[1][4],[$C[0],#16*4]
	ldp	$A[2][0],$A[2][1],[$C[0],#16*5]
	ldp	$A[2][2],$A[2][3],[$C[0],#16*6]
	ldp	$A[2][4],$A[3][0],[$C[0],#16*7]
	ldp	$A[3][1],$A[3][2],[$C[0],#16*8]
	ldp	$A[3][3],$A[3][4],[$C[0],#16*9]
	ldp	$A[4][0],$A[4][1],[$C[0],#16*10]
	ldp	$A[4][2],$A[4][3],[$C[0],#16*11]
	ldr	$A[4][4],[$C[0],#16*12]

	bl	KeccakF1600_int

	ldr	$C[0],[sp,#32]
	stp	$A[0][0],$A[0][1],[$C[0],#16*0]
	stp	$A[0][2],$A[0][3],[$C[0],#16*1]
	stp	$A[0][4],$A[1][0],[$C[0],#16*2]
	stp	$A[1][1],$A[1][2],[$C[0],#16*3]
	stp	$A[1][3],$A[1][4],[$C[0],#16*4]
	stp	$A[2][0],$A[2][1],[$C[0],#16*5]
	stp	$A[2][2],$A[2][3],[$C[0],#16*6]
	stp	$A[2][4],$A[3][0],[$C[0],#16*7]
	stp	$A[3][1],$A[3][2],[$C[0],#16*8]
	stp	$A[3][3],$A[3][4],[$C[0],#16*9]
	stp	$A[4][0],$A[4][1],[$C[0],#16*10]
	stp	$A[4][2],$A[4][3],[$C[0],#16*11]
	str	$A[4][4],[$C[0],#16*12]

	ldp	x19,x20,[x29,#16]
	add	sp,sp,#48
	ldp	x21,x22,[x29,#32]
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldp	x29,x30,[sp],#128
	.inst	0xd50323bf			// autiasp
	ret
.size	KeccakF1600,.-KeccakF1600

.globl	SHA3_absorb
.type	SHA3_absorb,%function
.align	5
SHA3_absorb:
	.inst	0xd503233f			// paciasp
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]
	sub	sp,sp,#64

	stp	x0,x1,[sp,#32]			// offload arguments
	stp	x2,x3,[sp,#48]

	mov	$C[0],x0			// uint64_t A[5][5]
	mov	$C[1],x1			// const void *inp
	mov	$C[2],x2			// size_t len
	mov	$C[3],x3			// size_t bsz
	ldp	$A[0][0],$A[0][1],[$C[0],#16*0]
	ldp	$A[0][2],$A[0][3],[$C[0],#16*1]
	ldp	$A[0][4],$A[1][0],[$C[0],#16*2]
	ldp	$A[1][1],$A[1][2],[$C[0],#16*3]
	ldp	$A[1][3],$A[1][4],[$C[0],#16*4]
	ldp	$A[2][0],$A[2][1],[$C[0],#16*5]
	ldp	$A[2][2],$A[2][3],[$C[0],#16*6]
	ldp	$A[2][4],$A[3][0],[$C[0],#16*7]
	ldp	$A[3][1],$A[3][2],[$C[0],#16*8]
	ldp	$A[3][3],$A[3][4],[$C[0],#16*9]
	ldp	$A[4][0],$A[4][1],[$C[0],#16*10]
	ldp	$A[4][2],$A[4][3],[$C[0],#16*11]
	ldr	$A[4][4],[$C[0],#16*12]
	b	.Loop_absorb

.align	4
.Loop_absorb:
	subs	$C[0],$C[2],$C[3]		// len - bsz
	blo	.Labsorbed

	str	$C[0],[sp,#48]			// save len - bsz
___
for (my $i=0; $i<24; $i+=2) {
my $j = $i+1;
$code.=<<___;
	ldr	$C[0],[$C[1]],#8		// *inp++
#ifdef	__AARCH64EB__
	rev	$C[0],$C[0]
#endif
	eor	$A[$i/5][$i%5],$A[$i/5][$i%5],$C[0]
	cmp	$C[3],#8*($i+2)
	blo	.Lprocess_block
	ldr	$C[0],[$C[1]],#8		// *inp++
#ifdef	__AARCH64EB__
	rev	$C[0],$C[0]
#endif
	eor	$A[$j/5][$j%5],$A[$j/5][$j%5],$C[0]
	beq	.Lprocess_block
___
}
$code.=<<___;
	ldr	$C[0],[$C[1]],#8		// *inp++
#ifdef	__AARCH64EB__
	rev	$C[0],$C[0]
#endif
	eor	$A[4][4],$A[4][4],$C[0]

.Lprocess_block:
	str	$C[1],[sp,#40]			// save inp

	bl	KeccakF1600_int

	ldr	$C[1],[sp,#40]			// restore arguments
	ldp	$C[2],$C[3],[sp,#48]
	b	.Loop_absorb

.align	4
.Labsorbed:
	ldr	$C[1],[sp,#32]
	stp	$A[0][0],$A[0][1],[$C[1],#16*0]
	stp	$A[0][2],$A[0][3],[$C[1],#16*1]
	stp	$A[0][4],$A[1][0],[$C[1],#16*2]
	stp	$A[1][1],$A[1][2],[$C[1],#16*3]
	stp	$A[1][3],$A[1][4],[$C[1],#16*4]
	stp	$A[2][0],$A[2][1],[$C[1],#16*5]
	stp	$A[2][2],$A[2][3],[$C[1],#16*6]
	stp	$A[2][4],$A[3][0],[$C[1],#16*7]
	stp	$A[3][1],$A[3][2],[$C[1],#16*8]
	stp	$A[3][3],$A[3][4],[$C[1],#16*9]
	stp	$A[4][0],$A[4][1],[$C[1],#16*10]
	stp	$A[4][2],$A[4][3],[$C[1],#16*11]
	str	$A[4][4],[$C[1],#16*12]

	mov	x0,$C[2]			// return value
	ldp	x19,x20,[x29,#16]
	add	sp,sp,#64
	ldp	x21,x22,[x29,#32]
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldp	x29,x30,[sp],#128
	.inst	0xd50323bf			// autiasp
	ret
.size	SHA3_absorb,.-SHA3_absorb
___
{
my ($A_flat,$out,$len,$bsz) = map("x$_",(19..22));
$code.=<<___;
.globl	SHA3_squeeze
.type	SHA3_squeeze,%function
.align	5
SHA3_squeeze:
	.inst	0xd503233f			// paciasp
	stp	x29,x30,[sp,#-48]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]

	mov	$A_flat,x0			// put aside arguments
	mov	$out,x1
	mov	$len,x2
	mov	$bsz,x3

.Loop_squeeze:
	ldr	x4,[x0],#8
	cmp	$len,#8
	blo	.Lsqueeze_tail
#ifdef	__AARCH64EB__
	rev	x4,x4
#endif
	str	x4,[$out],#8
	subs	$len,$len,#8
	beq	.Lsqueeze_done

	subs	x3,x3,#8
	bhi	.Loop_squeeze

	mov	x0,$A_flat
	bl	KeccakF1600
	mov	x0,$A_flat
	mov	x3,$bsz
	b	.Loop_squeeze

.align	4
.Lsqueeze_tail:
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1

.Lsqueeze_done:
	ldp	x19,x20,[sp,#16]
	ldp	x21,x22,[sp,#32]
	ldp	x29,x30,[sp],#48
	.inst	0xd50323bf			// autiasp
	ret
.size	SHA3_squeeze,.-SHA3_squeeze
___
}								}}}
								{{{
my @A = map([ "v".$_.".16b", "v".($_+1).".16b", "v".($_+2).".16b",
                             "v".($_+3).".16b", "v".($_+4).".16b" ],
            (0, 5, 10, 15, 20));

my @C = map("v$_.16b", (25..31));

$code.=<<___;
.type	KeccakF1600_ce,%function
.align	5
KeccakF1600_ce:
	mov	x9,#12
	adr	x10,iotas
	b	.Loop_ce
.align	4
.Loop_ce:
___
for($i=0; $i<2; $i++) {
$code.=<<___;
	////////////////////////////////////////////////// Theta
	eor3	$C[0],$A[0][0],$A[1][0],$A[2][0]
	eor3	$C[1],$A[0][1],$A[1][1],$A[2][1]
	eor3	$C[2],$A[0][2],$A[1][2],$A[2][2]
	eor3	$C[3],$A[0][3],$A[1][3],$A[2][3]
	eor3	$C[4],$A[0][4],$A[1][4],$A[2][4]
	eor3	$C[0],$C[0],   $A[3][0],$A[4][0]
	eor3	$C[1],$C[1],   $A[3][1],$A[4][1]
	eor3	$C[2],$C[2],   $A[3][2],$A[4][2]
	eor3	$C[3],$C[3],   $A[3][3],$A[4][3]
	eor3	$C[4],$C[4],   $A[3][4],$A[4][4]

	rax1	$C[5],$C[0],$C[2]			// D[1]
	rax1	$C[6],$C[1],$C[3]			// D[2]
	rax1	$C[2],$C[2],$C[4]			// D[3]
	rax1	$C[3],$C[3],$C[0]			// D[4]
	rax1	$C[4],$C[4],$C[1]			// D[0]

	////////////////////////////////////////////////// Theta+Rho+Pi
	xar	$C[0],   $A[1][1],$C[5],#64-$rhotates[1][1]	// C[0]=A[0][1]
	xar	$A[1][1],$A[1][4],$C[3],#64-$rhotates[1][4]
	xar	$A[1][4],$A[4][2],$C[6],#64-$rhotates[4][2]
	xar	$A[4][2],$A[2][4],$C[3],#64-$rhotates[2][4]
	xar	$A[2][4],$A[4][0],$C[4],#64-$rhotates[4][0]

	xar	$A[4][0],$A[0][2],$C[6],#64-$rhotates[0][2]

	xar	$A[0][2],$A[2][2],$C[6],#64-$rhotates[2][2]
	xar	$A[2][2],$A[2][3],$C[2],#64-$rhotates[2][3]
	xar	$A[2][3],$A[3][4],$C[3],#64-$rhotates[3][4]
	xar	$A[3][4],$A[4][3],$C[2],#64-$rhotates[4][3]
	xar	$A[4][3],$A[3][0],$C[4],#64-$rhotates[3][0]

	xar	$A[3][0],$A[0][4],$C[3],#64-$rhotates[0][4]

	eor	$A[0][0],$A[0][0],$C[4]
	ldr	x11,[x10],#8

	xar	$C[1],   $A[3][3],$C[2],#64-$rhotates[3][3]	// C[1]=A[0][3]
	xar	$A[3][3],$A[3][2],$C[6],#64-$rhotates[3][2]
	xar	$A[3][2],$A[2][1],$C[5],#64-$rhotates[2][1]
	xar	$A[2][1],$A[1][2],$C[6],#64-$rhotates[1][2]
	xar	$A[1][2],$A[2][0],$C[4],#64-$rhotates[2][0]

	xar	$A[2][0],$A[0][1],$C[5],#64-$rhotates[0][1]	// *

	xar	$A[0][4],$A[4][4],$C[3],#64-$rhotates[4][4]
	xar	$A[4][4],$A[4][1],$C[5],#64-$rhotates[4][1]
	xar	$A[4][1],$A[1][3],$C[2],#64-$rhotates[1][3]
	xar	$A[1][3],$A[3][1],$C[5],#64-$rhotates[3][1]
	xar	$A[3][1],$A[1][0],$C[4],#64-$rhotates[1][0]

	xar	$C[2],   $A[0][3],$C[2],#64-$rhotates[0][3]	// C[2]=A[1][0]

	////////////////////////////////////////////////// Chi+Iota
	dup	$C[6],x11				// borrow C[6]
	bcax	$C[3],   $A[0][0],$A[0][2],$C[0]	// *
	bcax	$A[0][1],$C[0],   $C[1],   $A[0][2]	// *
	bcax	$A[0][2],$A[0][2],$A[0][4],$C[1]
	bcax	$A[0][3],$C[1],   $A[0][0],$A[0][4]
	bcax	$A[0][4],$A[0][4],$C[0],   $A[0][0]

	bcax	$A[1][0],$C[2],   $A[1][2],$A[1][1]	// *
	bcax	$C[0],   $A[1][1],$A[1][3],$A[1][2]	// *
	bcax	$A[1][2],$A[1][2],$A[1][4],$A[1][3]
	bcax	$A[1][3],$A[1][3],$C[2],   $A[1][4]
	bcax	$A[1][4],$A[1][4],$A[1][1],$C[2]

	eor	$A[0][0],$C[3],$C[6]			// Iota

	bcax	$C[1],   $A[2][0],$A[2][2],$A[2][1]	// *
	bcax	$C[2],   $A[2][1],$A[2][3],$A[2][2]	// *
	bcax	$A[2][2],$A[2][2],$A[2][4],$A[2][3]
	bcax	$A[2][3],$A[2][3],$A[2][0],$A[2][4]
	bcax	$A[2][4],$A[2][4],$A[2][1],$A[2][0]

	bcax	$C[3],   $A[3][0],$A[3][2],$A[3][1]	// *
	bcax	$C[4],   $A[3][1],$A[3][3],$A[3][2]	// *
	bcax	$A[3][2],$A[3][2],$A[3][4],$A[3][3]
	bcax	$A[3][3],$A[3][3],$A[3][0],$A[3][4]
	bcax	$A[3][4],$A[3][4],$A[3][1],$A[3][0]

	bcax	$C[5],   $A[4][0],$A[4][2],$A[4][1]	// *
	bcax	$C[6],   $A[4][1],$A[4][3],$A[4][2]	// *
	bcax	$A[4][2],$A[4][2],$A[4][4],$A[4][3]
	bcax	$A[4][3],$A[4][3],$A[4][0],$A[4][4]
	bcax	$A[4][4],$A[4][4],$A[4][1],$A[4][0]
___
	(         $A[1][1],       $C[0]) = (      $C[0],          $A[1][1]);
	($A[2][0],$A[2][1], $C[1],$C[2]) = ($C[1],$C[2], $A[2][0],$A[2][1]);
	($A[3][0],$A[3][1], $C[3],$C[4]) = ($C[3],$C[4], $A[3][0],$A[3][1]);
	($A[4][0],$A[4][1], $C[5],$C[6]) = ($C[5],$C[6], $A[4][0],$A[4][1]);
}
$code.=<<___;
	subs	x9,x9,#1
	bne	.Loop_ce

	ret
.size	KeccakF1600_ce,.-KeccakF1600_ce

.type	KeccakF1600_cext,%function
.align	5
KeccakF1600_cext:
	.inst	0xd503233f		// paciasp
	stp	x29,x30,[sp,#-80]!
	add	x29,sp,#0
	stp	d8,d9,[sp,#16]		// per ABI requirement
	stp	d10,d11,[sp,#32]
	stp	d12,d13,[sp,#48]
	stp	d14,d15,[sp,#64]
___
for($i=0; $i<24; $i+=2) {		# load A[5][5]
my $j=$i+1;
$code.=<<___;
	ldp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	ldr	d24,[x0,#8*$i]
	bl	KeccakF1600_ce
	ldr	x30,[sp,#8]
___
for($i=0; $i<24; $i+=2) {		# store A[5][5]
my $j=$i+1;
$code.=<<___;
	stp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	str	d24,[x0,#8*$i]

	ldp	d8,d9,[sp,#16]
	ldp	d10,d11,[sp,#32]
	ldp	d12,d13,[sp,#48]
	ldp	d14,d15,[sp,#64]
	ldr	x29,[sp],#80
	.inst	0xd50323bf		// autiasp
	ret
.size	KeccakF1600_cext,.-KeccakF1600_cext
___

{
my ($ctx,$inp,$len,$bsz) = map("x$_",(0..3));

$code.=<<___;
.globl	SHA3_absorb_cext
.type	SHA3_absorb_cext,%function
.align	5
SHA3_absorb_cext:
	.inst	0xd503233f		// paciasp
	stp	x29,x30,[sp,#-80]!
	add	x29,sp,#0
	stp	d8,d9,[sp,#16]		// per ABI requirement
	stp	d10,d11,[sp,#32]
	stp	d12,d13,[sp,#48]
	stp	d14,d15,[sp,#64]
___
for($i=0; $i<24; $i+=2) {		# load A[5][5]
my $j=$i+1;
$code.=<<___;
	ldp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	ldr	d24,[x0,#8*$i]
	b	.Loop_absorb_ce

.align	4
.Loop_absorb_ce:
	subs	$len,$len,$bsz		// len - bsz
	blo	.Labsorbed_ce
___
for (my $i=0; $i<24; $i+=2) {
my $j = $i+1;
$code.=<<___;
	ldr	d31,[$inp],#8		// *inp++
#ifdef	__AARCH64EB__
	rev64	v31.16b,v31.16b
#endif
	eor	$A[$i/5][$i%5],$A[$i/5][$i%5],v31.16b
	cmp	$bsz,#8*($i+2)
	blo	.Lprocess_block_ce
	ldr	d31,[$inp],#8		// *inp++
#ifdef	__AARCH64EB__
	rev64	v31.16b,v31.16b
#endif
	eor	$A[$j/5][$j%5],$A[$j/5][$j%5],v31.16b
	beq	.Lprocess_block_ce
___
}
$code.=<<___;
	ldr	d31,[$inp],#8		// *inp++
#ifdef	__AARCH64EB__
	rev64	v31.16b,v31.16b
#endif
	eor	$A[4][4],$A[4][4],v31.16b

.Lprocess_block_ce:

	bl	KeccakF1600_ce

	b	.Loop_absorb_ce

.align	4
.Labsorbed_ce:
___
for($i=0; $i<24; $i+=2) {		# store A[5][5]
my $j=$i+1;
$code.=<<___;
	stp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	str	d24,[x0,#8*$i]
	add	x0,$len,$bsz		// return value

	ldp	d8,d9,[sp,#16]
	ldp	d10,d11,[sp,#32]
	ldp	d12,d13,[sp,#48]
	ldp	d14,d15,[sp,#64]
	ldp	x29,x30,[sp],#80
	.inst	0xd50323bf		// autiasp
	ret
.size	SHA3_absorb_cext,.-SHA3_absorb_cext
___
}
{
my ($ctx,$out,$len,$bsz) = map("x$_",(0..3));
$code.=<<___;
.globl	SHA3_squeeze_cext
.type	SHA3_squeeze_cext,%function
.align	5
SHA3_squeeze_cext:
	.inst	0xd503233f		// paciasp
	stp	x29,x30,[sp,#-16]!
	add	x29,sp,#0
	mov	x9,$ctx
	mov	x10,$bsz

.Loop_squeeze_ce:
	ldr	x4,[x9],#8
	cmp	$len,#8
	blo	.Lsqueeze_tail_ce
#ifdef	__AARCH64EB__
	rev	x4,x4
#endif
	str	x4,[$out],#8
	beq	.Lsqueeze_done_ce

	sub	$len,$len,#8
	subs	x10,x10,#8
	bhi	.Loop_squeeze_ce

	bl	KeccakF1600_cext
	ldr	x30,[sp,#8]
	mov	x9,$ctx
	mov	x10,$bsz
	b	.Loop_squeeze_ce

.align	4
.Lsqueeze_tail_ce:
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1

.Lsqueeze_done_ce:
	ldr	x29,[sp],#16
	.inst	0xd50323bf		// autiasp
	ret
.size	SHA3_squeeze_cext,.-SHA3_squeeze_cext
___
}								}}}
$code.=<<___;
.asciz	"Keccak-1600 absorb and squeeze for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
___

{   my  %opcode = (
	"rax1"	=> 0xce608c00,	"eor3"	=> 0xce000000,
	"bcax"	=> 0xce200000,	"xar"	=> 0xce800000	);

    sub unsha3 {
	my ($mnemonic,$arg)=@_;

	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/
	&&
	sprintf ".inst\t0x%08x\t//%s %s",
			$opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10),
			$mnemonic,$arg;
    }
}

foreach(split("\n",$code)) {

	s/\`([^\`]*)\`/eval($1)/ge;

	m/\bdup\b/ and s/\.16b/.2d/g	or
	s/\b(eor3|rax1|xar|bcax)\s+(v.*)/unsha3($1,$2)/ge;

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#!/usr/bin/env perl
# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for AVX2.
#
# July 2017.
#
# To paraphrase Gilles Van Assche, if you contemplate Fig. 2.3 on page
# 20 of The Keccak reference [or Fig. 5 of FIPS PUB 202], and load data
# other than A[0][0] in magic order into 6 [256-bit] registers, *each
# dedicated to one axis*, Pi permutation is reduced to intra-register
# shuffles...
#
# It makes other steps more intricate, but overall, is it a win? To be
# more specific index permutations organized by quadruples are:
#
#       [4][4] [3][3] [2][2] [1][1]<-+
#       [0][4] [0][3] [0][2] [0][1]<-+
#       [3][0] [1][0] [4][0] [2][0]  |
#       [4][3] [3][1] [2][4] [1][2]  |
#       [3][4] [1][3] [4][2] [2][1]  |
#       [2][3] [4][1] [1][4] [3][2]  |
#       [2][2] [4][4] [1][1] [3][3] -+
#
# This however is highly impractical for Theta and Chi. What would help
# Theta is if x indices were aligned column-wise, or in other words:
#
#       [0][4] [0][3] [0][2] [0][1]
#       [3][0] [1][0] [4][0] [2][0]
#vpermq([4][3] [3][1] [2][4] [1][2], 0b01110010)
#       [2][4] [4][3] [1][2] [3][1]
#vpermq([4][2] [3][4] [2][1] [1][3], 0b10001101)
#       [3][4] [1][3] [4][2] [2][1]
#vpermq([2][3] [4][1] [1][4] [3][2], 0b01110010)
#       [1][4] [2][3] [3][2] [4][1]
#vpermq([1][1] [2][2] [3][3] [4][4], 0b00011011)
#       [4][4] [3][3] [2][2] [1][1]
#
# So here we have it, lines not marked with vpermq() represent the magic
# order in which data is to be loaded and maintained. [And lines marked
# with vpermq() represent Pi circular permutation in chosen layout. Note
# that first step is permutation-free.] A[0][0] is loaded to register of
# its own, to all lanes. [A[0][0] is not part of Pi permutation or Rho.]
# Digits in variables' names denote right-most coordinates:

my ($A00,	# [0][0] [0][0] [0][0] [0][0]		# %ymm0
    $A01,	# [0][4] [0][3] [0][2] [0][1]		# %ymm1
    $A20,	# [3][0] [1][0] [4][0] [2][0]		# %ymm2
    $A31,	# [2][4] [4][3] [1][2] [3][1]		# %ymm3
    $A21,	# [3][4] [1][3] [4][2] [2][1]		# %ymm4
    $A41,	# [1][4] [2][3] [3][2] [4][1]		# %ymm5
    $A11) =	# [4][4] [3][3] [2][2] [1][1]		# %ymm6
    map("%ymm$_",(0..6));

# We also need to map the magic order into offsets within structure:

my @A_jagged = ([0,0], [1,0], [1,1], [1,2], [1,3],	# [0][0..4]
		[2,2], [6,0], [3,1], [4,2], [5,3],	# [1][0..4]
		[2,0], [4,0], [6,1], [5,2], [3,3],	# [2][0..4]
		[2,3], [3,0], [5,1], [6,2], [4,3],	# [3][0..4]
		[2,1], [5,0], [4,1], [3,2], [6,3]);	# [4][0..4]
   @A_jagged = map(8*($$_[0]*4+$$_[1]), @A_jagged);	# ... and now linear

# But on the other hand Chi is much better off if y indices were aligned
# column-wise, not x. For this reason we have to shuffle data prior
# Chi and revert it afterwards. Prior shuffle is naturally merged with
# Pi itself:
#
#       [0][4] [0][3] [0][2] [0][1]
#       [3][0] [1][0] [4][0] [2][0]
#vpermq([4][3] [3][1] [2][4] [1][2], 0b01110010)
#vpermq([2][4] [4][3] [1][2] [3][1], 0b00011011) = 0b10001101
#       [3][1] [1][2] [4][3] [2][4]
#vpermq([4][2] [3][4] [2][1] [1][3], 0b10001101)
#vpermq([3][4] [1][3] [4][2] [2][1], 0b11100100) = 0b10001101
#       [3][4] [1][3] [4][2] [2][1]
#vpermq([2][3] [4][1] [1][4] [3][2], 0b01110010)
#vpermq([1][4] [2][3] [3][2] [4][1], 0b01110010) = 0b00011011
#       [3][2] [1][4] [4][1] [2][3]
#vpermq([1][1] [2][2] [3][3] [4][4], 0b00011011)
#vpermq([4][4] [3][3] [2][2] [1][1], 0b10001101) = 0b01110010
#       [3][3] [1][1] [4][4] [2][2]
#
# And reverse post-Chi permutation:
#
#       [0][4] [0][3] [0][2] [0][1]
#       [3][0] [1][0] [4][0] [2][0]
#vpermq([3][1] [1][2] [4][3] [2][4], 0b00011011)
#       [2][4] [4][3] [1][2] [3][1]
#vpermq([3][4] [1][3] [4][2] [2][1], 0b11100100) = nop :-)
#       [3][4] [1][3] [4][2] [2][1]
#vpermq([3][2] [1][4] [4][1] [2][3], 0b10001101)
#       [1][4] [2][3] [3][2] [4][1]
#vpermq([3][3] [1][1] [4][4] [2][2], 0b01110010)
#       [4][4] [3][3] [2][2] [1][1]
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
#			r=1088(*)
#
# Haswell		8.7/+10%
# Skylake		7.8/+20%
# Ryzen			17(**)
#
# (*)	Corresponds to SHA3-256. Percentage after slash is improvement
#	coefficient in comparison to scalar keccak1600-x86_64.pl.
# (**)	It's expected that Ryzen performs poorly, because instruction
#	issue rate is limited to two AVX2 instructions per cycle and
#	in addition vpblendd is reportedly bound to specific port.
#	Obviously this code path should not be executed on Ryzen.

my @T = map("%ymm$_",(7..15));
my ($C14,$C00,$D00,$D14) = @T[5..8];

$code.=<<___;
.text

.type	__KeccakF1600,\@function
.align	32
__KeccakF1600:
	lea		rhotates_left+96(%rip),%r8
	lea		rhotates_right+96(%rip),%r9
	lea		iotas(%rip),%r10
	mov		\$24,%eax
	jmp		.Loop_avx2

.align	32
.Loop_avx2:
	######################################### Theta
	vpshufd		\$0b01001110,$A20,$C00
	vpxor		$A31,$A41,$C14
	vpxor		$A11,$A21,@T[2]
	vpxor		$A01,$C14,$C14
	vpxor		@T[2],$C14,$C14		# C[1..4]

	vpermq		\$0b10010011,$C14,@T[4]
	vpxor		$A20,$C00,$C00
	vpermq		\$0b01001110,$C00,@T[0]

	vpsrlq		\$63,$C14,@T[1]
	vpaddq		$C14,$C14,@T[2]
	vpor		@T[2],@T[1],@T[1]	# ROL64(C[1..4],1)

	vpermq		\$0b00111001,@T[1],$D14
	vpxor		@T[4],@T[1],$D00
	vpermq		\$0b00000000,$D00,$D00	# D[0..0] = ROL64(C[1],1) ^ C[4]

	vpxor		$A00,$C00,$C00
	vpxor		@T[0],$C00,$C00		# C[0..0]

	vpsrlq		\$63,$C00,@T[0]
	vpaddq		$C00,$C00,@T[1]
	vpor		@T[0],@T[1],@T[1]	# ROL64(C[0..0],1)

	vpxor		$D00,$A20,$A20		# ^= D[0..0]
	vpxor		$D00,$A00,$A00		# ^= D[0..0]

	vpblendd	\$0b11000000,@T[1],$D14,$D14
	vpblendd	\$0b00000011,$C00,@T[4],@T[4]
	vpxor		@T[4],$D14,$D14		# D[1..4] = ROL64(C[2..4,0),1) ^ C[0..3]

	######################################### Rho + Pi + pre-Chi shuffle
	vpsllvq		0*32-96(%r8),$A20,@T[3]
	vpsrlvq		0*32-96(%r9),$A20,$A20
	vpor		@T[3],$A20,$A20

	 vpxor		$D14,$A31,$A31		# ^= D[1..4] from Theta
	vpsllvq		2*32-96(%r8),$A31,@T[4]
	vpsrlvq		2*32-96(%r9),$A31,$A31
	vpor		@T[4],$A31,$A31

	 vpxor		$D14,$A21,$A21		# ^= D[1..4] from Theta
	vpsllvq		3*32-96(%r8),$A21,@T[5]
	vpsrlvq		3*32-96(%r9),$A21,$A21
	vpor		@T[5],$A21,$A21

	 vpxor		$D14,$A41,$A41		# ^= D[1..4] from Theta
	vpsllvq		4*32-96(%r8),$A41,@T[6]
	vpsrlvq		4*32-96(%r9),$A41,$A41
	vpor		@T[6],$A41,$A41

	 vpxor		$D14,$A11,$A11		# ^= D[1..4] from Theta
	 vpermq		\$0b10001101,$A20,@T[3]	# $A20 -> future $A31
	 vpermq		\$0b10001101,$A31,@T[4]	# $A31 -> future $A21
	vpsllvq		5*32-96(%r8),$A11,@T[7]
	vpsrlvq		5*32-96(%r9),$A11,@T[1]
	vpor		@T[7],@T[1],@T[1]	# $A11 -> future $A01

	 vpxor		$D14,$A01,$A01		# ^= D[1..4] from Theta
	 vpermq		\$0b00011011,$A21,@T[5]	# $A21 -> future $A41
	 vpermq		\$0b01110010,$A41,@T[6]	# $A41 -> future $A11
	vpsllvq		1*32-96(%r8),$A01,@T[8]
	vpsrlvq		1*32-96(%r9),$A01,@T[2]
	vpor		@T[8],@T[2],@T[2]	# $A01 -> future $A20

	######################################### Chi
	vpsrldq		\$8,@T[1],@T[7]
	vpandn		@T[7],@T[1],@T[0]	# tgting  [0][0] [0][0] [0][0] [0][0]

	vpblendd	\$0b00001100,@T[6],@T[2],$A31	#               [4][4] [2][0]
	vpblendd	\$0b00001100,@T[2],@T[4],@T[8]	#               [4][0] [2][1]
	 vpblendd	\$0b00001100,@T[4],@T[3],$A41	#               [4][2] [2][4]
	 vpblendd	\$0b00001100,@T[3],@T[2],@T[7]	#               [4][3] [2][0]
	vpblendd	\$0b00110000,@T[4],$A31,$A31	#        [1][3] [4][4] [2][0]
	vpblendd	\$0b00110000,@T[5],@T[8],@T[8]	#        [1][4] [4][0] [2][1]
	 vpblendd	\$0b00110000,@T[2],$A41,$A41	#        [1][0] [4][2] [2][4]
	 vpblendd	\$0b00110000,@T[6],@T[7],@T[7]	#        [1][1] [4][3] [2][0]
	vpblendd	\$0b11000000,@T[5],$A31,$A31	# [3][2] [1][3] [4][4] [2][0]
	vpblendd	\$0b11000000,@T[6],@T[8],@T[8]	# [3][3] [1][4] [4][0] [2][1]
	 vpblendd	\$0b11000000,@T[6],$A41,$A41	# [3][3] [1][0] [4][2] [2][4]
	 vpblendd	\$0b11000000,@T[4],@T[7],@T[7]	# [3][4] [1][1] [4][3] [2][0]
	vpandn		@T[8],$A31,$A31		# tgting  [3][1] [1][2] [4][3] [2][4]
	 vpandn		@T[7],$A41,$A41		# tgting  [3][2] [1][4] [4][1] [2][3]

	vpblendd	\$0b00001100,@T[2],@T[5],$A11	#               [4][0] [2][3]
	vpblendd	\$0b00001100,@T[5],@T[3],@T[8]	#               [4][1] [2][4]
	 vpxor		@T[3],$A31,$A31
	vpblendd	\$0b00110000,@T[3],$A11,$A11	#        [1][2] [4][0] [2][3]
	vpblendd	\$0b00110000,@T[4],@T[8],@T[8]	#        [1][3] [4][1] [2][4]
	 vpxor		@T[5],$A41,$A41
	vpblendd	\$0b11000000,@T[4],$A11,$A11	# [3][4] [1][2] [4][0] [2][3]
	vpblendd	\$0b11000000,@T[2],@T[8],@T[8]	# [3][0] [1][3] [4][1] [2][4]
	vpandn		@T[8],$A11,$A11		# tgting  [3][3] [1][1] [4][4] [2][2]
	vpxor		@T[6],$A11,$A11

	  vpermq	\$0b00011110,@T[1],$A21		# [0][1] [0][2] [0][4] [0][3]
	  vpblendd	\$0b00110000,$A00,$A21,@T[8]	# [0][1] [0][0] [0][4] [0][3]
	  vpermq	\$0b00111001,@T[1],$A01		# [0][1] [0][4] [0][3] [0][2]
	  vpblendd	\$0b11000000,$A00,$A01,$A01	# [0][0] [0][4] [0][3] [0][2]
	  vpandn	@T[8],$A01,$A01		# tgting  [0][4] [0][3] [0][2] [0][1]

	vpblendd	\$0b00001100,@T[5],@T[4],$A20	#               [4][1] [2][1]
	vpblendd	\$0b00001100,@T[4],@T[6],@T[7]	#               [4][2] [2][2]
	vpblendd	\$0b00110000,@T[6],$A20,$A20	#        [1][1] [4][1] [2][1]
	vpblendd	\$0b00110000,@T[3],@T[7],@T[7]	#        [1][2] [4][2] [2][2]
	vpblendd	\$0b11000000,@T[3],$A20,$A20	# [3][1] [1][1] [4][1] [2][1]
	vpblendd	\$0b11000000,@T[5],@T[7],@T[7]	# [3][2] [1][2] [4][2] [2][2]
	vpandn		@T[7],$A20,$A20		# tgting  [3][0] [1][0] [4][0] [2][0]
	vpxor		@T[2],$A20,$A20

	 vpermq		\$0b00000000,@T[0],@T[0]	# [0][0] [0][0] [0][0] [0][0]
	 vpermq		\$0b00011011,$A31,$A31	# post-Chi shuffle
	 vpermq		\$0b10001101,$A41,$A41
	 vpermq		\$0b01110010,$A11,$A11

	vpblendd	\$0b00001100,@T[3],@T[6],$A21	#               [4][3] [2][2]
	vpblendd	\$0b00001100,@T[6],@T[5],@T[7]	#               [4][4] [2][3]
	vpblendd	\$0b00110000,@T[5],$A21,$A21	#        [1][4] [4][3] [2][2]
	vpblendd	\$0b00110000,@T[2],@T[7],@T[7]	#        [1][0] [4][4] [2][3]
	vpblendd	\$0b11000000,@T[2],$A21,$A21	# [3][0] [1][4] [4][3] [2][2]
	vpblendd	\$0b11000000,@T[3],@T[7],@T[7]	# [3][1] [1][0] [4][4] [2][3]
	vpandn		@T[7],$A21,$A21		# tgting  [3][4] [1][3] [4][2] [2][1]

	vpxor		@T[0],$A00,$A00
	vpxor		@T[1],$A01,$A01
	vpxor		@T[4],$A21,$A21

	######################################### Iota
	vpxor		(%r10),$A00,$A00
	lea		32(%r10),%r10

	dec		%eax
	jnz		.Loop_avx2

	ret
.size	__KeccakF1600,.-__KeccakF1600
___
my ($A_flat,$inp,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
my  $out = $inp;	# in squeeze

$code.=<<___;
.globl	SHA3_absorb
.type	SHA3_absorb,\@function
.align	32
SHA3_absorb:
	mov	%rsp,%r11

	lea	-240(%rsp),%rsp
	and	\$-32,%rsp

	lea	96($A_flat),$A_flat
	lea	96($inp),$inp
	lea	96(%rsp),%r10

	vzeroupper

	vpbroadcastq	-96($A_flat),$A00	# load A[5][5]
	vmovdqu		8+32*0-96($A_flat),$A01
	vmovdqu		8+32*1-96($A_flat),$A20
	vmovdqu		8+32*2-96($A_flat),$A31
	vmovdqu		8+32*3-96($A_flat),$A21
	vmovdqu		8+32*4-96($A_flat),$A41
	vmovdqu		8+32*5-96($A_flat),$A11

	vpxor		@T[0],@T[0],@T[0]
	vmovdqa		@T[0],32*2-96(%r10)	# zero transfer area on stack
	vmovdqa		@T[0],32*3-96(%r10)
	vmovdqa		@T[0],32*4-96(%r10)
	vmovdqa		@T[0],32*5-96(%r10)
	vmovdqa		@T[0],32*6-96(%r10)

.Loop_absorb_avx2:
	mov		$bsz,%rax
	sub		$bsz,$len
	jc		.Ldone_absorb_avx2

	shr		\$3,%eax
	vpbroadcastq	0-96($inp),@T[0]
	vmovdqu		8-96($inp),@T[1]
	sub		\$4,%eax
___
for(my $i=5; $i<25; $i++) {
$code.=<<___
	dec	%eax
	jz	.Labsorved_avx2
	mov	8*$i-96($inp),%r8
	mov	%r8,$A_jagged[$i]-96(%r10)
___
}
$code.=<<___;
.Labsorved_avx2:
	lea	($inp,$bsz),$inp

	vpxor	@T[0],$A00,$A00
	vpxor	@T[1],$A01,$A01
	vpxor	32*2-96(%r10),$A20,$A20
	vpxor	32*3-96(%r10),$A31,$A31
	vpxor	32*4-96(%r10),$A21,$A21
	vpxor	32*5-96(%r10),$A41,$A41
	vpxor	32*6-96(%r10),$A11,$A11

	call	__KeccakF1600

	lea	96(%rsp),%r10
	jmp	.Loop_absorb_avx2

.Ldone_absorb_avx2:
	vmovq	%xmm0,-96($A_flat)
	vmovdqu	$A01,8+32*0-96($A_flat)
	vmovdqu	$A20,8+32*1-96($A_flat)
	vmovdqu	$A31,8+32*2-96($A_flat)
	vmovdqu	$A21,8+32*3-96($A_flat)
	vmovdqu	$A41,8+32*4-96($A_flat)
	vmovdqu	$A11,8+32*5-96($A_flat)

	vzeroupper

	lea	(%r11),%rsp
	lea	($len,$bsz),%rax		# return value
	ret
.size	SHA3_absorb,.-SHA3_absorb

.globl	SHA3_squeeze
.type	SHA3_squeeze,\@function
.align	32
SHA3_squeeze:
	mov	%rsp,%r11

	lea	96($A_flat),$A_flat
	shr	\$3,$bsz

	vzeroupper

	vpbroadcastq	-96($A_flat),$A00
	vpxor		@T[0],@T[0],@T[0]
	vmovdqu		8+32*0-96($A_flat),$A01
	vmovdqu		8+32*1-96($A_flat),$A20
	vmovdqu		8+32*2-96($A_flat),$A31
	vmovdqu		8+32*3-96($A_flat),$A21
	vmovdqu		8+32*4-96($A_flat),$A41
	vmovdqu		8+32*5-96($A_flat),$A11

	mov	$bsz,%rax

.Loop_squeeze_avx2:
	mov	@A_jagged[$i]-96($A_flat),%r8
___
for (my $i=0; $i<25; $i++) {
$code.=<<___;
	sub	\$8,$len
	jc	.Ltail_squeeze_avx2
	mov	%r8,($out)
	lea	8($out),$out
	je	.Ldone_squeeze_avx2
	dec	%eax
	je	.Lextend_output_avx2
	mov	@A_jagged[$i+1]-120($A_flat),%r8
___
}
$code.=<<___;
.Lextend_output_avx2:
	call	__KeccakF1600

	vmovq	%xmm0,-96($A_flat)
	vmovdqu	$A01,8+32*0-96($A_flat)
	vmovdqu	$A20,8+32*1-96($A_flat)
	vmovdqu	$A31,8+32*2-96($A_flat)
	vmovdqu	$A21,8+32*3-96($A_flat)
	vmovdqu	$A41,8+32*4-96($A_flat)
	vmovdqu	$A11,8+32*5-96($A_flat)

	mov	$bsz,%rax
	jmp	.Loop_squeeze_avx2

.Ltail_squeeze_avx2:
	add	\$8,$len
.Loop_tail_avx2:
	mov	%r8b,($out)
	lea	1($out),$out
	shr	\$8,%r8
	dec	$len
	jnz	.Loop_tail_avx2

.Ldone_squeeze_avx2:
	vzeroupper

	lea	(%r11),%rsp
	ret
.size	SHA3_squeeze,.-SHA3_squeeze

.align	64
rhotates_left:
	.quad	3,	18,	36,	41	# [2][0] [4][0] [1][0] [3][0]
	.quad	1,	62,	28,	27	# [0][1] [0][2] [0][3] [0][4]
	.quad	45,	6,	56,	39	# [3][1] [1][2] [4][3] [2][4]
	.quad	10,	61,	55,	8	# [2][1] [4][2] [1][3] [3][4]
	.quad	2,	15,	25,	20	# [4][1] [3][2] [2][3] [1][4]
	.quad	44,	43,	21,	14	# [1][1] [2][2] [3][3] [4][4]
rhotates_right:
	.quad	64-3,	64-18,	64-36,	64-41
	.quad	64-1,	64-62,	64-28,	64-27
	.quad	64-45,	64-6,	64-56,	64-39
	.quad	64-10,	64-61,	64-55,	64-8
	.quad	64-2,	64-15,	64-25,	64-20
	.quad	64-44,	64-43,	64-21,	64-14
iotas:
	.quad	0x0000000000000001, 0x0000000000000001, 0x0000000000000001, 0x0000000000000001
	.quad	0x0000000000008082, 0x0000000000008082, 0x0000000000008082, 0x0000000000008082
	.quad	0x800000000000808a, 0x800000000000808a, 0x800000000000808a, 0x800000000000808a
	.quad	0x8000000080008000, 0x8000000080008000, 0x8000000080008000, 0x8000000080008000
	.quad	0x000000000000808b, 0x000000000000808b, 0x000000000000808b, 0x000000000000808b
	.quad	0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001
	.quad	0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081
	.quad	0x8000000000008009, 0x8000000000008009, 0x8000000000008009, 0x8000000000008009
	.quad	0x000000000000008a, 0x000000000000008a, 0x000000000000008a, 0x000000000000008a
	.quad	0x0000000000000088, 0x0000000000000088, 0x0000000000000088, 0x0000000000000088
	.quad	0x0000000080008009, 0x0000000080008009, 0x0000000080008009, 0x0000000080008009
	.quad	0x000000008000000a, 0x000000008000000a, 0x000000008000000a, 0x000000008000000a
	.quad	0x000000008000808b, 0x000000008000808b, 0x000000008000808b, 0x000000008000808b
	.quad	0x800000000000008b, 0x800000000000008b, 0x800000000000008b, 0x800000000000008b
	.quad	0x8000000000008089, 0x8000000000008089, 0x8000000000008089, 0x8000000000008089
	.quad	0x8000000000008003, 0x8000000000008003, 0x8000000000008003, 0x8000000000008003
	.quad	0x8000000000008002, 0x8000000000008002, 0x8000000000008002, 0x8000000000008002
	.quad	0x8000000000000080, 0x8000000000000080, 0x8000000000000080, 0x8000000000000080
	.quad	0x000000000000800a, 0x000000000000800a, 0x000000000000800a, 0x000000000000800a
	.quad	0x800000008000000a, 0x800000008000000a, 0x800000008000000a, 0x800000008000000a
	.quad	0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081
	.quad	0x8000000000008080, 0x8000000000008080, 0x8000000000008080, 0x8000000000008080
	.quad	0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001
	.quad	0x8000000080008008, 0x8000000080008008, 0x8000000080008008, 0x8000000080008008

.asciz	"Keccak-1600 absorb and squeeze for AVX2, CRYPTOGAMS by <appro\@openssl.org>"
___

$output=pop;
open STDOUT,">$output";
print $code;
close STDOUT or die "error closing STDOUT: $!";