#! /usr/bin/env perl
# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# GHASH for for PowerISA v2.07.
#
# July 2014
#
# Accurate performance measurements are problematic, because it's
# always virtualized setup with possibly throttled processor.
# Relative comparison is therefore more informative. This initial
# version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x
# faster than "4-bit" integer-only compiler-generated 64-bit code.
# "Initial version" means that there is room for futher improvement.

# May 2016
#
# 2x aggregated reduction improves performance by 50% (resulting
# performance on POWER8 is 1 cycle per processed byte), and 4x
# aggregated reduction - by 170% or 2.7x (resulting in 0.55 cpb).

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

$flavour=shift;
$output =shift;

if ($flavour =~ /64/) {
	$SIZE_T=8;
	$LRSAVE=2*$SIZE_T;
	$STU="stdu";
	$POP="ld";
	$PUSH="std";
	$UCMP="cmpld";
	$SHRI="srdi";
} elsif ($flavour =~ /32/) {
	$SIZE_T=4;
	$LRSAVE=$SIZE_T;
	$STU="stwu";
	$POP="lwz";
	$PUSH="stw";
	$UCMP="cmplw";
	$SHRI="srwi";
} else { die "nonsense $flavour"; }

$sp="r1";
$FRAME=6*$SIZE_T+13*16;	# 13*16 is for v20-v31 offload

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";

open OUT,"| $^X \"$xlate\" $flavour \"$output\"" || die "can't call $xlate: $!";
*STDOUT=*OUT;

my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6));	# argument block

my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3));
my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12));
my ($Xl1,$Xm1,$Xh1,$IN1,$H2,$H2h,$H2l)=map("v$_",(13..19));
my $vrsave="r12";

$code=<<___;
.machine	"any"

.text

.globl	.gcm_init_p8
.align	5
.gcm_init_p8:
	li		r0,-4096
	li		r8,0x10
	mfspr		$vrsave,256
	li		r9,0x20
	mtspr		256,r0
	li		r10,0x30
	lvx_u		$H,0,r4			# load H

	vspltisb	$xC2,-16		# 0xf0
	vspltisb	$t0,1			# one
	vaddubm		$xC2,$xC2,$xC2		# 0xe0
	vxor		$zero,$zero,$zero
	vor		$xC2,$xC2,$t0		# 0xe1
	vsldoi		$xC2,$xC2,$zero,15	# 0xe1...
	vsldoi		$t1,$zero,$t0,1		# ...1
	vaddubm		$xC2,$xC2,$xC2		# 0xc2...
	vspltisb	$t2,7
	vor		$xC2,$xC2,$t1		# 0xc2....01
	vspltb		$t1,$H,0		# most significant byte
	vsl		$H,$H,$t0		# H<<=1
	vsrab		$t1,$t1,$t2		# broadcast carry bit
	vand		$t1,$t1,$xC2
	vxor		$IN,$H,$t1		# twisted H

	vsldoi		$H,$IN,$IN,8		# twist even more ...
	vsldoi		$xC2,$zero,$xC2,8	# 0xc2.0
	vsldoi		$Hl,$zero,$H,8		# ... and split
	vsldoi		$Hh,$H,$zero,8

	stvx_u		$xC2,0,r3		# save pre-computed table
	stvx_u		$Hl,r8,r3
	li		r8,0x40
	stvx_u		$H, r9,r3
	li		r9,0x50
	stvx_u		$Hh,r10,r3
	li		r10,0x60

	vpmsumd		$Xl,$IN,$Hl		# H.loÂ·H.lo
	vpmsumd		$Xm,$IN,$H		# H.hiÂ·H.lo+H.loÂ·H.hi
	vpmsumd		$Xh,$IN,$Hh		# H.hiÂ·H.hi

	vpmsumd		$t2,$Xl,$xC2		# 1st reduction phase

	vsldoi		$t0,$Xm,$zero,8
	vsldoi		$t1,$zero,$Xm,8
	vxor		$Xl,$Xl,$t0
	vxor		$Xh,$Xh,$t1

	vsldoi		$Xl,$Xl,$Xl,8
	vxor		$Xl,$Xl,$t2

	vsldoi		$t1,$Xl,$Xl,8		# 2nd reduction phase
	vpmsumd		$Xl,$Xl,$xC2
	vxor		$t1,$t1,$Xh
	vxor		$IN1,$Xl,$t1

	vsldoi		$H2,$IN1,$IN1,8
	vsldoi		$H2l,$zero,$H2,8
	vsldoi		$H2h,$H2,$zero,8

	stvx_u		$H2l,r8,r3		# save H^2
	li		r8,0x70
	stvx_u		$H2,r9,r3
	li		r9,0x80
	stvx_u		$H2h,r10,r3
	li		r10,0x90
___
{
my ($t4,$t5,$t6) = ($Hl,$H,$Hh);
$code.=<<___;
	vpmsumd		$Xl,$IN,$H2l		# H.loÂ·H^2.lo
	 vpmsumd	$Xl1,$IN1,$H2l		# H^2.loÂ·H^2.lo
	vpmsumd		$Xm,$IN,$H2		# H.hiÂ·H^2.lo+H.loÂ·H^2.hi
	 vpmsumd	$Xm1,$IN1,$H2		# H^2.hiÂ·H^2.lo+H^2.loÂ·H^2.hi
	vpmsumd		$Xh,$IN,$H2h		# H.hiÂ·H^2.hi
	 vpmsumd	$Xh1,$IN1,$H2h		# H^2.hiÂ·H^2.hi

	vpmsumd		$t2,$Xl,$xC2		# 1st reduction phase
	 vpmsumd	$t6,$Xl1,$xC2		# 1st reduction phase

	vsldoi		$t0,$Xm,$zero,8
	vsldoi		$t1,$zero,$Xm,8
	 vsldoi		$t4,$Xm1,$zero,8
	 vsldoi		$t5,$zero,$Xm1,8
	vxor		$Xl,$Xl,$t0
	vxor		$Xh,$Xh,$t1
	 vxor		$Xl1,$Xl1,$t4
	 vxor		$Xh1,$Xh1,$t5

	vsldoi		$Xl,$Xl,$Xl,8
	 vsldoi		$Xl1,$Xl1,$Xl1,8
	vxor		$Xl,$Xl,$t2
	 vxor		$Xl1,$Xl1,$t6

	vsldoi		$t1,$Xl,$Xl,8		# 2nd reduction phase
	 vsldoi		$t5,$Xl1,$Xl1,8		# 2nd reduction phase
	vpmsumd		$Xl,$Xl,$xC2
	 vpmsumd	$Xl1,$Xl1,$xC2
	vxor		$t1,$t1,$Xh
	 vxor		$t5,$t5,$Xh1
	vxor		$Xl,$Xl,$t1
	 vxor		$Xl1,$Xl1,$t5

	vsldoi		$H,$Xl,$Xl,8
	 vsldoi		$H2,$Xl1,$Xl1,8
	vsldoi		$Hl,$zero,$H,8
	vsldoi		$Hh,$H,$zero,8
	 vsldoi		$H2l,$zero,$H2,8
	 vsldoi		$H2h,$H2,$zero,8

	stvx_u		$Hl,r8,r3		# save H^3
	li		r8,0xa0
	stvx_u		$H,r9,r3
	li		r9,0xb0
	stvx_u		$Hh,r10,r3
	li		r10,0xc0
	 stvx_u		$H2l,r8,r3		# save H^4
	 stvx_u		$H2,r9,r3
	 stvx_u		$H2h,r10,r3

	mtspr		256,$vrsave
	blr
	.long		0
	.byte		0,12,0x14,0,0,0,2,0
	.long		0
.size	.gcm_init_p8,.-.gcm_init_p8
___
}
$code.=<<___;
.globl	.gcm_gmult_p8
.align	5
.gcm_gmult_p8:
	lis		r0,0xfff8
	li		r8,0x10
	mfspr		$vrsave,256
	li		r9,0x20
	mtspr		256,r0
	li		r10,0x30
	lvx_u		$IN,0,$Xip		# load Xi

	lvx_u		$Hl,r8,$Htbl		# load pre-computed table
	 le?lvsl	$lemask,r0,r0
	lvx_u		$H, r9,$Htbl
	 le?vspltisb	$t0,0x07
	lvx_u		$Hh,r10,$Htbl
	 le?vxor	$lemask,$lemask,$t0
	lvx_u		$xC2,0,$Htbl
	 le?vperm	$IN,$IN,$IN,$lemask
	vxor		$zero,$zero,$zero

	vpmsumd		$Xl,$IN,$Hl		# H.loÂ·Xi.lo
	vpmsumd		$Xm,$IN,$H		# H.hiÂ·Xi.lo+H.loÂ·Xi.hi
	vpmsumd		$Xh,$IN,$Hh		# H.hiÂ·Xi.hi

	vpmsumd		$t2,$Xl,$xC2		# 1st reduction phase

	vsldoi		$t0,$Xm,$zero,8
	vsldoi		$t1,$zero,$Xm,8
	vxor		$Xl,$Xl,$t0
	vxor		$Xh,$Xh,$t1

	vsldoi		$Xl,$Xl,$Xl,8
	vxor		$Xl,$Xl,$t2

	vsldoi		$t1,$Xl,$Xl,8		# 2nd reduction phase
	vpmsumd		$Xl,$Xl,$xC2
	vxor		$t1,$t1,$Xh
	vxor		$Xl,$Xl,$t1

	le?vperm	$Xl,$Xl,$Xl,$lemask
	stvx_u		$Xl,0,$Xip		# write out Xi

	mtspr		256,$vrsave
	blr
	.long		0
	.byte		0,12,0x14,0,0,0,2,0
	.long		0
.size	.gcm_gmult_p8,.-.gcm_gmult_p8

.globl	.gcm_ghash_p8
.align	5
.gcm_ghash_p8:
	li		r0,-4096
	li		r8,0x10
	mfspr		$vrsave,256
	li		r9,0x20
	mtspr		256,r0
	li		r10,0x30
	lvx_u		$Xl,0,$Xip		# load Xi

	lvx_u		$Hl,r8,$Htbl		# load pre-computed table
	li		r8,0x40
	 le?lvsl	$lemask,r0,r0
	lvx_u		$H, r9,$Htbl
	li		r9,0x50
	 le?vspltisb	$t0,0x07
	lvx_u		$Hh,r10,$Htbl
	li		r10,0x60
	 le?vxor	$lemask,$lemask,$t0
	lvx_u		$xC2,0,$Htbl
	 le?vperm	$Xl,$Xl,$Xl,$lemask
	vxor		$zero,$zero,$zero

	${UCMP}i	$len,64
	bge		Lgcm_ghash_p8_4x

	lvx_u		$IN,0,$inp
	addi		$inp,$inp,16
	subic.		$len,$len,16
	 le?vperm	$IN,$IN,$IN,$lemask
	vxor		$IN,$IN,$Xl
	beq		Lshort

	lvx_u		$H2l,r8,$Htbl		# load H^2
	li		r8,16
	lvx_u		$H2, r9,$Htbl
	add		r9,$inp,$len		# end of input
	lvx_u		$H2h,r10,$Htbl
	be?b		Loop_2x

.align	5
Loop_2x:
	lvx_u		$IN1,0,$inp
	le?vperm	$IN1,$IN1,$IN1,$lemask

	 subic		$len,$len,32
	vpmsumd		$Xl,$IN,$H2l		# H^2.loÂ·Xi.lo
	 vpmsumd	$Xl1,$IN1,$Hl		# H.loÂ·Xi+1.lo
	 subfe		r0,r0,r0		# borrow?-1:0
	vpmsumd		$Xm,$IN,$H2		# H^2.hiÂ·Xi.lo+H^2.loÂ·Xi.hi
	 vpmsumd	$Xm1,$IN1,$H		# H.hiÂ·Xi+1.lo+H.loÂ·Xi+1.hi
	 and		r0,r0,$len
	vpmsumd		$Xh,$IN,$H2h		# H^2.hiÂ·Xi.hi
	 vpmsumd	$Xh1,$IN1,$Hh		# H.hiÂ·Xi+1.hi
	 add		$inp,$inp,r0

	vxor		$Xl,$Xl,$Xl1
	vxor		$Xm,$Xm,$Xm1

	vpmsumd		$t2,$Xl,$xC2		# 1st reduction phase

	vsldoi		$t0,$Xm,$zero,8
	vsldoi		$t1,$zero,$Xm,8
	 vxor		$Xh,$Xh,$Xh1
	vxor		$Xl,$Xl,$t0
	vxor		$Xh,$Xh,$t1

	vsldoi		$Xl,$Xl,$Xl,8
	vxor		$Xl,$Xl,$t2
	 lvx_u		$IN,r8,$inp
	 addi		$inp,$inp,32

	vsldoi		$t1,$Xl,$Xl,8		# 2nd reduction phase
	vpmsumd		$Xl,$Xl,$xC2
	 le?vperm	$IN,$IN,$IN,$lemask
	vxor		$t1,$t1,$Xh
	vxor		$IN,$IN,$t1
	vxor		$IN,$IN,$Xl
	$UCMP		r9,$inp
	bgt		Loop_2x			# done yet?

	cmplwi		$len,0
	bne		Leven

Lshort:
	vpmsumd		$Xl,$IN,$Hl		# H.loÂ·Xi.lo
	vpmsumd		$Xm,$IN,$H		# H.hiÂ·Xi.lo+H.loÂ·Xi.hi
	vpmsumd		$Xh,$IN,$Hh		# H.hiÂ·Xi.hi

	vpmsumd		$t2,$Xl,$xC2		# 1st reduction phase

	vsldoi		$t0,$Xm,$zero,8
	vsldoi		$t1,$zero,$Xm,8
	vxor		$Xl,$Xl,$t0
	vxor		$Xh,$Xh,$t1

	vsldoi		$Xl,$Xl,$Xl,8
	vxor		$Xl,$Xl,$t2

	vsldoi		$t1,$Xl,$Xl,8		# 2nd reduction phase
	vpmsumd		$Xl,$Xl,$xC2
	vxor		$t1,$t1,$Xh

Leven:
	vxor		$Xl,$Xl,$t1
	le?vperm	$Xl,$Xl,$Xl,$lemask
	stvx_u		$Xl,0,$Xip		# write out Xi

	mtspr		256,$vrsave
	blr
	.long		0
	.byte		0,12,0x14,0,0,0,4,0
	.long		0
___
{
my ($Xl3,$Xm2,$IN2,$H3l,$H3,$H3h,
    $Xh3,$Xm3,$IN3,$H4l,$H4,$H4h) = map("v$_",(20..31));
my $IN0=$IN;
my ($H21l,$H21h,$loperm,$hiperm) = ($Hl,$Hh,$H2l,$H2h);

$code.=<<___;
.align	5
.gcm_ghash_p8_4x:
Lgcm_ghash_p8_4x:
	$STU		$sp,-$FRAME($sp)
	li		r10,`15+6*$SIZE_T`
	li		r11,`31+6*$SIZE_T`
	stvx		v20,r10,$sp
	addi		r10,r10,32
	stvx		v21,r11,$sp
	addi		r11,r11,32
	stvx		v22,r10,$sp
	addi		r10,r10,32
	stvx		v23,r11,$sp
	addi		r11,r11,32
	stvx		v24,r10,$sp
	addi		r10,r10,32
	stvx		v25,r11,$sp
	addi		r11,r11,32
	stvx		v26,r10,$sp
	addi		r10,r10,32
	stvx		v27,r11,$sp
	addi		r11,r11,32
	stvx		v28,r10,$sp
	addi		r10,r10,32
	stvx		v29,r11,$sp
	addi		r11,r11,32
	stvx		v30,r10,$sp
	li		r10,0x60
	stvx		v31,r11,$sp
	li		r0,-1
	stw		$vrsave,`$FRAME-4`($sp)	# save vrsave
	mtspr		256,r0			# preserve all AltiVec registers

	lvsl		$t0,0,r8		# 0x0001..0e0f
	#lvx_u		$H2l,r8,$Htbl		# load H^2
	li		r8,0x70
	lvx_u		$H2, r9,$Htbl
	li		r9,0x80
	vspltisb	$t1,8			# 0x0808..0808
	#lvx_u		$H2h,r10,$Htbl
	li		r10,0x90
	lvx_u		$H3l,r8,$Htbl		# load H^3
	li		r8,0xa0
	lvx_u		$H3, r9,$Htbl
	li		r9,0xb0
	lvx_u		$H3h,r10,$Htbl
	li		r10,0xc0
	lvx_u		$H4l,r8,$Htbl		# load H^4
	li		r8,0x10
	lvx_u		$H4, r9,$Htbl
	li		r9,0x20
	lvx_u		$H4h,r10,$Htbl
	li		r10,0x30

	vsldoi		$t2,$zero,$t1,8		# 0x0000..0808
	vaddubm		$hiperm,$t0,$t2		# 0x0001..1617
	vaddubm		$loperm,$t1,$hiperm	# 0x0809..1e1f

	$SHRI		$len,$len,4		# this allows to use sign bit
						# as carry
	lvx_u		$IN0,0,$inp		# load input
	lvx_u		$IN1,r8,$inp
	subic.		$len,$len,8
	lvx_u		$IN2,r9,$inp
	lvx_u		$IN3,r10,$inp
	addi		$inp,$inp,0x40
	le?vperm	$IN0,$IN0,$IN0,$lemask
	le?vperm	$IN1,$IN1,$IN1,$lemask
	le?vperm	$IN2,$IN2,$IN2,$lemask
	le?vperm	$IN3,$IN3,$IN3,$lemask

	vxor		$Xh,$IN0,$Xl

	 vpmsumd	$Xl1,$IN1,$H3l
	 vpmsumd	$Xm1,$IN1,$H3
	 vpmsumd	$Xh1,$IN1,$H3h

	 vperm		$H21l,$H2,$H,$hiperm
	 vperm		$t0,$IN2,$IN3,$loperm
	 vperm		$H21h,$H2,$H,$loperm
	 vperm		$t1,$IN2,$IN3,$hiperm
	 vpmsumd	$Xm2,$IN2,$H2		# H^2.loÂ·Xi+2.hi+H^2.hiÂ·Xi+2.lo
	 vpmsumd	$Xl3,$t0,$H21l		# H^2.loÂ·Xi+2.lo+H.loÂ·Xi+3.lo
	 vpmsumd	$Xm3,$IN3,$H		# H.hiÂ·Xi+3.lo  +H.loÂ·Xi+3.hi
	 vpmsumd	$Xh3,$t1,$H21h		# H^2.hiÂ·Xi+2.hi+H.hiÂ·Xi+3.hi

	 vxor		$Xm2,$Xm2,$Xm1
	 vxor		$Xl3,$Xl3,$Xl1
	 vxor		$Xm3,$Xm3,$Xm2
	 vxor		$Xh3,$Xh3,$Xh1

	blt		Ltail_4x

Loop_4x:
	lvx_u		$IN0,0,$inp
	lvx_u		$IN1,r8,$inp
	subic.		$len,$len,4
	lvx_u		$IN2,r9,$inp
	lvx_u		$IN3,r10,$inp
	addi		$inp,$inp,0x40
	le?vperm	$IN1,$IN1,$IN1,$lemask
	le?vperm	$IN2,$IN2,$IN2,$lemask
	le?vperm	$IN3,$IN3,$IN3,$lemask
	le?vperm	$IN0,$IN0,$IN0,$lemask

	vpmsumd		$Xl,$Xh,$H4l		# H^4.loÂ·Xi.lo
	vpmsumd		$Xm,$Xh,$H4		# H^4.hiÂ·Xi.lo+H^4.loÂ·Xi.hi
	vpmsumd		$Xh,$Xh,$H4h		# H^4.hiÂ·Xi.hi
	 vpmsumd	$Xl1,$IN1,$H3l
	 vpmsumd	$Xm1,$IN1,$H3
	 vpmsumd	$Xh1,$IN1,$H3h

	vxor		$Xl,$Xl,$Xl3
	vxor		$Xm,$Xm,$Xm3
	vxor		$Xh,$Xh,$Xh3
	 vperm		$t0,$IN2,$IN3,$loperm
	 vperm		$t1,$IN2,$IN3,$hiperm

	vpmsumd		$t2,$Xl,$xC2		# 1st reduction phase
	 vpmsumd	$Xl3,$t0,$H21l		# H.loÂ·Xi+3.lo  +H^2.loÂ·Xi+2.lo
	 vpmsumd	$Xh3,$t1,$H21h		# H.hiÂ·Xi+3.hi  +H^2.hiÂ·Xi+2.hi

	vsldoi		$t0,$Xm,$zero,8
	vsldoi		$t1,$zero,$Xm,8
	vxor		$Xl,$Xl,$t0
	vxor		$Xh,$Xh,$t1

	vsldoi		$Xl,$Xl,$Xl,8
	vxor		$Xl,$Xl,$t2

	vsldoi		$t1,$Xl,$Xl,8		# 2nd reduction phase
	 vpmsumd	$Xm2,$IN2,$H2		# H^2.hiÂ·Xi+2.lo+H^2.loÂ·Xi+2.hi
	 vpmsumd	$Xm3,$IN3,$H		# H.hiÂ·Xi+3.lo  +H.loÂ·Xi+3.hi
	vpmsumd		$Xl,$Xl,$xC2

	 vxor		$Xl3,$Xl3,$Xl1
	 vxor		$Xh3,$Xh3,$Xh1
	vxor		$Xh,$Xh,$IN0
	 vxor		$Xm2,$Xm2,$Xm1
	vxor		$Xh,$Xh,$t1
	 vxor		$Xm3,$Xm3,$Xm2
	vxor		$Xh,$Xh,$Xl
	bge		Loop_4x

Ltail_4x:
	vpmsumd		$Xl,$Xh,$H4l		# H^4.loÂ·Xi.lo
	vpmsumd		$Xm,$Xh,$H4		# H^4.hiÂ·Xi.lo+H^4.loÂ·Xi.hi
	vpmsumd		$Xh,$Xh,$H4h		# H^4.hiÂ·Xi.hi

	vxor		$Xl,$Xl,$Xl3
	vxor		$Xm,$Xm,$Xm3

	vpmsumd		$t2,$Xl,$xC2		# 1st reduction phase

	vsldoi		$t0,$Xm,$zero,8
	vsldoi		$t1,$zero,$Xm,8
	 vxor		$Xh,$Xh,$Xh3
	vxor		$Xl,$Xl,$t0
	vxor		$Xh,$Xh,$t1

	vsldoi		$Xl,$Xl,$Xl,8
	vxor		$Xl,$Xl,$t2

	vsldoi		$t1,$Xl,$Xl,8		# 2nd reduction phase
	vpmsumd		$Xl,$Xl,$xC2
	vxor		$t1,$t1,$Xh
	vxor		$Xl,$Xl,$t1

	addic.		$len,$len,4
	beq		Ldone_4x

	lvx_u		$IN0,0,$inp
	${UCMP}i	$len,2
	li		$len,-4
	blt		Lone
	lvx_u		$IN1,r8,$inp
	beq		Ltwo

Lthree:
	lvx_u		$IN2,r9,$inp
	le?vperm	$IN0,$IN0,$IN0,$lemask
	le?vperm	$IN1,$IN1,$IN1,$lemask
	le?vperm	$IN2,$IN2,$IN2,$lemask

	vxor		$Xh,$IN0,$Xl
	vmr		$H4l,$H3l
	vmr		$H4, $H3
	vmr		$H4h,$H3h

	vperm		$t0,$IN1,$IN2,$loperm
	vperm		$t1,$IN1,$IN2,$hiperm
	vpmsumd		$Xm2,$IN1,$H2		# H^2.loÂ·Xi+1.hi+H^2.hiÂ·Xi+1.lo
	vpmsumd		$Xm3,$IN2,$H		# H.hiÂ·Xi+2.lo  +H.loÂ·Xi+2.hi
	vpmsumd		$Xl3,$t0,$H21l		# H^2.loÂ·Xi+1.lo+H.loÂ·Xi+2.lo
	vpmsumd		$Xh3,$t1,$H21h		# H^2.hiÂ·Xi+1.hi+H.hiÂ·Xi+2.hi

	vxor		$Xm3,$Xm3,$Xm2
	b		Ltail_4x

.align	4
Ltwo:
	le?vperm	$IN0,$IN0,$IN0,$lemask
	le?vperm	$IN1,$IN1,$IN1,$lemask

	vxor		$Xh,$IN0,$Xl
	vperm		$t0,$zero,$IN1,$loperm
	vperm		$t1,$zero,$IN1,$hiperm

	vsldoi		$H4l,$zero,$H2,8
	vmr		$H4, $H2
	vsldoi		$H4h,$H2,$zero,8

	vpmsumd		$Xl3,$t0, $H21l		# H.loÂ·Xi+1.lo
	vpmsumd		$Xm3,$IN1,$H		# H.hiÂ·Xi+1.lo+H.loÂ·Xi+2.hi
	vpmsumd		$Xh3,$t1, $H21h		# H.hiÂ·Xi+1.hi

	b		Ltail_4x

.align	4
Lone:
	le?vperm	$IN0,$IN0,$IN0,$lemask

	vsldoi		$H4l,$zero,$H,8
	vmr		$H4, $H
	vsldoi		$H4h,$H,$zero,8

	vxor		$Xh,$IN0,$Xl
	vxor		$Xl3,$Xl3,$Xl3
	vxor		$Xm3,$Xm3,$Xm3
	vxor		$Xh3,$Xh3,$Xh3

	b		Ltail_4x

Ldone_4x:
	le?vperm	$Xl,$Xl,$Xl,$lemask
	stvx_u		$Xl,0,$Xip		# write out Xi

	li		r10,`15+6*$SIZE_T`
	li		r11,`31+6*$SIZE_T`
	mtspr		256,$vrsave
	lvx		v20,r10,$sp
	addi		r10,r10,32
	lvx		v21,r11,$sp
	addi		r11,r11,32
	lvx		v22,r10,$sp
	addi		r10,r10,32
	lvx		v23,r11,$sp
	addi		r11,r11,32
	lvx		v24,r10,$sp
	addi		r10,r10,32
	lvx		v25,r11,$sp
	addi		r11,r11,32
	lvx		v26,r10,$sp
	addi		r10,r10,32
	lvx		v27,r11,$sp
	addi		r11,r11,32
	lvx		v28,r10,$sp
	addi		r10,r10,32
	lvx		v29,r11,$sp
	addi		r11,r11,32
	lvx		v30,r10,$sp
	lvx		v31,r11,$sp
	addi		$sp,$sp,$FRAME
	blr
	.long		0
	.byte		0,12,0x04,0,0x80,0,4,0
	.long		0
___
}
$code.=<<___;
.size	.gcm_ghash_p8,.-.gcm_ghash_p8

.asciz  "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
.align  2
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/geo;

	if ($flavour =~ /le$/o) {	# little-endian
	    s/le\?//o		or
	    s/be\?/#be#/o;
	} else {
	    s/le\?/#le#/o	or
	    s/be\?//o;
	}
	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#! /usr/bin/env perl
# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# GHASH for ARMv8 Crypto Extension, 64-bit polynomial multiplication.
#
# June 2014
# Initial version was developed in tight cooperation with Ard Biesheuvel
# of Linaro from bits-n-pieces from other assembly modules. Just like
# aesv8-armx.pl this module supports both AArch32 and AArch64 execution modes.
#
# July 2014
# Implement 2x aggregated reduction [see ghash-x86.pl for background
# information].
#
# November 2017
#
# AArch64 register bank to "accommodate" 4x aggregated reduction and
# improve performance by 20-70% depending on processor.
#
# Current performance in cycles per processed byte:
#
#           64-bit PMULL  32-bit PMULL  32-bit NEON(*)
# Apple A7      0.58         0.92          5.62
# Cortex-A53    0.85         1.01          8.39
# Cortex-A57    0.73         1.17          7.61
# Denver        0.51         0.65          6.02
# Mongoose      0.65         1.10          8.06
# Kryo          0.76         1.16          8.00
#
# (*)	presented for reference/comparison purposes;

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

$flavour = shift;
$output  = shift;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

$Xi="x0";	# argument block
$Htbl="x1";
$inp="x2";
$len="x3";

$inc="x12";

{
my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
my ($t0,$t1,$t2,$xC2,$H,$Hhl,$H2)=map("q$_",(8..14));

$code=<<___;
#include <openssl/arm_arch.h>

#if __ARM_MAX_ARCH__>=7
.text
___
$code.=".arch	armv8-a+crypto\n"	if ($flavour =~ /64/);
$code.=<<___				if ($flavour !~ /64/);
.fpu	neon
.code	32
#undef	__thumb2__
___

################################################################################
# void gcm_init_v8(u128 Htable[16],const u64 H[2]);
#
# input:	128-bit H - secret parameter E(K,0^128)
# output:	precomputed table filled with degrees of twisted H;
#		H is twisted to handle reverse bitness of GHASH;
#		only few of 16 slots of Htable[16] are used;
#		data is opaque to outside world (which allows to
#		optimize the code independently);
#
$code.=<<___;
.global	gcm_init_v8
.type	gcm_init_v8,%function
.align	4
gcm_init_v8:
	AARCH64_VALID_CALL_TARGET
	vld1.64		{$t1},[x1]		@ load input H
	vmov.i8		$xC2,#0xe1
	vshl.i64	$xC2,$xC2,#57		@ 0xc2.0
	vext.8		$IN,$t1,$t1,#8
	vshr.u64	$t2,$xC2,#63
	vdup.32		$t1,${t1}[1]
	vext.8		$t0,$t2,$xC2,#8		@ t0=0xc2....01
	vshr.u64	$t2,$IN,#63
	vshr.s32	$t1,$t1,#31		@ broadcast carry bit
	vand		$t2,$t2,$t0
	vshl.i64	$IN,$IN,#1
	vext.8		$t2,$t2,$t2,#8
	vand		$t0,$t0,$t1
	vorr		$IN,$IN,$t2		@ H<<<=1
	veor		$H,$IN,$t0		@ twisted H
	vst1.64		{$H},[x0],#16		@ store Htable[0]

	@ calculate H^2
	vext.8		$t0,$H,$H,#8		@ Karatsuba pre-processing
	vpmull.p64	$Xl,$H,$H
	veor		$t0,$t0,$H
	vpmull2.p64	$Xh,$H,$H
	vpmull.p64	$Xm,$t0,$t0

	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	veor		$Xm,$Xm,$t2
	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase

	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	veor		$Xl,$Xm,$t2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$t2,$t2,$Xh
	veor		$H2,$Xl,$t2

	vext.8		$t1,$H2,$H2,#8		@ Karatsuba pre-processing
	veor		$t1,$t1,$H2
	vext.8		$Hhl,$t0,$t1,#8		@ pack Karatsuba pre-processed
	vst1.64		{$Hhl-$H2},[x0],#32	@ store Htable[1..2]
___
if ($flavour =~ /64/) {
my ($t3,$Yl,$Ym,$Yh) = map("q$_",(4..7));
my ($H3,$H34k,$H4,$H5,$H56k,$H6,$H7,$H78k,$H8) = map("q$_",(15..23));

$code.=<<___;
	@ calculate H^3 and H^4
	vpmull.p64	$Xl,$H, $H2
	vpmull.p64	$Yl,$H2,$H2
	vpmull2.p64	$Xh,$H, $H2
	vpmull2.p64	$Yh,$H2,$H2
	vpmull.p64	$Xm,$t0,$t1
	vpmull.p64	$Ym,$t1,$t1

	vext.8		$t0,$Xl,$Xh,#8		@ Karatsuba post-processing
	vext.8		$t1,$Yl,$Yh,#8
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t0
	veor		$t3,$Yl,$Yh
	veor		$Ym,$Ym,$t1
	veor		$Xm,$Xm,$t2
	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase
	veor		$Ym,$Ym,$t3
	vpmull.p64	$t3,$Yl,$xC2

	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Yh#lo,$Ym#hi
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	vmov		$Ym#hi,$Yl#lo
	veor		$Xl,$Xm,$t2
	veor		$Yl,$Ym,$t3

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase
	vext.8		$t3,$Yl,$Yl,#8
	vpmull.p64	$Xl,$Xl,$xC2
	vpmull.p64	$Yl,$Yl,$xC2
	veor		$t2,$t2,$Xh
	veor		$t3,$t3,$Yh

	veor		$H3, $Xl,$t2		@ H^3
	veor		$H4,$Yl,$t3			@ H^4

	vext.8		$t0,$H3, $H3,#8		@ Karatsuba pre-processing
	vext.8		$t1,$H4,$H4,#8
	vext.8		$t2,$H2,$H2,#8
	veor		$t0,$t0,$H3
	veor		$t1,$t1,$H4
	veor		$t2,$t2,$H2
	vext.8		$H34k,$t0,$t1,#8	@ pack Karatsuba pre-processed
	vst1.64		{$H3-$H4},[x0],#48	@ store Htable[3..5]

	@ calculate H^5 and H^6
	vpmull.p64	$Xl,$H2, $H3
	vpmull.p64	$Yl,$H3,$H3
	vpmull2.p64	$Xh,$H2, $H3
	vpmull2.p64	$Yh,$H3,$H3
	vpmull.p64	$Xm,$t0,$t2
	vpmull.p64	$Ym,$t0,$t0

	vext.8		$t0,$Xl,$Xh,#8		@ Karatsuba post-processing
	vext.8		$t1,$Yl,$Yh,#8
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t0
	veor		$t3,$Yl,$Yh
	veor		$Ym,$Ym,$t1
	veor		$Xm,$Xm,$t2
	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase
	veor		$Ym,$Ym,$t3
	vpmull.p64	$t3,$Yl,$xC2

	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Yh#lo,$Ym#hi
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	vmov		$Ym#hi,$Yl#lo
	veor		$Xl,$Xm,$t2
	veor		$Yl,$Ym,$t3

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase
	vext.8		$t3,$Yl,$Yl,#8
	vpmull.p64	$Xl,$Xl,$xC2
	vpmull.p64	$Yl,$Yl,$xC2
	veor		$t2,$t2,$Xh
	veor		$t3,$t3,$Yh
	veor		$H5,$Xl,$t2		    @ H^5
	veor		$H6,$Yl,$t3		    @ H^6

	vext.8		$t0,$H5, $H5,#8		@ Karatsuba pre-processing
	vext.8		$t1,$H6,$H6,#8
	vext.8		$t2,$H2,$H2,#8
	veor		$t0,$t0,$H5
	veor		$t1,$t1,$H6
	veor		$t2,$t2,$H2
	vext.8		$H56k,$t0,$t1,#8	@ pack Karatsuba pre-processed
	vst1.64		{$H5-$H6},[x0],#48	@ store Htable[6..8]

	@ calculate H^7 and H^8
	vpmull.p64	$Xl,$H2,$H5
	vpmull.p64	$Yl,$H2,$H6
	vpmull2.p64	$Xh,$H2,$H5
	vpmull2.p64	$Yh,$H2,$H6
	vpmull.p64	$Xm,$t0,$t2
	vpmull.p64	$Ym,$t1,$t2

	vext.8		$t0,$Xl,$Xh,#8		@ Karatsuba post-processing
	vext.8		$t1,$Yl,$Yh,#8
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t0
	veor		$t3,$Yl,$Yh
	veor		$Ym,$Ym,$t1
	veor		$Xm,$Xm,$t2
	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase
	veor		$Ym,$Ym,$t3
	vpmull.p64	$t3,$Yl,$xC2

	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Yh#lo,$Ym#hi
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	vmov		$Ym#hi,$Yl#lo
	veor		$Xl,$Xm,$t2
	veor		$Yl,$Ym,$t3

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase
	vext.8		$t3,$Yl,$Yl,#8
	vpmull.p64	$Xl,$Xl,$xC2
	vpmull.p64	$Yl,$Yl,$xC2
	veor		$t2,$t2,$Xh
	veor		$t3,$t3,$Yh
	veor		$H7,$Xl,$t2		    @ H^7
	veor		$H8,$Yl,$t3		    @ H^8

	vext.8		$t0,$H7,$H7,#8		@ Karatsuba pre-processing
	vext.8		$t1,$H8,$H8,#8
	veor		$t0,$t0,$H7
	veor		$t1,$t1,$H8
	vext.8		$H78k,$t0,$t1,#8	@ pack Karatsuba pre-processed
	vst1.64		{$H7-$H8},[x0]		@ store Htable[9..11]
___
}
$code.=<<___;
	ret
.size	gcm_init_v8,.-gcm_init_v8
___
################################################################################
# void gcm_gmult_v8(u64 Xi[2],const u128 Htable[16]);
#
# input:	Xi - current hash value;
#		Htable - table precomputed in gcm_init_v8;
# output:	Xi - next hash value Xi;
#
$code.=<<___;
.global	gcm_gmult_v8
.type	gcm_gmult_v8,%function
.align	4
gcm_gmult_v8:
	AARCH64_VALID_CALL_TARGET
	vld1.64		{$t1},[$Xi]		@ load Xi
	vmov.i8		$xC2,#0xe1
	vld1.64		{$H-$Hhl},[$Htbl]	@ load twisted H, ...
	vshl.u64	$xC2,$xC2,#57
#ifndef __ARMEB__
	vrev64.8	$t1,$t1
#endif
	vext.8		$IN,$t1,$t1,#8

	vpmull.p64	$Xl,$H,$IN		@ H.loÂ·Xi.lo
	veor		$t1,$t1,$IN		@ Karatsuba pre-processing
	vpmull2.p64	$Xh,$H,$IN		@ H.hiÂ·Xi.hi
	vpmull.p64	$Xm,$Hhl,$t1		@ (H.lo+H.hi)Â·(Xi.lo+Xi.hi)

	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	veor		$Xm,$Xm,$t2
	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction

	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	veor		$Xl,$Xm,$t2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$t2,$t2,$Xh
	veor		$Xl,$Xl,$t2

#ifndef __ARMEB__
	vrev64.8	$Xl,$Xl
#endif
	vext.8		$Xl,$Xl,$Xl,#8
	vst1.64		{$Xl},[$Xi]		@ write out Xi

	ret
.size	gcm_gmult_v8,.-gcm_gmult_v8
___
################################################################################
# void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
#
# input:	table precomputed in gcm_init_v8;
#		current hash value Xi;
#		pointer to input data;
#		length of input data in bytes, but divisible by block size;
# output:	next hash value Xi;
#
$code.=<<___;
.global	gcm_ghash_v8
.type	gcm_ghash_v8,%function
.align	4
gcm_ghash_v8:
	AARCH64_VALID_CALL_TARGET
___
$code.=<<___	if ($flavour =~ /64/);
	cmp		$len,#64
	b.hs		.Lgcm_ghash_v8_4x
___
$code.=<<___		if ($flavour !~ /64/);
	vstmdb		sp!,{d8-d15}		@ 32-bit ABI says so
___
$code.=<<___;
	vld1.64		{$Xl},[$Xi]		@ load [rotated] Xi
						@ "[rotated]" means that
						@ loaded value would have
						@ to be rotated in order to
						@ make it appear as in
						@ algorithm specification
	subs		$len,$len,#32		@ see if $len is 32 or larger
	mov		$inc,#16		@ $inc is used as post-
						@ increment for input pointer;
						@ as loop is modulo-scheduled
						@ $inc is zeroed just in time
						@ to preclude overstepping
						@ inp[len], which means that
						@ last block[s] are actually
						@ loaded twice, but last
						@ copy is not processed
	vld1.64		{$H-$Hhl},[$Htbl],#32	@ load twisted H, ..., H^2
	vmov.i8		$xC2,#0xe1
	vld1.64		{$H2},[$Htbl]
	cclr		$inc,eq			@ is it time to zero $inc?
	vext.8		$Xl,$Xl,$Xl,#8		@ rotate Xi
	vld1.64		{$t0},[$inp],#16	@ load [rotated] I[0]
	vshl.u64	$xC2,$xC2,#57		@ compose 0xc2.0 constant
#ifndef __ARMEB__
	vrev64.8	$t0,$t0
	vrev64.8	$Xl,$Xl
#endif
	vext.8		$IN,$t0,$t0,#8		@ rotate I[0]
	b.lo		.Lodd_tail_v8		@ $len was less than 32
___
{ my ($Xln,$Xmn,$Xhn,$In) = map("q$_",(4..7));
	#######
	# Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
	#	[(H*Ii+1) + (H*Xi+1)] mod P =
	#	[(H*Ii+1) + H^2*(Ii+Xi)] mod P
	#
$code.=<<___;
	vld1.64		{$t1},[$inp],$inc	@ load [rotated] I[1]
#ifndef __ARMEB__
	vrev64.8	$t1,$t1
#endif
	vext.8		$In,$t1,$t1,#8
	veor		$IN,$IN,$Xl		@ I[i]^=Xi
	vpmull.p64	$Xln,$H,$In		@ HÂ·Ii+1
	veor		$t1,$t1,$In		@ Karatsuba pre-processing
	vpmull2.p64	$Xhn,$H,$In
	b		.Loop_mod2x_v8

.align	4
.Loop_mod2x_v8:
	vext.8		$t2,$IN,$IN,#8
	subs		$len,$len,#32		@ is there more data?
	vpmull.p64	$Xl,$H2,$IN		@ H^2.loÂ·Xi.lo
	cclr		$inc,lo			@ is it time to zero $inc?

	vpmull.p64	$Xmn,$Hhl,$t1
	veor		$t2,$t2,$IN		@ Karatsuba pre-processing
	vpmull2.p64	$Xh,$H2,$IN		@ H^2.hiÂ·Xi.hi
	veor		$Xl,$Xl,$Xln		@ accumulate
	vpmull2.p64	$Xm,$Hhl,$t2		@ (H^2.lo+H^2.hi)Â·(Xi.lo+Xi.hi)
	vld1.64	{$t0},[$inp],$inc	@ load [rotated] I[i+2]

	veor		$Xh,$Xh,$Xhn
	cclr		$inc,eq			@ is it time to zero $inc?
	veor		$Xm,$Xm,$Xmn

	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	vld1.64	{$t1},[$inp],$inc	@ load [rotated] I[i+3]
#ifndef __ARMEB__
	vrev64.8	$t0,$t0
#endif
	veor		$Xm,$Xm,$t2
	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction

#ifndef __ARMEB__
	vrev64.8	$t1,$t1
#endif
	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	vext.8		$In,$t1,$t1,#8
	vext.8		$IN,$t0,$t0,#8
	veor		$Xl,$Xm,$t2
	vpmull.p64	$Xln,$H,$In		@ HÂ·Ii+1
	veor		$IN,$IN,$Xh		@ accumulate $IN early

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$IN,$IN,$t2
	veor		$t1,$t1,$In		@ Karatsuba pre-processing
	veor		$IN,$IN,$Xl
	vpmull2.p64	$Xhn,$H,$In
	b.hs		.Loop_mod2x_v8		@ there was at least 32 more bytes

	veor		$Xh,$Xh,$t2
	vext.8		$IN,$t0,$t0,#8		@ re-construct $IN
	adds		$len,$len,#32		@ re-construct $len
	veor		$Xl,$Xl,$Xh		@ re-construct $Xl
	b.eq		.Ldone_v8		@ is $len zero?
___
}
$code.=<<___;
.Lodd_tail_v8:
	vext.8		$t2,$Xl,$Xl,#8
	veor		$IN,$IN,$Xl		@ inp^=Xi
	veor		$t1,$t0,$t2		@ $t1 is rotated inp^Xi

	vpmull.p64	$Xl,$H,$IN		@ H.loÂ·Xi.lo
	veor		$t1,$t1,$IN		@ Karatsuba pre-processing
	vpmull2.p64	$Xh,$H,$IN		@ H.hiÂ·Xi.hi
	vpmull.p64	$Xm,$Hhl,$t1		@ (H.lo+H.hi)Â·(Xi.lo+Xi.hi)

	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	veor		$Xm,$Xm,$t2
	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction

	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	veor		$Xl,$Xm,$t2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$t2,$t2,$Xh
	veor		$Xl,$Xl,$t2

.Ldone_v8:
#ifndef __ARMEB__
	vrev64.8	$Xl,$Xl
#endif
	vext.8		$Xl,$Xl,$Xl,#8
	vst1.64		{$Xl},[$Xi]		@ write out Xi

___
$code.=<<___		if ($flavour !~ /64/);
	vldmia		sp!,{d8-d15}		@ 32-bit ABI says so
___
$code.=<<___;
	ret
.size	gcm_ghash_v8,.-gcm_ghash_v8
___

if ($flavour =~ /64/) {				# 4x subroutine
my ($I0,$j1,$j2,$j3,
    $I1,$I2,$I3,$H3,$H34,$H4,$Yl,$Ym,$Yh) = map("q$_",(4..7,15..23));

$code.=<<___;
.type	gcm_ghash_v8_4x,%function
.align	4
gcm_ghash_v8_4x:
.Lgcm_ghash_v8_4x:
	vld1.64		{$Xl},[$Xi]		@ load [rotated] Xi
	vld1.64		{$H-$H2},[$Htbl],#48	@ load twisted H, ..., H^2
	vmov.i8		$xC2,#0xe1
	vld1.64		{$H3-$H4},[$Htbl]	@ load twisted H^3, ..., H^4
	vshl.u64	$xC2,$xC2,#57		@ compose 0xc2.0 constant

	vld1.64		{$I0-$j3},[$inp],#64
#ifndef __ARMEB__
	vrev64.8	$Xl,$Xl
	vrev64.8	$j1,$j1
	vrev64.8	$j2,$j2
	vrev64.8	$j3,$j3
	vrev64.8	$I0,$I0
#endif
	vext.8		$I3,$j3,$j3,#8
	vext.8		$I2,$j2,$j2,#8
	vext.8		$I1,$j1,$j1,#8

	vpmull.p64	$Yl,$H,$I3		@ HÂ·Ii+3
	veor		$j3,$j3,$I3
	vpmull2.p64	$Yh,$H,$I3
	vpmull.p64	$Ym,$Hhl,$j3

	vpmull.p64	$t0,$H2,$I2		@ H^2Â·Ii+2
	veor		$j2,$j2,$I2
	vpmull2.p64	$I2,$H2,$I2
	vpmull2.p64	$j2,$Hhl,$j2

	veor		$Yl,$Yl,$t0
	veor		$Yh,$Yh,$I2
	veor		$Ym,$Ym,$j2

	vpmull.p64	$j3,$H3,$I1		@ H^3Â·Ii+1
	veor		$j1,$j1,$I1
	vpmull2.p64	$I1,$H3,$I1
	vpmull.p64	$j1,$H34,$j1

	veor		$Yl,$Yl,$j3
	veor		$Yh,$Yh,$I1
	veor		$Ym,$Ym,$j1

	subs		$len,$len,#128
	b.lo		.Ltail4x

	b		.Loop4x

.align	4
.Loop4x:
	veor		$t0,$I0,$Xl
	vld1.64	{$I0-$j3},[$inp],#64
	vext.8		$IN,$t0,$t0,#8
#ifndef __ARMEB__
	vrev64.8	$j1,$j1
	vrev64.8	$j2,$j2
	vrev64.8	$j3,$j3
	vrev64.8	$I0,$I0
#endif

	vpmull.p64	$Xl,$H4,$IN		@ H^4Â·(Xi+Ii)
	veor		$t0,$t0,$IN
	vpmull2.p64	$Xh,$H4,$IN
	vext.8		$I3,$j3,$j3,#8
	vpmull2.p64	$Xm,$H34,$t0

	veor		$Xl,$Xl,$Yl
	veor		$Xh,$Xh,$Yh
	vext.8		$I2,$j2,$j2,#8
	veor		$Xm,$Xm,$Ym
	vext.8		$I1,$j1,$j1,#8

	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	vpmull.p64	$Yl,$H,$I3		@ HÂ·Ii+3
	veor		$j3,$j3,$I3
	veor		$Xm,$Xm,$t1
	vpmull2.p64	$Yh,$H,$I3
	veor		$Xm,$Xm,$t2
	vpmull.p64	$Ym,$Hhl,$j3

	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	vpmull.p64	$t0,$H2,$I2		@ H^2Â·Ii+2
	veor		$j2,$j2,$I2
	vpmull2.p64	$I2,$H2,$I2
	veor		$Xl,$Xm,$t2
	vpmull2.p64	$j2,$Hhl,$j2

	veor		$Yl,$Yl,$t0
	veor		$Yh,$Yh,$I2
	veor		$Ym,$Ym,$j2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	vpmull.p64	$j3,$H3,$I1		@ H^3Â·Ii+1
	veor		$j1,$j1,$I1
	veor		$t2,$t2,$Xh
	vpmull2.p64	$I1,$H3,$I1
	vpmull.p64	$j1,$H34,$j1

	veor		$Xl,$Xl,$t2
	veor		$Yl,$Yl,$j3
	veor		$Yh,$Yh,$I1
	vext.8		$Xl,$Xl,$Xl,#8
	veor		$Ym,$Ym,$j1

	subs		$len,$len,#64
	b.hs		.Loop4x

.Ltail4x:
	veor		$t0,$I0,$Xl
	vext.8		$IN,$t0,$t0,#8

	vpmull.p64	$Xl,$H4,$IN		@ H^4Â·(Xi+Ii)
	veor		$t0,$t0,$IN
	vpmull2.p64	$Xh,$H4,$IN
	vpmull2.p64	$Xm,$H34,$t0

	veor		$Xl,$Xl,$Yl
	veor		$Xh,$Xh,$Yh
	veor		$Xm,$Xm,$Ym

	adds		$len,$len,#64
	b.eq		.Ldone4x

	cmp		$len,#32
	b.lo		.Lone
	b.eq		.Ltwo
.Lthree:
	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	vld1.64	{$I0-$j2},[$inp]
	veor		$Xm,$Xm,$t2
#ifndef	__ARMEB__
	vrev64.8	$j1,$j1
	vrev64.8	$j2,$j2
	vrev64.8	$I0,$I0
#endif

	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	vext.8		$I2,$j2,$j2,#8
	vext.8		$I1,$j1,$j1,#8
	veor		$Xl,$Xm,$t2

	vpmull.p64	$Yl,$H,$I2		@ HÂ·Ii+2
	veor		$j2,$j2,$I2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$t2,$t2,$Xh
	vpmull2.p64	$Yh,$H,$I2
	vpmull.p64	$Ym,$Hhl,$j2
	veor		$Xl,$Xl,$t2
	vpmull.p64	$j3,$H2,$I1		@ H^2Â·Ii+1
	veor		$j1,$j1,$I1
	vext.8		$Xl,$Xl,$Xl,#8

	vpmull2.p64	$I1,$H2,$I1
	veor		$t0,$I0,$Xl
	vpmull2.p64	$j1,$Hhl,$j1
	vext.8		$IN,$t0,$t0,#8

	veor		$Yl,$Yl,$j3
	veor		$Yh,$Yh,$I1
	veor		$Ym,$Ym,$j1

	vpmull.p64	$Xl,$H3,$IN		@ H^3Â·(Xi+Ii)
	veor		$t0,$t0,$IN
	vpmull2.p64	$Xh,$H3,$IN
	vpmull.p64	$Xm,$H34,$t0

	veor		$Xl,$Xl,$Yl
	veor		$Xh,$Xh,$Yh
	veor		$Xm,$Xm,$Ym
	b		.Ldone4x

.align	4
.Ltwo:
	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	vld1.64	{$I0-$j1},[$inp]
	veor		$Xm,$Xm,$t2
#ifndef	__ARMEB__
	vrev64.8	$j1,$j1
	vrev64.8	$I0,$I0
#endif

	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	vext.8		$I1,$j1,$j1,#8
	veor		$Xl,$Xm,$t2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$t2,$t2,$Xh
	veor		$Xl,$Xl,$t2
	vext.8		$Xl,$Xl,$Xl,#8

	vpmull.p64	$Yl,$H,$I1		@ HÂ·Ii+1
	veor		$j1,$j1,$I1

	veor		$t0,$I0,$Xl
	vext.8		$IN,$t0,$t0,#8

	vpmull2.p64	$Yh,$H,$I1
	vpmull.p64	$Ym,$Hhl,$j1

	vpmull.p64	$Xl,$H2,$IN		@ H^2Â·(Xi+Ii)
	veor		$t0,$t0,$IN
	vpmull2.p64	$Xh,$H2,$IN
	vpmull2.p64	$Xm,$Hhl,$t0

	veor		$Xl,$Xl,$Yl
	veor		$Xh,$Xh,$Yh
	veor		$Xm,$Xm,$Ym
	b		.Ldone4x

.align	4
.Lone:
	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	vld1.64	{$I0},[$inp]
	veor		$Xm,$Xm,$t2
#ifndef	__ARMEB__
	vrev64.8	$I0,$I0
#endif

	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	veor		$Xl,$Xm,$t2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$t2,$t2,$Xh
	veor		$Xl,$Xl,$t2
	vext.8		$Xl,$Xl,$Xl,#8

	veor		$t0,$I0,$Xl
	vext.8		$IN,$t0,$t0,#8

	vpmull.p64	$Xl,$H,$IN
	veor		$t0,$t0,$IN
	vpmull2.p64	$Xh,$H,$IN
	vpmull.p64	$Xm,$Hhl,$t0

.Ldone4x:
	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
	veor		$t2,$Xl,$Xh
	veor		$Xm,$Xm,$t1
	veor		$Xm,$Xm,$t2

	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
	veor		$Xl,$Xm,$t2

	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
	vpmull.p64	$Xl,$Xl,$xC2
	veor		$t2,$t2,$Xh
	veor		$Xl,$Xl,$t2
	vext.8		$Xl,$Xl,$Xl,#8

#ifndef __ARMEB__
	vrev64.8	$Xl,$Xl
#endif
	vst1.64		{$Xl},[$Xi]		@ write out Xi

	ret
.size	gcm_ghash_v8_4x,.-gcm_ghash_v8_4x
___

}
}
$code.=<<___;
.asciz  "GHASH for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align  2
#endif
___

if ($flavour =~ /64/) {			######## 64-bit code
    sub unvmov {
	my $arg=shift;

	$arg =~ m/q([0-9]+)#(lo|hi),\s*q([0-9]+)#(lo|hi)/o &&
	sprintf	"ins	v%d.d[%d],v%d.d[%d]",$1<8?$1:$1+8,($2 eq "lo")?0:1,
					     $3<8?$3:$3+8,($4 eq "lo")?0:1;
    }
    foreach(split("\n",$code)) {
	s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel	$1$2,$1zr,$1$2,$3/o	or
	s/vmov\.i8/movi/o		or	# fix up legacy mnemonics
	s/vmov\s+(.*)/unvmov($1)/geo	or
	s/vext\.8/ext/o			or
	s/vshr\.s/sshr\.s/o		or
	s/vshr/ushr/o			or
	s/^(\s+)v/$1/o			or	# strip off v prefix
	s/\bbx\s+lr\b/ret/o;

	s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo;	# old->new registers
	s/@\s/\/\//o;				# old->new style commentary

	# fix up remaining legacy suffixes
	s/\.[ui]?8(\s)/$1/o;
	s/\.[uis]?32//o and s/\.16b/\.4s/go;
	m/\.p64/o and s/\.16b/\.1q/o;		# 1st pmull argument
	m/l\.p64/o and s/\.16b/\.1d/go;		# 2nd and 3rd pmull arguments
	s/\.[uisp]?64//o and s/\.16b/\.2d/go;
	s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o;

	# Switch preprocessor checks to aarch64 versions.
	s/__ARME([BL])__/__AARCH64E$1__/go;

	print $_,"\n";
    }
} else {				######## 32-bit code
    sub unvdup32 {
	my $arg=shift;

	$arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o &&
	sprintf	"vdup.32	q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1;
    }
    sub unvpmullp64 {
	my ($mnemonic,$arg)=@_;

	if ($arg =~ m/q([0-9]+),\s*q([0-9]+),\s*q([0-9]+)/o) {
	    my $word = 0xf2a00e00|(($1&7)<<13)|(($1&8)<<19)
				 |(($2&7)<<17)|(($2&8)<<4)
				 |(($3&7)<<1) |(($3&8)<<2);
	    $word |= 0x00010001	 if ($mnemonic =~ "2");
	    # since ARMv7 instructions are always encoded little-endian.
	    # correct solution is to use .inst directive, but older
	    # assemblers don't implement it:-(
	    sprintf ".byte\t0x%02x,0x%02x,0x%02x,0x%02x\t@ %s %s",
			$word&0xff,($word>>8)&0xff,
			($word>>16)&0xff,($word>>24)&0xff,
			$mnemonic,$arg;
	}
    }

    foreach(split("\n",$code)) {
	s/\b[wx]([0-9]+)\b/r$1/go;		# new->old registers
	s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go;	# new->old registers
	s/\/\/\s?/@ /o;				# new->old style commentary

	# fix up remaining new-style suffixes
	s/\],#[0-9]+/]!/o;

	s/cclr\s+([^,]+),\s*([a-z]+)/mov$2	$1,#0/o			or
	s/vdup\.32\s+(.*)/unvdup32($1)/geo				or
	s/v?(pmull2?)\.p64\s+(.*)/unvpmullp64($1,$2)/geo		or
	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
	s/^(\s+)b\./$1b/o						or
	s/^(\s+)ret/$1bx\tlr/o;

	print $_,"\n";
    }
}

close STDOUT or die "error closing STDOUT"; # enforce flush


#!/usr/bin/env perl

# Copyright (c) 2015, Google Inc.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */

use strict;

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

my $flavour = shift;
my $output  = shift;

my $win64 = 0;
$win64 = 1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/;
my $dir = $1;
my $xlate;
( $xlate="${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

my ($out, $len, $tmp1, $tmp2) = $win64 ? ("%rcx", "%rdx", "%r8", "%r9")
                                       : ("%rdi", "%rsi", "%rdx", "%rcx");

print<<___;
.text

# CRYPTO_rdrand writes eight bytes of random data from the hardware RNG to
# |out|. It returns one on success or zero on hardware failure.
# int CRYPTO_rdrand(uint8_t out[8]);
.globl	CRYPTO_rdrand
.type	CRYPTO_rdrand,\@abi-omnipotent
.align	16
CRYPTO_rdrand:
.cfi_startproc
	xorq %rax, %rax
	rdrand $tmp1
	test $tmp1, $tmp1 # OLD cpu's: can use all 0s in output as error signal
	jz .Lerr
	cmp \$-1, $tmp1 # AMD bug: check if all returned bits by RDRAND is stuck on 1
	je .Lerr
	# An add-with-carry of zero effectively sets %rax to the carry flag.
	adcq %rax, %rax
	movq $tmp1, 0($out)
	retq
.Lerr:
	xorq %rax, %rax
	retq
.cfi_endproc
.size CRYPTO_rdrand,.-CRYPTO_rdrand

# CRYPTO_rdrand_multiple8_buf fills |len| bytes at |buf| with random data from
# the hardware RNG. The |len| argument must be a multiple of eight. It returns
# one on success and zero on hardware failure.
# int CRYPTO_rdrand_multiple8_buf(uint8_t *buf, size_t len);
.globl CRYPTO_rdrand_multiple8_buf
.type CRYPTO_rdrand_multiple8_buf,\@abi-omnipotent
.align 16
CRYPTO_rdrand_multiple8_buf:
.cfi_startproc
	test $len, $len
	jz .Lout
	movq \$8, $tmp1
.Lloop:
	rdrand $tmp2
	jnc .Lerr_multiple
	test $tmp2, $tmp2 # OLD cpu's: can use all 0s in output as error signal
	jz .Lerr_multiple
	cmp \$-1, $tmp2 # AMD bug: check if all returned bits by RDRAND is stuck on 1
	je .Lerr_multiple
	movq $tmp2, 0($out)
	addq $tmp1, $out
	subq $tmp1, $len
	jnz .Lloop
.Lout:
	movq \$1, %rax
	retq
.Lerr_multiple:
	xorq %rax, %rax
	retq
.cfi_endproc
.size CRYPTO_rdrand_multiple8_buf,.-CRYPTO_rdrand_multiple8_buf
___

close STDOUT or die "error closing STDOUT: $!";	# flush


#!/usr/bin/env perl
# Copyright 2017-2022 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for ARMv8.
#
# June 2017.
#
# This is straightforward KECCAK_1X_ALT implementation. It makes no
# sense to attempt SIMD/NEON implementation for following reason.
# 64-bit lanes of vector registers can't be addressed as easily as in
# 32-bit mode. This means that 64-bit NEON is bound to be slower than
# 32-bit NEON, and this implementation is faster than 32-bit NEON on
# same processor. Even though it takes more scalar xor's and andn's,
# it gets compensated by availability of rotate. Not to forget that
# most processors achieve higher issue rate with scalar instructions.
#
# February 2018.
#
# Add hardware-assisted ARMv8.2 implementation. It's KECCAK_1X_ALT
# variant with register permutation/rotation twist that allows to
# eliminate copies to temporary registers. If you look closely you'll
# notice that it uses only one lane of vector registers. The new
# instructions effectively facilitate parallel hashing, which we don't
# support [yet?]. But lowest-level core procedure is prepared for it.
# The inner round is 67 [vector] instructions, so it's not actually
# obvious that it will provide performance improvement [in serial
# hash] as long as vector instructions issue rate is limited to 1 per
# cycle...
#
######################################################################
# Numbers are cycles per processed byte.
#
#		r=1088(*)
#
# Cortex-A53	13
# Cortex-A57	12
# X-Gene	14
# Mongoose	10
# Kryo		12
# Denver	7.8
# Apple A7	7.2
# ThunderX2	9.7
#
# (*)	Corresponds to SHA3-256. No improvement coefficients are listed
#	because they vary too much from compiler to compiler. Newer
#	compiler does much better and improvement varies from 5% on
#	Cortex-A57 to 25% on Cortex-A53. While in comparison to older
#	compiler this code is at least 2x faster...

# File keccak1600-armv8.pl is imported from OpenSSL.
# https://github.com/openssl/openssl/blob/479b9adb88b9050186c1e9fc94879906f378b14b/crypto/sha/asm/keccak1600-armv8.pl

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

$flavour = shift;
$output = shift;

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
    *STDOUT=*OUT;
} else {
    open OUT,">$output";
    *STDOUT=*OUT;
}

my @subrhotates = ([  64,  63, 2, 36, 37 ],
                [ 28, 20,  58, 9, 44 ],
                [  61, 54, 21, 39, 25 ],
                [ 23, 19, 49, 43,  56 ],
                [ 46,  62, 3, 8, 50 ]);

$code.=<<___;
#include <openssl/arm_arch.h>
.text
.align 8	// strategic alignment and padding that allows to use
		// address value as loop termination condition...
	.quad	0,0,0,0,0,0,0,0
.type	iotas,%object
iotas:
	.quad	0x0000000000000001
	.quad	0x0000000000008082
	.quad	0x800000000000808a
	.quad	0x8000000080008000
	.quad	0x000000000000808b
	.quad	0x0000000080000001
	.quad	0x8000000080008081
	.quad	0x8000000000008009
	.quad	0x000000000000008a
	.quad	0x0000000000000088
	.quad	0x0000000080008009
	.quad	0x000000008000000a
	.quad	0x000000008000808b
	.quad	0x800000000000008b
	.quad	0x8000000000008089
	.quad	0x8000000000008003
	.quad	0x8000000000008002
	.quad	0x8000000000000080
	.quad	0x000000000000800a
	.quad	0x800000008000000a
	.quad	0x8000000080008081
	.quad	0x8000000000008080
	.quad	0x0000000080000001
	.quad	0x8000000080008008
.size	iotas,.-iotas
___
								{{{
my @A = map([ "x$_", "x".($_+1), "x".($_+2), "x".($_+3), "x".($_+4) ],
            (0, 5, 10, 15, 20));
   $A[3][3] = "x25"; # x18 is reserved

my @C = map("x$_", (26,27,28,30));

$code.=<<___;
.type	KeccakF1600_int,%function
.align	5
KeccakF1600_int:
	AARCH64_SIGN_LINK_REGISTER
	adr	$C[2],iotas
	stp	$C[2],x30,[sp,#16]		// 32 bytes on top are mine
	b	.Loop
.align	4
.Loop:
	////////////////////////////////////////// Theta
	eor	$C[0],$A[0][0],$A[1][0]
	stp	$A[0][4],$A[1][4],[sp,#0]	// offload pair...
	eor	$C[1],$A[0][1],$A[1][1]
	eor	$C[2],$A[0][2],$A[1][2]
	eor	$C[3],$A[0][3],$A[1][3]
___
	$C[4]=$A[0][4];
	$C[5]=$A[1][4];
$code.=<<___;
	eor	$C[4],$A[0][4],$A[1][4]
	eor	$C[0],$C[0],$A[2][0]
	eor	$C[1],$C[1],$A[2][1]
	eor	$C[2],$C[2],$A[2][2]
	eor	$C[3],$C[3],$A[2][3]
	eor	$C[4],$C[4],$A[2][4]
	eor	$C[0],$C[0],$A[3][0]
	eor	$C[1],$C[1],$A[3][1]
	eor	$C[2],$C[2],$A[3][2]
	eor	$C[3],$C[3],$A[3][3]
	eor	$C[4],$C[4],$A[3][4]
	eor	$C[0],$C[0],$A[4][0]
	eor	$C[2],$C[2],$A[4][2]
	eor	$C[1],$C[1],$A[4][1]
	eor	$C[3],$C[3],$A[4][3]
	eor	$C[4],$C[4],$A[4][4]
	eor	$C[5],$C[0],$C[2],ror#63
	eor	$A[0][1],$A[0][1],$C[5]
	eor	$A[1][1],$A[1][1],$C[5]
	eor	$A[2][1],$A[2][1],$C[5]
	eor	$A[3][1],$A[3][1],$C[5]
	eor	$A[4][1],$A[4][1],$C[5]
	eor	$C[5],$C[1],$C[3],ror#63
	eor	$C[2],$C[2],$C[4],ror#63
	eor	$C[3],$C[3],$C[0],ror#63
	eor	$C[4],$C[4],$C[1],ror#63
	eor	$C[1],   $A[0][2],$C[5]		// mov	$C[1],$A[0][2]
	eor	$A[1][2],$A[1][2],$C[5]
	eor	$A[2][2],$A[2][2],$C[5]
	eor	$A[3][2],$A[3][2],$C[5]
	eor	$A[4][2],$A[4][2],$C[5]
	eor	$A[0][0],$A[0][0],$C[4]
	eor	$A[1][0],$A[1][0],$C[4]
	eor	$A[2][0],$A[2][0],$C[4]
	eor	$A[3][0],$A[3][0],$C[4]
	eor	$A[4][0],$A[4][0],$C[4]
___
	$C[4]=undef;
	$C[5]=undef;
$code.=<<___;
	ldp	$A[0][4],$A[1][4],[sp,#0]	// re-load offloaded data
	eor	$C[0],   $A[0][3],$C[2]		// mov	$C[0],$A[0][3]
	eor	$A[1][3],$A[1][3],$C[2]
	eor	$A[2][3],$A[2][3],$C[2]
	eor	$A[3][3],$A[3][3],$C[2]
	eor	$A[4][3],$A[4][3],$C[2]
	eor	$C[2],   $A[0][4],$C[3]		// mov	$C[2],$A[0][4]
	eor	$A[1][4],$A[1][4],$C[3]
	eor	$A[2][4],$A[2][4],$C[3]
	eor	$A[3][4],$A[3][4],$C[3]
	eor	$A[4][4],$A[4][4],$C[3]
	////////////////////////////////////////// Rho+Pi
	mov	$C[3],$A[0][1]
	ror	$A[0][1],$A[1][1],#$subrhotates[1][1]
	//mov	$C[1],$A[0][2]
	ror	$A[0][2],$A[2][2],#$subrhotates[2][2]
	//mov	$C[0],$A[0][3]
	ror	$A[0][3],$A[3][3],#$subrhotates[3][3]
	//mov	$C[2],$A[0][4]
	ror	$A[0][4],$A[4][4],#$subrhotates[4][4]
	ror	$A[1][1],$A[1][4],#$subrhotates[1][4]
	ror	$A[2][2],$A[2][3],#$subrhotates[2][3]
	ror	$A[3][3],$A[3][2],#$subrhotates[3][2]
	ror	$A[4][4],$A[4][1],#$subrhotates[4][1]
	ror	$A[1][4],$A[4][2],#$subrhotates[4][2]
	ror	$A[2][3],$A[3][4],#$subrhotates[3][4]
	ror	$A[3][2],$A[2][1],#$subrhotates[2][1]
	ror	$A[4][1],$A[1][3],#$subrhotates[1][3]
	ror	$A[4][2],$A[2][4],#$subrhotates[2][4]
	ror	$A[3][4],$A[4][3],#$subrhotates[4][3]
	ror	$A[2][1],$A[1][2],#$subrhotates[1][2]
	ror	$A[1][3],$A[3][1],#$subrhotates[3][1]
	ror	$A[2][4],$A[4][0],#$subrhotates[4][0]
	ror	$A[4][3],$A[3][0],#$subrhotates[3][0]
	ror	$A[1][2],$A[2][0],#$subrhotates[2][0]
	ror	$A[3][1],$A[1][0],#$subrhotates[1][0]
	ror	$A[1][0],$C[0],#$subrhotates[0][3]
	ror	$A[2][0],$C[3],#$subrhotates[0][1]
	ror	$A[3][0],$C[2],#$subrhotates[0][4]
	ror	$A[4][0],$C[1],#$subrhotates[0][2]
	////////////////////////////////////////// Chi+Iota
	bic	$C[0],$A[0][2],$A[0][1]
	bic	$C[1],$A[0][3],$A[0][2]
	bic	$C[2],$A[0][0],$A[0][4]
	bic	$C[3],$A[0][1],$A[0][0]
	eor	$A[0][0],$A[0][0],$C[0]
	bic	$C[0],$A[0][4],$A[0][3]
	eor	$A[0][1],$A[0][1],$C[1]
	 ldr	$C[1],[sp,#16]
	eor	$A[0][3],$A[0][3],$C[2]
	eor	$A[0][4],$A[0][4],$C[3]
	eor	$A[0][2],$A[0][2],$C[0]
	 ldr	$C[3],[$C[1]],#8		// Iota[i++]
	bic	$C[0],$A[1][2],$A[1][1]
	 tst	$C[1],#255			// are we done?
	 str	$C[1],[sp,#16]
	bic	$C[1],$A[1][3],$A[1][2]
	bic	$C[2],$A[1][0],$A[1][4]
	 eor	$A[0][0],$A[0][0],$C[3]		// A[0][0] ^= Iota
	bic	$C[3],$A[1][1],$A[1][0]
	eor	$A[1][0],$A[1][0],$C[0]
	bic	$C[0],$A[1][4],$A[1][3]
	eor	$A[1][1],$A[1][1],$C[1]
	eor	$A[1][3],$A[1][3],$C[2]
	eor	$A[1][4],$A[1][4],$C[3]
	eor	$A[1][2],$A[1][2],$C[0]
	bic	$C[0],$A[2][2],$A[2][1]
	bic	$C[1],$A[2][3],$A[2][2]
	bic	$C[2],$A[2][0],$A[2][4]
	bic	$C[3],$A[2][1],$A[2][0]
	eor	$A[2][0],$A[2][0],$C[0]
	bic	$C[0],$A[2][4],$A[2][3]
	eor	$A[2][1],$A[2][1],$C[1]
	eor	$A[2][3],$A[2][3],$C[2]
	eor	$A[2][4],$A[2][4],$C[3]
	eor	$A[2][2],$A[2][2],$C[0]
	bic	$C[0],$A[3][2],$A[3][1]
	bic	$C[1],$A[3][3],$A[3][2]
	bic	$C[2],$A[3][0],$A[3][4]
	bic	$C[3],$A[3][1],$A[3][0]
	eor	$A[3][0],$A[3][0],$C[0]
	bic	$C[0],$A[3][4],$A[3][3]
	eor	$A[3][1],$A[3][1],$C[1]
	eor	$A[3][3],$A[3][3],$C[2]
	eor	$A[3][4],$A[3][4],$C[3]
	eor	$A[3][2],$A[3][2],$C[0]
	bic	$C[0],$A[4][2],$A[4][1]
	bic	$C[1],$A[4][3],$A[4][2]
	bic	$C[2],$A[4][0],$A[4][4]
	bic	$C[3],$A[4][1],$A[4][0]
	eor	$A[4][0],$A[4][0],$C[0]
	bic	$C[0],$A[4][4],$A[4][3]
	eor	$A[4][1],$A[4][1],$C[1]
	eor	$A[4][3],$A[4][3],$C[2]
	eor	$A[4][4],$A[4][4],$C[3]
	eor	$A[4][2],$A[4][2],$C[0]
	bne	.Loop
	ldr	x30,[sp,#24]
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	KeccakF1600_int,.-KeccakF1600_int
.type	KeccakF1600,%function
.align	5
KeccakF1600:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]
	sub	sp,sp,#48
	str	x0,[sp,#32]			// offload argument
	mov	$C[0],x0
	ldp	$A[0][0],$A[0][1],[x0,#16*0]
	ldp	$A[0][2],$A[0][3],[$C[0],#16*1]
	ldp	$A[0][4],$A[1][0],[$C[0],#16*2]
	ldp	$A[1][1],$A[1][2],[$C[0],#16*3]
	ldp	$A[1][3],$A[1][4],[$C[0],#16*4]
	ldp	$A[2][0],$A[2][1],[$C[0],#16*5]
	ldp	$A[2][2],$A[2][3],[$C[0],#16*6]
	ldp	$A[2][4],$A[3][0],[$C[0],#16*7]
	ldp	$A[3][1],$A[3][2],[$C[0],#16*8]
	ldp	$A[3][3],$A[3][4],[$C[0],#16*9]
	ldp	$A[4][0],$A[4][1],[$C[0],#16*10]
	ldp	$A[4][2],$A[4][3],[$C[0],#16*11]
	ldr	$A[4][4],[$C[0],#16*12]
	bl	KeccakF1600_int
	ldr	$C[0],[sp,#32]
	stp	$A[0][0],$A[0][1],[$C[0],#16*0]
	stp	$A[0][2],$A[0][3],[$C[0],#16*1]
	stp	$A[0][4],$A[1][0],[$C[0],#16*2]
	stp	$A[1][1],$A[1][2],[$C[0],#16*3]
	stp	$A[1][3],$A[1][4],[$C[0],#16*4]
	stp	$A[2][0],$A[2][1],[$C[0],#16*5]
	stp	$A[2][2],$A[2][3],[$C[0],#16*6]
	stp	$A[2][4],$A[3][0],[$C[0],#16*7]
	stp	$A[3][1],$A[3][2],[$C[0],#16*8]
	stp	$A[3][3],$A[3][4],[$C[0],#16*9]
	stp	$A[4][0],$A[4][1],[$C[0],#16*10]
	stp	$A[4][2],$A[4][3],[$C[0],#16*11]
	str	$A[4][4],[$C[0],#16*12]
	ldp	x19,x20,[x29,#16]
	add	sp,sp,#48
	ldp	x21,x22,[x29,#32]
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldp	x29,x30,[sp],#128
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	KeccakF1600,.-KeccakF1600
.globl	SHA3_Absorb_hw
.type	SHA3_Absorb_hw,%function
.align	5
SHA3_Absorb_hw:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]
	sub	sp,sp,#64
	stp	x0,x1,[sp,#32]			// offload arguments
	stp	x2,x3,[sp,#48]
	mov	$C[0],x0			// uint64_t A[5][5]
	mov	$C[1],x1			// const void *inp
	mov	$C[2],x2			// size_t len
	mov	$C[3],x3			// size_t bsz
	ldp	$A[0][0],$A[0][1],[$C[0],#16*0]
	ldp	$A[0][2],$A[0][3],[$C[0],#16*1]
	ldp	$A[0][4],$A[1][0],[$C[0],#16*2]
	ldp	$A[1][1],$A[1][2],[$C[0],#16*3]
	ldp	$A[1][3],$A[1][4],[$C[0],#16*4]
	ldp	$A[2][0],$A[2][1],[$C[0],#16*5]
	ldp	$A[2][2],$A[2][3],[$C[0],#16*6]
	ldp	$A[2][4],$A[3][0],[$C[0],#16*7]
	ldp	$A[3][1],$A[3][2],[$C[0],#16*8]
	ldp	$A[3][3],$A[3][4],[$C[0],#16*9]
	ldp	$A[4][0],$A[4][1],[$C[0],#16*10]
	ldp	$A[4][2],$A[4][3],[$C[0],#16*11]
	ldr	$A[4][4],[$C[0],#16*12]
	b	.Loop_absorb
.align	4
.Loop_absorb:
	subs	$C[0],$C[2],$C[3]		// len - bsz
	blo	.Labsorbed
	str	$C[0],[sp,#48]			// save len - bsz
___
for (my $i=0; $i<24; $i+=2) {
my $j = $i+1;
$code.=<<___;
	ldr	$C[0],[$C[1]],#8		// *inp++
#ifdef	__AARCH64EB__
	rev	$C[0],$C[0]
#endif
	eor	$A[$i/5][$i%5],$A[$i/5][$i%5],$C[0]
	cmp	$C[3],#8*($i+2)
	blo	.Lprocess_block
	ldr	$C[0],[$C[1]],#8		// *inp++
#ifdef	__AARCH64EB__
	rev	$C[0],$C[0]
#endif
	eor	$A[$j/5][$j%5],$A[$j/5][$j%5],$C[0]
	beq	.Lprocess_block
___
}
$code.=<<___;
	ldr	$C[0],[$C[1]],#8		// *inp++
#ifdef	__AARCH64EB__
	rev	$C[0],$C[0]
#endif
	eor	$A[4][4],$A[4][4],$C[0]
.Lprocess_block:
	str	$C[1],[sp,#40]			// save inp
	bl	KeccakF1600_int
	ldr	$C[1],[sp,#40]			// restore arguments
	ldp	$C[2],$C[3],[sp,#48]
	b	.Loop_absorb
.align	4
.Labsorbed:
	ldr	$C[1],[sp,#32]
	stp	$A[0][0],$A[0][1],[$C[1],#16*0]
	stp	$A[0][2],$A[0][3],[$C[1],#16*1]
	stp	$A[0][4],$A[1][0],[$C[1],#16*2]
	stp	$A[1][1],$A[1][2],[$C[1],#16*3]
	stp	$A[1][3],$A[1][4],[$C[1],#16*4]
	stp	$A[2][0],$A[2][1],[$C[1],#16*5]
	stp	$A[2][2],$A[2][3],[$C[1],#16*6]
	stp	$A[2][4],$A[3][0],[$C[1],#16*7]
	stp	$A[3][1],$A[3][2],[$C[1],#16*8]
	stp	$A[3][3],$A[3][4],[$C[1],#16*9]
	stp	$A[4][0],$A[4][1],[$C[1],#16*10]
	stp	$A[4][2],$A[4][3],[$C[1],#16*11]
	str	$A[4][4],[$C[1],#16*12]
	mov	x0,$C[2]			// return value
	ldp	x19,x20,[x29,#16]
	add	sp,sp,#64
	ldp	x21,x22,[x29,#32]
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldp	x29,x30,[sp],#128
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	SHA3_Absorb_hw,.-SHA3_Absorb_hw
___
{
my ($A_flat,$out,$len,$bsz) = map("x$_",(19..22));
$code.=<<___;
.globl	SHA3_Squeeze_hw
.type	SHA3_Squeeze_hw,%function
.align	5
SHA3_Squeeze_hw:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-48]!
	add	x29,sp,#0
	cmp	x2,#0
	beq	.Lsqueeze_abort
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	mov	$A_flat,x0			// put aside arguments
	mov	$out,x1
	mov	$len,x2
	mov	$bsz,x3
.Loop_squeeze:
	ldr	x4,[x0],#8
	cmp	$len,#8
	blo	.Lsqueeze_tail
#ifdef	__AARCH64EB__
	rev	x4,x4
#endif
	str	x4,[$out],#8
	subs	$len,$len,#8
	beq	.Lsqueeze_done
	subs	x3,x3,#8
	bhi	.Loop_squeeze
	mov	x0,$A_flat
	bl	KeccakF1600
	mov	x0,$A_flat
	mov	x3,$bsz
	b	.Loop_squeeze
.align	4
.Lsqueeze_tail:
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done
	strb	w4,[$out],#1
.Lsqueeze_done:
	ldp	x19,x20,[sp,#16]
	ldp	x21,x22,[sp,#32]
.Lsqueeze_abort:
	ldp	x29,x30,[sp],#48
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	SHA3_Squeeze_hw,.-SHA3_Squeeze_hw
___
}								}}}
								{{{
my @A = map([ "v".$_.".16b", "v".($_+1).".16b", "v".($_+2).".16b",
                             "v".($_+3).".16b", "v".($_+4).".16b" ],
            (0, 5, 10, 15, 20));

my @C = map("v$_.16b", (25..31));
my @D = @C[4,5,6,2,3];

$code.=<<___;
.type	KeccakF1600_ce,%function
.align	5
KeccakF1600_ce:
	mov	x9,#24
	adr	x10,iotas
	b	.Loop_ce
.align	4
.Loop_ce:
	////////////////////////////////////////////////// Theta
	eor3	$C[0],$A[4][0],$A[3][0],$A[2][0]
	eor3	$C[1],$A[4][1],$A[3][1],$A[2][1]
	eor3	$C[2],$A[4][2],$A[3][2],$A[2][2]
	eor3	$C[3],$A[4][3],$A[3][3],$A[2][3]
	eor3	$C[4],$A[4][4],$A[3][4],$A[2][4]
	eor3	$C[0],$C[0],   $A[1][0],$A[0][0]
	eor3	$C[1],$C[1],   $A[1][1],$A[0][1]
	eor3	$C[2],$C[2],   $A[1][2],$A[0][2]
	eor3	$C[3],$C[3],   $A[1][3],$A[0][3]
	eor3	$C[4],$C[4],   $A[1][4],$A[0][4]
	rax1	$C[5],$C[0],$C[2]			// D[1]
	rax1	$C[6],$C[1],$C[3]			// D[2]
	rax1	$C[2],$C[2],$C[4]			// D[3]
	rax1	$C[3],$C[3],$C[0]			// D[4]
	rax1	$C[4],$C[4],$C[1]			// D[0]
	////////////////////////////////////////////////// Theta+Rho+Pi
	xar	$C[0],   $A[0][1],$D[1],#$subrhotates[0][1] // C[0]=A[2][0]
	xar	$A[0][1],$A[1][1],$D[1],#$subrhotates[1][1]
	xar	$A[1][1],$A[1][4],$D[4],#$subrhotates[1][4]
	xar	$A[1][4],$A[4][2],$D[2],#$subrhotates[4][2]
	xar	$A[4][2],$A[2][4],$D[4],#$subrhotates[2][4]
	xar	$A[2][4],$A[4][0],$D[0],#$subrhotates[4][0]
	xar	$C[1],   $A[0][2],$D[2],#$subrhotates[0][2] // C[1]=A[4][0]
	xar	$A[0][2],$A[2][2],$D[2],#$subrhotates[2][2]
	xar	$A[2][2],$A[2][3],$D[3],#$subrhotates[2][3]
	xar	$A[2][3],$A[3][4],$D[4],#$subrhotates[3][4]
	xar	$A[3][4],$A[4][3],$D[3],#$subrhotates[4][3]
	xar	$A[4][3],$A[3][0],$D[0],#$subrhotates[3][0]
	xar	$A[3][0],$A[0][4],$D[4],#$subrhotates[0][4]
	xar	$D[4],   $A[4][4],$D[4],#$subrhotates[4][4] // D[4]=A[0][4]
	xar	$A[4][4],$A[4][1],$D[1],#$subrhotates[4][1]
	xar	$A[1][3],$A[1][3],$D[3],#$subrhotates[1][3] // A[1][3]=A[4][1]
	xar	$A[0][4],$A[3][1],$D[1],#$subrhotates[3][1] // A[0][4]=A[1][3]
	xar	$A[3][1],$A[1][0],$D[0],#$subrhotates[1][0]
	xar	$A[1][0],$A[0][3],$D[3],#$subrhotates[0][3]
	eor	$A[0][0],$A[0][0],$D[0]
	xar	$D[3],   $A[3][3],$D[3],#$subrhotates[3][3] // D[3]=A[0][3]
	xar	$A[0][3],$A[3][2],$D[2],#$subrhotates[3][2] // A[0][3]=A[3][3]
	xar	$D[1],   $A[2][1],$D[1],#$subrhotates[2][1] // D[1]=A[3][2]
	xar	$D[2],   $A[1][2],$D[2],#$subrhotates[1][2] // D[2]=A[2][1]
	xar	$D[0],   $A[2][0],$D[0],#$subrhotates[2][0] // D[0]=A[1][2]
	////////////////////////////////////////////////// Chi+Iota
	bcax	$A[4][0],$C[1],   $A[4][2],$A[1][3]	// A[1][3]=A[4][1]
	bcax	$A[4][1],$A[1][3],$A[4][3],$A[4][2]	// A[1][3]=A[4][1]
	bcax	$A[4][2],$A[4][2],$A[4][4],$A[4][3]
	bcax	$A[4][3],$A[4][3],$C[1],   $A[4][4]
	bcax	$A[4][4],$A[4][4],$A[1][3],$C[1]	// A[1][3]=A[4][1]
	ld1r	{$C[1]},[x10],#8
	bcax	$A[3][2],$D[1],   $A[3][4],$A[0][3]	// A[0][3]=A[3][3]
	bcax	$A[3][3],$A[0][3],$A[3][0],$A[3][4]	// A[0][3]=A[3][3]
	bcax	$A[3][4],$A[3][4],$A[3][1],$A[3][0]
	bcax	$A[3][0],$A[3][0],$D[1],   $A[3][1]
	bcax	$A[3][1],$A[3][1],$A[0][3],$D[1]	// A[0][3]=A[3][3]
	bcax	$A[2][0],$C[0],   $A[2][2],$D[2]
	bcax	$A[2][1],$D[2],   $A[2][3],$A[2][2]
	bcax	$A[2][2],$A[2][2],$A[2][4],$A[2][3]
	bcax	$A[2][3],$A[2][3],$C[0],   $A[2][4]
	bcax	$A[2][4],$A[2][4],$D[2],   $C[0]
	bcax	$A[1][2],$D[0],   $A[1][4],$A[0][4]	// A[0][4]=A[1][3]
	bcax	$A[1][3],$A[0][4],$A[1][0],$A[1][4]	// A[0][4]=A[1][3]
	bcax	$A[1][4],$A[1][4],$A[1][1],$A[1][0]
	bcax	$A[1][0],$A[1][0],$D[0],   $A[1][1]
	bcax	$A[1][1],$A[1][1],$A[0][4],$D[0]	// A[0][4]=A[1][3]
	bcax	$A[0][3],$D[3],   $A[0][0],$D[4]
	bcax	$A[0][4],$D[4],   $A[0][1],$A[0][0]
	bcax	$A[0][0],$A[0][0],$A[0][2],$A[0][1]
	bcax	$A[0][1],$A[0][1],$D[3],   $A[0][2]
	bcax	$A[0][2],$A[0][2],$D[4],   $D[3]
	eor	$A[0][0],$A[0][0],$C[1]
	subs	x9,x9,#1
	bne	.Loop_ce
	ret
.size	KeccakF1600_ce,.-KeccakF1600_ce
.type	KeccakF1600_cext,%function
.align	5
KeccakF1600_cext:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-80]!
	add	x29,sp,#0
	stp	d8,d9,[sp,#16]		// per ABI requirement
	stp	d10,d11,[sp,#32]
	stp	d12,d13,[sp,#48]
	stp	d14,d15,[sp,#64]
___
for($i=0; $i<24; $i+=2) {		# load A[5][5]
my $j=$i+1;
$code.=<<___;
	ldp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	ldr	d24,[x0,#8*$i]
	bl	KeccakF1600_ce
	ldr	x30,[sp,#8]
___
for($i=0; $i<24; $i+=2) {		# store A[5][5]
my $j=$i+1;
$code.=<<___;
	stp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	str	d24,[x0,#8*$i]
	ldp	d8,d9,[sp,#16]
	ldp	d10,d11,[sp,#32]
	ldp	d12,d13,[sp,#48]
	ldp	d14,d15,[sp,#64]
	ldr	x29,[sp],#80
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	KeccakF1600_cext,.-KeccakF1600_cext
___

{
my ($ctx,$inp,$len,$bsz) = map("x$_",(0..3));

$code.=<<___;
.globl	SHA3_Absorb_cext
.type	SHA3_Absorb_cext,%function
.align	5
SHA3_Absorb_cext:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-80]!
	add	x29,sp,#0
	stp	d8,d9,[sp,#16]		// per ABI requirement
	stp	d10,d11,[sp,#32]
	stp	d12,d13,[sp,#48]
	stp	d14,d15,[sp,#64]
___
for($i=0; $i<24; $i+=2) {		# load A[5][5]
my $j=$i+1;
$code.=<<___;
	ldp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	ldr	d24,[x0,#8*$i]
	b	.Loop_absorb_ce
.align	4
.Loop_absorb_ce:
	subs	$len,$len,$bsz		// len - bsz
	blo	.Labsorbed_ce
___
for (my $i=0; $i<24; $i+=2) {
my $j = $i+1;
$code.=<<___;
	ldr	d31,[$inp],#8		// *inp++
#ifdef	__AARCH64EB__
	rev64	v31.16b,v31.16b
#endif
	eor	$A[$i/5][$i%5],$A[$i/5][$i%5],v31.16b
	cmp	$bsz,#8*($i+2)
	blo	.Lprocess_block_ce
	ldr	d31,[$inp],#8		// *inp++
#ifdef	__AARCH64EB__
	rev64	v31.16b,v31.16b
#endif
	eor	$A[$j/5][$j%5],$A[$j/5][$j%5],v31.16b
	beq	.Lprocess_block_ce
___
}
$code.=<<___;
	ldr	d31,[$inp],#8		// *inp++
#ifdef	__AARCH64EB__
	rev64	v31.16b,v31.16b
#endif
	eor	$A[4][4],$A[4][4],v31.16b
.Lprocess_block_ce:
	bl	KeccakF1600_ce
	b	.Loop_absorb_ce
.align	4
.Labsorbed_ce:
___
for($i=0; $i<24; $i+=2) {		# store A[5][5]
my $j=$i+1;
$code.=<<___;
	stp	d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
	str	d24,[x0,#8*$i]
	add	x0,$len,$bsz		// return value
	ldp	d8,d9,[sp,#16]
	ldp	d10,d11,[sp,#32]
	ldp	d12,d13,[sp,#48]
	ldp	d14,d15,[sp,#64]
	ldp	x29,x30,[sp],#80
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	SHA3_Absorb_cext,.-SHA3_Absorb_cext
___
}
{
my ($ctx,$out,$len,$bsz) = map("x$_",(0..3));
$code.=<<___;
.globl	SHA3_Squeeze_cext
.type	SHA3_Squeeze_cext,%function
.align	5
SHA3_Squeeze_cext:
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-16]!
	add	x29,sp,#0
	cmp	$len,#0
	beq	.Lsqueeze_done_ce
	mov	x9,$ctx
	mov	x10,$bsz
.Loop_squeeze_ce:
	ldr	x4,[x9],#8
	cmp	$len,#8
	blo	.Lsqueeze_tail_ce
#ifdef	__AARCH64EB__
	rev	x4,x4
#endif
	str	x4,[$out],#8
	beq	.Lsqueeze_done_ce
	sub	$len,$len,#8
	subs	x10,x10,#8
	bhi	.Loop_squeeze_ce
	bl	KeccakF1600_cext
	ldr	x30,[sp,#8]
	mov	x9,$ctx
	mov	x10,$bsz
	b	.Loop_squeeze_ce
.align	4
.Lsqueeze_tail_ce:
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
	lsr	x4,x4,#8
	subs	$len,$len,#1
	beq	.Lsqueeze_done_ce
	strb	w4,[$out],#1
.Lsqueeze_done_ce:
	ldr	x29,[sp],#16
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	SHA3_Squeeze_cext,.-SHA3_Squeeze_cext
___
}								}}}
$code.=<<___;
.asciz	"Keccak-1600 absorb and squeeze for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
___

{   my  %opcode = (
	"rax1"	=> 0xce608c00,	"eor3"	=> 0xce000000,
	"bcax"	=> 0xce200000,	"xar"	=> 0xce800000	);

    sub unsha3 {
	my ($mnemonic,$arg)=@_;

	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/
	&&
	sprintf ".inst\t0x%08x\t//%s %s",
			$opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10),
			$mnemonic,$arg;
    }
}

foreach(split("\n",$code)) {

	s/\`([^\`]*)\`/eval($1)/ge;

	m/\bld1r\b/ and s/\.16b/.2d/g	or
	s/\b(eor3|rax1|xar|bcax)\s+(v.*)/unsha3($1,$2)/ge;

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";