#! /usr/bin/env perl
# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# sha1_block procedure for ARMv4.
#
# January 2007.

# Size/performance trade-off
# ====================================================================
# impl		size in bytes	comp cycles[*]	measured performance
# ====================================================================
# thumb		304		3212		4420
# armv4-small	392/+29%	1958/+64%	2250/+96%
# armv4-compact	740/+89%	1552/+26%	1840/+22%
# armv4-large	1420/+92%	1307/+19%	1370/+34%[***]
# full unroll	~5100/+260%	~1260/+4%	~1300/+5%
# ====================================================================
# thumb		= same as 'small' but in Thumb instructions[**] and
#		  with recurring code in two private functions;
# small		= detached Xload/update, loops are folded;
# compact	= detached Xload/update, 5x unroll;
# large		= interleaved Xload/update, 5x unroll;
# full unroll	= interleaved Xload/update, full unroll, estimated[!];
#
# [*]	Manually counted instructions in "grand" loop body. Measured
#	performance is affected by prologue and epilogue overhead,
#	i-cache availability, branch penalties, etc.
# [**]	While each Thumb instruction is twice smaller, they are not as
#	diverse as ARM ones: e.g., there are only two arithmetic
#	instructions with 3 arguments, no [fixed] rotate, addressing
#	modes are limited. As result it takes more instructions to do
#	the same job in Thumb, therefore the code is never twice as
#	small and always slower.
# [***]	which is also ~35% better than compiler generated code. Dual-
#	issue Cortex A8 core was measured to process input block in
#	~990 cycles.

# August 2010.
#
# Rescheduling for dual-issue pipeline resulted in 13% improvement on
# Cortex A8 core and in absolute terms ~870 cycles per input block
# [or 13.6 cycles per byte].

# February 2011.
#
# Profiler-assisted and platform-specific optimization resulted in 10%
# improvement on Cortex A8 core and 12.2 cycles per byte.

# September 2013.
#
# Add NEON implementation (see sha1-586.pl for background info). On
# Cortex A8 it was measured to process one byte in 6.7 cycles or >80%
# faster than integer-only code. Because [fully unrolled] NEON code
# is ~2.5x larger and there are some redundant instructions executed
# when processing last block, improvement is not as big for smallest
# blocks, only ~30%. Snapdragon S4 is a tad faster, 6.4 cycles per
# byte, which is also >80% faster than integer-only code. Cortex-A15
# is even faster spending 5.6 cycles per byte outperforming integer-
# only code by factor of 2.

# May 2014.
#
# Add ARMv8 code path performing at 2.35 cpb on Apple A7.

$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
    open STDOUT,">$output";
}

$ctx="r0";
$inp="r1";
$len="r2";
$a="r3";
$b="r4";
$c="r5";
$d="r6";
$e="r7";
$K="r8";
$t0="r9";
$t1="r10";
$t2="r11";
$t3="r12";
$Xi="r14";
@V=($a,$b,$c,$d,$e);

sub Xupdate {
my ($a,$b,$c,$d,$e,$opt1,$opt2)=@_;
$code.=<<___;
	ldr	$t0,[$Xi,#15*4]
	ldr	$t1,[$Xi,#13*4]
	ldr	$t2,[$Xi,#7*4]
	add	$e,$K,$e,ror#2			@ E+=K_xx_xx
	ldr	$t3,[$Xi,#2*4]
	eor	$t0,$t0,$t1
	eor	$t2,$t2,$t3			@ 1 cycle stall
	eor	$t1,$c,$d			@ F_xx_xx
	mov	$t0,$t0,ror#31
	add	$e,$e,$a,ror#27			@ E+=ROR(A,27)
	eor	$t0,$t0,$t2,ror#31
	str	$t0,[$Xi,#-4]!
	$opt1					@ F_xx_xx
	$opt2					@ F_xx_xx
	add	$e,$e,$t0			@ E+=X[i]
___
}

sub BODY_00_15 {
my ($a,$b,$c,$d,$e)=@_;
$code.=<<___;
#if __ARM_ARCH__<7
	ldrb	$t1,[$inp,#2]
	ldrb	$t0,[$inp,#3]
	ldrb	$t2,[$inp,#1]
	add	$e,$K,$e,ror#2			@ E+=K_00_19
	ldrb	$t3,[$inp],#4
	orr	$t0,$t0,$t1,lsl#8
	eor	$t1,$c,$d			@ F_xx_xx
	orr	$t0,$t0,$t2,lsl#16
	add	$e,$e,$a,ror#27			@ E+=ROR(A,27)
	orr	$t0,$t0,$t3,lsl#24
#else
	ldr	$t0,[$inp],#4			@ handles unaligned
	add	$e,$K,$e,ror#2			@ E+=K_00_19
	eor	$t1,$c,$d			@ F_xx_xx
	add	$e,$e,$a,ror#27			@ E+=ROR(A,27)
#ifdef __ARMEL__
	rev	$t0,$t0				@ byte swap
#endif
#endif
	and	$t1,$b,$t1,ror#2
	add	$e,$e,$t0			@ E+=X[i]
	eor	$t1,$t1,$d,ror#2		@ F_00_19(B,C,D)
	str	$t0,[$Xi,#-4]!
	add	$e,$e,$t1			@ E+=F_00_19(B,C,D)
___
}

sub BODY_16_19 {
my ($a,$b,$c,$d,$e)=@_;
	&Xupdate(@_,"and $t1,$b,$t1,ror#2");
$code.=<<___;
	eor	$t1,$t1,$d,ror#2		@ F_00_19(B,C,D)
	add	$e,$e,$t1			@ E+=F_00_19(B,C,D)
___
}

sub BODY_20_39 {
my ($a,$b,$c,$d,$e)=@_;
	&Xupdate(@_,"eor $t1,$b,$t1,ror#2");
$code.=<<___;
	add	$e,$e,$t1			@ E+=F_20_39(B,C,D)
___
}

sub BODY_40_59 {
my ($a,$b,$c,$d,$e)=@_;
	&Xupdate(@_,"and $t1,$b,$t1,ror#2","and $t2,$c,$d");
$code.=<<___;
	add	$e,$e,$t1			@ E+=F_40_59(B,C,D)
	add	$e,$e,$t2,ror#2
___
}

$code=<<___;
#include "arm_arch.h"

.text
#if defined(__thumb2__)
.syntax	unified
.thumb
#else
.code	32
#endif

.global	sha1_block_data_order
.type	sha1_block_data_order,%function

.align	5
sha1_block_data_order:
#if __ARM_MAX_ARCH__>=7
.Lsha1_block:
	adr	r3,.Lsha1_block
	ldr	r12,.LOPENSSL_armcap
	ldr	r12,[r3,r12]		@ OPENSSL_armcap_P
#ifdef	__APPLE__
	ldr	r12,[r12]
#endif
	tst	r12,#ARMV8_SHA1
	bne	.LARMv8
	tst	r12,#ARMV7_NEON
	bne	.LNEON
#endif
	stmdb	sp!,{r4-r12,lr}
	add	$len,$inp,$len,lsl#6	@ $len to point at the end of $inp
	ldmia	$ctx,{$a,$b,$c,$d,$e}
.Lloop:
	ldr	$K,.LK_00_19
	mov	$Xi,sp
	sub	sp,sp,#15*4
	mov	$c,$c,ror#30
	mov	$d,$d,ror#30
	mov	$e,$e,ror#30		@ [6]
.L_00_15:
___
for($i=0;$i<5;$i++) {
	&BODY_00_15(@V);	unshift(@V,pop(@V));
}
$code.=<<___;
#if defined(__thumb2__)
	mov	$t3,sp
	teq	$Xi,$t3
#else
	teq	$Xi,sp
#endif
	bne	.L_00_15		@ [((11+4)*5+2)*3]
	sub	sp,sp,#25*4
___
	&BODY_00_15(@V);	unshift(@V,pop(@V));
	&BODY_16_19(@V);	unshift(@V,pop(@V));
	&BODY_16_19(@V);	unshift(@V,pop(@V));
	&BODY_16_19(@V);	unshift(@V,pop(@V));
	&BODY_16_19(@V);	unshift(@V,pop(@V));
$code.=<<___;

	ldr	$K,.LK_20_39		@ [+15+16*4]
	cmn	sp,#0			@ [+3], clear carry to denote 20_39
.L_20_39_or_60_79:
___
for($i=0;$i<5;$i++) {
	&BODY_20_39(@V);	unshift(@V,pop(@V));
}
$code.=<<___;
#if defined(__thumb2__)
	mov	$t3,sp
	teq	$Xi,$t3
#else
	teq	$Xi,sp			@ preserve carry
#endif
	bne	.L_20_39_or_60_79	@ [+((12+3)*5+2)*4]
	bcs	.L_done			@ [+((12+3)*5+2)*4], spare 300 bytes

	ldr	$K,.LK_40_59
	sub	sp,sp,#20*4		@ [+2]
.L_40_59:
___
for($i=0;$i<5;$i++) {
	&BODY_40_59(@V);	unshift(@V,pop(@V));
}
$code.=<<___;
#if defined(__thumb2__)
	mov	$t3,sp
	teq	$Xi,$t3
#else
	teq	$Xi,sp
#endif
	bne	.L_40_59		@ [+((12+5)*5+2)*4]

	ldr	$K,.LK_60_79
	sub	sp,sp,#20*4
	cmp	sp,#0			@ set carry to denote 60_79
	b	.L_20_39_or_60_79	@ [+4], spare 300 bytes
.L_done:
	add	sp,sp,#80*4		@ "deallocate" stack frame
	ldmia	$ctx,{$K,$t0,$t1,$t2,$t3}
	add	$a,$K,$a
	add	$b,$t0,$b
	add	$c,$t1,$c,ror#2
	add	$d,$t2,$d,ror#2
	add	$e,$t3,$e,ror#2
	stmia	$ctx,{$a,$b,$c,$d,$e}
	teq	$inp,$len
	bne	.Lloop			@ [+18], total 1307

#if __ARM_ARCH__>=5
	ldmia	sp!,{r4-r12,pc}
#else
	ldmia	sp!,{r4-r12,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	sha1_block_data_order,.-sha1_block_data_order

.align	5
.LK_00_19:	.word	0x5a827999
.LK_20_39:	.word	0x6ed9eba1
.LK_40_59:	.word	0x8f1bbcdc
.LK_60_79:	.word	0xca62c1d6
#if __ARM_MAX_ARCH__>=7
.LOPENSSL_armcap:
.word	OPENSSL_armcap_P-.Lsha1_block
#endif
.asciz	"SHA1 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align	5
___
#####################################################################
# NEON stuff
#
{{{
my @V=($a,$b,$c,$d,$e);
my ($K_XX_XX,$Ki,$t0,$t1,$Xfer,$saved_sp)=map("r$_",(8..12,14));
my $Xi=4;
my @X=map("q$_",(8..11,0..3));
my @Tx=("q12","q13");
my ($K,$zero)=("q14","q15");
my $j=0;

sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
  my $arg = pop;
    $arg = "#$arg" if ($arg*1 eq $arg);
    $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
}

sub body_00_19 () {
	(
	'($a,$b,$c,$d,$e)=@V;'.		# '$code.="@ $j\n";'.
	'&bic	($t0,$d,$b)',
	'&add	($e,$e,$Ki)',		# e+=X[i]+K
	'&and	($t1,$c,$b)',
	'&ldr	($Ki,sprintf "[sp,#%d]",4*(($j+1)&15))',
	'&add	($e,$e,$a,"ror#27")',	# e+=ROR(A,27)
	'&eor	($t1,$t1,$t0)',		# F_00_19
	'&mov	($b,$b,"ror#2")',	# b=ROR(b,2)
	'&add	($e,$e,$t1);'.		# e+=F_00_19
	'$j++;	unshift(@V,pop(@V));'
	)
}
sub body_20_39 () {
	(
	'($a,$b,$c,$d,$e)=@V;'.		# '$code.="@ $j\n";'.
	'&eor	($t0,$b,$d)',
	'&add	($e,$e,$Ki)',		# e+=X[i]+K
	'&ldr	($Ki,sprintf "[sp,#%d]",4*(($j+1)&15)) if ($j<79)',
	'&eor	($t1,$t0,$c)',		# F_20_39
	'&add	($e,$e,$a,"ror#27")',	# e+=ROR(A,27)
	'&mov	($b,$b,"ror#2")',	# b=ROR(b,2)
	'&add	($e,$e,$t1);'.		# e+=F_20_39
	'$j++;	unshift(@V,pop(@V));'
	)
}
sub body_40_59 () {
	(
	'($a,$b,$c,$d,$e)=@V;'.		# '$code.="@ $j\n";'.
	'&add	($e,$e,$Ki)',		# e+=X[i]+K
	'&and	($t0,$c,$d)',
	'&ldr	($Ki,sprintf "[sp,#%d]",4*(($j+1)&15))',
	'&add	($e,$e,$a,"ror#27")',	# e+=ROR(A,27)
	'&eor	($t1,$c,$d)',
	'&add	($e,$e,$t0)',
	'&and	($t1,$t1,$b)',
	'&mov	($b,$b,"ror#2")',	# b=ROR(b,2)
	'&add	($e,$e,$t1);'.		# e+=F_40_59
	'$j++;	unshift(@V,pop(@V));'
	)
}

sub Xupdate_16_31 ()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e);

	&vext_8		(@X[0],@X[-4&7],@X[-3&7],8);	# compose "X[-14]" in "X[0]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vadd_i32	(@Tx[1],@X[-1&7],$K);
	 eval(shift(@insns));
	  &vld1_32	("{$K\[]}","[$K_XX_XX,:32]!")	if ($Xi%5==0);
	 eval(shift(@insns));
	&vext_8		(@Tx[0],@X[-1&7],$zero,4);	# "X[-3]", 3 words
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		(@X[0],@X[0],@X[-4&7]);		# "X[0]"^="X[-16]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		(@Tx[0],@Tx[0],@X[-2&7]);	# "X[-3]"^"X[-8]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		(@Tx[0],@Tx[0],@X[0]);		# "X[0]"^="X[-3]"^"X[-8]
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vst1_32	("{@Tx[1]}","[$Xfer,:128]!");	# X[]+K xfer
	  &sub		($Xfer,$Xfer,64)		if ($Xi%4==0);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vext_8		(@Tx[1],$zero,@Tx[0],4);	# "X[0]"<<96, extract one dword
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(@X[0],@Tx[0],@Tx[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vsri_32	(@X[0],@Tx[0],31);		# "X[0]"<<<=1
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	(@Tx[0],@Tx[1],30);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshl_u32	(@Tx[1],@Tx[1],2);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		(@X[0],@X[0],@Tx[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		(@X[0],@X[0],@Tx[1]);		# "X[0]"^=("X[0]">>96)<<<2

	foreach (@insns) { eval; }	# remaining instructions [if any]

  $Xi++;	push(@X,shift(@X));	# "rotate" X[]
}

sub Xupdate_32_79 ()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e);

	&vext_8		(@Tx[0],@X[-2&7],@X[-1&7],8);	# compose "X[-6]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		(@X[0],@X[0],@X[-4&7]);		# "X[0]"="X[-32]"^"X[-16]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		(@X[0],@X[0],@X[-7&7]);		# "X[0]"^="X[-28]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vadd_i32	(@Tx[1],@X[-1&7],$K);
	 eval(shift(@insns));
	  &vld1_32	("{$K\[]}","[$K_XX_XX,:32]!")	if ($Xi%5==0);
	 eval(shift(@insns));
	&veor		(@Tx[0],@Tx[0],@X[0]);		# "X[-6]"^="X[0]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	(@X[0],@Tx[0],30);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vst1_32	("{@Tx[1]}","[$Xfer,:128]!");	# X[]+K xfer
	  &sub		($Xfer,$Xfer,64)		if ($Xi%4==0);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vsli_32	(@X[0],@Tx[0],2);		# "X[0]"="X[-6]"<<<2

	foreach (@insns) { eval; }	# remaining instructions [if any]

  $Xi++;	push(@X,shift(@X));	# "rotate" X[]
}

sub Xuplast_80 ()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e);

	&vadd_i32	(@Tx[1],@X[-1&7],$K);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vst1_32	("{@Tx[1]}","[$Xfer,:128]!");
	&sub		($Xfer,$Xfer,64);

	&teq		($inp,$len);
	&sub		($K_XX_XX,$K_XX_XX,16);	# rewind $K_XX_XX
	&it		("eq");
	&subeq		($inp,$inp,64);		# reload last block to avoid SEGV
	&vld1_8		("{@X[-4&7]-@X[-3&7]}","[$inp]!");
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vld1_8		("{@X[-2&7]-@X[-1&7]}","[$inp]!");
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vld1_32	("{$K\[]}","[$K_XX_XX,:32]!");	# load K_00_19
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vrev32_8	(@X[-4&7],@X[-4&7]);

	foreach (@insns) { eval; }		# remaining instructions

   $Xi=0;
}

sub Xloop()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e);

	&vrev32_8	(@X[($Xi-3)&7],@X[($Xi-3)&7]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(@X[$Xi&7],@X[($Xi-4)&7],$K);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vst1_32	("{@X[$Xi&7]}","[$Xfer,:128]!");# X[]+K xfer to IALU

	foreach (@insns) { eval; }

  $Xi++;
}

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.arch	armv7-a
.fpu	neon

.type	sha1_block_data_order_neon,%function
.align	4
sha1_block_data_order_neon:
.LNEON:
	stmdb	sp!,{r4-r12,lr}
	add	$len,$inp,$len,lsl#6	@ $len to point at the end of $inp
	@ dmb				@ errata #451034 on early Cortex A8
	@ vstmdb	sp!,{d8-d15}	@ ABI specification says so
	mov	$saved_sp,sp
	sub	$Xfer,sp,#64
	adr	$K_XX_XX,.LK_00_19
	bic	$Xfer,$Xfer,#15		@ align for 128-bit stores

	ldmia	$ctx,{$a,$b,$c,$d,$e}	@ load context
	mov	sp,$Xfer		@ alloca

	vld1.8		{@X[-4&7]-@X[-3&7]},[$inp]!	@ handles unaligned
	veor		$zero,$zero,$zero
	vld1.8		{@X[-2&7]-@X[-1&7]},[$inp]!
	vld1.32		{${K}\[]},[$K_XX_XX,:32]!	@ load K_00_19
	vrev32.8	@X[-4&7],@X[-4&7]		@ yes, even on
	vrev32.8	@X[-3&7],@X[-3&7]		@ big-endian...
	vrev32.8	@X[-2&7],@X[-2&7]
	vadd.i32	@X[0],@X[-4&7],$K
	vrev32.8	@X[-1&7],@X[-1&7]
	vadd.i32	@X[1],@X[-3&7],$K
	vst1.32		{@X[0]},[$Xfer,:128]!
	vadd.i32	@X[2],@X[-2&7],$K
	vst1.32		{@X[1]},[$Xfer,:128]!
	vst1.32		{@X[2]},[$Xfer,:128]!
	ldr		$Ki,[sp]			@ big RAW stall

.Loop_neon:
___
	&Xupdate_16_31(\&body_00_19);
	&Xupdate_16_31(\&body_00_19);
	&Xupdate_16_31(\&body_00_19);
	&Xupdate_16_31(\&body_00_19);
	&Xupdate_32_79(\&body_00_19);
	&Xupdate_32_79(\&body_20_39);
	&Xupdate_32_79(\&body_20_39);
	&Xupdate_32_79(\&body_20_39);
	&Xupdate_32_79(\&body_20_39);
	&Xupdate_32_79(\&body_20_39);
	&Xupdate_32_79(\&body_40_59);
	&Xupdate_32_79(\&body_40_59);
	&Xupdate_32_79(\&body_40_59);
	&Xupdate_32_79(\&body_40_59);
	&Xupdate_32_79(\&body_40_59);
	&Xupdate_32_79(\&body_20_39);
	&Xuplast_80(\&body_20_39);
	&Xloop(\&body_20_39);
	&Xloop(\&body_20_39);
	&Xloop(\&body_20_39);
$code.=<<___;
	ldmia	$ctx,{$Ki,$t0,$t1,$Xfer}	@ accumulate context
	add	$a,$a,$Ki
	ldr	$Ki,[$ctx,#16]
	add	$b,$b,$t0
	add	$c,$c,$t1
	add	$d,$d,$Xfer
	it	eq
	moveq	sp,$saved_sp
	add	$e,$e,$Ki
	it	ne
	ldrne	$Ki,[sp]
	stmia	$ctx,{$a,$b,$c,$d,$e}
	itt	ne
	addne	$Xfer,sp,#3*16
	bne	.Loop_neon

	@ vldmia	sp!,{d8-d15}
	ldmia	sp!,{r4-r12,pc}
.size	sha1_block_data_order_neon,.-sha1_block_data_order_neon
#endif
___
}}}
#####################################################################
# ARMv8 stuff
#
{{{
my ($ABCD,$E,$E0,$E1)=map("q$_",(0..3));
my @MSG=map("q$_",(4..7));
my @Kxx=map("q$_",(8..11));
my ($W0,$W1,$ABCD_SAVE)=map("q$_",(12..14));

$code.=<<___;
#if __ARM_MAX_ARCH__>=7

# if defined(__thumb2__)
#  define INST(a,b,c,d)	.byte	c,d|0xf,a,b
# else
#  define INST(a,b,c,d)	.byte	a,b,c,d|0x10
# endif

.type	sha1_block_data_order_armv8,%function
.align	5
sha1_block_data_order_armv8:
.LARMv8:
	vstmdb	sp!,{d8-d15}		@ ABI specification says so

	veor	$E,$E,$E
	adr	r3,.LK_00_19
	vld1.32	{$ABCD},[$ctx]!
	vld1.32	{$E\[0]},[$ctx]
	sub	$ctx,$ctx,#16
	vld1.32	{@Kxx[0]\[]},[r3,:32]!
	vld1.32	{@Kxx[1]\[]},[r3,:32]!
	vld1.32	{@Kxx[2]\[]},[r3,:32]!
	vld1.32	{@Kxx[3]\[]},[r3,:32]

.Loop_v8:
	vld1.8		{@MSG[0]-@MSG[1]},[$inp]!
	vld1.8		{@MSG[2]-@MSG[3]},[$inp]!
	vrev32.8	@MSG[0],@MSG[0]
	vrev32.8	@MSG[1],@MSG[1]

	vadd.i32	$W0,@Kxx[0],@MSG[0]
	vrev32.8	@MSG[2],@MSG[2]
	vmov		$ABCD_SAVE,$ABCD	@ offload
	subs		$len,$len,#1

	vadd.i32	$W1,@Kxx[0],@MSG[1]
	vrev32.8	@MSG[3],@MSG[3]
	sha1h		$E1,$ABCD		@ 0
	sha1c		$ABCD,$E,$W0
	vadd.i32	$W0,@Kxx[$j],@MSG[2]
	sha1su0		@MSG[0],@MSG[1],@MSG[2]
___
for ($j=0,$i=1;$i<20-3;$i++) {
my $f=("c","p","m","p")[$i/5];
$code.=<<___;
	sha1h		$E0,$ABCD		@ $i
	sha1$f		$ABCD,$E1,$W1
	vadd.i32	$W1,@Kxx[$j],@MSG[3]
	sha1su1		@MSG[0],@MSG[3]
___
$code.=<<___ if ($i<20-4);
	sha1su0		@MSG[1],@MSG[2],@MSG[3]
___
	($E0,$E1)=($E1,$E0);	($W0,$W1)=($W1,$W0);
	push(@MSG,shift(@MSG));	$j++ if ((($i+3)%5)==0);
}
$code.=<<___;
	sha1h		$E0,$ABCD		@ $i
	sha1p		$ABCD,$E1,$W1
	vadd.i32	$W1,@Kxx[$j],@MSG[3]

	sha1h		$E1,$ABCD		@ 18
	sha1p		$ABCD,$E0,$W0

	sha1h		$E0,$ABCD		@ 19
	sha1p		$ABCD,$E1,$W1

	vadd.i32	$E,$E,$E0
	vadd.i32	$ABCD,$ABCD,$ABCD_SAVE
	bne		.Loop_v8

	vst1.32		{$ABCD},[$ctx]!
	vst1.32		{$E\[0]},[$ctx]

	vldmia	sp!,{d8-d15}
	ret					@ bx lr
.size	sha1_block_data_order_armv8,.-sha1_block_data_order_armv8
#endif
___
}}}
$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.comm	OPENSSL_armcap_P,4,4
#endif
___

{   my  %opcode = (
	"sha1c"		=> 0xf2000c40,	"sha1p"		=> 0xf2100c40,
	"sha1m"		=> 0xf2200c40,	"sha1su0"	=> 0xf2300c40,
	"sha1h"		=> 0xf3b902c0,	"sha1su1"	=> 0xf3ba0380	);

    sub unsha1 {
	my ($mnemonic,$arg)=@_;

	if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
	    my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
					 |(($2&7)<<17)|(($2&8)<<4)
					 |(($3&7)<<1) |(($3&8)<<2);
	    # since ARMv7 instructions are always encoded little-endian.
	    # correct solution is to use .inst directive, but older
	    # assemblers don't implement it:-(

	    # this fix-up provides Thumb encoding in conjunction with INST
	    $word &= ~0x10000000 if (($word & 0x0f000000) == 0x02000000);
	    sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
			$word&0xff,($word>>8)&0xff,
			($word>>16)&0xff,($word>>24)&0xff,
			$mnemonic,$arg;
	}
    }
}

foreach (split($/,$code)) {
	s/{q([0-9]+)\[\]}/sprintf "{d%d[],d%d[]}",2*$1,2*$1+1/eo	or
	s/{q([0-9]+)\[0\]}/sprintf "{d%d[0]}",2*$1/eo;

	s/\b(sha1\w+)\s+(q.*)/unsha1($1,$2)/geo;

	s/\bret\b/bx	lr/o		or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/o;	# make it possible to compile with -march=armv4

	print $_,$/;
}

close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#! /usr/bin/env perl
# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# SHA1 for ARMv8.
#
# Performance in cycles per processed byte and improvement coefficient
# over code generated with "default" compiler:
#
#		hardware-assisted	software(*)
# Apple A7	2.31			4.13 (+14%)
# Cortex-A53	2.24			8.03 (+97%)
# Cortex-A57	2.35			7.88 (+74%)
# Denver	2.13			3.97 (+0%)(**)
# X-Gene				8.80 (+200%)
# Mongoose	2.05			6.50 (+160%)
# Kryo		1.88			8.00 (+90%)
#
# (*)	Software results are presented mostly for reference purposes.
# (**)	Keep in mind that Denver relies on binary translation, which
#	optimizes compiler output at run-time.

$flavour = shift;
$output  = shift;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;

($ctx,$inp,$num)=("x0","x1","x2");
@Xw=map("w$_",(3..17,19));
@Xx=map("x$_",(3..17,19));
@V=($A,$B,$C,$D,$E)=map("w$_",(20..24));
($t0,$t1,$t2,$K)=map("w$_",(25..28));

sub BODY_00_19 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=($i+2)&15;

$code.=<<___ if ($i<15 && !($i&1));
	lsr	@Xx[$i+1],@Xx[$i],#32
___
$code.=<<___ if ($i<14 && !($i&1));
	ldr	@Xx[$i+2],[$inp,#`($i+2)*4-64`]
___
$code.=<<___ if ($i<14 && ($i&1));
#ifdef	__ARMEB__
	ror	@Xx[$i+1],@Xx[$i+1],#32
#else
	rev32	@Xx[$i+1],@Xx[$i+1]
#endif
___
$code.=<<___ if ($i<14);
	bic	$t0,$d,$b
	and	$t1,$c,$b
	ror	$t2,$a,#27
	add	$d,$d,$K		// future e+=K
	orr	$t0,$t0,$t1
	add	$e,$e,$t2		// e+=rot(a,5)
	ror	$b,$b,#2
	add	$d,$d,@Xw[($i+1)&15]	// future e+=X[i]
	add	$e,$e,$t0		// e+=F(b,c,d)
___
$code.=<<___ if ($i==19);
	movz	$K,#0xeba1
	movk	$K,#0x6ed9,lsl#16
___
$code.=<<___ if ($i>=14);
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+2)&15]
	bic	$t0,$d,$b
	and	$t1,$c,$b
	ror	$t2,$a,#27
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+8)&15]
	add	$d,$d,$K		// future e+=K
	orr	$t0,$t0,$t1
	add	$e,$e,$t2		// e+=rot(a,5)
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+13)&15]
	ror	$b,$b,#2
	add	$d,$d,@Xw[($i+1)&15]	// future e+=X[i]
	add	$e,$e,$t0		// e+=F(b,c,d)
	 ror	@Xw[$j],@Xw[$j],#31
___
}

sub BODY_40_59 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=($i+2)&15;

$code.=<<___ if ($i==59);
	movz	$K,#0xc1d6
	movk	$K,#0xca62,lsl#16
___
$code.=<<___;
	orr	$t0,$b,$c
	and	$t1,$b,$c
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+2)&15]
	ror	$t2,$a,#27
	and	$t0,$t0,$d
	add	$d,$d,$K		// future e+=K
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+8)&15]
	add	$e,$e,$t2		// e+=rot(a,5)
	orr	$t0,$t0,$t1
	ror	$b,$b,#2
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+13)&15]
	add	$d,$d,@Xw[($i+1)&15]	// future e+=X[i]
	add	$e,$e,$t0		// e+=F(b,c,d)
	 ror	@Xw[$j],@Xw[$j],#31
___
}

sub BODY_20_39 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=($i+2)&15;

$code.=<<___ if ($i==39);
	movz	$K,#0xbcdc
	movk	$K,#0x8f1b,lsl#16
___
$code.=<<___ if ($i<78);
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+2)&15]
	eor	$t0,$d,$b
	ror	$t2,$a,#27
	add	$d,$d,$K		// future e+=K
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+8)&15]
	eor	$t0,$t0,$c
	add	$e,$e,$t2		// e+=rot(a,5)
	ror	$b,$b,#2
	 eor	@Xw[$j],@Xw[$j],@Xw[($j+13)&15]
	add	$d,$d,@Xw[($i+1)&15]	// future e+=X[i]
	add	$e,$e,$t0		// e+=F(b,c,d)
	 ror	@Xw[$j],@Xw[$j],#31
___
$code.=<<___ if ($i==78);
	ldp	@Xw[1],@Xw[2],[$ctx]
	eor	$t0,$d,$b
	ror	$t2,$a,#27
	add	$d,$d,$K		// future e+=K
	eor	$t0,$t0,$c
	add	$e,$e,$t2		// e+=rot(a,5)
	ror	$b,$b,#2
	add	$d,$d,@Xw[($i+1)&15]	// future e+=X[i]
	add	$e,$e,$t0		// e+=F(b,c,d)
___
$code.=<<___ if ($i==79);
	ldp	@Xw[3],@Xw[4],[$ctx,#8]
	eor	$t0,$d,$b
	ror	$t2,$a,#27
	eor	$t0,$t0,$c
	add	$e,$e,$t2		// e+=rot(a,5)
	ror	$b,$b,#2
	ldr	@Xw[5],[$ctx,#16]
	add	$e,$e,$t0		// e+=F(b,c,d)
___
}

$code.=<<___;
#include "arm_arch.h"

.text

.extern	OPENSSL_armcap_P
.hidden OPENSSL_armcap_P
.globl	sha1_block_data_order
.type	sha1_block_data_order,%function
.align	6
sha1_block_data_order:
#ifdef	__ILP32__
	ldrsw	x16,.LOPENSSL_armcap_P
#else
	ldr	x16,.LOPENSSL_armcap_P
#endif
	adr	x17,.LOPENSSL_armcap_P
	add	x16,x16,x17
	ldr	w16,[x16]
	tst	w16,#ARMV8_SHA1
	b.ne	.Lv8_entry

	stp	x29,x30,[sp,#-96]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]

	ldp	$A,$B,[$ctx]
	ldp	$C,$D,[$ctx,#8]
	ldr	$E,[$ctx,#16]

.Loop:
	ldr	@Xx[0],[$inp],#64
	movz	$K,#0x7999
	sub	$num,$num,#1
	movk	$K,#0x5a82,lsl#16
#ifdef	__ARMEB__
	ror	$Xx[0],@Xx[0],#32
#else
	rev32	@Xx[0],@Xx[0]
#endif
	add	$E,$E,$K		// warm it up
	add	$E,$E,@Xw[0]
___
for($i=0;$i<20;$i++)	{ &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
for(;$i<40;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
for(;$i<60;$i++)	{ &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
for(;$i<80;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	add	$B,$B,@Xw[2]
	add	$C,$C,@Xw[3]
	add	$A,$A,@Xw[1]
	add	$D,$D,@Xw[4]
	add	$E,$E,@Xw[5]
	stp	$A,$B,[$ctx]
	stp	$C,$D,[$ctx,#8]
	str	$E,[$ctx,#16]
	cbnz	$num,.Loop

	ldp	x19,x20,[sp,#16]
	ldp	x21,x22,[sp,#32]
	ldp	x23,x24,[sp,#48]
	ldp	x25,x26,[sp,#64]
	ldp	x27,x28,[sp,#80]
	ldr	x29,[sp],#96
	ret
.size	sha1_block_data_order,.-sha1_block_data_order
___
{{{
my ($ABCD,$E,$E0,$E1)=map("v$_.16b",(0..3));
my @MSG=map("v$_.16b",(4..7));
my @Kxx=map("v$_.4s",(16..19));
my ($W0,$W1)=("v20.4s","v21.4s");
my $ABCD_SAVE="v22.16b";

$code.=<<___;
.type	sha1_block_armv8,%function
.align	6
sha1_block_armv8:
.Lv8_entry:
	stp	x29,x30,[sp,#-16]!
	add	x29,sp,#0

	adr	x4,.Lconst
	eor	$E,$E,$E
	ld1.32	{$ABCD},[$ctx],#16
	ld1.32	{$E}[0],[$ctx]
	sub	$ctx,$ctx,#16
	ld1.32	{@Kxx[0]-@Kxx[3]},[x4]

.Loop_hw:
	ld1	{@MSG[0]-@MSG[3]},[$inp],#64
	sub	$num,$num,#1
	rev32	@MSG[0],@MSG[0]
	rev32	@MSG[1],@MSG[1]

	add.i32	$W0,@Kxx[0],@MSG[0]
	rev32	@MSG[2],@MSG[2]
	orr	$ABCD_SAVE,$ABCD,$ABCD	// offload

	add.i32	$W1,@Kxx[0],@MSG[1]
	rev32	@MSG[3],@MSG[3]
	sha1h	$E1,$ABCD
	sha1c	$ABCD,$E,$W0		// 0
	add.i32	$W0,@Kxx[$j],@MSG[2]
	sha1su0	@MSG[0],@MSG[1],@MSG[2]
___
for ($j=0,$i=1;$i<20-3;$i++) {
my $f=("c","p","m","p")[$i/5];
$code.=<<___;
	sha1h	$E0,$ABCD		// $i
	sha1$f	$ABCD,$E1,$W1
	add.i32	$W1,@Kxx[$j],@MSG[3]
	sha1su1	@MSG[0],@MSG[3]
___
$code.=<<___ if ($i<20-4);
	sha1su0	@MSG[1],@MSG[2],@MSG[3]
___
	($E0,$E1)=($E1,$E0);		($W0,$W1)=($W1,$W0);
	push(@MSG,shift(@MSG));		$j++ if ((($i+3)%5)==0);
}
$code.=<<___;
	sha1h	$E0,$ABCD		// $i
	sha1p	$ABCD,$E1,$W1
	add.i32	$W1,@Kxx[$j],@MSG[3]

	sha1h	$E1,$ABCD		// 18
	sha1p	$ABCD,$E0,$W0

	sha1h	$E0,$ABCD		// 19
	sha1p	$ABCD,$E1,$W1

	add.i32	$E,$E,$E0
	add.i32	$ABCD,$ABCD,$ABCD_SAVE

	cbnz	$num,.Loop_hw

	st1.32	{$ABCD},[$ctx],#16
	st1.32	{$E}[0],[$ctx]

	ldr	x29,[sp],#16
	ret
.size	sha1_block_armv8,.-sha1_block_armv8
.align	6
.Lconst:
.long	0x5a827999,0x5a827999,0x5a827999,0x5a827999	//K_00_19
.long	0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1	//K_20_39
.long	0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc	//K_40_59
.long	0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6	//K_60_79
.LOPENSSL_armcap_P:
#ifdef	__ILP32__
.long	OPENSSL_armcap_P-.
#else
.quad	OPENSSL_armcap_P-.
#endif
.asciz	"SHA1 block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
___
}}}

{   my	%opcode = (
	"sha1c"		=> 0x5e000000,	"sha1p"		=> 0x5e001000,
	"sha1m"		=> 0x5e002000,	"sha1su0"	=> 0x5e003000,
	"sha1h"		=> 0x5e280800,	"sha1su1"	=> 0x5e281800	);

    sub unsha1 {
	my ($mnemonic,$arg)=@_;

	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
	&&
	sprintf ".inst\t0x%08x\t//%s %s",
			$opcode{$mnemonic}|$1|($2<<5)|($3<<16),
			$mnemonic,$arg;
    }
}

foreach(split("\n",$code)) {

	s/\`([^\`]*)\`/eval($1)/geo;

	s/\b(sha1\w+)\s+([qv].*)/unsha1($1,$2)/geo;

	s/\.\w?32\b//o		and s/\.16b/\.4s/go;
	m/(ld|st)1[^\[]+\[0\]/o	and s/\.4s/\.s/go;

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2012-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# SHA1 for C64x+.
#
# November 2011
#
# If compared to compiler-generated code with similar characteristics,
# i.e. compiled with OPENSSL_SMALL_FOOTPRINT and utilizing SPLOOPs,
# this implementation is 25% smaller and >2x faster. In absolute terms
# performance is (quite impressive) ~6.5 cycles per processed byte.
# Fully unrolled assembler would be ~5x larger and is likely to be
# ~15% faster. It would be free from references to intermediate ring
# buffer, but put more pressure on L1P [both because the code would be
# larger and won't be using SPLOOP buffer]. There are no plans to
# realize fully unrolled variant though...
#
# !!! Note that this module uses AMR, which means that all interrupt
# service routines are expected to preserve it and for own well-being
# zero it upon entry.

while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

($CTX,$INP,$NUM) = ("A4","B4","A6");		# arguments

($A,$B,$C,$D,$E, $Arot,$F,$F0,$T,$K) = map("A$_",(16..20, 21..25));
($X0,$X2,$X8,$X13) = ("A26","B26","A27","B27");
($TX0,$TX1,$TX2,$TX3) = map("B$_",(28..31));
($XPA,$XPB) = ("A5","B5");			# X circular buffer
($Actx,$Bctx,$Cctx,$Dctx,$Ectx) = map("A$_",(3,6..9));	# zaps $NUM

$code=<<___;
	.text

	.if	.ASSEMBLER_VERSION<7000000
	.asg	0,__TI_EABI__
	.endif
	.if	__TI_EABI__
	.asg	sha1_block_data_order,_sha1_block_data_order
	.endif

	.asg	B3,RA
	.asg	A15,FP
	.asg	B15,SP

	.if	.BIG_ENDIAN
	.asg	MV,SWAP2
	.asg	MV,SWAP4
	.endif

	.global	_sha1_block_data_order
_sha1_block_data_order:
	.asmfunc stack_usage(64)
	MV	$NUM,A0			; reassign $NUM
||	MVK	-64,B0
  [!A0]	BNOP	RA			; if ($NUM==0) return;
|| [A0]	STW	FP,*SP--[16]		; save frame pointer and alloca(64)
|| [A0]	MV	SP,FP
   [A0]	LDW	*${CTX}[0],$A		; load A-E...
|| [A0]	AND	B0,SP,SP		; align stack at 64 bytes
   [A0]	LDW	*${CTX}[1],$B
|| [A0]	SUBAW	SP,2,SP			; reserve two words above buffer
   [A0]	LDW	*${CTX}[2],$C
|| [A0]	MVK	0x00404,B0
   [A0]	LDW	*${CTX}[3],$D
|| [A0]	MVKH	0x50000,B0		; 0x050404, 64 bytes for $XP[AB]
   [A0]	LDW	*${CTX}[4],$E
|| [A0]	MVC	B0,AMR			; setup circular addressing
	LDNW	*${INP}++,$TX1		; pre-fetch input
	NOP	1

loop?:
	MVK	0x00007999,$K
||	ADDAW	SP,2,$XPA
||	SUB	A0,1,A0
||	MVK	13,B0
	MVKH	0x5a820000,$K		; K_00_19
||	ADDAW	SP,2,$XPB
||	MV	$A,$Actx
||	MV	$B,$Bctx
;;==================================================
	SPLOOPD	5			; BODY_00_13
||	MV	$C,$Cctx
||	MV	$D,$Dctx
||	MV	$E,$Ectx
||	MVC	B0,ILC

	ROTL	$A,5,$Arot
||	AND	$C,$B,$F
||	ANDN	$D,$B,$F0
||	ADD	$K,$E,$T		; T=E+K

	XOR	$F0,$F,$F		; F_00_19(B,C,D)
||	MV	$D,$E			; E=D
||	MV	$C,$D			; D=C
||	SWAP2	$TX1,$TX2
||	LDNW	*${INP}++,$TX1

	ADD	$F,$T,$T		; T+=F_00_19(B,C,D)
||	ROTL	$B,30,$C		; C=ROL(B,30)
||	SWAP4	$TX2,$TX3		; byte swap

	ADD	$Arot,$T,$T		; T+=ROL(A,5)
||	MV	$A,$B			; B=A

	ADD	$TX3,$T,$A		; A=T+Xi
||	STW	$TX3,*${XPB}++
	SPKERNEL
;;==================================================
	ROTL	$A,5,$Arot		; BODY_14
||	AND	$C,$B,$F
||	ANDN	$D,$B,$F0
||	ADD	$K,$E,$T		; T=E+K

	XOR	$F0,$F,$F		; F_00_19(B,C,D)
||	MV	$D,$E			; E=D
||	MV	$C,$D			; D=C
||	SWAP2	$TX1,$TX2
||	LDNW	*${INP}++,$TX1

	ADD	$F,$T,$T		; T+=F_00_19(B,C,D)
||	ROTL	$B,30,$C		; C=ROL(B,30)
||	SWAP4	$TX2,$TX2		; byte swap
||	LDW	*${XPA}++,$X0		; fetches from X ring buffer are
||	LDW	*${XPB}[4],$X2		; 2 iterations ahead

	ADD	$Arot,$T,$T		; T+=ROL(A,5)
||	MV	$A,$B			; B=A
||	LDW	*${XPA}[7],$X8
||	MV	$TX3,$X13		; ||	LDW	*${XPB}[15],$X13
||	MV	$TX2,$TX3

	ADD	$TX2,$T,$A		; A=T+Xi
||	STW	$TX2,*${XPB}++
;;==================================================
	ROTL	$A,5,$Arot		; BODY_15
||	AND	$C,$B,$F
||	ANDN	$D,$B,$F0
||	ADD	$K,$E,$T		; T=E+K

	XOR	$F0,$F,$F		; F_00_19(B,C,D)
||	MV	$D,$E			; E=D
||	MV	$C,$D			; D=C
||	SWAP2	$TX1,$TX2

	ADD	$F,$T,$T		; T+=F_00_19(B,C,D)
||	ROTL	$B,30,$C		; C=ROL(B,30)
||	SWAP4	$TX2,$TX2		; byte swap
||	XOR	$X0,$X2,$TX0		; Xupdate XORs are 1 iteration ahead
||	LDW	*${XPA}++,$X0
||	LDW	*${XPB}[4],$X2

	ADD	$Arot,$T,$T		; T+=ROL(A,5)
||	MV	$A,$B			; B=A
||	XOR	$X8,$X13,$TX1
||	LDW	*${XPA}[7],$X8
||	MV	$TX3,$X13		; ||	LDW	*${XPB}[15],$X13
||	MV	$TX2,$TX3

	ADD	$TX2,$T,$A		; A=T+Xi
||	STW	$TX2,*${XPB}++
||	XOR	$TX0,$TX1,$TX1
||	MVK	3,B0
;;==================================================
	SPLOOPD	5			; BODY_16_19
||	MVC	B0,ILC

	ROTL	$A,5,$Arot
||	AND	$C,$B,$F
||	ANDN	$D,$B,$F0
||	ADD	$K,$E,$T		; T=E+K
||	ROTL	$TX1,1,$TX2		; Xupdate output

	XOR	$F0,$F,$F		; F_00_19(B,C,D)
||	MV	$D,$E			; E=D
||	MV	$C,$D			; D=C

	ADD	$F,$T,$T		; T+=F_00_19(B,C,D)
||	ROTL	$B,30,$C		; C=ROL(B,30)
||	XOR	$X0,$X2,$TX0
||	LDW	*${XPA}++,$X0
||	LDW	*${XPB}[4],$X2

	ADD	$Arot,$T,$T		; T+=ROL(A,5)
||	MV	$A,$B			; B=A
||	XOR	$X8,$X13,$TX1
||	LDW	*${XPA}[7],$X8
||	MV	$TX3,$X13		; ||	LDW	*${XPB}[15],$X13
||	MV	$TX2,$TX3

	ADD	$TX2,$T,$A		; A=T+Xi
||	STW	$TX2,*${XPB}++
||	XOR	$TX0,$TX1,$TX1
	SPKERNEL

	MVK	0xffffeba1,$K
||	MVK	19,B0
	MVKH	0x6ed90000,$K		; K_20_39
___
sub BODY_20_39 {
$code.=<<___;
;;==================================================
	SPLOOPD	5			; BODY_20_39
||	MVC	B0,ILC

	ROTL	$A,5,$Arot
||	XOR	$B,$C,$F
||	ADD	$K,$E,$T		; T=E+K
||	ROTL	$TX1,1,$TX2		; Xupdate output

	XOR	$D,$F,$F		; F_20_39(B,C,D)
||	MV	$D,$E			; E=D
||	MV	$C,$D			; D=C

	ADD	$F,$T,$T		; T+=F_20_39(B,C,D)
||	ROTL	$B,30,$C		; C=ROL(B,30)
||	XOR	$X0,$X2,$TX0
||	LDW	*${XPA}++,$X0
||	LDW	*${XPB}[4],$X2

	ADD	$Arot,$T,$T		; T+=ROL(A,5)
||	MV	$A,$B			; B=A
||	XOR	$X8,$X13,$TX1
||	LDW	*${XPA}[7],$X8
||	MV	$TX3,$X13		; ||	LDW	*${XPB}[15],$X13
||	MV	$TX2,$TX3

	ADD	$TX2,$T,$A		; A=T+Xi
||	STW	$TX2,*${XPB}++		; last one is redundant
||	XOR	$TX0,$TX1,$TX1
	SPKERNEL
___
$code.=<<___ if (!shift);
	MVK	0xffffbcdc,$K
	MVKH	0x8f1b0000,$K		; K_40_59
___
}	&BODY_20_39();
$code.=<<___;
;;==================================================
	SPLOOPD	5			; BODY_40_59
||	MVC	B0,ILC
||	AND	$B,$C,$F
||	AND	$B,$D,$F0

	ROTL	$A,5,$Arot
||	XOR	$F0,$F,$F
||	AND	$C,$D,$F0
||	ADD	$K,$E,$T		; T=E+K
||	ROTL	$TX1,1,$TX2		; Xupdate output

	XOR	$F0,$F,$F		; F_40_59(B,C,D)
||	MV	$D,$E			; E=D
||	MV	$C,$D			; D=C

	ADD	$F,$T,$T		; T+=F_40_59(B,C,D)
||	ROTL	$B,30,$C		; C=ROL(B,30)
||	XOR	$X0,$X2,$TX0
||	LDW	*${XPA}++,$X0
||	LDW	*${XPB}[4],$X2

	ADD	$Arot,$T,$T		; T+=ROL(A,5)
||	MV	$A,$B			; B=A
||	XOR	$X8,$X13,$TX1
||	LDW	*${XPA}[7],$X8
||	MV	$TX3,$X13		; ||	LDW	*${XPB}[15],$X13
||	MV	$TX2,$TX3

	ADD	$TX2,$T,$A		; A=T+Xi
||	STW	$TX2,*${XPB}++
||	XOR	$TX0,$TX1,$TX1
||	AND	$B,$C,$F
||	AND	$B,$D,$F0
	SPKERNEL

	MVK	0xffffc1d6,$K
||	MVK	18,B0
	MVKH	0xca620000,$K		; K_60_79
___
	&BODY_20_39(-1);		# BODY_60_78
$code.=<<___;
;;==================================================
   [A0]	B	loop?
||	ROTL	$A,5,$Arot		; BODY_79
||	XOR	$B,$C,$F
||	ROTL	$TX1,1,$TX2		; Xupdate output

   [A0]	LDNW	*${INP}++,$TX1		; pre-fetch input
||	ADD	$K,$E,$T		; T=E+K
||	XOR	$D,$F,$F		; F_20_39(B,C,D)

	ADD	$F,$T,$T		; T+=F_20_39(B,C,D)
||	ADD	$Ectx,$D,$E		; E=D,E+=Ectx
||	ADD	$Dctx,$C,$D		; D=C,D+=Dctx
||	ROTL	$B,30,$C		; C=ROL(B,30)

	ADD	$Arot,$T,$T		; T+=ROL(A,5)
||	ADD	$Bctx,$A,$B		; B=A,B+=Bctx

	ADD	$TX2,$T,$A		; A=T+Xi

	ADD	$Actx,$A,$A		; A+=Actx
||	ADD	$Cctx,$C,$C		; C+=Cctx
;; end of loop?

	BNOP	RA			; return
||	MV	FP,SP			; restore stack pointer
||	LDW	*FP[0],FP		; restore frame pointer
	STW	$A,*${CTX}[0]		; emit A-E...
||	MVK	0,B0
	STW	$B,*${CTX}[1]
||	MVC	B0,AMR			; clear AMR
	STW	$C,*${CTX}[2]
	STW	$D,*${CTX}[3]
	STW	$E,*${CTX}[4]
	.endasmfunc

	.sect	.const
	.cstring "SHA1 block transform for C64x+, CRYPTOGAMS by <appro\@openssl.org>"
	.align	4
___

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Eternal question is what's wrong with compiler generated code? The
# trick is that it's possible to reduce the number of shifts required
# to perform rotations by maintaining copy of 32-bit value in upper
# bits of 64-bit register. Just follow mux2 and shrp instructions...
# Performance under big-endian OS such as HP-UX is 179MBps*1GHz, which
# is >50% better than HP C and >2x better than gcc.

$output = pop;

$code=<<___;
.ident  \"sha1-ia64.s, version 1.3\"
.ident  \"IA-64 ISA artwork by Andy Polyakov <appro\@fy.chalmers.se>\"
.explicit

___

if ($^O eq "hpux") {
    $ADDP="addp4";
    for (@ARGV) { $ADDP="add" if (/[\+DD|\-mlp]64/); }
} else { $ADDP="add"; }

#$human=1;
if ($human) {	# useful for visual code auditing...
	($A,$B,$C,$D,$E)   = ("A","B","C","D","E");
	($h0,$h1,$h2,$h3,$h4) = ("h0","h1","h2","h3","h4");
	($K_00_19, $K_20_39, $K_40_59, $K_60_79) =
	    (	"K_00_19","K_20_39","K_40_59","K_60_79"	);
	@X= (	"X0", "X1", "X2", "X3", "X4", "X5", "X6", "X7",
		"X8", "X9","X10","X11","X12","X13","X14","X15"	);
}
else {
	($A,$B,$C,$D,$E)   =    ("loc0","loc1","loc2","loc3","loc4");
	($h0,$h1,$h2,$h3,$h4) = ("loc5","loc6","loc7","loc8","loc9");
	($K_00_19, $K_20_39, $K_40_59, $K_60_79) =
	    (	"r14", "r15", "loc10", "loc11"	);
	@X= (	"r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
		"r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"	);
}

sub BODY_00_15 {
local	*code=shift;
my	($i,$a,$b,$c,$d,$e)=@_;
my	$j=$i+1;
my	$Xn=@X[$j%16];

$code.=<<___ if ($i==0);
{ .mmi;	ld1	$X[$i]=[inp],2		    // MSB
	ld1	tmp2=[tmp3],2		};;
{ .mmi;	ld1	tmp0=[inp],2
	ld1	tmp4=[tmp3],2		    // LSB
	dep	$X[$i]=$X[$i],tmp2,8,8	};;
___
if ($i<15) {
	$code.=<<___;
{ .mmi;	ld1	$Xn=[inp],2		    // forward Xload
	nop.m	0x0
	dep	tmp1=tmp0,tmp4,8,8	};;
{ .mmi;	ld1	tmp2=[tmp3],2		    // forward Xload
	and	tmp4=$c,$b
	dep	$X[$i]=$X[$i],tmp1,16,16} //;;
{ .mmi;	add	$e=$e,$K_00_19		    // e+=K_00_19
	andcm	tmp1=$d,$b
	dep.z	tmp5=$a,5,27		};; // a<<5
{ .mmi;	add	$e=$e,$X[$i]		    // e+=Xload
	or	tmp4=tmp4,tmp1		    // F_00_19(b,c,d)=(b&c)|(~b&d)
	extr.u	tmp1=$a,27,5		};; // a>>27
{ .mmi;	ld1	tmp0=[inp],2		    // forward Xload
	add	$e=$e,tmp4		    // e+=F_00_19(b,c,d)
	shrp	$b=tmp6,tmp6,2		}   // b=ROTATE(b,30)
{ .mmi;	ld1	tmp4=[tmp3],2		    // forward Xload
	or	tmp5=tmp1,tmp5		    // ROTATE(a,5)
	mux2	tmp6=$a,0x44		};; // see b in next iteration
{ .mii;	add	$e=$e,tmp5		    // e+=ROTATE(a,5)
	dep	$Xn=$Xn,tmp2,8,8	    // forward Xload
	mux2	$X[$i]=$X[$i],0x44	} //;;

___
	}
else	{
	$code.=<<___;
{ .mii;	and	tmp3=$c,$b
	dep	tmp1=tmp0,tmp4,8,8;;
	dep	$X[$i]=$X[$i],tmp1,16,16} //;;
{ .mmi;	add	$e=$e,$K_00_19		    // e+=K_00_19
	andcm	tmp1=$d,$b
	dep.z	tmp5=$a,5,27		};; // a<<5
{ .mmi;	add	$e=$e,$X[$i]		    // e+=Xupdate
	or	tmp4=tmp3,tmp1		    // F_00_19(b,c,d)=(b&c)|(~b&d)
	extr.u	tmp1=$a,27,5		}   // a>>27
{ .mmi;	xor	$Xn=$Xn,$X[($j+2)%16]	    // forward Xupdate
	xor	tmp3=$X[($j+8)%16],$X[($j+13)%16] // forward Xupdate
	nop.i	0			};;
{ .mmi;	add	$e=$e,tmp4		    // e+=F_00_19(b,c,d)
	xor	$Xn=$Xn,tmp3		    // forward Xupdate
	shrp	$b=tmp6,tmp6,2		}   // b=ROTATE(b,30)
{ .mmi; or	tmp1=tmp1,tmp5		    // ROTATE(a,5)
	mux2	tmp6=$a,0x44		};; // see b in next iteration
{ .mii;	add	$e=$e,tmp1		    // e+=ROTATE(a,5)
	shrp	$Xn=$Xn,$Xn,31		    // ROTATE(x[0]^x[2]^x[8]^x[13],1)
	mux2	$X[$i]=$X[$i],0x44	};;

___
	}
}

sub BODY_16_19 {
local	*code=shift;
my	($i,$a,$b,$c,$d,$e)=@_;
my	$j=$i+1;
my	$Xn=@X[$j%16];

$code.=<<___;
{ .mib;	add	$e=$e,$K_00_19		    // e+=K_00_19
	dep.z	tmp5=$a,5,27		}   // a<<5
{ .mib;	andcm	tmp1=$d,$b
	and	tmp0=$c,$b		};;
{ .mmi;	add	$e=$e,$X[$i%16]		    // e+=Xupdate
	or	tmp0=tmp0,tmp1		    // F_00_19(b,c,d)=(b&c)|(~b&d)
	extr.u	tmp1=$a,27,5		}   // a>>27
{ .mmi;	xor	$Xn=$Xn,$X[($j+2)%16]	    // forward Xupdate
	xor	tmp3=$X[($j+8)%16],$X[($j+13)%16]	// forward Xupdate
	nop.i	0			};;
{ .mmi;	add	$e=$e,tmp0		    // f+=F_00_19(b,c,d)
	xor	$Xn=$Xn,tmp3		    // forward Xupdate
	shrp	$b=tmp6,tmp6,2		}   // b=ROTATE(b,30)
{ .mmi;	or	tmp1=tmp1,tmp5		    // ROTATE(a,5)
	mux2	tmp6=$a,0x44		};; // see b in next iteration
{ .mii;	add	$e=$e,tmp1		    // e+=ROTATE(a,5)
	shrp	$Xn=$Xn,$Xn,31		    // ROTATE(x[0]^x[2]^x[8]^x[13],1)
	nop.i	0			};;

___
}

sub BODY_20_39 {
local	*code=shift;
my	($i,$a,$b,$c,$d,$e,$Konst)=@_;
	$Konst = $K_20_39 if (!defined($Konst));
my	$j=$i+1;
my	$Xn=@X[$j%16];

if ($i<79) {
$code.=<<___;
{ .mib;	add	$e=$e,$Konst		    // e+=K_XX_XX
	dep.z	tmp5=$a,5,27		}   // a<<5
{ .mib;	xor	tmp0=$c,$b
	xor	$Xn=$Xn,$X[($j+2)%16]	};; // forward Xupdate
{ .mib;	add	$e=$e,$X[$i%16]		    // e+=Xupdate
	extr.u	tmp1=$a,27,5		}   // a>>27
{ .mib;	xor	tmp0=tmp0,$d		    // F_20_39(b,c,d)=b^c^d
	xor	$Xn=$Xn,$X[($j+8)%16]	};; // forward Xupdate
{ .mmi;	add	$e=$e,tmp0		    // e+=F_20_39(b,c,d)
	xor	$Xn=$Xn,$X[($j+13)%16]	    // forward Xupdate
	shrp	$b=tmp6,tmp6,2		}   // b=ROTATE(b,30)
{ .mmi;	or	tmp1=tmp1,tmp5		    // ROTATE(a,5)
	mux2	tmp6=$a,0x44		};; // see b in next iteration
{ .mii;	add	$e=$e,tmp1		    // e+=ROTATE(a,5)
	shrp	$Xn=$Xn,$Xn,31		    // ROTATE(x[0]^x[2]^x[8]^x[13],1)
	nop.i	0			};;

___
}
else {
$code.=<<___;
{ .mib;	add	$e=$e,$Konst		    // e+=K_60_79
	dep.z	tmp5=$a,5,27		}   // a<<5
{ .mib;	xor	tmp0=$c,$b
	add	$h1=$h1,$a		};; // wrap up
{ .mib;	add	$e=$e,$X[$i%16]		    // e+=Xupdate
	extr.u	tmp1=$a,27,5		}   // a>>27
{ .mib;	xor	tmp0=tmp0,$d		    // F_20_39(b,c,d)=b^c^d
	add	$h3=$h3,$c		};; // wrap up
{ .mmi;	add	$e=$e,tmp0		    // e+=F_20_39(b,c,d)
	or	tmp1=tmp1,tmp5		    // ROTATE(a,5)
	shrp	$b=tmp6,tmp6,2		};; // b=ROTATE(b,30) ;;?
{ .mmi;	add	$e=$e,tmp1		    // e+=ROTATE(a,5)
	add	tmp3=1,inp		    // used in unaligned codepath
	add	$h4=$h4,$d		};; // wrap up

___
}
}

sub BODY_40_59 {
local	*code=shift;
my	($i,$a,$b,$c,$d,$e)=@_;
my	$j=$i+1;
my	$Xn=@X[$j%16];

$code.=<<___;
{ .mib;	add	$e=$e,$K_40_59		    // e+=K_40_59
	dep.z	tmp5=$a,5,27		}   // a<<5
{ .mib;	and	tmp1=$c,$d
	xor	tmp0=$c,$d		};;
{ .mmi;	add	$e=$e,$X[$i%16]		    // e+=Xupdate
	add	tmp5=tmp5,tmp1		    // a<<5+(c&d)
	extr.u	tmp1=$a,27,5		}   // a>>27
{ .mmi;	and	tmp0=tmp0,$b
	xor	$Xn=$Xn,$X[($j+2)%16]	    // forward Xupdate
	xor	tmp3=$X[($j+8)%16],$X[($j+13)%16] };;	// forward Xupdate
{ .mmi;	add	$e=$e,tmp0		    // e+=b&(c^d)
	add	tmp5=tmp5,tmp1		    // ROTATE(a,5)+(c&d)
	shrp	$b=tmp6,tmp6,2		}   // b=ROTATE(b,30)
{ .mmi;	xor	$Xn=$Xn,tmp3
	mux2	tmp6=$a,0x44		};; // see b in next iteration
{ .mii;	add	$e=$e,tmp5		    // e+=ROTATE(a,5)+(c&d)
	shrp	$Xn=$Xn,$Xn,31		    // ROTATE(x[0]^x[2]^x[8]^x[13],1)
	nop.i	0x0			};;

___
}
sub BODY_60_79	{ &BODY_20_39(@_,$K_60_79); }

$code.=<<___;
.text

tmp0=r8;
tmp1=r9;
tmp2=r10;
tmp3=r11;
ctx=r32;	// in0
inp=r33;	// in1

// void sha1_block_data_order(SHA_CTX *c,const void *p,size_t num);
.global	sha1_block_data_order#
.proc	sha1_block_data_order#
.align	32
sha1_block_data_order:
	.prologue
{ .mmi;	alloc	tmp1=ar.pfs,3,14,0,0
	$ADDP	tmp0=4,ctx
	.save	ar.lc,r3
	mov	r3=ar.lc		}
{ .mmi;	$ADDP	ctx=0,ctx
	$ADDP	inp=0,inp
	mov	r2=pr			};;
tmp4=in2;
tmp5=loc12;
tmp6=loc13;
	.body
{ .mlx;	ld4	$h0=[ctx],8
	movl	$K_00_19=0x5a827999	}
{ .mlx;	ld4	$h1=[tmp0],8
	movl	$K_20_39=0x6ed9eba1	};;
{ .mlx;	ld4	$h2=[ctx],8
	movl	$K_40_59=0x8f1bbcdc	}
{ .mlx;	ld4	$h3=[tmp0]
	movl	$K_60_79=0xca62c1d6	};;
{ .mmi;	ld4	$h4=[ctx],-16
	add	in2=-1,in2		    // adjust num for ar.lc
	mov	ar.ec=1			};;
{ .mmi;	nop.m	0
	add	tmp3=1,inp
	mov	ar.lc=in2		};; // brp.loop.imp: too far

.Ldtop:
{ .mmi;	mov	$A=$h0
	mov	$B=$h1
	mux2	tmp6=$h1,0x44		}
{ .mmi;	mov	$C=$h2
	mov	$D=$h3
	mov	$E=$h4			};;

___

{ my $i;
  my @V=($A,$B,$C,$D,$E);

	for($i=0;$i<16;$i++)	{ &BODY_00_15(\$code,$i,@V); unshift(@V,pop(@V)); }
	for(;$i<20;$i++)	{ &BODY_16_19(\$code,$i,@V); unshift(@V,pop(@V)); }
	for(;$i<40;$i++)	{ &BODY_20_39(\$code,$i,@V); unshift(@V,pop(@V)); }
	for(;$i<60;$i++)	{ &BODY_40_59(\$code,$i,@V); unshift(@V,pop(@V)); }
	for(;$i<80;$i++)	{ &BODY_60_79(\$code,$i,@V); unshift(@V,pop(@V)); }

	(($V[0] eq $A) and ($V[4] eq $E)) or die;	# double-check
}

$code.=<<___;
{ .mmb;	add	$h0=$h0,$A
	add	$h2=$h2,$C
	br.ctop.dptk.many	.Ldtop	};;
.Ldend:
{ .mmi;	add	tmp0=4,ctx
	mov	ar.lc=r3		};;
{ .mmi;	st4	[ctx]=$h0,8
	st4	[tmp0]=$h1,8		};;
{ .mmi;	st4	[ctx]=$h2,8
	st4	[tmp0]=$h3		};;
{ .mib;	st4	[ctx]=$h4,-16
	mov	pr=r2,0x1ffff
	br.ret.sptk.many	b0	};;
.endp	sha1_block_data_order#
stringz	"SHA1 block transform for IA64, CRYPTOGAMS by <appro\@openssl.org>"
___

open STDOUT,">$output" if $output;
print $code;


#! /usr/bin/env perl
# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# Multi-buffer SHA1 procedure processes n buffers in parallel by
# placing buffer data to designated lane of SIMD register. n is
# naturally limited to 4 on pre-AVX2 processors and to 8 on
# AVX2-capable processors such as Haswell.
#
#		this	+aesni(i)	sha1	aesni-sha1	gain(iv)
# -------------------------------------------------------------------
# Westmere(ii)	10.7/n	+1.28=3.96(n=4)	5.30	6.66		+68%
# Atom(ii)	18.1/n	+3.93=8.46(n=4)	9.37	12.8		+51%
# Sandy Bridge	(8.16	+5.15=13.3)/n	4.99	5.98		+80%
# Ivy Bridge	(8.08	+5.14=13.2)/n	4.60	5.54		+68%
# Haswell(iii)	(8.96	+5.00=14.0)/n	3.57	4.55		+160%
# Skylake	(8.70	+5.00=13.7)/n	3.64	4.20		+145%
# Bulldozer	(9.76	+5.76=15.5)/n	5.95	6.37		+64%
#
# (i)	multi-block CBC encrypt with 128-bit key;
# (ii)	(HASH+AES)/n does not apply to Westmere for n>3 and Atom,
#	because of lower AES-NI instruction throughput;
# (iii)	"this" is for n=8, when we gather twice as much data, result
#	for n=4 is 8.00+4.44=12.4;
# (iv)	presented improvement coefficients are asymptotic limits and
#	in real-life application are somewhat lower, e.g. for 2KB
#	fragments they range from 30% to 100% (on Haswell);

$flavour = shift;
$output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

$avx=0;

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.19) + ($1>=2.22);
}

if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
	   `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.09) + ($1>=2.10);
}

if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
	   `ml64 2>&1` =~ /Version ([0-9]+)\./) {
	$avx = ($1>=10) + ($1>=11);
}

if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
	$avx = ($2>=3.0) + ($2>3.0);
}

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

# void sha1_multi_block (
#     struct {	unsigned int A[8];
#		unsigned int B[8];
#		unsigned int C[8];
#		unsigned int D[8];
#		unsigned int E[8];	} *ctx,
#     struct {	void *ptr; int blocks;	} inp[8],
#     int num);		/* 1 or 2 */
#
$ctx="%rdi";	# 1st arg
$inp="%rsi";	# 2nd arg
$num="%edx";
@ptr=map("%r$_",(8..11));
$Tbl="%rbp";

@V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4));
($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9));
@Xi=map("%xmm$_",(10..14));
$K="%xmm15";

if (1) {
    # Atom-specific optimization aiming to eliminate pshufb with high
    # registers [and thus get rid of 48 cycles accumulated penalty]
    @Xi=map("%xmm$_",(0..4));
    ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
    @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
}

$REG_SZ=16;

sub Xi_off {
my $off = shift;

    $off %= 16; $off *= $REG_SZ;
    $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
}

sub BODY_00_19 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
my $k=$i+2;

# Loads are performed 2+3/4 iterations in advance. 3/4 means that out
# of 4 words you would expect to be loaded per given iteration one is
# spilled to next iteration. In other words indices in four input
# streams are distributed as following:
#
# $i==0:	0,0,0,0,1,1,1,1,2,2,2,
# $i==1:	2,3,3,3,
# $i==2:	3,4,4,4,
# ...
# $i==13:	14,15,15,15,
# $i==14:	15
#
# Then at $i==15 Xupdate is applied one iteration in advance...
$code.=<<___ if ($i==0);
	movd		(@ptr[0]),@Xi[0]
	 lea		`16*4`(@ptr[0]),@ptr[0]
	movd		(@ptr[1]),@Xi[2]	# borrow @Xi[2]
	 lea		`16*4`(@ptr[1]),@ptr[1]
	movd		(@ptr[2]),@Xi[3]	# borrow @Xi[3]
	 lea		`16*4`(@ptr[2]),@ptr[2]
	movd		(@ptr[3]),@Xi[4]	# borrow @Xi[4]
	 lea		`16*4`(@ptr[3]),@ptr[3]
	punpckldq	@Xi[3],@Xi[0]
	 movd		`4*$j-16*4`(@ptr[0]),@Xi[1]
	punpckldq	@Xi[4],@Xi[2]
	 movd		`4*$j-16*4`(@ptr[1]),$t3
	punpckldq	@Xi[2],@Xi[0]
	 movd		`4*$j-16*4`(@ptr[2]),$t2
	pshufb		$tx,@Xi[0]
___
$code.=<<___ if ($i<14);			# just load input
	 movd		`4*$j-16*4`(@ptr[3]),$t1
	 punpckldq	$t2,@Xi[1]
	movdqa	$a,$t2
	paddd	$K,$e				# e+=K_00_19
	 punpckldq	$t1,$t3
	movdqa	$b,$t1
	movdqa	$b,$t0
	pslld	\$5,$t2
	pandn	$d,$t1
	pand	$c,$t0
	 punpckldq	$t3,@Xi[1]
	movdqa	$a,$t3

	movdqa	@Xi[0],`&Xi_off($i)`
	paddd	@Xi[0],$e			# e+=X[i]
	 movd		`4*$k-16*4`(@ptr[0]),@Xi[2]
	psrld	\$27,$t3
	pxor	$t1,$t0				# Ch(b,c,d)
	movdqa	$b,$t1

	por	$t3,$t2				# rol(a,5)
	 movd		`4*$k-16*4`(@ptr[1]),$t3
	pslld	\$30,$t1
	paddd	$t0,$e				# e+=Ch(b,c,d)

	psrld	\$2,$b
	paddd	$t2,$e				# e+=rol(a,5)
	 pshufb	$tx,@Xi[1]
	 movd		`4*$k-16*4`(@ptr[2]),$t2
	por	$t1,$b				# b=rol(b,30)
___
$code.=<<___ if ($i==14);			# just load input
	 movd		`4*$j-16*4`(@ptr[3]),$t1
	 punpckldq	$t2,@Xi[1]
	movdqa	$a,$t2
	paddd	$K,$e				# e+=K_00_19
	 punpckldq	$t1,$t3
	movdqa	$b,$t1
	movdqa	$b,$t0
	pslld	\$5,$t2
	 prefetcht0	63(@ptr[0])
	pandn	$d,$t1
	pand	$c,$t0
	 punpckldq	$t3,@Xi[1]
	movdqa	$a,$t3

	movdqa	@Xi[0],`&Xi_off($i)`
	paddd	@Xi[0],$e			# e+=X[i]
	psrld	\$27,$t3
	pxor	$t1,$t0				# Ch(b,c,d)
	movdqa	$b,$t1
	 prefetcht0	63(@ptr[1])

	por	$t3,$t2				# rol(a,5)
	pslld	\$30,$t1
	paddd	$t0,$e				# e+=Ch(b,c,d)
	 prefetcht0	63(@ptr[2])

	psrld	\$2,$b
	paddd	$t2,$e				# e+=rol(a,5)
	 pshufb	$tx,@Xi[1]
	 prefetcht0	63(@ptr[3])
	por	$t1,$b				# b=rol(b,30)
___
$code.=<<___ if ($i>=13 && $i<15);
	movdqa	`&Xi_off($j+2)`,@Xi[3]		# preload "X[2]"
___
$code.=<<___ if ($i>=15);			# apply Xupdate
	pxor	@Xi[-2],@Xi[1]			# "X[13]"
	movdqa	`&Xi_off($j+2)`,@Xi[3]		# "X[2]"

	movdqa	$a,$t2
	 pxor	`&Xi_off($j+8)`,@Xi[1]
	paddd	$K,$e				# e+=K_00_19
	movdqa	$b,$t1
	pslld	\$5,$t2
	 pxor	@Xi[3],@Xi[1]
	movdqa	$b,$t0
	pandn	$d,$t1
	 movdqa	@Xi[1],$tx
	pand	$c,$t0
	movdqa	$a,$t3
	 psrld	\$31,$tx
	 paddd	@Xi[1],@Xi[1]

	movdqa	@Xi[0],`&Xi_off($i)`
	paddd	@Xi[0],$e			# e+=X[i]
	psrld	\$27,$t3
	pxor	$t1,$t0				# Ch(b,c,d)

	movdqa	$b,$t1
	por	$t3,$t2				# rol(a,5)
	pslld	\$30,$t1
	paddd	$t0,$e				# e+=Ch(b,c,d)

	psrld	\$2,$b
	paddd	$t2,$e				# e+=rol(a,5)
	 por	$tx,@Xi[1]			# rol	\$1,@Xi[1]
	por	$t1,$b				# b=rol(b,30)
___
push(@Xi,shift(@Xi));
}

sub BODY_20_39 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;

$code.=<<___ if ($i<79);
	pxor	@Xi[-2],@Xi[1]			# "X[13]"
	movdqa	`&Xi_off($j+2)`,@Xi[3]		# "X[2]"

	movdqa	$a,$t2
	movdqa	$d,$t0
	 pxor	`&Xi_off($j+8)`,@Xi[1]
	paddd	$K,$e				# e+=K_20_39
	pslld	\$5,$t2
	pxor	$b,$t0

	movdqa	$a,$t3
___
$code.=<<___ if ($i<72);
	movdqa	@Xi[0],`&Xi_off($i)`
___
$code.=<<___ if ($i<79);
	paddd	@Xi[0],$e			# e+=X[i]
	 pxor	@Xi[3],@Xi[1]
	psrld	\$27,$t3
	pxor	$c,$t0				# Parity(b,c,d)
	movdqa	$b,$t1

	pslld	\$30,$t1
	 movdqa	@Xi[1],$tx
	por	$t3,$t2				# rol(a,5)
	 psrld	\$31,$tx
	paddd	$t0,$e				# e+=Parity(b,c,d)
	 paddd	@Xi[1],@Xi[1]

	psrld	\$2,$b
	paddd	$t2,$e				# e+=rol(a,5)
	 por	$tx,@Xi[1]			# rol(@Xi[1],1)
	por	$t1,$b				# b=rol(b,30)
___
$code.=<<___ if ($i==79);
	movdqa	$a,$t2
	paddd	$K,$e				# e+=K_20_39
	movdqa	$d,$t0
	pslld	\$5,$t2
	pxor	$b,$t0

	movdqa	$a,$t3
	paddd	@Xi[0],$e			# e+=X[i]
	psrld	\$27,$t3
	movdqa	$b,$t1
	pxor	$c,$t0				# Parity(b,c,d)

	pslld	\$30,$t1
	por	$t3,$t2				# rol(a,5)
	paddd	$t0,$e				# e+=Parity(b,c,d)

	psrld	\$2,$b
	paddd	$t2,$e				# e+=rol(a,5)
	por	$t1,$b				# b=rol(b,30)
___
push(@Xi,shift(@Xi));
}

sub BODY_40_59 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;

$code.=<<___;
	pxor	@Xi[-2],@Xi[1]			# "X[13]"
	movdqa	`&Xi_off($j+2)`,@Xi[3]		# "X[2]"

	movdqa	$a,$t2
	movdqa	$d,$t1
	 pxor	`&Xi_off($j+8)`,@Xi[1]
	pxor	@Xi[3],@Xi[1]
	paddd	$K,$e				# e+=K_40_59
	pslld	\$5,$t2
	movdqa	$a,$t3
	pand	$c,$t1

	movdqa	$d,$t0
	 movdqa	@Xi[1],$tx
	psrld	\$27,$t3
	paddd	$t1,$e
	pxor	$c,$t0

	movdqa	@Xi[0],`&Xi_off($i)`
	paddd	@Xi[0],$e			# e+=X[i]
	por	$t3,$t2				# rol(a,5)
	 psrld	\$31,$tx
	pand	$b,$t0
	movdqa	$b,$t1

	pslld	\$30,$t1
	 paddd	@Xi[1],@Xi[1]
	paddd	$t0,$e				# e+=Maj(b,d,c)

	psrld	\$2,$b
	paddd	$t2,$e				# e+=rol(a,5)
	 por	$tx,@Xi[1]			# rol(@X[1],1)
	por	$t1,$b				# b=rol(b,30)
___
push(@Xi,shift(@Xi));
}

$code.=<<___;
.text

.extern	OPENSSL_ia32cap_P

.globl	sha1_multi_block
.type	sha1_multi_block,\@function,3
.align	32
sha1_multi_block:
.cfi_startproc
	mov	OPENSSL_ia32cap_P+4(%rip),%rcx
	bt	\$61,%rcx			# check SHA bit
	jc	_shaext_shortcut
___
$code.=<<___ if ($avx);
	test	\$`1<<28`,%ecx
	jnz	_avx_shortcut
___
$code.=<<___;
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbx
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,-0x78(%rax)
	movaps	%xmm11,-0x68(%rax)
	movaps	%xmm12,-0x58(%rax)
	movaps	%xmm13,-0x48(%rax)
	movaps	%xmm14,-0x38(%rax)
	movaps	%xmm15,-0x28(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`,%rsp
	and	\$-256,%rsp
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.cfi_cfa_expression	%rsp+`$REG_SZ*17`,deref,+8
.Lbody:
	lea	K_XX_XX(%rip),$Tbl
	lea	`$REG_SZ*16`(%rsp),%rbx

.Loop_grande:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldone

	movdqu	0x00($ctx),$A			# load context
	 lea	128(%rsp),%rax
	movdqu	0x20($ctx),$B
	movdqu	0x40($ctx),$C
	movdqu	0x60($ctx),$D
	movdqu	0x80($ctx),$E
	movdqa	0x60($Tbl),$tx			# pbswap_mask
	movdqa	-0x20($Tbl),$K			# K_00_19
	jmp	.Loop

.align	32
.Loop:
___
for($i=0;$i<20;$i++)	{ &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
$code.="	movdqa	0x00($Tbl),$K\n";	# K_20_39
for(;$i<40;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
$code.="	movdqa	0x20($Tbl),$K\n";	# K_40_59
for(;$i<60;$i++)	{ &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
$code.="	movdqa	0x40($Tbl),$K\n";	# K_60_79
for(;$i<80;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	movdqa	(%rbx),@Xi[0]			# pull counters
	mov	\$1,%ecx
	cmp	4*0(%rbx),%ecx			# examine counters
	pxor	$t2,$t2
	cmovge	$Tbl,@ptr[0]			# cancel input
	cmp	4*1(%rbx),%ecx
	movdqa	@Xi[0],@Xi[1]
	cmovge	$Tbl,@ptr[1]
	cmp	4*2(%rbx),%ecx
	pcmpgtd	$t2,@Xi[1]			# mask value
	cmovge	$Tbl,@ptr[2]
	cmp	4*3(%rbx),%ecx
	paddd	@Xi[1],@Xi[0]			# counters--
	cmovge	$Tbl,@ptr[3]

	movdqu	0x00($ctx),$t0
	pand	@Xi[1],$A
	movdqu	0x20($ctx),$t1
	pand	@Xi[1],$B
	paddd	$t0,$A
	movdqu	0x40($ctx),$t2
	pand	@Xi[1],$C
	paddd	$t1,$B
	movdqu	0x60($ctx),$t3
	pand	@Xi[1],$D
	paddd	$t2,$C
	movdqu	0x80($ctx),$tx
	pand	@Xi[1],$E
	movdqu	$A,0x00($ctx)
	paddd	$t3,$D
	movdqu	$B,0x20($ctx)
	paddd	$tx,$E
	movdqu	$C,0x40($ctx)
	movdqu	$D,0x60($ctx)
	movdqu	$E,0x80($ctx)

	movdqa	@Xi[0],(%rbx)			# save counters
	movdqa	0x60($Tbl),$tx			# pbswap_mask
	movdqa	-0x20($Tbl),$K			# K_00_19
	dec	$num
	jnz	.Loop

	mov	`$REG_SZ*17+8`(%rsp),$num
	lea	$REG_SZ($ctx),$ctx
	lea	`16*$REG_SZ/4`($inp),$inp
	dec	$num
	jnz	.Loop_grande

.Ldone:
	mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
.cfi_def_cfa	%rax,8
___
$code.=<<___ if ($win64);
	movaps	-0xb8(%rax),%xmm6
	movaps	-0xa8(%rax),%xmm7
	movaps	-0x98(%rax),%xmm8
	movaps	-0x88(%rax),%xmm9
	movaps	-0x78(%rax),%xmm10
	movaps	-0x68(%rax),%xmm11
	movaps	-0x58(%rax),%xmm12
	movaps	-0x48(%rax),%xmm13
	movaps	-0x38(%rax),%xmm14
	movaps	-0x28(%rax),%xmm15
___
$code.=<<___;
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue:
	ret
.cfi_endproc
.size	sha1_multi_block,.-sha1_multi_block
___
						{{{
my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
my @MSG0=map("%xmm$_",(4..7));
my @MSG1=map("%xmm$_",(11..14));

$code.=<<___;
.type	sha1_multi_block_shaext,\@function,3
.align	32
sha1_multi_block_shaext:
.cfi_startproc
_shaext_shortcut:
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,-0x78(%rax)
	movaps	%xmm11,-0x68(%rax)
	movaps	%xmm12,-0x58(%rax)
	movaps	%xmm13,-0x48(%rax)
	movaps	%xmm14,-0x38(%rax)
	movaps	%xmm15,-0x28(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`,%rsp
	shl	\$1,$num			# we process pair at a time
	and	\$-256,%rsp
	lea	0x40($ctx),$ctx			# size optimization
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.Lbody_shaext:
	lea	`$REG_SZ*16`(%rsp),%rbx
	movdqa	K_XX_XX+0x80(%rip),$BSWAP	# byte-n-word swap

.Loop_grande_shaext:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
___
for($i=0;$i<2;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	%rsp,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldone_shaext

	movq		0x00-0x40($ctx),$ABCD0	# a1.a0
	movq		0x20-0x40($ctx),@MSG0[0]# b1.b0
	movq		0x40-0x40($ctx),@MSG0[1]# c1.c0
	movq		0x60-0x40($ctx),@MSG0[2]# d1.d0
	movq		0x80-0x40($ctx),@MSG0[3]# e1.e0

	punpckldq	@MSG0[0],$ABCD0		# b1.a1.b0.a0
	punpckldq	@MSG0[2],@MSG0[1]	# d1.c1.d0.c0

	movdqa		$ABCD0,$ABCD1
	punpcklqdq	@MSG0[1],$ABCD0		# d0.c0.b0.a0
	punpckhqdq	@MSG0[1],$ABCD1		# d1.c1.b1.a1

	pshufd		\$0b00111111,@MSG0[3],$E0
	pshufd		\$0b01111111,@MSG0[3],$E1
	pshufd		\$0b00011011,$ABCD0,$ABCD0
	pshufd		\$0b00011011,$ABCD1,$ABCD1
	jmp		.Loop_shaext

.align	32
.Loop_shaext:
	movdqu		0x00(@ptr[0]),@MSG0[0]
	 movdqu		0x00(@ptr[1]),@MSG1[0]
	movdqu		0x10(@ptr[0]),@MSG0[1]
	 movdqu		0x10(@ptr[1]),@MSG1[1]
	movdqu		0x20(@ptr[0]),@MSG0[2]
	pshufb		$BSWAP,@MSG0[0]
	 movdqu		0x20(@ptr[1]),@MSG1[2]
	 pshufb		$BSWAP,@MSG1[0]
	movdqu		0x30(@ptr[0]),@MSG0[3]
	lea		0x40(@ptr[0]),@ptr[0]
	pshufb		$BSWAP,@MSG0[1]
	 movdqu		0x30(@ptr[1]),@MSG1[3]
	 lea		0x40(@ptr[1]),@ptr[1]
	 pshufb		$BSWAP,@MSG1[1]

	movdqa		$E0,0x50(%rsp)		# offload
	paddd		@MSG0[0],$E0
	 movdqa		$E1,0x70(%rsp)
	 paddd		@MSG1[0],$E1
	movdqa		$ABCD0,0x40(%rsp)	# offload
	movdqa		$ABCD0,$E0_
	 movdqa		$ABCD1,0x60(%rsp)
	 movdqa		$ABCD1,$E1_
	sha1rnds4	\$0,$E0,$ABCD0		# 0-3
	sha1nexte	@MSG0[1],$E0_
	 sha1rnds4	\$0,$E1,$ABCD1		# 0-3
	 sha1nexte	@MSG1[1],$E1_
	pshufb		$BSWAP,@MSG0[2]
	prefetcht0	127(@ptr[0])
	sha1msg1	@MSG0[1],@MSG0[0]
	 pshufb		$BSWAP,@MSG1[2]
	 prefetcht0	127(@ptr[1])
	 sha1msg1	@MSG1[1],@MSG1[0]

	pshufb		$BSWAP,@MSG0[3]
	movdqa		$ABCD0,$E0
	 pshufb		$BSWAP,@MSG1[3]
	 movdqa		$ABCD1,$E1
	sha1rnds4	\$0,$E0_,$ABCD0		# 4-7
	sha1nexte	@MSG0[2],$E0
	 sha1rnds4	\$0,$E1_,$ABCD1		# 4-7
	 sha1nexte	@MSG1[2],$E1
	pxor		@MSG0[2],@MSG0[0]
	sha1msg1	@MSG0[2],@MSG0[1]
	 pxor		@MSG1[2],@MSG1[0]
	 sha1msg1	@MSG1[2],@MSG1[1]
___
for($i=2;$i<20-4;$i++) {
$code.=<<___;
	movdqa		$ABCD0,$E0_
	 movdqa		$ABCD1,$E1_
	sha1rnds4	\$`int($i/5)`,$E0,$ABCD0	# 8-11
	sha1nexte	@MSG0[3],$E0_
	 sha1rnds4	\$`int($i/5)`,$E1,$ABCD1	# 8-11
	 sha1nexte	@MSG1[3],$E1_
	sha1msg2	@MSG0[3],@MSG0[0]
	 sha1msg2	@MSG1[3],@MSG1[0]
	pxor		@MSG0[3],@MSG0[1]
	sha1msg1	@MSG0[3],@MSG0[2]
	 pxor		@MSG1[3],@MSG1[1]
	 sha1msg1	@MSG1[3],@MSG1[2]
___
	($E0,$E0_)=($E0_,$E0);		($E1,$E1_)=($E1_,$E1);
	push(@MSG0,shift(@MSG0));	push(@MSG1,shift(@MSG1));
}
$code.=<<___;
	movdqa		$ABCD0,$E0_
	 movdqa		$ABCD1,$E1_
	sha1rnds4	\$3,$E0,$ABCD0		# 64-67
	sha1nexte	@MSG0[3],$E0_
	 sha1rnds4	\$3,$E1,$ABCD1		# 64-67
	 sha1nexte	@MSG1[3],$E1_
	sha1msg2	@MSG0[3],@MSG0[0]
	 sha1msg2	@MSG1[3],@MSG1[0]
	pxor		@MSG0[3],@MSG0[1]
	 pxor		@MSG1[3],@MSG1[1]

	mov		\$1,%ecx
	pxor		@MSG0[2],@MSG0[2]	# zero
	cmp		4*0(%rbx),%ecx		# examine counters
	cmovge		%rsp,@ptr[0]		# cancel input

	movdqa		$ABCD0,$E0
	 movdqa		$ABCD1,$E1
	sha1rnds4	\$3,$E0_,$ABCD0		# 68-71
	sha1nexte	@MSG0[0],$E0
	 sha1rnds4	\$3,$E1_,$ABCD1		# 68-71
	 sha1nexte	@MSG1[0],$E1
	sha1msg2	@MSG0[0],@MSG0[1]
	 sha1msg2	@MSG1[0],@MSG1[1]

	cmp		4*1(%rbx),%ecx
	cmovge		%rsp,@ptr[1]
	movq		(%rbx),@MSG0[0]		# pull counters

	movdqa		$ABCD0,$E0_
	 movdqa		$ABCD1,$E1_
	sha1rnds4	\$3,$E0,$ABCD0		# 72-75
	sha1nexte	@MSG0[1],$E0_
	 sha1rnds4	\$3,$E1,$ABCD1		# 72-75
	 sha1nexte	@MSG1[1],$E1_

	pshufd		\$0x00,@MSG0[0],@MSG1[2]
	pshufd		\$0x55,@MSG0[0],@MSG1[3]
	movdqa		@MSG0[0],@MSG0[1]
	pcmpgtd		@MSG0[2],@MSG1[2]
	pcmpgtd		@MSG0[2],@MSG1[3]

	movdqa		$ABCD0,$E0
	 movdqa		$ABCD1,$E1
	sha1rnds4	\$3,$E0_,$ABCD0		# 76-79
	sha1nexte	$MSG0[2],$E0
	 sha1rnds4	\$3,$E1_,$ABCD1		# 76-79
	 sha1nexte	$MSG0[2],$E1

	pcmpgtd		@MSG0[2],@MSG0[1]	# counter mask
	pand		@MSG1[2],$ABCD0
	pand		@MSG1[2],$E0
	 pand		@MSG1[3],$ABCD1
	 pand		@MSG1[3],$E1
	paddd		@MSG0[1],@MSG0[0]	# counters--

	paddd		0x40(%rsp),$ABCD0
	paddd		0x50(%rsp),$E0
	 paddd		0x60(%rsp),$ABCD1
	 paddd		0x70(%rsp),$E1

	movq		@MSG0[0],(%rbx)		# save counters
	dec		$num
	jnz		.Loop_shaext

	mov		`$REG_SZ*17+8`(%rsp),$num

	pshufd		\$0b00011011,$ABCD0,$ABCD0
	pshufd		\$0b00011011,$ABCD1,$ABCD1

	movdqa		$ABCD0,@MSG0[0]
	punpckldq	$ABCD1,$ABCD0		# b1.b0.a1.a0
	punpckhdq	$ABCD1,@MSG0[0]		# d1.d0.c1.c0
	punpckhdq	$E1,$E0			# e1.e0.xx.xx
	movq		$ABCD0,0x00-0x40($ctx)	# a1.a0
	psrldq		\$8,$ABCD0
	movq		@MSG0[0],0x40-0x40($ctx)# c1.c0
	psrldq		\$8,@MSG0[0]
	movq		$ABCD0,0x20-0x40($ctx)	# b1.b0
	psrldq		\$8,$E0
	movq		@MSG0[0],0x60-0x40($ctx)# d1.d0
	movq		$E0,0x80-0x40($ctx)	# e1.e0

	lea	`$REG_SZ/2`($ctx),$ctx
	lea	`16*2`($inp),$inp
	dec	$num
	jnz	.Loop_grande_shaext

.Ldone_shaext:
	#mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
___
$code.=<<___ if ($win64);
	movaps	-0xb8(%rax),%xmm6
	movaps	-0xa8(%rax),%xmm7
	movaps	-0x98(%rax),%xmm8
	movaps	-0x88(%rax),%xmm9
	movaps	-0x78(%rax),%xmm10
	movaps	-0x68(%rax),%xmm11
	movaps	-0x58(%rax),%xmm12
	movaps	-0x48(%rax),%xmm13
	movaps	-0x38(%rax),%xmm14
	movaps	-0x28(%rax),%xmm15
___
$code.=<<___;
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue_shaext:
	ret
.cfi_endproc
.size	sha1_multi_block_shaext,.-sha1_multi_block_shaext
___
						}}}

						if ($avx) {{{
sub BODY_00_19_avx {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
my $k=$i+2;
my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128";
my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4];

$code.=<<___ if ($i==0 && $REG_SZ==16);
	vmovd		(@ptr[0]),@Xi[0]
	 lea		`16*4`(@ptr[0]),@ptr[0]
	vmovd		(@ptr[1]),@Xi[2]	# borrow Xi[2]
	 lea		`16*4`(@ptr[1]),@ptr[1]
	vpinsrd		\$1,(@ptr[2]),@Xi[0],@Xi[0]
	 lea		`16*4`(@ptr[2]),@ptr[2]
	vpinsrd		\$1,(@ptr[3]),@Xi[2],@Xi[2]
	 lea		`16*4`(@ptr[3]),@ptr[3]
	 vmovd		`4*$j-16*4`(@ptr[0]),@Xi[1]
	vpunpckldq	@Xi[2],@Xi[0],@Xi[0]
	 vmovd		`4*$j-16*4`($ptr_n),$t3
	vpshufb		$tx,@Xi[0],@Xi[0]
___
$code.=<<___ if ($i<15 && $REG_SZ==16);		# just load input
	 vpinsrd	\$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
	 vpinsrd	\$1,`4*$j-16*4`(@ptr[3]),$t3,$t3
___
$code.=<<___ if ($i==0 && $REG_SZ==32);
	vmovd		(@ptr[0]),@Xi[0]
	 lea		`16*4`(@ptr[0]),@ptr[0]
	vmovd		(@ptr[4]),@Xi[2]	# borrow Xi[2]
	 lea		`16*4`(@ptr[4]),@ptr[4]
	vmovd		(@ptr[1]),$t2
	 lea		`16*4`(@ptr[1]),@ptr[1]
	vmovd		(@ptr[5]),$t1
	 lea		`16*4`(@ptr[5]),@ptr[5]
	vpinsrd		\$1,(@ptr[2]),@Xi[0],@Xi[0]
	 lea		`16*4`(@ptr[2]),@ptr[2]
	vpinsrd		\$1,(@ptr[6]),@Xi[2],@Xi[2]
	 lea		`16*4`(@ptr[6]),@ptr[6]
	vpinsrd		\$1,(@ptr[3]),$t2,$t2
	 lea		`16*4`(@ptr[3]),@ptr[3]
	vpunpckldq	$t2,@Xi[0],@Xi[0]
	vpinsrd		\$1,(@ptr[7]),$t1,$t1
	 lea		`16*4`(@ptr[7]),@ptr[7]
	vpunpckldq	$t1,@Xi[2],@Xi[2]
	 vmovd		`4*$j-16*4`(@ptr[0]),@Xi[1]
	vinserti128	@Xi[2],@Xi[0],@Xi[0]
	 vmovd		`4*$j-16*4`($ptr_n),$t3
	vpshufb		$tx,@Xi[0],@Xi[0]
___
$code.=<<___ if ($i<15 && $REG_SZ==32);		# just load input
	 vmovd		`4*$j-16*4`(@ptr[1]),$t2
	 vmovd		`4*$j-16*4`(@ptr[5]),$t1
	 vpinsrd	\$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
	 vpinsrd	\$1,`4*$j-16*4`(@ptr[6]),$t3,$t3
	 vpinsrd	\$1,`4*$j-16*4`(@ptr[3]),$t2,$t2
	 vpunpckldq	$t2,@Xi[1],@Xi[1]
	 vpinsrd	\$1,`4*$j-16*4`(@ptr[7]),$t1,$t1
	 vpunpckldq	$t1,$t3,$t3
___
$code.=<<___ if ($i<14);
	vpaddd	$K,$e,$e			# e+=K_00_19
	vpslld	\$5,$a,$t2
	vpandn	$d,$b,$t1
	vpand	$c,$b,$t0

	vmovdqa	@Xi[0],`&Xi_off($i)`
	vpaddd	@Xi[0],$e,$e			# e+=X[i]
	 $vpack		$t3,@Xi[1],@Xi[1]
	vpsrld	\$27,$a,$t3
	vpxor	$t1,$t0,$t0			# Ch(b,c,d)
	 vmovd		`4*$k-16*4`(@ptr[0]),@Xi[2]

	vpslld	\$30,$b,$t1
	vpor	$t3,$t2,$t2			# rol(a,5)
	 vmovd		`4*$k-16*4`($ptr_n),$t3
	vpaddd	$t0,$e,$e			# e+=Ch(b,c,d)

	vpsrld	\$2,$b,$b
	vpaddd	$t2,$e,$e			# e+=rol(a,5)
	 vpshufb	$tx,@Xi[1],@Xi[1]
	vpor	$t1,$b,$b			# b=rol(b,30)
___
$code.=<<___ if ($i==14);
	vpaddd	$K,$e,$e			# e+=K_00_19
	 prefetcht0	63(@ptr[0])
	vpslld	\$5,$a,$t2
	vpandn	$d,$b,$t1
	vpand	$c,$b,$t0

	vmovdqa	@Xi[0],`&Xi_off($i)`
	vpaddd	@Xi[0],$e,$e			# e+=X[i]
	 $vpack		$t3,@Xi[1],@Xi[1]
	vpsrld	\$27,$a,$t3
	 prefetcht0	63(@ptr[1])
	vpxor	$t1,$t0,$t0			# Ch(b,c,d)

	vpslld	\$30,$b,$t1
	vpor	$t3,$t2,$t2			# rol(a,5)
	 prefetcht0	63(@ptr[2])
	vpaddd	$t0,$e,$e			# e+=Ch(b,c,d)

	vpsrld	\$2,$b,$b
	vpaddd	$t2,$e,$e			# e+=rol(a,5)
	 prefetcht0	63(@ptr[3])
	 vpshufb	$tx,@Xi[1],@Xi[1]
	vpor	$t1,$b,$b			# b=rol(b,30)
___
$code.=<<___ if ($i>=13 && $i<15);
	vmovdqa	`&Xi_off($j+2)`,@Xi[3]		# preload "X[2]"
___
$code.=<<___ if ($i>=15);			# apply Xupdate
	vpxor	@Xi[-2],@Xi[1],@Xi[1]		# "X[13]"
	vmovdqa	`&Xi_off($j+2)`,@Xi[3]		# "X[2]"

	vpaddd	$K,$e,$e			# e+=K_00_19
	vpslld	\$5,$a,$t2
	vpandn	$d,$b,$t1
	 `"prefetcht0	63(@ptr[4])"		if ($i==15 && $REG_SZ==32)`
	vpand	$c,$b,$t0

	vmovdqa	@Xi[0],`&Xi_off($i)`
	vpaddd	@Xi[0],$e,$e			# e+=X[i]
	 vpxor	`&Xi_off($j+8)`,@Xi[1],@Xi[1]
	vpsrld	\$27,$a,$t3
	vpxor	$t1,$t0,$t0			# Ch(b,c,d)
	 vpxor	@Xi[3],@Xi[1],@Xi[1]
	 `"prefetcht0	63(@ptr[5])"		if ($i==15 && $REG_SZ==32)`

	vpslld	\$30,$b,$t1
	vpor	$t3,$t2,$t2			# rol(a,5)
	vpaddd	$t0,$e,$e			# e+=Ch(b,c,d)
	 `"prefetcht0	63(@ptr[6])"		if ($i==15 && $REG_SZ==32)`
	 vpsrld	\$31,@Xi[1],$tx
	 vpaddd	@Xi[1],@Xi[1],@Xi[1]

	vpsrld	\$2,$b,$b
	 `"prefetcht0	63(@ptr[7])"		if ($i==15 && $REG_SZ==32)`
	vpaddd	$t2,$e,$e			# e+=rol(a,5)
	 vpor	$tx,@Xi[1],@Xi[1]		# rol	\$1,@Xi[1]
	vpor	$t1,$b,$b			# b=rol(b,30)
___
push(@Xi,shift(@Xi));
}

sub BODY_20_39_avx {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;

$code.=<<___ if ($i<79);
	vpxor	@Xi[-2],@Xi[1],@Xi[1]		# "X[13]"
	vmovdqa	`&Xi_off($j+2)`,@Xi[3]		# "X[2]"

	vpslld	\$5,$a,$t2
	vpaddd	$K,$e,$e			# e+=K_20_39
	vpxor	$b,$d,$t0
___
$code.=<<___ if ($i<72);
	vmovdqa	@Xi[0],`&Xi_off($i)`
___
$code.=<<___ if ($i<79);
	vpaddd	@Xi[0],$e,$e			# e+=X[i]
	 vpxor	`&Xi_off($j+8)`,@Xi[1],@Xi[1]
	vpsrld	\$27,$a,$t3
	vpxor	$c,$t0,$t0			# Parity(b,c,d)
	 vpxor	@Xi[3],@Xi[1],@Xi[1]

	vpslld	\$30,$b,$t1
	vpor	$t3,$t2,$t2			# rol(a,5)
	vpaddd	$t0,$e,$e			# e+=Parity(b,c,d)
	 vpsrld	\$31,@Xi[1],$tx
	 vpaddd	@Xi[1],@Xi[1],@Xi[1]

	vpsrld	\$2,$b,$b
	vpaddd	$t2,$e,$e			# e+=rol(a,5)
	 vpor	$tx,@Xi[1],@Xi[1]		# rol(@Xi[1],1)
	vpor	$t1,$b,$b			# b=rol(b,30)
___
$code.=<<___ if ($i==79);
	vpslld	\$5,$a,$t2
	vpaddd	$K,$e,$e			# e+=K_20_39
	vpxor	$b,$d,$t0

	vpsrld	\$27,$a,$t3
	vpaddd	@Xi[0],$e,$e			# e+=X[i]
	vpxor	$c,$t0,$t0			# Parity(b,c,d)

	vpslld	\$30,$b,$t1
	vpor	$t3,$t2,$t2			# rol(a,5)
	vpaddd	$t0,$e,$e			# e+=Parity(b,c,d)

	vpsrld	\$2,$b,$b
	vpaddd	$t2,$e,$e			# e+=rol(a,5)
	vpor	$t1,$b,$b			# b=rol(b,30)
___
push(@Xi,shift(@Xi));
}

sub BODY_40_59_avx {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;

$code.=<<___;
	vpxor	@Xi[-2],@Xi[1],@Xi[1]		# "X[13]"
	vmovdqa	`&Xi_off($j+2)`,@Xi[3]		# "X[2]"

	vpaddd	$K,$e,$e			# e+=K_40_59
	vpslld	\$5,$a,$t2
	vpand	$c,$d,$t1
	 vpxor	`&Xi_off($j+8)`,@Xi[1],@Xi[1]

	vpaddd	$t1,$e,$e
	vpsrld	\$27,$a,$t3
	vpxor	$c,$d,$t0
	 vpxor	@Xi[3],@Xi[1],@Xi[1]

	vmovdqu	@Xi[0],`&Xi_off($i)`
	vpaddd	@Xi[0],$e,$e			# e+=X[i]
	vpor	$t3,$t2,$t2			# rol(a,5)
	 vpsrld	\$31,@Xi[1],$tx
	vpand	$b,$t0,$t0
	 vpaddd	@Xi[1],@Xi[1],@Xi[1]

	vpslld	\$30,$b,$t1
	vpaddd	$t0,$e,$e			# e+=Maj(b,d,c)

	vpsrld	\$2,$b,$b
	vpaddd	$t2,$e,$e			# e+=rol(a,5)
	 vpor	$tx,@Xi[1],@Xi[1]		# rol(@X[1],1)
	vpor	$t1,$b,$b			# b=rol(b,30)
___
push(@Xi,shift(@Xi));
}

$code.=<<___;
.type	sha1_multi_block_avx,\@function,3
.align	32
sha1_multi_block_avx:
.cfi_startproc
_avx_shortcut:
___
$code.=<<___ if ($avx>1);
	shr	\$32,%rcx
	cmp	\$2,$num
	jb	.Lavx
	test	\$`1<<5`,%ecx
	jnz	_avx2_shortcut
	jmp	.Lavx
.align	32
.Lavx:
___
$code.=<<___;
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,-0x78(%rax)
	movaps	%xmm11,-0x68(%rax)
	movaps	%xmm12,-0x58(%rax)
	movaps	%xmm13,-0x48(%rax)
	movaps	%xmm14,-0x38(%rax)
	movaps	%xmm15,-0x28(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`, %rsp
	and	\$-256,%rsp
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.cfi_cfa_expression	%rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx:
	lea	K_XX_XX(%rip),$Tbl
	lea	`$REG_SZ*16`(%rsp),%rbx

	vzeroupper
.Loop_grande_avx:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldone_avx

	vmovdqu	0x00($ctx),$A			# load context
	 lea	128(%rsp),%rax
	vmovdqu	0x20($ctx),$B
	vmovdqu	0x40($ctx),$C
	vmovdqu	0x60($ctx),$D
	vmovdqu	0x80($ctx),$E
	vmovdqu	0x60($Tbl),$tx			# pbswap_mask
	jmp	.Loop_avx

.align	32
.Loop_avx:
___
$code.="	vmovdqa	-0x20($Tbl),$K\n";	# K_00_19
for($i=0;$i<20;$i++)	{ &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
$code.="	vmovdqa	0x00($Tbl),$K\n";	# K_20_39
for(;$i<40;$i++)	{ &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
$code.="	vmovdqa	0x20($Tbl),$K\n";	# K_40_59
for(;$i<60;$i++)	{ &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
$code.="	vmovdqa	0x40($Tbl),$K\n";	# K_60_79
for(;$i<80;$i++)	{ &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	mov	\$1,%ecx
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	cmp	`4*$i`(%rbx),%ecx		# examine counters
	cmovge	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	vmovdqu	(%rbx),$t0			# pull counters
	vpxor	$t2,$t2,$t2
	vmovdqa	$t0,$t1
	vpcmpgtd $t2,$t1,$t1			# mask value
	vpaddd	$t1,$t0,$t0			# counters--

	vpand	$t1,$A,$A
	vpand	$t1,$B,$B
	vpaddd	0x00($ctx),$A,$A
	vpand	$t1,$C,$C
	vpaddd	0x20($ctx),$B,$B
	vpand	$t1,$D,$D
	vpaddd	0x40($ctx),$C,$C
	vpand	$t1,$E,$E
	vpaddd	0x60($ctx),$D,$D
	vpaddd	0x80($ctx),$E,$E
	vmovdqu	$A,0x00($ctx)
	vmovdqu	$B,0x20($ctx)
	vmovdqu	$C,0x40($ctx)
	vmovdqu	$D,0x60($ctx)
	vmovdqu	$E,0x80($ctx)

	vmovdqu	$t0,(%rbx)			# save counters
	vmovdqu	0x60($Tbl),$tx			# pbswap_mask
	dec	$num
	jnz	.Loop_avx

	mov	`$REG_SZ*17+8`(%rsp),$num
	lea	$REG_SZ($ctx),$ctx
	lea	`16*$REG_SZ/4`($inp),$inp
	dec	$num
	jnz	.Loop_grande_avx

.Ldone_avx:
	mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
.cfi_def_cfa	%rax,8
	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xb8(%rax),%xmm6
	movaps	-0xa8(%rax),%xmm7
	movaps	-0x98(%rax),%xmm8
	movaps	-0x88(%rax),%xmm9
	movaps	-0x78(%rax),%xmm10
	movaps	-0x68(%rax),%xmm11
	movaps	-0x58(%rax),%xmm12
	movaps	-0x48(%rax),%xmm13
	movaps	-0x38(%rax),%xmm14
	movaps	-0x28(%rax),%xmm15
___
$code.=<<___;
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue_avx:
	ret
.cfi_endproc
.size	sha1_multi_block_avx,.-sha1_multi_block_avx
___

						if ($avx>1) {
$code =~ s/\`([^\`]*)\`/eval $1/gem;

$REG_SZ=32;

@ptr=map("%r$_",(12..15,8..11));

@V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4));
($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9));
@Xi=map("%ymm$_",(10..14));
$K="%ymm15";

$code.=<<___;
.type	sha1_multi_block_avx2,\@function,3
.align	32
sha1_multi_block_avx2:
.cfi_startproc
_avx2_shortcut:
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,0x40(%rsp)
	movaps	%xmm11,0x50(%rsp)
	movaps	%xmm12,-0x78(%rax)
	movaps	%xmm13,-0x68(%rax)
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`, %rsp
	and	\$-256,%rsp
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.cfi_cfa_expression	%rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx2:
	lea	K_XX_XX(%rip),$Tbl
	shr	\$1,$num

	vzeroupper
.Loop_grande_avx2:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
	lea	`$REG_SZ*16`(%rsp),%rbx
___
for($i=0;$i<8;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	vmovdqu	0x00($ctx),$A			# load context
	 lea	128(%rsp),%rax
	vmovdqu	0x20($ctx),$B
	 lea	256+128(%rsp),%rbx
	vmovdqu	0x40($ctx),$C
	vmovdqu	0x60($ctx),$D
	vmovdqu	0x80($ctx),$E
	vmovdqu	0x60($Tbl),$tx			# pbswap_mask
	jmp	.Loop_avx2

.align	32
.Loop_avx2:
___
$code.="	vmovdqa	-0x20($Tbl),$K\n";	# K_00_19
for($i=0;$i<20;$i++)	{ &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
$code.="	vmovdqa	0x00($Tbl),$K\n";	# K_20_39
for(;$i<40;$i++)	{ &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
$code.="	vmovdqa	0x20($Tbl),$K\n";	# K_40_59
for(;$i<60;$i++)	{ &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
$code.="	vmovdqa	0x40($Tbl),$K\n";	# K_60_79
for(;$i<80;$i++)	{ &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	mov	\$1,%ecx
	lea	`$REG_SZ*16`(%rsp),%rbx
___
for($i=0;$i<8;$i++) {
    $code.=<<___;
	cmp	`4*$i`(%rbx),%ecx		# examine counters
	cmovge	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	vmovdqu	(%rbx),$t0		# pull counters
	vpxor	$t2,$t2,$t2
	vmovdqa	$t0,$t1
	vpcmpgtd $t2,$t1,$t1			# mask value
	vpaddd	$t1,$t0,$t0			# counters--

	vpand	$t1,$A,$A
	vpand	$t1,$B,$B
	vpaddd	0x00($ctx),$A,$A
	vpand	$t1,$C,$C
	vpaddd	0x20($ctx),$B,$B
	vpand	$t1,$D,$D
	vpaddd	0x40($ctx),$C,$C
	vpand	$t1,$E,$E
	vpaddd	0x60($ctx),$D,$D
	vpaddd	0x80($ctx),$E,$E
	vmovdqu	$A,0x00($ctx)
	vmovdqu	$B,0x20($ctx)
	vmovdqu	$C,0x40($ctx)
	vmovdqu	$D,0x60($ctx)
	vmovdqu	$E,0x80($ctx)

	vmovdqu	$t0,(%rbx)			# save counters
	lea	256+128(%rsp),%rbx
	vmovdqu	0x60($Tbl),$tx			# pbswap_mask
	dec	$num
	jnz	.Loop_avx2

	#mov	`$REG_SZ*17+8`(%rsp),$num
	#lea	$REG_SZ($ctx),$ctx
	#lea	`16*$REG_SZ/4`($inp),$inp
	#dec	$num
	#jnz	.Loop_grande_avx2

.Ldone_avx2:
	mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
.cfi_def_cfa	%rax,8
	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	movaps	-0x68(%rax),%xmm13
	movaps	-0x58(%rax),%xmm14
	movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue_avx2:
	ret
.cfi_endproc
.size	sha1_multi_block_avx2,.-sha1_multi_block_avx2
___
						}	}}}
$code.=<<___;

.align	256
	.long	0x5a827999,0x5a827999,0x5a827999,0x5a827999	# K_00_19
	.long	0x5a827999,0x5a827999,0x5a827999,0x5a827999	# K_00_19
K_XX_XX:
	.long	0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1	# K_20_39
	.long	0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1	# K_20_39
	.long	0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc	# K_40_59
	.long	0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc	# K_40_59
	.long	0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6	# K_60_79
	.long	0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6	# K_60_79
	.long	0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f	# pbswap
	.long	0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f	# pbswap
	.byte	0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
	.asciz	"SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
___

if ($win64) {
# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
$rec="%rcx";
$frame="%rdx";
$context="%r8";
$disp="%r9";

$code.=<<___;
.extern	__imp_RtlVirtualUnwind
.type	se_handler,\@abi-omnipotent
.align	16
se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# end of prologue label
	cmp	%r10,%rbx		# context->Rip<.Lbody
	jb	.Lin_prologue

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
	jae	.Lin_prologue

	mov	`16*17`(%rax),%rax	# pull saved stack pointer

	mov	-8(%rax),%rbx
	mov	-16(%rax),%rbp
	mov	%rbx,144($context)	# restore context->Rbx
	mov	%rbp,160($context)	# restore context->Rbp

	lea	-24-10*16(%rax),%rsi
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$20,%ecx
	.long	0xa548f3fc		# cld; rep movsq

.Lin_prologue:
	mov	8(%rax),%rdi
	mov	16(%rax),%rsi
	mov	%rax,152($context)	# restore context->Rsp
	mov	%rsi,168($context)	# restore context->Rsi
	mov	%rdi,176($context)	# restore context->Rdi

	mov	40($disp),%rdi		# disp->ContextRecord
	mov	$context,%rsi		# context
	mov	\$154,%ecx		# sizeof(CONTEXT)
	.long	0xa548f3fc		# cld; rep movsq

	mov	$disp,%rsi
	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
	mov	0(%rsi),%r8		# arg3, disp->ControlPc
	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
	mov	40(%rsi),%r10		# disp->ContextRecord
	lea	56(%rsi),%r11		# &disp->HandlerData
	lea	24(%rsi),%r12		# &disp->EstablisherFrame
	mov	%r10,32(%rsp)		# arg5
	mov	%r11,40(%rsp)		# arg6
	mov	%r12,48(%rsp)		# arg7
	mov	%rcx,56(%rsp)		# arg8, (NULL)
	call	*__imp_RtlVirtualUnwind(%rip)

	mov	\$1,%eax		# ExceptionContinueSearch
	add	\$64,%rsp
	popfq
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx
	pop	%rdi
	pop	%rsi
	ret
.size	se_handler,.-se_handler
___
$code.=<<___ if ($avx>1);
.type	avx2_handler,\@abi-omnipotent
.align	16
avx2_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# end of prologue label
	cmp	%r10,%rbx		# context->Rip<body label
	jb	.Lin_prologue

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=epilogue label
	jae	.Lin_prologue

	mov	`32*17`($context),%rax	# pull saved stack pointer

	mov	-8(%rax),%rbx
	mov	-16(%rax),%rbp
	mov	-24(%rax),%r12
	mov	-32(%rax),%r13
	mov	-40(%rax),%r14
	mov	-48(%rax),%r15
	mov	%rbx,144($context)	# restore context->Rbx
	mov	%rbp,160($context)	# restore context->Rbp
	mov	%r12,216($context)	# restore context->R12
	mov	%r13,224($context)	# restore context->R13
	mov	%r14,232($context)	# restore context->R14
	mov	%r15,240($context)	# restore context->R15

	lea	-56-10*16(%rax),%rsi
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$20,%ecx
	.long	0xa548f3fc		# cld; rep movsq

	jmp	.Lin_prologue
.size	avx2_handler,.-avx2_handler
___
$code.=<<___;
.section	.pdata
.align	4
	.rva	.LSEH_begin_sha1_multi_block
	.rva	.LSEH_end_sha1_multi_block
	.rva	.LSEH_info_sha1_multi_block
	.rva	.LSEH_begin_sha1_multi_block_shaext
	.rva	.LSEH_end_sha1_multi_block_shaext
	.rva	.LSEH_info_sha1_multi_block_shaext
___
$code.=<<___ if ($avx);
	.rva	.LSEH_begin_sha1_multi_block_avx
	.rva	.LSEH_end_sha1_multi_block_avx
	.rva	.LSEH_info_sha1_multi_block_avx
___
$code.=<<___ if ($avx>1);
	.rva	.LSEH_begin_sha1_multi_block_avx2
	.rva	.LSEH_end_sha1_multi_block_avx2
	.rva	.LSEH_info_sha1_multi_block_avx2
___
$code.=<<___;
.section	.xdata
.align	8
.LSEH_info_sha1_multi_block:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lbody,.Lepilogue			# HandlerData[]
.LSEH_info_sha1_multi_block_shaext:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lbody_shaext,.Lepilogue_shaext	# HandlerData[]
___
$code.=<<___ if ($avx);
.LSEH_info_sha1_multi_block_avx:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lbody_avx,.Lepilogue_avx		# HandlerData[]
___
$code.=<<___ if ($avx>1);
.LSEH_info_sha1_multi_block_avx2:
	.byte	9,0,0,0
	.rva	avx2_handler
	.rva	.Lbody_avx2,.Lepilogue_avx2		# HandlerData[]
___
}
####################################################################

sub rex {
  local *opcode=shift;
  my ($dst,$src)=@_;
  my $rex=0;

    $rex|=0x04			if ($dst>=8);
    $rex|=0x01			if ($src>=8);
    unshift @opcode,$rex|0x40	if ($rex);
}

sub sha1rnds4 {
    if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x0f,0x3a,0xcc);
	rex(\@opcode,$3,$2);
	push @opcode,0xc0|($2&7)|(($3&7)<<3);		# ModR/M
	my $c=$1;
	push @opcode,$c=~/^0/?oct($c):$c;
	return ".byte\t".join(',',@opcode);
    } else {
	return "sha1rnds4\t".@_[0];
    }
}

sub sha1op38 {
    my $instr = shift;
    my %opcodelet = (
		"sha1nexte" => 0xc8,
  		"sha1msg1"  => 0xc9,
		"sha1msg2"  => 0xca	);

    if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x0f,0x38);
	rex(\@opcode,$2,$1);
	push @opcode,$opcodelet{$instr};
	push @opcode,0xc0|($1&7)|(($2&7)<<3);		# ModR/M
	return ".byte\t".join(',',@opcode);
    } else {
	return $instr."\t".@_[0];
    }
}

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval($1)/ge;

	s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo		or
	s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo		or

	s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go		or
	s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go		or
	s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go	or
	s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go	or
	s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go		or
	s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# SHA1 block procedure for MIPS.

# Performance improvement is 30% on unaligned input. The "secret" is
# to deploy lwl/lwr pair to load unaligned input. One could have
# vectorized Xupdate on MIPSIII/IV, but the goal was to code MIPS32-
# compatible subroutine. There is room for minor optimization on
# little-endian platforms...

# September 2012.
#
# Add MIPS32r2 code (>25% less instructions).

######################################################################
# There is a number of MIPS ABI in use, O32 and N32/64 are most
# widely used. Then there is a new contender: NUBI. It appears that if
# one picks the latter, it's possible to arrange code in ABI neutral
# manner. Therefore let's stick to NUBI register layout:
#
($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25));
($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23));
($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31));
#
# The return value is placed in $a0. Following coding rules facilitate
# interoperability:
#
# - never ever touch $tp, "thread pointer", former $gp;
# - copy return value to $t0, former $v0 [or to $a0 if you're adapting
#   old code];
# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary;
#
# For reference here is register layout for N32/64 MIPS ABIs:
#
# ($zero,$at,$v0,$v1)=map("\$$_",(0..3));
# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25));
# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23));
# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31));
#
$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64

if ($flavour =~ /64|n32/i) {
	$PTR_ADD="daddu";	# incidentally works even on n32
	$PTR_SUB="dsubu";	# incidentally works even on n32
	$REG_S="sd";
	$REG_L="ld";
	$PTR_SLL="dsll";	# incidentally works even on n32
	$SZREG=8;
} else {
	$PTR_ADD="addu";
	$PTR_SUB="subu";
	$REG_S="sw";
	$REG_L="lw";
	$PTR_SLL="sll";
	$SZREG=4;
}
#
# <appro@openssl.org>
#
######################################################################

$big_endian=(`echo MIPSEB | $ENV{CC} -E -`=~/MIPSEB/)?0:1 if ($ENV{CC});

for (@ARGV) {	$output=$_ if (/\w[\w\-]*\.\w+$/);   }
open STDOUT,">$output";

if (!defined($big_endian))
            {   $big_endian=(unpack('L',pack('N',1))==1);   }

# offsets of the Most and Least Significant Bytes
$MSB=$big_endian?0:3;
$LSB=3&~$MSB;

@X=map("\$$_",(8..23));	# a4-a7,s0-s11

$ctx=$a0;
$inp=$a1;
$num=$a2;
$A="\$1";
$B="\$2";
$C="\$3";
$D="\$7";
$E="\$24";	@V=($A,$B,$C,$D,$E);
$t0="\$25";
$t1=$num;	# $num is offloaded to stack
$t2="\$30";	# fp
$K="\$31";	# ra

sub BODY_00_14 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
$code.=<<___	if (!$big_endian);
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
	wsbh	@X[$i],@X[$i]	# byte swap($i)
	rotr	@X[$i],@X[$i],16
#else
	srl	$t0,@X[$i],24	# byte swap($i)
	srl	$t1,@X[$i],8
	andi	$t2,@X[$i],0xFF00
	sll	@X[$i],@X[$i],24
	andi	$t1,0xFF00
	sll	$t2,$t2,8
	or	@X[$i],$t0
	or	$t1,$t2
	or	@X[$i],$t1
#endif
___
$code.=<<___;
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
	addu	$e,$K		# $i
	xor	$t0,$c,$d
	rotr	$t1,$a,27
	and	$t0,$b
	addu	$e,$t1
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
	 lw	@X[$j],$j*4($inp)
#else
	 lwl	@X[$j],$j*4+$MSB($inp)
	 lwr	@X[$j],$j*4+$LSB($inp)
#endif
	xor	$t0,$d
	addu	$e,@X[$i]
	rotr	$b,$b,2
	addu	$e,$t0
#else
	 lwl	@X[$j],$j*4+$MSB($inp)
	sll	$t0,$a,5	# $i
	addu	$e,$K
	 lwr	@X[$j],$j*4+$LSB($inp)
	srl	$t1,$a,27
	addu	$e,$t0
	xor	$t0,$c,$d
	addu	$e,$t1
	sll	$t2,$b,30
	and	$t0,$b
	srl	$b,$b,2
	xor	$t0,$d
	addu	$e,@X[$i]
	or	$b,$t2
	addu	$e,$t0
#endif
___
}

sub BODY_15_19 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;

$code.=<<___	if (!$big_endian && $i==15);
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
	wsbh	@X[$i],@X[$i]	# byte swap($i)
	rotr	@X[$i],@X[$i],16
#else
	srl	$t0,@X[$i],24	# byte swap($i)
	srl	$t1,@X[$i],8
	andi	$t2,@X[$i],0xFF00
	sll	@X[$i],@X[$i],24
	andi	$t1,0xFF00
	sll	$t2,$t2,8
	or	@X[$i],$t0
	or	@X[$i],$t1
	or	@X[$i],$t2
#endif
___
$code.=<<___;
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
	addu	$e,$K		# $i
	 xor	@X[$j%16],@X[($j+2)%16]
	xor	$t0,$c,$d
	rotr	$t1,$a,27
	 xor	@X[$j%16],@X[($j+8)%16]
	and	$t0,$b
	addu	$e,$t1
	 xor	@X[$j%16],@X[($j+13)%16]
	xor	$t0,$d
	addu	$e,@X[$i%16]
	 rotr	@X[$j%16],@X[$j%16],31
	rotr	$b,$b,2
	addu	$e,$t0
#else
	 xor	@X[$j%16],@X[($j+2)%16]
	sll	$t0,$a,5	# $i
	addu	$e,$K
	srl	$t1,$a,27
	addu	$e,$t0
	 xor	@X[$j%16],@X[($j+8)%16]
	xor	$t0,$c,$d
	addu	$e,$t1
	 xor	@X[$j%16],@X[($j+13)%16]
	sll	$t2,$b,30
	and	$t0,$b
	 srl	$t1,@X[$j%16],31
	 addu	@X[$j%16],@X[$j%16]
	srl	$b,$b,2
	xor	$t0,$d
	 or	@X[$j%16],$t1
	addu	$e,@X[$i%16]
	or	$b,$t2
	addu	$e,$t0
#endif
___
}

sub BODY_20_39 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
$code.=<<___ if ($i<79);
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
	 xor	@X[$j%16],@X[($j+2)%16]
	addu	$e,$K		# $i
	rotr	$t1,$a,27
	 xor	@X[$j%16],@X[($j+8)%16]
	xor	$t0,$c,$d
	addu	$e,$t1
	 xor	@X[$j%16],@X[($j+13)%16]
	xor	$t0,$b
	addu	$e,@X[$i%16]
	 rotr	@X[$j%16],@X[$j%16],31
	rotr	$b,$b,2
	addu	$e,$t0
#else
	 xor	@X[$j%16],@X[($j+2)%16]
	sll	$t0,$a,5	# $i
	addu	$e,$K
	srl	$t1,$a,27
	addu	$e,$t0
	 xor	@X[$j%16],@X[($j+8)%16]
	xor	$t0,$c,$d
	addu	$e,$t1
	 xor	@X[$j%16],@X[($j+13)%16]
	sll	$t2,$b,30
	xor	$t0,$b
	 srl	$t1,@X[$j%16],31
	 addu	@X[$j%16],@X[$j%16]
	srl	$b,$b,2
	addu	$e,@X[$i%16]
	 or	@X[$j%16],$t1
	or	$b,$t2
	addu	$e,$t0
#endif
___
$code.=<<___ if ($i==79);
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
	 lw	@X[0],0($ctx)
	addu	$e,$K		# $i
	 lw	@X[1],4($ctx)
	rotr	$t1,$a,27
	 lw	@X[2],8($ctx)
	xor	$t0,$c,$d
	addu	$e,$t1
	 lw	@X[3],12($ctx)
	xor	$t0,$b
	addu	$e,@X[$i%16]
	 lw	@X[4],16($ctx)
	rotr	$b,$b,2
	addu	$e,$t0
#else
	 lw	@X[0],0($ctx)
	sll	$t0,$a,5	# $i
	addu	$e,$K
	 lw	@X[1],4($ctx)
	srl	$t1,$a,27
	addu	$e,$t0
	 lw	@X[2],8($ctx)
	xor	$t0,$c,$d
	addu	$e,$t1
	 lw	@X[3],12($ctx)
	sll	$t2,$b,30
	xor	$t0,$b
	 lw	@X[4],16($ctx)
	srl	$b,$b,2
	addu	$e,@X[$i%16]
	or	$b,$t2
	addu	$e,$t0
#endif
___
}

sub BODY_40_59 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
$code.=<<___ if ($i<79);
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
	addu	$e,$K		# $i
	and	$t0,$c,$d
	 xor	@X[$j%16],@X[($j+2)%16]
	rotr	$t1,$a,27
	addu	$e,$t0
	 xor	@X[$j%16],@X[($j+8)%16]
	xor	$t0,$c,$d
	addu	$e,$t1
	 xor	@X[$j%16],@X[($j+13)%16]
	and	$t0,$b
	addu	$e,@X[$i%16]
	 rotr	@X[$j%16],@X[$j%16],31
	rotr	$b,$b,2
	addu	$e,$t0
#else
	 xor	@X[$j%16],@X[($j+2)%16]
	sll	$t0,$a,5	# $i
	addu	$e,$K
	srl	$t1,$a,27
	addu	$e,$t0
	 xor	@X[$j%16],@X[($j+8)%16]
	and	$t0,$c,$d
	addu	$e,$t1
	 xor	@X[$j%16],@X[($j+13)%16]
	sll	$t2,$b,30
	addu	$e,$t0
	 srl	$t1,@X[$j%16],31
	xor	$t0,$c,$d
	 addu	@X[$j%16],@X[$j%16]
	and	$t0,$b
	srl	$b,$b,2
	 or	@X[$j%16],$t1
	addu	$e,@X[$i%16]
	or	$b,$t2
	addu	$e,$t0
#endif
___
}

$FRAMESIZE=16;	# large enough to accommodate NUBI saved registers
$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0xc0fff008" : "0xc0ff0000";

$code=<<___;
#include "mips_arch.h"

.text

.set	noat
.set	noreorder
.align	5
.globl	sha1_block_data_order
.ent	sha1_block_data_order
sha1_block_data_order:
	.frame	$sp,$FRAMESIZE*$SZREG,$ra
	.mask	$SAVED_REGS_MASK,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,$FRAMESIZE*$SZREG
	$REG_S	$ra,($FRAMESIZE-1)*$SZREG($sp)
	$REG_S	$fp,($FRAMESIZE-2)*$SZREG($sp)
	$REG_S	$s11,($FRAMESIZE-3)*$SZREG($sp)
	$REG_S	$s10,($FRAMESIZE-4)*$SZREG($sp)
	$REG_S	$s9,($FRAMESIZE-5)*$SZREG($sp)
	$REG_S	$s8,($FRAMESIZE-6)*$SZREG($sp)
	$REG_S	$s7,($FRAMESIZE-7)*$SZREG($sp)
	$REG_S	$s6,($FRAMESIZE-8)*$SZREG($sp)
	$REG_S	$s5,($FRAMESIZE-9)*$SZREG($sp)
	$REG_S	$s4,($FRAMESIZE-10)*$SZREG($sp)
___
$code.=<<___ if ($flavour =~ /nubi/i);	# optimize non-nubi prologue
	$REG_S	$s3,($FRAMESIZE-11)*$SZREG($sp)
	$REG_S	$s2,($FRAMESIZE-12)*$SZREG($sp)
	$REG_S	$s1,($FRAMESIZE-13)*$SZREG($sp)
	$REG_S	$s0,($FRAMESIZE-14)*$SZREG($sp)
	$REG_S	$gp,($FRAMESIZE-15)*$SZREG($sp)
___
$code.=<<___;
	$PTR_SLL $num,6
	$PTR_ADD $num,$inp
	$REG_S	$num,0($sp)
	lw	$A,0($ctx)
	lw	$B,4($ctx)
	lw	$C,8($ctx)
	lw	$D,12($ctx)
	b	.Loop
	lw	$E,16($ctx)
.align	4
.Loop:
	.set	reorder
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
	lui	$K,0x5a82
	lw	@X[0],($inp)
	ori	$K,0x7999	# K_00_19
#else
	lwl	@X[0],$MSB($inp)
	lui	$K,0x5a82
	lwr	@X[0],$LSB($inp)
	ori	$K,0x7999	# K_00_19
#endif
___
for ($i=0;$i<15;$i++)	{ &BODY_00_14($i,@V); unshift(@V,pop(@V)); }
for (;$i<20;$i++)	{ &BODY_15_19($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	lui	$K,0x6ed9
	ori	$K,0xeba1	# K_20_39
___
for (;$i<40;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	lui	$K,0x8f1b
	ori	$K,0xbcdc	# K_40_59
___
for (;$i<60;$i++)	{ &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	lui	$K,0xca62
	ori	$K,0xc1d6	# K_60_79
___
for (;$i<80;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	$PTR_ADD $inp,64
	$REG_L	$num,0($sp)

	addu	$A,$X[0]
	addu	$B,$X[1]
	sw	$A,0($ctx)
	addu	$C,$X[2]
	addu	$D,$X[3]
	sw	$B,4($ctx)
	addu	$E,$X[4]
	sw	$C,8($ctx)
	sw	$D,12($ctx)
	sw	$E,16($ctx)
	.set	noreorder
	bne	$inp,$num,.Loop
	nop

	.set	noreorder
	$REG_L	$ra,($FRAMESIZE-1)*$SZREG($sp)
	$REG_L	$fp,($FRAMESIZE-2)*$SZREG($sp)
	$REG_L	$s11,($FRAMESIZE-3)*$SZREG($sp)
	$REG_L	$s10,($FRAMESIZE-4)*$SZREG($sp)
	$REG_L	$s9,($FRAMESIZE-5)*$SZREG($sp)
	$REG_L	$s8,($FRAMESIZE-6)*$SZREG($sp)
	$REG_L	$s7,($FRAMESIZE-7)*$SZREG($sp)
	$REG_L	$s6,($FRAMESIZE-8)*$SZREG($sp)
	$REG_L	$s5,($FRAMESIZE-9)*$SZREG($sp)
	$REG_L	$s4,($FRAMESIZE-10)*$SZREG($sp)
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$s3,($FRAMESIZE-11)*$SZREG($sp)
	$REG_L	$s2,($FRAMESIZE-12)*$SZREG($sp)
	$REG_L	$s1,($FRAMESIZE-13)*$SZREG($sp)
	$REG_L	$s0,($FRAMESIZE-14)*$SZREG($sp)
	$REG_L	$gp,($FRAMESIZE-15)*$SZREG($sp)
___
$code.=<<___;
	jr	$ra
	$PTR_ADD $sp,$FRAMESIZE*$SZREG
.end	sha1_block_data_order
.rdata
.asciiz	"SHA1 for MIPS, CRYPTOGAMS by <appro\@openssl.org>"
___
print $code;
close STDOUT or die "error closing STDOUT: $!";