#! /usr/bin/env perl
# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
#
# Permission to use under GPL terms is granted.
# ====================================================================

# SHA256 block procedure for ARMv4. May 2007.

# Performance is ~2x better than gcc 3.4 generated code and in "abso-
# lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
# byte [on single-issue Xscale PXA250 core].

# July 2010.
#
# Rescheduling for dual-issue pipeline resulted in 22% improvement on
# Cortex A8 core and ~20 cycles per processed byte.

# February 2011.
#
# Profiler-assisted and platform-specific optimization resulted in 16%
# improvement on Cortex A8 core and ~15.4 cycles per processed byte.

# September 2013.
#
# Add NEON implementation. On Cortex A8 it was measured to process one
# byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
# S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
# code (meaning that latter performs sub-optimally, nothing was done
# about it).

# May 2014.
#
# Add ARMv8 code path performing at 2.0 cpb on Apple A7.

$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
    open STDOUT,">$output";
}

$ctx="r0";	$t0="r0";
$inp="r1";	$t4="r1";
$len="r2";	$t1="r2";
$T1="r3";	$t3="r3";
$A="r4";
$B="r5";
$C="r6";
$D="r7";
$E="r8";
$F="r9";
$G="r10";
$H="r11";
@V=($A,$B,$C,$D,$E,$F,$G,$H);
$t2="r12";
$Ktbl="r14";

@Sigma0=( 2,13,22);
@Sigma1=( 6,11,25);
@sigma0=( 7,18, 3);
@sigma1=(17,19,10);

sub BODY_00_15 {
my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;

$code.=<<___ if ($i<16);
#if __ARM_ARCH__>=7
	@ ldr	$t1,[$inp],#4			@ $i
# if $i==15
	str	$inp,[sp,#17*4]			@ make room for $t4
# endif
	eor	$t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
	add	$a,$a,$t2			@ h+=Maj(a,b,c) from the past
	eor	$t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`	@ Sigma1(e)
# ifndef __ARMEB__
	rev	$t1,$t1
# endif
#else
	@ ldrb	$t1,[$inp,#3]			@ $i
	add	$a,$a,$t2			@ h+=Maj(a,b,c) from the past
	ldrb	$t2,[$inp,#2]
	ldrb	$t0,[$inp,#1]
	orr	$t1,$t1,$t2,lsl#8
	ldrb	$t2,[$inp],#4
	orr	$t1,$t1,$t0,lsl#16
# if $i==15
	str	$inp,[sp,#17*4]			@ make room for $t4
# endif
	eor	$t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
	orr	$t1,$t1,$t2,lsl#24
	eor	$t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`	@ Sigma1(e)
#endif
___
$code.=<<___;
	ldr	$t2,[$Ktbl],#4			@ *K256++
	add	$h,$h,$t1			@ h+=X[i]
	str	$t1,[sp,#`$i%16`*4]
	eor	$t1,$f,$g
	add	$h,$h,$t0,ror#$Sigma1[0]	@ h+=Sigma1(e)
	and	$t1,$t1,$e
	add	$h,$h,$t2			@ h+=K256[i]
	eor	$t1,$t1,$g			@ Ch(e,f,g)
	eor	$t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
	add	$h,$h,$t1			@ h+=Ch(e,f,g)
#if $i==31
	and	$t2,$t2,#0xff
	cmp	$t2,#0xf2			@ done?
#endif
#if $i<15
# if __ARM_ARCH__>=7
	ldr	$t1,[$inp],#4			@ prefetch
# else
	ldrb	$t1,[$inp,#3]
# endif
	eor	$t2,$a,$b			@ a^b, b^c in next round
#else
	ldr	$t1,[sp,#`($i+2)%16`*4]		@ from future BODY_16_xx
	eor	$t2,$a,$b			@ a^b, b^c in next round
	ldr	$t4,[sp,#`($i+15)%16`*4]	@ from future BODY_16_xx
#endif
	eor	$t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]`	@ Sigma0(a)
	and	$t3,$t3,$t2			@ (b^c)&=(a^b)
	add	$d,$d,$h			@ d+=h
	eor	$t3,$t3,$b			@ Maj(a,b,c)
	add	$h,$h,$t0,ror#$Sigma0[0]	@ h+=Sigma0(a)
	@ add	$h,$h,$t3			@ h+=Maj(a,b,c)
___
	($t2,$t3)=($t3,$t2);
}

sub BODY_16_XX {
my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;

$code.=<<___;
	@ ldr	$t1,[sp,#`($i+1)%16`*4]		@ $i
	@ ldr	$t4,[sp,#`($i+14)%16`*4]
	mov	$t0,$t1,ror#$sigma0[0]
	add	$a,$a,$t2			@ h+=Maj(a,b,c) from the past
	mov	$t2,$t4,ror#$sigma1[0]
	eor	$t0,$t0,$t1,ror#$sigma0[1]
	eor	$t2,$t2,$t4,ror#$sigma1[1]
	eor	$t0,$t0,$t1,lsr#$sigma0[2]	@ sigma0(X[i+1])
	ldr	$t1,[sp,#`($i+0)%16`*4]
	eor	$t2,$t2,$t4,lsr#$sigma1[2]	@ sigma1(X[i+14])
	ldr	$t4,[sp,#`($i+9)%16`*4]

	add	$t2,$t2,$t0
	eor	$t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`	@ from BODY_00_15
	add	$t1,$t1,$t2
	eor	$t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`	@ Sigma1(e)
	add	$t1,$t1,$t4			@ X[i]
___
	&BODY_00_15(@_);
}

$code=<<___;
#ifndef __KERNEL__
# include "arm_arch.h"
#else
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
# define __ARM_MAX_ARCH__ 7
#endif

.text
#if defined(__thumb2__)
.syntax unified
.thumb
#else
.code   32
#endif

.type	K256,%object
.align	5
K256:
.word	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
.word	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
.word	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
.word	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
.word	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
.word	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
.word	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
.word	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
.word	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
.word	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
.word	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
.word	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
.word	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
.word	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
.word	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
.word	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
.size	K256,.-K256
.word	0				@ terminator
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
.LOPENSSL_armcap:
.word	OPENSSL_armcap_P-.Lsha256_block_data_order
#endif
.align	5

.global	sha256_block_data_order
.type	sha256_block_data_order,%function
sha256_block_data_order:
.Lsha256_block_data_order:
#if __ARM_ARCH__<7 && !defined(__thumb2__)
	sub	r3,pc,#8		@ sha256_block_data_order
#else
	adr	r3,.Lsha256_block_data_order
#endif
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
	ldr	r12,.LOPENSSL_armcap
	ldr	r12,[r3,r12]		@ OPENSSL_armcap_P
#ifdef	__APPLE__
	ldr	r12,[r12]
#endif
	tst	r12,#ARMV8_SHA256
	bne	.LARMv8
	tst	r12,#ARMV7_NEON
	bne	.LNEON
#endif
	add	$len,$inp,$len,lsl#6	@ len to point at the end of inp
	stmdb	sp!,{$ctx,$inp,$len,r4-r11,lr}
	ldmia	$ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
	sub	$Ktbl,r3,#256+32	@ K256
	sub	sp,sp,#16*4		@ alloca(X[16])
.Loop:
# if __ARM_ARCH__>=7
	ldr	$t1,[$inp],#4
# else
	ldrb	$t1,[$inp,#3]
# endif
	eor	$t3,$B,$C		@ magic
	eor	$t2,$t2,$t2
___
for($i=0;$i<16;$i++)	{ &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
$code.=".Lrounds_16_xx:\n";
for (;$i<32;$i++)	{ &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
#ifdef	__thumb2__
	ite	eq			@ Thumb2 thing, sanity check in ARM
#endif
	ldreq	$t3,[sp,#16*4]		@ pull ctx
	bne	.Lrounds_16_xx

	add	$A,$A,$t2		@ h+=Maj(a,b,c) from the past
	ldr	$t0,[$t3,#0]
	ldr	$t1,[$t3,#4]
	ldr	$t2,[$t3,#8]
	add	$A,$A,$t0
	ldr	$t0,[$t3,#12]
	add	$B,$B,$t1
	ldr	$t1,[$t3,#16]
	add	$C,$C,$t2
	ldr	$t2,[$t3,#20]
	add	$D,$D,$t0
	ldr	$t0,[$t3,#24]
	add	$E,$E,$t1
	ldr	$t1,[$t3,#28]
	add	$F,$F,$t2
	ldr	$inp,[sp,#17*4]		@ pull inp
	ldr	$t2,[sp,#18*4]		@ pull inp+len
	add	$G,$G,$t0
	add	$H,$H,$t1
	stmia	$t3,{$A,$B,$C,$D,$E,$F,$G,$H}
	cmp	$inp,$t2
	sub	$Ktbl,$Ktbl,#256	@ rewind Ktbl
	bne	.Loop

	add	sp,sp,#`16+3`*4	@ destroy frame
#if __ARM_ARCH__>=5
	ldmia	sp!,{r4-r11,pc}
#else
	ldmia	sp!,{r4-r11,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	sha256_block_data_order,.-sha256_block_data_order
___
######################################################################
# NEON stuff
#
{{{
my @X=map("q$_",(0..3));
my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
my $Xfer=$t4;
my $j=0;

sub Dlo()   { shift=~m|q([1]?[0-9])|?"d".($1*2):"";     }
sub Dhi()   { shift=~m|q([1]?[0-9])|?"d".($1*2+1):"";   }

sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
  my $arg = pop;
    $arg = "#$arg" if ($arg*1 eq $arg);
    $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
}

sub Xupdate()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e,$f,$g,$h);

	&vext_8		($T0,@X[0],@X[1],4);	# X[1..4]
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vext_8		($T1,@X[2],@X[3],4);	# X[9..12]
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	($T2,$T0,$sigma0[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(@X[0],@X[0],$T1);	# X[0..3] += X[9..12]
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	($T1,$T0,$sigma0[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vsli_32	($T2,$T0,32-$sigma0[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vshr_u32	($T3,$T0,$sigma0[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		($T1,$T1,$T2);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vsli_32	($T3,$T0,32-$sigma0[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dhi(@X[3]),$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&veor		($T1,$T1,$T3);		# sigma0(X[1..4])
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dhi(@X[3]),32-$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T5,&Dhi(@X[3]),$sigma1[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(@X[0],@X[0],$T1);	# X[0..3] += sigma0(X[1..4])
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dhi(@X[3]),$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dhi(@X[3]),32-$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);		# sigma1(X[14..15])
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dlo(@X[0]),$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dlo(@X[0]),32-$sigma1[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T5,&Dlo(@X[0]),$sigma1[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vshr_u32	($T4,&Dlo(@X[0]),$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vld1_32	("{$T0}","[$Ktbl,:128]!");
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vsli_32	($T4,&Dlo(@X[0]),32-$sigma1[1]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &veor		($T5,$T5,$T4);		# sigma1(X[16..17])
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	(&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	($T0,$T0,@X[0]);
	 while($#insns>=2) { eval(shift(@insns)); }
	&vst1_32	("{$T0}","[$Xfer,:128]!");
	 eval(shift(@insns));
	 eval(shift(@insns));

	push(@X,shift(@X));		# "rotate" X[]
}

sub Xpreload()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);
  my ($a,$b,$c,$d,$e,$f,$g,$h);

	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vld1_32	("{$T0}","[$Ktbl,:128]!");
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vrev32_8	(@X[0],@X[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vadd_i32	($T0,$T0,@X[0]);
	 foreach (@insns) { eval; }	# remaining instructions
	&vst1_32	("{$T0}","[$Xfer,:128]!");

	push(@X,shift(@X));		# "rotate" X[]
}

sub body_00_15 () {
	(
	'($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
	'&add	($h,$h,$t1)',			# h+=X[i]+K[i]
	'&eor	($t1,$f,$g)',
	'&eor	($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
	'&add	($a,$a,$t2)',			# h+=Maj(a,b,c) from the past
	'&and	($t1,$t1,$e)',
	'&eor	($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',	# Sigma1(e)
	'&eor	($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
	'&eor	($t1,$t1,$g)',			# Ch(e,f,g)
	'&add	($h,$h,$t2,"ror#$Sigma1[0]")',	# h+=Sigma1(e)
	'&eor	($t2,$a,$b)',			# a^b, b^c in next round
	'&eor	($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',	# Sigma0(a)
	'&add	($h,$h,$t1)',			# h+=Ch(e,f,g)
	'&ldr	($t1,sprintf "[sp,#%d]",4*(($j+1)&15))	if (($j&15)!=15);'.
	'&ldr	($t1,"[$Ktbl]")				if ($j==15);'.
	'&ldr	($t1,"[sp,#64]")			if ($j==31)',
	'&and	($t3,$t3,$t2)',			# (b^c)&=(a^b)
	'&add	($d,$d,$h)',			# d+=h
	'&add	($h,$h,$t0,"ror#$Sigma0[0]");'.	# h+=Sigma0(a)
	'&eor	($t3,$t3,$b)',			# Maj(a,b,c)
	'$j++;	unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
	)
}

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.arch	armv7-a
.fpu	neon

.global	sha256_block_data_order_neon
.type	sha256_block_data_order_neon,%function
.align	5
.skip	16
sha256_block_data_order_neon:
.LNEON:
	stmdb	sp!,{r4-r12,lr}

	sub	$H,sp,#16*4+16
	adr	$Ktbl,K256
	bic	$H,$H,#15		@ align for 128-bit stores
	mov	$t2,sp
	mov	sp,$H			@ alloca
	add	$len,$inp,$len,lsl#6	@ len to point at the end of inp

	vld1.8		{@X[0]},[$inp]!
	vld1.8		{@X[1]},[$inp]!
	vld1.8		{@X[2]},[$inp]!
	vld1.8		{@X[3]},[$inp]!
	vld1.32		{$T0},[$Ktbl,:128]!
	vld1.32		{$T1},[$Ktbl,:128]!
	vld1.32		{$T2},[$Ktbl,:128]!
	vld1.32		{$T3},[$Ktbl,:128]!
	vrev32.8	@X[0],@X[0]		@ yes, even on
	str		$ctx,[sp,#64]
	vrev32.8	@X[1],@X[1]		@ big-endian
	str		$inp,[sp,#68]
	mov		$Xfer,sp
	vrev32.8	@X[2],@X[2]
	str		$len,[sp,#72]
	vrev32.8	@X[3],@X[3]
	str		$t2,[sp,#76]		@ save original sp
	vadd.i32	$T0,$T0,@X[0]
	vadd.i32	$T1,$T1,@X[1]
	vst1.32		{$T0},[$Xfer,:128]!
	vadd.i32	$T2,$T2,@X[2]
	vst1.32		{$T1},[$Xfer,:128]!
	vadd.i32	$T3,$T3,@X[3]
	vst1.32		{$T2},[$Xfer,:128]!
	vst1.32		{$T3},[$Xfer,:128]!

	ldmia		$ctx,{$A-$H}
	sub		$Xfer,$Xfer,#64
	ldr		$t1,[sp,#0]
	eor		$t2,$t2,$t2
	eor		$t3,$B,$C
	b		.L_00_48

.align	4
.L_00_48:
___
	&Xupdate(\&body_00_15);
	&Xupdate(\&body_00_15);
	&Xupdate(\&body_00_15);
	&Xupdate(\&body_00_15);
$code.=<<___;
	teq	$t1,#0				@ check for K256 terminator
	ldr	$t1,[sp,#0]
	sub	$Xfer,$Xfer,#64
	bne	.L_00_48

	ldr		$inp,[sp,#68]
	ldr		$t0,[sp,#72]
	sub		$Ktbl,$Ktbl,#256	@ rewind $Ktbl
	teq		$inp,$t0
	it		eq
	subeq		$inp,$inp,#64		@ avoid SEGV
	vld1.8		{@X[0]},[$inp]!		@ load next input block
	vld1.8		{@X[1]},[$inp]!
	vld1.8		{@X[2]},[$inp]!
	vld1.8		{@X[3]},[$inp]!
	it		ne
	strne		$inp,[sp,#68]
	mov		$Xfer,sp
___
	&Xpreload(\&body_00_15);
	&Xpreload(\&body_00_15);
	&Xpreload(\&body_00_15);
	&Xpreload(\&body_00_15);
$code.=<<___;
	ldr	$t0,[$t1,#0]
	add	$A,$A,$t2			@ h+=Maj(a,b,c) from the past
	ldr	$t2,[$t1,#4]
	ldr	$t3,[$t1,#8]
	ldr	$t4,[$t1,#12]
	add	$A,$A,$t0			@ accumulate
	ldr	$t0,[$t1,#16]
	add	$B,$B,$t2
	ldr	$t2,[$t1,#20]
	add	$C,$C,$t3
	ldr	$t3,[$t1,#24]
	add	$D,$D,$t4
	ldr	$t4,[$t1,#28]
	add	$E,$E,$t0
	str	$A,[$t1],#4
	add	$F,$F,$t2
	str	$B,[$t1],#4
	add	$G,$G,$t3
	str	$C,[$t1],#4
	add	$H,$H,$t4
	str	$D,[$t1],#4
	stmia	$t1,{$E-$H}

	ittte	ne
	movne	$Xfer,sp
	ldrne	$t1,[sp,#0]
	eorne	$t2,$t2,$t2
	ldreq	sp,[sp,#76]			@ restore original sp
	itt	ne
	eorne	$t3,$B,$C
	bne	.L_00_48

	ldmia	sp!,{r4-r12,pc}
.size	sha256_block_data_order_neon,.-sha256_block_data_order_neon
#endif
___
}}}
######################################################################
# ARMv8 stuff
#
{{{
my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
my @MSG=map("q$_",(8..11));
my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
my $Ktbl="r3";

$code.=<<___;
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)

# if defined(__thumb2__)
#  define INST(a,b,c,d)	.byte	c,d|0xc,a,b
# else
#  define INST(a,b,c,d)	.byte	a,b,c,d
# endif

.type	sha256_block_data_order_armv8,%function
.align	5
sha256_block_data_order_armv8:
.LARMv8:
	vld1.32	{$ABCD,$EFGH},[$ctx]
	sub	$Ktbl,$Ktbl,#256+32
	add	$len,$inp,$len,lsl#6	@ len to point at the end of inp
	b	.Loop_v8

.align	4
.Loop_v8:
	vld1.8		{@MSG[0]-@MSG[1]},[$inp]!
	vld1.8		{@MSG[2]-@MSG[3]},[$inp]!
	vld1.32		{$W0},[$Ktbl]!
	vrev32.8	@MSG[0],@MSG[0]
	vrev32.8	@MSG[1],@MSG[1]
	vrev32.8	@MSG[2],@MSG[2]
	vrev32.8	@MSG[3],@MSG[3]
	vmov		$ABCD_SAVE,$ABCD	@ offload
	vmov		$EFGH_SAVE,$EFGH
	teq		$inp,$len
___
for($i=0;$i<12;$i++) {
$code.=<<___;
	vld1.32		{$W1},[$Ktbl]!
	vadd.i32	$W0,$W0,@MSG[0]
	sha256su0	@MSG[0],@MSG[1]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W0
	sha256h2	$EFGH,$abcd,$W0
	sha256su1	@MSG[0],@MSG[2],@MSG[3]
___
	($W0,$W1)=($W1,$W0);	push(@MSG,shift(@MSG));
}
$code.=<<___;
	vld1.32		{$W1},[$Ktbl]!
	vadd.i32	$W0,$W0,@MSG[0]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W0
	sha256h2	$EFGH,$abcd,$W0

	vld1.32		{$W0},[$Ktbl]!
	vadd.i32	$W1,$W1,@MSG[1]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W1
	sha256h2	$EFGH,$abcd,$W1

	vld1.32		{$W1},[$Ktbl]
	vadd.i32	$W0,$W0,@MSG[2]
	sub		$Ktbl,$Ktbl,#256-16	@ rewind
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W0
	sha256h2	$EFGH,$abcd,$W0

	vadd.i32	$W1,$W1,@MSG[3]
	vmov		$abcd,$ABCD
	sha256h		$ABCD,$EFGH,$W1
	sha256h2	$EFGH,$abcd,$W1

	vadd.i32	$ABCD,$ABCD,$ABCD_SAVE
	vadd.i32	$EFGH,$EFGH,$EFGH_SAVE
	it		ne
	bne		.Loop_v8

	vst1.32		{$ABCD,$EFGH},[$ctx]

	ret		@ bx lr
.size	sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
#endif
___
}}}
$code.=<<___;
.asciz  "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
.comm   OPENSSL_armcap_P,4,4
#endif
___

open SELF,$0;
while(<SELF>) {
	next if (/^#!/);
	last if (!s/^#/@/ and !/^$/);
	print;
}
close SELF;

{   my  %opcode = (
	"sha256h"	=> 0xf3000c40,	"sha256h2"	=> 0xf3100c40,
	"sha256su0"	=> 0xf3ba03c0,	"sha256su1"	=> 0xf3200c40	);

    sub unsha256 {
	my ($mnemonic,$arg)=@_;

	if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
	    my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
					 |(($2&7)<<17)|(($2&8)<<4)
					 |(($3&7)<<1) |(($3&8)<<2);
	    # since ARMv7 instructions are always encoded little-endian.
	    # correct solution is to use .inst directive, but older
	    # assemblers don't implement it:-(
	    sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
			$word&0xff,($word>>8)&0xff,
			($word>>16)&0xff,($word>>24)&0xff,
			$mnemonic,$arg;
	}
    }
}

foreach (split($/,$code)) {

	s/\`([^\`]*)\`/eval $1/geo;

	s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;

	s/\bret\b/bx	lr/go		or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;	# make it possible to compile with -march=armv4

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#! /usr/bin/env perl
# Copyright 2012-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# SHA256 for C64x+.
#
# January 2012
#
# Performance is just below 10 cycles per processed byte, which is
# almost 40% faster than compiler-generated code. Unroll is unlikely
# to give more than ~8% improvement...
#
# !!! Note that this module uses AMR, which means that all interrupt
# service routines are expected to preserve it and for own well-being
# zero it upon entry.

while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

($CTXA,$INP,$NUM) = ("A4","B4","A6");            # arguments
 $K256="A3";

($A,$Actx,$B,$Bctx,$C,$Cctx,$D,$Dctx,$T2,$S0,$s1,$t0a,$t1a,$t2a,$X9,$X14)
	=map("A$_",(16..31));
($E,$Ectx,$F,$Fctx,$G,$Gctx,$H,$Hctx,$T1,$S1,$s0,$t0e,$t1e,$t2e,$X1,$X15)
	=map("B$_",(16..31));

($Xia,$Xib)=("A5","B5");			# circular/ring buffer
 $CTXB=$t2e;

($Xn,$X0,$K)=("B7","B8","B9");
($Maj,$Ch)=($T2,"B6");

$code.=<<___;
	.text

	.if	.ASSEMBLER_VERSION<7000000
	.asg	0,__TI_EABI__
	.endif
	.if	__TI_EABI__
	.nocmp
	.asg	sha256_block_data_order,_sha256_block_data_order
	.endif

	.asg	B3,RA
	.asg	A15,FP
	.asg	B15,SP

	.if	.BIG_ENDIAN
	.asg	SWAP2,MV
	.asg	SWAP4,MV
	.endif

	.global	_sha256_block_data_order
_sha256_block_data_order:
__sha256_block:
	.asmfunc stack_usage(64)
	MV	$NUM,A0				; reassign $NUM
||	MVK	-64,B0
  [!A0]	BNOP	RA				; if ($NUM==0) return;
|| [A0]	STW	FP,*SP--[16]			; save frame pointer and alloca(64)
|| [A0]	MV	SP,FP
   [A0]	ADDKPC	__sha256_block,B2
|| [A0]	AND	B0,SP,SP			; align stack at 64 bytes
	.if	__TI_EABI__
   [A0]	MVK	0x00404,B1
|| [A0]	MVKL	\$PCR_OFFSET(K256,__sha256_block),$K256
   [A0]	MVKH	0x50000,B1
|| [A0]	MVKH	\$PCR_OFFSET(K256,__sha256_block),$K256
	.else
   [A0]	MVK	0x00404,B1
|| [A0]	MVKL	(K256-__sha256_block),$K256
   [A0]	MVKH	0x50000,B1
|| [A0]	MVKH	(K256-__sha256_block),$K256
	.endif
   [A0]	MVC	B1,AMR				; setup circular addressing
|| [A0]	MV	SP,$Xia
   [A0]	MV	SP,$Xib
|| [A0]	ADD	B2,$K256,$K256
|| [A0]	MV	$CTXA,$CTXB
|| [A0]	SUBAW	SP,2,SP				; reserve two words above buffer
	LDW	*${CTXA}[0],$A			; load ctx
||	LDW	*${CTXB}[4],$E
	LDW	*${CTXA}[1],$B
||	LDW	*${CTXB}[5],$F
	LDW	*${CTXA}[2],$C
||	LDW	*${CTXB}[6],$G
	LDW	*${CTXA}[3],$D
||	LDW	*${CTXB}[7],$H

	LDNW	*$INP++,$Xn			; pre-fetch input
	LDW	*$K256++,$K			; pre-fetch K256[0]
	MVK	14,B0				; loop counters
	MVK	47,B1
||	ADDAW	$Xia,9,$Xia
outerloop?:
	SUB	A0,1,A0
||	MV	$A,$Actx
||	MV	$E,$Ectx
||	MVD	$B,$Bctx
||	MVD	$F,$Fctx
	MV	$C,$Cctx
||	MV	$G,$Gctx
||	MVD	$D,$Dctx
||	MVD	$H,$Hctx
||	SWAP4	$Xn,$X0

	SPLOOPD	8				; BODY_00_14
||	MVC	B0,ILC
||	SWAP2	$X0,$X0

	LDNW	*$INP++,$Xn
||	ROTL	$A,30,$S0
||	OR	$A,$B,$Maj
||	AND	$A,$B,$t2a
||	ROTL	$E,26,$S1
||	AND	$F,$E,$Ch
||	ANDN	$G,$E,$t2e
	ROTL	$A,19,$t0a
||	AND	$C,$Maj,$Maj
||	ROTL	$E,21,$t0e
||	XOR	$t2e,$Ch,$Ch			; Ch(e,f,g) = (e&f)^(~e&g)
	ROTL	$A,10,$t1a
||	OR	$t2a,$Maj,$Maj			; Maj(a,b,c) = ((a|b)&c)|(a&b)
||	ROTL	$E,7,$t1e
||	ADD	$K,$H,$T1			; T1 = h + K256[i]
	ADD	$X0,$T1,$T1			; T1 += X[i];
||	STW	$X0,*$Xib++
||	XOR	$t0a,$S0,$S0
||	XOR	$t0e,$S1,$S1
	XOR	$t1a,$S0,$S0			; Sigma0(a)
||	XOR	$t1e,$S1,$S1			; Sigma1(e)
||	LDW	*$K256++,$K			; pre-fetch K256[i+1]
||	ADD	$Ch,$T1,$T1			; T1 += Ch(e,f,g)
	ADD	$S1,$T1,$T1			; T1 += Sigma1(e)
||	ADD	$S0,$Maj,$T2			; T2 = Sigma0(a) + Maj(a,b,c)
||	ROTL	$G,0,$H				; h = g
||	MV	$F,$G				; g = f
||	MV	$X0,$X14
||	SWAP4	$Xn,$X0
	SWAP2	$X0,$X0
||	MV	$E,$F				; f = e
||	ADD	$D,$T1,$E			; e = d + T1
||	MV	$C,$D				; d = c
	MV	$B,$C				; c = b
||	MV	$A,$B				; b = a
||	ADD	$T1,$T2,$A			; a = T1 + T2
	SPKERNEL

	ROTL	$A,30,$S0			; BODY_15
||	OR	$A,$B,$Maj
||	AND	$A,$B,$t2a
||	ROTL	$E,26,$S1
||	AND	$F,$E,$Ch
||	ANDN	$G,$E,$t2e
||	LDW	*${Xib}[1],$Xn			; modulo-scheduled
	ROTL	$A,19,$t0a
||	AND	$C,$Maj,$Maj
||	ROTL	$E,21,$t0e
||	XOR	$t2e,$Ch,$Ch			; Ch(e,f,g) = (e&f)^(~e&g)
||	LDW	*${Xib}[2],$X1			; modulo-scheduled
	ROTL	$A,10,$t1a
||	OR	$t2a,$Maj,$Maj			; Maj(a,b,c) = ((a|b)&c)|(a&b)
||	ROTL	$E,7,$t1e
||	ADD	$K,$H,$T1			; T1 = h + K256[i]
	ADD	$X0,$T1,$T1			; T1 += X[i];
||	STW	$X0,*$Xib++
||	XOR	$t0a,$S0,$S0
||	XOR	$t0e,$S1,$S1
	XOR	$t1a,$S0,$S0			; Sigma0(a)
||	XOR	$t1e,$S1,$S1			; Sigma1(e)
||	LDW	*$K256++,$K			; pre-fetch K256[i+1]
||	ADD	$Ch,$T1,$T1			; T1 += Ch(e,f,g)
	ADD	$S1,$T1,$T1			; T1 += Sigma1(e)
||	ADD	$S0,$Maj,$T2			; T2 = Sigma0(a) + Maj(a,b,c)
||	ROTL	$G,0,$H				; h = g
||	MV	$F,$G				; g = f
||	MV	$X0,$X15
	MV	$E,$F				; f = e
||	ADD	$D,$T1,$E			; e = d + T1
||	MV	$C,$D				; d = c
||	MV	$Xn,$X0				; modulo-scheduled
||	LDW	*$Xia,$X9			; modulo-scheduled
||	ROTL	$X1,25,$t0e			; modulo-scheduled
||	ROTL	$X14,15,$t0a			; modulo-scheduled
	SHRU	$X1,3,$s0			; modulo-scheduled
||	SHRU	$X14,10,$s1			; modulo-scheduled
||	ROTL	$B,0,$C				; c = b
||	MV	$A,$B				; b = a
||	ADD	$T1,$T2,$A			; a = T1 + T2

	SPLOOPD	10				; BODY_16_63
||	MVC	B1,ILC
||	ROTL	$X1,14,$t1e			; modulo-scheduled
||	ROTL	$X14,13,$t1a			; modulo-scheduled

	XOR	$t0e,$s0,$s0
||	XOR	$t0a,$s1,$s1
||	MV	$X15,$X14
||	MV	$X1,$Xn
	XOR	$t1e,$s0,$s0			; sigma0(X[i+1])
||	XOR	$t1a,$s1,$s1			; sigma1(X[i+14])
||	LDW	*${Xib}[2],$X1			; module-scheduled
	ROTL	$A,30,$S0
||	OR	$A,$B,$Maj
||	AND	$A,$B,$t2a
||	ROTL	$E,26,$S1
||	AND	$F,$E,$Ch
||	ANDN	$G,$E,$t2e
||	ADD	$X9,$X0,$X0			; X[i] += X[i+9]
	ROTL	$A,19,$t0a
||	AND	$C,$Maj,$Maj
||	ROTL	$E,21,$t0e
||	XOR	$t2e,$Ch,$Ch			; Ch(e,f,g) = (e&f)^(~e&g)
||	ADD	$s0,$X0,$X0			; X[i] += sigma1(X[i+1])
	ROTL	$A,10,$t1a
||	OR	$t2a,$Maj,$Maj			; Maj(a,b,c) = ((a|b)&c)|(a&b)
||	ROTL	$E,7,$t1e
||	ADD	$H,$K,$T1			; T1 = h + K256[i]
||	ADD	$s1,$X0,$X0			; X[i] += sigma1(X[i+14])
	XOR	$t0a,$S0,$S0
||	XOR	$t0e,$S1,$S1
||	ADD	$X0,$T1,$T1			; T1 += X[i]
||	STW	$X0,*$Xib++
	XOR	$t1a,$S0,$S0			; Sigma0(a)
||	XOR	$t1e,$S1,$S1			; Sigma1(e)
||	ADD	$Ch,$T1,$T1			; T1 += Ch(e,f,g)
||	MV	$X0,$X15
||	ROTL	$G,0,$H				; h = g
||	LDW	*$K256++,$K			; pre-fetch K256[i+1]
	ADD	$S1,$T1,$T1			; T1 += Sigma1(e)
||	ADD	$S0,$Maj,$T2			; T2 = Sigma0(a) + Maj(a,b,c)
||	MV	$F,$G				; g = f
||	MV	$Xn,$X0				; modulo-scheduled
||	LDW	*++$Xia,$X9			; modulo-scheduled
||	ROTL	$X1,25,$t0e			; module-scheduled
||	ROTL	$X14,15,$t0a			; modulo-scheduled
	ROTL	$X1,14,$t1e			; modulo-scheduled
||	ROTL	$X14,13,$t1a			; modulo-scheduled
||	MV	$E,$F				; f = e
||	ADD	$D,$T1,$E			; e = d + T1
||	MV	$C,$D				; d = c
||	MV	$B,$C				; c = b
	MV	$A,$B				; b = a
||	ADD	$T1,$T2,$A			; a = T1 + T2
||	SHRU	$X1,3,$s0			; modulo-scheduled
||	SHRU	$X14,10,$s1			; modulo-scheduled
	SPKERNEL

   [A0]	B	outerloop?
|| [A0]	LDNW	*$INP++,$Xn			; pre-fetch input
|| [A0]	ADDK	-260,$K256			; rewind K256
||	ADD	$Actx,$A,$A			; accumulate ctx
||	ADD	$Ectx,$E,$E
||	ADD	$Bctx,$B,$B
	ADD	$Fctx,$F,$F
||	ADD	$Cctx,$C,$C
||	ADD	$Gctx,$G,$G
||	ADD	$Dctx,$D,$D
||	ADD	$Hctx,$H,$H
|| [A0]	LDW	*$K256++,$K			; pre-fetch K256[0]

  [!A0]	BNOP	RA
||[!A0]	MV	$CTXA,$CTXB
  [!A0]	MV	FP,SP				; restore stack pointer
||[!A0]	LDW	*FP[0],FP			; restore frame pointer
  [!A0]	STW	$A,*${CTXA}[0]  		; save ctx
||[!A0]	STW	$E,*${CTXB}[4]
||[!A0]	MVK	0,B0
  [!A0]	STW	$B,*${CTXA}[1]
||[!A0]	STW	$F,*${CTXB}[5]
||[!A0]	MVC	B0,AMR				; clear AMR
	STW	$C,*${CTXA}[2]
||	STW	$G,*${CTXB}[6]
	STW	$D,*${CTXA}[3]
||	STW	$H,*${CTXB}[7]
	.endasmfunc

	.if	__TI_EABI__
	.sect	".text:sha_asm.const"
	.else
	.sect	".const:sha_asm"
	.endif
	.align	128
K256:
	.uword	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
	.uword	0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
	.uword	0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
	.uword	0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
	.uword	0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
	.uword	0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
	.uword	0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
	.uword	0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
	.uword	0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
	.uword	0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
	.uword	0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
	.uword	0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
	.uword	0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
	.uword	0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
	.uword	0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
	.uword	0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
	.cstring "SHA256 block transform for C64x+, CRYPTOGAMS by <appro\@openssl.org>"
	.align	4

___

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# Multi-buffer SHA256 procedure processes n buffers in parallel by
# placing buffer data to designated lane of SIMD register. n is
# naturally limited to 4 on pre-AVX2 processors and to 8 on
# AVX2-capable processors such as Haswell.
#
#		this	+aesni(i)	sha256	aesni-sha256	gain(iv)
# -------------------------------------------------------------------
# Westmere(ii)	23.3/n	+1.28=7.11(n=4)	12.3	+3.75=16.1	+126%
# Atom(ii)	38.7/n	+3.93=13.6(n=4)	20.8	+5.69=26.5	+95%
# Sandy Bridge	(20.5	+5.15=25.7)/n	11.6	13.0		+103%
# Ivy Bridge	(20.4	+5.14=25.5)/n	10.3	11.6		+82%
# Haswell(iii)	(21.0	+5.00=26.0)/n	7.80	8.79		+170%
# Skylake	(18.9	+5.00=23.9)/n	7.70	8.17		+170%
# Bulldozer	(21.6	+5.76=27.4)/n	13.6	13.7		+100%
#
# (i)	multi-block CBC encrypt with 128-bit key;
# (ii)	(HASH+AES)/n does not apply to Westmere for n>3 and Atom,
#	because of lower AES-NI instruction throughput, nor is there
#	AES-NI-SHA256 stitch for these processors;
# (iii)	"this" is for n=8, when we gather twice as much data, result
#	for n=4 is 20.3+4.44=24.7;
# (iv)	presented improvement coefficients are asymptotic limits and
#	in real-life application are somewhat lower, e.g. for 2KB
#	fragments they range from 75% to 130% (on Haswell);

$flavour = shift;
$output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

$avx=0;

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.19) + ($1>=2.22);
}

if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
	   `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.09) + ($1>=2.10);
}

if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
	   `ml64 2>&1` =~ /Version ([0-9]+)\./) {
	$avx = ($1>=10) + ($1>=11);
}

if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
	$avx = ($2>=3.0) + ($2>3.0);
}

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

# void sha256_multi_block (
#     struct {	unsigned int A[8];
#		unsigned int B[8];
#		unsigned int C[8];
#		unsigned int D[8];
#		unsigned int E[8];
#		unsigned int F[8];
#		unsigned int G[8];
#		unsigned int H[8];	} *ctx,
#     struct {	void *ptr; int blocks;	} inp[8],
#     int num);		/* 1 or 2 */
#
$ctx="%rdi";	# 1st arg
$inp="%rsi";	# 2nd arg
$num="%edx";	# 3rd arg
@ptr=map("%r$_",(8..11));
$Tbl="%rbp";

@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%xmm$_",(8..15));
($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%xmm$_",(0..7));

$REG_SZ=16;

sub Xi_off {
my $off = shift;

    $off %= 16; $off *= $REG_SZ;
    $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
}

sub ROUND_00_15 {
my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;

$code.=<<___ if ($i<15);
	movd		`4*$i`(@ptr[0]),$Xi
	movd		`4*$i`(@ptr[1]),$t1
	movd		`4*$i`(@ptr[2]),$t2
	movd		`4*$i`(@ptr[3]),$t3
	punpckldq	$t2,$Xi
	punpckldq	$t3,$t1
	punpckldq	$t1,$Xi
___
$code.=<<___ if ($i==15);
	movd		`4*$i`(@ptr[0]),$Xi
	 lea		`16*4`(@ptr[0]),@ptr[0]
	movd		`4*$i`(@ptr[1]),$t1
	 lea		`16*4`(@ptr[1]),@ptr[1]
	movd		`4*$i`(@ptr[2]),$t2
	 lea		`16*4`(@ptr[2]),@ptr[2]
	movd		`4*$i`(@ptr[3]),$t3
	 lea		`16*4`(@ptr[3]),@ptr[3]
	punpckldq	$t2,$Xi
	punpckldq	$t3,$t1
	punpckldq	$t1,$Xi
___
$code.=<<___;
	movdqa	$e,$sigma
	`"pshufb	$Xn,$Xi"		if ($i<=15 && ($i&1)==0)`
	movdqa	$e,$t3
	`"pshufb	$Xn,$Xi"		if ($i<=15 && ($i&1)==1)`
	psrld	\$6,$sigma
	movdqa	$e,$t2
	pslld	\$7,$t3
	movdqa	$Xi,`&Xi_off($i)`
	 paddd	$h,$Xi				# Xi+=h

	psrld	\$11,$t2
	pxor	$t3,$sigma
	pslld	\$21-7,$t3
	 paddd	`32*($i%8)-128`($Tbl),$Xi	# Xi+=K[round]
	pxor	$t2,$sigma

	psrld	\$25-11,$t2
	 movdqa	$e,$t1
	 `"prefetcht0	63(@ptr[0])"		if ($i==15)`
	pxor	$t3,$sigma
	 movdqa	$e,$axb				# borrow $axb
	pslld	\$26-21,$t3
	 pandn	$g,$t1
	 pand	$f,$axb
	pxor	$t2,$sigma

	 `"prefetcht0	63(@ptr[1])"		if ($i==15)`
	movdqa	$a,$t2
	pxor	$t3,$sigma			# Sigma1(e)
	movdqa	$a,$t3
	psrld	\$2,$t2
	paddd	$sigma,$Xi			# Xi+=Sigma1(e)
	 pxor	$axb,$t1			# Ch(e,f,g)
	 movdqa	$b,$axb
	movdqa	$a,$sigma
	pslld	\$10,$t3
	 pxor	$a,$axb				# a^b, b^c in next round

	 `"prefetcht0	63(@ptr[2])"		if ($i==15)`
	psrld	\$13,$sigma
	pxor	$t3,$t2
	 paddd	$t1,$Xi				# Xi+=Ch(e,f,g)
	pslld	\$19-10,$t3
	 pand	$axb,$bxc
	pxor	$sigma,$t2

	 `"prefetcht0	63(@ptr[3])"		if ($i==15)`
	psrld	\$22-13,$sigma
	pxor	$t3,$t2
	 movdqa	$b,$h
	pslld	\$30-19,$t3
	pxor	$t2,$sigma
	 pxor	$bxc,$h				# h=Maj(a,b,c)=Ch(a^b,c,b)
	 paddd	$Xi,$d				# d+=Xi
	pxor	$t3,$sigma			# Sigma0(a)

	paddd	$Xi,$h				# h+=Xi
	paddd	$sigma,$h			# h+=Sigma0(a)
___
$code.=<<___ if (($i%8)==7);
	lea	`32*8`($Tbl),$Tbl
___
	($axb,$bxc)=($bxc,$axb);
}

sub ROUND_16_XX {
my $i=shift;

$code.=<<___;
	movdqa	`&Xi_off($i+1)`,$Xn
	paddd	`&Xi_off($i+9)`,$Xi		# Xi+=X[i+9]

	movdqa	$Xn,$sigma
	movdqa	$Xn,$t2
	psrld	\$3,$sigma
	movdqa	$Xn,$t3

	psrld	\$7,$t2
	movdqa	`&Xi_off($i+14)`,$t1
	pslld	\$14,$t3
	pxor	$t2,$sigma
	psrld	\$18-7,$t2
	movdqa	$t1,$axb			# borrow $axb
	pxor	$t3,$sigma
	pslld	\$25-14,$t3
	pxor	$t2,$sigma
	psrld	\$10,$t1
	movdqa	$axb,$t2

	psrld	\$17,$axb
	pxor	$t3,$sigma			# sigma0(X[i+1])
	pslld	\$13,$t2
	 paddd	$sigma,$Xi			# Xi+=sigma0(e)
	pxor	$axb,$t1
	psrld	\$19-17,$axb
	pxor	$t2,$t1
	pslld	\$15-13,$t2
	pxor	$axb,$t1
	pxor	$t2,$t1				# sigma0(X[i+14])
	paddd	$t1,$Xi				# Xi+=sigma1(X[i+14])
___
	&ROUND_00_15($i,@_);
	($Xi,$Xn)=($Xn,$Xi);
}

$code.=<<___;
.text

.extern	OPENSSL_ia32cap_P

.globl	sha256_multi_block
.type	sha256_multi_block,\@function,3
.align	32
sha256_multi_block:
.cfi_startproc
	mov	OPENSSL_ia32cap_P+4(%rip),%rcx
	bt	\$61,%rcx			# check SHA bit
	jc	_shaext_shortcut
___
$code.=<<___ if ($avx);
	test	\$`1<<28`,%ecx
	jnz	_avx_shortcut
___
$code.=<<___;
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,-0x78(%rax)
	movaps	%xmm11,-0x68(%rax)
	movaps	%xmm12,-0x58(%rax)
	movaps	%xmm13,-0x48(%rax)
	movaps	%xmm14,-0x38(%rax)
	movaps	%xmm15,-0x28(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`, %rsp
	and	\$-256,%rsp
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.cfi_cfa_expression	%rsp+`$REG_SZ*17`,deref,+8
.Lbody:
	lea	K256+128(%rip),$Tbl
	lea	`$REG_SZ*16`(%rsp),%rbx
	lea	0x80($ctx),$ctx			# size optimization

.Loop_grande:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldone

	movdqu	0x00-0x80($ctx),$A		# load context
	 lea	128(%rsp),%rax
	movdqu	0x20-0x80($ctx),$B
	movdqu	0x40-0x80($ctx),$C
	movdqu	0x60-0x80($ctx),$D
	movdqu	0x80-0x80($ctx),$E
	movdqu	0xa0-0x80($ctx),$F
	movdqu	0xc0-0x80($ctx),$G
	movdqu	0xe0-0x80($ctx),$H
	movdqu	.Lpbswap(%rip),$Xn
	jmp	.Loop

.align	32
.Loop:
	movdqa	$C,$bxc
	pxor	$B,$bxc				# magic seed
___
for($i=0;$i<16;$i++)	{ &ROUND_00_15($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	movdqu	`&Xi_off($i)`,$Xi
	mov	\$3,%ecx
	jmp	.Loop_16_xx
.align	32
.Loop_16_xx:
___
for(;$i<32;$i++)	{ &ROUND_16_XX($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	dec	%ecx
	jnz	.Loop_16_xx

	mov	\$1,%ecx
	lea	K256+128(%rip),$Tbl

	movdqa	(%rbx),$sigma			# pull counters
	cmp	4*0(%rbx),%ecx			# examine counters
	pxor	$t1,$t1
	cmovge	$Tbl,@ptr[0]			# cancel input
	cmp	4*1(%rbx),%ecx
	movdqa	$sigma,$Xn
	cmovge	$Tbl,@ptr[1]
	cmp	4*2(%rbx),%ecx
	pcmpgtd	$t1,$Xn				# mask value
	cmovge	$Tbl,@ptr[2]
	cmp	4*3(%rbx),%ecx
	paddd	$Xn,$sigma			# counters--
	cmovge	$Tbl,@ptr[3]

	movdqu	0x00-0x80($ctx),$t1
	pand	$Xn,$A
	movdqu	0x20-0x80($ctx),$t2
	pand	$Xn,$B
	movdqu	0x40-0x80($ctx),$t3
	pand	$Xn,$C
	movdqu	0x60-0x80($ctx),$Xi
	pand	$Xn,$D
	paddd	$t1,$A
	movdqu	0x80-0x80($ctx),$t1
	pand	$Xn,$E
	paddd	$t2,$B
	movdqu	0xa0-0x80($ctx),$t2
	pand	$Xn,$F
	paddd	$t3,$C
	movdqu	0xc0-0x80($ctx),$t3
	pand	$Xn,$G
	paddd	$Xi,$D
	movdqu	0xe0-0x80($ctx),$Xi
	pand	$Xn,$H
	paddd	$t1,$E
	paddd	$t2,$F
	movdqu	$A,0x00-0x80($ctx)
	paddd	$t3,$G
	movdqu	$B,0x20-0x80($ctx)
	paddd	$Xi,$H
	movdqu	$C,0x40-0x80($ctx)
	movdqu	$D,0x60-0x80($ctx)
	movdqu	$E,0x80-0x80($ctx)
	movdqu	$F,0xa0-0x80($ctx)
	movdqu	$G,0xc0-0x80($ctx)
	movdqu	$H,0xe0-0x80($ctx)

	movdqa	$sigma,(%rbx)			# save counters
	movdqa	.Lpbswap(%rip),$Xn
	dec	$num
	jnz	.Loop

	mov	`$REG_SZ*17+8`(%rsp),$num
	lea	$REG_SZ($ctx),$ctx
	lea	`16*$REG_SZ/4`($inp),$inp
	dec	$num
	jnz	.Loop_grande

.Ldone:
	mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
.cfi_def_cfa	%rax,8
___
$code.=<<___ if ($win64);
	movaps	-0xb8(%rax),%xmm6
	movaps	-0xa8(%rax),%xmm7
	movaps	-0x98(%rax),%xmm8
	movaps	-0x88(%rax),%xmm9
	movaps	-0x78(%rax),%xmm10
	movaps	-0x68(%rax),%xmm11
	movaps	-0x58(%rax),%xmm12
	movaps	-0x48(%rax),%xmm13
	movaps	-0x38(%rax),%xmm14
	movaps	-0x28(%rax),%xmm15
___
$code.=<<___;
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue:
	ret
.cfi_endproc
.size	sha256_multi_block,.-sha256_multi_block
___
						{{{
my ($Wi,$TMP0,$TMP1,$TMPx,$ABEF0,$CDGH0,$ABEF1,$CDGH1)=map("%xmm$_",(0..3,12..15));
my @MSG0=map("%xmm$_",(4..7));
my @MSG1=map("%xmm$_",(8..11));

$code.=<<___;
.type	sha256_multi_block_shaext,\@function,3
.align	32
sha256_multi_block_shaext:
.cfi_startproc
_shaext_shortcut:
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,-0x78(%rax)
	movaps	%xmm11,-0x68(%rax)
	movaps	%xmm12,-0x58(%rax)
	movaps	%xmm13,-0x48(%rax)
	movaps	%xmm14,-0x38(%rax)
	movaps	%xmm15,-0x28(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`,%rsp
	shl	\$1,$num			# we process pair at a time
	and	\$-256,%rsp
	lea	0x80($ctx),$ctx			# size optimization
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.Lbody_shaext:
	lea	`$REG_SZ*16`(%rsp),%rbx
	lea	K256_shaext+0x80(%rip),$Tbl

.Loop_grande_shaext:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
___
for($i=0;$i<2;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	%rsp,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldone_shaext

	movq		0x00-0x80($ctx),$ABEF0		# A1.A0
	movq		0x20-0x80($ctx),@MSG0[0]	# B1.B0
	movq		0x40-0x80($ctx),$CDGH0		# C1.C0
	movq		0x60-0x80($ctx),@MSG0[1]	# D1.D0
	movq		0x80-0x80($ctx),@MSG1[0]	# E1.E0
	movq		0xa0-0x80($ctx),@MSG1[1]	# F1.F0
	movq		0xc0-0x80($ctx),@MSG1[2]	# G1.G0
	movq		0xe0-0x80($ctx),@MSG1[3]	# H1.H0

	punpckldq	@MSG0[0],$ABEF0			# B1.A1.B0.A0
	punpckldq	@MSG0[1],$CDGH0			# D1.C1.D0.C0
	punpckldq	@MSG1[1],@MSG1[0]		# F1.E1.F0.E0
	punpckldq	@MSG1[3],@MSG1[2]		# H1.G1.H0.G0
	movdqa		K256_shaext-0x10(%rip),$TMPx	# byte swap

	movdqa		$ABEF0,$ABEF1
	movdqa		$CDGH0,$CDGH1
	punpcklqdq	@MSG1[0],$ABEF0			# F0.E0.B0.A0
	punpcklqdq	@MSG1[2],$CDGH0			# H0.G0.D0.C0
	punpckhqdq	@MSG1[0],$ABEF1			# F1.E1.B1.A1
	punpckhqdq	@MSG1[2],$CDGH1			# H1.G1.D1.C1

	pshufd		\$0b00011011,$ABEF0,$ABEF0
	pshufd		\$0b00011011,$CDGH0,$CDGH0
	pshufd		\$0b00011011,$ABEF1,$ABEF1
	pshufd		\$0b00011011,$CDGH1,$CDGH1
	jmp		.Loop_shaext

.align	32
.Loop_shaext:
	movdqu		0x00(@ptr[0]),@MSG0[0]
	 movdqu		0x00(@ptr[1]),@MSG1[0]
	movdqu		0x10(@ptr[0]),@MSG0[1]
	 movdqu		0x10(@ptr[1]),@MSG1[1]
	movdqu		0x20(@ptr[0]),@MSG0[2]
	pshufb		$TMPx,@MSG0[0]
	 movdqu		0x20(@ptr[1]),@MSG1[2]
	 pshufb		$TMPx,@MSG1[0]
	movdqu		0x30(@ptr[0]),@MSG0[3]
	lea		0x40(@ptr[0]),@ptr[0]
	 movdqu		0x30(@ptr[1]),@MSG1[3]
	 lea		0x40(@ptr[1]),@ptr[1]

	movdqa		0*16-0x80($Tbl),$Wi
	pshufb		$TMPx,@MSG0[1]
	paddd		@MSG0[0],$Wi
	pxor		$ABEF0,@MSG0[0]		# black magic
	movdqa		$Wi,$TMP0
	 movdqa		0*16-0x80($Tbl),$TMP1
	 pshufb		$TMPx,@MSG1[1]
	 paddd		@MSG1[0],$TMP1
	movdqa		$CDGH0,0x50(%rsp)	# offload
	sha256rnds2	$ABEF0,$CDGH0		# 0-3
	 pxor		$ABEF1,@MSG1[0]		# black magic
	 movdqa		$TMP1,$Wi
	 movdqa		$CDGH1,0x70(%rsp)
	 sha256rnds2	$ABEF1,$CDGH1		# 0-3
	pshufd		\$0x0e,$TMP0,$Wi
	pxor		$ABEF0,@MSG0[0]		# black magic
	movdqa		$ABEF0,0x40(%rsp)	# offload
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	 pxor		$ABEF1,@MSG1[0]		# black magic
	 movdqa		$ABEF1,0x60(%rsp)
	movdqa		1*16-0x80($Tbl),$TMP0
	paddd		@MSG0[1],$TMP0
	pshufb		$TMPx,@MSG0[2]
	 sha256rnds2	$CDGH1,$ABEF1

	movdqa		$TMP0,$Wi
	 movdqa		1*16-0x80($Tbl),$TMP1
	 paddd		@MSG1[1],$TMP1
	sha256rnds2	$ABEF0,$CDGH0		# 4-7
	 movdqa		$TMP1,$Wi
	prefetcht0	127(@ptr[0])
	pshufb		$TMPx,@MSG0[3]
	 pshufb		$TMPx,@MSG1[2]
	 prefetcht0	127(@ptr[1])
	 sha256rnds2	$ABEF1,$CDGH1		# 4-7
	pshufd		\$0x0e,$TMP0,$Wi
	 pshufb		$TMPx,@MSG1[3]
	sha256msg1	@MSG0[1],@MSG0[0]
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	movdqa		2*16-0x80($Tbl),$TMP0
	paddd		@MSG0[2],$TMP0
	 sha256rnds2	$CDGH1,$ABEF1

	movdqa		$TMP0,$Wi
	 movdqa		2*16-0x80($Tbl),$TMP1
	 paddd		@MSG1[2],$TMP1
	sha256rnds2	$ABEF0,$CDGH0		# 8-11
	 sha256msg1	@MSG1[1],@MSG1[0]
	 movdqa		$TMP1,$Wi
	movdqa		@MSG0[3],$TMPx
	 sha256rnds2	$ABEF1,$CDGH1		# 8-11
	pshufd		\$0x0e,$TMP0,$Wi
	palignr		\$4,@MSG0[2],$TMPx
	paddd		$TMPx,@MSG0[0]
	 movdqa		@MSG1[3],$TMPx
	 palignr	\$4,@MSG1[2],$TMPx
	sha256msg1	@MSG0[2],@MSG0[1]
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	movdqa		3*16-0x80($Tbl),$TMP0
	paddd		@MSG0[3],$TMP0
	 sha256rnds2	$CDGH1,$ABEF1
	 sha256msg1	@MSG1[2],@MSG1[1]

	movdqa		$TMP0,$Wi
	 movdqa		3*16-0x80($Tbl),$TMP1
	 paddd		$TMPx,@MSG1[0]
	 paddd		@MSG1[3],$TMP1
	sha256msg2	@MSG0[3],@MSG0[0]
	sha256rnds2	$ABEF0,$CDGH0		# 12-15
	 movdqa		$TMP1,$Wi
	movdqa		@MSG0[0],$TMPx
	palignr		\$4,@MSG0[3],$TMPx
	 sha256rnds2	$ABEF1,$CDGH1		# 12-15
	 sha256msg2	@MSG1[3],@MSG1[0]
	pshufd		\$0x0e,$TMP0,$Wi
	paddd		$TMPx,@MSG0[1]
	 movdqa		@MSG1[0],$TMPx
	 palignr	\$4,@MSG1[3],$TMPx
	sha256msg1	@MSG0[3],@MSG0[2]
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	movdqa		4*16-0x80($Tbl),$TMP0
	paddd		@MSG0[0],$TMP0
	 sha256rnds2	$CDGH1,$ABEF1
	 sha256msg1	@MSG1[3],@MSG1[2]
___
for($i=4;$i<16-3;$i++) {
$code.=<<___;
	movdqa		$TMP0,$Wi
	 movdqa		$i*16-0x80($Tbl),$TMP1
	 paddd		$TMPx,@MSG1[1]
	 paddd		@MSG1[0],$TMP1
	sha256msg2	@MSG0[0],@MSG0[1]
	sha256rnds2	$ABEF0,$CDGH0		# 16-19...
	 movdqa		$TMP1,$Wi
	movdqa		@MSG0[1],$TMPx
	palignr		\$4,@MSG0[0],$TMPx
	 sha256rnds2	$ABEF1,$CDGH1		# 16-19...
	 sha256msg2	@MSG1[0],@MSG1[1]
	pshufd		\$0x0e,$TMP0,$Wi
	paddd		$TMPx,@MSG0[2]
	 movdqa		@MSG1[1],$TMPx
	 palignr	\$4,@MSG1[0],$TMPx
	sha256msg1	@MSG0[0],@MSG0[3]
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	movdqa		`($i+1)*16`-0x80($Tbl),$TMP0
	paddd		@MSG0[1],$TMP0
	 sha256rnds2	$CDGH1,$ABEF1
	 sha256msg1	@MSG1[0],@MSG1[3]
___
	push(@MSG0,shift(@MSG0));	push(@MSG1,shift(@MSG1));
}
$code.=<<___;
	movdqa		$TMP0,$Wi
	 movdqa		13*16-0x80($Tbl),$TMP1
	 paddd		$TMPx,@MSG1[1]
	 paddd		@MSG1[0],$TMP1
	sha256msg2	@MSG0[0],@MSG0[1]
	sha256rnds2	$ABEF0,$CDGH0		# 52-55
	 movdqa		$TMP1,$Wi
	movdqa		@MSG0[1],$TMPx
	palignr		\$4,@MSG0[0],$TMPx
	 sha256rnds2	$ABEF1,$CDGH1		# 52-55
	 sha256msg2	@MSG1[0],@MSG1[1]
	pshufd		\$0x0e,$TMP0,$Wi
	paddd		$TMPx,@MSG0[2]
	 movdqa		@MSG1[1],$TMPx
	 palignr	\$4,@MSG1[0],$TMPx
	nop
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	movdqa		14*16-0x80($Tbl),$TMP0
	paddd		@MSG0[1],$TMP0
	 sha256rnds2	$CDGH1,$ABEF1

	movdqa		$TMP0,$Wi
	 movdqa		14*16-0x80($Tbl),$TMP1
	 paddd		$TMPx,@MSG1[2]
	 paddd		@MSG1[1],$TMP1
	sha256msg2	@MSG0[1],@MSG0[2]
	nop
	sha256rnds2	$ABEF0,$CDGH0		# 56-59
	 movdqa		$TMP1,$Wi
	  mov		\$1,%ecx
	  pxor		@MSG0[1],@MSG0[1]	# zero
	 sha256rnds2	$ABEF1,$CDGH1		# 56-59
	 sha256msg2	@MSG1[1],@MSG1[2]
	pshufd		\$0x0e,$TMP0,$Wi
	movdqa		15*16-0x80($Tbl),$TMP0
	paddd		@MSG0[2],$TMP0
	  movq		(%rbx),@MSG0[2]		# pull counters
	  nop
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	 movdqa		15*16-0x80($Tbl),$TMP1
	 paddd		@MSG1[2],$TMP1
	 sha256rnds2	$CDGH1,$ABEF1

	movdqa		$TMP0,$Wi
	  cmp		4*0(%rbx),%ecx		# examine counters
	  cmovge	%rsp,@ptr[0]		# cancel input
	  cmp		4*1(%rbx),%ecx
	  cmovge	%rsp,@ptr[1]
	  pshufd	\$0x00,@MSG0[2],@MSG1[0]
	sha256rnds2	$ABEF0,$CDGH0		# 60-63
	 movdqa		$TMP1,$Wi
	  pshufd	\$0x55,@MSG0[2],@MSG1[1]
	  movdqa	@MSG0[2],@MSG1[2]
	 sha256rnds2	$ABEF1,$CDGH1		# 60-63
	pshufd		\$0x0e,$TMP0,$Wi
	  pcmpgtd	@MSG0[1],@MSG1[0]
	  pcmpgtd	@MSG0[1],@MSG1[1]
	sha256rnds2	$CDGH0,$ABEF0
	 pshufd		\$0x0e,$TMP1,$Wi
	  pcmpgtd	@MSG0[1],@MSG1[2]	# counter mask
	  movdqa	K256_shaext-0x10(%rip),$TMPx
	 sha256rnds2	$CDGH1,$ABEF1

	pand		@MSG1[0],$CDGH0
	 pand		@MSG1[1],$CDGH1
	pand		@MSG1[0],$ABEF0
	 pand		@MSG1[1],$ABEF1
	paddd		@MSG0[2],@MSG1[2]	# counters--

	paddd		0x50(%rsp),$CDGH0
	 paddd		0x70(%rsp),$CDGH1
	paddd		0x40(%rsp),$ABEF0
	 paddd		0x60(%rsp),$ABEF1

	movq		@MSG1[2],(%rbx)		# save counters
	dec		$num
	jnz		.Loop_shaext

	mov		`$REG_SZ*17+8`(%rsp),$num

	pshufd		\$0b00011011,$ABEF0,$ABEF0
	pshufd		\$0b00011011,$CDGH0,$CDGH0
	pshufd		\$0b00011011,$ABEF1,$ABEF1
	pshufd		\$0b00011011,$CDGH1,$CDGH1

	movdqa		$ABEF0,@MSG0[0]
	movdqa		$CDGH0,@MSG0[1]
	punpckldq	$ABEF1,$ABEF0			# B1.B0.A1.A0
	punpckhdq	$ABEF1,@MSG0[0]			# F1.F0.E1.E0
	punpckldq	$CDGH1,$CDGH0			# D1.D0.C1.C0
	punpckhdq	$CDGH1,@MSG0[1]			# H1.H0.G1.G0

	movq		$ABEF0,0x00-0x80($ctx)		# A1.A0
	psrldq		\$8,$ABEF0
	movq		@MSG0[0],0x80-0x80($ctx)	# E1.E0
	psrldq		\$8,@MSG0[0]
	movq		$ABEF0,0x20-0x80($ctx)		# B1.B0
	movq		@MSG0[0],0xa0-0x80($ctx)	# F1.F0

	movq		$CDGH0,0x40-0x80($ctx)		# C1.C0
	psrldq		\$8,$CDGH0
	movq		@MSG0[1],0xc0-0x80($ctx)	# G1.G0
	psrldq		\$8,@MSG0[1]
	movq		$CDGH0,0x60-0x80($ctx)		# D1.D0
	movq		@MSG0[1],0xe0-0x80($ctx)	# H1.H0

	lea	`$REG_SZ/2`($ctx),$ctx
	lea	`16*2`($inp),$inp
	dec	$num
	jnz	.Loop_grande_shaext

.Ldone_shaext:
	#mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
___
$code.=<<___ if ($win64);
	movaps	-0xb8(%rax),%xmm6
	movaps	-0xa8(%rax),%xmm7
	movaps	-0x98(%rax),%xmm8
	movaps	-0x88(%rax),%xmm9
	movaps	-0x78(%rax),%xmm10
	movaps	-0x68(%rax),%xmm11
	movaps	-0x58(%rax),%xmm12
	movaps	-0x48(%rax),%xmm13
	movaps	-0x38(%rax),%xmm14
	movaps	-0x28(%rax),%xmm15
___
$code.=<<___;
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue_shaext:
	ret
.cfi_endproc
.size	sha256_multi_block_shaext,.-sha256_multi_block_shaext
___
						}}}
						if ($avx) {{{
sub ROUND_00_15_avx {
my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;

$code.=<<___ if ($i<15 && $REG_SZ==16);
	vmovd		`4*$i`(@ptr[0]),$Xi
	vmovd		`4*$i`(@ptr[1]),$t1
	vpinsrd		\$1,`4*$i`(@ptr[2]),$Xi,$Xi
	vpinsrd		\$1,`4*$i`(@ptr[3]),$t1,$t1
	vpunpckldq	$t1,$Xi,$Xi
	vpshufb		$Xn,$Xi,$Xi
___
$code.=<<___ if ($i==15 && $REG_SZ==16);
	vmovd		`4*$i`(@ptr[0]),$Xi
	 lea		`16*4`(@ptr[0]),@ptr[0]
	vmovd		`4*$i`(@ptr[1]),$t1
	 lea		`16*4`(@ptr[1]),@ptr[1]
	vpinsrd		\$1,`4*$i`(@ptr[2]),$Xi,$Xi
	 lea		`16*4`(@ptr[2]),@ptr[2]
	vpinsrd		\$1,`4*$i`(@ptr[3]),$t1,$t1
	 lea		`16*4`(@ptr[3]),@ptr[3]
	vpunpckldq	$t1,$Xi,$Xi
	vpshufb		$Xn,$Xi,$Xi
___
$code.=<<___ if ($i<15 && $REG_SZ==32);
	vmovd		`4*$i`(@ptr[0]),$Xi
	vmovd		`4*$i`(@ptr[4]),$t1
	vmovd		`4*$i`(@ptr[1]),$t2
	vmovd		`4*$i`(@ptr[5]),$t3
	vpinsrd		\$1,`4*$i`(@ptr[2]),$Xi,$Xi
	vpinsrd		\$1,`4*$i`(@ptr[6]),$t1,$t1
	vpinsrd		\$1,`4*$i`(@ptr[3]),$t2,$t2
	vpunpckldq	$t2,$Xi,$Xi
	vpinsrd		\$1,`4*$i`(@ptr[7]),$t3,$t3
	vpunpckldq	$t3,$t1,$t1
	vinserti128	$t1,$Xi,$Xi
	vpshufb		$Xn,$Xi,$Xi
___
$code.=<<___ if ($i==15 && $REG_SZ==32);
	vmovd		`4*$i`(@ptr[0]),$Xi
	 lea		`16*4`(@ptr[0]),@ptr[0]
	vmovd		`4*$i`(@ptr[4]),$t1
	 lea		`16*4`(@ptr[4]),@ptr[4]
	vmovd		`4*$i`(@ptr[1]),$t2
	 lea		`16*4`(@ptr[1]),@ptr[1]
	vmovd		`4*$i`(@ptr[5]),$t3
	 lea		`16*4`(@ptr[5]),@ptr[5]
	vpinsrd		\$1,`4*$i`(@ptr[2]),$Xi,$Xi
	 lea		`16*4`(@ptr[2]),@ptr[2]
	vpinsrd		\$1,`4*$i`(@ptr[6]),$t1,$t1
	 lea		`16*4`(@ptr[6]),@ptr[6]
	vpinsrd		\$1,`4*$i`(@ptr[3]),$t2,$t2
	 lea		`16*4`(@ptr[3]),@ptr[3]
	vpunpckldq	$t2,$Xi,$Xi
	vpinsrd		\$1,`4*$i`(@ptr[7]),$t3,$t3
	 lea		`16*4`(@ptr[7]),@ptr[7]
	vpunpckldq	$t3,$t1,$t1
	vinserti128	$t1,$Xi,$Xi
	vpshufb		$Xn,$Xi,$Xi
___
$code.=<<___;
	vpsrld	\$6,$e,$sigma
	vpslld	\$26,$e,$t3
	vmovdqu	$Xi,`&Xi_off($i)`
	 vpaddd	$h,$Xi,$Xi			# Xi+=h

	vpsrld	\$11,$e,$t2
	vpxor	$t3,$sigma,$sigma
	vpslld	\$21,$e,$t3
	 vpaddd	`32*($i%8)-128`($Tbl),$Xi,$Xi	# Xi+=K[round]
	vpxor	$t2,$sigma,$sigma

	vpsrld	\$25,$e,$t2
	vpxor	$t3,$sigma,$sigma
	 `"prefetcht0	63(@ptr[0])"		if ($i==15)`
	vpslld	\$7,$e,$t3
	 vpandn	$g,$e,$t1
	 vpand	$f,$e,$axb			# borrow $axb
	 `"prefetcht0	63(@ptr[1])"		if ($i==15)`
	vpxor	$t2,$sigma,$sigma

	vpsrld	\$2,$a,$h			# borrow $h
	vpxor	$t3,$sigma,$sigma		# Sigma1(e)
	 `"prefetcht0	63(@ptr[2])"		if ($i==15)`
	vpslld	\$30,$a,$t2
	 vpxor	$axb,$t1,$t1			# Ch(e,f,g)
	 vpxor	$a,$b,$axb			# a^b, b^c in next round
	 `"prefetcht0	63(@ptr[3])"		if ($i==15)`
	vpxor	$t2,$h,$h
	vpaddd	$sigma,$Xi,$Xi			# Xi+=Sigma1(e)

	vpsrld	\$13,$a,$t2
	 `"prefetcht0	63(@ptr[4])"		if ($i==15 && $REG_SZ==32)`
	vpslld	\$19,$a,$t3
	 vpaddd	$t1,$Xi,$Xi			# Xi+=Ch(e,f,g)
	 vpand	$axb,$bxc,$bxc
	 `"prefetcht0	63(@ptr[5])"		if ($i==15 && $REG_SZ==32)`
	vpxor	$t2,$h,$sigma

	vpsrld	\$22,$a,$t2
	vpxor	$t3,$sigma,$sigma
	 `"prefetcht0	63(@ptr[6])"		if ($i==15 && $REG_SZ==32)`
	vpslld	\$10,$a,$t3
	 vpxor	$bxc,$b,$h			# h=Maj(a,b,c)=Ch(a^b,c,b)
	 vpaddd	$Xi,$d,$d			# d+=Xi
	 `"prefetcht0	63(@ptr[7])"		if ($i==15 && $REG_SZ==32)`
	vpxor	$t2,$sigma,$sigma
	vpxor	$t3,$sigma,$sigma		# Sigma0(a)

	vpaddd	$Xi,$h,$h			# h+=Xi
	vpaddd	$sigma,$h,$h			# h+=Sigma0(a)
___
$code.=<<___ if (($i%8)==7);
	add	\$`32*8`,$Tbl
___
	($axb,$bxc)=($bxc,$axb);
}

sub ROUND_16_XX_avx {
my $i=shift;

$code.=<<___;
	vmovdqu	`&Xi_off($i+1)`,$Xn
	vpaddd	`&Xi_off($i+9)`,$Xi,$Xi		# Xi+=X[i+9]

	vpsrld	\$3,$Xn,$sigma
	vpsrld	\$7,$Xn,$t2
	vpslld	\$25,$Xn,$t3
	vpxor	$t2,$sigma,$sigma
	vpsrld	\$18,$Xn,$t2
	vpxor	$t3,$sigma,$sigma
	vpslld	\$14,$Xn,$t3
	vmovdqu	`&Xi_off($i+14)`,$t1
	vpsrld	\$10,$t1,$axb			# borrow $axb

	vpxor	$t2,$sigma,$sigma
	vpsrld	\$17,$t1,$t2
	vpxor	$t3,$sigma,$sigma		# sigma0(X[i+1])
	vpslld	\$15,$t1,$t3
	 vpaddd	$sigma,$Xi,$Xi			# Xi+=sigma0(e)
	vpxor	$t2,$axb,$sigma
	vpsrld	\$19,$t1,$t2
	vpxor	$t3,$sigma,$sigma
	vpslld	\$13,$t1,$t3
	vpxor	$t2,$sigma,$sigma
	vpxor	$t3,$sigma,$sigma		# sigma0(X[i+14])
	vpaddd	$sigma,$Xi,$Xi			# Xi+=sigma1(X[i+14])
___
	&ROUND_00_15_avx($i,@_);
	($Xi,$Xn)=($Xn,$Xi);
}

$code.=<<___;
.type	sha256_multi_block_avx,\@function,3
.align	32
sha256_multi_block_avx:
.cfi_startproc
_avx_shortcut:
___
$code.=<<___ if ($avx>1);
	shr	\$32,%rcx
	cmp	\$2,$num
	jb	.Lavx
	test	\$`1<<5`,%ecx
	jnz	_avx2_shortcut
	jmp	.Lavx
.align	32
.Lavx:
___
$code.=<<___;
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,-0x78(%rax)
	movaps	%xmm11,-0x68(%rax)
	movaps	%xmm12,-0x58(%rax)
	movaps	%xmm13,-0x48(%rax)
	movaps	%xmm14,-0x38(%rax)
	movaps	%xmm15,-0x28(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`, %rsp
	and	\$-256,%rsp
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.cfi_cfa_expression	%rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx:
	lea	K256+128(%rip),$Tbl
	lea	`$REG_SZ*16`(%rsp),%rbx
	lea	0x80($ctx),$ctx			# size optimization

.Loop_grande_avx:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldone_avx

	vmovdqu	0x00-0x80($ctx),$A		# load context
	 lea	128(%rsp),%rax
	vmovdqu	0x20-0x80($ctx),$B
	vmovdqu	0x40-0x80($ctx),$C
	vmovdqu	0x60-0x80($ctx),$D
	vmovdqu	0x80-0x80($ctx),$E
	vmovdqu	0xa0-0x80($ctx),$F
	vmovdqu	0xc0-0x80($ctx),$G
	vmovdqu	0xe0-0x80($ctx),$H
	vmovdqu	.Lpbswap(%rip),$Xn
	jmp	.Loop_avx

.align	32
.Loop_avx:
	vpxor	$B,$C,$bxc			# magic seed
___
for($i=0;$i<16;$i++)	{ &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	vmovdqu	`&Xi_off($i)`,$Xi
	mov	\$3,%ecx
	jmp	.Loop_16_xx_avx
.align	32
.Loop_16_xx_avx:
___
for(;$i<32;$i++)	{ &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	dec	%ecx
	jnz	.Loop_16_xx_avx

	mov	\$1,%ecx
	lea	K256+128(%rip),$Tbl
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	cmp	`4*$i`(%rbx),%ecx		# examine counters
	cmovge	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	vmovdqa	(%rbx),$sigma			# pull counters
	vpxor	$t1,$t1,$t1
	vmovdqa	$sigma,$Xn
	vpcmpgtd $t1,$Xn,$Xn			# mask value
	vpaddd	$Xn,$sigma,$sigma		# counters--

	vmovdqu	0x00-0x80($ctx),$t1
	vpand	$Xn,$A,$A
	vmovdqu	0x20-0x80($ctx),$t2
	vpand	$Xn,$B,$B
	vmovdqu	0x40-0x80($ctx),$t3
	vpand	$Xn,$C,$C
	vmovdqu	0x60-0x80($ctx),$Xi
	vpand	$Xn,$D,$D
	vpaddd	$t1,$A,$A
	vmovdqu	0x80-0x80($ctx),$t1
	vpand	$Xn,$E,$E
	vpaddd	$t2,$B,$B
	vmovdqu	0xa0-0x80($ctx),$t2
	vpand	$Xn,$F,$F
	vpaddd	$t3,$C,$C
	vmovdqu	0xc0-0x80($ctx),$t3
	vpand	$Xn,$G,$G
	vpaddd	$Xi,$D,$D
	vmovdqu	0xe0-0x80($ctx),$Xi
	vpand	$Xn,$H,$H
	vpaddd	$t1,$E,$E
	vpaddd	$t2,$F,$F
	vmovdqu	$A,0x00-0x80($ctx)
	vpaddd	$t3,$G,$G
	vmovdqu	$B,0x20-0x80($ctx)
	vpaddd	$Xi,$H,$H
	vmovdqu	$C,0x40-0x80($ctx)
	vmovdqu	$D,0x60-0x80($ctx)
	vmovdqu	$E,0x80-0x80($ctx)
	vmovdqu	$F,0xa0-0x80($ctx)
	vmovdqu	$G,0xc0-0x80($ctx)
	vmovdqu	$H,0xe0-0x80($ctx)

	vmovdqu	$sigma,(%rbx)			# save counters
	vmovdqu	.Lpbswap(%rip),$Xn
	dec	$num
	jnz	.Loop_avx

	mov	`$REG_SZ*17+8`(%rsp),$num
	lea	$REG_SZ($ctx),$ctx
	lea	`16*$REG_SZ/4`($inp),$inp
	dec	$num
	jnz	.Loop_grande_avx

.Ldone_avx:
	mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
.cfi_def_cfa	%rax,8
	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xb8(%rax),%xmm6
	movaps	-0xa8(%rax),%xmm7
	movaps	-0x98(%rax),%xmm8
	movaps	-0x88(%rax),%xmm9
	movaps	-0x78(%rax),%xmm10
	movaps	-0x68(%rax),%xmm11
	movaps	-0x58(%rax),%xmm12
	movaps	-0x48(%rax),%xmm13
	movaps	-0x38(%rax),%xmm14
	movaps	-0x28(%rax),%xmm15
___
$code.=<<___;
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue_avx:
	ret
.cfi_endproc
.size	sha256_multi_block_avx,.-sha256_multi_block_avx
___
						if ($avx>1) {
$code =~ s/\`([^\`]*)\`/eval $1/gem;

$REG_SZ=32;
@ptr=map("%r$_",(12..15,8..11));

@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%ymm$_",(8..15));
($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%ymm$_",(0..7));

$code.=<<___;
.type	sha256_multi_block_avx2,\@function,3
.align	32
sha256_multi_block_avx2:
.cfi_startproc
_avx2_shortcut:
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,0x40(%rsp)
	movaps	%xmm11,0x50(%rsp)
	movaps	%xmm12,-0x78(%rax)
	movaps	%xmm13,-0x68(%rax)
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
___
$code.=<<___;
	sub	\$`$REG_SZ*18`, %rsp
	and	\$-256,%rsp
	mov	%rax,`$REG_SZ*17`(%rsp)		# original %rsp
.cfi_cfa_expression	%rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx2:
	lea	K256+128(%rip),$Tbl
	lea	0x80($ctx),$ctx			# size optimization

.Loop_grande_avx2:
	mov	$num,`$REG_SZ*17+8`(%rsp)	# original $num
	xor	$num,$num
	lea	`$REG_SZ*16`(%rsp),%rbx
___
for($i=0;$i<8;$i++) {
    $code.=<<___;
	mov	`16*$i+0`($inp),@ptr[$i]	# input pointer
	mov	`16*$i+8`($inp),%ecx		# number of blocks
	cmp	$num,%ecx
	cmovg	%ecx,$num			# find maximum
	test	%ecx,%ecx
	mov	%ecx,`4*$i`(%rbx)		# initialize counters
	cmovle	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	vmovdqu	0x00-0x80($ctx),$A		# load context
	 lea	128(%rsp),%rax
	vmovdqu	0x20-0x80($ctx),$B
	 lea	256+128(%rsp),%rbx
	vmovdqu	0x40-0x80($ctx),$C
	vmovdqu	0x60-0x80($ctx),$D
	vmovdqu	0x80-0x80($ctx),$E
	vmovdqu	0xa0-0x80($ctx),$F
	vmovdqu	0xc0-0x80($ctx),$G
	vmovdqu	0xe0-0x80($ctx),$H
	vmovdqu	.Lpbswap(%rip),$Xn
	jmp	.Loop_avx2

.align	32
.Loop_avx2:
	vpxor	$B,$C,$bxc			# magic seed
___
for($i=0;$i<16;$i++)	{ &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	vmovdqu	`&Xi_off($i)`,$Xi
	mov	\$3,%ecx
	jmp	.Loop_16_xx_avx2
.align	32
.Loop_16_xx_avx2:
___
for(;$i<32;$i++)	{ &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
	dec	%ecx
	jnz	.Loop_16_xx_avx2

	mov	\$1,%ecx
	lea	`$REG_SZ*16`(%rsp),%rbx
	lea	K256+128(%rip),$Tbl
___
for($i=0;$i<8;$i++) {
    $code.=<<___;
	cmp	`4*$i`(%rbx),%ecx		# examine counters
	cmovge	$Tbl,@ptr[$i]			# cancel input
___
}
$code.=<<___;
	vmovdqa	(%rbx),$sigma			# pull counters
	vpxor	$t1,$t1,$t1
	vmovdqa	$sigma,$Xn
	vpcmpgtd $t1,$Xn,$Xn			# mask value
	vpaddd	$Xn,$sigma,$sigma		# counters--

	vmovdqu	0x00-0x80($ctx),$t1
	vpand	$Xn,$A,$A
	vmovdqu	0x20-0x80($ctx),$t2
	vpand	$Xn,$B,$B
	vmovdqu	0x40-0x80($ctx),$t3
	vpand	$Xn,$C,$C
	vmovdqu	0x60-0x80($ctx),$Xi
	vpand	$Xn,$D,$D
	vpaddd	$t1,$A,$A
	vmovdqu	0x80-0x80($ctx),$t1
	vpand	$Xn,$E,$E
	vpaddd	$t2,$B,$B
	vmovdqu	0xa0-0x80($ctx),$t2
	vpand	$Xn,$F,$F
	vpaddd	$t3,$C,$C
	vmovdqu	0xc0-0x80($ctx),$t3
	vpand	$Xn,$G,$G
	vpaddd	$Xi,$D,$D
	vmovdqu	0xe0-0x80($ctx),$Xi
	vpand	$Xn,$H,$H
	vpaddd	$t1,$E,$E
	vpaddd	$t2,$F,$F
	vmovdqu	$A,0x00-0x80($ctx)
	vpaddd	$t3,$G,$G
	vmovdqu	$B,0x20-0x80($ctx)
	vpaddd	$Xi,$H,$H
	vmovdqu	$C,0x40-0x80($ctx)
	vmovdqu	$D,0x60-0x80($ctx)
	vmovdqu	$E,0x80-0x80($ctx)
	vmovdqu	$F,0xa0-0x80($ctx)
	vmovdqu	$G,0xc0-0x80($ctx)
	vmovdqu	$H,0xe0-0x80($ctx)

	vmovdqu	$sigma,(%rbx)			# save counters
	lea	256+128(%rsp),%rbx
	vmovdqu	.Lpbswap(%rip),$Xn
	dec	$num
	jnz	.Loop_avx2

	#mov	`$REG_SZ*17+8`(%rsp),$num
	#lea	$REG_SZ($ctx),$ctx
	#lea	`16*$REG_SZ/4`($inp),$inp
	#dec	$num
	#jnz	.Loop_grande_avx2

.Ldone_avx2:
	mov	`$REG_SZ*17`(%rsp),%rax		# original %rsp
.cfi_def_cfa	%rax,8
	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	movaps	-0x68(%rax),%xmm13
	movaps	-0x58(%rax),%xmm14
	movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lepilogue_avx2:
	ret
.cfi_endproc
.size	sha256_multi_block_avx2,.-sha256_multi_block_avx2
___
					}	}}}
$code.=<<___;
.align	256
K256:
___
sub TABLE {
    foreach (@_) {
	$code.=<<___;
	.long	$_,$_,$_,$_
	.long	$_,$_,$_,$_
___
    }
}
&TABLE(	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,
	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,
	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,
	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,
	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,
	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,
	0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,
	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,
	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 );
$code.=<<___;
.Lpbswap:
	.long	0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f	# pbswap
	.long	0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f	# pbswap
K256_shaext:
	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
	.asciz	"SHA256 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
___

if ($win64) {
# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
$rec="%rcx";
$frame="%rdx";
$context="%r8";
$disp="%r9";

$code.=<<___;
.extern	__imp_RtlVirtualUnwind
.type	se_handler,\@abi-omnipotent
.align	16
se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# end of prologue label
	cmp	%r10,%rbx		# context->Rip<.Lbody
	jb	.Lin_prologue

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
	jae	.Lin_prologue

	mov	`16*17`(%rax),%rax	# pull saved stack pointer

	mov	-8(%rax),%rbx
	mov	-16(%rax),%rbp
	mov	%rbx,144($context)	# restore context->Rbx
	mov	%rbp,160($context)	# restore context->Rbp

	lea	-24-10*16(%rax),%rsi
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$20,%ecx
	.long	0xa548f3fc		# cld; rep movsq

.Lin_prologue:
	mov	8(%rax),%rdi
	mov	16(%rax),%rsi
	mov	%rax,152($context)	# restore context->Rsp
	mov	%rsi,168($context)	# restore context->Rsi
	mov	%rdi,176($context)	# restore context->Rdi

	mov	40($disp),%rdi		# disp->ContextRecord
	mov	$context,%rsi		# context
	mov	\$154,%ecx		# sizeof(CONTEXT)
	.long	0xa548f3fc		# cld; rep movsq

	mov	$disp,%rsi
	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
	mov	0(%rsi),%r8		# arg3, disp->ControlPc
	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
	mov	40(%rsi),%r10		# disp->ContextRecord
	lea	56(%rsi),%r11		# &disp->HandlerData
	lea	24(%rsi),%r12		# &disp->EstablisherFrame
	mov	%r10,32(%rsp)		# arg5
	mov	%r11,40(%rsp)		# arg6
	mov	%r12,48(%rsp)		# arg7
	mov	%rcx,56(%rsp)		# arg8, (NULL)
	call	*__imp_RtlVirtualUnwind(%rip)

	mov	\$1,%eax		# ExceptionContinueSearch
	add	\$64,%rsp
	popfq
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx
	pop	%rdi
	pop	%rsi
	ret
.size	se_handler,.-se_handler
___
$code.=<<___ if ($avx>1);
.type	avx2_handler,\@abi-omnipotent
.align	16
avx2_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# end of prologue label
	cmp	%r10,%rbx		# context->Rip<body label
	jb	.Lin_prologue

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=epilogue label
	jae	.Lin_prologue

	mov	`32*17`($context),%rax	# pull saved stack pointer

	mov	-8(%rax),%rbx
	mov	-16(%rax),%rbp
	mov	-24(%rax),%r12
	mov	-32(%rax),%r13
	mov	-40(%rax),%r14
	mov	-48(%rax),%r15
	mov	%rbx,144($context)	# restore context->Rbx
	mov	%rbp,160($context)	# restore context->Rbp
	mov	%r12,216($context)	# restore context->R12
	mov	%r13,224($context)	# restore context->R13
	mov	%r14,232($context)	# restore context->R14
	mov	%r15,240($context)	# restore context->R15

	lea	-56-10*16(%rax),%rsi
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$20,%ecx
	.long	0xa548f3fc		# cld; rep movsq

	jmp	.Lin_prologue
.size	avx2_handler,.-avx2_handler
___
$code.=<<___;
.section	.pdata
.align	4
	.rva	.LSEH_begin_sha256_multi_block
	.rva	.LSEH_end_sha256_multi_block
	.rva	.LSEH_info_sha256_multi_block
	.rva	.LSEH_begin_sha256_multi_block_shaext
	.rva	.LSEH_end_sha256_multi_block_shaext
	.rva	.LSEH_info_sha256_multi_block_shaext
___
$code.=<<___ if ($avx);
	.rva	.LSEH_begin_sha256_multi_block_avx
	.rva	.LSEH_end_sha256_multi_block_avx
	.rva	.LSEH_info_sha256_multi_block_avx
___
$code.=<<___ if ($avx>1);
	.rva	.LSEH_begin_sha256_multi_block_avx2
	.rva	.LSEH_end_sha256_multi_block_avx2
	.rva	.LSEH_info_sha256_multi_block_avx2
___
$code.=<<___;
.section	.xdata
.align	8
.LSEH_info_sha256_multi_block:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lbody,.Lepilogue			# HandlerData[]
.LSEH_info_sha256_multi_block_shaext:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lbody_shaext,.Lepilogue_shaext		# HandlerData[]
___
$code.=<<___ if ($avx);
.LSEH_info_sha256_multi_block_avx:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lbody_avx,.Lepilogue_avx		# HandlerData[]
___
$code.=<<___ if ($avx>1);
.LSEH_info_sha256_multi_block_avx2:
	.byte	9,0,0,0
	.rva	avx2_handler
	.rva	.Lbody_avx2,.Lepilogue_avx2		# HandlerData[]
___
}
####################################################################

sub rex {
  local *opcode=shift;
  my ($dst,$src)=@_;
  my $rex=0;

    $rex|=0x04			if ($dst>=8);
    $rex|=0x01			if ($src>=8);
    unshift @opcode,$rex|0x40	if ($rex);
}

sub sha256op38 {
    my $instr = shift;
    my %opcodelet = (
		"sha256rnds2" => 0xcb,
  		"sha256msg1"  => 0xcc,
		"sha256msg2"  => 0xcd	);

    if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x0f,0x38);
	rex(\@opcode,$2,$1);
	push @opcode,$opcodelet{$instr};
	push @opcode,0xc0|($1&7)|(($2&7)<<3);		# ModR/M
	return ".byte\t".join(',',@opcode);
    } else {
	return $instr."\t".@_[0];
    }
}

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval($1)/ge;

	s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo		or

	s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go		or
	s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go		or
	s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go	or
	s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go	or
	s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go		or
	s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# SHA512 block transform for x86. September 2007.
#
# May 2013.
#
# Add SSSE3 code path, 20-25% improvement [over original SSE2 code].
#
# Performance in clock cycles per processed byte (less is better):
#
#		gcc	icc	x86 asm	SIMD(*)	x86_64(**)
# Pentium	100	97	61	-	-
# PIII		75	77	56	-	-
# P4		116	95	82	34.6	30.8
# AMD K8	54	55	36	20.7	9.57
# Core2		66	57	40	15.9	9.97
# Westmere	70	-	38	12.2	9.58
# Sandy Bridge	58	-	35	11.9	11.2
# Ivy Bridge	50	-	33	11.5	8.17
# Haswell	46	-	29	11.3	7.66
# Skylake	40	-	26	13.3	7.25
# Bulldozer	121	-	50	14.0	13.5
# VIA Nano	91	-	52	33	14.7
# Atom		126	-	68	48(***)	14.7
# Silvermont	97	-	58	42(***)	17.5
# Goldmont	80	-	48	19.5	12.0
#
# (*)	whichever best applicable.
# (**)	x86_64 assembler performance is presented for reference
#	purposes, the results are for integer-only code.
# (***)	paddq is incredibly slow on Atom.
#
# IALU code-path is optimized for elder Pentiums. On vanilla Pentium
# performance improvement over compiler generated code reaches ~60%,
# while on PIII - ~35%. On newer Âµ-archs improvement varies from 15%
# to 50%, but it's less important as they are expected to execute SSE2
# code-path, which is commonly ~2-3x faster [than compiler generated
# code]. SSE2 code-path is as fast as original sha512-sse2.pl, even
# though it does not use 128-bit operations. The latter means that
# SSE2-aware kernel is no longer required to execute the code. Another
# difference is that new code optimizes amount of writes, but at the
# cost of increased data cache "footprint" by 1/2KB.

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";

$output=pop;
open STDOUT,">$output";

&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");

$sse2=0;
for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }

&external_label("OPENSSL_ia32cap_P") if ($sse2);

$Tlo=&DWP(0,"esp");	$Thi=&DWP(4,"esp");
$Alo=&DWP(8,"esp");	$Ahi=&DWP(8+4,"esp");
$Blo=&DWP(16,"esp");	$Bhi=&DWP(16+4,"esp");
$Clo=&DWP(24,"esp");	$Chi=&DWP(24+4,"esp");
$Dlo=&DWP(32,"esp");	$Dhi=&DWP(32+4,"esp");
$Elo=&DWP(40,"esp");	$Ehi=&DWP(40+4,"esp");
$Flo=&DWP(48,"esp");	$Fhi=&DWP(48+4,"esp");
$Glo=&DWP(56,"esp");	$Ghi=&DWP(56+4,"esp");
$Hlo=&DWP(64,"esp");	$Hhi=&DWP(64+4,"esp");
$K512="ebp";

$Asse2=&QWP(0,"esp");
$Bsse2=&QWP(8,"esp");
$Csse2=&QWP(16,"esp");
$Dsse2=&QWP(24,"esp");
$Esse2=&QWP(32,"esp");
$Fsse2=&QWP(40,"esp");
$Gsse2=&QWP(48,"esp");
$Hsse2=&QWP(56,"esp");

$A="mm0";	# B-D and
$E="mm4";	# F-H are commonly loaded to respectively mm1-mm3 and
		# mm5-mm7, but it's done on on-demand basis...
$BxC="mm2";	# ... except for B^C

sub BODY_00_15_sse2 {
    my $phase=shift;

	#&movq	("mm5",$Fsse2);			# load f
	#&movq	("mm6",$Gsse2);			# load g

	&movq	("mm1",$E);			# %mm1 is sliding right
	 &pxor	("mm5","mm6");			# f^=g
	&psrlq	("mm1",14);
	 &movq	($Esse2,$E);			# modulo-scheduled save e
	 &pand	("mm5",$E);			# f&=e
	&psllq	($E,23);			# $E is sliding left
	 &movq	($A,"mm3")			if ($phase<2);
	 &movq	(&QWP(8*9,"esp"),"mm7")		# save X[i]
	&movq	("mm3","mm1");			# %mm3 is T1
	 &psrlq	("mm1",4);
	 &pxor	("mm5","mm6");			# Ch(e,f,g)
	&pxor	("mm3",$E);
	 &psllq	($E,23);
	&pxor	("mm3","mm1");
	 &movq	($Asse2,$A);			# modulo-scheduled save a
	 &paddq	("mm7","mm5");			# X[i]+=Ch(e,f,g)
	&pxor	("mm3",$E);
	 &psrlq	("mm1",23);
	 &paddq	("mm7",$Hsse2);			# X[i]+=h
	&pxor	("mm3","mm1");
	 &psllq	($E,4);
	 &paddq	("mm7",QWP(0,$K512));		# X[i]+=K512[i]
	&pxor	("mm3",$E);			# T1=Sigma1_512(e)

	 &movq	($E,$Dsse2);			# e = load d, e in next round
	&paddq	("mm3","mm7");			# T1+=X[i]
	 &movq	("mm5",$A);			# %mm5 is sliding right
	 &psrlq	("mm5",28);
	&paddq	($E,"mm3");			# d += T1
	 &movq	("mm6",$A);			# %mm6 is sliding left
	 &movq	("mm7","mm5");
	 &psllq	("mm6",25);
	&movq	("mm1",$Bsse2);			# load b
	 &psrlq	("mm5",6);
	 &pxor	("mm7","mm6");
	&sub	("esp",8);
	 &psllq	("mm6",5);
	 &pxor	("mm7","mm5");
	&pxor	($A,"mm1");			# a^b, b^c in next round
	 &psrlq	("mm5",5);
	 &pxor	("mm7","mm6");
	&pand	($BxC,$A);			# (b^c)&(a^b)
	 &psllq	("mm6",6);
	 &pxor	("mm7","mm5");
	&pxor	($BxC,"mm1");			# [h=]Maj(a,b,c)
	 &pxor	("mm6","mm7");			# Sigma0_512(a)
	 &movq	("mm7",&QWP(8*(9+16-1),"esp"))	if ($phase!=0);	# pre-fetch
	 &movq	("mm5",$Fsse2)			if ($phase==0);	# load f

    if ($phase>1) {
	&paddq	($BxC,"mm6");			# h+=Sigma0(a)
	 &add	($K512,8);
	#&paddq	($BxC,"mm3");			# h+=T1

	($A,$BxC) = ($BxC,$A);			# rotate registers
    } else {
	&paddq	("mm3",$BxC);			# T1+=Maj(a,b,c)
	 &movq	($BxC,$A);
	 &add	($K512,8);
	&paddq	("mm3","mm6");			# T1+=Sigma0(a)
	 &movq	("mm6",$Gsse2)			if ($phase==0);	# load g
	#&movq	($A,"mm3");			# h=T1
    }
}

sub BODY_00_15_x86 {
	#define Sigma1(x)	(ROTR((x),14) ^ ROTR((x),18)  ^ ROTR((x),41))
	#	LO		lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23
	#	HI		hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23
	&mov	("ecx",$Elo);
	&mov	("edx",$Ehi);
	&mov	("esi","ecx");

	&shr	("ecx",9);	# lo>>9
	&mov	("edi","edx");
	&shr	("edx",9);	# hi>>9
	&mov	("ebx","ecx");
	&shl	("esi",14);	# lo<<14
	&mov	("eax","edx");
	&shl	("edi",14);	# hi<<14
	&xor	("ebx","esi");

	&shr	("ecx",14-9);	# lo>>14
	&xor	("eax","edi");
	&shr	("edx",14-9);	# hi>>14
	&xor	("eax","ecx");
	&shl	("esi",18-14);	# lo<<18
	&xor	("ebx","edx");
	&shl	("edi",18-14);	# hi<<18
	&xor	("ebx","esi");

	&shr	("ecx",18-14);	# lo>>18
	&xor	("eax","edi");
	&shr	("edx",18-14);	# hi>>18
	&xor	("eax","ecx");
	&shl	("esi",23-18);	# lo<<23
	&xor	("ebx","edx");
	&shl	("edi",23-18);	# hi<<23
	&xor	("eax","esi");
	&xor	("ebx","edi");			# T1 = Sigma1(e)

	&mov	("ecx",$Flo);
	&mov	("edx",$Fhi);
	&mov	("esi",$Glo);
	&mov	("edi",$Ghi);
	 &add	("eax",$Hlo);
	 &adc	("ebx",$Hhi);			# T1 += h
	&xor	("ecx","esi");
	&xor	("edx","edi");
	&and	("ecx",$Elo);
	&and	("edx",$Ehi);
	 &add	("eax",&DWP(8*(9+15)+0,"esp"));
	 &adc	("ebx",&DWP(8*(9+15)+4,"esp"));	# T1 += X[0]
	&xor	("ecx","esi");
	&xor	("edx","edi");			# Ch(e,f,g) = (f^g)&e)^g

	&mov	("esi",&DWP(0,$K512));
	&mov	("edi",&DWP(4,$K512));		# K[i]
	&add	("eax","ecx");
	&adc	("ebx","edx");			# T1 += Ch(e,f,g)
	&mov	("ecx",$Dlo);
	&mov	("edx",$Dhi);
	&add	("eax","esi");
	&adc	("ebx","edi");			# T1 += K[i]
	&mov	($Tlo,"eax");
	&mov	($Thi,"ebx");			# put T1 away
	&add	("eax","ecx");
	&adc	("ebx","edx");			# d += T1

	#define Sigma0(x)	(ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39))
	#	LO		lo>>28^hi<<4  ^ hi>>2^lo<<30 ^ hi>>7^lo<<25
	#	HI		hi>>28^lo<<4  ^ lo>>2^hi<<30 ^ lo>>7^hi<<25
	&mov	("ecx",$Alo);
	&mov	("edx",$Ahi);
	&mov	($Dlo,"eax");
	&mov	($Dhi,"ebx");
	&mov	("esi","ecx");

	&shr	("ecx",2);	# lo>>2
	&mov	("edi","edx");
	&shr	("edx",2);	# hi>>2
	&mov	("ebx","ecx");
	&shl	("esi",4);	# lo<<4
	&mov	("eax","edx");
	&shl	("edi",4);	# hi<<4
	&xor	("ebx","esi");

	&shr	("ecx",7-2);	# lo>>7
	&xor	("eax","edi");
	&shr	("edx",7-2);	# hi>>7
	&xor	("ebx","ecx");
	&shl	("esi",25-4);	# lo<<25
	&xor	("eax","edx");
	&shl	("edi",25-4);	# hi<<25
	&xor	("eax","esi");

	&shr	("ecx",28-7);	# lo>>28
	&xor	("ebx","edi");
	&shr	("edx",28-7);	# hi>>28
	&xor	("eax","ecx");
	&shl	("esi",30-25);	# lo<<30
	&xor	("ebx","edx");
	&shl	("edi",30-25);	# hi<<30
	&xor	("eax","esi");
	&xor	("ebx","edi");			# Sigma0(a)

	&mov	("ecx",$Alo);
	&mov	("edx",$Ahi);
	&mov	("esi",$Blo);
	&mov	("edi",$Bhi);
	&add	("eax",$Tlo);
	&adc	("ebx",$Thi);			# T1 = Sigma0(a)+T1
	&or	("ecx","esi");
	&or	("edx","edi");
	&and	("ecx",$Clo);
	&and	("edx",$Chi);
	&and	("esi",$Alo);
	&and	("edi",$Ahi);
	&or	("ecx","esi");
	&or	("edx","edi");			# Maj(a,b,c) = ((a|b)&c)|(a&b)

	&add	("eax","ecx");
	&adc	("ebx","edx");			# T1 += Maj(a,b,c)
	&mov	($Tlo,"eax");
	&mov	($Thi,"ebx");

	&mov	(&LB("edx"),&BP(0,$K512));	# pre-fetch LSB of *K
	&sub	("esp",8);
	&lea	($K512,&DWP(8,$K512));		# K++
}

&function_begin("sha512_block_data_order");
	&mov	("esi",wparam(0));	# ctx
	&mov	("edi",wparam(1));	# inp
	&mov	("eax",wparam(2));	# num
	&mov	("ebx","esp");		# saved sp

	&call	(&label("pic_point"));	# make it PIC!
&set_label("pic_point");
	&blindpop($K512);
	&lea	($K512,&DWP(&label("K512")."-".&label("pic_point"),$K512));

	&sub	("esp",16);
	&and	("esp",-64);

	&shl	("eax",7);
	&add	("eax","edi");
	&mov	(&DWP(0,"esp"),"esi");	# ctx
	&mov	(&DWP(4,"esp"),"edi");	# inp
	&mov	(&DWP(8,"esp"),"eax");	# inp+num*128
	&mov	(&DWP(12,"esp"),"ebx");	# saved sp

if ($sse2) {
	&picmeup("edx","OPENSSL_ia32cap_P",$K512,&label("K512"));
	&mov	("ecx",&DWP(0,"edx"));
	&test	("ecx",1<<26);
	&jz	(&label("loop_x86"));

	&mov	("edx",&DWP(4,"edx"));

	# load ctx->h[0-7]
	&movq	($A,&QWP(0,"esi"));
	 &and	("ecx",1<<24);		# XMM registers availability
	&movq	("mm1",&QWP(8,"esi"));
	 &and	("edx",1<<9);		# SSSE3 bit
	&movq	($BxC,&QWP(16,"esi"));
	 &or	("ecx","edx");
	&movq	("mm3",&QWP(24,"esi"));
	&movq	($E,&QWP(32,"esi"));
	&movq	("mm5",&QWP(40,"esi"));
	&movq	("mm6",&QWP(48,"esi"));
	&movq	("mm7",&QWP(56,"esi"));
	&cmp	("ecx",1<<24|1<<9);
	&je	(&label("SSSE3"));
	&sub	("esp",8*10);
	&jmp	(&label("loop_sse2"));

&set_label("loop_sse2",16);
	#&movq	($Asse2,$A);
	&movq	($Bsse2,"mm1");
	&movq	($Csse2,$BxC);
	&movq	($Dsse2,"mm3");
	#&movq	($Esse2,$E);
	&movq	($Fsse2,"mm5");
	&movq	($Gsse2,"mm6");
	&pxor	($BxC,"mm1");			# magic
	&movq	($Hsse2,"mm7");
	&movq	("mm3",$A);			# magic

	&mov	("eax",&DWP(0,"edi"));
	&mov	("ebx",&DWP(4,"edi"));
	&add	("edi",8);
	&mov	("edx",15);			# counter
	&bswap	("eax");
	&bswap	("ebx");
	&jmp	(&label("00_14_sse2"));

&set_label("00_14_sse2",16);
	&movd	("mm1","eax");
	&mov	("eax",&DWP(0,"edi"));
	&movd	("mm7","ebx");
	&mov	("ebx",&DWP(4,"edi"));
	&add	("edi",8);
	&bswap	("eax");
	&bswap	("ebx");
	&punpckldq("mm7","mm1");

	&BODY_00_15_sse2();

	&dec	("edx");
	&jnz	(&label("00_14_sse2"));

	&movd	("mm1","eax");
	&movd	("mm7","ebx");
	&punpckldq("mm7","mm1");

	&BODY_00_15_sse2(1);

	&pxor	($A,$A);			# A is in %mm3
	&mov	("edx",32);			# counter
	&jmp	(&label("16_79_sse2"));

&set_label("16_79_sse2",16);
    for ($j=0;$j<2;$j++) {			# 2x unroll
	#&movq	("mm7",&QWP(8*(9+16-1),"esp"));	# prefetched in BODY_00_15
	&movq	("mm5",&QWP(8*(9+16-14),"esp"));
	&movq	("mm1","mm7");
	&psrlq	("mm7",1);
	 &movq	("mm6","mm5");
	 &psrlq	("mm5",6);
	&psllq	("mm1",56);
	 &paddq	($A,"mm3");			# from BODY_00_15
	 &movq	("mm3","mm7");
	&psrlq	("mm7",7-1);
	 &pxor	("mm3","mm1");
	 &psllq	("mm1",63-56);
	&pxor	("mm3","mm7");
	 &psrlq	("mm7",8-7);
	&pxor	("mm3","mm1");
	 &movq	("mm1","mm5");
	 &psrlq	("mm5",19-6);
	&pxor	("mm7","mm3");			# sigma0

	 &psllq	("mm6",3);
	 &pxor	("mm1","mm5");
	&paddq	("mm7",&QWP(8*(9+16),"esp"));
	 &pxor	("mm1","mm6");
	 &psrlq	("mm5",61-19);
	&paddq	("mm7",&QWP(8*(9+16-9),"esp"));
	 &pxor	("mm1","mm5");
	 &psllq	("mm6",45-3);
	&movq	("mm5",$Fsse2);			# load f
	 &pxor	("mm1","mm6");			# sigma1
	&movq	("mm6",$Gsse2);			# load g

	&paddq	("mm7","mm1");			# X[i]
	#&movq	(&QWP(8*9,"esp"),"mm7");	# moved to BODY_00_15

	&BODY_00_15_sse2(2);
    }
	&dec	("edx");
	&jnz	(&label("16_79_sse2"));

	#&movq	($A,$Asse2);
	&paddq	($A,"mm3");			# from BODY_00_15
	&movq	("mm1",$Bsse2);
	#&movq	($BxC,$Csse2);
	&movq	("mm3",$Dsse2);
	#&movq	($E,$Esse2);
	&movq	("mm5",$Fsse2);
	&movq	("mm6",$Gsse2);
	&movq	("mm7",$Hsse2);

	&pxor	($BxC,"mm1");			# de-magic
	&paddq	($A,&QWP(0,"esi"));
	&paddq	("mm1",&QWP(8,"esi"));
	&paddq	($BxC,&QWP(16,"esi"));
	&paddq	("mm3",&QWP(24,"esi"));
	&paddq	($E,&QWP(32,"esi"));
	&paddq	("mm5",&QWP(40,"esi"));
	&paddq	("mm6",&QWP(48,"esi"));
	&paddq	("mm7",&QWP(56,"esi"));

	&mov	("eax",8*80);
	&movq	(&QWP(0,"esi"),$A);
	&movq	(&QWP(8,"esi"),"mm1");
	&movq	(&QWP(16,"esi"),$BxC);
	&movq	(&QWP(24,"esi"),"mm3");
	&movq	(&QWP(32,"esi"),$E);
	&movq	(&QWP(40,"esi"),"mm5");
	&movq	(&QWP(48,"esi"),"mm6");
	&movq	(&QWP(56,"esi"),"mm7");

	&lea	("esp",&DWP(0,"esp","eax"));	# destroy frame
	&sub	($K512,"eax");			# rewind K

	&cmp	("edi",&DWP(8*10+8,"esp"));	# are we done yet?
	&jb	(&label("loop_sse2"));

	&mov	("esp",&DWP(8*10+12,"esp"));	# restore sp
	&emms	();
&function_end_A();

&set_label("SSSE3",32);
{ my ($cnt,$frame)=("ecx","edx");
  my @X=map("xmm$_",(0..7));
  my $j;
  my $i=0;

	&lea	($frame,&DWP(-64,"esp"));
	&sub	("esp",256);

	# fixed stack frame layout
	#
	# +0	A B C D E F G H		# backing store
	# +64	X[0]+K[i] .. X[15]+K[i]	# XMM->MM xfer area
	# +192				# XMM off-load ring buffer
	# +256				# saved parameters

	&movdqa		(@X[1],&QWP(80*8,$K512));		# byte swap mask
	&movdqu		(@X[0],&QWP(0,"edi"));
	&pshufb		(@X[0],@X[1]);
    for ($j=0;$j<8;$j++) {
	&movdqa		(&QWP(16*(($j-1)%4),$frame),@X[3])	if ($j>4); # off-load
	&movdqa		(@X[3],&QWP(16*($j%8),$K512));
	&movdqa		(@X[2],@X[1])				if ($j<7); # perpetuate byte swap mask
	&movdqu		(@X[1],&QWP(16*($j+1),"edi"))		if ($j<7); # next input
	&movdqa		(@X[1],&QWP(16*(($j+1)%4),$frame))	if ($j==7);# restore @X[0]
	&paddq		(@X[3],@X[0]);
	&pshufb		(@X[1],@X[2])				if ($j<7);
	&movdqa		(&QWP(16*($j%8)-128,$frame),@X[3]);	# xfer X[i]+K[i]

	push(@X,shift(@X));					# rotate(@X)
    }
	#&jmp		(&label("loop_ssse3"));
	&nop		();

&set_label("loop_ssse3",32);
	&movdqa		(@X[2],&QWP(16*(($j+1)%4),$frame));	# pre-restore @X[1]
	&movdqa		(&QWP(16*(($j-1)%4),$frame),@X[3]);	# off-load @X[3]
	&lea		($K512,&DWP(16*8,$K512));

	#&movq	($Asse2,$A);			# off-load A-H
	&movq	($Bsse2,"mm1");
	 &mov	("ebx","edi");
	&movq	($Csse2,$BxC);
	 &lea	("edi",&DWP(128,"edi"));	# advance input
	&movq	($Dsse2,"mm3");
	 &cmp	("edi","eax");
	#&movq	($Esse2,$E);
	&movq	($Fsse2,"mm5");
	 &cmovb	("ebx","edi");
	&movq	($Gsse2,"mm6");
	 &mov	("ecx",4);			# loop counter
	&pxor	($BxC,"mm1");			# magic
	&movq	($Hsse2,"mm7");
	&pxor	("mm3","mm3");			# magic

	&jmp		(&label("00_47_ssse3"));

sub BODY_00_15_ssse3 {		# "phase-less" copy of BODY_00_15_sse2
	(
	'&movq	("mm1",$E)',				# %mm1 is sliding right
	'&movq	("mm7",&QWP(((-8*$i)%128)-128,$frame))',# X[i]+K[i]
	 '&pxor	("mm5","mm6")',				# f^=g
	'&psrlq	("mm1",14)',
	 '&movq	(&QWP(8*($i+4)%64,"esp"),$E)',		# modulo-scheduled save e
	 '&pand	("mm5",$E)',				# f&=e
	'&psllq	($E,23)',				# $E is sliding left
	'&paddq	($A,"mm3")',				# [h+=Maj(a,b,c)]
	'&movq	("mm3","mm1")',				# %mm3 is T1
	 '&psrlq("mm1",4)',
	 '&pxor	("mm5","mm6")',				# Ch(e,f,g)
	'&pxor	("mm3",$E)',
	 '&psllq($E,23)',
	'&pxor	("mm3","mm1")',
	 '&movq	(&QWP(8*$i%64,"esp"),$A)',		# modulo-scheduled save a
	 '&paddq("mm7","mm5")',				# X[i]+=Ch(e,f,g)
	'&pxor	("mm3",$E)',
	 '&psrlq("mm1",23)',
	 '&paddq("mm7",&QWP(8*($i+7)%64,"esp"))',	# X[i]+=h
	'&pxor	("mm3","mm1")',
	 '&psllq($E,4)',
	'&pxor	("mm3",$E)',				# T1=Sigma1_512(e)

	 '&movq	($E,&QWP(8*($i+3)%64,"esp"))',		# e = load d, e in next round
	'&paddq	("mm3","mm7")',				# T1+=X[i]
	 '&movq	("mm5",$A)',				# %mm5 is sliding right
	 '&psrlq("mm5",28)',
	'&paddq	($E,"mm3")',				# d += T1
	 '&movq	("mm6",$A)',				# %mm6 is sliding left
	 '&movq	("mm7","mm5")',
	 '&psllq("mm6",25)',
	'&movq	("mm1",&QWP(8*($i+1)%64,"esp"))',	# load b
	 '&psrlq("mm5",6)',
	 '&pxor	("mm7","mm6")',
	 '&psllq("mm6",5)',
	 '&pxor	("mm7","mm5")',
	'&pxor	($A,"mm1")',				# a^b, b^c in next round
	 '&psrlq("mm5",5)',
	 '&pxor	("mm7","mm6")',
	'&pand	($BxC,$A)',				# (b^c)&(a^b)
	 '&psllq("mm6",6)',
	 '&pxor	("mm7","mm5")',
	'&pxor	($BxC,"mm1")',				# [h=]Maj(a,b,c)
	 '&pxor	("mm6","mm7")',				# Sigma0_512(a)
	 '&movq	("mm5",&QWP(8*($i+5-1)%64,"esp"))',	# pre-load f
	'&paddq	($BxC,"mm6")',				# h+=Sigma0(a)
	 '&movq	("mm6",&QWP(8*($i+6-1)%64,"esp"))',	# pre-load g

	'($A,$BxC) = ($BxC,$A); $i--;'
	);
}

&set_label("00_47_ssse3",32);

    for(;$j<16;$j++) {
	my ($t0,$t2,$t1)=@X[2..4];
	my @insns = (&BODY_00_15_ssse3(),&BODY_00_15_ssse3());

	&movdqa		($t2,@X[5]);
	&movdqa		(@X[1],$t0);			# restore @X[1]
	&palignr	($t0,@X[0],8);			# X[1..2]
	&movdqa		(&QWP(16*($j%4),$frame),@X[4]);	# off-load @X[4]
	 &palignr	($t2,@X[4],8);			# X[9..10]

	&movdqa		($t1,$t0);
	&psrlq		($t0,7);
	 &paddq		(@X[0],$t2);			# X[0..1] += X[9..10]
	&movdqa		($t2,$t1);
	&psrlq		($t1,1);
	&psllq		($t2,64-8);
	&pxor		($t0,$t1);
	&psrlq		($t1,8-1);
	&pxor		($t0,$t2);
	&psllq		($t2,8-1);
	&pxor		($t0,$t1);
	 &movdqa	($t1,@X[7]);
	&pxor		($t0,$t2);			# sigma0(X[1..2])
	 &movdqa	($t2,@X[7]);
	 &psrlq		($t1,6);
	&paddq		(@X[0],$t0);			# X[0..1] += sigma0(X[1..2])

	&movdqa		($t0,@X[7]);
	&psrlq		($t2,19);
	&psllq		($t0,64-61);
	&pxor		($t1,$t2);
	&psrlq		($t2,61-19);
	&pxor		($t1,$t0);
	&psllq		($t0,61-19);
	&pxor		($t1,$t2);
	&movdqa		($t2,&QWP(16*(($j+2)%4),$frame));# pre-restore @X[1]
	&pxor		($t1,$t0);			# sigma0(X[1..2])
	&movdqa		($t0,&QWP(16*($j%8),$K512));
	 eval(shift(@insns));
	&paddq		(@X[0],$t1);			# X[0..1] += sigma0(X[14..15])
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&paddq		($t0,@X[0]);
	 foreach(@insns) { eval; }
	&movdqa		(&QWP(16*($j%8)-128,$frame),$t0);# xfer X[i]+K[i]

	push(@X,shift(@X));				# rotate(@X)
    }
	&lea		($K512,&DWP(16*8,$K512));
	&dec		("ecx");
	&jnz		(&label("00_47_ssse3"));

	&movdqa		(@X[1],&QWP(0,$K512));		# byte swap mask
	&lea		($K512,&DWP(-80*8,$K512));	# rewind
	&movdqu		(@X[0],&QWP(0,"ebx"));
	&pshufb		(@X[0],@X[1]);

    for ($j=0;$j<8;$j++) {	# load next or same block
	my @insns = (&BODY_00_15_ssse3(),&BODY_00_15_ssse3());

	&movdqa		(&QWP(16*(($j-1)%4),$frame),@X[3])	if ($j>4); # off-load
	&movdqa		(@X[3],&QWP(16*($j%8),$K512));
	&movdqa		(@X[2],@X[1])				if ($j<7); # perpetuate byte swap mask
	&movdqu		(@X[1],&QWP(16*($j+1),"ebx"))		if ($j<7); # next input
	&movdqa		(@X[1],&QWP(16*(($j+1)%4),$frame))	if ($j==7);# restore @X[0]
	&paddq		(@X[3],@X[0]);
	&pshufb		(@X[1],@X[2])				if ($j<7);
	 foreach(@insns) { eval; }
	&movdqa		(&QWP(16*($j%8)-128,$frame),@X[3]);# xfer X[i]+K[i]

	push(@X,shift(@X));				# rotate(@X)
    }

	#&movq	($A,$Asse2);			# load A-H
	&movq	("mm1",$Bsse2);
	&paddq	($A,"mm3");			# from BODY_00_15
	#&movq	($BxC,$Csse2);
	&movq	("mm3",$Dsse2);
	#&movq	($E,$Esse2);
	#&movq	("mm5",$Fsse2);
	#&movq	("mm6",$Gsse2);
	&movq	("mm7",$Hsse2);

	&pxor	($BxC,"mm1");			# de-magic
	&paddq	($A,&QWP(0,"esi"));
	&paddq	("mm1",&QWP(8,"esi"));
	&paddq	($BxC,&QWP(16,"esi"));
	&paddq	("mm3",&QWP(24,"esi"));
	&paddq	($E,&QWP(32,"esi"));
	&paddq	("mm5",&QWP(40,"esi"));
	&paddq	("mm6",&QWP(48,"esi"));
	&paddq	("mm7",&QWP(56,"esi"));

	&movq	(&QWP(0,"esi"),$A);
	&movq	(&QWP(8,"esi"),"mm1");
	&movq	(&QWP(16,"esi"),$BxC);
	&movq	(&QWP(24,"esi"),"mm3");
	&movq	(&QWP(32,"esi"),$E);
	&movq	(&QWP(40,"esi"),"mm5");
	&movq	(&QWP(48,"esi"),"mm6");
	&movq	(&QWP(56,"esi"),"mm7");

    	&cmp	("edi","eax")			# are we done yet?
	&jb	(&label("loop_ssse3"));

	&mov	("esp",&DWP(64+12,$frame));	# restore sp
	&emms	();
}
&function_end_A();
}
&set_label("loop_x86",16);
    # copy input block to stack reversing byte and qword order
    for ($i=0;$i<8;$i++) {
	&mov	("eax",&DWP($i*16+0,"edi"));
	&mov	("ebx",&DWP($i*16+4,"edi"));
	&mov	("ecx",&DWP($i*16+8,"edi"));
	&mov	("edx",&DWP($i*16+12,"edi"));
	&bswap	("eax");
	&bswap	("ebx");
	&bswap	("ecx");
	&bswap	("edx");
	&push	("eax");
	&push	("ebx");
	&push	("ecx");
	&push	("edx");
    }
	&add	("edi",128);
	&sub	("esp",9*8);		# place for T,A,B,C,D,E,F,G,H
	&mov	(&DWP(8*(9+16)+4,"esp"),"edi");

	# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
	&lea	("edi",&DWP(8,"esp"));
	&mov	("ecx",16);
	&data_word(0xA5F3F689);		# rep movsd

&set_label("00_15_x86",16);
	&BODY_00_15_x86();

	&cmp	(&LB("edx"),0x94);
	&jne	(&label("00_15_x86"));

&set_label("16_79_x86",16);
	#define sigma0(x)	(ROTR((x),1)  ^ ROTR((x),8)  ^ ((x)>>7))
	#	LO		lo>>1^hi<<31  ^ lo>>8^hi<<24 ^ lo>>7^hi<<25
	#	HI		hi>>1^lo<<31  ^ hi>>8^lo<<24 ^ hi>>7
	&mov	("ecx",&DWP(8*(9+15+16-1)+0,"esp"));
	&mov	("edx",&DWP(8*(9+15+16-1)+4,"esp"));
	&mov	("esi","ecx");

	&shr	("ecx",1);	# lo>>1
	&mov	("edi","edx");
	&shr	("edx",1);	# hi>>1
	&mov	("eax","ecx");
	&shl	("esi",24);	# lo<<24
	&mov	("ebx","edx");
	&shl	("edi",24);	# hi<<24
	&xor	("ebx","esi");

	&shr	("ecx",7-1);	# lo>>7
	&xor	("eax","edi");
	&shr	("edx",7-1);	# hi>>7
	&xor	("eax","ecx");
	&shl	("esi",31-24);	# lo<<31
	&xor	("ebx","edx");
	&shl	("edi",25-24);	# hi<<25
	&xor	("ebx","esi");

	&shr	("ecx",8-7);	# lo>>8
	&xor	("eax","edi");
	&shr	("edx",8-7);	# hi>>8
	&xor	("eax","ecx");
	&shl	("edi",31-25);	# hi<<31
	&xor	("ebx","edx");
	&xor	("eax","edi");			# T1 = sigma0(X[-15])

	&mov	(&DWP(0,"esp"),"eax");
	&mov	(&DWP(4,"esp"),"ebx");		# put T1 away

	#define sigma1(x)	(ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6))
	#	LO		lo>>19^hi<<13 ^ hi>>29^lo<<3 ^ lo>>6^hi<<26
	#	HI		hi>>19^lo<<13 ^ lo>>29^hi<<3 ^ hi>>6
	&mov	("ecx",&DWP(8*(9+15+16-14)+0,"esp"));
	&mov	("edx",&DWP(8*(9+15+16-14)+4,"esp"));
	&mov	("esi","ecx");

	&shr	("ecx",6);	# lo>>6
	&mov	("edi","edx");
	&shr	("edx",6);	# hi>>6
	&mov	("eax","ecx");
	&shl	("esi",3);	# lo<<3
	&mov	("ebx","edx");
	&shl	("edi",3);	# hi<<3
	&xor	("eax","esi");

	&shr	("ecx",19-6);	# lo>>19
	&xor	("ebx","edi");
	&shr	("edx",19-6);	# hi>>19
	&xor	("eax","ecx");
	&shl	("esi",13-3);	# lo<<13
	&xor	("ebx","edx");
	&shl	("edi",13-3);	# hi<<13
	&xor	("ebx","esi");

	&shr	("ecx",29-19);	# lo>>29
	&xor	("eax","edi");
	&shr	("edx",29-19);	# hi>>29
	&xor	("ebx","ecx");
	&shl	("edi",26-13);	# hi<<26
	&xor	("eax","edx");
	&xor	("eax","edi");			# sigma1(X[-2])

	&mov	("ecx",&DWP(8*(9+15+16)+0,"esp"));
	&mov	("edx",&DWP(8*(9+15+16)+4,"esp"));
	&add	("eax",&DWP(0,"esp"));
	&adc	("ebx",&DWP(4,"esp"));		# T1 = sigma1(X[-2])+T1
	&mov	("esi",&DWP(8*(9+15+16-9)+0,"esp"));
	&mov	("edi",&DWP(8*(9+15+16-9)+4,"esp"));
	&add	("eax","ecx");
	&adc	("ebx","edx");			# T1 += X[-16]
	&add	("eax","esi");
	&adc	("ebx","edi");			# T1 += X[-7]
	&mov	(&DWP(8*(9+15)+0,"esp"),"eax");
	&mov	(&DWP(8*(9+15)+4,"esp"),"ebx");	# save X[0]

	&BODY_00_15_x86();

	&cmp	(&LB("edx"),0x17);
	&jne	(&label("16_79_x86"));

	&mov	("esi",&DWP(8*(9+16+80)+0,"esp"));# ctx
	&mov	("edi",&DWP(8*(9+16+80)+4,"esp"));# inp
    for($i=0;$i<4;$i++) {
	&mov	("eax",&DWP($i*16+0,"esi"));
	&mov	("ebx",&DWP($i*16+4,"esi"));
	&mov	("ecx",&DWP($i*16+8,"esi"));
	&mov	("edx",&DWP($i*16+12,"esi"));
	&add	("eax",&DWP(8+($i*16)+0,"esp"));
	&adc	("ebx",&DWP(8+($i*16)+4,"esp"));
	&mov	(&DWP($i*16+0,"esi"),"eax");
	&mov	(&DWP($i*16+4,"esi"),"ebx");
	&add	("ecx",&DWP(8+($i*16)+8,"esp"));
	&adc	("edx",&DWP(8+($i*16)+12,"esp"));
	&mov	(&DWP($i*16+8,"esi"),"ecx");
	&mov	(&DWP($i*16+12,"esi"),"edx");
    }
	&add	("esp",8*(9+16+80));		# destroy frame
	&sub	($K512,8*80);			# rewind K

	&cmp	("edi",&DWP(8,"esp"));		# are we done yet?
	&jb	(&label("loop_x86"));

	&mov	("esp",&DWP(12,"esp"));		# restore sp
&function_end_A();

&set_label("K512",64);	# Yes! I keep it in the code segment!
	&data_word(0xd728ae22,0x428a2f98);	# u64
	&data_word(0x23ef65cd,0x71374491);	# u64
	&data_word(0xec4d3b2f,0xb5c0fbcf);	# u64
	&data_word(0x8189dbbc,0xe9b5dba5);	# u64
	&data_word(0xf348b538,0x3956c25b);	# u64
	&data_word(0xb605d019,0x59f111f1);	# u64
	&data_word(0xaf194f9b,0x923f82a4);	# u64
	&data_word(0xda6d8118,0xab1c5ed5);	# u64
	&data_word(0xa3030242,0xd807aa98);	# u64
	&data_word(0x45706fbe,0x12835b01);	# u64
	&data_word(0x4ee4b28c,0x243185be);	# u64
	&data_word(0xd5ffb4e2,0x550c7dc3);	# u64
	&data_word(0xf27b896f,0x72be5d74);	# u64
	&data_word(0x3b1696b1,0x80deb1fe);	# u64
	&data_word(0x25c71235,0x9bdc06a7);	# u64
	&data_word(0xcf692694,0xc19bf174);	# u64
	&data_word(0x9ef14ad2,0xe49b69c1);	# u64
	&data_word(0x384f25e3,0xefbe4786);	# u64
	&data_word(0x8b8cd5b5,0x0fc19dc6);	# u64
	&data_word(0x77ac9c65,0x240ca1cc);	# u64
	&data_word(0x592b0275,0x2de92c6f);	# u64
	&data_word(0x6ea6e483,0x4a7484aa);	# u64
	&data_word(0xbd41fbd4,0x5cb0a9dc);	# u64
	&data_word(0x831153b5,0x76f988da);	# u64
	&data_word(0xee66dfab,0x983e5152);	# u64
	&data_word(0x2db43210,0xa831c66d);	# u64
	&data_word(0x98fb213f,0xb00327c8);	# u64
	&data_word(0xbeef0ee4,0xbf597fc7);	# u64
	&data_word(0x3da88fc2,0xc6e00bf3);	# u64
	&data_word(0x930aa725,0xd5a79147);	# u64
	&data_word(0xe003826f,0x06ca6351);	# u64
	&data_word(0x0a0e6e70,0x14292967);	# u64
	&data_word(0x46d22ffc,0x27b70a85);	# u64
	&data_word(0x5c26c926,0x2e1b2138);	# u64
	&data_word(0x5ac42aed,0x4d2c6dfc);	# u64
	&data_word(0x9d95b3df,0x53380d13);	# u64
	&data_word(0x8baf63de,0x650a7354);	# u64
	&data_word(0x3c77b2a8,0x766a0abb);	# u64
	&data_word(0x47edaee6,0x81c2c92e);	# u64
	&data_word(0x1482353b,0x92722c85);	# u64
	&data_word(0x4cf10364,0xa2bfe8a1);	# u64
	&data_word(0xbc423001,0xa81a664b);	# u64
	&data_word(0xd0f89791,0xc24b8b70);	# u64
	&data_word(0x0654be30,0xc76c51a3);	# u64
	&data_word(0xd6ef5218,0xd192e819);	# u64
	&data_word(0x5565a910,0xd6990624);	# u64
	&data_word(0x5771202a,0xf40e3585);	# u64
	&data_word(0x32bbd1b8,0x106aa070);	# u64
	&data_word(0xb8d2d0c8,0x19a4c116);	# u64
	&data_word(0x5141ab53,0x1e376c08);	# u64
	&data_word(0xdf8eeb99,0x2748774c);	# u64
	&data_word(0xe19b48a8,0x34b0bcb5);	# u64
	&data_word(0xc5c95a63,0x391c0cb3);	# u64
	&data_word(0xe3418acb,0x4ed8aa4a);	# u64
	&data_word(0x7763e373,0x5b9cca4f);	# u64
	&data_word(0xd6b2b8a3,0x682e6ff3);	# u64
	&data_word(0x5defb2fc,0x748f82ee);	# u64
	&data_word(0x43172f60,0x78a5636f);	# u64
	&data_word(0xa1f0ab72,0x84c87814);	# u64
	&data_word(0x1a6439ec,0x8cc70208);	# u64
	&data_word(0x23631e28,0x90befffa);	# u64
	&data_word(0xde82bde9,0xa4506ceb);	# u64
	&data_word(0xb2c67915,0xbef9a3f7);	# u64
	&data_word(0xe372532b,0xc67178f2);	# u64
	&data_word(0xea26619c,0xca273ece);	# u64
	&data_word(0x21c0c207,0xd186b8c7);	# u64
	&data_word(0xcde0eb1e,0xeada7dd6);	# u64
	&data_word(0xee6ed178,0xf57d4f7f);	# u64
	&data_word(0x72176fba,0x06f067aa);	# u64
	&data_word(0xa2c898a6,0x0a637dc5);	# u64
	&data_word(0xbef90dae,0x113f9804);	# u64
	&data_word(0x131c471b,0x1b710b35);	# u64
	&data_word(0x23047d84,0x28db77f5);	# u64
	&data_word(0x40c72493,0x32caab7b);	# u64
	&data_word(0x15c9bebc,0x3c9ebe0a);	# u64
	&data_word(0x9c100d4c,0x431d67c4);	# u64
	&data_word(0xcb3e42b6,0x4cc5d4be);	# u64
	&data_word(0xfc657e2a,0x597f299c);	# u64
	&data_word(0x3ad6faec,0x5fcb6fab);	# u64
	&data_word(0x4a475817,0x6c44198c);	# u64

	&data_word(0x04050607,0x00010203);	# byte swap
	&data_word(0x0c0d0e0f,0x08090a0b);	# mask
&function_end_B("sha512_block_data_order");
&asciz("SHA512 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>");

&asm_finish();

close STDOUT or die "error closing STDOUT: $!";