#!/usr/bin/env perl
# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for x86_64.
#
# June 2017.
#
# Below code is [lane complementing] KECCAK_2X implementation (see
# sha/keccak1600.c) with C[5] and D[5] held in register bank. Though
# instead of actually unrolling the loop pair-wise I simply flip
# pointers to T[][] and A[][] at the end of round. Since number of
# rounds is even, last round writes to A[][] and everything works out.
# How does it compare to x86_64 assembly module in Keccak Code Package?
# Depending on processor it's either as fast or faster by up to 15%...
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
#			r=1088(*)
#
# P4			25.8
# Core 2		12.9
# Westmere		13.7
# Sandy Bridge		12.9(**)
# Haswell		9.6
# Skylake		9.4
# Silvermont		22.8
# Goldmont		15.8
# VIA Nano		17.3
# Sledgehammer		13.3
# Bulldozer		16.5
# Ryzen			8.8
#
# (*)	Corresponds to SHA3-256. Improvement over compiler-generate
#	varies a lot, most common coefficient is 15% in comparison to
#	gcc-5.x, 50% for gcc-4.x, 90% for gcc-3.x.
# (**)	Sandy Bridge has broken rotate instruction. Performance can be
#	improved by 14% by replacing rotates with double-precision
#	shift with same register as source and destination.

$flavour = shift;
$output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

my @A = map([ 8*$_-100, 8*($_+1)-100, 8*($_+2)-100,
              8*($_+3)-100, 8*($_+4)-100 ], (0,5,10,15,20));

my @C = ("%rax","%rbx","%rcx","%rdx","%rbp");
my @D = map("%r$_",(8..12));
my @T = map("%r$_",(13..14));
my $iotas = "%r15";

my @rhotates = ([  0,  1, 62, 28, 27 ],
                [ 36, 44,  6, 55, 20 ],
                [  3, 10, 43, 25, 39 ],
                [ 41, 45, 15, 21,  8 ],
                [ 18,  2, 61, 56, 14 ]);

$code.=<<___;
.text

.type	__KeccakF1600,\@abi-omnipotent
.align	32
__KeccakF1600:
.cfi_startproc
	mov	$A[4][0](%rdi),@C[0]
	mov	$A[4][1](%rdi),@C[1]
	mov	$A[4][2](%rdi),@C[2]
	mov	$A[4][3](%rdi),@C[3]
	mov	$A[4][4](%rdi),@C[4]
	jmp	.Loop

.align	32
.Loop:
	mov	$A[0][0](%rdi),@D[0]
	mov	$A[1][1](%rdi),@D[1]
	mov	$A[2][2](%rdi),@D[2]
	mov	$A[3][3](%rdi),@D[3]

	xor	$A[0][2](%rdi),@C[2]
	xor	$A[0][3](%rdi),@C[3]
	xor	@D[0],         @C[0]
	xor	$A[0][1](%rdi),@C[1]
	 xor	$A[1][2](%rdi),@C[2]
	 xor	$A[1][0](%rdi),@C[0]
	mov	@C[4],@D[4]
	xor	$A[0][4](%rdi),@C[4]

	xor	@D[2],         @C[2]
	xor	$A[2][0](%rdi),@C[0]
	 xor	$A[1][3](%rdi),@C[3]
	 xor	@D[1],         @C[1]
	 xor	$A[1][4](%rdi),@C[4]

	xor	$A[3][2](%rdi),@C[2]
	xor	$A[3][0](%rdi),@C[0]
	 xor	$A[2][3](%rdi),@C[3]
	 xor	$A[2][1](%rdi),@C[1]
	 xor	$A[2][4](%rdi),@C[4]

	mov	@C[2],@T[0]
	rol	\$1,@C[2]
	xor	@C[0],@C[2]		# D[1] = ROL64(C[2], 1) ^ C[0]
	 xor	@D[3],         @C[3]

	rol	\$1,@C[0]
	xor	@C[3],@C[0]		# D[4] = ROL64(C[0], 1) ^ C[3]
	 xor	$A[3][1](%rdi),@C[1]

	rol	\$1,@C[3]
	xor	@C[1],@C[3]		# D[2] = ROL64(C[3], 1) ^ C[1]
	 xor	$A[3][4](%rdi),@C[4]

	rol	\$1,@C[1]
	xor	@C[4],@C[1]		# D[0] = ROL64(C[1], 1) ^ C[4]

	rol	\$1,@C[4]
	xor	@T[0],@C[4]		# D[3] = ROL64(C[4], 1) ^ C[2]
___
	(@D[0..4], @C) = (@C[1..4,0], @D);
$code.=<<___;
	xor	@D[1],@C[1]
	xor	@D[2],@C[2]
	rol	\$$rhotates[1][1],@C[1]
	xor	@D[3],@C[3]
	xor	@D[4],@C[4]
	rol	\$$rhotates[2][2],@C[2]
	xor	@D[0],@C[0]
	 mov	@C[1],@T[0]
	rol	\$$rhotates[3][3],@C[3]
	 or	@C[2],@C[1]
	 xor	@C[0],@C[1]		#           C[0] ^ ( C[1] | C[2])
	rol	\$$rhotates[4][4],@C[4]

	 xor	($iotas),@C[1]
	 lea	8($iotas),$iotas

	mov	@C[4],@T[1]
	and	@C[3],@C[4]
	 mov	@C[1],$A[0][0](%rsi)	# R[0][0] = C[0] ^ ( C[1] | C[2]) ^ iotas[i]
	xor	@C[2],@C[4]		#           C[2] ^ ( C[4] & C[3])
	not	@C[2]
	mov	@C[4],$A[0][2](%rsi)	# R[0][2] = C[2] ^ ( C[4] & C[3])

	or	@C[3],@C[2]
	  mov	$A[4][2](%rdi),@C[4]
	xor	@T[0],@C[2]		#           C[1] ^ (~C[2] | C[3])
	mov	@C[2],$A[0][1](%rsi)	# R[0][1] = C[1] ^ (~C[2] | C[3])

	and	@C[0],@T[0]
	  mov	$A[1][4](%rdi),@C[1]
	xor	@T[1],@T[0]		#           C[4] ^ ( C[1] & C[0])
	  mov	$A[2][0](%rdi),@C[2]
	mov	@T[0],$A[0][4](%rsi)	# R[0][4] = C[4] ^ ( C[1] & C[0])

	or	@C[0],@T[1]
	  mov	$A[0][3](%rdi),@C[0]
	xor	@C[3],@T[1]		#           C[3] ^ ( C[4] | C[0])
	  mov	$A[3][1](%rdi),@C[3]
	mov	@T[1],$A[0][3](%rsi)	# R[0][3] = C[3] ^ ( C[4] | C[0])

	xor	@D[3],@C[0]
	xor	@D[2],@C[4]
	rol	\$$rhotates[0][3],@C[0]
	xor	@D[1],@C[3]
	xor	@D[4],@C[1]
	rol	\$$rhotates[4][2],@C[4]
	rol	\$$rhotates[3][1],@C[3]
	xor	@D[0],@C[2]
	rol	\$$rhotates[1][4],@C[1]
	 mov	@C[0],@T[0]
	 or	@C[4],@C[0]
	rol	\$$rhotates[2][0],@C[2]

	xor	@C[3],@C[0]		#           C[3] ^ (C[0] |  C[4])
	mov	@C[0],$A[1][3](%rsi)	# R[1][3] = C[3] ^ (C[0] |  C[4])

	mov	@C[1],@T[1]
	and	@T[0],@C[1]
	  mov	$A[0][1](%rdi),@C[0]
	xor	@C[4],@C[1]		#           C[4] ^ (C[1] &  C[0])
	not	@C[4]
	mov	@C[1],$A[1][4](%rsi)	# R[1][4] = C[4] ^ (C[1] &  C[0])

	or	@C[3],@C[4]
	  mov	$A[1][2](%rdi),@C[1]
	xor	@C[2],@C[4]		#           C[2] ^ (~C[4] | C[3])
	mov	@C[4],$A[1][2](%rsi)	# R[1][2] = C[2] ^ (~C[4] | C[3])

	and	@C[2],@C[3]
	  mov	$A[4][0](%rdi),@C[4]
	xor	@T[1],@C[3]		#           C[1] ^ (C[3] &  C[2])
	mov	@C[3],$A[1][1](%rsi)	# R[1][1] = C[1] ^ (C[3] &  C[2])

	or	@C[2],@T[1]
	  mov	$A[2][3](%rdi),@C[2]
	xor	@T[0],@T[1]		#           C[0] ^ (C[1] |  C[2])
	  mov	$A[3][4](%rdi),@C[3]
	mov	@T[1],$A[1][0](%rsi)	# R[1][0] = C[0] ^ (C[1] |  C[2])

	xor	@D[3],@C[2]
	xor	@D[4],@C[3]
	rol	\$$rhotates[2][3],@C[2]
	xor	@D[2],@C[1]
	rol	\$$rhotates[3][4],@C[3]
	xor	@D[0],@C[4]
	rol	\$$rhotates[1][2],@C[1]
	xor	@D[1],@C[0]
	rol	\$$rhotates[4][0],@C[4]
	 mov	@C[2],@T[0]
	 and	@C[3],@C[2]
	rol	\$$rhotates[0][1],@C[0]

	not	@C[3]
	xor	@C[1],@C[2]		#            C[1] ^ ( C[2] & C[3])
	mov	@C[2],$A[2][1](%rsi)	# R[2][1] =  C[1] ^ ( C[2] & C[3])

	mov	@C[4],@T[1]
	and	@C[3],@C[4]
	  mov	$A[2][1](%rdi),@C[2]
	xor	@T[0],@C[4]		#            C[2] ^ ( C[4] & ~C[3])
	mov	@C[4],$A[2][2](%rsi)	# R[2][2] =  C[2] ^ ( C[4] & ~C[3])

	or	@C[1],@T[0]
	  mov	$A[4][3](%rdi),@C[4]
	xor	@C[0],@T[0]		#            C[0] ^ ( C[2] | C[1])
	mov	@T[0],$A[2][0](%rsi)	# R[2][0] =  C[0] ^ ( C[2] | C[1])

	and	@C[0],@C[1]
	xor	@T[1],@C[1]		#            C[4] ^ ( C[1] & C[0])
	mov	@C[1],$A[2][4](%rsi)	# R[2][4] =  C[4] ^ ( C[1] & C[0])

	or	@C[0],@T[1]
	  mov	$A[1][0](%rdi),@C[1]
	xor	@C[3],@T[1]		#           ~C[3] ^ ( C[0] | C[4])
	  mov	$A[3][2](%rdi),@C[3]
	mov	@T[1],$A[2][3](%rsi)	# R[2][3] = ~C[3] ^ ( C[0] | C[4])

	mov	$A[0][4](%rdi),@C[0]

	xor	@D[1],@C[2]
	xor	@D[2],@C[3]
	rol	\$$rhotates[2][1],@C[2]
	xor	@D[0],@C[1]
	rol	\$$rhotates[3][2],@C[3]
	xor	@D[3],@C[4]
	rol	\$$rhotates[1][0],@C[1]
	xor	@D[4],@C[0]
	rol	\$$rhotates[4][3],@C[4]
	 mov	@C[2],@T[0]
	 or	@C[3],@C[2]
	rol	\$$rhotates[0][4],@C[0]

	not	@C[3]
	xor	@C[1],@C[2]		#            C[1] ^ ( C[2] | C[3])
	mov	@C[2],$A[3][1](%rsi)	# R[3][1] =  C[1] ^ ( C[2] | C[3])

	mov	@C[4],@T[1]
	or	@C[3],@C[4]
	xor	@T[0],@C[4]		#            C[2] ^ ( C[4] | ~C[3])
	mov	@C[4],$A[3][2](%rsi)	# R[3][2] =  C[2] ^ ( C[4] | ~C[3])

	and	@C[1],@T[0]
	xor	@C[0],@T[0]		#            C[0] ^ ( C[2] & C[1])
	mov	@T[0],$A[3][0](%rsi)	# R[3][0] =  C[0] ^ ( C[2] & C[1])

	or	@C[0],@C[1]
	xor	@T[1],@C[1]		#            C[4] ^ ( C[1] | C[0])
	mov	@C[1],$A[3][4](%rsi)	# R[3][4] =  C[4] ^ ( C[1] | C[0])

	and	@T[1],@C[0]
	xor	@C[3],@C[0]		#           ~C[3] ^ ( C[0] & C[4])
	mov	@C[0],$A[3][3](%rsi)	# R[3][3] = ~C[3] ^ ( C[0] & C[4])

	xor	$A[0][2](%rdi),@D[2]
	xor	$A[1][3](%rdi),@D[3]
	rol	\$$rhotates[0][2],@D[2]
	xor	$A[4][1](%rdi),@D[1]
	rol	\$$rhotates[1][3],@D[3]
	xor	$A[2][4](%rdi),@D[4]
	rol	\$$rhotates[4][1],@D[1]
	xor	$A[3][0](%rdi),@D[0]
	xchg	%rsi,%rdi
	rol	\$$rhotates[2][4],@D[4]
	rol	\$$rhotates[3][0],@D[0]
___
	@C = @D[2..4,0,1];
$code.=<<___;
	mov	@C[0],@T[0]
	and	@C[1],@C[0]
	not	@C[1]
	xor	@C[4],@C[0]		#            C[4] ^ ( C[0] & C[1])
	mov	@C[0],$A[4][4](%rdi)	# R[4][4] =  C[4] ^ ( C[0] & C[1])

	mov	@C[2],@T[1]
	and	@C[1],@C[2]
	xor	@T[0],@C[2]		#            C[0] ^ ( C[2] & ~C[1])
	mov	@C[2],$A[4][0](%rdi)	# R[4][0] =  C[0] ^ ( C[2] & ~C[1])

	or	@C[4],@T[0]
	xor	@C[3],@T[0]		#            C[3] ^ ( C[0] | C[4])
	mov	@T[0],$A[4][3](%rdi)	# R[4][3] =  C[3] ^ ( C[0] | C[4])

	and	@C[3],@C[4]
	xor	@T[1],@C[4]		#            C[2] ^ ( C[4] & C[3])
	mov	@C[4],$A[4][2](%rdi)	# R[4][2] =  C[2] ^ ( C[4] & C[3])

	or	@T[1],@C[3]
	xor	@C[1],@C[3]		#           ~C[1] ^ ( C[2] | C[3])
	mov	@C[3],$A[4][1](%rdi)	# R[4][1] = ~C[1] ^ ( C[2] | C[3])

	mov	@C[0],@C[1]		# harmonize with the loop top
	mov	@T[0],@C[0]

	test	\$255,$iotas
	jnz	.Loop

	lea	-192($iotas),$iotas	# rewind iotas
	ret
.cfi_endproc
.size	__KeccakF1600,.-__KeccakF1600

.type	KeccakF1600,\@abi-omnipotent
.align	32
KeccakF1600:
.cfi_startproc
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15

	lea	100(%rdi),%rdi		# size optimization
	sub	\$200,%rsp
.cfi_adjust_cfa_offset	200

	notq	$A[0][1](%rdi)
	notq	$A[0][2](%rdi)
	notq	$A[1][3](%rdi)
	notq	$A[2][2](%rdi)
	notq	$A[3][2](%rdi)
	notq	$A[4][0](%rdi)

	lea	iotas(%rip),$iotas
	lea	100(%rsp),%rsi		# size optimization

	call	__KeccakF1600

	notq	$A[0][1](%rdi)
	notq	$A[0][2](%rdi)
	notq	$A[1][3](%rdi)
	notq	$A[2][2](%rdi)
	notq	$A[3][2](%rdi)
	notq	$A[4][0](%rdi)
	lea	-100(%rdi),%rdi		# preserve A[][]

	add	\$200,%rsp
.cfi_adjust_cfa_offset	-200

	pop	%r15
.cfi_pop	%r15
	pop	%r14
.cfi_pop	%r14
	pop	%r13
.cfi_pop	%r13
	pop	%r12
.cfi_pop	%r12
	pop	%rbp
.cfi_pop	%rbp
	pop	%rbx
.cfi_pop	%rbx
	ret
.cfi_endproc
.size	KeccakF1600,.-KeccakF1600
___

{ my ($A_flat,$inp,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
     ($A_flat,$inp) = ("%r8","%r9");
$code.=<<___;
.globl	SHA3_absorb
.type	SHA3_absorb,\@function,4
.align	32
SHA3_absorb:
.cfi_startproc
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15

	lea	100(%rdi),%rdi		# size optimization
	sub	\$232,%rsp
.cfi_adjust_cfa_offset	232

	mov	%rsi,$inp
	lea	100(%rsp),%rsi		# size optimization

	notq	$A[0][1](%rdi)
	notq	$A[0][2](%rdi)
	notq	$A[1][3](%rdi)
	notq	$A[2][2](%rdi)
	notq	$A[3][2](%rdi)
	notq	$A[4][0](%rdi)
	lea	iotas(%rip),$iotas

	mov	$bsz,216-100(%rsi)	# save bsz

.Loop_absorb:
	cmp	$bsz,$len
	jc	.Ldone_absorb

	shr	\$3,$bsz
	lea	-100(%rdi),$A_flat

.Lblock_absorb:
	mov	($inp),%rax
	lea	8($inp),$inp
	xor	($A_flat),%rax
	lea	8($A_flat),$A_flat
	sub	\$8,$len
	mov	%rax,-8($A_flat)
	sub	\$1,$bsz
	jnz	.Lblock_absorb

	mov	$inp,200-100(%rsi)	# save inp
	mov	$len,208-100(%rsi)	# save len
	call	__KeccakF1600
	mov	200-100(%rsi),$inp	# pull inp
	mov	208-100(%rsi),$len	# pull len
	mov	216-100(%rsi),$bsz	# pull bsz
	jmp	.Loop_absorb

.align	32
.Ldone_absorb:
	mov	$len,%rax		# return value

	notq	$A[0][1](%rdi)
	notq	$A[0][2](%rdi)
	notq	$A[1][3](%rdi)
	notq	$A[2][2](%rdi)
	notq	$A[3][2](%rdi)
	notq	$A[4][0](%rdi)

	add	\$232,%rsp
.cfi_adjust_cfa_offset	-232

	pop	%r15
.cfi_pop	%r15
	pop	%r14
.cfi_pop	%r14
	pop	%r13
.cfi_pop	%r13
	pop	%r12
.cfi_pop	%r12
	pop	%rbp
.cfi_pop	%rbp
	pop	%rbx
.cfi_pop	%rbx
	ret
.cfi_endproc
.size	SHA3_absorb,.-SHA3_absorb
___
}
{ my ($A_flat,$out,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
     ($out,$len,$bsz) = ("%r12","%r13","%r14");

$code.=<<___;
.globl	SHA3_squeeze
.type	SHA3_squeeze,\@function,4
.align	32
SHA3_squeeze:
.cfi_startproc
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14

	shr	\$3,%rcx
	mov	$A_flat,%r8
	mov	%rsi,$out
	mov	%rdx,$len
	mov	%rcx,$bsz
	jmp	.Loop_squeeze

.align	32
.Loop_squeeze:
	cmp	\$8,$len
	jb	.Ltail_squeeze

	mov	(%r8),%rax
	lea	8(%r8),%r8
	mov	%rax,($out)
	lea	8($out),$out
	sub	\$8,$len		# len -= 8
	jz	.Ldone_squeeze

	sub	\$1,%rcx		# bsz--
	jnz	.Loop_squeeze

	call	KeccakF1600
	mov	$A_flat,%r8
	mov	$bsz,%rcx
	jmp	.Loop_squeeze

.Ltail_squeeze:
	mov	%r8, %rsi
	mov	$out,%rdi
	mov	$len,%rcx
	.byte	0xf3,0xa4		# rep	movsb

.Ldone_squeeze:
	pop	%r14
.cfi_pop	%r14
	pop	%r13
.cfi_pop	%r13
	pop	%r12
.cfi_pop	%r13
	ret
.cfi_endproc
.size	SHA3_squeeze,.-SHA3_squeeze
___
}
$code.=<<___;
.align	256
	.quad	0,0,0,0,0,0,0,0
.type	iotas,\@object
iotas:
	.quad	0x0000000000000001
	.quad	0x0000000000008082
	.quad	0x800000000000808a
	.quad	0x8000000080008000
	.quad	0x000000000000808b
	.quad	0x0000000080000001
	.quad	0x8000000080008081
	.quad	0x8000000000008009
	.quad	0x000000000000008a
	.quad	0x0000000000000088
	.quad	0x0000000080008009
	.quad	0x000000008000000a
	.quad	0x000000008000808b
	.quad	0x800000000000008b
	.quad	0x8000000000008089
	.quad	0x8000000000008003
	.quad	0x8000000000008002
	.quad	0x8000000000000080
	.quad	0x000000000000800a
	.quad	0x800000008000000a
	.quad	0x8000000080008081
	.quad	0x8000000000008080
	.quad	0x0000000080000001
	.quad	0x8000000080008008
.size	iotas,.-iotas
.asciz	"Keccak-1600 absorb and squeeze for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
___

foreach (split("\n",$code)) {
	# Below replacement results in 11.2 on Sandy Bridge, 9.4 on
	# Haswell, but it hurts other processors by up to 2-3-4x...
	#s/rol\s+(\$[0-9]+),(%[a-z][a-z0-9]+)/shld\t$1,$2,$2/;
	# Below replacement results in 9.3 on Haswell [as well as
	# on Ryzen, i.e. it *hurts* Ryzen]...
	#s/rol\s+\$([0-9]+),(%[a-z][a-z0-9]+)/rorx\t\$64-$1,$2,$2/;

	print $_, "\n";
}

close STDOUT or die "error closing STDOUT: $!";


#!/usr/bin/env perl
# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for PowerISA 2.07.
#
# June 2017.
#
# This is straightforward KECCAK_1X_ALT SIMD implementation, but with
# disjoint Rho and Pi. The module is ABI-bitness- and endian-neutral.
# POWER8 processor spends 9.8 cycles to process byte out of large
# buffer for r=1088, which matches SHA3-256. This is 17% better than
# scalar PPC64 code. It probably should be noted that if POWER8's
# successor can achieve higher scalar instruction issue rate, then
# this module will loose... And it does on POWER9 with 12.0 vs. 9.4.

$flavour = shift;

if ($flavour =~ /64/) {
	$SIZE_T	=8;
	$LRSAVE	=2*$SIZE_T;
	$UCMP	="cmpld";
	$STU	="stdu";
	$POP	="ld";
	$PUSH	="std";
} elsif ($flavour =~ /32/) {
	$SIZE_T	=4;
	$LRSAVE	=$SIZE_T;
	$STU	="stwu";
	$POP	="lwz";
	$PUSH	="stw";
	$UCMP	="cmplw";
} else { die "nonsense $flavour"; }

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";

open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";

$FRAME=6*$SIZE_T+13*16;	# 13*16 is for v20-v31 offload

my $sp ="r1";

my $iotas = "r12";

########################################################################
# Register layout:
#
# v0		A[0][0] A[1][0]
# v1		A[0][1] A[1][1]
# v2		A[0][2] A[1][2]
# v3		A[0][3] A[1][3]
# v4		A[0][4] A[1][4]
#
# v5		A[2][0] A[3][0]
# v6		A[2][1] A[3][1]
# v7		A[2][2] A[3][2]
# v8		A[2][3] A[3][3]
# v9		A[2][4] A[3][4]
#
# v10		A[4][0] A[4][1]
# v11		A[4][2] A[4][3]
# v12		A[4][4] A[4][4]
#
# v13..25	rhotates[][]
# v26..31	volatile
#
$code.=<<___;
.machine	"any"
.text

.type	KeccakF1600_int,\@function
.align	5
KeccakF1600_int:
	li	r0,24
	mtctr	r0
	li	r0,0
	b	.Loop

.align	4
.Loop:
	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Theta
	vxor	v26,v0, v5		; A[0..1][0]^A[2..3][0]
	vxor	v27,v1, v6		; A[0..1][1]^A[2..3][1]
	vxor	v28,v2, v7		; A[0..1][2]^A[2..3][2]
	vxor	v29,v3, v8		; A[0..1][3]^A[2..3][3]
	vxor	v30,v4, v9		; A[0..1][4]^A[2..3][4]
	vpermdi	v31,v26,v27,0b00	; A[0][0..1]^A[2][0..1]
	vpermdi	v26,v26,v27,0b11	; A[1][0..1]^A[3][0..1]
	vpermdi	v27,v28,v29,0b00	; A[0][2..3]^A[2][2..3]
	vpermdi	v28,v28,v29,0b11	; A[1][2..3]^A[3][2..3]
	vpermdi	v29,v30,v30,0b10	; A[1..0][4]^A[3..2][4]
	vxor	v26,v26,v31		; C[0..1]
	vxor	v27,v27,v28		; C[2..3]
	vxor	v28,v29,v30		; C[4..4]
	vspltisb v31,1
	vxor	v26,v26,v10		; C[0..1] ^= A[4][0..1]
	vxor	v27,v27,v11		; C[2..3] ^= A[4][2..3]
	vxor	v28,v28,v12		; C[4..4] ^= A[4][4..4], low!

	vrld	v29,v26,v31		; ROL64(C[0..1],1)
	vrld	v30,v27,v31		; ROL64(C[2..3],1)
	vrld	v31,v28,v31		; ROL64(C[4..4],1)
	vpermdi	v31,v31,v29,0b10
	vxor	v26,v26,v30		; C[0..1] ^= ROL64(C[2..3],1)
	vxor	v27,v27,v31		; C[2..3] ^= ROL64(C[4..0],1)
	vxor	v28,v28,v29		; C[4..4] ^= ROL64(C[0..1],1), low!

	vpermdi	v29,v26,v26,0b00	; C[0..0]
	vpermdi	v30,v28,v26,0b10	; C[4..0]
	vpermdi	v31,v28,v28,0b11	; C[4..4]
	vxor	v1, v1, v29		; A[0..1][1] ^= C[0..0]
	vxor	v6, v6, v29		; A[2..3][1] ^= C[0..0]
	vxor	v10,v10,v30		; A[4][0..1] ^= C[4..0]
	vxor	v0, v0, v31		; A[0..1][0] ^= C[4..4]
	vxor	v5, v5, v31		; A[2..3][0] ^= C[4..4]

	vpermdi	v29,v27,v27,0b00	; C[2..2]
	vpermdi	v30,v26,v26,0b11	; C[1..1]
	vpermdi	v31,v26,v27,0b10	; C[1..2]
	vxor	v3, v3, v29		; A[0..1][3] ^= C[2..2]
	vxor	v8, v8, v29		; A[2..3][3] ^= C[2..2]
	vxor	v2, v2, v30		; A[0..1][2] ^= C[1..1]
	vxor	v7, v7, v30		; A[2..3][2] ^= C[1..1]
	vxor	v11,v11,v31		; A[4][2..3] ^= C[1..2]

	vpermdi	v29,v27,v27,0b11	; C[3..3]
	vxor	v4, v4, v29		; A[0..1][4] ^= C[3..3]
	vxor	v9, v9, v29		; A[2..3][4] ^= C[3..3]
	vxor	v12,v12,v29		; A[4..4][4] ^= C[3..3]

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Rho
	vrld	v26,v0, v13		; v0
	vrld	v1, v1, v14
	vrld	v27,v2, v15		; v2
	vrld	v28,v3, v16		; v3
	vrld	v4, v4, v17
	vrld	v5, v5, v18
	vrld	v6, v6, v19
	vrld	v29,v7, v20		; v7
	vrld	v8, v8, v21
	vrld	v9, v9, v22
	vrld	v10,v10,v23
	vrld	v30,v11,v24		; v11
	vrld	v12,v12,v25

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Pi
	vpermdi	v0, v26,v28,0b00	; [0][0] [1][0] < [0][0] [0][3]
	vpermdi	v2, v29,v5, 0b00	; [0][2] [1][2] < [2][2] [2][0]
	vpermdi	v11,v9, v5, 0b01	; [4][2] [4][3] < [2][4] [3][0]
	vpermdi	v5, v1, v4, 0b00	; [2][0] [3][0] < [0][1] [0][4]
	vpermdi	v1, v1, v4, 0b11	; [0][1] [1][1] < [1][1] [1][4]
	vpermdi	v3, v8, v6, 0b11	; [0][3] [1][3] < [3][3] [3][1]
	vpermdi	v4, v12,v30,0b10	; [0][4] [1][4] < [4][4] [4][2]
	vpermdi	v7, v8, v6, 0b00	; [2][2] [3][2] < [2][3] [2][1]
	vpermdi	v6, v27,v26,0b11	; [2][1] [3][1] < [1][2] [1][0]
	vpermdi	v8, v9, v29,0b11	; [2][3] [3][3] < [3][4] [3][2]
	vpermdi	v12,v10,v10,0b11	; [4][4] [4][4] < [4][1] [4][1]
	vpermdi	v9, v10,v30,0b01	; [2][4] [3][4] < [4][0] [4][3]
	vpermdi	v10,v27,v28,0b01	; [4][0] [4][1] < [0][2] [1][3]

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Chi + Iota
	lvx_u	v31,$iotas,r0		; iotas[index]
	addic	r0,r0,16		; index++

	vandc	v26,v2, v1		; (~A[0..1][1] & A[0..1][2])
	vandc	v27,v3, v2		; (~A[0..1][2] & A[0..1][3])
	vandc	v28,v4, v3		; (~A[0..1][3] & A[0..1][4])
	vandc	v29,v0, v4		; (~A[0..1][4] & A[0..1][0])
	vandc	v30,v1, v0		; (~A[0..1][0] & A[0..1][1])
	vxor	v0, v0, v26		; A[0..1][0] ^= (~A[0..1][1] & A[0..1][2])
	vxor	v1, v1, v27		; A[0..1][1] ^= (~A[0..1][2] & A[0..1][3])
	vxor	v2, v2, v28		; A[0..1][2] ^= (~A[0..1][3] & A[0..1][4])
	vxor	v3, v3, v29		; A[0..1][3] ^= (~A[0..1][4] & A[0..1][0])
	vxor	v4, v4, v30		; A[0..1][4] ^= (~A[0..1][0] & A[0..1][1])

	vandc	v26,v7, v6		; (~A[2..3][1] & A[2..3][2])
	vandc	v27,v8, v7		; (~A[2..3][2] & A[2..3][3])
	vandc	v28,v9, v8		; (~A[2..3][3] & A[2..3][4])
	vandc	v29,v5, v9		; (~A[2..3][4] & A[2..3][0])
	vandc	v30,v6, v5		; (~A[2..3][0] & A[2..3][1])
	vxor	v5, v5, v26		; A[2..3][0] ^= (~A[2..3][1] & A[2..3][2])
	vxor	v6, v6, v27		; A[2..3][1] ^= (~A[2..3][2] & A[2..3][3])
	vxor	v7, v7, v28		; A[2..3][2] ^= (~A[2..3][3] & A[2..3][4])
	vxor	v8, v8, v29		; A[2..3][3] ^= (~A[2..3][4] & A[2..3][0])
	vxor	v9, v9, v30		; A[2..3][4] ^= (~A[2..3][0] & A[2..3][1])

	vxor	v0, v0, v31		; A[0][0] ^= iotas[index++]

	vpermdi	v26,v10,v11,0b10	; A[4][1..2]
	vpermdi	v27,v12,v10,0b00	; A[4][4..0]
	vpermdi	v28,v11,v12,0b10	; A[4][3..4]
	vpermdi	v29,v10,v10,0b10	; A[4][1..0]
	vandc	v26,v11,v26		; (~A[4][1..2] & A[4][2..3])
	vandc	v27,v27,v28		; (~A[4][3..4] & A[4][4..0])
	vandc	v28,v10,v29		; (~A[4][1..0] & A[4][0..1])
	vxor	v10,v10,v26		; A[4][0..1] ^= (~A[4][1..2] & A[4][2..3])
	vxor	v11,v11,v27		; A[4][2..3] ^= (~A[4][3..4] & A[4][4..0])
	vxor	v12,v12,v28		; A[4][4..4] ^= (~A[4][0..1] & A[4][1..0])

	bdnz	.Loop

	vpermdi	v12,v12,v12,0b11	; broadcast A[4][4]
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,0,0
.size	KeccakF1600_int,.-KeccakF1600_int

.type	KeccakF1600,\@function
.align	5
KeccakF1600:
	$STU	$sp,-$FRAME($sp)
	li	r10,`15+6*$SIZE_T`
	li	r11,`31+6*$SIZE_T`
	mflr	r8
	mfspr	r7, 256			; save vrsave
	stvx	v20,r10,$sp
	addi	r10,r10,32
	stvx	v21,r11,$sp
	addi	r11,r11,32
	stvx	v22,r10,$sp
	addi	r10,r10,32
	stvx	v23,r11,$sp
	addi	r11,r11,32
	stvx	v24,r10,$sp
	addi	r10,r10,32
	stvx	v25,r11,$sp
	addi	r11,r11,32
	stvx	v26,r10,$sp
	addi	r10,r10,32
	stvx	v27,r11,$sp
	addi	r11,r11,32
	stvx	v28,r10,$sp
	addi	r10,r10,32
	stvx	v29,r11,$sp
	addi	r11,r11,32
	stvx	v30,r10,$sp
	stvx	v31,r11,$sp
	stw	r7,`$FRAME-4`($sp)	; save vrsave
	li	r0, -1
	$PUSH	r8,`$FRAME+$LRSAVE`($sp)
	mtspr	256, r0			; preserve all AltiVec registers

	li	r11,16
	lvx_4w	v0,0,r3			; load A[5][5]
	li	r10,32
	lvx_4w	v1,r11,r3
	addi	r11,r11,32
	lvx_4w	v2,r10,r3
	addi	r10,r10,32
	lvx_4w	v3,r11,r3
	addi	r11,r11,32
	lvx_4w	v4,r10,r3
	addi	r10,r10,32
	lvx_4w	v5,r11,r3
	addi	r11,r11,32
	lvx_4w	v6,r10,r3
	addi	r10,r10,32
	lvx_4w	v7,r11,r3
	addi	r11,r11,32
	lvx_4w	v8,r10,r3
	addi	r10,r10,32
	lvx_4w	v9,r11,r3
	addi	r11,r11,32
	lvx_4w	v10,r10,r3
	addi	r10,r10,32
	lvx_4w	v11,r11,r3
	lvx_splt v12,r10,r3

	bl	PICmeup

	li	r11,16
	lvx_u	v13,0,r12		; load rhotates
	li	r10,32
	lvx_u	v14,r11,r12
	addi	r11,r11,32
	lvx_u	v15,r10,r12
	addi	r10,r10,32
	lvx_u	v16,r11,r12
	addi	r11,r11,32
	lvx_u	v17,r10,r12
	addi	r10,r10,32
	lvx_u	v18,r11,r12
	addi	r11,r11,32
	lvx_u	v19,r10,r12
	addi	r10,r10,32
	lvx_u	v20,r11,r12
	addi	r11,r11,32
	lvx_u	v21,r10,r12
	addi	r10,r10,32
	lvx_u	v22,r11,r12
	addi	r11,r11,32
	lvx_u	v23,r10,r12
	addi	r10,r10,32
	lvx_u	v24,r11,r12
	lvx_u	v25,r10,r12
	addi	r12,r12,`16*16`		; points at iotas

	bl	KeccakF1600_int

	li	r11,16
	stvx_4w	v0,0,r3			; return A[5][5]
	li	r10,32
	stvx_4w	v1,r11,r3
	addi	r11,r11,32
	stvx_4w	v2,r10,r3
	addi	r10,r10,32
	stvx_4w	v3,r11,r3
	addi	r11,r11,32
	stvx_4w	v4,r10,r3
	addi	r10,r10,32
	stvx_4w	v5,r11,r3
	addi	r11,r11,32
	stvx_4w	v6,r10,r3
	addi	r10,r10,32
	stvx_4w	v7,r11,r3
	addi	r11,r11,32
	stvx_4w	v8,r10,r3
	addi	r10,r10,32
	stvx_4w	v9,r11,r3
	addi	r11,r11,32
	stvx_4w	v10,r10,r3
	addi	r10,r10,32
	stvx_4w	v11,r11,r3
	stvdx_u v12,r10,r3

	li	r10,`15+6*$SIZE_T`
	li	r11,`31+6*$SIZE_T`
	mtlr	r8
	mtspr	256, r7			; restore vrsave
	lvx	v20,r10,$sp
	addi	r10,r10,32
	lvx	v21,r11,$sp
	addi	r11,r11,32
	lvx	v22,r10,$sp
	addi	r10,r10,32
	lvx	v23,r11,$sp
	addi	r11,r11,32
	lvx	v24,r10,$sp
	addi	r10,r10,32
	lvx	v25,r11,$sp
	addi	r11,r11,32
	lvx	v26,r10,$sp
	addi	r10,r10,32
	lvx	v27,r11,$sp
	addi	r11,r11,32
	lvx	v28,r10,$sp
	addi	r10,r10,32
	lvx	v29,r11,$sp
	addi	r11,r11,32
	lvx	v30,r10,$sp
	lvx	v31,r11,$sp
	addi	$sp,$sp,$FRAME
	blr
	.long	0
	.byte	0,12,0x04,1,0x80,0,1,0
	.long	0
.size	KeccakF1600,.-KeccakF1600
___
{
my ($A_jagged,$inp,$len,$bsz) = map("r$_",(3..6));

$code.=<<___;
.globl	SHA3_absorb
.type	SHA3_absorb,\@function
.align	5
SHA3_absorb:
	$STU	$sp,-$FRAME($sp)
	li	r10,`15+6*$SIZE_T`
	li	r11,`31+6*$SIZE_T`
	mflr	r8
	mfspr	r7, 256			; save vrsave
	stvx	v20,r10,$sp
	addi	r10,r10,32
	stvx	v21,r11,$sp
	addi	r11,r11,32
	stvx	v22,r10,$sp
	addi	r10,r10,32
	stvx	v23,r11,$sp
	addi	r11,r11,32
	stvx	v24,r10,$sp
	addi	r10,r10,32
	stvx	v25,r11,$sp
	addi	r11,r11,32
	stvx	v26,r10,$sp
	addi	r10,r10,32
	stvx	v27,r11,$sp
	addi	r11,r11,32
	stvx	v28,r10,$sp
	addi	r10,r10,32
	stvx	v29,r11,$sp
	addi	r11,r11,32
	stvx	v30,r10,$sp
	stvx	v31,r11,$sp
	stw	r7,`$FRAME-4`($sp)	; save vrsave
	li	r0, -1
	$PUSH	r8,`$FRAME+$LRSAVE`($sp)
	mtspr	256, r0			; preserve all AltiVec registers

	li	r11,16
	lvx_4w	v0,0,$A_jagged		; load A[5][5]
	li	r10,32
	lvx_4w	v1,r11,$A_jagged
	addi	r11,r11,32
	lvx_4w	v2,r10,$A_jagged
	addi	r10,r10,32
	lvx_4w	v3,r11,$A_jagged
	addi	r11,r11,32
	lvx_4w	v4,r10,$A_jagged
	addi	r10,r10,32
	lvx_4w	v5,r11,$A_jagged
	addi	r11,r11,32
	lvx_4w	v6,r10,$A_jagged
	addi	r10,r10,32
	lvx_4w	v7,r11,$A_jagged
	addi	r11,r11,32
	lvx_4w	v8,r10,$A_jagged
	addi	r10,r10,32
	lvx_4w	v9,r11,$A_jagged
	addi	r11,r11,32
	lvx_4w	v10,r10,$A_jagged
	addi	r10,r10,32
	lvx_4w	v11,r11,$A_jagged
	lvx_splt v12,r10,$A_jagged

	bl	PICmeup

	li	r11,16
	lvx_u	v13,0,r12		; load rhotates
	li	r10,32
	lvx_u	v14,r11,r12
	addi	r11,r11,32
	lvx_u	v15,r10,r12
	addi	r10,r10,32
	lvx_u	v16,r11,r12
	addi	r11,r11,32
	lvx_u	v17,r10,r12
	addi	r10,r10,32
	lvx_u	v18,r11,r12
	addi	r11,r11,32
	lvx_u	v19,r10,r12
	addi	r10,r10,32
	lvx_u	v20,r11,r12
	addi	r11,r11,32
	lvx_u	v21,r10,r12
	addi	r10,r10,32
	lvx_u	v22,r11,r12
	addi	r11,r11,32
	lvx_u	v23,r10,r12
	addi	r10,r10,32
	lvx_u	v24,r11,r12
	lvx_u	v25,r10,r12
	li	r10,-32
	li	r11,-16
	addi	r12,r12,`16*16`		; points at iotas
	b	.Loop_absorb

.align	4
.Loop_absorb:
	$UCMP	$len,$bsz		; len < bsz?
	blt	.Labsorbed

	sub	$len,$len,$bsz		; len -= bsz
	srwi	r0,$bsz,3
	mtctr	r0

	lvx_u	v30,r10,r12		; permutation masks
	lvx_u	v31,r11,r12
	?vspltisb v27,7			; prepare masks for byte swap
	?vxor	v30,v30,v27		; on big-endian
	?vxor	v31,v31,v27

	vxor	v27,v27,v27		; zero
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v0, v0, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v1, v1, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v2, v2, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v3, v3, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v4, v4, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v0, v0, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v1, v1, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v2, v2, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v3, v3, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v4, v4, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v5, v5, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v6, v6, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v7, v7, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v8, v8, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v9, v9, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v5, v5, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v6, v6, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v7, v7, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v8, v8, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v9, v9, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v10, v10, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v10, v10, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v30
	vxor	v11, v11, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v11, v11, v26
	bdz	.Lprocess_block
	lvdx_u	v26,0,$inp
	addi	$inp,$inp,8
	vperm	v26,v26,v27,v31
	vxor	v12, v12, v26

.Lprocess_block:
	bl	KeccakF1600_int

	b	.Loop_absorb

.align	4
.Labsorbed:
	li	r11,16
	stvx_4w	v0,0,$A_jagged		; return A[5][5]
	li	r10,32
	stvx_4w	v1,r11,$A_jagged
	addi	r11,r11,32
	stvx_4w	v2,r10,$A_jagged
	addi	r10,r10,32
	stvx_4w	v3,r11,$A_jagged
	addi	r11,r11,32
	stvx_4w	v4,r10,$A_jagged
	addi	r10,r10,32
	stvx_4w	v5,r11,$A_jagged
	addi	r11,r11,32
	stvx_4w	v6,r10,$A_jagged
	addi	r10,r10,32
	stvx_4w	v7,r11,$A_jagged
	addi	r11,r11,32
	stvx_4w	v8,r10,$A_jagged
	addi	r10,r10,32
	stvx_4w	v9,r11,$A_jagged
	addi	r11,r11,32
	stvx_4w	v10,r10,$A_jagged
	addi	r10,r10,32
	stvx_4w	v11,r11,$A_jagged
	stvdx_u v12,r10,$A_jagged

	mr	r3,$len			; return value
	li	r10,`15+6*$SIZE_T`
	li	r11,`31+6*$SIZE_T`
	mtlr	r8
	mtspr	256, r7			; restore vrsave
	lvx	v20,r10,$sp
	addi	r10,r10,32
	lvx	v21,r11,$sp
	addi	r11,r11,32
	lvx	v22,r10,$sp
	addi	r10,r10,32
	lvx	v23,r11,$sp
	addi	r11,r11,32
	lvx	v24,r10,$sp
	addi	r10,r10,32
	lvx	v25,r11,$sp
	addi	r11,r11,32
	lvx	v26,r10,$sp
	addi	r10,r10,32
	lvx	v27,r11,$sp
	addi	r11,r11,32
	lvx	v28,r10,$sp
	addi	r10,r10,32
	lvx	v29,r11,$sp
	addi	r11,r11,32
	lvx	v30,r10,$sp
	lvx	v31,r11,$sp
	addi	$sp,$sp,$FRAME
	blr
	.long	0
	.byte	0,12,0x04,1,0x80,0,4,0
	.long	0
.size	SHA3_absorb,.-SHA3_absorb
___
}
{
my ($A_jagged,$out,$len,$bsz) = map("r$_",(3..6));

$code.=<<___;
.globl	SHA3_squeeze
.type	SHA3_squeeze,\@function
.align	5
SHA3_squeeze:
	mflr	r9			; r9 is not touched by KeccakF1600
	subi	$out,$out,1		; prepare for stbu
	addi	r8,$A_jagged,4		; prepare volatiles
	mr	r10,$bsz
	li	r11,0
	b	.Loop_squeeze
.align	4
.Loop_squeeze:
	lwzx	r7,r11,r8		; lo
	lwzx	r0,r11,$A_jagged	; hi
	${UCMP}i $len,8
	blt	.Lsqueeze_tail

	stbu	r7,1($out)		; write lo
	srwi	r7,r7,8
	stbu	r7,1($out)
	srwi	r7,r7,8
	stbu	r7,1($out)
	srwi	r7,r7,8
	stbu	r7,1($out)
	stbu	r0,1($out)		; write hi
	srwi	r0,r0,8
	stbu	r0,1($out)
	srwi	r0,r0,8
	stbu	r0,1($out)
	srwi	r0,r0,8
	stbu	r0,1($out)

	subic.	$len,$len,8
	beqlr				; return if done

	subic.	r10,r10,8
	ble	.Loutput_expand

	addi	r11,r11,16		; calculate jagged index
	cmplwi	r11,`16*5`
	blt	.Loop_squeeze
	subi	r11,r11,72
	beq	.Loop_squeeze
	addi	r11,r11,72
	cmplwi	r11,`16*5+8`
	subi	r11,r11,8
	beq	.Loop_squeeze
	addi	r11,r11,8
	cmplwi	r11,`16*10`
	subi	r11,r11,72
	beq	.Loop_squeeze
	addi	r11,r11,72
	blt	.Loop_squeeze
	subi	r11,r11,8
	b	.Loop_squeeze

.align	4
.Loutput_expand:
	bl	KeccakF1600
	mtlr	r9

	addi	r8,$A_jagged,4		; restore volatiles
	mr	r10,$bsz
	li	r11,0
	b	.Loop_squeeze

.align	4
.Lsqueeze_tail:
	mtctr	$len
	subic.	$len,$len,4
	ble	.Loop_tail_lo
	li	r8,4
	mtctr	r8
.Loop_tail_lo:
	stbu	r7,1($out)
	srdi	r7,r7,8
	bdnz	.Loop_tail_lo
	ble	.Lsqueeze_done
	mtctr	$len
.Loop_tail_hi:
	stbu	r0,1($out)
	srdi	r0,r0,8
	bdnz	.Loop_tail_hi

.Lsqueeze_done:
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,4,0
	.long	0
.size	SHA3_squeeze,.-SHA3_squeeze
___
}
$code.=<<___;
.align	6
PICmeup:
	mflr	r0
	bcl	20,31,\$+4
	mflr	r12   ; vvvvvv "distance" between . and 1st data entry
	addi	r12,r12,`64-8`
	mtlr	r0
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,0,0
	.space	`64-9*4`
.type	rhotates,\@object
.align	6
rhotates:
	.quad	0,  36
	.quad	1,  44
	.quad	62,  6
	.quad	28, 55
	.quad	27, 20
	.quad	3,  41
	.quad	10, 45
	.quad	43, 15
	.quad	25, 21
	.quad	39,  8
	.quad	18,  2
	.quad	61, 56
	.quad	14, 14
.size	rhotates,.-rhotates
	.quad	0,0
	.quad	0x0001020304050607,0x1011121314151617
	.quad	0x1011121314151617,0x0001020304050607
.type	iotas,\@object
iotas:
	.quad	0x0000000000000001,0
	.quad	0x0000000000008082,0
	.quad	0x800000000000808a,0
	.quad	0x8000000080008000,0
	.quad	0x000000000000808b,0
	.quad	0x0000000080000001,0
	.quad	0x8000000080008081,0
	.quad	0x8000000000008009,0
	.quad	0x000000000000008a,0
	.quad	0x0000000000000088,0
	.quad	0x0000000080008009,0
	.quad	0x000000008000000a,0
	.quad	0x000000008000808b,0
	.quad	0x800000000000008b,0
	.quad	0x8000000000008089,0
	.quad	0x8000000000008003,0
	.quad	0x8000000000008002,0
	.quad	0x8000000000000080,0
	.quad	0x000000000000800a,0
	.quad	0x800000008000000a,0
	.quad	0x8000000080008081,0
	.quad	0x8000000000008080,0
	.quad	0x0000000080000001,0
	.quad	0x8000000080008008,0
.size	iotas,.-iotas
.asciz	"Keccak-1600 absorb and squeeze for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
___

foreach  (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/ge;

	if ($flavour =~ /le$/) {	# little-endian
	    s/\?([a-z]+)/;$1/;
	} else {			# big-endian
	    s/\?([a-z]+)/$1/;
	}

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 1998-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# [Re]written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# "[Re]written" was achieved in two major overhauls. In 2004 BODY_*
# functions were re-implemented to address P4 performance issue [see
# commentary below], and in 2006 the rest was rewritten in order to
# gain freedom to liberate licensing terms.

# January, September 2004.
#
# It was noted that Intel IA-32 C compiler generates code which
# performs ~30% *faster* on P4 CPU than original *hand-coded*
# SHA1 assembler implementation. To address this problem (and
# prove that humans are still better than machines:-), the
# original code was overhauled, which resulted in following
# performance changes:
#
#		compared with original	compared with Intel cc
#		assembler impl.		generated code
# Pentium	-16%			+48%
# PIII/AMD	+8%			+16%
# P4		+85%(!)			+45%
#
# As you can see Pentium came out as looser:-( Yet I reckoned that
# improvement on P4 outweighs the loss and incorporate this
# re-tuned code to 0.9.7 and later.
# ----------------------------------------------------------------

# August 2009.
#
# George Spelvin has tipped that F_40_59(b,c,d) can be rewritten as
# '(c&d) + (b&(c^d))', which allows to accumulate partial results
# and lighten "pressure" on scratch registers. This resulted in
# >12% performance improvement on contemporary AMD cores (with no
# degradation on other CPUs:-). Also, the code was revised to maximize
# "distance" between instructions producing input to 'lea' instruction
# and the 'lea' instruction itself, which is essential for Intel Atom
# core and resulted in ~15% improvement.

# October 2010.
#
# Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it
# is to offload message schedule denoted by Wt in NIST specification,
# or Xupdate in OpenSSL source, to SIMD unit. The idea is not novel,
# and in SSE2 context was first explored by Dean Gaudet in 2004, see
# http://arctic.org/~dean/crypto/sha1.html. Since then several things
# have changed that made it interesting again:
#
# a) XMM units became faster and wider;
# b) instruction set became more versatile;
# c) an important observation was made by Max Locktykhin, which made
#    it possible to reduce amount of instructions required to perform
#    the operation in question, for further details see
#    http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/.

# April 2011.
#
# Add AVX code path, probably most controversial... The thing is that
# switch to AVX alone improves performance by as little as 4% in
# comparison to SSSE3 code path. But below result doesn't look like
# 4% improvement... Trouble is that Sandy Bridge decodes 'ro[rl]' as
# pair of Âµ-ops, and it's the additional Âµ-ops, two per round, that
# make it run slower than Core2 and Westmere. But 'sh[rl]d' is decoded
# as single Âµ-op by Sandy Bridge and it's replacing 'ro[rl]' with
# equivalent 'sh[rl]d' that is responsible for the impressive 5.1
# cycles per processed byte. But 'sh[rl]d' is not something that used
# to be fast, nor does it appear to be fast in upcoming Bulldozer
# [according to its optimization manual]. Which is why AVX code path
# is guarded by *both* AVX and synthetic bit denoting Intel CPUs.
# One can argue that it's unfair to AMD, but without 'sh[rl]d' it
# makes no sense to keep the AVX code path. If somebody feels that
# strongly, it's probably more appropriate to discuss possibility of
# using vector rotate XOP on AMD...

# March 2014.
#
# Add support for Intel SHA Extensions.

######################################################################
# Current performance is summarized in following table. Numbers are
# CPU clock cycles spent to process single byte (less is better).
#
#		x86		SSSE3		AVX
# Pentium	15.7		-
# PIII		11.5		-
# P4		10.6		-
# AMD K8	7.1		-
# Core2		7.3		6.0/+22%	-
# Westmere	7.3		5.5/+33%	-
# Sandy Bridge	8.8		6.2/+40%	5.1(**)/+73%
# Ivy Bridge	7.2		4.8/+51%	4.7(**)/+53%
# Haswell	6.5		4.3/+51%	4.1(**)/+58%
# Skylake	6.4		4.1/+55%	4.1(**)/+55%
# Bulldozer	11.6		6.0/+92%
# VIA Nano	10.6		7.5/+41%
# Atom		12.5		9.3(*)/+35%
# Silvermont	14.5		9.9(*)/+46%
# Goldmont	8.8		6.7/+30%	1.7(***)/+415%
#
# (*)	Loop is 1056 instructions long and expected result is ~8.25.
#	The discrepancy is because of front-end limitations, so
#	called MS-ROM penalties, and on Silvermont even rotate's
#	limited parallelism.
#
# (**)	As per above comment, the result is for AVX *plus* sh[rl]d.
#
# (***)	SHAEXT result

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";

$output=pop;
open STDOUT,">$output";

&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");

$xmm=$ymm=0;
for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }

$ymm=1 if ($xmm &&
		`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
			=~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
		$1>=2.19);	# first version supporting AVX

$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
		`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
		$1>=2.03);	# first version supporting AVX

$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" &&
		`ml 2>&1` =~ /Version ([0-9]+)\./ &&
		$1>=10);	# first version supporting AVX

$ymm=1 if ($xmm && !$ymm && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|based on LLVM) ([0-9]+\.[0-9]+)/ &&
		$2>=3.0);	# first version supporting AVX

$shaext=$xmm;	### set to zero if compiling for 1.0.1

&external_label("OPENSSL_ia32cap_P") if ($xmm);

$A="eax";
$B="ebx";
$C="ecx";
$D="edx";
$E="edi";
$T="esi";
$tmp1="ebp";

@V=($A,$B,$C,$D,$E,$T);

$alt=0;	# 1 denotes alternative IALU implementation, which performs
	# 8% *worse* on P4, same on Westmere and Atom, 2% better on
	# Sandy Bridge...

sub BODY_00_15
	{
	local($n,$a,$b,$c,$d,$e,$f)=@_;

	&comment("00_15 $n");

	&mov($f,$c);			# f to hold F_00_19(b,c,d)
	 if ($n==0)  { &mov($tmp1,$a); }
	 else        { &mov($a,$tmp1); }
	&rotl($tmp1,5);			# tmp1=ROTATE(a,5)
	 &xor($f,$d);
	&add($tmp1,$e);			# tmp1+=e;
	 &mov($e,&swtmp($n%16));	# e becomes volatile and is loaded
	 				# with xi, also note that e becomes
					# f in next round...
	&and($f,$b);
	&rotr($b,2);			# b=ROTATE(b,30)
	 &xor($f,$d);			# f holds F_00_19(b,c,d)
	&lea($tmp1,&DWP(0x5a827999,$tmp1,$e));	# tmp1+=K_00_19+xi

	if ($n==15) { &mov($e,&swtmp(($n+1)%16));# pre-fetch f for next round
		      &add($f,$tmp1); }	# f+=tmp1
	else        { &add($tmp1,$f); }	# f becomes a in next round
	&mov($tmp1,$a)			if ($alt && $n==15);
	}

sub BODY_16_19
	{
	local($n,$a,$b,$c,$d,$e,$f)=@_;

	&comment("16_19 $n");

if ($alt) {
	&xor($c,$d);
	 &xor($f,&swtmp(($n+2)%16));	# f to hold Xupdate(xi,xa,xb,xc,xd)
	&and($tmp1,$c);			# tmp1 to hold F_00_19(b,c,d), b&=c^d
	 &xor($f,&swtmp(($n+8)%16));
	&xor($tmp1,$d);			# tmp1=F_00_19(b,c,d)
	 &xor($f,&swtmp(($n+13)%16));	# f holds xa^xb^xc^xd
	&rotl($f,1);			# f=ROTATE(f,1)
	 &add($e,$tmp1);		# e+=F_00_19(b,c,d)
	&xor($c,$d);			# restore $c
	 &mov($tmp1,$a);		# b in next round
	&rotr($b,$n==16?2:7);		# b=ROTATE(b,30)
	 &mov(&swtmp($n%16),$f);	# xi=f
	&rotl($a,5);			# ROTATE(a,5)
	 &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e
	&mov($e,&swtmp(($n+1)%16));	# pre-fetch f for next round
	 &add($f,$a);			# f+=ROTATE(a,5)
} else {
	&mov($tmp1,$c);			# tmp1 to hold F_00_19(b,c,d)
	 &xor($f,&swtmp(($n+2)%16));	# f to hold Xupdate(xi,xa,xb,xc,xd)
	&xor($tmp1,$d);
	 &xor($f,&swtmp(($n+8)%16));
	&and($tmp1,$b);
	 &xor($f,&swtmp(($n+13)%16));	# f holds xa^xb^xc^xd
	&rotl($f,1);			# f=ROTATE(f,1)
	 &xor($tmp1,$d);		# tmp1=F_00_19(b,c,d)
	&add($e,$tmp1);			# e+=F_00_19(b,c,d)
	 &mov($tmp1,$a);
	&rotr($b,2);			# b=ROTATE(b,30)
	 &mov(&swtmp($n%16),$f);	# xi=f
	&rotl($tmp1,5);			# ROTATE(a,5)
	 &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e
	&mov($e,&swtmp(($n+1)%16));	# pre-fetch f for next round
	 &add($f,$tmp1);		# f+=ROTATE(a,5)
}
	}

sub BODY_20_39
	{
	local($n,$a,$b,$c,$d,$e,$f)=@_;
	local $K=($n<40)?0x6ed9eba1:0xca62c1d6;

	&comment("20_39 $n");

if ($alt) {
	&xor($tmp1,$c);			# tmp1 to hold F_20_39(b,c,d), b^=c
	 &xor($f,&swtmp(($n+2)%16));	# f to hold Xupdate(xi,xa,xb,xc,xd)
	&xor($tmp1,$d);			# tmp1 holds F_20_39(b,c,d)
	 &xor($f,&swtmp(($n+8)%16));
	&add($e,$tmp1);			# e+=F_20_39(b,c,d)
	 &xor($f,&swtmp(($n+13)%16));	# f holds xa^xb^xc^xd
	&rotl($f,1);			# f=ROTATE(f,1)
	 &mov($tmp1,$a);		# b in next round
	&rotr($b,7);			# b=ROTATE(b,30)
	 &mov(&swtmp($n%16),$f)		if($n<77);# xi=f
	&rotl($a,5);			# ROTATE(a,5)
	 &xor($b,$c)			if($n==39);# warm up for BODY_40_59
	&and($tmp1,$b)			if($n==39);
	 &lea($f,&DWP($K,$f,$e));	# f+=e+K_XX_YY
	&mov($e,&swtmp(($n+1)%16))	if($n<79);# pre-fetch f for next round
	 &add($f,$a);			# f+=ROTATE(a,5)
	&rotr($a,5)			if ($n==79);
} else {
	&mov($tmp1,$b);			# tmp1 to hold F_20_39(b,c,d)
	 &xor($f,&swtmp(($n+2)%16));	# f to hold Xupdate(xi,xa,xb,xc,xd)
	&xor($tmp1,$c);
	 &xor($f,&swtmp(($n+8)%16));
	&xor($tmp1,$d);			# tmp1 holds F_20_39(b,c,d)
	 &xor($f,&swtmp(($n+13)%16));	# f holds xa^xb^xc^xd
	&rotl($f,1);			# f=ROTATE(f,1)
	 &add($e,$tmp1);		# e+=F_20_39(b,c,d)
	&rotr($b,2);			# b=ROTATE(b,30)
	 &mov($tmp1,$a);
	&rotl($tmp1,5);			# ROTATE(a,5)
	 &mov(&swtmp($n%16),$f) if($n<77);# xi=f
	&lea($f,&DWP($K,$f,$e));	# f+=e+K_XX_YY
	 &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round
	&add($f,$tmp1);			# f+=ROTATE(a,5)
}
	}

sub BODY_40_59
	{
	local($n,$a,$b,$c,$d,$e,$f)=@_;

	&comment("40_59 $n");

if ($alt) {
	&add($e,$tmp1);			# e+=b&(c^d)
	 &xor($f,&swtmp(($n+2)%16));	# f to hold Xupdate(xi,xa,xb,xc,xd)
	&mov($tmp1,$d);
	 &xor($f,&swtmp(($n+8)%16));
	&xor($c,$d);			# restore $c
	 &xor($f,&swtmp(($n+13)%16));	# f holds xa^xb^xc^xd
	&rotl($f,1);			# f=ROTATE(f,1)
	 &and($tmp1,$c);
	&rotr($b,7);			# b=ROTATE(b,30)
	 &add($e,$tmp1);		# e+=c&d
	&mov($tmp1,$a);			# b in next round
	 &mov(&swtmp($n%16),$f);	# xi=f
	&rotl($a,5);			# ROTATE(a,5)
	 &xor($b,$c)			if ($n<59);
	&and($tmp1,$b)			if ($n<59);# tmp1 to hold F_40_59(b,c,d)
	 &lea($f,&DWP(0x8f1bbcdc,$f,$e));# f+=K_40_59+e+(b&(c^d))
	&mov($e,&swtmp(($n+1)%16));	# pre-fetch f for next round
	 &add($f,$a);			# f+=ROTATE(a,5)
} else {
	&mov($tmp1,$c);			# tmp1 to hold F_40_59(b,c,d)
	 &xor($f,&swtmp(($n+2)%16));	# f to hold Xupdate(xi,xa,xb,xc,xd)
	&xor($tmp1,$d);
	 &xor($f,&swtmp(($n+8)%16));
	&and($tmp1,$b);
	 &xor($f,&swtmp(($n+13)%16));	# f holds xa^xb^xc^xd
	&rotl($f,1);			# f=ROTATE(f,1)
	 &add($tmp1,$e);		# b&(c^d)+=e
	&rotr($b,2);			# b=ROTATE(b,30)
	 &mov($e,$a);			# e becomes volatile
	&rotl($e,5);			# ROTATE(a,5)
	 &mov(&swtmp($n%16),$f);	# xi=f
	&lea($f,&DWP(0x8f1bbcdc,$f,$tmp1));# f+=K_40_59+e+(b&(c^d))
	 &mov($tmp1,$c);
	&add($f,$e);			# f+=ROTATE(a,5)
	 &and($tmp1,$d);
	&mov($e,&swtmp(($n+1)%16));	# pre-fetch f for next round
	 &add($f,$tmp1);		# f+=c&d
}
	}

&function_begin("sha1_block_data_order");
if ($xmm) {
  &static_label("shaext_shortcut")	if ($shaext);
  &static_label("ssse3_shortcut");
  &static_label("avx_shortcut")		if ($ymm);
  &static_label("K_XX_XX");

	&call	(&label("pic_point"));	# make it PIC!
  &set_label("pic_point");
	&blindpop($tmp1);
	&picmeup($T,"OPENSSL_ia32cap_P",$tmp1,&label("pic_point"));
	&lea	($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1));

	&mov	($A,&DWP(0,$T));
	&mov	($D,&DWP(4,$T));
	&test	($D,1<<9);		# check SSSE3 bit
	&jz	(&label("x86"));
	&mov	($C,&DWP(8,$T));
	&test	($A,1<<24);		# check FXSR bit
	&jz	(&label("x86"));
	if ($shaext) {
		&test	($C,1<<29);		# check SHA bit
		&jnz	(&label("shaext_shortcut"));
	}
	if ($ymm) {
		&and	($D,1<<28);		# mask AVX bit
		&and	($A,1<<30);		# mask "Intel CPU" bit
		&or	($A,$D);
		&cmp	($A,1<<28|1<<30);
		&je	(&label("avx_shortcut"));
	}
	&jmp	(&label("ssse3_shortcut"));
  &set_label("x86",16);
}
	&mov($tmp1,&wparam(0));	# SHA_CTX *c
	&mov($T,&wparam(1));	# const void *input
	&mov($A,&wparam(2));	# size_t num
	&stack_push(16+3);	# allocate X[16]
	&shl($A,6);
	&add($A,$T);
	&mov(&wparam(2),$A);	# pointer beyond the end of input
	&mov($E,&DWP(16,$tmp1));# pre-load E
	&jmp(&label("loop"));

&set_label("loop",16);

	# copy input chunk to X, but reversing byte order!
	for ($i=0; $i<16; $i+=4)
		{
		&mov($A,&DWP(4*($i+0),$T));
		&mov($B,&DWP(4*($i+1),$T));
		&mov($C,&DWP(4*($i+2),$T));
		&mov($D,&DWP(4*($i+3),$T));
		&bswap($A);
		&bswap($B);
		&bswap($C);
		&bswap($D);
		&mov(&swtmp($i+0),$A);
		&mov(&swtmp($i+1),$B);
		&mov(&swtmp($i+2),$C);
		&mov(&swtmp($i+3),$D);
		}
	&mov(&wparam(1),$T);	# redundant in 1st spin

	&mov($A,&DWP(0,$tmp1));	# load SHA_CTX
	&mov($B,&DWP(4,$tmp1));
	&mov($C,&DWP(8,$tmp1));
	&mov($D,&DWP(12,$tmp1));
	# E is pre-loaded

	for($i=0;$i<16;$i++)	{ &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
	for(;$i<20;$i++)	{ &BODY_16_19($i,@V); unshift(@V,pop(@V)); }
	for(;$i<40;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
	for(;$i<60;$i++)	{ &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
	for(;$i<80;$i++)	{ &BODY_20_39($i,@V); unshift(@V,pop(@V)); }

	(($V[5] eq $D) and ($V[0] eq $E)) or die;	# double-check

	&mov($tmp1,&wparam(0));	# re-load SHA_CTX*
	&mov($D,&wparam(1));	# D is last "T" and is discarded

	&add($E,&DWP(0,$tmp1));	# E is last "A"...
	&add($T,&DWP(4,$tmp1));
	&add($A,&DWP(8,$tmp1));
	&add($B,&DWP(12,$tmp1));
	&add($C,&DWP(16,$tmp1));

	&mov(&DWP(0,$tmp1),$E);	# update SHA_CTX
	 &add($D,64);		# advance input pointer
	&mov(&DWP(4,$tmp1),$T);
	 &cmp($D,&wparam(2));	# have we reached the end yet?
	&mov(&DWP(8,$tmp1),$A);
	 &mov($E,$C);		# C is last "E" which needs to be "pre-loaded"
	&mov(&DWP(12,$tmp1),$B);
	 &mov($T,$D);		# input pointer
	&mov(&DWP(16,$tmp1),$C);
	&jb(&label("loop"));

	&stack_pop(16+3);
&function_end("sha1_block_data_order");

if ($xmm) {
if ($shaext) {
######################################################################
# Intel SHA Extensions implementation of SHA1 update function.
#
my ($ctx,$inp,$num)=("edi","esi","ecx");
my ($ABCD,$E,$E_,$BSWAP)=map("xmm$_",(0..3));
my @MSG=map("xmm$_",(4..7));

sub sha1rnds4 {
 my ($dst,$src,$imm)=@_;
    if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
    {	&data_byte(0x0f,0x3a,0xcc,0xc0|($1<<3)|$2,$imm);	}
}
sub sha1op38 {
 my ($opcodelet,$dst,$src)=@_;
    if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
    {	&data_byte(0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2);	}
}
sub sha1nexte	{ sha1op38(0xc8,@_); }
sub sha1msg1	{ sha1op38(0xc9,@_); }
sub sha1msg2	{ sha1op38(0xca,@_); }

&function_begin("_sha1_block_data_order_shaext");
	&call	(&label("pic_point"));	# make it PIC!
	&set_label("pic_point");
	&blindpop($tmp1);
	&lea	($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1));
&set_label("shaext_shortcut");
	&mov	($ctx,&wparam(0));
	&mov	("ebx","esp");
	&mov	($inp,&wparam(1));
	&mov	($num,&wparam(2));
	&sub	("esp",32);

	&movdqu	($ABCD,&QWP(0,$ctx));
	&movd	($E,&DWP(16,$ctx));
	&and	("esp",-32);
	&movdqa	($BSWAP,&QWP(0x50,$tmp1));	# byte-n-word swap

	&movdqu	(@MSG[0],&QWP(0,$inp));
	&pshufd	($ABCD,$ABCD,0b00011011);	# flip word order
	&movdqu	(@MSG[1],&QWP(0x10,$inp));
	&pshufd	($E,$E,0b00011011);		# flip word order
	&movdqu	(@MSG[2],&QWP(0x20,$inp));
	&pshufb	(@MSG[0],$BSWAP);
	&movdqu	(@MSG[3],&QWP(0x30,$inp));
	&pshufb	(@MSG[1],$BSWAP);
	&pshufb	(@MSG[2],$BSWAP);
	&pshufb	(@MSG[3],$BSWAP);
	&jmp	(&label("loop_shaext"));

&set_label("loop_shaext",16);
	&dec		($num);
	&lea		("eax",&DWP(0x40,$inp));
	&movdqa		(&QWP(0,"esp"),$E);	# offload $E
	&paddd		($E,@MSG[0]);
	&cmovne		($inp,"eax");
	&movdqa		(&QWP(16,"esp"),$ABCD);	# offload $ABCD

for($i=0;$i<20-4;$i+=2) {
	&sha1msg1	(@MSG[0],@MSG[1]);
	&movdqa		($E_,$ABCD);
	&sha1rnds4	($ABCD,$E,int($i/5));	# 0-3...
	&sha1nexte	($E_,@MSG[1]);
	&pxor		(@MSG[0],@MSG[2]);
	&sha1msg1	(@MSG[1],@MSG[2]);
	&sha1msg2	(@MSG[0],@MSG[3]);

	&movdqa		($E,$ABCD);
	&sha1rnds4	($ABCD,$E_,int(($i+1)/5));
	&sha1nexte	($E,@MSG[2]);
	&pxor		(@MSG[1],@MSG[3]);
	&sha1msg2	(@MSG[1],@MSG[0]);

	push(@MSG,shift(@MSG));	push(@MSG,shift(@MSG));
}
	&movdqu		(@MSG[0],&QWP(0,$inp));
	&movdqa		($E_,$ABCD);
	&sha1rnds4	($ABCD,$E,3);		# 64-67
	&sha1nexte	($E_,@MSG[1]);
	&movdqu		(@MSG[1],&QWP(0x10,$inp));
	&pshufb		(@MSG[0],$BSWAP);

	&movdqa		($E,$ABCD);
	&sha1rnds4	($ABCD,$E_,3);		# 68-71
	&sha1nexte	($E,@MSG[2]);
	&movdqu		(@MSG[2],&QWP(0x20,$inp));
	&pshufb		(@MSG[1],$BSWAP);

	&movdqa		($E_,$ABCD);
	&sha1rnds4	($ABCD,$E,3);		# 72-75
	&sha1nexte	($E_,@MSG[3]);
	&movdqu		(@MSG[3],&QWP(0x30,$inp));
	&pshufb		(@MSG[2],$BSWAP);

	&movdqa		($E,$ABCD);
	&sha1rnds4	($ABCD,$E_,3);		# 76-79
	&movdqa		($E_,&QWP(0,"esp"));
	&pshufb		(@MSG[3],$BSWAP);
	&sha1nexte	($E,$E_);
	&paddd		($ABCD,&QWP(16,"esp"));

	&jnz		(&label("loop_shaext"));

	&pshufd	($ABCD,$ABCD,0b00011011);
	&pshufd	($E,$E,0b00011011);
	&movdqu	(&QWP(0,$ctx),$ABCD)
	&movd	(&DWP(16,$ctx),$E);
	&mov	("esp","ebx");
&function_end("_sha1_block_data_order_shaext");
}
######################################################################
# The SSSE3 implementation.
#
# %xmm[0-7] are used as ring @X[] buffer containing quadruples of last
# 32 elements of the message schedule or Xupdate outputs. First 4
# quadruples are simply byte-swapped input, next 4 are calculated
# according to method originally suggested by Dean Gaudet (modulo
# being implemented in SSSE3). Once 8 quadruples or 32 elements are
# collected, it switches to routine proposed by Max Locktyukhin.
#
# Calculations inevitably require temporary registers, and there are
# no %xmm registers left to spare. For this reason part of the ring
# buffer, X[2..4] to be specific, is offloaded to 3 quadriples ring
# buffer on the stack. Keep in mind that X[2] is alias X[-6], X[3] -
# X[-5], and X[4] - X[-4]...
#
# Another notable optimization is aggressive stack frame compression
# aiming to minimize amount of 9-byte instructions...
#
# Yet another notable optimization is "jumping" $B variable. It means
# that there is no register permanently allocated for $B value. This
# allowed to eliminate one instruction from body_20_39...
#
my $Xi=4;			# 4xSIMD Xupdate round, start pre-seeded
my @X=map("xmm$_",(4..7,0..3));	# pre-seeded for $Xi=4
my @V=($A,$B,$C,$D,$E);
my $j=0;			# hash round
my $rx=0;
my @T=($T,$tmp1);
my $inp;

my $_rol=sub { &rol(@_) };
my $_ror=sub { &ror(@_) };

&function_begin("_sha1_block_data_order_ssse3");
	&call	(&label("pic_point"));	# make it PIC!
	&set_label("pic_point");
	&blindpop($tmp1);
	&lea	($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1));
&set_label("ssse3_shortcut");

	&movdqa	(@X[3],&QWP(0,$tmp1));		# K_00_19
	&movdqa	(@X[4],&QWP(16,$tmp1));		# K_20_39
	&movdqa	(@X[5],&QWP(32,$tmp1));		# K_40_59
	&movdqa	(@X[6],&QWP(48,$tmp1));		# K_60_79
	&movdqa	(@X[2],&QWP(64,$tmp1));		# pbswap mask

	&mov	($E,&wparam(0));		# load argument block
	&mov	($inp=@T[1],&wparam(1));
	&mov	($D,&wparam(2));
	&mov	(@T[0],"esp");

	# stack frame layout
	#
	# +0	X[0]+K	X[1]+K	X[2]+K	X[3]+K	# XMM->IALU xfer area
	#	X[4]+K	X[5]+K	X[6]+K	X[7]+K
	#	X[8]+K	X[9]+K	X[10]+K	X[11]+K
	#	X[12]+K	X[13]+K	X[14]+K	X[15]+K
	#
	# +64	X[0]	X[1]	X[2]	X[3]	# XMM->XMM backtrace area
	#	X[4]	X[5]	X[6]	X[7]
	#	X[8]	X[9]	X[10]	X[11]	# even borrowed for K_00_19
	#
	# +112	K_20_39	K_20_39	K_20_39	K_20_39	# constants
	#	K_40_59	K_40_59	K_40_59	K_40_59
	#	K_60_79	K_60_79	K_60_79	K_60_79
	#	K_00_19	K_00_19	K_00_19	K_00_19
	#	pbswap mask
	#
	# +192	ctx				# argument block
	# +196	inp
	# +200	end
	# +204	esp
	&sub	("esp",208);
	&and	("esp",-64);

	&movdqa	(&QWP(112+0,"esp"),@X[4]);	# copy constants
	&movdqa	(&QWP(112+16,"esp"),@X[5]);
	&movdqa	(&QWP(112+32,"esp"),@X[6]);
	&shl	($D,6);				# len*64
	&movdqa	(&QWP(112+48,"esp"),@X[3]);
	&add	($D,$inp);			# end of input
	&movdqa	(&QWP(112+64,"esp"),@X[2]);
	&add	($inp,64);
	&mov	(&DWP(192+0,"esp"),$E);		# save argument block
	&mov	(&DWP(192+4,"esp"),$inp);
	&mov	(&DWP(192+8,"esp"),$D);
	&mov	(&DWP(192+12,"esp"),@T[0]);	# save original %esp

	&mov	($A,&DWP(0,$E));		# load context
	&mov	($B,&DWP(4,$E));
	&mov	($C,&DWP(8,$E));
	&mov	($D,&DWP(12,$E));
	&mov	($E,&DWP(16,$E));
	&mov	(@T[0],$B);			# magic seed

	&movdqu	(@X[-4&7],&QWP(-64,$inp));	# load input to %xmm[0-3]
	&movdqu	(@X[-3&7],&QWP(-48,$inp));
	&movdqu	(@X[-2&7],&QWP(-32,$inp));
	&movdqu	(@X[-1&7],&QWP(-16,$inp));
	&pshufb	(@X[-4&7],@X[2]);		# byte swap
	&pshufb	(@X[-3&7],@X[2]);
	&pshufb	(@X[-2&7],@X[2]);
	&movdqa	(&QWP(112-16,"esp"),@X[3]);	# borrow last backtrace slot
	&pshufb	(@X[-1&7],@X[2]);
	&paddd	(@X[-4&7],@X[3]);		# add K_00_19
	&paddd	(@X[-3&7],@X[3]);
	&paddd	(@X[-2&7],@X[3]);
	&movdqa	(&QWP(0,"esp"),@X[-4&7]);	# X[]+K xfer to IALU
	&psubd	(@X[-4&7],@X[3]);		# restore X[]
	&movdqa	(&QWP(0+16,"esp"),@X[-3&7]);
	&psubd	(@X[-3&7],@X[3]);
	&movdqa	(&QWP(0+32,"esp"),@X[-2&7]);
	&mov	(@T[1],$C);
	&psubd	(@X[-2&7],@X[3]);
	&xor	(@T[1],$D);
	&pshufd	(@X[0],@X[-4&7],0xee);		# was &movdqa	(@X[0],@X[-3&7]);
	&and	(@T[0],@T[1]);
	&jmp	(&label("loop"));

######################################################################
# SSE instruction sequence is first broken to groups of independent
# instructions, independent in respect to their inputs and shifter
# (not all architectures have more than one). Then IALU instructions
# are "knitted in" between the SSE groups. Distance is maintained for
# SSE latency of 2 in hope that it fits better upcoming AMD Bulldozer
# [which allegedly also implements SSSE3]...
#
# Temporary registers usage. X[2] is volatile at the entry and at the
# end is restored from backtrace ring buffer. X[3] is expected to
# contain current K_XX_XX constant and is used to calculate X[-1]+K
# from previous round, it becomes volatile the moment the value is
# saved to stack for transfer to IALU. X[4] becomes volatile whenever
# X[-4] is accumulated and offloaded to backtrace ring buffer, at the
# end it is loaded with next K_XX_XX [which becomes X[3] in next
# round]...
#
sub Xupdate_ssse3_16_31()		# recall that $Xi starts with 4
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 40 instructions
  my ($a,$b,$c,$d,$e);

	 eval(shift(@insns));		# ror
	 eval(shift(@insns));
	 eval(shift(@insns));
	&punpcklqdq(@X[0],@X[-3&7]);	# compose "X[-14]" in "X[0]", was &palignr(@X[0],@X[-4&7],8);
	&movdqa	(@X[2],@X[-1&7]);
	 eval(shift(@insns));
	 eval(shift(@insns));

	  &paddd	(@X[3],@X[-1&7]);
	  &movdqa	(&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer
	 eval(shift(@insns));		# rol
	 eval(shift(@insns));
	&psrldq	(@X[2],4);		# "X[-3]", 3 dwords
	 eval(shift(@insns));
	 eval(shift(@insns));
	&pxor	(@X[0],@X[-4&7]);	# "X[0]"^="X[-16]"
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror

	&pxor	(@X[2],@X[-2&7]);	# "X[-3]"^"X[-8]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	&pxor	(@X[0],@X[2]);		# "X[0]"^="X[-3]"^"X[-8]"
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol
	  &movdqa	(&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]);	# X[]+K xfer to IALU
	 eval(shift(@insns));
	 eval(shift(@insns));

	&movdqa	(@X[4],@X[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	&movdqa (@X[2],@X[0]);
	 eval(shift(@insns));

	&pslldq	(@X[4],12);		# "X[0]"<<96, extract one dword
	&paddd	(@X[0],@X[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));

	&psrld	(@X[2],31);
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol
	&movdqa	(@X[3],@X[4]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	&psrld	(@X[4],30);
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	&por	(@X[0],@X[2]);		# "X[0]"<<<=1
	 eval(shift(@insns));
	  &movdqa	(@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5);	# restore X[] from backtrace buffer
	 eval(shift(@insns));
	 eval(shift(@insns));

	&pslld	(@X[3],2);
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol
	&pxor   (@X[0],@X[4]);
	  &movdqa	(@X[4],&QWP(112-16+16*(($Xi)/5),"esp"));	# K_XX_XX
	 eval(shift(@insns));
	 eval(shift(@insns));

	&pxor	(@X[0],@X[3]);		# "X[0]"^=("X[0]"<<96)<<<2
	  &pshufd	(@X[1],@X[-3&7],0xee)	if ($Xi<7);	# was &movdqa	(@X[1],@X[-2&7])
	  &pshufd	(@X[3],@X[-1&7],0xee)	if ($Xi==7);
	 eval(shift(@insns));
	 eval(shift(@insns));

	 foreach (@insns) { eval; }	# remaining instructions [if any]

  $Xi++;	push(@X,shift(@X));	# "rotate" X[]
}

sub Xupdate_ssse3_32_79()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 to 44 instructions
  my ($a,$b,$c,$d,$e);

	 eval(shift(@insns));		# body_20_39
	&pxor	(@X[0],@X[-4&7]);	# "X[0]"="X[-32]"^"X[-16]"
	&punpcklqdq(@X[2],@X[-1&7]);	# compose "X[-6]", was &palignr(@X[2],@X[-2&7],8)
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol

	&pxor	(@X[0],@X[-7&7]);	# "X[0]"^="X[-28]"
	  &movdqa	(&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);	# save X[] to backtrace buffer
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns))		if (@insns[0] =~ /_rol/);
	 if ($Xi%5) {
	  &movdqa	(@X[4],@X[3]);	# "perpetuate" K_XX_XX...
	 } else {			# ... or load next one
	  &movdqa	(@X[4],&QWP(112-16+16*($Xi/5),"esp"));
	 }
	 eval(shift(@insns));		# ror
	  &paddd	(@X[3],@X[-1&7]);
	 eval(shift(@insns));

	&pxor	(@X[0],@X[2]);		# "X[0]"^="X[-6]"
	 eval(shift(@insns));		# body_20_39
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol

	&movdqa	(@X[2],@X[0]);
	  &movdqa	(&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]);	# X[]+K xfer to IALU
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	 eval(shift(@insns));
	 eval(shift(@insns))		if (@insns[0] =~ /_rol/);

	&pslld	(@X[0],2);
	 eval(shift(@insns));		# body_20_39
	 eval(shift(@insns));
	&psrld	(@X[2],30);
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	 eval(shift(@insns));
	 eval(shift(@insns))		if (@insns[1] =~ /_rol/);
	 eval(shift(@insns))		if (@insns[0] =~ /_rol/);

	&por	(@X[0],@X[2]);		# "X[0]"<<<=2
	 eval(shift(@insns));		# body_20_39
	 eval(shift(@insns));
	  &movdqa	(@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19);	# restore X[] from backtrace buffer
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	  &pshufd	(@X[3],@X[-1],0xee)	if ($Xi<19);	# was &movdqa	(@X[3],@X[0])
	 eval(shift(@insns));

	 foreach (@insns) { eval; }	# remaining instructions

  $Xi++;	push(@X,shift(@X));	# "rotate" X[]
}

sub Xuplast_ssse3_80()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 instructions
  my ($a,$b,$c,$d,$e);

	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &paddd	(@X[3],@X[-1&7]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	  &movdqa	(&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]);	# X[]+K xfer IALU

	 foreach (@insns) { eval; }		# remaining instructions

	&mov	($inp=@T[1],&DWP(192+4,"esp"));
	&cmp	($inp,&DWP(192+8,"esp"));
	&je	(&label("done"));

	&movdqa	(@X[3],&QWP(112+48,"esp"));	# K_00_19
	&movdqa	(@X[2],&QWP(112+64,"esp"));	# pbswap mask
	&movdqu	(@X[-4&7],&QWP(0,$inp));	# load input
	&movdqu	(@X[-3&7],&QWP(16,$inp));
	&movdqu	(@X[-2&7],&QWP(32,$inp));
	&movdqu	(@X[-1&7],&QWP(48,$inp));
	&add	($inp,64);
	&pshufb	(@X[-4&7],@X[2]);		# byte swap
	&mov	(&DWP(192+4,"esp"),$inp);
	&movdqa	(&QWP(112-16,"esp"),@X[3]);	# borrow last backtrace slot

  $Xi=0;
}

sub Xloop_ssse3()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 instructions
  my ($a,$b,$c,$d,$e);

	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&pshufb	(@X[($Xi-3)&7],@X[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&paddd	(@X[($Xi-4)&7],@X[3]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&movdqa	(&QWP(0+16*$Xi,"esp"),@X[($Xi-4)&7]);	# X[]+K xfer to IALU
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&psubd	(@X[($Xi-4)&7],@X[3]);

	foreach (@insns) { eval; }
  $Xi++;
}

sub Xtail_ssse3()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 instructions
  my ($a,$b,$c,$d,$e);

	foreach (@insns) { eval; }
}

sub body_00_19 () {	# ((c^d)&b)^d
	# on start @T[0]=(c^d)&b
	return &body_20_39()	if ($rx==19);	$rx++;
	(
	'($a,$b,$c,$d,$e)=@V;'.
	'&$_ror	($b,$j?7:2);',	# $b>>>2
	'&xor	(@T[0],$d);',
	'&mov	(@T[1],$a);',	# $b in next round

	'&add	($e,&DWP(4*($j&15),"esp"));',	# X[]+K xfer
	'&xor	($b,$c);',	# $c^$d for next round

	'&$_rol	($a,5);',
	'&add	($e,@T[0]);',
	'&and	(@T[1],$b);',	# ($b&($c^$d)) for next round

	'&xor	($b,$c);',	# restore $b
	'&add	($e,$a);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
	);
}

sub body_20_39 () {	# b^d^c
	# on entry @T[0]=b^d
	return &body_40_59()	if ($rx==39);	$rx++;
	(
	'($a,$b,$c,$d,$e)=@V;'.
	'&add	($e,&DWP(4*($j&15),"esp"));',	# X[]+K xfer
	'&xor	(@T[0],$d)	if($j==19);'.
	'&xor	(@T[0],$c)	if($j> 19);',	# ($b^$d^$c)
	'&mov	(@T[1],$a);',	# $b in next round

	'&$_rol	($a,5);',
	'&add	($e,@T[0]);',
	'&xor	(@T[1],$c)	if ($j< 79);',	# $b^$d for next round

	'&$_ror	($b,7);',	# $b>>>2
	'&add	($e,$a);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
	);
}

sub body_40_59 () {	# ((b^c)&(c^d))^c
	# on entry @T[0]=(b^c), (c^=d)
	$rx++;
	(
	'($a,$b,$c,$d,$e)=@V;'.
	'&add	($e,&DWP(4*($j&15),"esp"));',	# X[]+K xfer
	'&and	(@T[0],$c)	if ($j>=40);',	# (b^c)&(c^d)
	'&xor	($c,$d)		if ($j>=40);',	# restore $c

	'&$_ror	($b,7);',	# $b>>>2
	'&mov	(@T[1],$a);',	# $b for next round
	'&xor	(@T[0],$c);',

	'&$_rol	($a,5);',
	'&add	($e,@T[0]);',
	'&xor	(@T[1],$c)	if ($j==59);'.
	'&xor	(@T[1],$b)	if ($j< 59);',	# b^c for next round

	'&xor	($b,$c)		if ($j< 59);',	# c^d for next round
	'&add	($e,$a);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
	);
}
######
sub bodyx_00_19 () {	# ((c^d)&b)^d
	# on start @T[0]=(b&c)^(~b&d), $e+=X[]+K
	return &bodyx_20_39()	if ($rx==19);	$rx++;
	(
	'($a,$b,$c,$d,$e)=@V;'.

	'&rorx	($b,$b,2)			if ($j==0);'.	# $b>>>2
	'&rorx	($b,@T[1],7)			if ($j!=0);',	# $b>>>2
	'&lea	($e,&DWP(0,$e,@T[0]));',
	'&rorx	(@T[0],$a,5);',

	'&andn	(@T[1],$a,$c);',
	'&and	($a,$b)',
	'&add	($d,&DWP(4*(($j+1)&15),"esp"));',	# X[]+K xfer

	'&xor	(@T[1],$a)',
	'&add	($e,@T[0]);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
	);
}

sub bodyx_20_39 () {	# b^d^c
	# on start $b=b^c^d
	return &bodyx_40_59()	if ($rx==39);	$rx++;
	(
	'($a,$b,$c,$d,$e)=@V;'.

	'&add	($e,($j==19?@T[0]:$b))',
	'&rorx	($b,@T[1],7);',	# $b>>>2
	'&rorx	(@T[0],$a,5);',

	'&xor	($a,$b)				if ($j<79);',
	'&add	($d,&DWP(4*(($j+1)&15),"esp"))	if ($j<79);',	# X[]+K xfer
	'&xor	($a,$c)				if ($j<79);',
	'&add	($e,@T[0]);'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
	);
}

sub bodyx_40_59 () {	# ((b^c)&(c^d))^c
	# on start $b=((b^c)&(c^d))^c
	return &bodyx_20_39()	if ($rx==59);	$rx++;
	(
	'($a,$b,$c,$d,$e)=@V;'.

	'&rorx	(@T[0],$a,5)',
	'&lea	($e,&DWP(0,$e,$b))',
	'&rorx	($b,@T[1],7)',	# $b>>>2
	'&add	($d,&DWP(4*(($j+1)&15),"esp"))',	# X[]+K xfer

	'&mov	(@T[1],$c)',
	'&xor	($a,$b)',	# b^c for next round
	'&xor	(@T[1],$b)',	# c^d for next round

	'&and	($a,@T[1])',
	'&add	($e,@T[0])',
	'&xor	($a,$b)'	.'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
	);
}

&set_label("loop",16);
	&Xupdate_ssse3_16_31(\&body_00_19);
	&Xupdate_ssse3_16_31(\&body_00_19);
	&Xupdate_ssse3_16_31(\&body_00_19);
	&Xupdate_ssse3_16_31(\&body_00_19);
	&Xupdate_ssse3_32_79(\&body_00_19);
	&Xupdate_ssse3_32_79(\&body_20_39);
	&Xupdate_ssse3_32_79(\&body_20_39);
	&Xupdate_ssse3_32_79(\&body_20_39);
	&Xupdate_ssse3_32_79(\&body_20_39);
	&Xupdate_ssse3_32_79(\&body_20_39);
	&Xupdate_ssse3_32_79(\&body_40_59);
	&Xupdate_ssse3_32_79(\&body_40_59);
	&Xupdate_ssse3_32_79(\&body_40_59);
	&Xupdate_ssse3_32_79(\&body_40_59);
	&Xupdate_ssse3_32_79(\&body_40_59);
	&Xupdate_ssse3_32_79(\&body_20_39);
	&Xuplast_ssse3_80(\&body_20_39);	# can jump to "done"

				$saved_j=$j; @saved_V=@V;

	&Xloop_ssse3(\&body_20_39);
	&Xloop_ssse3(\&body_20_39);
	&Xloop_ssse3(\&body_20_39);

	&mov	(@T[1],&DWP(192,"esp"));	# update context
	&add	($A,&DWP(0,@T[1]));
	&add	(@T[0],&DWP(4,@T[1]));		# $b
	&add	($C,&DWP(8,@T[1]));
	&mov	(&DWP(0,@T[1]),$A);
	&add	($D,&DWP(12,@T[1]));
	&mov	(&DWP(4,@T[1]),@T[0]);
	&add	($E,&DWP(16,@T[1]));
	&mov	(&DWP(8,@T[1]),$C);
	&mov	($B,$C);
	&mov	(&DWP(12,@T[1]),$D);
	&xor	($B,$D);
	&mov	(&DWP(16,@T[1]),$E);
	&mov	(@T[1],@T[0]);
	&pshufd	(@X[0],@X[-4&7],0xee);		# was &movdqa	(@X[0],@X[-3&7]);
	&and	(@T[0],$B);
	&mov	($B,$T[1]);

	&jmp	(&label("loop"));

&set_label("done",16);		$j=$saved_j; @V=@saved_V;

	&Xtail_ssse3(\&body_20_39);
	&Xtail_ssse3(\&body_20_39);
	&Xtail_ssse3(\&body_20_39);

	&mov	(@T[1],&DWP(192,"esp"));	# update context
	&add	($A,&DWP(0,@T[1]));
	&mov	("esp",&DWP(192+12,"esp"));	# restore %esp
	&add	(@T[0],&DWP(4,@T[1]));		# $b
	&add	($C,&DWP(8,@T[1]));
	&mov	(&DWP(0,@T[1]),$A);
	&add	($D,&DWP(12,@T[1]));
	&mov	(&DWP(4,@T[1]),@T[0]);
	&add	($E,&DWP(16,@T[1]));
	&mov	(&DWP(8,@T[1]),$C);
	&mov	(&DWP(12,@T[1]),$D);
	&mov	(&DWP(16,@T[1]),$E);

&function_end("_sha1_block_data_order_ssse3");

$rx=0;	# reset

if ($ymm) {
my $Xi=4;			# 4xSIMD Xupdate round, start pre-seeded
my @X=map("xmm$_",(4..7,0..3));	# pre-seeded for $Xi=4
my @V=($A,$B,$C,$D,$E);
my $j=0;			# hash round
my @T=($T,$tmp1);
my $inp;

my $_rol=sub { &shld(@_[0],@_) };
my $_ror=sub { &shrd(@_[0],@_) };

&function_begin("_sha1_block_data_order_avx");
	&call	(&label("pic_point"));	# make it PIC!
	&set_label("pic_point");
	&blindpop($tmp1);
	&lea	($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1));
&set_label("avx_shortcut");
	&vzeroall();

	&vmovdqa(@X[3],&QWP(0,$tmp1));		# K_00_19
	&vmovdqa(@X[4],&QWP(16,$tmp1));		# K_20_39
	&vmovdqa(@X[5],&QWP(32,$tmp1));		# K_40_59
	&vmovdqa(@X[6],&QWP(48,$tmp1));		# K_60_79
	&vmovdqa(@X[2],&QWP(64,$tmp1));		# pbswap mask

	&mov	($E,&wparam(0));		# load argument block
	&mov	($inp=@T[1],&wparam(1));
	&mov	($D,&wparam(2));
	&mov	(@T[0],"esp");

	# stack frame layout
	#
	# +0	X[0]+K	X[1]+K	X[2]+K	X[3]+K	# XMM->IALU xfer area
	#	X[4]+K	X[5]+K	X[6]+K	X[7]+K
	#	X[8]+K	X[9]+K	X[10]+K	X[11]+K
	#	X[12]+K	X[13]+K	X[14]+K	X[15]+K
	#
	# +64	X[0]	X[1]	X[2]	X[3]	# XMM->XMM backtrace area
	#	X[4]	X[5]	X[6]	X[7]
	#	X[8]	X[9]	X[10]	X[11]	# even borrowed for K_00_19
	#
	# +112	K_20_39	K_20_39	K_20_39	K_20_39	# constants
	#	K_40_59	K_40_59	K_40_59	K_40_59
	#	K_60_79	K_60_79	K_60_79	K_60_79
	#	K_00_19	K_00_19	K_00_19	K_00_19
	#	pbswap mask
	#
	# +192	ctx				# argument block
	# +196	inp
	# +200	end
	# +204	esp
	&sub	("esp",208);
	&and	("esp",-64);

	&vmovdqa(&QWP(112+0,"esp"),@X[4]);	# copy constants
	&vmovdqa(&QWP(112+16,"esp"),@X[5]);
	&vmovdqa(&QWP(112+32,"esp"),@X[6]);
	&shl	($D,6);				# len*64
	&vmovdqa(&QWP(112+48,"esp"),@X[3]);
	&add	($D,$inp);			# end of input
	&vmovdqa(&QWP(112+64,"esp"),@X[2]);
	&add	($inp,64);
	&mov	(&DWP(192+0,"esp"),$E);		# save argument block
	&mov	(&DWP(192+4,"esp"),$inp);
	&mov	(&DWP(192+8,"esp"),$D);
	&mov	(&DWP(192+12,"esp"),@T[0]);	# save original %esp

	&mov	($A,&DWP(0,$E));		# load context
	&mov	($B,&DWP(4,$E));
	&mov	($C,&DWP(8,$E));
	&mov	($D,&DWP(12,$E));
	&mov	($E,&DWP(16,$E));
	&mov	(@T[0],$B);			# magic seed

	&vmovdqu(@X[-4&7],&QWP(-64,$inp));	# load input to %xmm[0-3]
	&vmovdqu(@X[-3&7],&QWP(-48,$inp));
	&vmovdqu(@X[-2&7],&QWP(-32,$inp));
	&vmovdqu(@X[-1&7],&QWP(-16,$inp));
	&vpshufb(@X[-4&7],@X[-4&7],@X[2]);	# byte swap
	&vpshufb(@X[-3&7],@X[-3&7],@X[2]);
	&vpshufb(@X[-2&7],@X[-2&7],@X[2]);
	&vmovdqa(&QWP(112-16,"esp"),@X[3]);	# borrow last backtrace slot
	&vpshufb(@X[-1&7],@X[-1&7],@X[2]);
	&vpaddd	(@X[0],@X[-4&7],@X[3]);		# add K_00_19
	&vpaddd	(@X[1],@X[-3&7],@X[3]);
	&vpaddd	(@X[2],@X[-2&7],@X[3]);
	&vmovdqa(&QWP(0,"esp"),@X[0]);		# X[]+K xfer to IALU
	&mov	(@T[1],$C);
	&vmovdqa(&QWP(0+16,"esp"),@X[1]);
	&xor	(@T[1],$D);
	&vmovdqa(&QWP(0+32,"esp"),@X[2]);
	&and	(@T[0],@T[1]);
	&jmp	(&label("loop"));

sub Xupdate_avx_16_31()		# recall that $Xi starts with 4
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 40 instructions
  my ($a,$b,$c,$d,$e);

	 eval(shift(@insns));
	 eval(shift(@insns));
	&vpalignr(@X[0],@X[-3&7],@X[-4&7],8);	# compose "X[-14]" in "X[0]"
	 eval(shift(@insns));
	 eval(shift(@insns));

	  &vpaddd	(@X[3],@X[3],@X[-1&7]);
	  &vmovdqa	(&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vpsrldq(@X[2],@X[-1&7],4);		# "X[-3]", 3 dwords
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vpxor	(@X[0],@X[0],@X[-4&7]);		# "X[0]"^="X[-16]"
	 eval(shift(@insns));
	 eval(shift(@insns));

	&vpxor	(@X[2],@X[2],@X[-2&7]);		# "X[-3]"^"X[-8]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vmovdqa	(&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]);	# X[]+K xfer to IALU
	 eval(shift(@insns));
	 eval(shift(@insns));

	&vpxor	(@X[0],@X[0],@X[2]);		# "X[0]"^="X[-3]"^"X[-8]"
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	&vpsrld	(@X[2],@X[0],31);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	&vpslldq(@X[4],@X[0],12);		# "X[0]"<<96, extract one dword
	&vpaddd	(@X[0],@X[0],@X[0]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	&vpsrld	(@X[3],@X[4],30);
	&vpor	(@X[0],@X[0],@X[2]);		# "X[0]"<<<=1
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	&vpslld	(@X[4],@X[4],2);
	  &vmovdqa	(@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5);	# restore X[] from backtrace buffer
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vpxor	(@X[0],@X[0],@X[3]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	&vpxor	(@X[0],@X[0],@X[4]);		# "X[0]"^=("X[0]"<<96)<<<2
	 eval(shift(@insns));
	 eval(shift(@insns));
	  &vmovdqa	(@X[4],&QWP(112-16+16*(($Xi)/5),"esp"));	# K_XX_XX
	 eval(shift(@insns));
	 eval(shift(@insns));

	 foreach (@insns) { eval; }	# remaining instructions [if any]

  $Xi++;	push(@X,shift(@X));	# "rotate" X[]
}

sub Xupdate_avx_32_79()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 to 44 instructions
  my ($a,$b,$c,$d,$e);

	&vpalignr(@X[2],@X[-1&7],@X[-2&7],8);	# compose "X[-6]"
	&vpxor	(@X[0],@X[0],@X[-4&7]);	# "X[0]"="X[-32]"^"X[-16]"
	 eval(shift(@insns));		# body_20_39
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol

	&vpxor	(@X[0],@X[0],@X[-7&7]);	# "X[0]"^="X[-28]"
	  &vmovdqa	(&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);	# save X[] to backtrace buffer
	 eval(shift(@insns));
	 eval(shift(@insns));
	 if ($Xi%5) {
	  &vmovdqa	(@X[4],@X[3]);	# "perpetuate" K_XX_XX...
	 } else {			# ... or load next one
	  &vmovdqa	(@X[4],&QWP(112-16+16*($Xi/5),"esp"));
	 }
	  &vpaddd	(@X[3],@X[3],@X[-1&7]);
	 eval(shift(@insns));		# ror
	 eval(shift(@insns));

	&vpxor	(@X[0],@X[0],@X[2]);		# "X[0]"^="X[-6]"
	 eval(shift(@insns));		# body_20_39
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol

	&vpsrld	(@X[2],@X[0],30);
	  &vmovdqa	(&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]);	# X[]+K xfer to IALU
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	 eval(shift(@insns));

	&vpslld	(@X[0],@X[0],2);
	 eval(shift(@insns));		# body_20_39
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	 eval(shift(@insns));

	&vpor	(@X[0],@X[0],@X[2]);	# "X[0]"<<<=2
	 eval(shift(@insns));		# body_20_39
	 eval(shift(@insns));
	  &vmovdqa	(@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19);	# restore X[] from backtrace buffer
	 eval(shift(@insns));
	 eval(shift(@insns));		# rol
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));		# ror
	 eval(shift(@insns));

	 foreach (@insns) { eval; }	# remaining instructions

  $Xi++;	push(@X,shift(@X));	# "rotate" X[]
}

sub Xuplast_avx_80()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 instructions
  my ($a,$b,$c,$d,$e);

	 eval(shift(@insns));
	  &vpaddd	(@X[3],@X[3],@X[-1&7]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));

	  &vmovdqa	(&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]);	# X[]+K xfer IALU

	 foreach (@insns) { eval; }		# remaining instructions

	&mov	($inp=@T[1],&DWP(192+4,"esp"));
	&cmp	($inp,&DWP(192+8,"esp"));
	&je	(&label("done"));

	&vmovdqa(@X[3],&QWP(112+48,"esp"));	# K_00_19
	&vmovdqa(@X[2],&QWP(112+64,"esp"));	# pbswap mask
	&vmovdqu(@X[-4&7],&QWP(0,$inp));	# load input
	&vmovdqu(@X[-3&7],&QWP(16,$inp));
	&vmovdqu(@X[-2&7],&QWP(32,$inp));
	&vmovdqu(@X[-1&7],&QWP(48,$inp));
	&add	($inp,64);
	&vpshufb(@X[-4&7],@X[-4&7],@X[2]);		# byte swap
	&mov	(&DWP(192+4,"esp"),$inp);
	&vmovdqa(&QWP(112-16,"esp"),@X[3]);	# borrow last backtrace slot

  $Xi=0;
}

sub Xloop_avx()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 instructions
  my ($a,$b,$c,$d,$e);

	 eval(shift(@insns));
	 eval(shift(@insns));
	&vpshufb	(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vpaddd	(@X[$Xi&7],@X[($Xi-4)&7],@X[3]);
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	 eval(shift(@insns));
	&vmovdqa	(&QWP(0+16*$Xi,"esp"),@X[$Xi&7]);	# X[]+K xfer to IALU
	 eval(shift(@insns));
	 eval(shift(@insns));

	foreach (@insns) { eval; }
  $Xi++;
}

sub Xtail_avx()
{ use integer;
  my $body = shift;
  my @insns = (&$body,&$body,&$body,&$body);	# 32 instructions
  my ($a,$b,$c,$d,$e);

	foreach (@insns) { eval; }
}

&set_label("loop",16);
	&Xupdate_avx_16_31(\&body_00_19);
	&Xupdate_avx_16_31(\&body_00_19);
	&Xupdate_avx_16_31(\&body_00_19);
	&Xupdate_avx_16_31(\&body_00_19);
	&Xupdate_avx_32_79(\&body_00_19);
	&Xupdate_avx_32_79(\&body_20_39);
	&Xupdate_avx_32_79(\&body_20_39);
	&Xupdate_avx_32_79(\&body_20_39);
	&Xupdate_avx_32_79(\&body_20_39);
	&Xupdate_avx_32_79(\&body_20_39);
	&Xupdate_avx_32_79(\&body_40_59);
	&Xupdate_avx_32_79(\&body_40_59);
	&Xupdate_avx_32_79(\&body_40_59);
	&Xupdate_avx_32_79(\&body_40_59);
	&Xupdate_avx_32_79(\&body_40_59);
	&Xupdate_avx_32_79(\&body_20_39);
	&Xuplast_avx_80(\&body_20_39);	# can jump to "done"

				$saved_j=$j; @saved_V=@V;

	&Xloop_avx(\&body_20_39);
	&Xloop_avx(\&body_20_39);
	&Xloop_avx(\&body_20_39);

	&mov	(@T[1],&DWP(192,"esp"));	# update context
	&add	($A,&DWP(0,@T[1]));
	&add	(@T[0],&DWP(4,@T[1]));		# $b
	&add	($C,&DWP(8,@T[1]));
	&mov	(&DWP(0,@T[1]),$A);
	&add	($D,&DWP(12,@T[1]));
	&mov	(&DWP(4,@T[1]),@T[0]);
	&add	($E,&DWP(16,@T[1]));
	&mov	($B,$C);
	&mov	(&DWP(8,@T[1]),$C);
	&xor	($B,$D);
	&mov	(&DWP(12,@T[1]),$D);
	&mov	(&DWP(16,@T[1]),$E);
	&mov	(@T[1],@T[0]);
	&and	(@T[0],$B);
	&mov	($B,@T[1]);

	&jmp	(&label("loop"));

&set_label("done",16);		$j=$saved_j; @V=@saved_V;

	&Xtail_avx(\&body_20_39);
	&Xtail_avx(\&body_20_39);
	&Xtail_avx(\&body_20_39);

	&vzeroall();

	&mov	(@T[1],&DWP(192,"esp"));	# update context
	&add	($A,&DWP(0,@T[1]));
	&mov	("esp",&DWP(192+12,"esp"));	# restore %esp
	&add	(@T[0],&DWP(4,@T[1]));		# $b
	&add	($C,&DWP(8,@T[1]));
	&mov	(&DWP(0,@T[1]),$A);
	&add	($D,&DWP(12,@T[1]));
	&mov	(&DWP(4,@T[1]),@T[0]);
	&add	($E,&DWP(16,@T[1]));
	&mov	(&DWP(8,@T[1]),$C);
	&mov	(&DWP(12,@T[1]),$D);
	&mov	(&DWP(16,@T[1]),$E);
&function_end("_sha1_block_data_order_avx");
}
&set_label("K_XX_XX",64);
&data_word(0x5a827999,0x5a827999,0x5a827999,0x5a827999);	# K_00_19
&data_word(0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1);	# K_20_39
&data_word(0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc);	# K_40_59
&data_word(0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6);	# K_60_79
&data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f);	# pbswap mask
&data_byte(0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0);
}
&asciz("SHA1 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>");

&asm_finish();

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# SHA1 block procedure for Alpha.

# On 21264 performance is 33% better than code generated by vendor
# compiler, and 75% better than GCC [3.4], and in absolute terms is
# 8.7 cycles per processed byte. Implementation features vectorized
# byte swap, but not Xupdate.

@X=(	"\$0",	"\$1",	"\$2",	"\$3",	"\$4",	"\$5",	"\$6",	"\$7",
	"\$8",	"\$9",	"\$10",	"\$11",	"\$12",	"\$13",	"\$14",	"\$15");
$ctx="a0";	# $16
$inp="a1";
$num="a2";
$A="a3";
$B="a4";	# 20
$C="a5";
$D="t8";
$E="t9";	@V=($A,$B,$C,$D,$E);
$t0="t10";	# 24
$t1="t11";
$t2="ra";
$t3="t12";
$K="AT";	# 28

sub BODY_00_19 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
$code.=<<___ if ($i==0);
	ldq_u	@X[0],0+0($inp)
	ldq_u	@X[1],0+7($inp)
___
$code.=<<___ if (!($i&1) && $i<14);
	ldq_u	@X[$i+2],($i+2)*4+0($inp)
	ldq_u	@X[$i+3],($i+2)*4+7($inp)
___
$code.=<<___ if (!($i&1) && $i<15);
	extql	@X[$i],$inp,@X[$i]
	extqh	@X[$i+1],$inp,@X[$i+1]

	or	@X[$i+1],@X[$i],@X[$i]	# pair of 32-bit values are fetched

	srl	@X[$i],24,$t0		# vectorized byte swap
	srl	@X[$i],8,$t2

	sll	@X[$i],8,$t3
	sll	@X[$i],24,@X[$i]
	zapnot	$t0,0x11,$t0
	zapnot	$t2,0x22,$t2

	zapnot	@X[$i],0x88,@X[$i]
	or	$t0,$t2,$t0
	zapnot	$t3,0x44,$t3
	sll	$a,5,$t1

	or	@X[$i],$t0,@X[$i]
	addl	$K,$e,$e
	and	$b,$c,$t2
	zapnot	$a,0xf,$a

	or	@X[$i],$t3,@X[$i]
	srl	$a,27,$t0
	bic	$d,$b,$t3
	sll	$b,30,$b

	extll	@X[$i],4,@X[$i+1]	# extract upper half
	or	$t2,$t3,$t2
	addl	@X[$i],$e,$e

	addl	$t1,$e,$e
	srl	$b,32,$t3
	zapnot	@X[$i],0xf,@X[$i]

	addl	$t0,$e,$e
	addl	$t2,$e,$e
	or	$t3,$b,$b
___
$code.=<<___ if (($i&1) && $i<15);
	sll	$a,5,$t1
	addl	$K,$e,$e
	and	$b,$c,$t2
	zapnot	$a,0xf,$a

	srl	$a,27,$t0
	addl	@X[$i%16],$e,$e
	bic	$d,$b,$t3
	sll	$b,30,$b

	or	$t2,$t3,$t2
	addl	$t1,$e,$e
	srl	$b,32,$t3
	zapnot	@X[$i],0xf,@X[$i]

	addl	$t0,$e,$e
	addl	$t2,$e,$e
	or	$t3,$b,$b
___
$code.=<<___ if ($i>=15);	# with forward Xupdate
	sll	$a,5,$t1
	addl	$K,$e,$e
	and	$b,$c,$t2
	xor	@X[($j+2)%16],@X[$j%16],@X[$j%16]

	zapnot	$a,0xf,$a
	addl	@X[$i%16],$e,$e
	bic	$d,$b,$t3
	xor	@X[($j+8)%16],@X[$j%16],@X[$j%16]

	srl	$a,27,$t0
	addl	$t1,$e,$e
	or	$t2,$t3,$t2
	xor	@X[($j+13)%16],@X[$j%16],@X[$j%16]

	sll	$b,30,$b
	addl	$t0,$e,$e
	srl	@X[$j%16],31,$t1

	addl	$t2,$e,$e
	srl	$b,32,$t3
	addl	@X[$j%16],@X[$j%16],@X[$j%16]

	or	$t3,$b,$b
	zapnot	@X[$i%16],0xf,@X[$i%16]
	or	$t1,@X[$j%16],@X[$j%16]
___
}

sub BODY_20_39 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
$code.=<<___ if ($i<79);	# with forward Xupdate
	sll	$a,5,$t1
	addl	$K,$e,$e
	zapnot	$a,0xf,$a
	xor	@X[($j+2)%16],@X[$j%16],@X[$j%16]

	sll	$b,30,$t3
	addl	$t1,$e,$e
	xor	$b,$c,$t2
	xor	@X[($j+8)%16],@X[$j%16],@X[$j%16]

	srl	$b,2,$b
	addl	@X[$i%16],$e,$e
	xor	$d,$t2,$t2
	xor	@X[($j+13)%16],@X[$j%16],@X[$j%16]

	srl	@X[$j%16],31,$t1
	addl	$t2,$e,$e
	srl	$a,27,$t0
	addl	@X[$j%16],@X[$j%16],@X[$j%16]

	or	$t3,$b,$b
	addl	$t0,$e,$e
	or	$t1,@X[$j%16],@X[$j%16]
___
$code.=<<___ if ($i<77);
	zapnot	@X[$i%16],0xf,@X[$i%16]
___
$code.=<<___ if ($i==79);	# with context fetch
	sll	$a,5,$t1
	addl	$K,$e,$e
	zapnot	$a,0xf,$a
	ldl	@X[0],0($ctx)

	sll	$b,30,$t3
	addl	$t1,$e,$e
	xor	$b,$c,$t2
	ldl	@X[1],4($ctx)

	srl	$b,2,$b
	addl	@X[$i%16],$e,$e
	xor	$d,$t2,$t2
	ldl	@X[2],8($ctx)

	srl	$a,27,$t0
	addl	$t2,$e,$e
	ldl	@X[3],12($ctx)

	or	$t3,$b,$b
	addl	$t0,$e,$e
	ldl	@X[4],16($ctx)
___
}

sub BODY_40_59 {
my ($i,$a,$b,$c,$d,$e)=@_;
my $j=$i+1;
$code.=<<___;	# with forward Xupdate
	sll	$a,5,$t1
	addl	$K,$e,$e
	zapnot	$a,0xf,$a
	xor	@X[($j+2)%16],@X[$j%16],@X[$j%16]

	srl	$a,27,$t0
	and	$b,$c,$t2
	and	$b,$d,$t3
	xor	@X[($j+8)%16],@X[$j%16],@X[$j%16]

	sll	$b,30,$b
	addl	$t1,$e,$e
	xor	@X[($j+13)%16],@X[$j%16],@X[$j%16]

	srl	@X[$j%16],31,$t1
	addl	$t0,$e,$e
	or	$t2,$t3,$t2
	and	$c,$d,$t3

	or	$t2,$t3,$t2
	srl	$b,32,$t3
	addl	@X[$i%16],$e,$e
	addl	@X[$j%16],@X[$j%16],@X[$j%16]

	or	$t3,$b,$b
	addl	$t2,$e,$e
	or	$t1,@X[$j%16],@X[$j%16]
	zapnot	@X[$i%16],0xf,@X[$i%16]
___
}

$code=<<___;
#ifdef __linux__
#include <asm/regdef.h>
#else
#include <asm.h>
#include <regdef.h>
#endif

.text

.set	noat
.set	noreorder
.globl	sha1_block_data_order
.align	5
.ent	sha1_block_data_order
sha1_block_data_order:
	lda	sp,-64(sp)
	stq	ra,0(sp)
	stq	s0,8(sp)
	stq	s1,16(sp)
	stq	s2,24(sp)
	stq	s3,32(sp)
	stq	s4,40(sp)
	stq	s5,48(sp)
	stq	fp,56(sp)
	.mask	0x0400fe00,-64
	.frame	sp,64,ra
	.prologue 0

	ldl	$A,0($ctx)
	ldl	$B,4($ctx)
	sll	$num,6,$num
	ldl	$C,8($ctx)
	ldl	$D,12($ctx)
	ldl	$E,16($ctx)
	addq	$inp,$num,$num

.Lloop:
	.set	noreorder
	ldah	$K,23170(zero)
	zapnot	$B,0xf,$B
	lda	$K,31129($K)	# K_00_19
___
for ($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }

$code.=<<___;
	ldah	$K,28378(zero)
	lda	$K,-5215($K)	# K_20_39
___
for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }

$code.=<<___;
	ldah	$K,-28900(zero)
	lda	$K,-17188($K)	# K_40_59
___
for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }

$code.=<<___;
	ldah	$K,-13725(zero)
	lda	$K,-15914($K)	# K_60_79
___
for (;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }

$code.=<<___;
	addl	@X[0],$A,$A
	addl	@X[1],$B,$B
	addl	@X[2],$C,$C
	addl	@X[3],$D,$D
	addl	@X[4],$E,$E
	stl	$A,0($ctx)
	stl	$B,4($ctx)
	addq	$inp,64,$inp
	stl	$C,8($ctx)
	stl	$D,12($ctx)
	stl	$E,16($ctx)
	cmpult	$inp,$num,$t1
	bne	$t1,.Lloop

	.set	noreorder
	ldq	ra,0(sp)
	ldq	s0,8(sp)
	ldq	s1,16(sp)
	ldq	s2,24(sp)
	ldq	s3,32(sp)
	ldq	s4,40(sp)
	ldq	s5,48(sp)
	ldq	fp,56(sp)
	lda	sp,64(sp)
	ret	(ra)
.end	sha1_block_data_order
.ascii	"SHA1 block transform for Alpha, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
___
$output=pop and open STDOUT,">$output";
print $code;
close STDOUT or die "error closing STDOUT: $!";