#! /usr/bin/env perl
# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
#
# AES-NI-CTR+GHASH stitch.
#
# February 2013
#
# OpenSSL GCM implementation is organized in such way that its
# performance is rather close to the sum of its streamed components,
# in the context parallelized AES-NI CTR and modulo-scheduled
# PCLMULQDQ-enabled GHASH. Unfortunately, as no stitch implementation
# was observed to perform significantly better than the sum of the
# components on contemporary CPUs, the effort was deemed impossible to
# justify. This module is based on combination of Intel submissions,
# [1] and [2], with MOVBE twist suggested by Ilya Albrekht and Max
# Locktyukhin of Intel Corp. who verified that it reduces shuffles
# pressure with notable relative improvement, achieving 1.0 cycle per
# byte processed with 128-bit key on Haswell processor, 0.74 - on
# Broadwell, 0.63 - on Skylake... [Mentioned results are raw profiled
# measurements for favourable packet size, one divisible by 96.
# Applications using the EVP interface will observe a few percent
# worse performance.]
#
# Knights Landing processes 1 byte in 1.25 cycles (measured with EVP).
#
# [1] http://rt.openssl.org/Ticket/Display.html?id=2900&user=guest&pass=guest
# [2] http://www.intel.com/content/dam/www/public/us/en/documents/software-support/enabling-high-performance-gcm.pdf

$flavour = shift;
$output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.20) + ($1>=2.22);
}

if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.09) + ($1>=2.10);
}

if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
	$avx = ($1>=10) + ($1>=11);
}

if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
	$avx = ($2>=3.0) + ($2>3.0);
}

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

if ($avx>1) {{{

($inp,$out,$len,$key,$ivp,$Xip)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9");

($Ii,$T1,$T2,$Hkey,
 $Z0,$Z1,$Z2,$Z3,$Xi) = map("%xmm$_",(0..8));

($inout0,$inout1,$inout2,$inout3,$inout4,$inout5,$rndkey) = map("%xmm$_",(9..15));

($counter,$rounds,$ret,$const,$in0,$end0)=("%ebx","%ebp","%r10","%r11","%r14","%r15");

$code=<<___;
.text

.type	_aesni_ctr32_ghash_6x,\@abi-omnipotent
.align	32
_aesni_ctr32_ghash_6x:
.cfi_startproc
	vmovdqu		0x20($const),$T2	# borrow $T2, .Lone_msb
	sub		\$6,$len
	vpxor		$Z0,$Z0,$Z0		# $Z0   = 0
	vmovdqu		0x00-0x80($key),$rndkey
	vpaddb		$T2,$T1,$inout1
	vpaddb		$T2,$inout1,$inout2
	vpaddb		$T2,$inout2,$inout3
	vpaddb		$T2,$inout3,$inout4
	vpaddb		$T2,$inout4,$inout5
	vpxor		$rndkey,$T1,$inout0
	vmovdqu		$Z0,16+8(%rsp)		# "$Z3" = 0
	jmp		.Loop6x

.align	32
.Loop6x:
	add		\$`6<<24`,$counter
	jc		.Lhandle_ctr32		# discard $inout[1-5]?
	vmovdqu		0x00-0x20($Xip),$Hkey	# $Hkey^1
	  vpaddb	$T2,$inout5,$T1		# next counter value
	  vpxor		$rndkey,$inout1,$inout1
	  vpxor		$rndkey,$inout2,$inout2

.Lresume_ctr32:
	vmovdqu		$T1,($ivp)		# save next counter value
	vpclmulqdq	\$0x10,$Hkey,$Z3,$Z1
	  vpxor		$rndkey,$inout3,$inout3
	  vmovups	0x10-0x80($key),$T2	# borrow $T2 for $rndkey
	vpclmulqdq	\$0x01,$Hkey,$Z3,$Z2
	xor		%r12,%r12
	cmp		$in0,$end0

	  vaesenc	$T2,$inout0,$inout0
	vmovdqu		0x30+8(%rsp),$Ii	# I[4]
	  vpxor		$rndkey,$inout4,$inout4
	vpclmulqdq	\$0x00,$Hkey,$Z3,$T1
	  vaesenc	$T2,$inout1,$inout1
	  vpxor		$rndkey,$inout5,$inout5
	setnc		%r12b
	vpclmulqdq	\$0x11,$Hkey,$Z3,$Z3
	  vaesenc	$T2,$inout2,$inout2
	vmovdqu		0x10-0x20($Xip),$Hkey	# $Hkey^2
	neg		%r12
	  vaesenc	$T2,$inout3,$inout3
	 vpxor		$Z1,$Z2,$Z2
	vpclmulqdq	\$0x00,$Hkey,$Ii,$Z1
	 vpxor		$Z0,$Xi,$Xi		# modulo-scheduled
	  vaesenc	$T2,$inout4,$inout4
	 vpxor		$Z1,$T1,$Z0
	and		\$0x60,%r12
	  vmovups	0x20-0x80($key),$rndkey
	vpclmulqdq	\$0x10,$Hkey,$Ii,$T1
	  vaesenc	$T2,$inout5,$inout5

	vpclmulqdq	\$0x01,$Hkey,$Ii,$T2
	lea		($in0,%r12),$in0
	  vaesenc	$rndkey,$inout0,$inout0
	 vpxor		16+8(%rsp),$Xi,$Xi	# modulo-scheduled [vpxor $Z3,$Xi,$Xi]
	vpclmulqdq	\$0x11,$Hkey,$Ii,$Hkey
	 vmovdqu	0x40+8(%rsp),$Ii	# I[3]
	  vaesenc	$rndkey,$inout1,$inout1
	movbe		0x58($in0),%r13
	  vaesenc	$rndkey,$inout2,$inout2
	movbe		0x50($in0),%r12
	  vaesenc	$rndkey,$inout3,$inout3
	mov		%r13,0x20+8(%rsp)
	  vaesenc	$rndkey,$inout4,$inout4
	mov		%r12,0x28+8(%rsp)
	vmovdqu		0x30-0x20($Xip),$Z1	# borrow $Z1 for $Hkey^3
	  vaesenc	$rndkey,$inout5,$inout5

	  vmovups	0x30-0x80($key),$rndkey
	 vpxor		$T1,$Z2,$Z2
	vpclmulqdq	\$0x00,$Z1,$Ii,$T1
	  vaesenc	$rndkey,$inout0,$inout0
	 vpxor		$T2,$Z2,$Z2
	vpclmulqdq	\$0x10,$Z1,$Ii,$T2
	  vaesenc	$rndkey,$inout1,$inout1
	 vpxor		$Hkey,$Z3,$Z3
	vpclmulqdq	\$0x01,$Z1,$Ii,$Hkey
	  vaesenc	$rndkey,$inout2,$inout2
	vpclmulqdq	\$0x11,$Z1,$Ii,$Z1
	 vmovdqu	0x50+8(%rsp),$Ii	# I[2]
	  vaesenc	$rndkey,$inout3,$inout3
	  vaesenc	$rndkey,$inout4,$inout4
	 vpxor		$T1,$Z0,$Z0
	vmovdqu		0x40-0x20($Xip),$T1	# borrow $T1 for $Hkey^4
	  vaesenc	$rndkey,$inout5,$inout5

	  vmovups	0x40-0x80($key),$rndkey
	 vpxor		$T2,$Z2,$Z2
	vpclmulqdq	\$0x00,$T1,$Ii,$T2
	  vaesenc	$rndkey,$inout0,$inout0
	 vpxor		$Hkey,$Z2,$Z2
	vpclmulqdq	\$0x10,$T1,$Ii,$Hkey
	  vaesenc	$rndkey,$inout1,$inout1
	movbe		0x48($in0),%r13
	 vpxor		$Z1,$Z3,$Z3
	vpclmulqdq	\$0x01,$T1,$Ii,$Z1
	  vaesenc	$rndkey,$inout2,$inout2
	movbe		0x40($in0),%r12
	vpclmulqdq	\$0x11,$T1,$Ii,$T1
	 vmovdqu	0x60+8(%rsp),$Ii	# I[1]
	  vaesenc	$rndkey,$inout3,$inout3
	mov		%r13,0x30+8(%rsp)
	  vaesenc	$rndkey,$inout4,$inout4
	mov		%r12,0x38+8(%rsp)
	 vpxor		$T2,$Z0,$Z0
	vmovdqu		0x60-0x20($Xip),$T2	# borrow $T2 for $Hkey^5
	  vaesenc	$rndkey,$inout5,$inout5

	  vmovups	0x50-0x80($key),$rndkey
	 vpxor		$Hkey,$Z2,$Z2
	vpclmulqdq	\$0x00,$T2,$Ii,$Hkey
	  vaesenc	$rndkey,$inout0,$inout0
	 vpxor		$Z1,$Z2,$Z2
	vpclmulqdq	\$0x10,$T2,$Ii,$Z1
	  vaesenc	$rndkey,$inout1,$inout1
	movbe		0x38($in0),%r13
	 vpxor		$T1,$Z3,$Z3
	vpclmulqdq	\$0x01,$T2,$Ii,$T1
	 vpxor		0x70+8(%rsp),$Xi,$Xi	# accumulate I[0]
	  vaesenc	$rndkey,$inout2,$inout2
	movbe		0x30($in0),%r12
	vpclmulqdq	\$0x11,$T2,$Ii,$T2
	  vaesenc	$rndkey,$inout3,$inout3
	mov		%r13,0x40+8(%rsp)
	  vaesenc	$rndkey,$inout4,$inout4
	mov		%r12,0x48+8(%rsp)
	 vpxor		$Hkey,$Z0,$Z0
	 vmovdqu	0x70-0x20($Xip),$Hkey	# $Hkey^6
	  vaesenc	$rndkey,$inout5,$inout5

	  vmovups	0x60-0x80($key),$rndkey
	 vpxor		$Z1,$Z2,$Z2
	vpclmulqdq	\$0x10,$Hkey,$Xi,$Z1
	  vaesenc	$rndkey,$inout0,$inout0
	 vpxor		$T1,$Z2,$Z2
	vpclmulqdq	\$0x01,$Hkey,$Xi,$T1
	  vaesenc	$rndkey,$inout1,$inout1
	movbe		0x28($in0),%r13
	 vpxor		$T2,$Z3,$Z3
	vpclmulqdq	\$0x00,$Hkey,$Xi,$T2
	  vaesenc	$rndkey,$inout2,$inout2
	movbe		0x20($in0),%r12
	vpclmulqdq	\$0x11,$Hkey,$Xi,$Xi
	  vaesenc	$rndkey,$inout3,$inout3
	mov		%r13,0x50+8(%rsp)
	  vaesenc	$rndkey,$inout4,$inout4
	mov		%r12,0x58+8(%rsp)
	vpxor		$Z1,$Z2,$Z2
	  vaesenc	$rndkey,$inout5,$inout5
	vpxor		$T1,$Z2,$Z2

	  vmovups	0x70-0x80($key),$rndkey
	vpslldq		\$8,$Z2,$Z1
	vpxor		$T2,$Z0,$Z0
	vmovdqu		0x10($const),$Hkey	# .Lpoly

	  vaesenc	$rndkey,$inout0,$inout0
	vpxor		$Xi,$Z3,$Z3
	  vaesenc	$rndkey,$inout1,$inout1
	vpxor		$Z1,$Z0,$Z0
	movbe		0x18($in0),%r13
	  vaesenc	$rndkey,$inout2,$inout2
	movbe		0x10($in0),%r12
	vpalignr	\$8,$Z0,$Z0,$Ii		# 1st phase
	vpclmulqdq	\$0x10,$Hkey,$Z0,$Z0
	mov		%r13,0x60+8(%rsp)
	  vaesenc	$rndkey,$inout3,$inout3
	mov		%r12,0x68+8(%rsp)
	  vaesenc	$rndkey,$inout4,$inout4
	  vmovups	0x80-0x80($key),$T1	# borrow $T1 for $rndkey
	  vaesenc	$rndkey,$inout5,$inout5

	  vaesenc	$T1,$inout0,$inout0
	  vmovups	0x90-0x80($key),$rndkey
	  vaesenc	$T1,$inout1,$inout1
	vpsrldq		\$8,$Z2,$Z2
	  vaesenc	$T1,$inout2,$inout2
	vpxor		$Z2,$Z3,$Z3
	  vaesenc	$T1,$inout3,$inout3
	vpxor		$Ii,$Z0,$Z0
	movbe		0x08($in0),%r13
	  vaesenc	$T1,$inout4,$inout4
	movbe		0x00($in0),%r12
	  vaesenc	$T1,$inout5,$inout5
	  vmovups	0xa0-0x80($key),$T1
	  cmp		\$11,$rounds
	  jb		.Lenc_tail		# 128-bit key

	  vaesenc	$rndkey,$inout0,$inout0
	  vaesenc	$rndkey,$inout1,$inout1
	  vaesenc	$rndkey,$inout2,$inout2
	  vaesenc	$rndkey,$inout3,$inout3
	  vaesenc	$rndkey,$inout4,$inout4
	  vaesenc	$rndkey,$inout5,$inout5

	  vaesenc	$T1,$inout0,$inout0
	  vaesenc	$T1,$inout1,$inout1
	  vaesenc	$T1,$inout2,$inout2
	  vaesenc	$T1,$inout3,$inout3
	  vaesenc	$T1,$inout4,$inout4
	  vmovups	0xb0-0x80($key),$rndkey
	  vaesenc	$T1,$inout5,$inout5
	  vmovups	0xc0-0x80($key),$T1
	  je		.Lenc_tail		# 192-bit key

	  vaesenc	$rndkey,$inout0,$inout0
	  vaesenc	$rndkey,$inout1,$inout1
	  vaesenc	$rndkey,$inout2,$inout2
	  vaesenc	$rndkey,$inout3,$inout3
	  vaesenc	$rndkey,$inout4,$inout4
	  vaesenc	$rndkey,$inout5,$inout5

	  vaesenc	$T1,$inout0,$inout0
	  vaesenc	$T1,$inout1,$inout1
	  vaesenc	$T1,$inout2,$inout2
	  vaesenc	$T1,$inout3,$inout3
	  vaesenc	$T1,$inout4,$inout4
	  vmovups	0xd0-0x80($key),$rndkey
	  vaesenc	$T1,$inout5,$inout5
	  vmovups	0xe0-0x80($key),$T1
	  jmp		.Lenc_tail		# 256-bit key

.align	32
.Lhandle_ctr32:
	vmovdqu		($const),$Ii		# borrow $Ii for .Lbswap_mask
	  vpshufb	$Ii,$T1,$Z2		# byte-swap counter
	  vmovdqu	0x30($const),$Z1	# borrow $Z1, .Ltwo_lsb
	  vpaddd	0x40($const),$Z2,$inout1	# .Lone_lsb
	  vpaddd	$Z1,$Z2,$inout2
	vmovdqu		0x00-0x20($Xip),$Hkey	# $Hkey^1
	  vpaddd	$Z1,$inout1,$inout3
	  vpshufb	$Ii,$inout1,$inout1
	  vpaddd	$Z1,$inout2,$inout4
	  vpshufb	$Ii,$inout2,$inout2
	  vpxor		$rndkey,$inout1,$inout1
	  vpaddd	$Z1,$inout3,$inout5
	  vpshufb	$Ii,$inout3,$inout3
	  vpxor		$rndkey,$inout2,$inout2
	  vpaddd	$Z1,$inout4,$T1		# byte-swapped next counter value
	  vpshufb	$Ii,$inout4,$inout4
	  vpshufb	$Ii,$inout5,$inout5
	  vpshufb	$Ii,$T1,$T1		# next counter value
	jmp		.Lresume_ctr32

.align	32
.Lenc_tail:
	  vaesenc	$rndkey,$inout0,$inout0
	vmovdqu		$Z3,16+8(%rsp)		# postpone vpxor $Z3,$Xi,$Xi
	vpalignr	\$8,$Z0,$Z0,$Xi		# 2nd phase
	  vaesenc	$rndkey,$inout1,$inout1
	vpclmulqdq	\$0x10,$Hkey,$Z0,$Z0
	  vpxor		0x00($inp),$T1,$T2
	  vaesenc	$rndkey,$inout2,$inout2
	  vpxor		0x10($inp),$T1,$Ii
	  vaesenc	$rndkey,$inout3,$inout3
	  vpxor		0x20($inp),$T1,$Z1
	  vaesenc	$rndkey,$inout4,$inout4
	  vpxor		0x30($inp),$T1,$Z2
	  vaesenc	$rndkey,$inout5,$inout5
	  vpxor		0x40($inp),$T1,$Z3
	  vpxor		0x50($inp),$T1,$Hkey
	  vmovdqu	($ivp),$T1		# load next counter value

	  vaesenclast	$T2,$inout0,$inout0
	  vmovdqu	0x20($const),$T2	# borrow $T2, .Lone_msb
	  vaesenclast	$Ii,$inout1,$inout1
	 vpaddb		$T2,$T1,$Ii
	mov		%r13,0x70+8(%rsp)
	lea		0x60($inp),$inp
	  vaesenclast	$Z1,$inout2,$inout2
	 vpaddb		$T2,$Ii,$Z1
	mov		%r12,0x78+8(%rsp)
	lea		0x60($out),$out
	  vmovdqu	0x00-0x80($key),$rndkey
	  vaesenclast	$Z2,$inout3,$inout3
	 vpaddb		$T2,$Z1,$Z2
	  vaesenclast	$Z3, $inout4,$inout4
	 vpaddb		$T2,$Z2,$Z3
	  vaesenclast	$Hkey,$inout5,$inout5
	 vpaddb		$T2,$Z3,$Hkey

	add		\$0x60,$ret
	sub		\$0x6,$len
	jc		.L6x_done

	  vmovups	$inout0,-0x60($out)	# save output
	 vpxor		$rndkey,$T1,$inout0
	  vmovups	$inout1,-0x50($out)
	 vmovdqa	$Ii,$inout1		# 0 latency
	  vmovups	$inout2,-0x40($out)
	 vmovdqa	$Z1,$inout2		# 0 latency
	  vmovups	$inout3,-0x30($out)
	 vmovdqa	$Z2,$inout3		# 0 latency
	  vmovups	$inout4,-0x20($out)
	 vmovdqa	$Z3,$inout4		# 0 latency
	  vmovups	$inout5,-0x10($out)
	 vmovdqa	$Hkey,$inout5		# 0 latency
	vmovdqu		0x20+8(%rsp),$Z3	# I[5]
	jmp		.Loop6x

.L6x_done:
	vpxor		16+8(%rsp),$Xi,$Xi	# modulo-scheduled
	vpxor		$Z0,$Xi,$Xi		# modulo-scheduled

	ret
.cfi_endproc
.size	_aesni_ctr32_ghash_6x,.-_aesni_ctr32_ghash_6x
___
######################################################################
#
# size_t aesni_gcm_[en|de]crypt(const void *inp, void *out, size_t len,
#		const AES_KEY *key, unsigned char iv[16],
#		struct { u128 Xi,H,Htbl[9]; } *Xip);
$code.=<<___;
.globl	aesni_gcm_decrypt
.type	aesni_gcm_decrypt,\@function,6
.align	32
aesni_gcm_decrypt:
.cfi_startproc
	xor	$ret,$ret
	cmp	\$0x60,$len			# minimal accepted length
	jb	.Lgcm_dec_abort

	lea	(%rsp),%rax			# save stack pointer
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,-0xd8(%rax)
	movaps	%xmm7,-0xc8(%rax)
	movaps	%xmm8,-0xb8(%rax)
	movaps	%xmm9,-0xa8(%rax)
	movaps	%xmm10,-0x98(%rax)
	movaps	%xmm11,-0x88(%rax)
	movaps	%xmm12,-0x78(%rax)
	movaps	%xmm13,-0x68(%rax)
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
.Lgcm_dec_body:
___
$code.=<<___;
	vzeroupper

	vmovdqu		($ivp),$T1		# input counter value
	add		\$-128,%rsp
	mov		12($ivp),$counter
	lea		.Lbswap_mask(%rip),$const
	lea		-0x80($key),$in0	# borrow $in0
	mov		\$0xf80,$end0		# borrow $end0
	vmovdqu		($Xip),$Xi		# load Xi
	and		\$-128,%rsp		# ensure stack alignment
	vmovdqu		($const),$Ii		# borrow $Ii for .Lbswap_mask
	lea		0x80($key),$key		# size optimization
	lea		0x20+0x20($Xip),$Xip	# size optimization
	mov		0xf0-0x80($key),$rounds
	vpshufb		$Ii,$Xi,$Xi

	and		$end0,$in0
	and		%rsp,$end0
	sub		$in0,$end0
	jc		.Ldec_no_key_aliasing
	cmp		\$768,$end0
	jnc		.Ldec_no_key_aliasing
	sub		$end0,%rsp		# avoid aliasing with key
.Ldec_no_key_aliasing:

	vmovdqu		0x50($inp),$Z3		# I[5]
	lea		($inp),$in0
	vmovdqu		0x40($inp),$Z0
	lea		-0xc0($inp,$len),$end0
	vmovdqu		0x30($inp),$Z1
	shr		\$4,$len
	xor		$ret,$ret
	vmovdqu		0x20($inp),$Z2
	 vpshufb	$Ii,$Z3,$Z3		# passed to _aesni_ctr32_ghash_6x
	vmovdqu		0x10($inp),$T2
	 vpshufb	$Ii,$Z0,$Z0
	vmovdqu		($inp),$Hkey
	 vpshufb	$Ii,$Z1,$Z1
	vmovdqu		$Z0,0x30(%rsp)
	 vpshufb	$Ii,$Z2,$Z2
	vmovdqu		$Z1,0x40(%rsp)
	 vpshufb	$Ii,$T2,$T2
	vmovdqu		$Z2,0x50(%rsp)
	 vpshufb	$Ii,$Hkey,$Hkey
	vmovdqu		$T2,0x60(%rsp)
	vmovdqu		$Hkey,0x70(%rsp)

	call		_aesni_ctr32_ghash_6x

	vmovups		$inout0,-0x60($out)	# save output
	vmovups		$inout1,-0x50($out)
	vmovups		$inout2,-0x40($out)
	vmovups		$inout3,-0x30($out)
	vmovups		$inout4,-0x20($out)
	vmovups		$inout5,-0x10($out)

	vpshufb		($const),$Xi,$Xi	# .Lbswap_mask
	vmovdqu		$Xi,-0x40($Xip)		# output Xi

	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	movaps	-0x68(%rax),%xmm13
	movaps	-0x58(%rax),%xmm14
	movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp		# restore %rsp
.cfi_def_cfa_register	%rsp
.Lgcm_dec_abort:
	mov	$ret,%rax		# return value
	ret
.cfi_endproc
.size	aesni_gcm_decrypt,.-aesni_gcm_decrypt
___

$code.=<<___;
.type	_aesni_ctr32_6x,\@abi-omnipotent
.align	32
_aesni_ctr32_6x:
.cfi_startproc
	vmovdqu		0x00-0x80($key),$Z0	# borrow $Z0 for $rndkey
	vmovdqu		0x20($const),$T2	# borrow $T2, .Lone_msb
	lea		-1($rounds),%r13
	vmovups		0x10-0x80($key),$rndkey
	lea		0x20-0x80($key),%r12
	vpxor		$Z0,$T1,$inout0
	add		\$`6<<24`,$counter
	jc		.Lhandle_ctr32_2
	vpaddb		$T2,$T1,$inout1
	vpaddb		$T2,$inout1,$inout2
	vpxor		$Z0,$inout1,$inout1
	vpaddb		$T2,$inout2,$inout3
	vpxor		$Z0,$inout2,$inout2
	vpaddb		$T2,$inout3,$inout4
	vpxor		$Z0,$inout3,$inout3
	vpaddb		$T2,$inout4,$inout5
	vpxor		$Z0,$inout4,$inout4
	vpaddb		$T2,$inout5,$T1
	vpxor		$Z0,$inout5,$inout5
	jmp		.Loop_ctr32

.align	16
.Loop_ctr32:
	vaesenc		$rndkey,$inout0,$inout0
	vaesenc		$rndkey,$inout1,$inout1
	vaesenc		$rndkey,$inout2,$inout2
	vaesenc		$rndkey,$inout3,$inout3
	vaesenc		$rndkey,$inout4,$inout4
	vaesenc		$rndkey,$inout5,$inout5
	vmovups		(%r12),$rndkey
	lea		0x10(%r12),%r12
	dec		%r13d
	jnz		.Loop_ctr32

	vmovdqu		(%r12),$Hkey		# last round key
	vaesenc		$rndkey,$inout0,$inout0
	vpxor		0x00($inp),$Hkey,$Z0
	vaesenc		$rndkey,$inout1,$inout1
	vpxor		0x10($inp),$Hkey,$Z1
	vaesenc		$rndkey,$inout2,$inout2
	vpxor		0x20($inp),$Hkey,$Z2
	vaesenc		$rndkey,$inout3,$inout3
	vpxor		0x30($inp),$Hkey,$Xi
	vaesenc		$rndkey,$inout4,$inout4
	vpxor		0x40($inp),$Hkey,$T2
	vaesenc		$rndkey,$inout5,$inout5
	vpxor		0x50($inp),$Hkey,$Hkey
	lea		0x60($inp),$inp

	vaesenclast	$Z0,$inout0,$inout0
	vaesenclast	$Z1,$inout1,$inout1
	vaesenclast	$Z2,$inout2,$inout2
	vaesenclast	$Xi,$inout3,$inout3
	vaesenclast	$T2,$inout4,$inout4
	vaesenclast	$Hkey,$inout5,$inout5
	vmovups		$inout0,0x00($out)
	vmovups		$inout1,0x10($out)
	vmovups		$inout2,0x20($out)
	vmovups		$inout3,0x30($out)
	vmovups		$inout4,0x40($out)
	vmovups		$inout5,0x50($out)
	lea		0x60($out),$out

	ret
.align	32
.Lhandle_ctr32_2:
	vpshufb		$Ii,$T1,$Z2		# byte-swap counter
	vmovdqu		0x30($const),$Z1	# borrow $Z1, .Ltwo_lsb
	vpaddd		0x40($const),$Z2,$inout1	# .Lone_lsb
	vpaddd		$Z1,$Z2,$inout2
	vpaddd		$Z1,$inout1,$inout3
	vpshufb		$Ii,$inout1,$inout1
	vpaddd		$Z1,$inout2,$inout4
	vpshufb		$Ii,$inout2,$inout2
	vpxor		$Z0,$inout1,$inout1
	vpaddd		$Z1,$inout3,$inout5
	vpshufb		$Ii,$inout3,$inout3
	vpxor		$Z0,$inout2,$inout2
	vpaddd		$Z1,$inout4,$T1		# byte-swapped next counter value
	vpshufb		$Ii,$inout4,$inout4
	vpxor		$Z0,$inout3,$inout3
	vpshufb		$Ii,$inout5,$inout5
	vpxor		$Z0,$inout4,$inout4
	vpshufb		$Ii,$T1,$T1		# next counter value
	vpxor		$Z0,$inout5,$inout5
	jmp	.Loop_ctr32
.cfi_endproc
.size	_aesni_ctr32_6x,.-_aesni_ctr32_6x

.globl	aesni_gcm_encrypt
.type	aesni_gcm_encrypt,\@function,6
.align	32
aesni_gcm_encrypt:
.cfi_startproc
	xor	$ret,$ret
	cmp	\$0x60*3,$len			# minimal accepted length
	jb	.Lgcm_enc_abort

	lea	(%rsp),%rax			# save stack pointer
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,-0xd8(%rax)
	movaps	%xmm7,-0xc8(%rax)
	movaps	%xmm8,-0xb8(%rax)
	movaps	%xmm9,-0xa8(%rax)
	movaps	%xmm10,-0x98(%rax)
	movaps	%xmm11,-0x88(%rax)
	movaps	%xmm12,-0x78(%rax)
	movaps	%xmm13,-0x68(%rax)
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
.Lgcm_enc_body:
___
$code.=<<___;
	vzeroupper

	vmovdqu		($ivp),$T1		# input counter value
	add		\$-128,%rsp
	mov		12($ivp),$counter
	lea		.Lbswap_mask(%rip),$const
	lea		-0x80($key),$in0	# borrow $in0
	mov		\$0xf80,$end0		# borrow $end0
	lea		0x80($key),$key		# size optimization
	vmovdqu		($const),$Ii		# borrow $Ii for .Lbswap_mask
	and		\$-128,%rsp		# ensure stack alignment
	mov		0xf0-0x80($key),$rounds

	and		$end0,$in0
	and		%rsp,$end0
	sub		$in0,$end0
	jc		.Lenc_no_key_aliasing
	cmp		\$768,$end0
	jnc		.Lenc_no_key_aliasing
	sub		$end0,%rsp		# avoid aliasing with key
.Lenc_no_key_aliasing:

	lea		($out),$in0
	lea		-0xc0($out,$len),$end0
	shr		\$4,$len

	call		_aesni_ctr32_6x
	vpshufb		$Ii,$inout0,$Xi		# save bswapped output on stack
	vpshufb		$Ii,$inout1,$T2
	vmovdqu		$Xi,0x70(%rsp)
	vpshufb		$Ii,$inout2,$Z0
	vmovdqu		$T2,0x60(%rsp)
	vpshufb		$Ii,$inout3,$Z1
	vmovdqu		$Z0,0x50(%rsp)
	vpshufb		$Ii,$inout4,$Z2
	vmovdqu		$Z1,0x40(%rsp)
	vpshufb		$Ii,$inout5,$Z3		# passed to _aesni_ctr32_ghash_6x
	vmovdqu		$Z2,0x30(%rsp)

	call		_aesni_ctr32_6x

	vmovdqu		($Xip),$Xi		# load Xi
	lea		0x20+0x20($Xip),$Xip	# size optimization
	sub		\$12,$len
	mov		\$0x60*2,$ret
	vpshufb		$Ii,$Xi,$Xi

	call		_aesni_ctr32_ghash_6x
	vmovdqu		0x20(%rsp),$Z3		# I[5]
	 vmovdqu	($const),$Ii		# borrow $Ii for .Lbswap_mask
	vmovdqu		0x00-0x20($Xip),$Hkey	# $Hkey^1
	vpunpckhqdq	$Z3,$Z3,$T1
	vmovdqu		0x20-0x20($Xip),$rndkey	# borrow $rndkey for $HK
	 vmovups	$inout0,-0x60($out)	# save output
	 vpshufb	$Ii,$inout0,$inout0	# but keep bswapped copy
	vpxor		$Z3,$T1,$T1
	 vmovups	$inout1,-0x50($out)
	 vpshufb	$Ii,$inout1,$inout1
	 vmovups	$inout2,-0x40($out)
	 vpshufb	$Ii,$inout2,$inout2
	 vmovups	$inout3,-0x30($out)
	 vpshufb	$Ii,$inout3,$inout3
	 vmovups	$inout4,-0x20($out)
	 vpshufb	$Ii,$inout4,$inout4
	 vmovups	$inout5,-0x10($out)
	 vpshufb	$Ii,$inout5,$inout5
	 vmovdqu	$inout0,0x10(%rsp)	# free $inout0
___
{ my ($HK,$T3)=($rndkey,$inout0);

$code.=<<___;
	 vmovdqu	0x30(%rsp),$Z2		# I[4]
	 vmovdqu	0x10-0x20($Xip),$Ii	# borrow $Ii for $Hkey^2
	 vpunpckhqdq	$Z2,$Z2,$T2
	vpclmulqdq	\$0x00,$Hkey,$Z3,$Z1
	 vpxor		$Z2,$T2,$T2
	vpclmulqdq	\$0x11,$Hkey,$Z3,$Z3
	vpclmulqdq	\$0x00,$HK,$T1,$T1

	 vmovdqu	0x40(%rsp),$T3		# I[3]
	vpclmulqdq	\$0x00,$Ii,$Z2,$Z0
	 vmovdqu	0x30-0x20($Xip),$Hkey	# $Hkey^3
	vpxor		$Z1,$Z0,$Z0
	 vpunpckhqdq	$T3,$T3,$Z1
	vpclmulqdq	\$0x11,$Ii,$Z2,$Z2
	 vpxor		$T3,$Z1,$Z1
	vpxor		$Z3,$Z2,$Z2
	vpclmulqdq	\$0x10,$HK,$T2,$T2
	 vmovdqu	0x50-0x20($Xip),$HK
	vpxor		$T1,$T2,$T2

	 vmovdqu	0x50(%rsp),$T1		# I[2]
	vpclmulqdq	\$0x00,$Hkey,$T3,$Z3
	 vmovdqu	0x40-0x20($Xip),$Ii	# borrow $Ii for $Hkey^4
	vpxor		$Z0,$Z3,$Z3
	 vpunpckhqdq	$T1,$T1,$Z0
	vpclmulqdq	\$0x11,$Hkey,$T3,$T3
	 vpxor		$T1,$Z0,$Z0
	vpxor		$Z2,$T3,$T3
	vpclmulqdq	\$0x00,$HK,$Z1,$Z1
	vpxor		$T2,$Z1,$Z1

	 vmovdqu	0x60(%rsp),$T2		# I[1]
	vpclmulqdq	\$0x00,$Ii,$T1,$Z2
	 vmovdqu	0x60-0x20($Xip),$Hkey	# $Hkey^5
	vpxor		$Z3,$Z2,$Z2
	 vpunpckhqdq	$T2,$T2,$Z3
	vpclmulqdq	\$0x11,$Ii,$T1,$T1
	 vpxor		$T2,$Z3,$Z3
	vpxor		$T3,$T1,$T1
	vpclmulqdq	\$0x10,$HK,$Z0,$Z0
	 vmovdqu	0x80-0x20($Xip),$HK
	vpxor		$Z1,$Z0,$Z0

	 vpxor		0x70(%rsp),$Xi,$Xi	# accumulate I[0]
	vpclmulqdq	\$0x00,$Hkey,$T2,$Z1
	 vmovdqu	0x70-0x20($Xip),$Ii	# borrow $Ii for $Hkey^6
	 vpunpckhqdq	$Xi,$Xi,$T3
	vpxor		$Z2,$Z1,$Z1
	vpclmulqdq	\$0x11,$Hkey,$T2,$T2
	 vpxor		$Xi,$T3,$T3
	vpxor		$T1,$T2,$T2
	vpclmulqdq	\$0x00,$HK,$Z3,$Z3
	vpxor		$Z0,$Z3,$Z0

	vpclmulqdq	\$0x00,$Ii,$Xi,$Z2
	 vmovdqu	0x00-0x20($Xip),$Hkey	# $Hkey^1
	 vpunpckhqdq	$inout5,$inout5,$T1
	vpclmulqdq	\$0x11,$Ii,$Xi,$Xi
	 vpxor		$inout5,$T1,$T1
	vpxor		$Z1,$Z2,$Z1
	vpclmulqdq	\$0x10,$HK,$T3,$T3
	 vmovdqu	0x20-0x20($Xip),$HK
	vpxor		$T2,$Xi,$Z3
	vpxor		$Z0,$T3,$Z2

	 vmovdqu	0x10-0x20($Xip),$Ii	# borrow $Ii for $Hkey^2
	  vpxor		$Z1,$Z3,$T3		# aggregated Karatsuba post-processing
	vpclmulqdq	\$0x00,$Hkey,$inout5,$Z0
	  vpxor		$T3,$Z2,$Z2
	 vpunpckhqdq	$inout4,$inout4,$T2
	vpclmulqdq	\$0x11,$Hkey,$inout5,$inout5
	 vpxor		$inout4,$T2,$T2
	  vpslldq	\$8,$Z2,$T3
	vpclmulqdq	\$0x00,$HK,$T1,$T1
	  vpxor		$T3,$Z1,$Xi
	  vpsrldq	\$8,$Z2,$Z2
	  vpxor		$Z2,$Z3,$Z3

	vpclmulqdq	\$0x00,$Ii,$inout4,$Z1
	 vmovdqu	0x30-0x20($Xip),$Hkey	# $Hkey^3
	vpxor		$Z0,$Z1,$Z1
	 vpunpckhqdq	$inout3,$inout3,$T3
	vpclmulqdq	\$0x11,$Ii,$inout4,$inout4
	 vpxor		$inout3,$T3,$T3
	vpxor		$inout5,$inout4,$inout4
	  vpalignr	\$8,$Xi,$Xi,$inout5	# 1st phase
	vpclmulqdq	\$0x10,$HK,$T2,$T2
	 vmovdqu	0x50-0x20($Xip),$HK
	vpxor		$T1,$T2,$T2

	vpclmulqdq	\$0x00,$Hkey,$inout3,$Z0
	 vmovdqu	0x40-0x20($Xip),$Ii	# borrow $Ii for $Hkey^4
	vpxor		$Z1,$Z0,$Z0
	 vpunpckhqdq	$inout2,$inout2,$T1
	vpclmulqdq	\$0x11,$Hkey,$inout3,$inout3
	 vpxor		$inout2,$T1,$T1
	vpxor		$inout4,$inout3,$inout3
	  vxorps	0x10(%rsp),$Z3,$Z3	# accumulate $inout0
	vpclmulqdq	\$0x00,$HK,$T3,$T3
	vpxor		$T2,$T3,$T3

	  vpclmulqdq	\$0x10,0x10($const),$Xi,$Xi
	  vxorps	$inout5,$Xi,$Xi

	vpclmulqdq	\$0x00,$Ii,$inout2,$Z1
	 vmovdqu	0x60-0x20($Xip),$Hkey	# $Hkey^5
	vpxor		$Z0,$Z1,$Z1
	 vpunpckhqdq	$inout1,$inout1,$T2
	vpclmulqdq	\$0x11,$Ii,$inout2,$inout2
	 vpxor		$inout1,$T2,$T2
	  vpalignr	\$8,$Xi,$Xi,$inout5	# 2nd phase
	vpxor		$inout3,$inout2,$inout2
	vpclmulqdq	\$0x10,$HK,$T1,$T1
	 vmovdqu	0x80-0x20($Xip),$HK
	vpxor		$T3,$T1,$T1

	  vxorps	$Z3,$inout5,$inout5
	  vpclmulqdq	\$0x10,0x10($const),$Xi,$Xi
	  vxorps	$inout5,$Xi,$Xi

	vpclmulqdq	\$0x00,$Hkey,$inout1,$Z0
	 vmovdqu	0x70-0x20($Xip),$Ii	# borrow $Ii for $Hkey^6
	vpxor		$Z1,$Z0,$Z0
	 vpunpckhqdq	$Xi,$Xi,$T3
	vpclmulqdq	\$0x11,$Hkey,$inout1,$inout1
	 vpxor		$Xi,$T3,$T3
	vpxor		$inout2,$inout1,$inout1
	vpclmulqdq	\$0x00,$HK,$T2,$T2
	vpxor		$T1,$T2,$T2

	vpclmulqdq	\$0x00,$Ii,$Xi,$Z1
	vpclmulqdq	\$0x11,$Ii,$Xi,$Z3
	vpxor		$Z0,$Z1,$Z1
	vpclmulqdq	\$0x10,$HK,$T3,$Z2
	vpxor		$inout1,$Z3,$Z3
	vpxor		$T2,$Z2,$Z2

	vpxor		$Z1,$Z3,$Z0		# aggregated Karatsuba post-processing
	vpxor		$Z0,$Z2,$Z2
	vpslldq		\$8,$Z2,$T1
	vmovdqu		0x10($const),$Hkey	# .Lpoly
	vpsrldq		\$8,$Z2,$Z2
	vpxor		$T1,$Z1,$Xi
	vpxor		$Z2,$Z3,$Z3

	vpalignr	\$8,$Xi,$Xi,$T2		# 1st phase
	vpclmulqdq	\$0x10,$Hkey,$Xi,$Xi
	vpxor		$T2,$Xi,$Xi

	vpalignr	\$8,$Xi,$Xi,$T2		# 2nd phase
	vpclmulqdq	\$0x10,$Hkey,$Xi,$Xi
	vpxor		$Z3,$T2,$T2
	vpxor		$T2,$Xi,$Xi
___
}
$code.=<<___;
	vpshufb		($const),$Xi,$Xi	# .Lbswap_mask
	vmovdqu		$Xi,-0x40($Xip)		# output Xi

	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	movaps	-0x68(%rax),%xmm13
	movaps	-0x58(%rax),%xmm14
	movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp		# restore %rsp
.cfi_def_cfa_register	%rsp
.Lgcm_enc_abort:
	mov	$ret,%rax		# return value
	ret
.cfi_endproc
.size	aesni_gcm_encrypt,.-aesni_gcm_encrypt
___

$code.=<<___;
.align	64
.Lbswap_mask:
	.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
.Lpoly:
	.byte	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
.Lone_msb:
	.byte	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
.Ltwo_lsb:
	.byte	2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
.Lone_lsb:
	.byte	1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
.asciz	"AES-NI GCM module for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
.align	64
___
if ($win64) {
$rec="%rcx";
$frame="%rdx";
$context="%r8";
$disp="%r9";

$code.=<<___
.extern	__imp_RtlVirtualUnwind
.type	gcm_se_handler,\@abi-omnipotent
.align	16
gcm_se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# prologue label
	cmp	%r10,%rbx		# context->Rip<prologue label
	jb	.Lcommon_seh_tail

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=epilogue label
	jae	.Lcommon_seh_tail

	mov	120($context),%rax	# pull context->Rax

	mov	-48(%rax),%r15
	mov	-40(%rax),%r14
	mov	-32(%rax),%r13
	mov	-24(%rax),%r12
	mov	-16(%rax),%rbp
	mov	-8(%rax),%rbx
	mov	%r15,240($context)
	mov	%r14,232($context)
	mov	%r13,224($context)
	mov	%r12,216($context)
	mov	%rbp,160($context)
	mov	%rbx,144($context)

	lea	-0xd8(%rax),%rsi	# %xmm save area
	lea	512($context),%rdi	# & context.Xmm6
	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)
	.long	0xa548f3fc		# cld; rep movsq

.Lcommon_seh_tail:
	mov	8(%rax),%rdi
	mov	16(%rax),%rsi
	mov	%rax,152($context)	# restore context->Rsp
	mov	%rsi,168($context)	# restore context->Rsi
	mov	%rdi,176($context)	# restore context->Rdi

	mov	40($disp),%rdi		# disp->ContextRecord
	mov	$context,%rsi		# context
	mov	\$154,%ecx		# sizeof(CONTEXT)
	.long	0xa548f3fc		# cld; rep movsq

	mov	$disp,%rsi
	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
	mov	0(%rsi),%r8		# arg3, disp->ControlPc
	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
	mov	40(%rsi),%r10		# disp->ContextRecord
	lea	56(%rsi),%r11		# &disp->HandlerData
	lea	24(%rsi),%r12		# &disp->EstablisherFrame
	mov	%r10,32(%rsp)		# arg5
	mov	%r11,40(%rsp)		# arg6
	mov	%r12,48(%rsp)		# arg7
	mov	%rcx,56(%rsp)		# arg8, (NULL)
	call	*__imp_RtlVirtualUnwind(%rip)

	mov	\$1,%eax		# ExceptionContinueSearch
	add	\$64,%rsp
	popfq
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx
	pop	%rdi
	pop	%rsi
	ret
.size	gcm_se_handler,.-gcm_se_handler

.section	.pdata
.align	4
	.rva	.LSEH_begin_aesni_gcm_decrypt
	.rva	.LSEH_end_aesni_gcm_decrypt
	.rva	.LSEH_gcm_dec_info

	.rva	.LSEH_begin_aesni_gcm_encrypt
	.rva	.LSEH_end_aesni_gcm_encrypt
	.rva	.LSEH_gcm_enc_info
.section	.xdata
.align	8
.LSEH_gcm_dec_info:
	.byte	9,0,0,0
	.rva	gcm_se_handler
	.rva	.Lgcm_dec_body,.Lgcm_dec_abort
.LSEH_gcm_enc_info:
	.byte	9,0,0,0
	.rva	gcm_se_handler
	.rva	.Lgcm_enc_body,.Lgcm_enc_abort
___
}
}}} else {{{
$code=<<___;	# assembler is too old
.text

.globl	aesni_gcm_encrypt
.type	aesni_gcm_encrypt,\@abi-omnipotent
aesni_gcm_encrypt:
.cfi_startproc
	xor	%eax,%eax
	ret
.cfi_endproc
.size	aesni_gcm_encrypt,.-aesni_gcm_encrypt

.globl	aesni_gcm_decrypt
.type	aesni_gcm_decrypt,\@abi-omnipotent
aesni_gcm_decrypt:
.cfi_startproc
	xor	%eax,%eax
	ret
.cfi_endproc
.size	aesni_gcm_decrypt,.-aesni_gcm_decrypt
___
}}}

$code =~ s/\`([^\`]*)\`/eval($1)/gem;

print $code;

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# March 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+128 bytes shared table]. Even though
# loops are aggressively modulo-scheduled in respect to references to
# Htbl and Z.hi updates for 8 cycles per byte, measured performance is
# ~12 cycles per processed byte on 21264 CPU. It seems to be a dynamic
# scheduling "glitch," because uprofile(1) indicates uniform sample
# distribution, as if all instruction bundles execute in 1.5 cycles.
# Meaning that it could have been even faster, yet 12 cycles is ~60%
# better than gcc-generated code and ~80% than code generated by vendor
# compiler.

$cnt="v0";	# $0
$t0="t0";
$t1="t1";
$t2="t2";
$Thi0="t3";	# $4
$Tlo0="t4";
$Thi1="t5";
$Tlo1="t6";
$rem="t7";	# $8
#################
$Xi="a0";	# $16, input argument block
$Htbl="a1";
$inp="a2";
$len="a3";
$nlo="a4";	# $20
$nhi="a5";
$Zhi="t8";
$Zlo="t9";
$Xhi="t10";	# $24
$Xlo="t11";
$remp="t12";
$rem_4bit="AT";	# $28

{ my $N;
  sub loop() {

	$N++;
$code.=<<___;
.align	4
	extbl	$Xlo,7,$nlo
	and	$nlo,0xf0,$nhi
	sll	$nlo,4,$nlo
	and	$nlo,0xf0,$nlo

	addq	$nlo,$Htbl,$nlo
	ldq	$Zlo,8($nlo)
	addq	$nhi,$Htbl,$nhi
	ldq	$Zhi,0($nlo)

	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	lda	$cnt,6(zero)
	extbl	$Xlo,6,$nlo

	ldq	$Tlo1,8($nhi)
	s8addq	$remp,$rem_4bit,$remp
	ldq	$Thi1,0($nhi)
	srl	$Zlo,4,$Zlo

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	xor	$t0,$Zlo,$Zlo
	and	$nlo,0xf0,$nhi

	xor	$Tlo1,$Zlo,$Zlo
	sll	$nlo,4,$nlo
	xor	$Thi1,$Zhi,$Zhi
	and	$nlo,0xf0,$nlo

	addq	$nlo,$Htbl,$nlo
	ldq	$Tlo0,8($nlo)
	addq	$nhi,$Htbl,$nhi
	ldq	$Thi0,0($nlo)

.Looplo$N:
	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	subq	$cnt,1,$cnt
	srl	$Zlo,4,$Zlo

	ldq	$Tlo1,8($nhi)
	xor	$rem,$Zhi,$Zhi
	ldq	$Thi1,0($nhi)
	s8addq	$remp,$rem_4bit,$remp

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	xor	$t0,$Zlo,$Zlo
	extbl	$Xlo,$cnt,$nlo

	and	$nlo,0xf0,$nhi
	xor	$Thi0,$Zhi,$Zhi
	xor	$Tlo0,$Zlo,$Zlo
	sll	$nlo,4,$nlo

	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	and	$nlo,0xf0,$nlo
	srl	$Zlo,4,$Zlo

	s8addq	$remp,$rem_4bit,$remp
	xor	$rem,$Zhi,$Zhi
	addq	$nlo,$Htbl,$nlo
	addq	$nhi,$Htbl,$nhi

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	ldq	$Tlo0,8($nlo)
	xor	$t0,$Zlo,$Zlo

	xor	$Tlo1,$Zlo,$Zlo
	xor	$Thi1,$Zhi,$Zhi
	ldq	$Thi0,0($nlo)
	bne	$cnt,.Looplo$N

	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	lda	$cnt,7(zero)
	srl	$Zlo,4,$Zlo

	ldq	$Tlo1,8($nhi)
	xor	$rem,$Zhi,$Zhi
	ldq	$Thi1,0($nhi)
	s8addq	$remp,$rem_4bit,$remp

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	xor	$t0,$Zlo,$Zlo
	extbl	$Xhi,$cnt,$nlo

	and	$nlo,0xf0,$nhi
	xor	$Thi0,$Zhi,$Zhi
	xor	$Tlo0,$Zlo,$Zlo
	sll	$nlo,4,$nlo

	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	and	$nlo,0xf0,$nlo
	srl	$Zlo,4,$Zlo

	s8addq	$remp,$rem_4bit,$remp
	xor	$rem,$Zhi,$Zhi
	addq	$nlo,$Htbl,$nlo
	addq	$nhi,$Htbl,$nhi

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	ldq	$Tlo0,8($nlo)
	xor	$t0,$Zlo,$Zlo

	xor	$Tlo1,$Zlo,$Zlo
	xor	$Thi1,$Zhi,$Zhi
	ldq	$Thi0,0($nlo)
	unop

.Loophi$N:
	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	subq	$cnt,1,$cnt
	srl	$Zlo,4,$Zlo

	ldq	$Tlo1,8($nhi)
	xor	$rem,$Zhi,$Zhi
	ldq	$Thi1,0($nhi)
	s8addq	$remp,$rem_4bit,$remp

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	xor	$t0,$Zlo,$Zlo
	extbl	$Xhi,$cnt,$nlo

	and	$nlo,0xf0,$nhi
	xor	$Thi0,$Zhi,$Zhi
	xor	$Tlo0,$Zlo,$Zlo
	sll	$nlo,4,$nlo

	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	and	$nlo,0xf0,$nlo
	srl	$Zlo,4,$Zlo

	s8addq	$remp,$rem_4bit,$remp
	xor	$rem,$Zhi,$Zhi
	addq	$nlo,$Htbl,$nlo
	addq	$nhi,$Htbl,$nhi

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	ldq	$Tlo0,8($nlo)
	xor	$t0,$Zlo,$Zlo

	xor	$Tlo1,$Zlo,$Zlo
	xor	$Thi1,$Zhi,$Zhi
	ldq	$Thi0,0($nlo)
	bne	$cnt,.Loophi$N

	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	srl	$Zlo,4,$Zlo

	ldq	$Tlo1,8($nhi)
	xor	$rem,$Zhi,$Zhi
	ldq	$Thi1,0($nhi)
	s8addq	$remp,$rem_4bit,$remp

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	xor	$t0,$Zlo,$Zlo

	xor	$Tlo0,$Zlo,$Zlo
	xor	$Thi0,$Zhi,$Zhi

	and	$Zlo,0x0f,$remp
	sll	$Zhi,60,$t0
	srl	$Zlo,4,$Zlo

	s8addq	$remp,$rem_4bit,$remp
	xor	$rem,$Zhi,$Zhi

	ldq	$rem,0($remp)
	srl	$Zhi,4,$Zhi
	xor	$Tlo1,$Zlo,$Zlo
	xor	$Thi1,$Zhi,$Zhi
	xor	$t0,$Zlo,$Zlo
	xor	$rem,$Zhi,$Zhi
___
}}

$code=<<___;
#ifdef __linux__
#include <asm/regdef.h>
#else
#include <asm.h>
#include <regdef.h>
#endif

.text

.set	noat
.set	noreorder
.globl	gcm_gmult_4bit
.align	4
.ent	gcm_gmult_4bit
gcm_gmult_4bit:
	.frame	sp,0,ra
	.prologue 0

	ldq	$Xlo,8($Xi)
	ldq	$Xhi,0($Xi)

	bsr	$t0,picmeup
	nop
___

	&loop();

$code.=<<___;
	srl	$Zlo,24,$t0	# byte swap
	srl	$Zlo,8,$t1

	sll	$Zlo,8,$t2
	sll	$Zlo,24,$Zlo
	zapnot	$t0,0x11,$t0
	zapnot	$t1,0x22,$t1

	zapnot	$Zlo,0x88,$Zlo
	or	$t0,$t1,$t0
	zapnot	$t2,0x44,$t2

	or	$Zlo,$t0,$Zlo
	srl	$Zhi,24,$t0
	srl	$Zhi,8,$t1

	or	$Zlo,$t2,$Zlo
	sll	$Zhi,8,$t2
	sll	$Zhi,24,$Zhi

	srl	$Zlo,32,$Xlo
	sll	$Zlo,32,$Zlo

	zapnot	$t0,0x11,$t0
	zapnot	$t1,0x22,$t1
	or	$Zlo,$Xlo,$Xlo

	zapnot	$Zhi,0x88,$Zhi
	or	$t0,$t1,$t0
	zapnot	$t2,0x44,$t2

	or	$Zhi,$t0,$Zhi
	or	$Zhi,$t2,$Zhi

	srl	$Zhi,32,$Xhi
	sll	$Zhi,32,$Zhi

	or	$Zhi,$Xhi,$Xhi
	stq	$Xlo,8($Xi)
	stq	$Xhi,0($Xi)

	ret	(ra)
.end	gcm_gmult_4bit
___

$inhi="s0";
$inlo="s1";

$code.=<<___;
.globl	gcm_ghash_4bit
.align	4
.ent	gcm_ghash_4bit
gcm_ghash_4bit:
	lda	sp,-32(sp)
	stq	ra,0(sp)
	stq	s0,8(sp)
	stq	s1,16(sp)
	.mask	0x04000600,-32
	.frame	sp,32,ra
	.prologue 0

	ldq_u	$inhi,0($inp)
	ldq_u	$Thi0,7($inp)
	ldq_u	$inlo,8($inp)
	ldq_u	$Tlo0,15($inp)
	ldq	$Xhi,0($Xi)
	ldq	$Xlo,8($Xi)

	bsr	$t0,picmeup
	nop

.Louter:
	extql	$inhi,$inp,$inhi
	extqh	$Thi0,$inp,$Thi0
	or	$inhi,$Thi0,$inhi
	lda	$inp,16($inp)

	extql	$inlo,$inp,$inlo
	extqh	$Tlo0,$inp,$Tlo0
	or	$inlo,$Tlo0,$inlo
	subq	$len,16,$len

	xor	$Xlo,$inlo,$Xlo
	xor	$Xhi,$inhi,$Xhi
___

	&loop();

$code.=<<___;
	srl	$Zlo,24,$t0	# byte swap
	srl	$Zlo,8,$t1

	sll	$Zlo,8,$t2
	sll	$Zlo,24,$Zlo
	zapnot	$t0,0x11,$t0
	zapnot	$t1,0x22,$t1

	zapnot	$Zlo,0x88,$Zlo
	or	$t0,$t1,$t0
	zapnot	$t2,0x44,$t2

	or	$Zlo,$t0,$Zlo
	srl	$Zhi,24,$t0
	srl	$Zhi,8,$t1

	or	$Zlo,$t2,$Zlo
	sll	$Zhi,8,$t2
	sll	$Zhi,24,$Zhi

	srl	$Zlo,32,$Xlo
	sll	$Zlo,32,$Zlo
	beq	$len,.Ldone

	zapnot	$t0,0x11,$t0
	zapnot	$t1,0x22,$t1
	or	$Zlo,$Xlo,$Xlo
	ldq_u	$inhi,0($inp)

	zapnot	$Zhi,0x88,$Zhi
	or	$t0,$t1,$t0
	zapnot	$t2,0x44,$t2
	ldq_u	$Thi0,7($inp)

	or	$Zhi,$t0,$Zhi
	or	$Zhi,$t2,$Zhi
	ldq_u	$inlo,8($inp)
	ldq_u	$Tlo0,15($inp)

	srl	$Zhi,32,$Xhi
	sll	$Zhi,32,$Zhi

	or	$Zhi,$Xhi,$Xhi
	br	zero,.Louter

.Ldone:
	zapnot	$t0,0x11,$t0
	zapnot	$t1,0x22,$t1
	or	$Zlo,$Xlo,$Xlo

	zapnot	$Zhi,0x88,$Zhi
	or	$t0,$t1,$t0
	zapnot	$t2,0x44,$t2

	or	$Zhi,$t0,$Zhi
	or	$Zhi,$t2,$Zhi

	srl	$Zhi,32,$Xhi
	sll	$Zhi,32,$Zhi

	or	$Zhi,$Xhi,$Xhi

	stq	$Xlo,8($Xi)
	stq	$Xhi,0($Xi)

	.set	noreorder
	/*ldq	ra,0(sp)*/
	ldq	s0,8(sp)
	ldq	s1,16(sp)
	lda	sp,32(sp)
	ret	(ra)
.end	gcm_ghash_4bit

.align	4
.ent	picmeup
picmeup:
	.frame	sp,0,$t0
	.prologue 0
	br	$rem_4bit,.Lpic
.Lpic:	lda	$rem_4bit,12($rem_4bit)
	ret	($t0)
.end	picmeup
	nop
rem_4bit:
	.long	0,0x0000<<16, 0,0x1C20<<16, 0,0x3840<<16, 0,0x2460<<16
	.long	0,0x7080<<16, 0,0x6CA0<<16, 0,0x48C0<<16, 0,0x54E0<<16
	.long	0,0xE100<<16, 0,0xFD20<<16, 0,0xD940<<16, 0,0xC560<<16
	.long	0,0x9180<<16, 0,0x8DA0<<16, 0,0xA9C0<<16, 0,0xB5E0<<16
.ascii	"GHASH for Alpha, CRYPTOGAMS by <appro\@openssl.org>"
.align	4

___
$output=pop and open STDOUT,">$output";
print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# April 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+32 bytes shared table]. There is no
# experimental performance data available yet. The only approximation
# that can be made at this point is based on code size. Inner loop is
# 32 instructions long and on single-issue core should execute in <40
# cycles. Having verified that gcc 3.4 didn't unroll corresponding
# loop, this assembler loop body was found to be ~3x smaller than
# compiler-generated one...
#
# July 2010
#
# Rescheduling for dual-issue pipeline resulted in 8.5% improvement on
# Cortex A8 core and ~25 cycles per processed byte (which was observed
# to be ~3 times faster than gcc-generated code:-)
#
# February 2011
#
# Profiler-assisted and platform-specific optimization resulted in 7%
# improvement on Cortex A8 core and ~23.5 cycles per byte.
#
# March 2011
#
# Add NEON implementation featuring polynomial multiplication, i.e. no
# lookup tables involved. On Cortex A8 it was measured to process one
# byte in 15 cycles or 55% faster than integer-only code.
#
# April 2014
#
# Switch to multiplication algorithm suggested in paper referred
# below and combine it with reduction algorithm from x86 module.
# Performance improvement over previous version varies from 65% on
# Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8
# processes one byte in 8.45 cycles, A9 - in 10.2, A15 - in 7.63,
# Snapdragon S4 - in 9.33.
#
# CÃ¢mara, D.; GouvÃªa, C. P. L.; LÃ³pez, J. & Dahab, R.: Fast Software
# Polynomial Multiplication on ARM Processors using the NEON Engine.
#
# http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf

# ====================================================================
# Note about "528B" variant. In ARM case it makes lesser sense to
# implement it for following reasons:
#
# - performance improvement won't be anywhere near 50%, because 128-
#   bit shift operation is neatly fused with 128-bit xor here, and
#   "538B" variant would eliminate only 4-5 instructions out of 32
#   in the inner loop (meaning that estimated improvement is ~15%);
# - ARM-based systems are often embedded ones and extra memory
#   consumption might be unappreciated (for so little improvement);
#
# Byte order [in]dependence. =========================================
#
# Caller is expected to maintain specific *dword* order in Htable,
# namely with *least* significant dword of 128-bit value at *lower*
# address. This differs completely from C code and has everything to
# do with ldm instruction and order in which dwords are "consumed" by
# algorithm. *Byte* order within these dwords in turn is whatever
# *native* byte order on current platform. See gcm128.c for working
# example...

$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
    open STDOUT,">$output";
}

$Xi="r0";	# argument block
$Htbl="r1";
$inp="r2";
$len="r3";

$Zll="r4";	# variables
$Zlh="r5";
$Zhl="r6";
$Zhh="r7";
$Tll="r8";
$Tlh="r9";
$Thl="r10";
$Thh="r11";
$nlo="r12";
################# r13 is stack pointer
$nhi="r14";
################# r15 is program counter

$rem_4bit=$inp;	# used in gcm_gmult_4bit
$cnt=$len;

sub Zsmash() {
  my $i=12;
  my @args=@_;
  for ($Zll,$Zlh,$Zhl,$Zhh) {
    $code.=<<___;
#if __ARM_ARCH__>=7 && defined(__ARMEL__)
	rev	$_,$_
	str	$_,[$Xi,#$i]
#elif defined(__ARMEB__)
	str	$_,[$Xi,#$i]
#else
	mov	$Tlh,$_,lsr#8
	strb	$_,[$Xi,#$i+3]
	mov	$Thl,$_,lsr#16
	strb	$Tlh,[$Xi,#$i+2]
	mov	$Thh,$_,lsr#24
	strb	$Thl,[$Xi,#$i+1]
	strb	$Thh,[$Xi,#$i]
#endif
___
    $code.="\t".shift(@args)."\n";
    $i-=4;
  }
}

$code=<<___;
#include "arm_arch.h"

.text
#if defined(__thumb2__) || defined(__clang__)
.syntax	unified
#define ldrplb  ldrbpl
#define ldrneb  ldrbne
#endif
#if defined(__thumb2__)
.thumb
#else
.code	32
#endif

.type	rem_4bit,%object
.align	5
rem_4bit:
.short	0x0000,0x1C20,0x3840,0x2460
.short	0x7080,0x6CA0,0x48C0,0x54E0
.short	0xE100,0xFD20,0xD940,0xC560
.short	0x9180,0x8DA0,0xA9C0,0xB5E0
.size	rem_4bit,.-rem_4bit

.type	rem_4bit_get,%function
rem_4bit_get:
#if defined(__thumb2__)
	adr	$rem_4bit,rem_4bit
#else
	sub	$rem_4bit,pc,#8+32	@ &rem_4bit
#endif
	b	.Lrem_4bit_got
	nop
	nop
.size	rem_4bit_get,.-rem_4bit_get

.global	gcm_ghash_4bit
.type	gcm_ghash_4bit,%function
.align	4
gcm_ghash_4bit:
#if defined(__thumb2__)
	adr	r12,rem_4bit
#else
	sub	r12,pc,#8+48		@ &rem_4bit
#endif
	add	$len,$inp,$len		@ $len to point at the end
	stmdb	sp!,{r3-r11,lr}		@ save $len/end too

	ldmia	r12,{r4-r11}		@ copy rem_4bit ...
	stmdb	sp!,{r4-r11}		@ ... to stack

	ldrb	$nlo,[$inp,#15]
	ldrb	$nhi,[$Xi,#15]
.Louter:
	eor	$nlo,$nlo,$nhi
	and	$nhi,$nlo,#0xf0
	and	$nlo,$nlo,#0x0f
	mov	$cnt,#14

	add	$Zhh,$Htbl,$nlo,lsl#4
	ldmia	$Zhh,{$Zll-$Zhh}	@ load Htbl[nlo]
	add	$Thh,$Htbl,$nhi
	ldrb	$nlo,[$inp,#14]

	and	$nhi,$Zll,#0xf		@ rem
	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
	add	$nhi,$nhi,$nhi
	eor	$Zll,$Tll,$Zll,lsr#4
	ldrh	$Tll,[sp,$nhi]		@ rem_4bit[rem]
	eor	$Zll,$Zll,$Zlh,lsl#28
	ldrb	$nhi,[$Xi,#14]
	eor	$Zlh,$Tlh,$Zlh,lsr#4
	eor	$Zlh,$Zlh,$Zhl,lsl#28
	eor	$Zhl,$Thl,$Zhl,lsr#4
	eor	$Zhl,$Zhl,$Zhh,lsl#28
	eor	$Zhh,$Thh,$Zhh,lsr#4
	eor	$nlo,$nlo,$nhi
	and	$nhi,$nlo,#0xf0
	and	$nlo,$nlo,#0x0f
	eor	$Zhh,$Zhh,$Tll,lsl#16

.Linner:
	add	$Thh,$Htbl,$nlo,lsl#4
	and	$nlo,$Zll,#0xf		@ rem
	subs	$cnt,$cnt,#1
	add	$nlo,$nlo,$nlo
	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nlo]
	eor	$Zll,$Tll,$Zll,lsr#4
	eor	$Zll,$Zll,$Zlh,lsl#28
	eor	$Zlh,$Tlh,$Zlh,lsr#4
	eor	$Zlh,$Zlh,$Zhl,lsl#28
	ldrh	$Tll,[sp,$nlo]		@ rem_4bit[rem]
	eor	$Zhl,$Thl,$Zhl,lsr#4
#ifdef	__thumb2__
	it	pl
#endif
	ldrplb	$nlo,[$inp,$cnt]
	eor	$Zhl,$Zhl,$Zhh,lsl#28
	eor	$Zhh,$Thh,$Zhh,lsr#4

	add	$Thh,$Htbl,$nhi
	and	$nhi,$Zll,#0xf		@ rem
	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
	add	$nhi,$nhi,$nhi
	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
	eor	$Zll,$Tll,$Zll,lsr#4
#ifdef	__thumb2__
	it	pl
#endif
	ldrplb	$Tll,[$Xi,$cnt]
	eor	$Zll,$Zll,$Zlh,lsl#28
	eor	$Zlh,$Tlh,$Zlh,lsr#4
	ldrh	$Tlh,[sp,$nhi]
	eor	$Zlh,$Zlh,$Zhl,lsl#28
	eor	$Zhl,$Thl,$Zhl,lsr#4
	eor	$Zhl,$Zhl,$Zhh,lsl#28
#ifdef	__thumb2__
	it	pl
#endif
	eorpl	$nlo,$nlo,$Tll
	eor	$Zhh,$Thh,$Zhh,lsr#4
#ifdef	__thumb2__
	itt	pl
#endif
	andpl	$nhi,$nlo,#0xf0
	andpl	$nlo,$nlo,#0x0f
	eor	$Zhh,$Zhh,$Tlh,lsl#16	@ ^= rem_4bit[rem]
	bpl	.Linner

	ldr	$len,[sp,#32]		@ re-load $len/end
	add	$inp,$inp,#16
	mov	$nhi,$Zll
___
	&Zsmash("cmp\t$inp,$len","\n".
				 "#ifdef __thumb2__\n".
				 "	it	ne\n".
				 "#endif\n".
				 "	ldrneb	$nlo,[$inp,#15]");
$code.=<<___;
	bne	.Louter

	add	sp,sp,#36
#if __ARM_ARCH__>=5
	ldmia	sp!,{r4-r11,pc}
#else
	ldmia	sp!,{r4-r11,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	gcm_ghash_4bit,.-gcm_ghash_4bit

.global	gcm_gmult_4bit
.type	gcm_gmult_4bit,%function
gcm_gmult_4bit:
	stmdb	sp!,{r4-r11,lr}
	ldrb	$nlo,[$Xi,#15]
	b	rem_4bit_get
.Lrem_4bit_got:
	and	$nhi,$nlo,#0xf0
	and	$nlo,$nlo,#0x0f
	mov	$cnt,#14

	add	$Zhh,$Htbl,$nlo,lsl#4
	ldmia	$Zhh,{$Zll-$Zhh}	@ load Htbl[nlo]
	ldrb	$nlo,[$Xi,#14]

	add	$Thh,$Htbl,$nhi
	and	$nhi,$Zll,#0xf		@ rem
	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
	add	$nhi,$nhi,$nhi
	eor	$Zll,$Tll,$Zll,lsr#4
	ldrh	$Tll,[$rem_4bit,$nhi]	@ rem_4bit[rem]
	eor	$Zll,$Zll,$Zlh,lsl#28
	eor	$Zlh,$Tlh,$Zlh,lsr#4
	eor	$Zlh,$Zlh,$Zhl,lsl#28
	eor	$Zhl,$Thl,$Zhl,lsr#4
	eor	$Zhl,$Zhl,$Zhh,lsl#28
	eor	$Zhh,$Thh,$Zhh,lsr#4
	and	$nhi,$nlo,#0xf0
	eor	$Zhh,$Zhh,$Tll,lsl#16
	and	$nlo,$nlo,#0x0f

.Loop:
	add	$Thh,$Htbl,$nlo,lsl#4
	and	$nlo,$Zll,#0xf		@ rem
	subs	$cnt,$cnt,#1
	add	$nlo,$nlo,$nlo
	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nlo]
	eor	$Zll,$Tll,$Zll,lsr#4
	eor	$Zll,$Zll,$Zlh,lsl#28
	eor	$Zlh,$Tlh,$Zlh,lsr#4
	eor	$Zlh,$Zlh,$Zhl,lsl#28
	ldrh	$Tll,[$rem_4bit,$nlo]	@ rem_4bit[rem]
	eor	$Zhl,$Thl,$Zhl,lsr#4
#ifdef	__thumb2__
	it	pl
#endif
	ldrplb	$nlo,[$Xi,$cnt]
	eor	$Zhl,$Zhl,$Zhh,lsl#28
	eor	$Zhh,$Thh,$Zhh,lsr#4

	add	$Thh,$Htbl,$nhi
	and	$nhi,$Zll,#0xf		@ rem
	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
	add	$nhi,$nhi,$nhi
	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
	eor	$Zll,$Tll,$Zll,lsr#4
	eor	$Zll,$Zll,$Zlh,lsl#28
	eor	$Zlh,$Tlh,$Zlh,lsr#4
	ldrh	$Tll,[$rem_4bit,$nhi]	@ rem_4bit[rem]
	eor	$Zlh,$Zlh,$Zhl,lsl#28
	eor	$Zhl,$Thl,$Zhl,lsr#4
	eor	$Zhl,$Zhl,$Zhh,lsl#28
	eor	$Zhh,$Thh,$Zhh,lsr#4
#ifdef	__thumb2__
	itt	pl
#endif
	andpl	$nhi,$nlo,#0xf0
	andpl	$nlo,$nlo,#0x0f
	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
	bpl	.Loop
___
	&Zsmash();
$code.=<<___;
#if __ARM_ARCH__>=5
	ldmia	sp!,{r4-r11,pc}
#else
	ldmia	sp!,{r4-r11,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	gcm_gmult_4bit,.-gcm_gmult_4bit
___
{
my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
my ($t0,$t1,$t2,$t3)=map("q$_",(8..12));
my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31));

sub clmul64x64 {
my ($r,$a,$b)=@_;
$code.=<<___;
	vext.8		$t0#lo, $a, $a, #1	@ A1
	vmull.p8	$t0, $t0#lo, $b		@ F = A1*B
	vext.8		$r#lo, $b, $b, #1	@ B1
	vmull.p8	$r, $a, $r#lo		@ E = A*B1
	vext.8		$t1#lo, $a, $a, #2	@ A2
	vmull.p8	$t1, $t1#lo, $b		@ H = A2*B
	vext.8		$t3#lo, $b, $b, #2	@ B2
	vmull.p8	$t3, $a, $t3#lo		@ G = A*B2
	vext.8		$t2#lo, $a, $a, #3	@ A3
	veor		$t0, $t0, $r		@ L = E + F
	vmull.p8	$t2, $t2#lo, $b		@ J = A3*B
	vext.8		$r#lo, $b, $b, #3	@ B3
	veor		$t1, $t1, $t3		@ M = G + H
	vmull.p8	$r, $a, $r#lo		@ I = A*B3
	veor		$t0#lo, $t0#lo, $t0#hi	@ t0 = (L) (P0 + P1) << 8
	vand		$t0#hi, $t0#hi, $k48
	vext.8		$t3#lo, $b, $b, #4	@ B4
	veor		$t1#lo, $t1#lo, $t1#hi	@ t1 = (M) (P2 + P3) << 16
	vand		$t1#hi, $t1#hi, $k32
	vmull.p8	$t3, $a, $t3#lo		@ K = A*B4
	veor		$t2, $t2, $r		@ N = I + J
	veor		$t0#lo, $t0#lo, $t0#hi
	veor		$t1#lo, $t1#lo, $t1#hi
	veor		$t2#lo, $t2#lo, $t2#hi	@ t2 = (N) (P4 + P5) << 24
	vand		$t2#hi, $t2#hi, $k16
	vext.8		$t0, $t0, $t0, #15
	veor		$t3#lo, $t3#lo, $t3#hi	@ t3 = (K) (P6 + P7) << 32
	vmov.i64	$t3#hi, #0
	vext.8		$t1, $t1, $t1, #14
	veor		$t2#lo, $t2#lo, $t2#hi
	vmull.p8	$r, $a, $b		@ D = A*B
	vext.8		$t3, $t3, $t3, #12
	vext.8		$t2, $t2, $t2, #13
	veor		$t0, $t0, $t1
	veor		$t2, $t2, $t3
	veor		$r, $r, $t0
	veor		$r, $r, $t2
___
}

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.arch	armv7-a
.fpu	neon

.global	gcm_init_neon
.type	gcm_init_neon,%function
.align	4
gcm_init_neon:
	vld1.64		$IN#hi,[r1]!		@ load H
	vmov.i8		$t0,#0xe1
	vld1.64		$IN#lo,[r1]
	vshl.i64	$t0#hi,#57
	vshr.u64	$t0#lo,#63		@ t0=0xc2....01
	vdup.8		$t1,$IN#hi[7]
	vshr.u64	$Hlo,$IN#lo,#63
	vshr.s8		$t1,#7			@ broadcast carry bit
	vshl.i64	$IN,$IN,#1
	vand		$t0,$t0,$t1
	vorr		$IN#hi,$Hlo		@ H<<<=1
	veor		$IN,$IN,$t0		@ twisted H
	vstmia		r0,{$IN}

	ret					@ bx lr
.size	gcm_init_neon,.-gcm_init_neon

.global	gcm_gmult_neon
.type	gcm_gmult_neon,%function
.align	4
gcm_gmult_neon:
	vld1.64		$IN#hi,[$Xi]!		@ load Xi
	vld1.64		$IN#lo,[$Xi]!
	vmov.i64	$k48,#0x0000ffffffffffff
	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
	vmov.i64	$k32,#0x00000000ffffffff
#ifdef __ARMEL__
	vrev64.8	$IN,$IN
#endif
	vmov.i64	$k16,#0x000000000000ffff
	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing
	mov		$len,#16
	b		.Lgmult_neon
.size	gcm_gmult_neon,.-gcm_gmult_neon

.global	gcm_ghash_neon
.type	gcm_ghash_neon,%function
.align	4
gcm_ghash_neon:
	vld1.64		$Xl#hi,[$Xi]!		@ load Xi
	vld1.64		$Xl#lo,[$Xi]!
	vmov.i64	$k48,#0x0000ffffffffffff
	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
	vmov.i64	$k32,#0x00000000ffffffff
#ifdef __ARMEL__
	vrev64.8	$Xl,$Xl
#endif
	vmov.i64	$k16,#0x000000000000ffff
	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing

.Loop_neon:
	vld1.64		$IN#hi,[$inp]!		@ load inp
	vld1.64		$IN#lo,[$inp]!
#ifdef __ARMEL__
	vrev64.8	$IN,$IN
#endif
	veor		$IN,$Xl			@ inp^=Xi
.Lgmult_neon:
___
	&clmul64x64	($Xl,$Hlo,"$IN#lo");	# H.loÂ·Xi.lo
$code.=<<___;
	veor		$IN#lo,$IN#lo,$IN#hi	@ Karatsuba pre-processing
___
	&clmul64x64	($Xm,$Hhl,"$IN#lo");	# (H.lo+H.hi)Â·(Xi.lo+Xi.hi)
	&clmul64x64	($Xh,$Hhi,"$IN#hi");	# H.hiÂ·Xi.hi
$code.=<<___;
	veor		$Xm,$Xm,$Xl		@ Karatsuba post-processing
	veor		$Xm,$Xm,$Xh
	veor		$Xl#hi,$Xl#hi,$Xm#lo
	veor		$Xh#lo,$Xh#lo,$Xm#hi	@ Xh|Xl - 256-bit result

	@ equivalent of reduction_avx from ghash-x86_64.pl
	vshl.i64	$t1,$Xl,#57		@ 1st phase
	vshl.i64	$t2,$Xl,#62
	veor		$t2,$t2,$t1		@
	vshl.i64	$t1,$Xl,#63
	veor		$t2, $t2, $t1		@
 	veor		$Xl#hi,$Xl#hi,$t2#lo	@
	veor		$Xh#lo,$Xh#lo,$t2#hi

	vshr.u64	$t2,$Xl,#1		@ 2nd phase
	veor		$Xh,$Xh,$Xl
	veor		$Xl,$Xl,$t2		@
	vshr.u64	$t2,$t2,#6
	vshr.u64	$Xl,$Xl,#1		@
	veor		$Xl,$Xl,$Xh		@
	veor		$Xl,$Xl,$t2		@

	subs		$len,#16
	bne		.Loop_neon

#ifdef __ARMEL__
	vrev64.8	$Xl,$Xl
#endif
	sub		$Xi,#16
	vst1.64		$Xl#hi,[$Xi]!		@ write out Xi
	vst1.64		$Xl#lo,[$Xi]

	ret					@ bx lr
.size	gcm_ghash_neon,.-gcm_ghash_neon
#endif
___
}
$code.=<<___;
.asciz  "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
.align  2
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/geo;

	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
	s/\bret\b/bx	lr/go		or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;    # make it possible to compile with -march=armv4

	print $_,"\n";
}
close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#! /usr/bin/env perl
# Copyright 2012-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# December 2011
#
# The module implements GCM GHASH function and underlying single
# multiplication operation in GF(2^128). Even though subroutines
# have _4bit suffix, they are not using any tables, but rely on
# hardware Galois Field Multiply support. Streamed GHASH processes
# byte in ~7 cycles, which is >6x faster than "4-bit" table-driven
# code compiled with TI's cl6x 6.0 with -mv6400+ -o2 flags. We are
# comparing apples vs. oranges, but compiler surely could have done
# better, because theoretical [though not necessarily achievable]
# estimate for "4-bit" table-driven implementation is ~12 cycles.

while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

($Xip,$Htable,$inp,$len)=("A4","B4","A6","B6");	# arguments

($Z0,$Z1,$Z2,$Z3,	$H0, $H1, $H2, $H3,
			$H0x,$H1x,$H2x,$H3x)=map("A$_",(16..27));
($H01u,$H01y,$H2u,$H3u,	$H0y,$H1y,$H2y,$H3y,
			$H0z,$H1z,$H2z,$H3z)=map("B$_",(16..27));
($FF000000,$E10000)=("B30","B31");
($xip,$x0,$x1,$xib)=map("B$_",(6..9));	# $xip zaps $len
 $xia="A9";
($rem,$res)=("B4","B5");		# $rem zaps $Htable

$code.=<<___;
	.text

	.if	.ASSEMBLER_VERSION<7000000
	.asg	0,__TI_EABI__
	.endif
	.if	__TI_EABI__
	.asg	gcm_gmult_1bit,_gcm_gmult_1bit
	.asg	gcm_gmult_4bit,_gcm_gmult_4bit
	.asg	gcm_ghash_4bit,_gcm_ghash_4bit
	.endif

	.asg	B3,RA

	.if	0
	.global	_gcm_gmult_1bit
_gcm_gmult_1bit:
	ADDAD	$Htable,2,$Htable
	.endif
	.global	_gcm_gmult_4bit
_gcm_gmult_4bit:
	.asmfunc
	LDDW	*${Htable}[-1],$H1:$H0	; H.lo
	LDDW	*${Htable}[-2],$H3:$H2	; H.hi
||	MV	$Xip,${xip}		; reassign Xi
||	MVK	15,B1			; SPLOOPD constant

	MVK	0xE1,$E10000
||	LDBU	*++${xip}[15],$x1	; Xi[15]
	MVK	0xFF,$FF000000
||	LDBU	*--${xip},$x0		; Xi[14]
	SHL	$E10000,16,$E10000	; [pre-shifted] reduction polynomial
	SHL	$FF000000,24,$FF000000	; upper byte mask
||	BNOP	ghash_loop?
||	MVK	1,B0			; take a single spin

	PACKH2	$H0,$H1,$xia		; pack H0' and H1's upper bytes
	AND	$H2,$FF000000,$H2u	; H2's upper byte
	AND	$H3,$FF000000,$H3u	; H3's upper byte
||	SHRU	$H2u,8,$H2u
	SHRU	$H3u,8,$H3u
||	ZERO	$Z1:$Z0
	SHRU2	$xia,8,$H01u
||	ZERO	$Z3:$Z2
	.endasmfunc

	.global	_gcm_ghash_4bit
_gcm_ghash_4bit:
	.asmfunc
	LDDW	*${Htable}[-1],$H1:$H0	; H.lo
||	SHRU	$len,4,B0		; reassign len
	LDDW	*${Htable}[-2],$H3:$H2	; H.hi
||	MV	$Xip,${xip}		; reassign Xi
||	MVK	15,B1			; SPLOOPD constant

	MVK	0xE1,$E10000
|| [B0]	LDNDW	*${inp}[1],$H1x:$H0x
	MVK	0xFF,$FF000000
|| [B0]	LDNDW	*${inp}++[2],$H3x:$H2x
	SHL	$E10000,16,$E10000	; [pre-shifted] reduction polynomial
||	LDDW	*${xip}[1],$Z1:$Z0
	SHL	$FF000000,24,$FF000000	; upper byte mask
||	LDDW	*${xip}[0],$Z3:$Z2

	PACKH2	$H0,$H1,$xia		; pack H0' and H1's upper bytes
	AND	$H2,$FF000000,$H2u	; H2's upper byte
	AND	$H3,$FF000000,$H3u	; H3's upper byte
||	SHRU	$H2u,8,$H2u
	SHRU	$H3u,8,$H3u
	SHRU2	$xia,8,$H01u

|| [B0]	XOR	$H0x,$Z0,$Z0		; Xi^=inp
|| [B0]	XOR	$H1x,$Z1,$Z1
	.if	.LITTLE_ENDIAN
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	SHRU	$Z1,24,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0]	SHRU	$Z1,16,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.else
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	MV	$Z0,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0] SHRU	$Z0,8,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.endif
	STDW	$Z3:$Z2,*${xip}[0]
|| [B0]	ZERO	$Z3:$Z2
|| [B0]	MV	$xia,$x1
   [B0]	ADDK	14,${xip}

ghash_loop?:
	SPLOOPD	6			; 6*16+7
||	MVC	B1,ILC
|| [B0]	SUB	B0,1,B0
||	ZERO	A0
||	ADD	$x1,$x1,$xib		; SHL	$x1,1,$xib
||	SHL	$x1,1,$xia
___

########____________________________
#  0    D2.     M1          M2      |
#  1            M1                  |
#  2            M1          M2      |
#  3        D1. M1          M2      |
#  4        S1. L1                  |
#  5    S2  S1x L1          D2  L2  |____________________________
#  6/0          L1  S1      L2  S2x |D2.     M1          M2      |
#  7/1          L1  S1  D1x S2  M2  |        M1                  |
#  8/2              S1  L1x S2      |        M1          M2      |
#  9/3              S1  L1x         |    D1. M1          M2      |
# 10/4                  D1x         |    S1. L1                  |
# 11/5                              |S2  S1x L1          D2  L2  |____________
# 12/6/0                D1x       __|        L1  S1      L2  S2x |D2.     ....
#    7/1                                     L1  S1  D1x S2  M2  |        ....
#    8/2                                         S1  L1x S2      |        ....
#####...                                         ................|............
$code.=<<___;
	XORMPY	$H0,$xia,$H0x		; 0	; HÂ·(Xi[i]<<1)
||	XORMPY	$H01u,$xib,$H01y
|| [A0]	LDBU	*--${xip},$x0
	XORMPY	$H1,$xia,$H1x		; 1
	XORMPY	$H2,$xia,$H2x		; 2
||	XORMPY	$H2u,$xib,$H2y
	XORMPY	$H3,$xia,$H3x		; 3
||	XORMPY	$H3u,$xib,$H3y
||[!A0]	MVK.D	15,A0				; *--${xip} counter
	XOR.L	$H0x,$Z0,$Z0		; 4	; Z^=HÂ·(Xi[i]<<1)
|| [A0]	SUB.S	A0,1,A0
	XOR.L	$H1x,$Z1,$Z1		; 5
||	AND.D	$H01y,$FF000000,$H0z
||	SWAP2.L	$H01y,$H1y		;	; SHL	$H01y,16,$H1y
||	SHL	$x0,1,$xib
||	SHL	$x0,1,$xia

	XOR.L	$H2x,$Z2,$Z2		; 6/0	; [0,0] in epilogue
||	SHL	$Z0,1,$rem		;	; rem=Z<<1
||	SHRMB.S	$Z1,$Z0,$Z0		;	; Z>>=8
||	AND.L	$H1y,$FF000000,$H1z
	XOR.L	$H3x,$Z3,$Z3		; 7/1
||	SHRMB.S	$Z2,$Z1,$Z1
||	XOR.D	$H0z,$Z0,$Z0			; merge upper byte products
||	AND.S	$H2y,$FF000000,$H2z
||	XORMPY	$E10000,$rem,$res	;	; implicit rem&0x1FE
	XOR.L	$H1z,$Z1,$Z1		; 8/2
||	SHRMB.S	$Z3,$Z2,$Z2
||	AND.S	$H3y,$FF000000,$H3z
	XOR.L	$H2z,$Z2,$Z2		; 9/3
||	SHRU	$Z3,8,$Z3
	XOR.D	$H3z,$Z3,$Z3		; 10/4
	NOP				; 11/5

	SPKERNEL 0,2
||	XOR.D	$res,$Z3,$Z3		; 12/6/0; Z^=res

	; input pre-fetch is possible where D1 slot is available...
   [B0]	LDNDW	*${inp}[1],$H1x:$H0x	; 8/-
   [B0]	LDNDW	*${inp}++[2],$H3x:$H2x	; 9/-
	NOP				; 10/-
	.if	.LITTLE_ENDIAN
	SWAP2	$Z0,$Z1			; 11/-
||	SWAP4	$Z1,$Z0
	SWAP4	$Z1,$Z1			; 12/-
||	SWAP2	$Z0,$Z0
	SWAP2	$Z2,$Z3
||	SWAP4	$Z3,$Z2
||[!B0]	BNOP	RA
	SWAP4	$Z3,$Z3
||	SWAP2	$Z2,$Z2
|| [B0]	BNOP	ghash_loop?
   [B0]	XOR	$H0x,$Z0,$Z0		; Xi^=inp
|| [B0]	XOR	$H1x,$Z1,$Z1
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	SHRU	$Z1,24,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0]	SHRU	$Z1,16,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.else
  [!B0]	BNOP	RA			; 11/-
   [B0]	BNOP	ghash_loop?		; 12/-
   [B0]	XOR	$H0x,$Z0,$Z0		; Xi^=inp
|| [B0]	XOR	$H1x,$Z1,$Z1
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	MV	$Z0,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0] SHRU	$Z0,8,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.endif
	STDW	$Z3:$Z2,*${xip}[0]
|| [B0]	ZERO	$Z3:$Z2
|| [B0]	MV	$xia,$x1
   [B0]	ADDK	14,${xip}
	.endasmfunc

	.sect	.const
	.cstring "GHASH for C64x+, CRYPTOGAMS by <appro\@openssl.org>"
	.align	4
___

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# March 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+128 bytes shared table]. Streamed
# GHASH performance was measured to be 6.67 cycles per processed byte
# on Itanium 2, which is >90% better than Microsoft compiler generated
# code. To anchor to something else sha1-ia64.pl module processes one
# byte in 5.7 cycles. On Itanium GHASH should run at ~8.5 cycles per
# byte.

# September 2010
#
# It was originally thought that it makes lesser sense to implement
# "528B" variant on Itanium 2 for following reason. Because number of
# functional units is naturally limited, it appeared impossible to
# implement "528B" loop in 4 cycles, only in 5. This would mean that
# theoretically performance improvement couldn't be more than 20%.
# But occasionally you prove yourself wrong:-) I figured out a way to
# fold couple of instructions and having freed yet another instruction
# slot by unrolling the loop... Resulting performance is 4.45 cycles
# per processed byte and 50% better than "256B" version. On original
# Itanium performance should remain the same as the "256B" version,
# i.e. ~8.5 cycles.

$output=pop and (open STDOUT,">$output" or die "can't open $output: $!");

if ($^O eq "hpux") {
    $ADDP="addp4";
    for (@ARGV) { $ADDP="add" if (/[\+DD|\-mlp]64/); }
} else { $ADDP="add"; }
for (@ARGV)  {  $big_endian=1 if (/\-DB_ENDIAN/);
                $big_endian=0 if (/\-DL_ENDIAN/);  }
if (!defined($big_endian))
             {  $big_endian=(unpack('L',pack('N',1))==1);  }

sub loop() {
my $label=shift;
my ($p16,$p17)=(shift)?("p63","p63"):("p16","p17"); # mask references to inp

# Loop is scheduled for 6 ticks on Itanium 2 and 8 on Itanium, i.e.
# in scalable manner;-) Naturally assuming data in L1 cache...
# Special note about 'dep' instruction, which is used to construct
# &rem_4bit[Zlo&0xf]. It works, because rem_4bit is aligned at 128
# bytes boundary and lower 7 bits of its address are guaranteed to
# be zero.
$code.=<<___;
$label:
{ .mfi;	(p18)	ld8	Hlo=[Hi[1]],-8
	(p19)	dep	rem=Zlo,rem_4bitp,3,4	}
{ .mfi;	(p19)	xor	Zhi=Zhi,Hhi
	($p17)	xor	xi[1]=xi[1],in[1]	};;
{ .mfi;	(p18)	ld8	Hhi=[Hi[1]]
	(p19)	shrp	Zlo=Zhi,Zlo,4		}
{ .mfi;	(p19)	ld8	rem=[rem]
	(p18)	and	Hi[1]=mask0xf0,xi[2]	};;
{ .mmi;	($p16)	ld1	in[0]=[inp],-1
	(p18)	xor	Zlo=Zlo,Hlo
	(p19)	shr.u	Zhi=Zhi,4		}
{ .mib;	(p19)	xor	Hhi=Hhi,rem
	(p18)	add	Hi[1]=Htbl,Hi[1]	};;

{ .mfi;	(p18)	ld8	Hlo=[Hi[1]],-8
	(p18)	dep	rem=Zlo,rem_4bitp,3,4	}
{ .mfi;	(p17)	shladd	Hi[0]=xi[1],4,r0
	(p18)	xor	Zhi=Zhi,Hhi		};;
{ .mfi;	(p18)	ld8	Hhi=[Hi[1]]
	(p18)	shrp	Zlo=Zhi,Zlo,4		}
{ .mfi;	(p18)	ld8	rem=[rem]
	(p17)	and	Hi[0]=mask0xf0,Hi[0]	};;
{ .mmi;	(p16)	ld1	xi[0]=[Xi],-1
	(p18)	xor	Zlo=Zlo,Hlo
	(p18)	shr.u	Zhi=Zhi,4		}
{ .mib;	(p18)	xor	Hhi=Hhi,rem
	(p17)	add	Hi[0]=Htbl,Hi[0]
	br.ctop.sptk	$label			};;
___
}

$code=<<___;
.explicit
.text

prevfs=r2;	prevlc=r3;	prevpr=r8;
mask0xf0=r21;
rem=r22;	rem_4bitp=r23;
Xi=r24;		Htbl=r25;
inp=r26;	end=r27;
Hhi=r28;	Hlo=r29;
Zhi=r30;	Zlo=r31;

.align	128
.skip	16					// aligns loop body
.global	gcm_gmult_4bit#
.proc	gcm_gmult_4bit#
gcm_gmult_4bit:
	.prologue
{ .mmi;	.save	ar.pfs,prevfs
	alloc	prevfs=ar.pfs,2,6,0,8
	$ADDP	Xi=15,in0			// &Xi[15]
	mov	rem_4bitp=ip		}
{ .mii;	$ADDP	Htbl=8,in1			// &Htbl[0].lo
	.save	ar.lc,prevlc
	mov	prevlc=ar.lc
	.save	pr,prevpr
	mov	prevpr=pr		};;

	.body
	.rotr	in[3],xi[3],Hi[2]

{ .mib;	ld1	xi[2]=[Xi],-1			// Xi[15]
	mov	mask0xf0=0xf0
	brp.loop.imp	.Loop1,.Lend1-16};;
{ .mmi;	ld1	xi[1]=[Xi],-1			// Xi[14]
					};;
{ .mii;	shladd	Hi[1]=xi[2],4,r0
	mov	pr.rot=0x7<<16
	mov	ar.lc=13		};;
{ .mii;	and	Hi[1]=mask0xf0,Hi[1]
	mov	ar.ec=3
	xor	Zlo=Zlo,Zlo		};;
{ .mii;	add	Hi[1]=Htbl,Hi[1]		// &Htbl[nlo].lo
	add	rem_4bitp=rem_4bit#-gcm_gmult_4bit#,rem_4bitp
	xor	Zhi=Zhi,Zhi		};;
___
	&loop	(".Loop1",1);
$code.=<<___;
.Lend1:
{ .mib;	xor	Zhi=Zhi,Hhi		};;	// modulo-scheduling artefact
{ .mib;	mux1	Zlo=Zlo,\@rev		};;
{ .mib;	mux1	Zhi=Zhi,\@rev		};;
{ .mmi;	add	Hlo=9,Xi;;			// ;; is here to prevent
	add	Hhi=1,Xi		};;	// pipeline flush on Itanium
{ .mib;	st8	[Hlo]=Zlo
	mov	pr=prevpr,0x1ffff	};;
{ .mib;	st8	[Hhi]=Zhi
	mov	ar.lc=prevlc
	br.ret.sptk.many	b0	};;
.endp	gcm_gmult_4bit#
___

######################################################################
# "528B" (well, "512B" actually) streamed GHASH
#
$Xip="in0";
$Htbl="in1";
$inp="in2";
$len="in3";
$rem_8bit="loc0";
$mask0xff="loc1";
($sum,$rum) = $big_endian ? ("nop.m","nop.m") : ("sum","rum");

sub load_htable() {
    for (my $i=0;$i<8;$i++) {
	$code.=<<___;
{ .mmi;	ld8	r`16+2*$i+1`=[r8],16		// Htable[$i].hi
	ld8	r`16+2*$i`=[r9],16	}	// Htable[$i].lo
{ .mmi;	ldf8	f`32+2*$i+1`=[r10],16		// Htable[`8+$i`].hi
	ldf8	f`32+2*$i`=[r11],16		// Htable[`8+$i`].lo
___
	$code.=shift	if (($i+$#_)==7);
	$code.="\t};;\n"
    }
}

$code.=<<___;
prevsp=r3;

.align	32
.skip	16					// aligns loop body
.global	gcm_ghash_4bit#
.proc	gcm_ghash_4bit#
gcm_ghash_4bit:
	.prologue
{ .mmi;	.save	ar.pfs,prevfs
	alloc	prevfs=ar.pfs,4,2,0,0
	.vframe	prevsp
	mov	prevsp=sp
	mov	$rem_8bit=ip		};;
	.body
{ .mfi;	$ADDP	r8=0+0,$Htbl
	$ADDP	r9=0+8,$Htbl		}
{ .mfi;	$ADDP	r10=128+0,$Htbl
	$ADDP	r11=128+8,$Htbl		};;
___
	&load_htable(
	"	$ADDP	$Xip=15,$Xip",		# &Xi[15]
	"	$ADDP	$len=$len,$inp",	# &inp[len]
	"	$ADDP	$inp=15,$inp",		# &inp[15]
	"	mov	$mask0xff=0xff",
	"	add	sp=-512,sp",
	"	andcm	sp=sp,$mask0xff",	# align stack frame
	"	add	r14=0,sp",
	"	add	r15=8,sp");
$code.=<<___;
{ .mmi;	$sum	1<<1				// go big-endian
	add	r8=256+0,sp
	add	r9=256+8,sp		}
{ .mmi;	add	r10=256+128+0,sp
	add	r11=256+128+8,sp
	add	$len=-17,$len		};;
___
for($i=0;$i<8;$i++) {	# generate first half of Hshr4[]
my ($rlo,$rhi)=("r".eval(16+2*$i),"r".eval(16+2*$i+1));
$code.=<<___;
{ .mmi;	st8	[r8]=$rlo,16			// Htable[$i].lo
	st8	[r9]=$rhi,16			// Htable[$i].hi
	shrp	$rlo=$rhi,$rlo,4	}//;;
{ .mmi;	stf8	[r10]=f`32+2*$i`,16		// Htable[`8+$i`].lo
	stf8	[r11]=f`32+2*$i+1`,16		// Htable[`8+$i`].hi
	shr.u	$rhi=$rhi,4		};;
{ .mmi;	st8	[r14]=$rlo,16			// Htable[$i].lo>>4
	st8	[r15]=$rhi,16		}//;;	// Htable[$i].hi>>4
___
}
$code.=<<___;
{ .mmi;	ld8	r16=[r8],16			// Htable[8].lo
	ld8	r17=[r9],16		};;	// Htable[8].hi
{ .mmi;	ld8	r18=[r8],16			// Htable[9].lo
	ld8	r19=[r9],16		}	// Htable[9].hi
{ .mmi;	rum	1<<5				// clear um.mfh
	shrp	r16=r17,r16,4		};;
___
for($i=0;$i<6;$i++) {	# generate second half of Hshr4[]
$code.=<<___;
{ .mmi;	ld8	r`20+2*$i`=[r8],16		// Htable[`10+$i`].lo
	ld8	r`20+2*$i+1`=[r9],16		// Htable[`10+$i`].hi
	shr.u	r`16+2*$i+1`=r`16+2*$i+1`,4	};;
{ .mmi;	st8	[r14]=r`16+2*$i`,16		// Htable[`8+$i`].lo>>4
	st8	[r15]=r`16+2*$i+1`,16		// Htable[`8+$i`].hi>>4
	shrp	r`18+2*$i`=r`18+2*$i+1`,r`18+2*$i`,4	}
___
}
$code.=<<___;
{ .mmi;	shr.u	r`16+2*$i+1`=r`16+2*$i+1`,4	};;
{ .mmi;	st8	[r14]=r`16+2*$i`,16		// Htable[`8+$i`].lo>>4
	st8	[r15]=r`16+2*$i+1`,16		// Htable[`8+$i`].hi>>4
	shrp	r`18+2*$i`=r`18+2*$i+1`,r`18+2*$i`,4	}
{ .mmi;	add	$Htbl=256,sp			// &Htable[0]
	add	$rem_8bit=rem_8bit#-gcm_ghash_4bit#,$rem_8bit
	shr.u	r`18+2*$i+1`=r`18+2*$i+1`,4	};;
{ .mmi;	st8	[r14]=r`18+2*$i`		// Htable[`8+$i`].lo>>4
	st8	[r15]=r`18+2*$i+1`	}	// Htable[`8+$i`].hi>>4
___

$in="r15";
@xi=("r16","r17");
@rem=("r18","r19");
($Alo,$Ahi,$Blo,$Bhi,$Zlo,$Zhi)=("r20","r21","r22","r23","r24","r25");
($Atbl,$Btbl)=("r26","r27");

$code.=<<___;	# (p16)
{ .mmi;	ld1	$in=[$inp],-1			//(p16) *inp--
	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	cmp.eq	p0,p6=r0,r0		};;	//	clear p6
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p16),(p17)
{ .mmi;	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mii;	ld1	$in=[$inp],-1			//(p16) *inp--
	dep	$Atbl=$xi[1],$Htbl,4,4		//(p17) &Htable[nlo].lo
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
.align	32
.LOOP:
{ .mmi;
(p6)	st8	[$Xip]=$Zhi,13
	xor	$Zlo=$Zlo,$Zlo
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi].lo
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p16),(p17),(p18)
{ .mmi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	ld8	$rem[0]=[$Btbl],-256		//(p18) Htable[nhi].lo,&Hshr4[nhi].lo
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mfi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	dep	$Atbl=$xi[1],$Htbl,4,4	}	//(p17) &Htable[nlo].lo
{ .mfi;	shladd	$rem[0]=$rem[0],4,r0		//(p18) Htable[nhi].lo<<4
	xor	$Zlo=$Zlo,$Alo		};;	//(p18) Z.lo^=Htable[nlo].lo
{ .mmi;	ld8	$Blo=[$Btbl],8			//(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
	ld1	$in=[$inp],-1		}	//(p16) *inp--
{ .mmi;	xor	$rem[0]=$rem[0],$Zlo		//(p18) Z.lo^(Htable[nhi].lo<<4)
	mov	$Zhi=$Ahi			//(p18) Z.hi^=Htable[nlo].hi
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
{ .mmi;	ld8	$Bhi=[$Btbl]			//(p18) Hshr4[nhi].hi
	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	shrp	$Zlo=$Zhi,$Zlo,8	}	//(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
{ .mmi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi]
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

for ($i=1;$i<14;$i++) {
# Above and below fragments are derived from this one by removing
# unsuitable (p??) instructions.
$code.=<<___;	# (p16),(p17),(p18),(p19)
{ .mmi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	ld8	$rem[0]=[$Btbl],-256		//(p18) Htable[nhi].lo,&Hshr4[nhi].lo
	shr.u	$Zhi=$Zhi,8		}	//(p19) Z.hi>>=8
{ .mmi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo			//(p19) Z.lo^=Hshr4[nhi].lo
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mmi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
	dep	$Atbl=$xi[1],$Htbl,4,4	}	//(p17) &Htable[nlo].lo
{ .mmi;	shladd	$rem[0]=$rem[0],4,r0		//(p18) Htable[nhi].lo<<4
	xor	$Zlo=$Zlo,$Alo			//(p18) Z.lo^=Htable[nlo].lo
	xor	$Zhi=$Zhi,$Bhi		};;	//(p19) Z.hi^=Hshr4[nhi].hi
{ .mmi;	ld8	$Blo=[$Btbl],8			//(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
	ld1	$in=[$inp],-1			//(p16) *inp--
	shl	$rem[1]=$rem[1],48	}	//(p19) rem_8bit[rem]<<48
{ .mmi;	xor	$rem[0]=$rem[0],$Zlo		//(p18) Z.lo^(Htable[nhi].lo<<4)
	xor	$Zhi=$Zhi,$Ahi			//(p18) Z.hi^=Htable[nlo].hi
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
{ .mmi;	ld8	$Bhi=[$Btbl]			//(p18) Hshr4[nhi].hi
	ld1	$xi[0]=[$Xip],-1		//(p16) *Xi--
	shrp	$Zlo=$Zhi,$Zlo,8	}	//(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
{ .mmi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	xor	$Zhi=$Zhi,$rem[1]		//(p19) Z.hi^=rem_8bit[rem]<<48
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi]
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers
}

$code.=<<___;	# (p17),(p18),(p19)
{ .mmi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	ld8	$rem[0]=[$Btbl],-256		//(p18) Htable[nhi].lo,&Hshr4[nhi].lo
	shr.u	$Zhi=$Zhi,8		}	//(p19) Z.hi>>=8
{ .mmi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo			//(p19) Z.lo^=Hshr4[nhi].lo
	xor	$xi[1]=$xi[1],$in	};;	//(p17) xi=$xi[i]^inp[i]
{ .mmi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
	dep	$Atbl=$xi[1],$Htbl,4,4	};;	//(p17) &Htable[nlo].lo
{ .mmi;	shladd	$rem[0]=$rem[0],4,r0		//(p18) Htable[nhi].lo<<4
	xor	$Zlo=$Zlo,$Alo			//(p18) Z.lo^=Htable[nlo].lo
	xor	$Zhi=$Zhi,$Bhi		};;	//(p19) Z.hi^=Hshr4[nhi].hi
{ .mmi;	ld8	$Blo=[$Btbl],8			//(p18) Hshr4[nhi].lo,&Hshr4[nhi].hi
	shl	$rem[1]=$rem[1],48	}	//(p19) rem_8bit[rem]<<48
{ .mmi;	xor	$rem[0]=$rem[0],$Zlo		//(p18) Z.lo^(Htable[nhi].lo<<4)
	xor	$Zhi=$Zhi,$Ahi			//(p18) Z.hi^=Htable[nlo].hi
	and	$xi[1]=-16,$xi[1]	};;	//(p17) nhi=xi&0xf0
{ .mmi;	ld8	$Bhi=[$Btbl]			//(p18) Hshr4[nhi].hi
	shrp	$Zlo=$Zhi,$Zlo,8	}	//(p18) Z.lo=(Z.hi<<56)|(Z.lo>>8)
{ .mmi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	xor	$Zhi=$Zhi,$rem[1]		//(p19) Z.hi^=rem_8bit[rem]<<48
	add	$Btbl=$xi[1],$Htbl	};;	//(p17) &Htable[nhi]
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p18),(p19)
{ .mfi;	ld8	$Alo=[$Atbl],8			//(p18) Htable[nlo].lo,&Htable[nlo].hi
	shr.u	$Zhi=$Zhi,8		}	//(p19) Z.hi>>=8
{ .mfi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo		};;	//(p19) Z.lo^=Hshr4[nhi].lo
{ .mfi;	ld8	$Ahi=[$Atbl]			//(p18) Htable[nlo].hi
	xor	$Zlo=$Zlo,$Alo		}	//(p18) Z.lo^=Htable[nlo].lo
{ .mfi;	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
	xor	$Zhi=$Zhi,$Bhi		};;	//(p19) Z.hi^=Hshr4[nhi].hi
{ .mfi;	ld8	$Blo=[$Btbl],8			//(p18) Htable[nhi].lo,&Htable[nhi].hi
	shl	$rem[1]=$rem[1],48	}	//(p19) rem_8bit[rem]<<48
{ .mfi;	shladd	$rem[0]=$Zlo,4,r0		//(p18) Z.lo<<4
	xor	$Zhi=$Zhi,$Ahi		};;	//(p18) Z.hi^=Htable[nlo].hi
{ .mfi;	ld8	$Bhi=[$Btbl]			//(p18) Htable[nhi].hi
	shrp	$Zlo=$Zhi,$Zlo,4	}	//(p18) Z.lo=(Z.hi<<60)|(Z.lo>>4)
{ .mfi;	and	$rem[0]=$rem[0],$mask0xff	//(p18) rem=($Zlo^(Htable[nhi].lo<<4))&0xff
	xor	$Zhi=$Zhi,$rem[1]	};;	//(p19) Z.hi^=rem_8bit[rem]<<48
___
push (@xi,shift(@xi)); push (@rem,shift(@rem));	# "rotate" registers

$code.=<<___;	# (p19)
{ .mmi;	cmp.ltu	p6,p0=$inp,$len
	add	$inp=32,$inp
	shr.u	$Zhi=$Zhi,4		}	//(p19) Z.hi>>=4
{ .mmi;	shladd	$rem[1]=$rem[1],1,$rem_8bit	//(p19) &rem_8bit[rem]
	xor	$Zlo=$Zlo,$Blo			//(p19) Z.lo^=Hshr4[nhi].lo
	add	$Xip=9,$Xip		};;	//	&Xi.lo
{ .mmi;	ld2	$rem[1]=[$rem[1]]		//(p19) rem_8bit[rem]
(p6)	ld1	$in=[$inp],-1			//[p16] *inp--
(p6)	extr.u	$xi[1]=$Zlo,8,8		}	//[p17] Xi[14]
{ .mmi;	xor	$Zhi=$Zhi,$Bhi			//(p19) Z.hi^=Hshr4[nhi].hi
(p6)	and	$xi[0]=$Zlo,$mask0xff	};;	//[p16] Xi[15]
{ .mmi;	st8	[$Xip]=$Zlo,-8
(p6)	xor	$xi[0]=$xi[0],$in		//[p17] xi=$xi[i]^inp[i]
	shl	$rem[1]=$rem[1],48	};;	//(p19) rem_8bit[rem]<<48
{ .mmi;
(p6)	ld1	$in=[$inp],-1			//[p16] *inp--
	xor	$Zhi=$Zhi,$rem[1]		//(p19) Z.hi^=rem_8bit[rem]<<48
(p6)	dep	$Atbl=$xi[0],$Htbl,4,4	}	//[p17] &Htable[nlo].lo
{ .mib;
(p6)	and	$xi[0]=-16,$xi[0]		//[p17] nhi=xi&0xf0
(p6)	br.cond.dptk.many	.LOOP	};;

{ .mib;	st8	[$Xip]=$Zhi		};;
{ .mib;	$rum	1<<1				// return to little-endian
	.restore	sp
	mov	sp=prevsp
	br.ret.sptk.many	b0	};;
.endp	gcm_ghash_4bit#
___
$code.=<<___;
.align	128
.type	rem_4bit#,\@object
rem_4bit:
        data8	0x0000<<48, 0x1C20<<48, 0x3840<<48, 0x2460<<48
        data8	0x7080<<48, 0x6CA0<<48, 0x48C0<<48, 0x54E0<<48
        data8	0xE100<<48, 0xFD20<<48, 0xD940<<48, 0xC560<<48
        data8	0x9180<<48, 0x8DA0<<48, 0xA9C0<<48, 0xB5E0<<48
.size	rem_4bit#,128
.type	rem_8bit#,\@object
rem_8bit:
	data1	0x00,0x00, 0x01,0xC2, 0x03,0x84, 0x02,0x46, 0x07,0x08, 0x06,0xCA, 0x04,0x8C, 0x05,0x4E
	data1	0x0E,0x10, 0x0F,0xD2, 0x0D,0x94, 0x0C,0x56, 0x09,0x18, 0x08,0xDA, 0x0A,0x9C, 0x0B,0x5E
	data1	0x1C,0x20, 0x1D,0xE2, 0x1F,0xA4, 0x1E,0x66, 0x1B,0x28, 0x1A,0xEA, 0x18,0xAC, 0x19,0x6E
	data1	0x12,0x30, 0x13,0xF2, 0x11,0xB4, 0x10,0x76, 0x15,0x38, 0x14,0xFA, 0x16,0xBC, 0x17,0x7E
	data1	0x38,0x40, 0x39,0x82, 0x3B,0xC4, 0x3A,0x06, 0x3F,0x48, 0x3E,0x8A, 0x3C,0xCC, 0x3D,0x0E
	data1	0x36,0x50, 0x37,0x92, 0x35,0xD4, 0x34,0x16, 0x31,0x58, 0x30,0x9A, 0x32,0xDC, 0x33,0x1E
	data1	0x24,0x60, 0x25,0xA2, 0x27,0xE4, 0x26,0x26, 0x23,0x68, 0x22,0xAA, 0x20,0xEC, 0x21,0x2E
	data1	0x2A,0x70, 0x2B,0xB2, 0x29,0xF4, 0x28,0x36, 0x2D,0x78, 0x2C,0xBA, 0x2E,0xFC, 0x2F,0x3E
	data1	0x70,0x80, 0x71,0x42, 0x73,0x04, 0x72,0xC6, 0x77,0x88, 0x76,0x4A, 0x74,0x0C, 0x75,0xCE
	data1	0x7E,0x90, 0x7F,0x52, 0x7D,0x14, 0x7C,0xD6, 0x79,0x98, 0x78,0x5A, 0x7A,0x1C, 0x7B,0xDE
	data1	0x6C,0xA0, 0x6D,0x62, 0x6F,0x24, 0x6E,0xE6, 0x6B,0xA8, 0x6A,0x6A, 0x68,0x2C, 0x69,0xEE
	data1	0x62,0xB0, 0x63,0x72, 0x61,0x34, 0x60,0xF6, 0x65,0xB8, 0x64,0x7A, 0x66,0x3C, 0x67,0xFE
	data1	0x48,0xC0, 0x49,0x02, 0x4B,0x44, 0x4A,0x86, 0x4F,0xC8, 0x4E,0x0A, 0x4C,0x4C, 0x4D,0x8E
	data1	0x46,0xD0, 0x47,0x12, 0x45,0x54, 0x44,0x96, 0x41,0xD8, 0x40,0x1A, 0x42,0x5C, 0x43,0x9E
	data1	0x54,0xE0, 0x55,0x22, 0x57,0x64, 0x56,0xA6, 0x53,0xE8, 0x52,0x2A, 0x50,0x6C, 0x51,0xAE
	data1	0x5A,0xF0, 0x5B,0x32, 0x59,0x74, 0x58,0xB6, 0x5D,0xF8, 0x5C,0x3A, 0x5E,0x7C, 0x5F,0xBE
	data1	0xE1,0x00, 0xE0,0xC2, 0xE2,0x84, 0xE3,0x46, 0xE6,0x08, 0xE7,0xCA, 0xE5,0x8C, 0xE4,0x4E
	data1	0xEF,0x10, 0xEE,0xD2, 0xEC,0x94, 0xED,0x56, 0xE8,0x18, 0xE9,0xDA, 0xEB,0x9C, 0xEA,0x5E
	data1	0xFD,0x20, 0xFC,0xE2, 0xFE,0xA4, 0xFF,0x66, 0xFA,0x28, 0xFB,0xEA, 0xF9,0xAC, 0xF8,0x6E
	data1	0xF3,0x30, 0xF2,0xF2, 0xF0,0xB4, 0xF1,0x76, 0xF4,0x38, 0xF5,0xFA, 0xF7,0xBC, 0xF6,0x7E
	data1	0xD9,0x40, 0xD8,0x82, 0xDA,0xC4, 0xDB,0x06, 0xDE,0x48, 0xDF,0x8A, 0xDD,0xCC, 0xDC,0x0E
	data1	0xD7,0x50, 0xD6,0x92, 0xD4,0xD4, 0xD5,0x16, 0xD0,0x58, 0xD1,0x9A, 0xD3,0xDC, 0xD2,0x1E
	data1	0xC5,0x60, 0xC4,0xA2, 0xC6,0xE4, 0xC7,0x26, 0xC2,0x68, 0xC3,0xAA, 0xC1,0xEC, 0xC0,0x2E
	data1	0xCB,0x70, 0xCA,0xB2, 0xC8,0xF4, 0xC9,0x36, 0xCC,0x78, 0xCD,0xBA, 0xCF,0xFC, 0xCE,0x3E
	data1	0x91,0x80, 0x90,0x42, 0x92,0x04, 0x93,0xC6, 0x96,0x88, 0x97,0x4A, 0x95,0x0C, 0x94,0xCE
	data1	0x9F,0x90, 0x9E,0x52, 0x9C,0x14, 0x9D,0xD6, 0x98,0x98, 0x99,0x5A, 0x9B,0x1C, 0x9A,0xDE
	data1	0x8D,0xA0, 0x8C,0x62, 0x8E,0x24, 0x8F,0xE6, 0x8A,0xA8, 0x8B,0x6A, 0x89,0x2C, 0x88,0xEE
	data1	0x83,0xB0, 0x82,0x72, 0x80,0x34, 0x81,0xF6, 0x84,0xB8, 0x85,0x7A, 0x87,0x3C, 0x86,0xFE
	data1	0xA9,0xC0, 0xA8,0x02, 0xAA,0x44, 0xAB,0x86, 0xAE,0xC8, 0xAF,0x0A, 0xAD,0x4C, 0xAC,0x8E
	data1	0xA7,0xD0, 0xA6,0x12, 0xA4,0x54, 0xA5,0x96, 0xA0,0xD8, 0xA1,0x1A, 0xA3,0x5C, 0xA2,0x9E
	data1	0xB5,0xE0, 0xB4,0x22, 0xB6,0x64, 0xB7,0xA6, 0xB2,0xE8, 0xB3,0x2A, 0xB1,0x6C, 0xB0,0xAE
	data1	0xBB,0xF0, 0xBA,0x32, 0xB8,0x74, 0xB9,0xB6, 0xBC,0xF8, 0xBD,0x3A, 0xBF,0x7C, 0xBE,0xBE
.size	rem_8bit#,512
stringz	"GHASH for IA64, CRYPTOGAMS by <appro\@openssl.org>"
___

$code =~ s/mux1(\s+)\S+\@rev/nop.i$1 0x0/gm      if ($big_endian);
$code =~ s/\`([^\`]*)\`/eval $1/gem;

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# April 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+128 bytes shared table]. On PA-7100LC
# it processes one byte in 19.6 cycles, which is more than twice as
# fast as code generated by gcc 3.2. PA-RISC 2.0 loop is scheduled for
# 8 cycles, but measured performance on PA-8600 system is ~9 cycles per
# processed byte. This is ~2.2x faster than 64-bit code generated by
# vendor compiler (which used to be very hard to beat:-).
#
# Special thanks to polarhome.com for providing HP-UX account.

$flavour = shift;
$output = shift;
open STDOUT,">$output";

if ($flavour =~ /64/) {
	$LEVEL		="2.0W";
	$SIZE_T		=8;
	$FRAME_MARKER	=80;
	$SAVED_RP	=16;
	$PUSH		="std";
	$PUSHMA		="std,ma";
	$POP		="ldd";
	$POPMB		="ldd,mb";
	$NREGS		=6;
} else {
	$LEVEL		="1.0";	#"\n\t.ALLOW\t2.0";
	$SIZE_T		=4;
	$FRAME_MARKER	=48;
	$SAVED_RP	=20;
	$PUSH		="stw";
	$PUSHMA		="stwm";
	$POP		="ldw";
	$POPMB		="ldwm";
	$NREGS		=11;
}

$FRAME=10*$SIZE_T+$FRAME_MARKER;# NREGS saved regs + frame marker
				#                 [+ argument transfer]

################# volatile registers
$Xi="%r26";	# argument block
$Htbl="%r25";
$inp="%r24";
$len="%r23";
$Hhh=$Htbl;	# variables
$Hll="%r22";
$Zhh="%r21";
$Zll="%r20";
$cnt="%r19";
$rem_4bit="%r28";
$rem="%r29";
$mask0xf0="%r31";

################# preserved registers
$Thh="%r1";
$Tll="%r2";
$nlo="%r3";
$nhi="%r4";
$byte="%r5";
if ($SIZE_T==4) {
	$Zhl="%r6";
	$Zlh="%r7";
	$Hhl="%r8";
	$Hlh="%r9";
	$Thl="%r10";
	$Tlh="%r11";
}
$rem2="%r6";	# used in PA-RISC 2.0 code

$code.=<<___;
	.LEVEL	$LEVEL
	.SPACE	\$TEXT\$
	.SUBSPA	\$CODE\$,QUAD=0,ALIGN=8,ACCESS=0x2C,CODE_ONLY

	.EXPORT	gcm_gmult_4bit,ENTRY,ARGW0=GR,ARGW1=GR
	.ALIGN	64
gcm_gmult_4bit
	.PROC
	.CALLINFO	FRAME=`$FRAME-10*$SIZE_T`,NO_CALLS,SAVE_RP,ENTRY_GR=$NREGS
	.ENTRY
	$PUSH	%r2,-$SAVED_RP(%sp)	; standard prologue
	$PUSHMA	%r3,$FRAME(%sp)
	$PUSH	%r4,`-$FRAME+1*$SIZE_T`(%sp)
	$PUSH	%r5,`-$FRAME+2*$SIZE_T`(%sp)
	$PUSH	%r6,`-$FRAME+3*$SIZE_T`(%sp)
___
$code.=<<___ if ($SIZE_T==4);
	$PUSH	%r7,`-$FRAME+4*$SIZE_T`(%sp)
	$PUSH	%r8,`-$FRAME+5*$SIZE_T`(%sp)
	$PUSH	%r9,`-$FRAME+6*$SIZE_T`(%sp)
	$PUSH	%r10,`-$FRAME+7*$SIZE_T`(%sp)
	$PUSH	%r11,`-$FRAME+8*$SIZE_T`(%sp)
___
$code.=<<___;
	blr	%r0,$rem_4bit
	ldi	3,$rem
L\$pic_gmult
	andcm	$rem_4bit,$rem,$rem_4bit
	addl	$inp,$len,$len
	ldo	L\$rem_4bit-L\$pic_gmult($rem_4bit),$rem_4bit
	ldi	0xf0,$mask0xf0
___
$code.=<<___ if ($SIZE_T==4);
	ldi	31,$rem
	mtctl	$rem,%cr11
	extrd,u,*= $rem,%sar,1,$rem	; executes on PA-RISC 1.0
	b	L\$parisc1_gmult
	nop
___

$code.=<<___;
	ldb	15($Xi),$nlo
	ldo	8($Htbl),$Hll

	and	$mask0xf0,$nlo,$nhi
	depd,z	$nlo,59,4,$nlo

	ldd	$nlo($Hll),$Zll
	ldd	$nlo($Hhh),$Zhh

	depd,z	$Zll,60,4,$rem
	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldb	14($Xi),$nlo

	ldd	$nhi($Hll),$Tll
	ldd	$nhi($Hhh),$Thh
	and	$mask0xf0,$nlo,$nhi
	depd,z	$nlo,59,4,$nlo

	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh
	ldd	$rem($rem_4bit),$rem
	b	L\$oop_gmult_pa2
	ldi	13,$cnt

	.ALIGN	8
L\$oop_gmult_pa2
	xor	$rem,$Zhh,$Zhh		; moved here to work around gas bug
	depd,z	$Zll,60,4,$rem

	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldd	$nlo($Hll),$Tll
	ldd	$nlo($Hhh),$Thh

	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh
	ldd	$rem($rem_4bit),$rem

	xor	$rem,$Zhh,$Zhh
	depd,z	$Zll,60,4,$rem
	ldbx	$cnt($Xi),$nlo

	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldd	$nhi($Hll),$Tll
	ldd	$nhi($Hhh),$Thh

	and	$mask0xf0,$nlo,$nhi
	depd,z	$nlo,59,4,$nlo
	ldd	$rem($rem_4bit),$rem

	xor	$Tll,$Zll,$Zll
	addib,uv -1,$cnt,L\$oop_gmult_pa2
	xor	$Thh,$Zhh,$Zhh

	xor	$rem,$Zhh,$Zhh
	depd,z	$Zll,60,4,$rem

	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldd	$nlo($Hll),$Tll
	ldd	$nlo($Hhh),$Thh

	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh
	ldd	$rem($rem_4bit),$rem

	xor	$rem,$Zhh,$Zhh
	depd,z	$Zll,60,4,$rem

	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldd	$nhi($Hll),$Tll
	ldd	$nhi($Hhh),$Thh

	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh
	ldd	$rem($rem_4bit),$rem

	xor	$rem,$Zhh,$Zhh
	std	$Zll,8($Xi)
	std	$Zhh,0($Xi)
___

$code.=<<___ if ($SIZE_T==4);
	b	L\$done_gmult
	nop

L\$parisc1_gmult
	ldb	15($Xi),$nlo
	ldo	12($Htbl),$Hll
	ldo	8($Htbl),$Hlh
	ldo	4($Htbl),$Hhl

	and	$mask0xf0,$nlo,$nhi
	zdep	$nlo,27,4,$nlo

	ldwx	$nlo($Hll),$Zll
	ldwx	$nlo($Hlh),$Zlh
	ldwx	$nlo($Hhl),$Zhl
	ldwx	$nlo($Hhh),$Zhh
	zdep	$Zll,28,4,$rem
	ldb	14($Xi),$nlo
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$nhi($Hll),$Tll
	shrpw	$Zhl,$Zlh,4,$Zlh
	ldwx	$nhi($Hlh),$Tlh
	shrpw	$Zhh,$Zhl,4,$Zhl
	ldwx	$nhi($Hhl),$Thl
	extru	$Zhh,27,28,$Zhh
	ldwx	$nhi($Hhh),$Thh
	xor	$rem,$Zhh,$Zhh
	and	$mask0xf0,$nlo,$nhi
	zdep	$nlo,27,4,$nlo

	xor	$Tll,$Zll,$Zll
	ldwx	$nlo($Hll),$Tll
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nlo($Hlh),$Tlh
	xor	$Thl,$Zhl,$Zhl
	b	L\$oop_gmult_pa1
	ldi	13,$cnt

	.ALIGN	8
L\$oop_gmult_pa1
	zdep	$Zll,28,4,$rem
	ldwx	$nlo($Hhl),$Thl
	xor	$Thh,$Zhh,$Zhh
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$nlo($Hhh),$Thh
	shrpw	$Zhl,$Zlh,4,$Zlh
	ldbx	$cnt($Xi),$nlo
	xor	$Tll,$Zll,$Zll
	ldwx	$nhi($Hll),$Tll
	shrpw	$Zhh,$Zhl,4,$Zhl
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nhi($Hlh),$Tlh
	extru	$Zhh,27,28,$Zhh
	xor	$Thl,$Zhl,$Zhl
	ldwx	$nhi($Hhl),$Thl
	xor	$rem,$Zhh,$Zhh
	zdep	$Zll,28,4,$rem
	xor	$Thh,$Zhh,$Zhh
	ldwx	$nhi($Hhh),$Thh
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zhl,$Zlh,4,$Zlh
	shrpw	$Zhh,$Zhl,4,$Zhl
	and	$mask0xf0,$nlo,$nhi
	extru	$Zhh,27,28,$Zhh
	zdep	$nlo,27,4,$nlo
	xor	$Tll,$Zll,$Zll
	ldwx	$nlo($Hll),$Tll
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nlo($Hlh),$Tlh
	xor	$rem,$Zhh,$Zhh
	addib,uv -1,$cnt,L\$oop_gmult_pa1
	xor	$Thl,$Zhl,$Zhl

	zdep	$Zll,28,4,$rem
	ldwx	$nlo($Hhl),$Thl
	xor	$Thh,$Zhh,$Zhh
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$nlo($Hhh),$Thh
	shrpw	$Zhl,$Zlh,4,$Zlh
	xor	$Tll,$Zll,$Zll
	ldwx	$nhi($Hll),$Tll
	shrpw	$Zhh,$Zhl,4,$Zhl
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nhi($Hlh),$Tlh
	extru	$Zhh,27,28,$Zhh
	xor	$rem,$Zhh,$Zhh
	xor	$Thl,$Zhl,$Zhl
	ldwx	$nhi($Hhl),$Thl
	xor	$Thh,$Zhh,$Zhh
	ldwx	$nhi($Hhh),$Thh
	zdep	$Zll,28,4,$rem
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	shrpw	$Zhl,$Zlh,4,$Zlh
	shrpw	$Zhh,$Zhl,4,$Zhl
	extru	$Zhh,27,28,$Zhh
	xor	$Tll,$Zll,$Zll
	xor	$Tlh,$Zlh,$Zlh
	xor	$rem,$Zhh,$Zhh
	stw	$Zll,12($Xi)
	xor	$Thl,$Zhl,$Zhl
	stw	$Zlh,8($Xi)
	xor	$Thh,$Zhh,$Zhh
	stw	$Zhl,4($Xi)
	stw	$Zhh,0($Xi)
___
$code.=<<___;
L\$done_gmult
	$POP	`-$FRAME-$SAVED_RP`(%sp),%r2		; standard epilogue
	$POP	`-$FRAME+1*$SIZE_T`(%sp),%r4
	$POP	`-$FRAME+2*$SIZE_T`(%sp),%r5
	$POP	`-$FRAME+3*$SIZE_T`(%sp),%r6
___
$code.=<<___ if ($SIZE_T==4);
	$POP	`-$FRAME+4*$SIZE_T`(%sp),%r7
	$POP	`-$FRAME+5*$SIZE_T`(%sp),%r8
	$POP	`-$FRAME+6*$SIZE_T`(%sp),%r9
	$POP	`-$FRAME+7*$SIZE_T`(%sp),%r10
	$POP	`-$FRAME+8*$SIZE_T`(%sp),%r11
___
$code.=<<___;
	bv	(%r2)
	.EXIT
	$POPMB	-$FRAME(%sp),%r3
	.PROCEND

	.EXPORT	gcm_ghash_4bit,ENTRY,ARGW0=GR,ARGW1=GR,ARGW2=GR,ARGW3=GR
	.ALIGN	64
gcm_ghash_4bit
	.PROC
	.CALLINFO	FRAME=`$FRAME-10*$SIZE_T`,NO_CALLS,SAVE_RP,ENTRY_GR=11
	.ENTRY
	$PUSH	%r2,-$SAVED_RP(%sp)	; standard prologue
	$PUSHMA	%r3,$FRAME(%sp)
	$PUSH	%r4,`-$FRAME+1*$SIZE_T`(%sp)
	$PUSH	%r5,`-$FRAME+2*$SIZE_T`(%sp)
	$PUSH	%r6,`-$FRAME+3*$SIZE_T`(%sp)
___
$code.=<<___ if ($SIZE_T==4);
	$PUSH	%r7,`-$FRAME+4*$SIZE_T`(%sp)
	$PUSH	%r8,`-$FRAME+5*$SIZE_T`(%sp)
	$PUSH	%r9,`-$FRAME+6*$SIZE_T`(%sp)
	$PUSH	%r10,`-$FRAME+7*$SIZE_T`(%sp)
	$PUSH	%r11,`-$FRAME+8*$SIZE_T`(%sp)
___
$code.=<<___;
	blr	%r0,$rem_4bit
	ldi	3,$rem
L\$pic_ghash
	andcm	$rem_4bit,$rem,$rem_4bit
	addl	$inp,$len,$len
	ldo	L\$rem_4bit-L\$pic_ghash($rem_4bit),$rem_4bit
	ldi	0xf0,$mask0xf0
___
$code.=<<___ if ($SIZE_T==4);
	ldi	31,$rem
	mtctl	$rem,%cr11
	extrd,u,*= $rem,%sar,1,$rem	; executes on PA-RISC 1.0
	b	L\$parisc1_ghash
	nop
___

$code.=<<___;
	ldb	15($Xi),$nlo
	ldo	8($Htbl),$Hll

L\$outer_ghash_pa2
	ldb	15($inp),$nhi
	xor	$nhi,$nlo,$nlo
	and	$mask0xf0,$nlo,$nhi
	depd,z	$nlo,59,4,$nlo

	ldd	$nlo($Hll),$Zll
	ldd	$nlo($Hhh),$Zhh

	depd,z	$Zll,60,4,$rem
	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldb	14($Xi),$nlo
	ldb	14($inp),$byte

	ldd	$nhi($Hll),$Tll
	ldd	$nhi($Hhh),$Thh
	xor	$byte,$nlo,$nlo
	and	$mask0xf0,$nlo,$nhi
	depd,z	$nlo,59,4,$nlo

	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh
	ldd	$rem($rem_4bit),$rem
	b	L\$oop_ghash_pa2
	ldi	13,$cnt

	.ALIGN	8
L\$oop_ghash_pa2
	xor	$rem,$Zhh,$Zhh		; moved here to work around gas bug
	depd,z	$Zll,60,4,$rem2

	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldd	$nlo($Hll),$Tll
	ldd	$nlo($Hhh),$Thh

	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh
	ldbx	$cnt($Xi),$nlo
	ldbx	$cnt($inp),$byte

	depd,z	$Zll,60,4,$rem
	shrpd	$Zhh,$Zll,4,$Zll
	ldd	$rem2($rem_4bit),$rem2

	xor	$rem2,$Zhh,$Zhh
	xor	$byte,$nlo,$nlo
	ldd	$nhi($Hll),$Tll
	ldd	$nhi($Hhh),$Thh

	and	$mask0xf0,$nlo,$nhi
	depd,z	$nlo,59,4,$nlo

	extrd,u	$Zhh,59,60,$Zhh
	xor	$Tll,$Zll,$Zll

	ldd	$rem($rem_4bit),$rem
	addib,uv -1,$cnt,L\$oop_ghash_pa2
	xor	$Thh,$Zhh,$Zhh

	xor	$rem,$Zhh,$Zhh
	depd,z	$Zll,60,4,$rem2

	shrpd	$Zhh,$Zll,4,$Zll
	extrd,u	$Zhh,59,60,$Zhh
	ldd	$nlo($Hll),$Tll
	ldd	$nlo($Hhh),$Thh

	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh

	depd,z	$Zll,60,4,$rem
	shrpd	$Zhh,$Zll,4,$Zll
	ldd	$rem2($rem_4bit),$rem2

	xor	$rem2,$Zhh,$Zhh
	ldd	$nhi($Hll),$Tll
	ldd	$nhi($Hhh),$Thh

	extrd,u	$Zhh,59,60,$Zhh
	xor	$Tll,$Zll,$Zll
	xor	$Thh,$Zhh,$Zhh
	ldd	$rem($rem_4bit),$rem

	xor	$rem,$Zhh,$Zhh
	std	$Zll,8($Xi)
	ldo	16($inp),$inp
	std	$Zhh,0($Xi)
	cmpb,*<> $inp,$len,L\$outer_ghash_pa2
	copy	$Zll,$nlo
___

$code.=<<___ if ($SIZE_T==4);
	b	L\$done_ghash
	nop

L\$parisc1_ghash
	ldb	15($Xi),$nlo
	ldo	12($Htbl),$Hll
	ldo	8($Htbl),$Hlh
	ldo	4($Htbl),$Hhl

L\$outer_ghash_pa1
	ldb	15($inp),$byte
	xor	$byte,$nlo,$nlo
	and	$mask0xf0,$nlo,$nhi
	zdep	$nlo,27,4,$nlo

	ldwx	$nlo($Hll),$Zll
	ldwx	$nlo($Hlh),$Zlh
	ldwx	$nlo($Hhl),$Zhl
	ldwx	$nlo($Hhh),$Zhh
	zdep	$Zll,28,4,$rem
	ldb	14($Xi),$nlo
	ldb	14($inp),$byte
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$nhi($Hll),$Tll
	shrpw	$Zhl,$Zlh,4,$Zlh
	ldwx	$nhi($Hlh),$Tlh
	shrpw	$Zhh,$Zhl,4,$Zhl
	ldwx	$nhi($Hhl),$Thl
	extru	$Zhh,27,28,$Zhh
	ldwx	$nhi($Hhh),$Thh
	xor	$byte,$nlo,$nlo
	xor	$rem,$Zhh,$Zhh
	and	$mask0xf0,$nlo,$nhi
	zdep	$nlo,27,4,$nlo

	xor	$Tll,$Zll,$Zll
	ldwx	$nlo($Hll),$Tll
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nlo($Hlh),$Tlh
	xor	$Thl,$Zhl,$Zhl
	b	L\$oop_ghash_pa1
	ldi	13,$cnt

	.ALIGN	8
L\$oop_ghash_pa1
	zdep	$Zll,28,4,$rem
	ldwx	$nlo($Hhl),$Thl
	xor	$Thh,$Zhh,$Zhh
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$nlo($Hhh),$Thh
	shrpw	$Zhl,$Zlh,4,$Zlh
	ldbx	$cnt($Xi),$nlo
	xor	$Tll,$Zll,$Zll
	ldwx	$nhi($Hll),$Tll
	shrpw	$Zhh,$Zhl,4,$Zhl
	ldbx	$cnt($inp),$byte
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nhi($Hlh),$Tlh
	extru	$Zhh,27,28,$Zhh
	xor	$Thl,$Zhl,$Zhl
	ldwx	$nhi($Hhl),$Thl
	xor	$rem,$Zhh,$Zhh
	zdep	$Zll,28,4,$rem
	xor	$Thh,$Zhh,$Zhh
	ldwx	$nhi($Hhh),$Thh
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zhl,$Zlh,4,$Zlh
	xor	$byte,$nlo,$nlo
	shrpw	$Zhh,$Zhl,4,$Zhl
	and	$mask0xf0,$nlo,$nhi
	extru	$Zhh,27,28,$Zhh
	zdep	$nlo,27,4,$nlo
	xor	$Tll,$Zll,$Zll
	ldwx	$nlo($Hll),$Tll
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nlo($Hlh),$Tlh
	xor	$rem,$Zhh,$Zhh
	addib,uv -1,$cnt,L\$oop_ghash_pa1
	xor	$Thl,$Zhl,$Zhl

	zdep	$Zll,28,4,$rem
	ldwx	$nlo($Hhl),$Thl
	xor	$Thh,$Zhh,$Zhh
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	ldwx	$nlo($Hhh),$Thh
	shrpw	$Zhl,$Zlh,4,$Zlh
	xor	$Tll,$Zll,$Zll
	ldwx	$nhi($Hll),$Tll
	shrpw	$Zhh,$Zhl,4,$Zhl
	xor	$Tlh,$Zlh,$Zlh
	ldwx	$nhi($Hlh),$Tlh
	extru	$Zhh,27,28,$Zhh
	xor	$rem,$Zhh,$Zhh
	xor	$Thl,$Zhl,$Zhl
	ldwx	$nhi($Hhl),$Thl
	xor	$Thh,$Zhh,$Zhh
	ldwx	$nhi($Hhh),$Thh
	zdep	$Zll,28,4,$rem
	ldwx	$rem($rem_4bit),$rem
	shrpw	$Zlh,$Zll,4,$Zll
	shrpw	$Zhl,$Zlh,4,$Zlh
	shrpw	$Zhh,$Zhl,4,$Zhl
	extru	$Zhh,27,28,$Zhh
	xor	$Tll,$Zll,$Zll
	xor	$Tlh,$Zlh,$Zlh
	xor	$rem,$Zhh,$Zhh
	stw	$Zll,12($Xi)
	xor	$Thl,$Zhl,$Zhl
	stw	$Zlh,8($Xi)
	xor	$Thh,$Zhh,$Zhh
	stw	$Zhl,4($Xi)
	ldo	16($inp),$inp
	stw	$Zhh,0($Xi)
	comb,<>	$inp,$len,L\$outer_ghash_pa1
	copy	$Zll,$nlo
___
$code.=<<___;
L\$done_ghash
	$POP	`-$FRAME-$SAVED_RP`(%sp),%r2		; standard epilogue
	$POP	`-$FRAME+1*$SIZE_T`(%sp),%r4
	$POP	`-$FRAME+2*$SIZE_T`(%sp),%r5
	$POP	`-$FRAME+3*$SIZE_T`(%sp),%r6
___
$code.=<<___ if ($SIZE_T==4);
	$POP	`-$FRAME+4*$SIZE_T`(%sp),%r7
	$POP	`-$FRAME+5*$SIZE_T`(%sp),%r8
	$POP	`-$FRAME+6*$SIZE_T`(%sp),%r9
	$POP	`-$FRAME+7*$SIZE_T`(%sp),%r10
	$POP	`-$FRAME+8*$SIZE_T`(%sp),%r11
___
$code.=<<___;
	bv	(%r2)
	.EXIT
	$POPMB	-$FRAME(%sp),%r3
	.PROCEND

	.ALIGN	64
L\$rem_4bit
	.WORD	`0x0000<<16`,0,`0x1C20<<16`,0,`0x3840<<16`,0,`0x2460<<16`,0
	.WORD	`0x7080<<16`,0,`0x6CA0<<16`,0,`0x48C0<<16`,0,`0x54E0<<16`,0
	.WORD	`0xE100<<16`,0,`0xFD20<<16`,0,`0xD940<<16`,0,`0xC560<<16`,0
	.WORD	`0x9180<<16`,0,`0x8DA0<<16`,0,`0xA9C0<<16`,0,`0xB5E0<<16`,0
	.STRINGZ "GHASH for PA-RISC, GRYPTOGAMS by <appro\@openssl.org>"
	.ALIGN	64
___

# Explicitly encode PA-RISC 2.0 instructions used in this module, so
# that it can be compiled with .LEVEL 1.0. It should be noted that I
# wouldn't have to do this, if GNU assembler understood .ALLOW 2.0
# directive...

my $ldd = sub {
  my ($mod,$args) = @_;
  my $orig = "ldd$mod\t$args";

    if ($args =~ /%r([0-9]+)\(%r([0-9]+)\),%r([0-9]+)/)		# format 4
    {	my $opcode=(0x03<<26)|($2<<21)|($1<<16)|(3<<6)|$3;
	sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig;
    }
    elsif ($args =~ /(\-?[0-9]+)\(%r([0-9]+)\),%r([0-9]+)/)	# format 5
    {	my $opcode=(0x03<<26)|($2<<21)|(1<<12)|(3<<6)|$3;
	$opcode|=(($1&0xF)<<17)|(($1&0x10)<<12);		# encode offset
	$opcode|=(1<<5)  if ($mod =~ /^,m/);
	$opcode|=(1<<13) if ($mod =~ /^,mb/);
	sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig;
    }
    else { "\t".$orig; }
};

my $std = sub {
  my ($mod,$args) = @_;
  my $orig = "std$mod\t$args";

    if ($args =~ /%r([0-9]+),(\-?[0-9]+)\(%r([0-9]+)\)/) # format 3 suffices
    {	my $opcode=(0x1c<<26)|($3<<21)|($1<<16)|(($2&0x1FF8)<<1)|(($2>>13)&1);
	sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig;
    }
    else { "\t".$orig; }
};

my $extrd = sub {
  my ($mod,$args) = @_;
  my $orig = "extrd$mod\t$args";

    # I only have ",u" completer, it's implicitly encoded...
    if ($args =~ /%r([0-9]+),([0-9]+),([0-9]+),%r([0-9]+)/)	# format 15
    {	my $opcode=(0x36<<26)|($1<<21)|($4<<16);
	my $len=32-$3;
	$opcode |= (($2&0x20)<<6)|(($2&0x1f)<<5);		# encode pos
	$opcode |= (($len&0x20)<<7)|($len&0x1f);		# encode len
	sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig;
    }
    elsif ($args =~ /%r([0-9]+),%sar,([0-9]+),%r([0-9]+)/)	# format 12
    {	my $opcode=(0x34<<26)|($1<<21)|($3<<16)|(2<<11)|(1<<9);
	my $len=32-$2;
	$opcode |= (($len&0x20)<<3)|($len&0x1f);		# encode len
	$opcode |= (1<<13) if ($mod =~ /,\**=/);
	sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig;
    }
    else { "\t".$orig; }
};

my $shrpd = sub {
  my ($mod,$args) = @_;
  my $orig = "shrpd$mod\t$args";

    if ($args =~ /%r([0-9]+),%r([0-9]+),([0-9]+),%r([0-9]+)/)	# format 14
    {	my $opcode=(0x34<<26)|($2<<21)|($1<<16)|(1<<10)|$4;
	my $cpos=63-$3;
	$opcode |= (($cpos&0x20)<<6)|(($cpos&0x1f)<<5);		# encode sa
	sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig;
    }
    elsif ($args =~ /%r([0-9]+),%r([0-9]+),%sar,%r([0-9]+)/)	# format 11
    {	sprintf "\t.WORD\t0x%08x\t; %s",
		(0x34<<26)|($2<<21)|($1<<16)|(1<<9)|$3,$orig;
    }
    else { "\t".$orig; }
};

my $depd = sub {
  my ($mod,$args) = @_;
  my $orig = "depd$mod\t$args";

    # I only have ",z" completer, it's implicitly encoded...
    if ($args =~ /%r([0-9]+),([0-9]+),([0-9]+),%r([0-9]+)/)	# format 16
    {	my $opcode=(0x3c<<26)|($4<<21)|($1<<16);
    	my $cpos=63-$2;
	my $len=32-$3;
	$opcode |= (($cpos&0x20)<<6)|(($cpos&0x1f)<<5);		# encode pos
	$opcode |= (($len&0x20)<<7)|($len&0x1f);		# encode len
	sprintf "\t.WORD\t0x%08x\t; %s",$opcode,$orig;
    }
    else { "\t".$orig; }
};

sub assemble {
  my ($mnemonic,$mod,$args)=@_;
  my $opcode = eval("\$$mnemonic");

    ref($opcode) eq 'CODE' ? &$opcode($mod,$args) : "\t$mnemonic$mod\t$args";
}

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
	=~ /GNU assembler/) {
    $gnuas = 1;
}

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/ge;
	if ($SIZE_T==4) {
		s/^\s+([a-z]+)([\S]*)\s+([\S]*)/&assemble($1,$2,$3)/e;
		s/cmpb,\*/comb,/;
		s/,\*/,/;
	}

	s/(\.LEVEL\s+2\.0)W/$1w/	if ($gnuas && $SIZE_T==8);
	s/\.SPACE\s+\$TEXT\$/.text/	if ($gnuas && $SIZE_T==8);
	s/\.SUBSPA.*//			if ($gnuas && $SIZE_T==8);
	s/\bbv\b/bve/			if ($SIZE_T==8);

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";