#! /usr/bin/env perl
# Copyright 2010-2018 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# April 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+32 bytes shared table]. There is no
# experimental performance data available yet. The only approximation
# that can be made at this point is based on code size. Inner loop is
# 32 instructions long and on single-issue core should execute in <40
# cycles. Having verified that gcc 3.4 didn't unroll corresponding
# loop, this assembler loop body was found to be ~3x smaller than
# compiler-generated one...
#
# July 2010
#
# Rescheduling for dual-issue pipeline resulted in 8.5% improvement on
# Cortex A8 core and ~25 cycles per processed byte (which was observed
# to be ~3 times faster than gcc-generated code:-)
#
# February 2011
#
# Profiler-assisted and platform-specific optimization resulted in 7%
# improvement on Cortex A8 core and ~23.5 cycles per byte.
#
# March 2011
#
# Add NEON implementation featuring polynomial multiplication, i.e. no
# lookup tables involved. On Cortex A8 it was measured to process one
# byte in 15 cycles or 55% faster than integer-only code.
#
# April 2014
#
# Switch to multiplication algorithm suggested in paper referred
# below and combine it with reduction algorithm from x86 module.
# Performance improvement over previous version varies from 65% on
# Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8
# processes one byte in 8.45 cycles, A9 - in 10.2, A15 - in 7.63,
# Snapdragon S4 - in 9.33.
#
# CÃ¢mara, D.; GouvÃªa, C. P. L.; LÃ³pez, J. & Dahab, R.: Fast Software
# Polynomial Multiplication on ARM Processors using the NEON Engine.
#
# http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf

# ====================================================================
# Note about "528B" variant. In ARM case it makes lesser sense to
# implement it for following reasons:
#
# - performance improvement won't be anywhere near 50%, because 128-
#   bit shift operation is neatly fused with 128-bit xor here, and
#   "538B" variant would eliminate only 4-5 instructions out of 32
#   in the inner loop (meaning that estimated improvement is ~15%);
# - ARM-based systems are often embedded ones and extra memory
#   consumption might be unappreciated (for so little improvement);
#
# Byte order [in]dependence. =========================================
#
# Caller is expected to maintain specific *dword* order in Htable,
# namely with *least* significant dword of 128-bit value at *lower*
# address. This differs completely from C code and has everything to
# do with ldm instruction and order in which dwords are "consumed" by
# algorithm. *Byte* order within these dwords in turn is whatever
# *native* byte order on current platform. See gcm128.c for working
# example...

# This file was patched in BoringSSL to remove the variable-time 4-bit
# implementation.

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

$flavour = shift;
$output = shift;

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
    *STDOUT=*OUT;
} else {
    open OUT,">$output";
    *STDOUT=*OUT;
}

$Xi="r0";	# argument block
$Htbl="r1";
$inp="r2";
$len="r3";

$code=<<___;
#include <openssl/arm_arch.h>

@ Silence ARMv8 deprecated IT instruction warnings. This file is used by both
@ ARMv7 and ARMv8 processors and does not use ARMv8 instructions. (ARMv8 PMULL
@ instructions are in aesv8-armx.pl.)
.arch  armv7-a

.text
#if defined(__thumb2__) || defined(__clang__)
.syntax	unified
#define ldrplb  ldrbpl
#define ldrneb  ldrbne
#endif
#if defined(__thumb2__)
.thumb
#else
.code	32
#endif
___
{
my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
my ($t0,$t1,$t2,$t3)=map("q$_",(8..12));
my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31));

sub clmul64x64 {
my ($r,$a,$b)=@_;
$code.=<<___;
	vext.8		$t0#lo, $a, $a, #1	@ A1
	vmull.p8	$t0, $t0#lo, $b		@ F = A1*B
	vext.8		$r#lo, $b, $b, #1	@ B1
	vmull.p8	$r, $a, $r#lo		@ E = A*B1
	vext.8		$t1#lo, $a, $a, #2	@ A2
	vmull.p8	$t1, $t1#lo, $b		@ H = A2*B
	vext.8		$t3#lo, $b, $b, #2	@ B2
	vmull.p8	$t3, $a, $t3#lo		@ G = A*B2
	vext.8		$t2#lo, $a, $a, #3	@ A3
	veor		$t0, $t0, $r		@ L = E + F
	vmull.p8	$t2, $t2#lo, $b		@ J = A3*B
	vext.8		$r#lo, $b, $b, #3	@ B3
	veor		$t1, $t1, $t3		@ M = G + H
	vmull.p8	$r, $a, $r#lo		@ I = A*B3
	veor		$t0#lo, $t0#lo, $t0#hi	@ t0 = (L) (P0 + P1) << 8
	vand		$t0#hi, $t0#hi, $k48
	vext.8		$t3#lo, $b, $b, #4	@ B4
	veor		$t1#lo, $t1#lo, $t1#hi	@ t1 = (M) (P2 + P3) << 16
	vand		$t1#hi, $t1#hi, $k32
	vmull.p8	$t3, $a, $t3#lo		@ K = A*B4
	veor		$t2, $t2, $r		@ N = I + J
	veor		$t0#lo, $t0#lo, $t0#hi
	veor		$t1#lo, $t1#lo, $t1#hi
	veor		$t2#lo, $t2#lo, $t2#hi	@ t2 = (N) (P4 + P5) << 24
	vand		$t2#hi, $t2#hi, $k16
	vext.8		$t0, $t0, $t0, #15
	veor		$t3#lo, $t3#lo, $t3#hi	@ t3 = (K) (P6 + P7) << 32
	vmov.i64	$t3#hi, #0
	vext.8		$t1, $t1, $t1, #14
	veor		$t2#lo, $t2#lo, $t2#hi
	vmull.p8	$r, $a, $b		@ D = A*B
	vext.8		$t3, $t3, $t3, #12
	vext.8		$t2, $t2, $t2, #13
	veor		$t0, $t0, $t1
	veor		$t2, $t2, $t3
	veor		$r, $r, $t0
	veor		$r, $r, $t2
___
}

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.arch	armv7-a
.fpu	neon

.global	gcm_init_neon
.type	gcm_init_neon,%function
.align	4
gcm_init_neon:
	vld1.64		$IN#hi,[r1]!		@ load H
	vmov.i8		$t0,#0xe1
	vld1.64		$IN#lo,[r1]
	vshl.i64	$t0#hi,#57
	vshr.u64	$t0#lo,#63		@ t0=0xc2....01
	vdup.8		$t1,$IN#hi[7]
	vshr.u64	$Hlo,$IN#lo,#63
	vshr.s8		$t1,#7			@ broadcast carry bit
	vshl.i64	$IN,$IN,#1
	vand		$t0,$t0,$t1
	vorr		$IN#hi,$Hlo		@ H<<<=1
	veor		$IN,$IN,$t0		@ twisted H
	vstmia		r0,{$IN}

	ret					@ bx lr
.size	gcm_init_neon,.-gcm_init_neon

.global	gcm_gmult_neon
.type	gcm_gmult_neon,%function
.align	4
gcm_gmult_neon:
	vld1.64		$IN#hi,[$Xi]!		@ load Xi
	vld1.64		$IN#lo,[$Xi]!
	vmov.i64	$k48,#0x0000ffffffffffff
	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
	vmov.i64	$k32,#0x00000000ffffffff
#ifdef __ARMEL__
	vrev64.8	$IN,$IN
#endif
	vmov.i64	$k16,#0x000000000000ffff
	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing
	mov		$len,#16
	b		.Lgmult_neon
.size	gcm_gmult_neon,.-gcm_gmult_neon

.global	gcm_ghash_neon
.type	gcm_ghash_neon,%function
.align	4
gcm_ghash_neon:
	vld1.64		$Xl#hi,[$Xi]!		@ load Xi
	vld1.64		$Xl#lo,[$Xi]!
	vmov.i64	$k48,#0x0000ffffffffffff
	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
	vmov.i64	$k32,#0x00000000ffffffff
#ifdef __ARMEL__
	vrev64.8	$Xl,$Xl
#endif
	vmov.i64	$k16,#0x000000000000ffff
	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing

.Loop_neon:
	vld1.64		$IN#hi,[$inp]!		@ load inp
	vld1.64		$IN#lo,[$inp]!
#ifdef __ARMEL__
	vrev64.8	$IN,$IN
#endif
	veor		$IN,$Xl			@ inp^=Xi
.Lgmult_neon:
___
	&clmul64x64	($Xl,$Hlo,"$IN#lo");	# H.loÂ·Xi.lo
$code.=<<___;
	veor		$IN#lo,$IN#lo,$IN#hi	@ Karatsuba pre-processing
___
	&clmul64x64	($Xm,$Hhl,"$IN#lo");	# (H.lo+H.hi)Â·(Xi.lo+Xi.hi)
	&clmul64x64	($Xh,$Hhi,"$IN#hi");	# H.hiÂ·Xi.hi
$code.=<<___;
	veor		$Xm,$Xm,$Xl		@ Karatsuba post-processing
	veor		$Xm,$Xm,$Xh
	veor		$Xl#hi,$Xl#hi,$Xm#lo
	veor		$Xh#lo,$Xh#lo,$Xm#hi	@ Xh|Xl - 256-bit result

	@ equivalent of reduction_avx from ghash-x86_64.pl
	vshl.i64	$t1,$Xl,#57		@ 1st phase
	vshl.i64	$t2,$Xl,#62
	veor		$t2,$t2,$t1		@
	vshl.i64	$t1,$Xl,#63
	veor		$t2, $t2, $t1		@
 	veor		$Xl#hi,$Xl#hi,$t2#lo	@
	veor		$Xh#lo,$Xh#lo,$t2#hi

	vshr.u64	$t2,$Xl,#1		@ 2nd phase
	veor		$Xh,$Xh,$Xl
	veor		$Xl,$Xl,$t2		@
	vshr.u64	$t2,$t2,#6
	vshr.u64	$Xl,$Xl,#1		@
	veor		$Xl,$Xl,$Xh		@
	veor		$Xl,$Xl,$t2		@

	subs		$len,#16
	bne		.Loop_neon

#ifdef __ARMEL__
	vrev64.8	$Xl,$Xl
#endif
	sub		$Xi,#16
	vst1.64		$Xl#hi,[$Xi]!		@ write out Xi
	vst1.64		$Xl#lo,[$Xi]

	ret					@ bx lr
.size	gcm_ghash_neon,.-gcm_ghash_neon
#endif
___
}
$code.=<<___;
.asciz  "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
.align  2
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/geo;

	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
	s/\bret\b/bx	lr/go		or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;    # make it possible to compile with -march=armv4

	print $_,"\n";
}
close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#! /usr/bin/env perl
# Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# This file was adapted to AArch64 from the 32-bit version in ghash-armv4.pl. It
# implements the multiplication algorithm described in:
#
# CÃ¢mara, D.; GouvÃªa, C. P. L.; LÃ³pez, J. & Dahab, R.: Fast Software
# Polynomial Multiplication on ARM Processors using the NEON Engine.
#
# http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf
#
# The main distinction to keep in mind between 32-bit NEON and AArch64 SIMD is
# AArch64 cannot compute over the upper halves of SIMD registers. In 32-bit
# NEON, the low and high halves of the 128-bit register q0 are accessible as
# 64-bit registers d0 and d1, respectively. In AArch64, dN is the lower half of
# vN. Where the 32-bit version would use the upper half, this file must keep
# halves in separate registers.
#
# The other distinction is in syntax. 32-bit NEON embeds lane information in the
# instruction name, while AArch64 uses suffixes on the registers. For instance,
# left-shifting 64-bit lanes of a SIMD register in 32-bit would be written:
#
#     vshl.i64 q0, q0, #1
#
# in 64-bit, it would be written:
#
#     shl v0.2d, v0.2d, #1
#
# See Programmer's Guide for ARMv8-A, section 7 for details.
# http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf
#
# Finally, note the 8-bit and 64-bit polynomial multipliers in AArch64 differ
# only by suffix. pmull vR.8h, vA.8b, vB.8b multiplies eight 8-bit polynomials
# and is always available. pmull vR.1q, vA.1d, vB.1d multiplies a 64-bit
# polynomial and is conditioned on the PMULL extension. This file emulates the
# latter with the former.

use strict;

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

my $flavour = shift;
my $output = shift;

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/;
    my $dir = $1;
    my $xlate;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
    *STDOUT=*OUT;
} else {
    open OUT,">$output";
    *STDOUT=*OUT;
}

my ($Xi, $Htbl, $inp, $len) = map("x$_", (0..3));	# argument block
my ($Xl, $Xm, $Xh, $INlo, $INhi) = map("v$_", (0..4));
my ($Hlo, $Hhi, $Hhl) = map("v$_", (5..7));
# d8-d15 are callee-saved, so avoid v8-v15. AArch64 SIMD has plenty of registers
# to spare.
my ($t0, $t1, $t2, $t3) = map("v$_", (16..19));
my ($t0l_t1l, $t0h_t1h, $t2l_t3l, $t2h_t3h) = map("v$_", (20..23));
my ($k48_k32, $k16_k0) = map("v$_", (24..25));

my $code = "";

# clmul64x64 emits code which emulates pmull $r.1q, $a.1d, $b.1d. $r, $a, and $b
# must be distinct from $t* and $k*. $t* are clobbered by the emitted code.
sub clmul64x64 {
my ($r, $a, $b) = @_;
$code .= <<___;
	ext	$t0.8b, $a.8b, $a.8b, #1	// A1
	pmull	$t0.8h, $t0.8b, $b.8b		// F = A1*B
	ext	$r.8b, $b.8b, $b.8b, #1		// B1
	pmull	$r.8h, $a.8b, $r.8b		// E = A*B1
	ext	$t1.8b, $a.8b, $a.8b, #2	// A2
	pmull	$t1.8h, $t1.8b, $b.8b		// H = A2*B
	ext	$t3.8b, $b.8b, $b.8b, #2	// B2
	pmull	$t3.8h, $a.8b, $t3.8b		// G = A*B2
	ext	$t2.8b, $a.8b, $a.8b, #3	// A3
	eor	$t0.16b, $t0.16b, $r.16b	// L = E + F
	pmull	$t2.8h, $t2.8b, $b.8b		// J = A3*B
	ext	$r.8b, $b.8b, $b.8b, #3		// B3
	eor	$t1.16b, $t1.16b, $t3.16b	// M = G + H
	pmull	$r.8h, $a.8b, $r.8b		// I = A*B3

	// Here we diverge from the 32-bit version. It computes the following
	// (instructions reordered for clarity):
	//
	//     veor	\$t0#lo, \$t0#lo, \$t0#hi	@ t0 = P0 + P1 (L)
	//     vand	\$t0#hi, \$t0#hi, \$k48
	//     veor	\$t0#lo, \$t0#lo, \$t0#hi
	//
	//     veor	\$t1#lo, \$t1#lo, \$t1#hi	@ t1 = P2 + P3 (M)
	//     vand	\$t1#hi, \$t1#hi, \$k32
	//     veor	\$t1#lo, \$t1#lo, \$t1#hi
	//
	//     veor	\$t2#lo, \$t2#lo, \$t2#hi	@ t2 = P4 + P5 (N)
	//     vand	\$t2#hi, \$t2#hi, \$k16
	//     veor	\$t2#lo, \$t2#lo, \$t2#hi
	//
	//     veor	\$t3#lo, \$t3#lo, \$t3#hi	@ t3 = P6 + P7 (K)
	//     vmov.i64	\$t3#hi, #0
	//
	// \$kN is a mask with the bottom N bits set. AArch64 cannot compute on
	// upper halves of SIMD registers, so we must split each half into
	// separate registers. To compensate, we pair computations up and
	// parallelize.

	ext	$t3.8b, $b.8b, $b.8b, #4	// B4
	eor	$t2.16b, $t2.16b, $r.16b	// N = I + J
	pmull	$t3.8h, $a.8b, $t3.8b		// K = A*B4

	// This can probably be scheduled more efficiently. For now, we just
	// pair up independent instructions.
	zip1	$t0l_t1l.2d, $t0.2d, $t1.2d
	zip1	$t2l_t3l.2d, $t2.2d, $t3.2d
	zip2	$t0h_t1h.2d, $t0.2d, $t1.2d
	zip2	$t2h_t3h.2d, $t2.2d, $t3.2d
	eor	$t0l_t1l.16b, $t0l_t1l.16b, $t0h_t1h.16b
	eor	$t2l_t3l.16b, $t2l_t3l.16b, $t2h_t3h.16b
	and	$t0h_t1h.16b, $t0h_t1h.16b, $k48_k32.16b
	and	$t2h_t3h.16b, $t2h_t3h.16b, $k16_k0.16b
	eor	$t0l_t1l.16b, $t0l_t1l.16b, $t0h_t1h.16b
	eor	$t2l_t3l.16b, $t2l_t3l.16b, $t2h_t3h.16b
	zip1	$t0.2d, $t0l_t1l.2d, $t0h_t1h.2d
	zip1	$t2.2d, $t2l_t3l.2d, $t2h_t3h.2d
	zip2	$t1.2d, $t0l_t1l.2d, $t0h_t1h.2d
	zip2	$t3.2d, $t2l_t3l.2d, $t2h_t3h.2d

	ext	$t0.16b, $t0.16b, $t0.16b, #15	// t0 = t0 << 8
	ext	$t1.16b, $t1.16b, $t1.16b, #14	// t1 = t1 << 16
	pmull	$r.8h, $a.8b, $b.8b		// D = A*B
	ext	$t3.16b, $t3.16b, $t3.16b, #12	// t3 = t3 << 32
	ext	$t2.16b, $t2.16b, $t2.16b, #13	// t2 = t2 << 24
	eor	$t0.16b, $t0.16b, $t1.16b
	eor	$t2.16b, $t2.16b, $t3.16b
	eor	$r.16b, $r.16b, $t0.16b
	eor	$r.16b, $r.16b, $t2.16b
___
}

$code .= <<___;
#include <openssl/arm_arch.h>

.text

.global	gcm_init_neon
.type	gcm_init_neon,%function
.align	4
gcm_init_neon:
	AARCH64_VALID_CALL_TARGET
	// This function is adapted from gcm_init_v8. xC2 is t3.
	ld1	{$t1.2d}, [x1]			// load H
	movi	$t3.16b, #0xe1
	shl	$t3.2d, $t3.2d, #57		// 0xc2.0
	ext	$INlo.16b, $t1.16b, $t1.16b, #8
	ushr	$t2.2d, $t3.2d, #63
	dup	$t1.4s, $t1.s[1]
	ext	$t0.16b, $t2.16b, $t3.16b, #8	// t0=0xc2....01
	ushr	$t2.2d, $INlo.2d, #63
	sshr	$t1.4s, $t1.4s, #31		// broadcast carry bit
	and	$t2.16b, $t2.16b, $t0.16b
	shl	$INlo.2d, $INlo.2d, #1
	ext	$t2.16b, $t2.16b, $t2.16b, #8
	and	$t0.16b, $t0.16b, $t1.16b
	orr	$INlo.16b, $INlo.16b, $t2.16b	// H<<<=1
	eor	$Hlo.16b, $INlo.16b, $t0.16b	// twisted H
	st1	{$Hlo.2d}, [x0]			// store Htable[0]
	ret
.size	gcm_init_neon,.-gcm_init_neon

.global	gcm_gmult_neon
.type	gcm_gmult_neon,%function
.align	4
gcm_gmult_neon:
	AARCH64_VALID_CALL_TARGET
	ld1	{$INlo.16b}, [$Xi]		// load Xi
	ld1	{$Hlo.1d}, [$Htbl], #8		// load twisted H
	ld1	{$Hhi.1d}, [$Htbl]
	adrp	x9, :pg_hi21:.Lmasks		// load constants
	add	x9, x9, :lo12:.Lmasks
	ld1	{$k48_k32.2d, $k16_k0.2d}, [x9]
	rev64	$INlo.16b, $INlo.16b		// byteswap Xi
	ext	$INlo.16b, $INlo.16b, $INlo.16b, #8
	eor	$Hhl.8b, $Hlo.8b, $Hhi.8b	// Karatsuba pre-processing

	mov	$len, #16
	b	.Lgmult_neon
.size	gcm_gmult_neon,.-gcm_gmult_neon

.global	gcm_ghash_neon
.type	gcm_ghash_neon,%function
.align	4
gcm_ghash_neon:
	AARCH64_VALID_CALL_TARGET
	ld1	{$Xl.16b}, [$Xi]		// load Xi
	ld1	{$Hlo.1d}, [$Htbl], #8		// load twisted H
	ld1	{$Hhi.1d}, [$Htbl]
	adrp	x9, :pg_hi21:.Lmasks		// load constants
	add	x9, x9, :lo12:.Lmasks
	ld1	{$k48_k32.2d, $k16_k0.2d}, [x9]
	rev64	$Xl.16b, $Xl.16b		// byteswap Xi
	ext	$Xl.16b, $Xl.16b, $Xl.16b, #8
	eor	$Hhl.8b, $Hlo.8b, $Hhi.8b	// Karatsuba pre-processing

.Loop_neon:
	ld1	{$INlo.16b}, [$inp], #16	// load inp
	rev64	$INlo.16b, $INlo.16b		// byteswap inp
	ext	$INlo.16b, $INlo.16b, $INlo.16b, #8
	eor	$INlo.16b, $INlo.16b, $Xl.16b	// inp ^= Xi

.Lgmult_neon:
	// Split the input into $INlo and $INhi. (The upper halves are unused,
	// so it is okay to leave them alone.)
	ins	$INhi.d[0], $INlo.d[1]
___
&clmul64x64	($Xl, $Hlo, $INlo);		# H.loÂ·Xi.lo
$code .= <<___;
	eor	$INlo.8b, $INlo.8b, $INhi.8b	// Karatsuba pre-processing
___
&clmul64x64	($Xm, $Hhl, $INlo);		# (H.lo+H.hi)Â·(Xi.lo+Xi.hi)
&clmul64x64	($Xh, $Hhi, $INhi);		# H.hiÂ·Xi.hi
$code .= <<___;
	ext	$t0.16b, $Xl.16b, $Xh.16b, #8
	eor	$Xm.16b, $Xm.16b, $Xl.16b	// Karatsuba post-processing
	eor	$Xm.16b, $Xm.16b, $Xh.16b
	eor	$Xm.16b, $Xm.16b, $t0.16b	// Xm overlaps Xh.lo and Xl.hi
	ins	$Xl.d[1], $Xm.d[0]		// Xh|Xl - 256-bit result
	// This is a no-op due to the ins instruction below.
	// ins	$Xh.d[0], $Xm.d[1]

	// equivalent of reduction_avx from ghash-x86_64.pl
	shl	$t1.2d, $Xl.2d, #57		// 1st phase
	shl	$t2.2d, $Xl.2d, #62
	eor	$t2.16b, $t2.16b, $t1.16b	//
	shl	$t1.2d, $Xl.2d, #63
	eor	$t2.16b, $t2.16b, $t1.16b	//
	// Note Xm contains {Xl.d[1], Xh.d[0]}.
	eor	$t2.16b, $t2.16b, $Xm.16b
	ins	$Xl.d[1], $t2.d[0]		// Xl.d[1] ^= t2.d[0]
	ins	$Xh.d[0], $t2.d[1]		// Xh.d[0] ^= t2.d[1]

	ushr	$t2.2d, $Xl.2d, #1		// 2nd phase
	eor	$Xh.16b, $Xh.16b,$Xl.16b
	eor	$Xl.16b, $Xl.16b,$t2.16b	//
	ushr	$t2.2d, $t2.2d, #6
	ushr	$Xl.2d, $Xl.2d, #1		//
	eor	$Xl.16b, $Xl.16b, $Xh.16b	//
	eor	$Xl.16b, $Xl.16b, $t2.16b	//

	subs	$len, $len, #16
	bne	.Loop_neon

	rev64	$Xl.16b, $Xl.16b		// byteswap Xi and write
	ext	$Xl.16b, $Xl.16b, $Xl.16b, #8
	st1	{$Xl.16b}, [$Xi]

	ret
.size	gcm_ghash_neon,.-gcm_ghash_neon

.section	.rodata
.align	4
.Lmasks:
.quad	0x0000ffffffffffff	// k48
.quad	0x00000000ffffffff	// k32
.quad	0x000000000000ffff	// k16
.quad	0x0000000000000000	// k0
.asciz  "GHASH for ARMv8, derived from ARMv4 version by <appro\@openssl.org>"
.align  2
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/geo;

	print $_,"\n";
}
close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#!/usr/bin/env perl
# Copyright (c) 2019, Google Inc.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# ghash-ssse3-x86.pl is a constant-time variant of the traditional 4-bit
# table-based GHASH implementation. It requires SSSE3 instructions.
#
# For background, the table-based strategy is a 4-bit windowed multiplication.
# It precomputes all 4-bit multiples of H (this is 16 128-bit rows), then loops
# over 4-bit windows of the input and indexes them up into the table. Visually,
# it multiplies as in the schoolbook multiplication diagram below, but with
# more terms. (Each term is 4 bits, so there are 32 terms in each row.) First
# it incorporates the terms labeled '1' by indexing the most significant term
# of X into the table. Then it shifts and repeats for '2' and so on.
#
#        hhhhhh
#  *     xxxxxx
#  ============
#        666666
#       555555
#      444444
#     333333
#    222222
#   111111
#
# This implementation changes the order. We treat the table as a 16Ã—16 matrix
# and transpose it. The first row is then the first byte of each multiple of H,
# and so on. We then reorder terms as below. Observe that the terms labeled '1'
# and '2' are all lookups into the first row, etc. This maps well to the SSSE3
# pshufb instruction, using alternating terms of X in parallel as indices. This
# alternation is needed because pshufb maps 4 bits to 8 bits. Then we shift and
# repeat for each row.
#
#        hhhhhh
#  *     xxxxxx
#  ============
#        224466
#       113355
#      224466
#     113355
#    224466
#   113355
#
# Next we account for GCM's confusing bit order. The "first" bit is the least
# significant coefficient, but GCM treats the most sigificant bit within a byte
# as first. Bytes are little-endian, and bits are big-endian. We reverse the
# bytes in XMM registers for a consistent bit and byte ordering, but this means
# the least significant bit is the most significant coefficient and vice versa.
#
# For consistency, "low", "high", "left-shift", and "right-shift" refer to the
# bit ordering within the XMM register, rather than the reversed coefficient
# ordering. Low bits are less significant bits and more significant
# coefficients. Right-shifts move from MSB to the LSB and correspond to
# increasing the power of each coefficient.
#
# Note this bit reversal enters into the table's column indices. H*1 is stored
# in column 0b1000 and H*x^3 is stored in column 0b0001. It also means earlier
# table rows contain more significant coefficients, so we iterate forwards.

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../../perlasm");
require "x86asm.pl";

$output = $ARGV[1];
open STDOUT, ">$output";

&asm_init($ARGV[0]);

my ($Xi, $Htable, $in, $len) = ("edi", "esi", "edx", "ecx");
&static_label("reverse_bytes");
&static_label("low4_mask");

my $call_counter = 0;
# process_rows emits assembly code to process $rows rows of the table. On
# input, $Htable stores the pointer to the next row. xmm0 and xmm1 store the
# low and high halves of the input. The result so far is passed in xmm2. xmm3
# must be zero. On output, $Htable is advanced to the next row and xmm2 is
# updated. xmm3 remains zero. It clobbers eax, xmm4, xmm5, and xmm6.
sub process_rows {
	my ($rows) = @_;
	$call_counter++;

	# Shifting whole XMM registers by bits is complex. psrldq shifts by
	# bytes, and psrlq shifts the two 64-bit halves separately. Each row
	# produces 8 bits of carry, and the reduction needs an additional 7-bit
	# shift. This must fit in 64 bits so reduction can use psrlq. This
	# allows up to 7 rows at a time.
	die "Carry register would overflow 64 bits." if ($rows*8 + 7 > 64);

	&mov("eax", $rows);
&set_label("loop_row_$call_counter");
	&movdqa("xmm4", &QWP(0, $Htable));
	&lea($Htable, &DWP(16, $Htable));

	# Right-shift xmm2 and xmm3 by 8 bytes.
	&movdqa("xmm6", "xmm2");
	&palignr("xmm6", "xmm3", 1);
	&movdqa("xmm3", "xmm6");
	&psrldq("xmm2", 1);

	# Load the next table row and index the low and high bits of the input.
	# Note the low (respectively, high) half corresponds to more
	# (respectively, less) significant coefficients.
	&movdqa("xmm5", "xmm4");
	&pshufb("xmm4", "xmm0");
	&pshufb("xmm5", "xmm1");

	# Add the high half (xmm5) without shifting.
	&pxor("xmm2", "xmm5");

	# Add the low half (xmm4). This must be right-shifted by 4 bits. First,
	# add into the carry register (xmm3).
	&movdqa("xmm5", "xmm4");
	&psllq("xmm5", 60);
	&movdqa("xmm6", "xmm5");
	&pslldq("xmm6", 8);
	&pxor("xmm3", "xmm6");

	# Next, add into xmm2.
	&psrldq("xmm5", 8);
	&pxor("xmm2", "xmm5");
	&psrlq("xmm4", 4);
	&pxor("xmm2", "xmm4");

	&sub("eax", 1);
	&jnz(&label("loop_row_$call_counter"));

	# Reduce the carry register. The reduction polynomial is 1 + x + x^2 +
	# x^7, so we shift and XOR four times.
	&pxor("xmm2", "xmm3");	# x^0 = 0
	&psrlq("xmm3", 1);
	&pxor("xmm2", "xmm3");	# x^1 = x
	&psrlq("xmm3", 1);
	&pxor("xmm2", "xmm3");	# x^(1+1) = x^2
	&psrlq("xmm3", 5);
	&pxor("xmm2", "xmm3");	# x^(1+1+5) = x^7
	&pxor("xmm3", "xmm3");
____
}

# gcm_gmult_ssse3 multiplies |Xi| by |Htable| and writes the result to |Xi|.
# |Xi| is represented in GHASH's serialized byte representation. |Htable| is
# formatted as described above.
# void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]);
&function_begin("gcm_gmult_ssse3");
	&mov($Xi, &wparam(0));
	&mov($Htable, &wparam(1));

	&movdqu("xmm0", &QWP(0, $Xi));
	&call(&label("pic_point"));
&set_label("pic_point");
	&blindpop("eax");
	&movdqa("xmm7", &QWP(&label("reverse_bytes")."-".&label("pic_point"), "eax"));
	&movdqa("xmm2", &QWP(&label("low4_mask")."-".&label("pic_point"), "eax"));

	# Reverse input bytes to deserialize.
	&pshufb("xmm0", "xmm7");

	# Split each byte into low (xmm0) and high (xmm1) halves.
	&movdqa("xmm1", "xmm2");
	&pandn("xmm1", "xmm0");
	&psrld("xmm1", 4);
	&pand("xmm0", "xmm2");

	# Maintain the result in xmm2 (the value) and xmm3 (carry bits). Note
	# that, due to bit reversal, xmm3 contains bits that fall off when
	# right-shifting, not left-shifting.
	&pxor("xmm2", "xmm2");
	&pxor("xmm3", "xmm3");

	# We must reduce at least once every 7 rows, so divide into three
	# chunks.
	&process_rows(5);
	&process_rows(5);
	&process_rows(6);

	# Store the result. Reverse bytes to serialize.
	&pshufb("xmm2", "xmm7");
	&movdqu(&QWP(0, $Xi), "xmm2");

	# Zero any registers which contain secrets.
	&pxor("xmm0", "xmm0");
	&pxor("xmm1", "xmm1");
	&pxor("xmm2", "xmm2");
	&pxor("xmm3", "xmm3");
	&pxor("xmm4", "xmm4");
	&pxor("xmm5", "xmm5");
	&pxor("xmm6", "xmm6");
&function_end("gcm_gmult_ssse3");

# gcm_ghash_ssse3 incorporates |len| bytes from |in| to |Xi|, using |Htable| as
# the key. It writes the result back to |Xi|. |Xi| is represented in GHASH's
# serialized byte representation. |Htable| is formatted as described above.
# void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
#                      size_t len);
&function_begin("gcm_ghash_ssse3");
	&mov($Xi, &wparam(0));
	&mov($Htable, &wparam(1));
	&mov($in, &wparam(2));
	&mov($len, &wparam(3));

	&movdqu("xmm0", &QWP(0, $Xi));
	&call(&label("pic_point"));
&set_label("pic_point");
	&blindpop("ebx");
	&movdqa("xmm7", &QWP(&label("reverse_bytes")."-".&label("pic_point"), "ebx"));

	# This function only processes whole blocks.
	&and($len, -16);

	# Reverse input bytes to deserialize. We maintain the running
	# total in xmm0.
	&pshufb("xmm0", "xmm7");

	# Iterate over each block. On entry to each iteration, xmm3 is zero.
	&pxor("xmm3", "xmm3");
&set_label("loop_ghash");
	&movdqa("xmm2", &QWP(&label("low4_mask")."-".&label("pic_point"), "ebx"));

	# Incorporate the next block of input.
	&movdqu("xmm1", &QWP(0, $in));
	&pshufb("xmm1", "xmm7");	# Reverse bytes.
	&pxor("xmm0", "xmm1");

	# Split each byte into low (xmm0) and high (xmm1) halves.
	&movdqa("xmm1", "xmm2");
	&pandn("xmm1", "xmm0");
	&psrld("xmm1", 4);
	&pand("xmm0", "xmm2");

	# Maintain the result in xmm2 (the value) and xmm3 (carry bits). Note
	# that, due to bit reversal, xmm3 contains bits that fall off when
	# right-shifting, not left-shifting.
	&pxor("xmm2", "xmm2");
	# xmm3 is already zero at this point.

	# We must reduce at least once every 7 rows, so divide into three
	# chunks.
	&process_rows(5);
	&process_rows(5);
	&process_rows(6);

	&movdqa("xmm0", "xmm2");

	# Rewind $Htable for the next iteration.
	&lea($Htable, &DWP(-256, $Htable));

	# Advance input and continue.
	&lea($in, &DWP(16, $in));
	&sub($len, 16);
	&jnz(&label("loop_ghash"));

	# Reverse bytes and store the result.
	&pshufb("xmm0", "xmm7");
	&movdqu(&QWP(0, $Xi), "xmm0");

	# Zero any registers which contain secrets.
	&pxor("xmm0", "xmm0");
	&pxor("xmm1", "xmm1");
	&pxor("xmm2", "xmm2");
	&pxor("xmm3", "xmm3");
	&pxor("xmm4", "xmm4");
	&pxor("xmm5", "xmm5");
	&pxor("xmm6", "xmm6");
&function_end("gcm_ghash_ssse3");

# reverse_bytes is a permutation which, if applied with pshufb, reverses the
# bytes in an XMM register.
&set_label("reverse_bytes", 16);
&data_byte(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
# low4_mask is an XMM mask which selects the low four bits of each byte.
&set_label("low4_mask", 16);
&data_word(0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f);

&asm_finish();

close STDOUT or die "error closing STDOUT: $!";


#!/usr/bin/env perl
# Copyright (c) 2019, Google Inc.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# ghash-ssse3-x86_64.pl is a constant-time variant of the traditional 4-bit
# table-based GHASH implementation. It requires SSSE3 instructions.
#
# For background, the table-based strategy is a 4-bit windowed multiplication.
# It precomputes all 4-bit multiples of H (this is 16 128-bit rows), then loops
# over 4-bit windows of the input and indexes them up into the table. Visually,
# it multiplies as in the schoolbook multiplication diagram below, but with
# more terms. (Each term is 4 bits, so there are 32 terms in each row.) First
# it incorporates the terms labeled '1' by indexing the most significant term
# of X into the table. Then it shifts and repeats for '2' and so on.
#
#        hhhhhh
#  *     xxxxxx
#  ============
#        666666
#       555555
#      444444
#     333333
#    222222
#   111111
#
# This implementation changes the order. We treat the table as a 16Ã—16 matrix
# and transpose it. The first row is then the first byte of each multiple of H,
# and so on. We then reorder terms as below. Observe that the terms labeled '1'
# and '2' are all lookups into the first row, etc. This maps well to the SSSE3
# pshufb instruction, using alternating terms of X in parallel as indices. This
# alternation is needed because pshufb maps 4 bits to 8 bits. Then we shift and
# repeat for each row.
#
#        hhhhhh
#  *     xxxxxx
#  ============
#        224466
#       113355
#      224466
#     113355
#    224466
#   113355
#
# Next we account for GCM's confusing bit order. The "first" bit is the least
# significant coefficient, but GCM treats the most sigificant bit within a byte
# as first. Bytes are little-endian, and bits are big-endian. We reverse the
# bytes in XMM registers for a consistent bit and byte ordering, but this means
# the least significant bit is the most significant coefficient and vice versa.
#
# For consistency, "low", "high", "left-shift", and "right-shift" refer to the
# bit ordering within the XMM register, rather than the reversed coefficient
# ordering. Low bits are less significant bits and more significant
# coefficients. Right-shifts move from MSB to the LSB and correspond to
# increasing the power of each coefficient.
#
# Note this bit reversal enters into the table's column indices. H*1 is stored
# in column 0b1000 and H*x^3 is stored in column 0b0001. It also means earlier
# table rows contain more significant coefficients, so we iterate forwards.

use strict;

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

my $flavour = shift;
my $output  = shift;

my $win64 = 0;
$win64 = 1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/;
my $dir = $1;
my $xlate;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT = *OUT;

my ($Xi, $Htable, $in, $len) = $win64 ? ("%rcx", "%rdx", "%r8", "%r9") :
                                        ("%rdi", "%rsi", "%rdx", "%rcx");

my $code = <<____;
.text

# gcm_gmult_ssse3 multiplies |Xi| by |Htable| and writes the result to |Xi|.
# |Xi| is represented in GHASH's serialized byte representation. |Htable| is
# formatted as described above.
# void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]);
.type	gcm_gmult_ssse3, \@abi-omnipotent
.globl	gcm_gmult_ssse3
.align	16
gcm_gmult_ssse3:
.cfi_startproc
.seh_startproc
____
$code .= <<____ if ($win64);
	subq	\$40, %rsp
.seh_allocstack	40
	movdqa	%xmm6, (%rsp)
.seh_savexmm128	%xmm6, 0
	movdqa	%xmm10, 16(%rsp)
.seh_savexmm128	%xmm10, 16
____
$code .= <<____;
	movdqu	($Xi), %xmm0
	movdqa	.Lreverse_bytes(%rip), %xmm10
	movdqa	.Llow4_mask(%rip), %xmm2

	# Reverse input bytes to deserialize.
	pshufb	%xmm10, %xmm0

	# Split each byte into low (%xmm0) and high (%xmm1) halves.
	movdqa	%xmm2, %xmm1
	pandn	%xmm0, %xmm1
	psrld	\$4, %xmm1
	pand	%xmm2, %xmm0

	# Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note
	# that, due to bit reversal, %xmm3 contains bits that fall off when
	# right-shifting, not left-shifting.
	pxor	%xmm2, %xmm2
	pxor	%xmm3, %xmm3
____

my $call_counter = 0;
# process_rows returns assembly code to process $rows rows of the table. On
# input, $Htable stores the pointer to the next row. %xmm0 and %xmm1 store the
# low and high halves of the input. The result so far is passed in %xmm2. %xmm3
# must be zero. On output, $Htable is advanced to the next row and %xmm2 is
# updated. %xmm3 remains zero. It clobbers %rax, %xmm4, %xmm5, and %xmm6.
sub process_rows {
    my ($rows) = @_;
    $call_counter++;

    # Shifting whole XMM registers by bits is complex. psrldq shifts by bytes,
    # and psrlq shifts the two 64-bit halves separately. Each row produces 8
    # bits of carry, and the reduction needs an additional 7-bit shift. This
    # must fit in 64 bits so reduction can use psrlq. This allows up to 7 rows
    # at a time.
    die "Carry register would overflow 64 bits." if ($rows*8 + 7 > 64);

    return <<____;
	movq	\$$rows, %rax
.Loop_row_$call_counter:
	movdqa	($Htable), %xmm4
	leaq	16($Htable), $Htable

	# Right-shift %xmm2 and %xmm3 by 8 bytes.
	movdqa	%xmm2, %xmm6
	palignr	\$1, %xmm3, %xmm6
	movdqa	%xmm6, %xmm3
	psrldq	\$1, %xmm2

	# Load the next table row and index the low and high bits of the input.
	# Note the low (respectively, high) half corresponds to more
	# (respectively, less) significant coefficients.
	movdqa	%xmm4, %xmm5
	pshufb	%xmm0, %xmm4
	pshufb	%xmm1, %xmm5

	# Add the high half (%xmm5) without shifting.
	pxor	%xmm5, %xmm2

	# Add the low half (%xmm4). This must be right-shifted by 4 bits. First,
	# add into the carry register (%xmm3).
	movdqa	%xmm4, %xmm5
	psllq	\$60, %xmm5
	movdqa	%xmm5, %xmm6
	pslldq	\$8, %xmm6
	pxor	%xmm6, %xmm3

	# Next, add into %xmm2.
	psrldq	\$8, %xmm5
	pxor	%xmm5, %xmm2
	psrlq	\$4, %xmm4
	pxor	%xmm4, %xmm2

	subq	\$1, %rax
	jnz	.Loop_row_$call_counter

	# Reduce the carry register. The reduction polynomial is 1 + x + x^2 +
	# x^7, so we shift and XOR four times.
	pxor	%xmm3, %xmm2	# x^0 = 0
	psrlq	\$1, %xmm3
	pxor	%xmm3, %xmm2	# x^1 = x
	psrlq	\$1, %xmm3
	pxor	%xmm3, %xmm2	# x^(1+1) = x^2
	psrlq	\$5, %xmm3
	pxor	%xmm3, %xmm2	# x^(1+1+5) = x^7
	pxor	%xmm3, %xmm3
____
}

# We must reduce at least once every 7 rows, so divide into three chunks.
$code .= process_rows(5);
$code .= process_rows(5);
$code .= process_rows(6);

$code .= <<____;
	# Store the result. Reverse bytes to serialize.
	pshufb	%xmm10, %xmm2
	movdqu	%xmm2, ($Xi)

	# Zero any registers which contain secrets.
	pxor	%xmm0, %xmm0
	pxor	%xmm1, %xmm1
	pxor	%xmm2, %xmm2
	pxor	%xmm3, %xmm3
	pxor	%xmm4, %xmm4
	pxor	%xmm5, %xmm5
	pxor	%xmm6, %xmm6
____
$code .= <<____ if ($win64);
	movdqa	(%rsp), %xmm6
	movdqa	16(%rsp), %xmm10
	addq	\$40, %rsp
____
$code .= <<____;
	ret
.cfi_endproc
.seh_endproc
.size	gcm_gmult_ssse3,.-gcm_gmult_ssse3
____

$code .= <<____;
# gcm_ghash_ssse3 incorporates |len| bytes from |in| to |Xi|, using |Htable| as
# the key. It writes the result back to |Xi|. |Xi| is represented in GHASH's
# serialized byte representation. |Htable| is formatted as described above.
# void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
#                      size_t len);
.type	gcm_ghash_ssse3, \@abi-omnipotent
.globl	gcm_ghash_ssse3
.align	16
gcm_ghash_ssse3:
.cfi_startproc
.seh_startproc
____
$code .= <<____ if ($win64);
	subq	\$56, %rsp
.seh_allocstack	56
	movdqa	%xmm6, (%rsp)
.seh_savexmm128	%xmm6, 0
	movdqa	%xmm10, 16(%rsp)
.seh_savexmm128	%xmm10, 16
	movdqa	%xmm11, 32(%rsp)
.seh_savexmm128	%xmm11, 32
____
$code .= <<____;
	movdqu	($Xi), %xmm0
	movdqa	.Lreverse_bytes(%rip), %xmm10
	movdqa	.Llow4_mask(%rip), %xmm11

	# This function only processes whole blocks.
	andq	\$-16, $len

	# Reverse input bytes to deserialize. We maintain the running
	# total in %xmm0.
	pshufb	%xmm10, %xmm0

	# Iterate over each block. On entry to each iteration, %xmm3 is zero.
	pxor	%xmm3, %xmm3
.Loop_ghash:
	# Incorporate the next block of input.
	movdqu	($in), %xmm1
	pshufb	%xmm10, %xmm1	# Reverse bytes.
	pxor	%xmm1, %xmm0

	# Split each byte into low (%xmm0) and high (%xmm1) halves.
	movdqa	%xmm11, %xmm1
	pandn	%xmm0, %xmm1
	psrld	\$4, %xmm1
	pand	%xmm11, %xmm0

	# Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note
	# that, due to bit reversal, %xmm3 contains bits that fall off when
	# right-shifting, not left-shifting.
	pxor	%xmm2, %xmm2
	# %xmm3 is already zero at this point.
____

# We must reduce at least once every 7 rows, so divide into three chunks.
$code .= process_rows(5);
$code .= process_rows(5);
$code .= process_rows(6);

$code .= <<____;
	movdqa	%xmm2, %xmm0

	# Rewind $Htable for the next iteration.
	leaq	-256($Htable), $Htable

	# Advance input and continue.
	leaq	16($in), $in
	subq	\$16, $len
	jnz	.Loop_ghash

	# Reverse bytes and store the result.
	pshufb	%xmm10, %xmm0
	movdqu	%xmm0, ($Xi)

	# Zero any registers which contain secrets.
	pxor	%xmm0, %xmm0
	pxor	%xmm1, %xmm1
	pxor	%xmm2, %xmm2
	pxor	%xmm3, %xmm3
	pxor	%xmm4, %xmm4
	pxor	%xmm5, %xmm5
	pxor	%xmm6, %xmm6
____
$code .= <<____ if ($win64);
	movdqa	(%rsp), %xmm6
	movdqa	16(%rsp), %xmm10
	movdqa	32(%rsp), %xmm11
	addq	\$56, %rsp
____
$code .= <<____;
	ret
.cfi_endproc
.seh_endproc
.size	gcm_ghash_ssse3,.-gcm_ghash_ssse3

.section .rodata
.align	16
# .Lreverse_bytes is a permutation which, if applied with pshufb, reverses the
# bytes in an XMM register.
.Lreverse_bytes:
.byte	15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
# .Llow4_mask is an XMM mask which selects the low four bits of each byte.
.Llow4_mask:
.quad	0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f
.text
____

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# March, May, June 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+64/128 bytes fixed table]. It has two
# code paths: vanilla x86 and vanilla SSE. Former will be executed on
# 486 and Pentium, latter on all others. SSE GHASH features so called
# "528B" variant of "4-bit" method utilizing additional 256+16 bytes
# of per-key storage [+512 bytes shared table]. Performance results
# are for streamed GHASH subroutine and are expressed in cycles per
# processed byte, less is better:
#
#		gcc 2.95.3(*)	SSE assembler	x86 assembler
#
# Pentium	105/111(**)	-		50
# PIII		68 /75		12.2		24
# P4		125/125		17.8		84(***)
# Opteron	66 /70		10.1		30
# Core2		54 /67		8.4		18
# Atom		105/105		16.8		53
# VIA Nano	69 /71		13.0		27
#
# (*)	gcc 3.4.x was observed to generate few percent slower code,
#	which is one of reasons why 2.95.3 results were chosen,
#	another reason is lack of 3.4.x results for older CPUs;
#	comparison with SSE results is not completely fair, because C
#	results are for vanilla "256B" implementation, while
#	assembler results are for "528B";-)
# (**)	second number is result for code compiled with -fPIC flag,
#	which is actually more relevant, because assembler code is
#	position-independent;
# (***)	see comment in non-MMX routine for further details;
#
# To summarize, it's >2-5 times faster than gcc-generated code. To
# anchor it to something else SHA1 assembler processes one byte in
# ~7 cycles on contemporary x86 cores. As for choice of MMX/SSE
# in particular, see comment at the end of the file...

# May 2010
#
# Add PCLMULQDQ version performing at 2.10 cycles per processed byte.
# The question is how close is it to theoretical limit? The pclmulqdq
# instruction latency appears to be 14 cycles and there can't be more
# than 2 of them executing at any given time. This means that single
# Karatsuba multiplication would take 28 cycles *plus* few cycles for
# pre- and post-processing. Then multiplication has to be followed by
# modulo-reduction. Given that aggregated reduction method [see
# "Carry-less Multiplication and Its Usage for Computing the GCM Mode"
# white paper by Intel] allows you to perform reduction only once in
# a while we can assume that asymptotic performance can be estimated
# as (28+Tmod/Naggr)/16, where Tmod is time to perform reduction
# and Naggr is the aggregation factor.
#
# Before we proceed to this implementation let's have closer look at
# the best-performing code suggested by Intel in their white paper.
# By tracing inter-register dependencies Tmod is estimated as ~19
# cycles and Naggr chosen by Intel is 4, resulting in 2.05 cycles per
# processed byte. As implied, this is quite optimistic estimate,
# because it does not account for Karatsuba pre- and post-processing,
# which for a single multiplication is ~5 cycles. Unfortunately Intel
# does not provide performance data for GHASH alone. But benchmarking
# AES_GCM_encrypt ripped out of Fig. 15 of the white paper with aadt
# alone resulted in 2.46 cycles per byte of out 16KB buffer. Note that
# the result accounts even for pre-computing of degrees of the hash
# key H, but its portion is negligible at 16KB buffer size.
#
# Moving on to the implementation in question. Tmod is estimated as
# ~13 cycles and Naggr is 2, giving asymptotic performance of ...
# 2.16. How is it possible that measured performance is better than
# optimistic theoretical estimate? There is one thing Intel failed
# to recognize. By serializing GHASH with CTR in same subroutine
# former's performance is really limited to above (Tmul + Tmod/Naggr)
# equation. But if GHASH procedure is detached, the modulo-reduction
# can be interleaved with Naggr-1 multiplications at instruction level
# and under ideal conditions even disappear from the equation. So that
# optimistic theoretical estimate for this implementation is ...
# 28/16=1.75, and not 2.16. Well, it's probably way too optimistic,
# at least for such small Naggr. I'd argue that (28+Tproc/Naggr),
# where Tproc is time required for Karatsuba pre- and post-processing,
# is more realistic estimate. In this case it gives ... 1.91 cycles.
# Or in other words, depending on how well we can interleave reduction
# and one of the two multiplications the performance should be between
# 1.91 and 2.16. As already mentioned, this implementation processes
# one byte out of 8KB buffer in 2.10 cycles, while x86_64 counterpart
# - in 2.02. x86_64 performance is better, because larger register
# bank allows to interleave reduction and multiplication better.
#
# Does it make sense to increase Naggr? To start with it's virtually
# impossible in 32-bit mode, because of limited register bank
# capacity. Otherwise improvement has to be weighed against slower
# setup, as well as code size and complexity increase. As even
# optimistic estimate doesn't promise 30% performance improvement,
# there are currently no plans to increase Naggr.
#
# Special thanks to David Woodhouse for providing access to a
# Westmere-based system on behalf of Intel Open Source Technology Centre.

# January 2010
#
# Tweaked to optimize transitions between integer and FP operations
# on same XMM register, PCLMULQDQ subroutine was measured to process
# one byte in 2.07 cycles on Sandy Bridge, and in 2.12 - on Westmere.
# The minor regression on Westmere is outweighed by ~15% improvement
# on Sandy Bridge. Strangely enough attempt to modify 64-bit code in
# similar manner resulted in almost 20% degradation on Sandy Bridge,
# where original 64-bit code processes one byte in 1.95 cycles.

#####################################################################
# For reference, AMD Bulldozer processes one byte in 1.98 cycles in
# 32-bit mode and 1.89 in 64-bit.

# February 2013
#
# Overhaul: aggregate Karatsuba post-processing, improve ILP in
# reduction_alg9. Resulting performance is 1.96 cycles per byte on
# Westmere, 1.95 - on Sandy/Ivy Bridge, 1.76 - on Bulldozer.

# This file was patched in BoringSSL to remove the variable-time 4-bit
# implementation.

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../../perlasm");
require "x86asm.pl";

$output=$ARGV[1];
open STDOUT,">$output";

&asm_init($ARGV[0],$x86only = $ARGV[$#ARGV] eq "386");

$sse2=0;
for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }

if (!$x86only) {{{
if ($sse2) {{
######################################################################
# PCLMULQDQ version.

$Xip="eax";
$Htbl="edx";
$const="ecx";
$inp="esi";
$len="ebx";

($Xi,$Xhi)=("xmm0","xmm1");	$Hkey="xmm2";
($T1,$T2,$T3)=("xmm3","xmm4","xmm5");
($Xn,$Xhn)=("xmm6","xmm7");

&static_label("bswap");

sub clmul64x64_T2 {	# minimal "register" pressure
my ($Xhi,$Xi,$Hkey,$HK)=@_;

	&movdqa		($Xhi,$Xi);		#
	&pshufd		($T1,$Xi,0b01001110);
	&pshufd		($T2,$Hkey,0b01001110)	if (!defined($HK));
	&pxor		($T1,$Xi);		#
	&pxor		($T2,$Hkey)		if (!defined($HK));
			$HK=$T2			if (!defined($HK));

	&pclmulqdq	($Xi,$Hkey,0x00);	#######
	&pclmulqdq	($Xhi,$Hkey,0x11);	#######
	&pclmulqdq	($T1,$HK,0x00);		#######
	&xorps		($T1,$Xi);		#
	&xorps		($T1,$Xhi);		#

	&movdqa		($T2,$T1);		#
	&psrldq		($T1,8);
	&pslldq		($T2,8);		#
	&pxor		($Xhi,$T1);
	&pxor		($Xi,$T2);		#
}

sub clmul64x64_T3 {
# Even though this subroutine offers visually better ILP, it
# was empirically found to be a tad slower than above version.
# At least in gcm_ghash_clmul context. But it's just as well,
# because loop modulo-scheduling is possible only thanks to
# minimized "register" pressure...
my ($Xhi,$Xi,$Hkey)=@_;

	&movdqa		($T1,$Xi);		#
	&movdqa		($Xhi,$Xi);
	&pclmulqdq	($Xi,$Hkey,0x00);	#######
	&pclmulqdq	($Xhi,$Hkey,0x11);	#######
	&pshufd		($T2,$T1,0b01001110);	#
	&pshufd		($T3,$Hkey,0b01001110);
	&pxor		($T2,$T1);		#
	&pxor		($T3,$Hkey);
	&pclmulqdq	($T2,$T3,0x00);		#######
	&pxor		($T2,$Xi);		#
	&pxor		($T2,$Xhi);		#

	&movdqa		($T3,$T2);		#
	&psrldq		($T2,8);
	&pslldq		($T3,8);		#
	&pxor		($Xhi,$T2);
	&pxor		($Xi,$T3);		#
}

if (1) {		# Algorithm 9 with <<1 twist.
			# Reduction is shorter and uses only two
			# temporary registers, which makes it better
			# candidate for interleaving with 64x64
			# multiplication. Pre-modulo-scheduled loop
			# was found to be ~20% faster than Algorithm 5
			# below. Algorithm 9 was therefore chosen for
			# further optimization...

sub reduction_alg9 {	# 17/11 times faster than Intel version
my ($Xhi,$Xi) = @_;

	# 1st phase
	&movdqa		($T2,$Xi);		#
	&movdqa		($T1,$Xi);
	&psllq		($Xi,5);
	&pxor		($T1,$Xi);		#
	&psllq		($Xi,1);
	&pxor		($Xi,$T1);		#
	&psllq		($Xi,57);		#
	&movdqa		($T1,$Xi);		#
	&pslldq		($Xi,8);
	&psrldq		($T1,8);		#
	&pxor		($Xi,$T2);
	&pxor		($Xhi,$T1);		#

	# 2nd phase
	&movdqa		($T2,$Xi);
	&psrlq		($Xi,1);
	&pxor		($Xhi,$T2);		#
	&pxor		($T2,$Xi);
	&psrlq		($Xi,5);
	&pxor		($Xi,$T2);		#
	&psrlq		($Xi,1);		#
	&pxor		($Xi,$Xhi)		#
}

&function_begin_B("gcm_init_clmul");
	&mov		($Htbl,&wparam(0));
	&mov		($Xip,&wparam(1));

	&call		(&label("pic"));
&set_label("pic");
	&blindpop	($const);
	&lea		($const,&DWP(&label("bswap")."-".&label("pic"),$const));

	&movdqu		($Hkey,&QWP(0,$Xip));
	&pshufd		($Hkey,$Hkey,0b01001110);# dword swap

	# <<1 twist
	&pshufd		($T2,$Hkey,0b11111111);	# broadcast uppermost dword
	&movdqa		($T1,$Hkey);
	&psllq		($Hkey,1);
	&pxor		($T3,$T3);		#
	&psrlq		($T1,63);
	&pcmpgtd	($T3,$T2);		# broadcast carry bit
	&pslldq		($T1,8);
	&por		($Hkey,$T1);		# H<<=1

	# magic reduction
	&pand		($T3,&QWP(16,$const));	# 0x1c2_polynomial
	&pxor		($Hkey,$T3);		# if(carry) H^=0x1c2_polynomial

	# calculate H^2
	&movdqa		($Xi,$Hkey);
	&clmul64x64_T2	($Xhi,$Xi,$Hkey);
	&reduction_alg9	($Xhi,$Xi);

	&pshufd		($T1,$Hkey,0b01001110);
	&pshufd		($T2,$Xi,0b01001110);
	&pxor		($T1,$Hkey);		# Karatsuba pre-processing
	&movdqu		(&QWP(0,$Htbl),$Hkey);	# save H
	&pxor		($T2,$Xi);		# Karatsuba pre-processing
	&movdqu		(&QWP(16,$Htbl),$Xi);	# save H^2
	&palignr	($T2,$T1,8);		# low part is H.lo^H.hi
	&movdqu		(&QWP(32,$Htbl),$T2);	# save Karatsuba "salt"

	&ret		();
&function_end_B("gcm_init_clmul");

&function_begin_B("gcm_gmult_clmul");
	&mov		($Xip,&wparam(0));
	&mov		($Htbl,&wparam(1));

	&call		(&label("pic"));
&set_label("pic");
	&blindpop	($const);
	&lea		($const,&DWP(&label("bswap")."-".&label("pic"),$const));

	&movdqu		($Xi,&QWP(0,$Xip));
	&movdqa		($T3,&QWP(0,$const));
	&movups		($Hkey,&QWP(0,$Htbl));
	&pshufb		($Xi,$T3);
	&movups		($T2,&QWP(32,$Htbl));

	&clmul64x64_T2	($Xhi,$Xi,$Hkey,$T2);
	&reduction_alg9	($Xhi,$Xi);

	&pshufb		($Xi,$T3);
	&movdqu		(&QWP(0,$Xip),$Xi);

	&ret	();
&function_end_B("gcm_gmult_clmul");

&function_begin("gcm_ghash_clmul");
	&mov		($Xip,&wparam(0));
	&mov		($Htbl,&wparam(1));
	&mov		($inp,&wparam(2));
	&mov		($len,&wparam(3));

	&call		(&label("pic"));
&set_label("pic");
	&blindpop	($const);
	&lea		($const,&DWP(&label("bswap")."-".&label("pic"),$const));

	&movdqu		($Xi,&QWP(0,$Xip));
	&movdqa		($T3,&QWP(0,$const));
	&movdqu		($Hkey,&QWP(0,$Htbl));
	&pshufb		($Xi,$T3);

	&sub		($len,0x10);
	&jz		(&label("odd_tail"));

	#######
	# Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
	#	[(H*Ii+1) + (H*Xi+1)] mod P =
	#	[(H*Ii+1) + H^2*(Ii+Xi)] mod P
	#
	&movdqu		($T1,&QWP(0,$inp));	# Ii
	&movdqu		($Xn,&QWP(16,$inp));	# Ii+1
	&pshufb		($T1,$T3);
	&pshufb		($Xn,$T3);
	&movdqu		($T3,&QWP(32,$Htbl));
	&pxor		($Xi,$T1);		# Ii+Xi

	&pshufd		($T1,$Xn,0b01001110);	# H*Ii+1
	&movdqa		($Xhn,$Xn);
	&pxor		($T1,$Xn);		#
	&lea		($inp,&DWP(32,$inp));	# i+=2

	&pclmulqdq	($Xn,$Hkey,0x00);	#######
	&pclmulqdq	($Xhn,$Hkey,0x11);	#######
	&pclmulqdq	($T1,$T3,0x00);		#######
	&movups		($Hkey,&QWP(16,$Htbl));	# load H^2
	&nop		();

	&sub		($len,0x20);
	&jbe		(&label("even_tail"));
	&jmp		(&label("mod_loop"));

&set_label("mod_loop",32);
	&pshufd		($T2,$Xi,0b01001110);	# H^2*(Ii+Xi)
	&movdqa		($Xhi,$Xi);
	&pxor		($T2,$Xi);		#
	&nop		();

	&pclmulqdq	($Xi,$Hkey,0x00);	#######
	&pclmulqdq	($Xhi,$Hkey,0x11);	#######
	&pclmulqdq	($T2,$T3,0x10);		#######
	&movups		($Hkey,&QWP(0,$Htbl));	# load H

	&xorps		($Xi,$Xn);		# (H*Ii+1) + H^2*(Ii+Xi)
	&movdqa		($T3,&QWP(0,$const));
	&xorps		($Xhi,$Xhn);
	 &movdqu	($Xhn,&QWP(0,$inp));	# Ii
	&pxor		($T1,$Xi);		# aggregated Karatsuba post-processing
	 &movdqu	($Xn,&QWP(16,$inp));	# Ii+1
	&pxor		($T1,$Xhi);		#

	 &pshufb	($Xhn,$T3);
	&pxor		($T2,$T1);		#

	&movdqa		($T1,$T2);		#
	&psrldq		($T2,8);
	&pslldq		($T1,8);		#
	&pxor		($Xhi,$T2);
	&pxor		($Xi,$T1);		#
	 &pshufb	($Xn,$T3);
	 &pxor		($Xhi,$Xhn);		# "Ii+Xi", consume early

	&movdqa		($Xhn,$Xn);		#&clmul64x64_TX	($Xhn,$Xn,$Hkey); H*Ii+1
	  &movdqa	($T2,$Xi);		#&reduction_alg9($Xhi,$Xi); 1st phase
	  &movdqa	($T1,$Xi);
	  &psllq	($Xi,5);
	  &pxor		($T1,$Xi);		#
	  &psllq	($Xi,1);
	  &pxor		($Xi,$T1);		#
	&pclmulqdq	($Xn,$Hkey,0x00);	#######
	&movups		($T3,&QWP(32,$Htbl));
	  &psllq	($Xi,57);		#
	  &movdqa	($T1,$Xi);		#
	  &pslldq	($Xi,8);
	  &psrldq	($T1,8);		#
	  &pxor		($Xi,$T2);
	  &pxor		($Xhi,$T1);		#
	&pshufd		($T1,$Xhn,0b01001110);
	  &movdqa	($T2,$Xi);		# 2nd phase
	  &psrlq	($Xi,1);
	&pxor		($T1,$Xhn);
	  &pxor		($Xhi,$T2);		#
	&pclmulqdq	($Xhn,$Hkey,0x11);	#######
	&movups		($Hkey,&QWP(16,$Htbl));	# load H^2
	  &pxor		($T2,$Xi);
	  &psrlq	($Xi,5);
	  &pxor		($Xi,$T2);		#
	  &psrlq	($Xi,1);		#
	  &pxor		($Xi,$Xhi)		#
	&pclmulqdq	($T1,$T3,0x00);		#######

	&lea		($inp,&DWP(32,$inp));
	&sub		($len,0x20);
	&ja		(&label("mod_loop"));

&set_label("even_tail");
	&pshufd		($T2,$Xi,0b01001110);	# H^2*(Ii+Xi)
	&movdqa		($Xhi,$Xi);
	&pxor		($T2,$Xi);		#

	&pclmulqdq	($Xi,$Hkey,0x00);	#######
	&pclmulqdq	($Xhi,$Hkey,0x11);	#######
	&pclmulqdq	($T2,$T3,0x10);		#######
	&movdqa		($T3,&QWP(0,$const));

	&xorps		($Xi,$Xn);		# (H*Ii+1) + H^2*(Ii+Xi)
	&xorps		($Xhi,$Xhn);
	&pxor		($T1,$Xi);		# aggregated Karatsuba post-processing
	&pxor		($T1,$Xhi);		#

	&pxor		($T2,$T1);		#

	&movdqa		($T1,$T2);		#
	&psrldq		($T2,8);
	&pslldq		($T1,8);		#
	&pxor		($Xhi,$T2);
	&pxor		($Xi,$T1);		#

	&reduction_alg9	($Xhi,$Xi);

	&test		($len,$len);
	&jnz		(&label("done"));

	&movups		($Hkey,&QWP(0,$Htbl));	# load H
&set_label("odd_tail");
	&movdqu		($T1,&QWP(0,$inp));	# Ii
	&pshufb		($T1,$T3);
	&pxor		($Xi,$T1);		# Ii+Xi

	&clmul64x64_T2	($Xhi,$Xi,$Hkey);	# H*(Ii+Xi)
	&reduction_alg9	($Xhi,$Xi);

&set_label("done");
	&pshufb		($Xi,$T3);
	&movdqu		(&QWP(0,$Xip),$Xi);
&function_end("gcm_ghash_clmul");

} else {		# Algorithm 5. Kept for reference purposes.

sub reduction_alg5 {	# 19/16 times faster than Intel version
my ($Xhi,$Xi)=@_;

	# <<1
	&movdqa		($T1,$Xi);		#
	&movdqa		($T2,$Xhi);
	&pslld		($Xi,1);
	&pslld		($Xhi,1);		#
	&psrld		($T1,31);
	&psrld		($T2,31);		#
	&movdqa		($T3,$T1);
	&pslldq		($T1,4);
	&psrldq		($T3,12);		#
	&pslldq		($T2,4);
	&por		($Xhi,$T3);		#
	&por		($Xi,$T1);
	&por		($Xhi,$T2);		#

	# 1st phase
	&movdqa		($T1,$Xi);
	&movdqa		($T2,$Xi);
	&movdqa		($T3,$Xi);		#
	&pslld		($T1,31);
	&pslld		($T2,30);
	&pslld		($Xi,25);		#
	&pxor		($T1,$T2);
	&pxor		($T1,$Xi);		#
	&movdqa		($T2,$T1);		#
	&pslldq		($T1,12);
	&psrldq		($T2,4);		#
	&pxor		($T3,$T1);

	# 2nd phase
	&pxor		($Xhi,$T3);		#
	&movdqa		($Xi,$T3);
	&movdqa		($T1,$T3);
	&psrld		($Xi,1);		#
	&psrld		($T1,2);
	&psrld		($T3,7);		#
	&pxor		($Xi,$T1);
	&pxor		($Xhi,$T2);
	&pxor		($Xi,$T3);		#
	&pxor		($Xi,$Xhi);		#
}

&function_begin_B("gcm_init_clmul");
	&mov		($Htbl,&wparam(0));
	&mov		($Xip,&wparam(1));

	&call		(&label("pic"));
&set_label("pic");
	&blindpop	($const);
	&lea		($const,&DWP(&label("bswap")."-".&label("pic"),$const));

	&movdqu		($Hkey,&QWP(0,$Xip));
	&pshufd		($Hkey,$Hkey,0b01001110);# dword swap

	# calculate H^2
	&movdqa		($Xi,$Hkey);
	&clmul64x64_T3	($Xhi,$Xi,$Hkey);
	&reduction_alg5	($Xhi,$Xi);

	&movdqu		(&QWP(0,$Htbl),$Hkey);	# save H
	&movdqu		(&QWP(16,$Htbl),$Xi);	# save H^2

	&ret		();
&function_end_B("gcm_init_clmul");

&function_begin_B("gcm_gmult_clmul");
	&mov		($Xip,&wparam(0));
	&mov		($Htbl,&wparam(1));

	&call		(&label("pic"));
&set_label("pic");
	&blindpop	($const);
	&lea		($const,&DWP(&label("bswap")."-".&label("pic"),$const));

	&movdqu		($Xi,&QWP(0,$Xip));
	&movdqa		($Xn,&QWP(0,$const));
	&movdqu		($Hkey,&QWP(0,$Htbl));
	&pshufb		($Xi,$Xn);

	&clmul64x64_T3	($Xhi,$Xi,$Hkey);
	&reduction_alg5	($Xhi,$Xi);

	&pshufb		($Xi,$Xn);
	&movdqu		(&QWP(0,$Xip),$Xi);

	&ret	();
&function_end_B("gcm_gmult_clmul");

&function_begin("gcm_ghash_clmul");
	&mov		($Xip,&wparam(0));
	&mov		($Htbl,&wparam(1));
	&mov		($inp,&wparam(2));
	&mov		($len,&wparam(3));

	&call		(&label("pic"));
&set_label("pic");
	&blindpop	($const);
	&lea		($const,&DWP(&label("bswap")."-".&label("pic"),$const));

	&movdqu		($Xi,&QWP(0,$Xip));
	&movdqa		($T3,&QWP(0,$const));
	&movdqu		($Hkey,&QWP(0,$Htbl));
	&pshufb		($Xi,$T3);

	&sub		($len,0x10);
	&jz		(&label("odd_tail"));

	#######
	# Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
	#	[(H*Ii+1) + (H*Xi+1)] mod P =
	#	[(H*Ii+1) + H^2*(Ii+Xi)] mod P
	#
	&movdqu		($T1,&QWP(0,$inp));	# Ii
	&movdqu		($Xn,&QWP(16,$inp));	# Ii+1
	&pshufb		($T1,$T3);
	&pshufb		($Xn,$T3);
	&pxor		($Xi,$T1);		# Ii+Xi

	&clmul64x64_T3	($Xhn,$Xn,$Hkey);	# H*Ii+1
	&movdqu		($Hkey,&QWP(16,$Htbl));	# load H^2

	&sub		($len,0x20);
	&lea		($inp,&DWP(32,$inp));	# i+=2
	&jbe		(&label("even_tail"));

&set_label("mod_loop");
	&clmul64x64_T3	($Xhi,$Xi,$Hkey);	# H^2*(Ii+Xi)
	&movdqu		($Hkey,&QWP(0,$Htbl));	# load H

	&pxor		($Xi,$Xn);		# (H*Ii+1) + H^2*(Ii+Xi)
	&pxor		($Xhi,$Xhn);

	&reduction_alg5	($Xhi,$Xi);

	#######
	&movdqa		($T3,&QWP(0,$const));
	&movdqu		($T1,&QWP(0,$inp));	# Ii
	&movdqu		($Xn,&QWP(16,$inp));	# Ii+1
	&pshufb		($T1,$T3);
	&pshufb		($Xn,$T3);
	&pxor		($Xi,$T1);		# Ii+Xi

	&clmul64x64_T3	($Xhn,$Xn,$Hkey);	# H*Ii+1
	&movdqu		($Hkey,&QWP(16,$Htbl));	# load H^2

	&sub		($len,0x20);
	&lea		($inp,&DWP(32,$inp));
	&ja		(&label("mod_loop"));

&set_label("even_tail");
	&clmul64x64_T3	($Xhi,$Xi,$Hkey);	# H^2*(Ii+Xi)

	&pxor		($Xi,$Xn);		# (H*Ii+1) + H^2*(Ii+Xi)
	&pxor		($Xhi,$Xhn);

	&reduction_alg5	($Xhi,$Xi);

	&movdqa		($T3,&QWP(0,$const));
	&test		($len,$len);
	&jnz		(&label("done"));

	&movdqu		($Hkey,&QWP(0,$Htbl));	# load H
&set_label("odd_tail");
	&movdqu		($T1,&QWP(0,$inp));	# Ii
	&pshufb		($T1,$T3);
	&pxor		($Xi,$T1);		# Ii+Xi

	&clmul64x64_T3	($Xhi,$Xi,$Hkey);	# H*(Ii+Xi)
	&reduction_alg5	($Xhi,$Xi);

	&movdqa		($T3,&QWP(0,$const));
&set_label("done");
	&pshufb		($Xi,$T3);
	&movdqu		(&QWP(0,$Xip),$Xi);
&function_end("gcm_ghash_clmul");

}

&set_label("bswap",64);
	&data_byte(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);
	&data_byte(1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2);	# 0x1c2_polynomial
}}	# $sse2
}}}	# !$x86only

&asciz("GHASH for x86, CRYPTOGAMS by <appro\@openssl.org>");
&asm_finish();

close STDOUT or die "error closing STDOUT: $!";

# A question was risen about choice of vanilla MMX. Or rather why wasn't
# SSE2 chosen instead? In addition to the fact that MMX runs on legacy
# CPUs such as PIII, "4-bit" MMX version was observed to provide better
# performance than *corresponding* SSE2 one even on contemporary CPUs.
# SSE2 results were provided by Peter-Michael Hager. He maintains SSE2
# implementation featuring full range of lookup-table sizes, but with
# per-invocation lookup table setup. Latter means that table size is
# chosen depending on how much data is to be hashed in every given call,
# more data - larger table. Best reported result for Core2 is ~4 cycles
# per processed byte out of 64KB block. This number accounts even for
# 64KB table setup overhead. As discussed in gcm128.c we choose to be
# more conservative in respect to lookup table sizes, but how do the
# results compare? Minimalistic "256B" MMX version delivers ~11 cycles
# on same platform. As also discussed in gcm128.c, next in line "8-bit
# Shoup's" or "4KB" method should deliver twice the performance of
# "256B" one, in other words not worse than ~6 cycles per byte. It
# should be also be noted that in SSE2 case improvement can be "super-
# linear," i.e. more than twice, mostly because >>8 maps to single
# instruction on SSE2 register. This is unlike "4-bit" case when >>4
# maps to same amount of instructions in both MMX and SSE2 cases.
# Bottom line is that switch to SSE2 is considered to be justifiable
# only in case we choose to implement "8-bit" method...


#! /usr/bin/env perl
# Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# March, June 2010
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that
# it uses 256 bytes per-key table [+128 bytes shared table]. GHASH
# function features so called "528B" variant utilizing additional
# 256+16 bytes of per-key storage [+512 bytes shared table].
# Performance results are for this streamed GHASH subroutine and are
# expressed in cycles per processed byte, less is better:
#
#		gcc 3.4.x(*)	assembler
#
# P4		28.6		14.0		+100%
# Opteron	19.3		7.7		+150%
# Core2		17.8		8.1(**)		+120%
# Atom		31.6		16.8		+88%
# VIA Nano	21.8		10.1		+115%
#
# (*)	comparison is not completely fair, because C results are
#	for vanilla "256B" implementation, while assembler results
#	are for "528B";-)
# (**)	it's mystery [to me] why Core2 result is not same as for
#	Opteron;

# May 2010
#
# Add PCLMULQDQ version performing at 2.02 cycles per processed byte.
# See ghash-x86.pl for background information and details about coding
# techniques.
#
# Special thanks to David Woodhouse for providing access to a
# Westmere-based system on behalf of Intel Open Source Technology Centre.

# December 2012
#
# Overhaul: aggregate Karatsuba post-processing, improve ILP in
# reduction_alg9, increase reduction aggregate factor to 4x. As for
# the latter. ghash-x86.pl discusses that it makes lesser sense to
# increase aggregate factor. Then why increase here? Critical path
# consists of 3 independent pclmulqdq instructions, Karatsuba post-
# processing and reduction. "On top" of this we lay down aggregated
# multiplication operations, triplets of independent pclmulqdq's. As
# issue rate for pclmulqdq is limited, it makes lesser sense to
# aggregate more multiplications than it takes to perform remaining
# non-multiplication operations. 2x is near-optimal coefficient for
# contemporary Intel CPUs (therefore modest improvement coefficient),
# but not for Bulldozer. Latter is because logical SIMD operations
# are twice as slow in comparison to Intel, so that critical path is
# longer. A CPU with higher pclmulqdq issue rate would also benefit
# from higher aggregate factor...
#
# Westmere	1.78(+13%)
# Sandy Bridge	1.80(+8%)
# Ivy Bridge	1.80(+7%)
# Haswell	0.55(+93%) (if system doesn't support AVX)
# Broadwell	0.45(+110%)(if system doesn't support AVX)
# Skylake	0.44(+110%)(if system doesn't support AVX)
# Bulldozer	1.49(+27%)
# Silvermont	2.88(+13%)
# Knights L	2.12(-)    (if system doesn't support AVX)
# Goldmont	1.08(+24%)

# March 2013
#
# ... 8x aggregate factor AVX code path is using reduction algorithm
# suggested by Shay Gueron[1]. Even though contemporary AVX-capable
# CPUs such as Sandy and Ivy Bridge can execute it, the code performs
# sub-optimally in comparison to above mentioned version. But thanks
# to Ilya Albrekht and Max Locktyukhin of Intel Corp. we knew that
# it performs in 0.41 cycles per byte on Haswell processor, in
# 0.29 on Broadwell, and in 0.36 on Skylake.
#
# Knights Landing achieves 1.09 cpb.
#
# [1] http://rt.openssl.org/Ticket/Display.html?id=2900&user=guest&pass=guest

# This file was patched in BoringSSL to remove the variable-time 4-bit
# implementation.

# The first two arguments should always be the flavour and output file path.
if ($#ARGV < 1) { die "Not enough arguments provided.
  Two arguments are necessary: the flavour and the output file path."; }

$flavour = shift;
$output  = shift;

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

# See the notes about |$avx| in aesni-gcm-x86_64.pl; otherwise tags will be
# computed incorrectly.
#
# In upstream, this is controlled by shelling out to the compiler to check
# versions, but BoringSSL is intended to be used with pre-generated perlasm
# output, so this isn't useful anyway.
$avx = 1;
for (@ARGV) { $avx = 0 if (/-DMY_ASSEMBLER_IS_TOO_OLD_FOR_AVX/); }

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

$do4xaggr=1;

$code=<<___;
.text
.extern	OPENSSL_ia32cap_P
___

######################################################################
# PCLMULQDQ version.

@_4args=$win64?	("%rcx","%rdx","%r8", "%r9") :	# Win64 order
		("%rdi","%rsi","%rdx","%rcx");	# Unix order

($Xi,$Xhi)=("%xmm0","%xmm1");	$Hkey="%xmm2";
($T1,$T2,$T3)=("%xmm3","%xmm4","%xmm5");

sub clmul64x64_T2 {	# minimal register pressure
my ($Xhi,$Xi,$Hkey,$HK)=@_;

if (!defined($HK)) {	$HK = $T2;
$code.=<<___;
	movdqa		$Xi,$Xhi		#
	pshufd		\$0b01001110,$Xi,$T1
	pshufd		\$0b01001110,$Hkey,$T2
	pxor		$Xi,$T1			#
	pxor		$Hkey,$T2
___
} else {
$code.=<<___;
	movdqa		$Xi,$Xhi		#
	pshufd		\$0b01001110,$Xi,$T1
	pxor		$Xi,$T1			#
___
}
$code.=<<___;
	pclmulqdq	\$0x00,$Hkey,$Xi	#######
	pclmulqdq	\$0x11,$Hkey,$Xhi	#######
	pclmulqdq	\$0x00,$HK,$T1		#######
	pxor		$Xi,$T1			#
	pxor		$Xhi,$T1		#

	movdqa		$T1,$T2			#
	psrldq		\$8,$T1
	pslldq		\$8,$T2			#
	pxor		$T1,$Xhi
	pxor		$T2,$Xi			#
___
}

sub reduction_alg9 {	# 17/11 times faster than Intel version
my ($Xhi,$Xi) = @_;

$code.=<<___;
	# 1st phase
	movdqa		$Xi,$T2			#
	movdqa		$Xi,$T1
	psllq		\$5,$Xi
	pxor		$Xi,$T1			#
	psllq		\$1,$Xi
	pxor		$T1,$Xi			#
	psllq		\$57,$Xi		#
	movdqa		$Xi,$T1			#
	pslldq		\$8,$Xi
	psrldq		\$8,$T1			#
	pxor		$T2,$Xi
	pxor		$T1,$Xhi		#

	# 2nd phase
	movdqa		$Xi,$T2
	psrlq		\$1,$Xi
	pxor		$T2,$Xhi		#
	pxor		$Xi,$T2
	psrlq		\$5,$Xi
	pxor		$T2,$Xi			#
	psrlq		\$1,$Xi			#
	pxor		$Xhi,$Xi		#
___
}

{ my ($Htbl,$Xip)=@_4args;
  my $HK="%xmm6";

$code.=<<___;
.globl	gcm_init_clmul
.type	gcm_init_clmul,\@abi-omnipotent
.align	16
gcm_init_clmul:
.cfi_startproc
.seh_startproc
.L_init_clmul:
___
$code.=<<___ if ($win64);
	sub	\$0x18,%rsp
.seh_allocstack	0x18
	movaps	%xmm6,(%rsp)
.seh_savexmm128	%xmm6, 0
___
$code.=<<___;
	movdqu		($Xip),$Hkey
	pshufd		\$0b01001110,$Hkey,$Hkey	# dword swap

	# <<1 twist
	pshufd		\$0b11111111,$Hkey,$T2	# broadcast uppermost dword
	movdqa		$Hkey,$T1
	psllq		\$1,$Hkey
	pxor		$T3,$T3			#
	psrlq		\$63,$T1
	pcmpgtd		$T2,$T3			# broadcast carry bit
	pslldq		\$8,$T1
	por		$T1,$Hkey		# H<<=1

	# magic reduction
	pand		.L0x1c2_polynomial(%rip),$T3
	pxor		$T3,$Hkey		# if(carry) H^=0x1c2_polynomial

	# calculate H^2
	pshufd		\$0b01001110,$Hkey,$HK
	movdqa		$Hkey,$Xi
	pxor		$Hkey,$HK
___
	&clmul64x64_T2	($Xhi,$Xi,$Hkey,$HK);
	&reduction_alg9	($Xhi,$Xi);
$code.=<<___;
	pshufd		\$0b01001110,$Hkey,$T1
	pshufd		\$0b01001110,$Xi,$T2
	pxor		$Hkey,$T1		# Karatsuba pre-processing
	movdqu		$Hkey,0x00($Htbl)	# save H
	pxor		$Xi,$T2			# Karatsuba pre-processing
	movdqu		$Xi,0x10($Htbl)		# save H^2
	palignr		\$8,$T1,$T2		# low part is H.lo^H.hi...
	movdqu		$T2,0x20($Htbl)		# save Karatsuba "salt"
___
if ($do4xaggr) {
	&clmul64x64_T2	($Xhi,$Xi,$Hkey,$HK);	# H^3
	&reduction_alg9	($Xhi,$Xi);
$code.=<<___;
	movdqa		$Xi,$T3
___
	&clmul64x64_T2	($Xhi,$Xi,$Hkey,$HK);	# H^4
	&reduction_alg9	($Xhi,$Xi);
$code.=<<___;
	pshufd		\$0b01001110,$T3,$T1
	pshufd		\$0b01001110,$Xi,$T2
	pxor		$T3,$T1			# Karatsuba pre-processing
	movdqu		$T3,0x30($Htbl)		# save H^3
	pxor		$Xi,$T2			# Karatsuba pre-processing
	movdqu		$Xi,0x40($Htbl)		# save H^4
	palignr		\$8,$T1,$T2		# low part is H^3.lo^H^3.hi...
	movdqu		$T2,0x50($Htbl)		# save Karatsuba "salt"
___
}
$code.=<<___ if ($win64);
	movaps	(%rsp),%xmm6
	lea	0x18(%rsp),%rsp
___
$code.=<<___;
	ret
.cfi_endproc
.seh_endproc
.size	gcm_init_clmul,.-gcm_init_clmul
___
}

{ my ($Xip,$Htbl)=@_4args;

$code.=<<___;
.globl	gcm_gmult_clmul
.type	gcm_gmult_clmul,\@abi-omnipotent
.align	16
gcm_gmult_clmul:
.cfi_startproc
.L_gmult_clmul:
	movdqu		($Xip),$Xi
	movdqa		.Lbswap_mask(%rip),$T3
	movdqu		($Htbl),$Hkey
	movdqu		0x20($Htbl),$T2
	pshufb		$T3,$Xi
___
	&clmul64x64_T2	($Xhi,$Xi,$Hkey,$T2);
$code.=<<___ if (0 || (&reduction_alg9($Xhi,$Xi)&&0));
	# experimental alternative. special thing about is that there
	# no dependency between the two multiplications...
	mov		\$`0xE1<<1`,%eax
	mov		\$0xA040608020C0E000,%r10	# ((7..0)Â·0xE0)&0xff
	mov		\$0x07,%r11d
	movq		%rax,$T1
	movq		%r10,$T2
	movq		%r11,$T3		# borrow $T3
	pand		$Xi,$T3
	pshufb		$T3,$T2			# ($Xi&7)Â·0xE0
	movq		%rax,$T3
	pclmulqdq	\$0x00,$Xi,$T1		# Â·(0xE1<<1)
	pxor		$Xi,$T2
	pslldq		\$15,$T2
	paddd		$T2,$T2			# <<(64+56+1)
	pxor		$T2,$Xi
	pclmulqdq	\$0x01,$T3,$Xi
	movdqa		.Lbswap_mask(%rip),$T3	# reload $T3
	psrldq		\$1,$T1
	pxor		$T1,$Xhi
	pslldq		\$7,$Xi
	pxor		$Xhi,$Xi
___
$code.=<<___;
	pshufb		$T3,$Xi
	movdqu		$Xi,($Xip)
	ret
.cfi_endproc
.size	gcm_gmult_clmul,.-gcm_gmult_clmul
___
}

{ my ($Xip,$Htbl,$inp,$len)=@_4args;
  my ($Xln,$Xmn,$Xhn,$Hkey2,$HK) = map("%xmm$_",(3..7));
  my ($T1,$T2,$T3)=map("%xmm$_",(8..10));

$code.=<<___;
.globl	gcm_ghash_clmul
.type	gcm_ghash_clmul,\@abi-omnipotent
.align	32
gcm_ghash_clmul:
.cfi_startproc
.seh_startproc
.L_ghash_clmul:
___
$code.=<<___ if ($win64);
	lea	-0x88(%rsp),%rax
	lea	-0x20(%rax),%rsp
.seh_allocstack	0x20+0x88
	movaps	%xmm6,-0x20(%rax)
.seh_savexmm128	%xmm6, 0x20-0x20
	movaps	%xmm7,-0x10(%rax)
.seh_savexmm128	%xmm7, 0x20-0x10
	movaps	%xmm8,0(%rax)
.seh_savexmm128	%xmm8, 0x20+0
	movaps	%xmm9,0x10(%rax)
.seh_savexmm128	%xmm9, 0x20+0x10
	movaps	%xmm10,0x20(%rax)
.seh_savexmm128	%xmm10, 0x20+0x20
	movaps	%xmm11,0x30(%rax)
.seh_savexmm128	%xmm11, 0x20+0x30
	movaps	%xmm12,0x40(%rax)
.seh_savexmm128	%xmm12, 0x20+0x40
	movaps	%xmm13,0x50(%rax)
.seh_savexmm128	%xmm13, 0x20+0x50
	movaps	%xmm14,0x60(%rax)
.seh_savexmm128	%xmm14, 0x20+0x60
	movaps	%xmm15,0x70(%rax)
.seh_savexmm128	%xmm15, 0x20+0x70
___
$code.=<<___;
	movdqa		.Lbswap_mask(%rip),$T3

	movdqu		($Xip),$Xi
	movdqu		($Htbl),$Hkey
	movdqu		0x20($Htbl),$HK
	pshufb		$T3,$Xi

	sub		\$0x10,$len
	jz		.Lodd_tail

	movdqu		0x10($Htbl),$Hkey2
___
if ($do4xaggr) {
my ($Xl,$Xm,$Xh,$Hkey3,$Hkey4)=map("%xmm$_",(11..15));

$code.=<<___;
	leaq		OPENSSL_ia32cap_P(%rip),%rax
	mov		4(%rax),%eax
	cmp		\$0x30,$len
	jb		.Lskip4x

	and		\$`1<<26|1<<22`,%eax	# isolate MOVBE+XSAVE
	cmp		\$`1<<22`,%eax		# check for MOVBE without XSAVE
	je		.Lskip4x

	sub		\$0x30,$len
	mov		\$0xA040608020C0E000,%rax	# ((7..0)Â·0xE0)&0xff
	movdqu		0x30($Htbl),$Hkey3
	movdqu		0x40($Htbl),$Hkey4

	#######
	# Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P
	#
	movdqu		0x30($inp),$Xln
	 movdqu		0x20($inp),$Xl
	pshufb		$T3,$Xln
	 pshufb		$T3,$Xl
	movdqa		$Xln,$Xhn
	pshufd		\$0b01001110,$Xln,$Xmn
	pxor		$Xln,$Xmn
	pclmulqdq	\$0x00,$Hkey,$Xln
	pclmulqdq	\$0x11,$Hkey,$Xhn
	pclmulqdq	\$0x00,$HK,$Xmn

	movdqa		$Xl,$Xh
	pshufd		\$0b01001110,$Xl,$Xm
	pxor		$Xl,$Xm
	pclmulqdq	\$0x00,$Hkey2,$Xl
	pclmulqdq	\$0x11,$Hkey2,$Xh
	pclmulqdq	\$0x10,$HK,$Xm
	xorps		$Xl,$Xln
	xorps		$Xh,$Xhn
	movups		0x50($Htbl),$HK
	xorps		$Xm,$Xmn

	movdqu		0x10($inp),$Xl
	 movdqu		0($inp),$T1
	pshufb		$T3,$Xl
	 pshufb		$T3,$T1
	movdqa		$Xl,$Xh
	pshufd		\$0b01001110,$Xl,$Xm
	 pxor		$T1,$Xi
	pxor		$Xl,$Xm
	pclmulqdq	\$0x00,$Hkey3,$Xl
	 movdqa		$Xi,$Xhi
	 pshufd		\$0b01001110,$Xi,$T1
	 pxor		$Xi,$T1
	pclmulqdq	\$0x11,$Hkey3,$Xh
	pclmulqdq	\$0x00,$HK,$Xm
	xorps		$Xl,$Xln
	xorps		$Xh,$Xhn

	lea	0x40($inp),$inp
	sub	\$0x40,$len
	jc	.Ltail4x

	jmp	.Lmod4_loop
.align	32
.Lmod4_loop:
	pclmulqdq	\$0x00,$Hkey4,$Xi
	xorps		$Xm,$Xmn
	 movdqu		0x30($inp),$Xl
	 pshufb		$T3,$Xl
	pclmulqdq	\$0x11,$Hkey4,$Xhi
	xorps		$Xln,$Xi
	 movdqu		0x20($inp),$Xln
	 movdqa		$Xl,$Xh
	pclmulqdq	\$0x10,$HK,$T1
	 pshufd		\$0b01001110,$Xl,$Xm
	xorps		$Xhn,$Xhi
	 pxor		$Xl,$Xm
	 pshufb		$T3,$Xln
	movups		0x20($Htbl),$HK
	xorps		$Xmn,$T1
	 pclmulqdq	\$0x00,$Hkey,$Xl
	 pshufd		\$0b01001110,$Xln,$Xmn

	pxor		$Xi,$T1			# aggregated Karatsuba post-processing
	 movdqa		$Xln,$Xhn
	pxor		$Xhi,$T1		#
	 pxor		$Xln,$Xmn
	movdqa		$T1,$T2			#
	 pclmulqdq	\$0x11,$Hkey,$Xh
	pslldq		\$8,$T1
	psrldq		\$8,$T2			#
	pxor		$T1,$Xi
	movdqa		.L7_mask(%rip),$T1
	pxor		$T2,$Xhi		#
	movq		%rax,$T2

	pand		$Xi,$T1			# 1st phase
	pshufb		$T1,$T2			#
	pxor		$Xi,$T2			#
	 pclmulqdq	\$0x00,$HK,$Xm
	psllq		\$57,$T2		#
	movdqa		$T2,$T1			#
	pslldq		\$8,$T2
	 pclmulqdq	\$0x00,$Hkey2,$Xln
	psrldq		\$8,$T1			#
	pxor		$T2,$Xi
	pxor		$T1,$Xhi		#
	movdqu		0($inp),$T1

	movdqa		$Xi,$T2			# 2nd phase
	psrlq		\$1,$Xi
	 pclmulqdq	\$0x11,$Hkey2,$Xhn
	 xorps		$Xl,$Xln
	 movdqu		0x10($inp),$Xl
	 pshufb		$T3,$Xl
	 pclmulqdq	\$0x10,$HK,$Xmn
	 xorps		$Xh,$Xhn
	 movups		0x50($Htbl),$HK
	pshufb		$T3,$T1
	pxor		$T2,$Xhi		#
	pxor		$Xi,$T2
	psrlq		\$5,$Xi

	 movdqa		$Xl,$Xh
	 pxor		$Xm,$Xmn
	 pshufd		\$0b01001110,$Xl,$Xm
	pxor		$T2,$Xi			#
	pxor		$T1,$Xhi
	 pxor		$Xl,$Xm
	 pclmulqdq	\$0x00,$Hkey3,$Xl
	psrlq		\$1,$Xi			#
	pxor		$Xhi,$Xi		#
	movdqa		$Xi,$Xhi
	 pclmulqdq	\$0x11,$Hkey3,$Xh
	 xorps		$Xl,$Xln
	pshufd		\$0b01001110,$Xi,$T1
	pxor		$Xi,$T1

	 pclmulqdq	\$0x00,$HK,$Xm
	 xorps		$Xh,$Xhn

	lea	0x40($inp),$inp
	sub	\$0x40,$len
	jnc	.Lmod4_loop

.Ltail4x:
	pclmulqdq	\$0x00,$Hkey4,$Xi
	pclmulqdq	\$0x11,$Hkey4,$Xhi
	pclmulqdq	\$0x10,$HK,$T1
	xorps		$Xm,$Xmn
	xorps		$Xln,$Xi
	xorps		$Xhn,$Xhi
	pxor		$Xi,$Xhi		# aggregated Karatsuba post-processing
	pxor		$Xmn,$T1

	pxor		$Xhi,$T1		#
	pxor		$Xi,$Xhi

	movdqa		$T1,$T2			#
	psrldq		\$8,$T1
	pslldq		\$8,$T2			#
	pxor		$T1,$Xhi
	pxor		$T2,$Xi			#
___
	&reduction_alg9($Xhi,$Xi);
$code.=<<___;
	add	\$0x40,$len
	jz	.Ldone
	movdqu	0x20($Htbl),$HK
	sub	\$0x10,$len
	jz	.Lodd_tail
.Lskip4x:
___
}
$code.=<<___;
	#######
	# Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
	#	[(H*Ii+1) + (H*Xi+1)] mod P =
	#	[(H*Ii+1) + H^2*(Ii+Xi)] mod P
	#
	movdqu		($inp),$T1		# Ii
	movdqu		16($inp),$Xln		# Ii+1
	pshufb		$T3,$T1
	pshufb		$T3,$Xln
	pxor		$T1,$Xi			# Ii+Xi

	movdqa		$Xln,$Xhn
	pshufd		\$0b01001110,$Xln,$Xmn
	pxor		$Xln,$Xmn
	pclmulqdq	\$0x00,$Hkey,$Xln
	pclmulqdq	\$0x11,$Hkey,$Xhn
	pclmulqdq	\$0x00,$HK,$Xmn

	lea		32($inp),$inp		# i+=2
	nop
	sub		\$0x20,$len
	jbe		.Leven_tail
	nop
	jmp		.Lmod_loop

.align	32
.Lmod_loop:
	movdqa		$Xi,$Xhi
	movdqa		$Xmn,$T1
	pshufd		\$0b01001110,$Xi,$Xmn	#
	pxor		$Xi,$Xmn		#

	pclmulqdq	\$0x00,$Hkey2,$Xi
	pclmulqdq	\$0x11,$Hkey2,$Xhi
	pclmulqdq	\$0x10,$HK,$Xmn

	pxor		$Xln,$Xi		# (H*Ii+1) + H^2*(Ii+Xi)
	pxor		$Xhn,$Xhi
	  movdqu	($inp),$T2		# Ii
	pxor		$Xi,$T1			# aggregated Karatsuba post-processing
	  pshufb	$T3,$T2
	  movdqu	16($inp),$Xln		# Ii+1

	pxor		$Xhi,$T1
	  pxor		$T2,$Xhi		# "Ii+Xi", consume early
	pxor		$T1,$Xmn
	 pshufb		$T3,$Xln
	movdqa		$Xmn,$T1		#
	psrldq		\$8,$T1
	pslldq		\$8,$Xmn		#
	pxor		$T1,$Xhi
	pxor		$Xmn,$Xi		#

	movdqa		$Xln,$Xhn		#

	  movdqa	$Xi,$T2			# 1st phase
	  movdqa	$Xi,$T1
	  psllq		\$5,$Xi
	  pxor		$Xi,$T1			#
	pclmulqdq	\$0x00,$Hkey,$Xln	#######
	  psllq		\$1,$Xi
	  pxor		$T1,$Xi			#
	  psllq		\$57,$Xi		#
	  movdqa	$Xi,$T1			#
	  pslldq	\$8,$Xi
	  psrldq	\$8,$T1			#
	  pxor		$T2,$Xi
	pshufd		\$0b01001110,$Xhn,$Xmn
	  pxor		$T1,$Xhi		#
	pxor		$Xhn,$Xmn		#

	  movdqa	$Xi,$T2			# 2nd phase
	  psrlq		\$1,$Xi
	pclmulqdq	\$0x11,$Hkey,$Xhn	#######
	  pxor		$T2,$Xhi		#
	  pxor		$Xi,$T2
	  psrlq		\$5,$Xi
	  pxor		$T2,$Xi			#
	lea		32($inp),$inp
	  psrlq		\$1,$Xi			#
	pclmulqdq	\$0x00,$HK,$Xmn		#######
	  pxor		$Xhi,$Xi		#

	sub		\$0x20,$len
	ja		.Lmod_loop

.Leven_tail:
	 movdqa		$Xi,$Xhi
	 movdqa		$Xmn,$T1
	 pshufd		\$0b01001110,$Xi,$Xmn	#
	 pxor		$Xi,$Xmn		#

	pclmulqdq	\$0x00,$Hkey2,$Xi
	pclmulqdq	\$0x11,$Hkey2,$Xhi
	pclmulqdq	\$0x10,$HK,$Xmn

	pxor		$Xln,$Xi		# (H*Ii+1) + H^2*(Ii+Xi)
	pxor		$Xhn,$Xhi
	pxor		$Xi,$T1
	pxor		$Xhi,$T1
	pxor		$T1,$Xmn
	movdqa		$Xmn,$T1		#
	psrldq		\$8,$T1
	pslldq		\$8,$Xmn		#
	pxor		$T1,$Xhi
	pxor		$Xmn,$Xi		#
___
	&reduction_alg9	($Xhi,$Xi);
$code.=<<___;
	test		$len,$len
	jnz		.Ldone

.Lodd_tail:
	movdqu		($inp),$T1		# Ii
	pshufb		$T3,$T1
	pxor		$T1,$Xi			# Ii+Xi
___
	&clmul64x64_T2	($Xhi,$Xi,$Hkey,$HK);	# H*(Ii+Xi)
	&reduction_alg9	($Xhi,$Xi);
$code.=<<___;
.Ldone:
	pshufb		$T3,$Xi
	movdqu		$Xi,($Xip)
___
$code.=<<___ if ($win64);
	movaps	(%rsp),%xmm6
	movaps	0x10(%rsp),%xmm7
	movaps	0x20(%rsp),%xmm8
	movaps	0x30(%rsp),%xmm9
	movaps	0x40(%rsp),%xmm10
	movaps	0x50(%rsp),%xmm11
	movaps	0x60(%rsp),%xmm12
	movaps	0x70(%rsp),%xmm13
	movaps	0x80(%rsp),%xmm14
	movaps	0x90(%rsp),%xmm15
	lea	0xa8(%rsp),%rsp
___
$code.=<<___;
	ret
.cfi_endproc
.seh_endproc
.size	gcm_ghash_clmul,.-gcm_ghash_clmul
___
}

$code.=<<___;
.globl	gcm_init_avx
.type	gcm_init_avx,\@abi-omnipotent
.align	32
gcm_init_avx:
.cfi_startproc
___
if ($avx) {
my ($Htbl,$Xip)=@_4args;
my $HK="%xmm6";

$code.=<<___ if ($win64);
.seh_startproc
	sub	\$0x18,%rsp
.seh_allocstack	0x18
	movaps	%xmm6,(%rsp)
.seh_savexmm128	%xmm6, 0
___
$code.=<<___;
	vzeroupper

	vmovdqu		($Xip),$Hkey
	vpshufd		\$0b01001110,$Hkey,$Hkey	# dword swap

	# <<1 twist
	vpshufd		\$0b11111111,$Hkey,$T2	# broadcast uppermost dword
	vpsrlq		\$63,$Hkey,$T1
	vpsllq		\$1,$Hkey,$Hkey
	vpxor		$T3,$T3,$T3		#
	vpcmpgtd	$T2,$T3,$T3		# broadcast carry bit
	vpslldq		\$8,$T1,$T1
	vpor		$T1,$Hkey,$Hkey		# H<<=1

	# magic reduction
	vpand		.L0x1c2_polynomial(%rip),$T3,$T3
	vpxor		$T3,$Hkey,$Hkey		# if(carry) H^=0x1c2_polynomial

	vpunpckhqdq	$Hkey,$Hkey,$HK
	vmovdqa		$Hkey,$Xi
	vpxor		$Hkey,$HK,$HK
	mov		\$4,%r10		# up to H^8
	jmp		.Linit_start_avx
___

sub clmul64x64_avx {
my ($Xhi,$Xi,$Hkey,$HK)=@_;

if (!defined($HK)) {	$HK = $T2;
$code.=<<___;
	vpunpckhqdq	$Xi,$Xi,$T1
	vpunpckhqdq	$Hkey,$Hkey,$T2
	vpxor		$Xi,$T1,$T1		#
	vpxor		$Hkey,$T2,$T2
___
} else {
$code.=<<___;
	vpunpckhqdq	$Xi,$Xi,$T1
	vpxor		$Xi,$T1,$T1		#
___
}
$code.=<<___;
	vpclmulqdq	\$0x11,$Hkey,$Xi,$Xhi	#######
	vpclmulqdq	\$0x00,$Hkey,$Xi,$Xi	#######
	vpclmulqdq	\$0x00,$HK,$T1,$T1	#######
	vpxor		$Xi,$Xhi,$T2		#
	vpxor		$T2,$T1,$T1		#

	vpslldq		\$8,$T1,$T2		#
	vpsrldq		\$8,$T1,$T1
	vpxor		$T2,$Xi,$Xi		#
	vpxor		$T1,$Xhi,$Xhi
___
}

sub reduction_avx {
my ($Xhi,$Xi) = @_;

$code.=<<___;
	vpsllq		\$57,$Xi,$T1		# 1st phase
	vpsllq		\$62,$Xi,$T2
	vpxor		$T1,$T2,$T2		#
	vpsllq		\$63,$Xi,$T1
	vpxor		$T1,$T2,$T2		#
	vpslldq		\$8,$T2,$T1		#
	vpsrldq		\$8,$T2,$T2
	vpxor		$T1,$Xi,$Xi		#
	vpxor		$T2,$Xhi,$Xhi

	vpsrlq		\$1,$Xi,$T2		# 2nd phase
	vpxor		$Xi,$Xhi,$Xhi
	vpxor		$T2,$Xi,$Xi		#
	vpsrlq		\$5,$T2,$T2
	vpxor		$T2,$Xi,$Xi		#
	vpsrlq		\$1,$Xi,$Xi		#
	vpxor		$Xhi,$Xi,$Xi		#
___
}

$code.=<<___;
.align	32
.Linit_loop_avx:
	vpalignr	\$8,$T1,$T2,$T3		# low part is H.lo^H.hi...
	vmovdqu		$T3,-0x10($Htbl)	# save Karatsuba "salt"
___
	&clmul64x64_avx	($Xhi,$Xi,$Hkey,$HK);	# calculate H^3,5,7
	&reduction_avx	($Xhi,$Xi);
$code.=<<___;
.Linit_start_avx:
	vmovdqa		$Xi,$T3
___
	&clmul64x64_avx	($Xhi,$Xi,$Hkey,$HK);	# calculate H^2,4,6,8
	&reduction_avx	($Xhi,$Xi);
$code.=<<___;
	vpshufd		\$0b01001110,$T3,$T1
	vpshufd		\$0b01001110,$Xi,$T2
	vpxor		$T3,$T1,$T1		# Karatsuba pre-processing
	vmovdqu		$T3,0x00($Htbl)		# save H^1,3,5,7
	vpxor		$Xi,$T2,$T2		# Karatsuba pre-processing
	vmovdqu		$Xi,0x10($Htbl)		# save H^2,4,6,8
	lea		0x30($Htbl),$Htbl
	sub		\$1,%r10
	jnz		.Linit_loop_avx

	vpalignr	\$8,$T2,$T1,$T3		# last "salt" is flipped
	vmovdqu		$T3,-0x10($Htbl)

	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	(%rsp),%xmm6
	lea	0x18(%rsp),%rsp
___
$code.=<<___;
	ret
.seh_endproc
.cfi_endproc
.size	gcm_init_avx,.-gcm_init_avx
___
} else {
$code.=<<___;
	jmp	.L_init_clmul
.cfi_endproc
.size	gcm_init_avx,.-gcm_init_avx
___
}

$code.=<<___;
.globl	gcm_gmult_avx
.type	gcm_gmult_avx,\@abi-omnipotent
.align	32
gcm_gmult_avx:
.cfi_startproc
	jmp	.L_gmult_clmul
.cfi_endproc
.size	gcm_gmult_avx,.-gcm_gmult_avx
___

$code.=<<___;
.globl	gcm_ghash_avx
.type	gcm_ghash_avx,\@abi-omnipotent
.align	32
gcm_ghash_avx:
.cfi_startproc
___
if ($avx) {
my ($Xip,$Htbl,$inp,$len)=@_4args;
my ($Xlo,$Xhi,$Xmi,
    $Zlo,$Zhi,$Zmi,
    $Hkey,$HK,$T1,$T2,
    $Xi,$Xo,$Tred,$bswap,$Ii,$Ij) = map("%xmm$_",(0..15));

$code.=<<___ if ($win64);
.seh_startproc
	lea	-0x88(%rsp),%rax
	lea	-0x20(%rax),%rsp
.seh_allocstack	0x20+0x88
	movaps	%xmm6,-0x20(%rax)
.seh_savexmm128	%xmm6, 0x20-0x20
	movaps	%xmm7,-0x10(%rax)
.seh_savexmm128	%xmm7, 0x20-0x10
	movaps	%xmm8,0(%rax)
.seh_savexmm128	%xmm8, 0x20+0
	movaps	%xmm9,0x10(%rax)
.seh_savexmm128	%xmm9, 0x20+0x10
	movaps	%xmm10,0x20(%rax)
.seh_savexmm128	%xmm10, 0x20+0x20
	movaps	%xmm11,0x30(%rax)
.seh_savexmm128	%xmm11, 0x20+0x30
	movaps	%xmm12,0x40(%rax)
.seh_savexmm128	%xmm12, 0x20+0x40
	movaps	%xmm13,0x50(%rax)
.seh_savexmm128	%xmm13, 0x20+0x50
	movaps	%xmm14,0x60(%rax)
.seh_savexmm128	%xmm14, 0x20+0x60
	movaps	%xmm15,0x70(%rax)
.seh_savexmm128	%xmm15, 0x20+0x70
___
$code.=<<___;
	vzeroupper

	vmovdqu		($Xip),$Xi		# load $Xi
	lea		.L0x1c2_polynomial(%rip),%r10
	lea		0x40($Htbl),$Htbl	# size optimization
	vmovdqu		.Lbswap_mask(%rip),$bswap
	vpshufb		$bswap,$Xi,$Xi
	cmp		\$0x80,$len
	jb		.Lshort_avx
	sub		\$0x80,$len

	vmovdqu		0x70($inp),$Ii		# I[7]
	vmovdqu		0x00-0x40($Htbl),$Hkey	# $Hkey^1
	vpshufb		$bswap,$Ii,$Ii
	vmovdqu		0x20-0x40($Htbl),$HK

	vpunpckhqdq	$Ii,$Ii,$T2
	 vmovdqu	0x60($inp),$Ij		# I[6]
	vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	vpxor		$Ii,$T2,$T2
	 vpshufb	$bswap,$Ij,$Ij
	vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	 vmovdqu	0x10-0x40($Htbl),$Hkey	# $Hkey^2
	 vpunpckhqdq	$Ij,$Ij,$T1
	 vmovdqu	0x50($inp),$Ii		# I[5]
	vpclmulqdq	\$0x00,$HK,$T2,$Xmi
	 vpxor		$Ij,$T1,$T1

	 vpshufb	$bswap,$Ii,$Ii
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Zlo
	 vpunpckhqdq	$Ii,$Ii,$T2
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Zhi
	 vmovdqu	0x30-0x40($Htbl),$Hkey	# $Hkey^3
	 vpxor		$Ii,$T2,$T2
	 vmovdqu	0x40($inp),$Ij		# I[4]
	vpclmulqdq	\$0x10,$HK,$T1,$Zmi
	 vmovdqu	0x50-0x40($Htbl),$HK

	 vpshufb	$bswap,$Ij,$Ij
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	vpxor		$Xhi,$Zhi,$Zhi
	 vpunpckhqdq	$Ij,$Ij,$T1
	vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	 vmovdqu	0x40-0x40($Htbl),$Hkey	# $Hkey^4
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T2,$Xmi
	 vpxor		$Ij,$T1,$T1

	 vmovdqu	0x30($inp),$Ii		# I[3]
	vpxor		$Zlo,$Xlo,$Xlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Zlo
	vpxor		$Zhi,$Xhi,$Xhi
	 vpshufb	$bswap,$Ii,$Ii
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Zhi
	 vmovdqu	0x60-0x40($Htbl),$Hkey	# $Hkey^5
	vpxor		$Zmi,$Xmi,$Xmi
	 vpunpckhqdq	$Ii,$Ii,$T2
	vpclmulqdq	\$0x10,$HK,$T1,$Zmi
	 vmovdqu	0x80-0x40($Htbl),$HK
	 vpxor		$Ii,$T2,$T2

	 vmovdqu	0x20($inp),$Ij		# I[2]
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	vpxor		$Xhi,$Zhi,$Zhi
	 vpshufb	$bswap,$Ij,$Ij
	vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	 vmovdqu	0x70-0x40($Htbl),$Hkey	# $Hkey^6
	vpxor		$Xmi,$Zmi,$Zmi
	 vpunpckhqdq	$Ij,$Ij,$T1
	vpclmulqdq	\$0x00,$HK,$T2,$Xmi
	 vpxor		$Ij,$T1,$T1

	 vmovdqu	0x10($inp),$Ii		# I[1]
	vpxor		$Zlo,$Xlo,$Xlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Zlo
	vpxor		$Zhi,$Xhi,$Xhi
	 vpshufb	$bswap,$Ii,$Ii
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Zhi
	 vmovdqu	0x90-0x40($Htbl),$Hkey	# $Hkey^7
	vpxor		$Zmi,$Xmi,$Xmi
	 vpunpckhqdq	$Ii,$Ii,$T2
	vpclmulqdq	\$0x10,$HK,$T1,$Zmi
	 vmovdqu	0xb0-0x40($Htbl),$HK
	 vpxor		$Ii,$T2,$T2

	 vmovdqu	($inp),$Ij		# I[0]
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	vpxor		$Xhi,$Zhi,$Zhi
	 vpshufb	$bswap,$Ij,$Ij
	vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	 vmovdqu	0xa0-0x40($Htbl),$Hkey	# $Hkey^8
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x10,$HK,$T2,$Xmi

	lea		0x80($inp),$inp
	cmp		\$0x80,$len
	jb		.Ltail_avx

	vpxor		$Xi,$Ij,$Ij		# accumulate $Xi
	sub		\$0x80,$len
	jmp		.Loop8x_avx

.align	32
.Loop8x_avx:
	vpunpckhqdq	$Ij,$Ij,$T1
	 vmovdqu	0x70($inp),$Ii		# I[7]
	vpxor		$Xlo,$Zlo,$Zlo
	vpxor		$Ij,$T1,$T1
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xi
	 vpshufb	$bswap,$Ii,$Ii
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xo
	 vmovdqu	0x00-0x40($Htbl),$Hkey	# $Hkey^1
	 vpunpckhqdq	$Ii,$Ii,$T2
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Tred
	 vmovdqu	0x20-0x40($Htbl),$HK
	 vpxor		$Ii,$T2,$T2

	  vmovdqu	0x60($inp),$Ij		# I[6]
	 vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	vpxor		$Zlo,$Xi,$Xi		# collect result
	  vpshufb	$bswap,$Ij,$Ij
	 vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	vxorps		$Zhi,$Xo,$Xo
	  vmovdqu	0x10-0x40($Htbl),$Hkey	# $Hkey^2
	 vpunpckhqdq	$Ij,$Ij,$T1
	 vpclmulqdq	\$0x00,$HK,  $T2,$Xmi
	vpxor		$Zmi,$Tred,$Tred
	 vxorps		$Ij,$T1,$T1

	  vmovdqu	0x50($inp),$Ii		# I[5]
	vpxor		$Xi,$Tred,$Tred		# aggregated Karatsuba post-processing
	 vpclmulqdq	\$0x00,$Hkey,$Ij,$Zlo
	vpxor		$Xo,$Tred,$Tred
	vpslldq		\$8,$Tred,$T2
	 vpxor		$Xlo,$Zlo,$Zlo
	 vpclmulqdq	\$0x11,$Hkey,$Ij,$Zhi
	vpsrldq		\$8,$Tred,$Tred
	vpxor		$T2, $Xi, $Xi
	  vmovdqu	0x30-0x40($Htbl),$Hkey	# $Hkey^3
	  vpshufb	$bswap,$Ii,$Ii
	vxorps		$Tred,$Xo, $Xo
	 vpxor		$Xhi,$Zhi,$Zhi
	 vpunpckhqdq	$Ii,$Ii,$T2
	 vpclmulqdq	\$0x10,$HK,  $T1,$Zmi
	  vmovdqu	0x50-0x40($Htbl),$HK
	 vpxor		$Ii,$T2,$T2
	 vpxor		$Xmi,$Zmi,$Zmi

	  vmovdqu	0x40($inp),$Ij		# I[4]
	vpalignr	\$8,$Xi,$Xi,$Tred	# 1st phase
	 vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	  vpshufb	$bswap,$Ij,$Ij
	 vpxor		$Zlo,$Xlo,$Xlo
	 vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	  vmovdqu	0x40-0x40($Htbl),$Hkey	# $Hkey^4
	 vpunpckhqdq	$Ij,$Ij,$T1
	 vpxor		$Zhi,$Xhi,$Xhi
	 vpclmulqdq	\$0x00,$HK,  $T2,$Xmi
	 vxorps		$Ij,$T1,$T1
	 vpxor		$Zmi,$Xmi,$Xmi

	  vmovdqu	0x30($inp),$Ii		# I[3]
	vpclmulqdq	\$0x10,(%r10),$Xi,$Xi
	 vpclmulqdq	\$0x00,$Hkey,$Ij,$Zlo
	  vpshufb	$bswap,$Ii,$Ii
	 vpxor		$Xlo,$Zlo,$Zlo
	 vpclmulqdq	\$0x11,$Hkey,$Ij,$Zhi
	  vmovdqu	0x60-0x40($Htbl),$Hkey	# $Hkey^5
	 vpunpckhqdq	$Ii,$Ii,$T2
	 vpxor		$Xhi,$Zhi,$Zhi
	 vpclmulqdq	\$0x10,$HK,  $T1,$Zmi
	  vmovdqu	0x80-0x40($Htbl),$HK
	 vpxor		$Ii,$T2,$T2
	 vpxor		$Xmi,$Zmi,$Zmi

	  vmovdqu	0x20($inp),$Ij		# I[2]
	 vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	  vpshufb	$bswap,$Ij,$Ij
	 vpxor		$Zlo,$Xlo,$Xlo
	 vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	  vmovdqu	0x70-0x40($Htbl),$Hkey	# $Hkey^6
	 vpunpckhqdq	$Ij,$Ij,$T1
	 vpxor		$Zhi,$Xhi,$Xhi
	 vpclmulqdq	\$0x00,$HK,  $T2,$Xmi
	 vpxor		$Ij,$T1,$T1
	 vpxor		$Zmi,$Xmi,$Xmi
	vxorps		$Tred,$Xi,$Xi

	  vmovdqu	0x10($inp),$Ii		# I[1]
	vpalignr	\$8,$Xi,$Xi,$Tred	# 2nd phase
	 vpclmulqdq	\$0x00,$Hkey,$Ij,$Zlo
	  vpshufb	$bswap,$Ii,$Ii
	 vpxor		$Xlo,$Zlo,$Zlo
	 vpclmulqdq	\$0x11,$Hkey,$Ij,$Zhi
	  vmovdqu	0x90-0x40($Htbl),$Hkey	# $Hkey^7
	vpclmulqdq	\$0x10,(%r10),$Xi,$Xi
	vxorps		$Xo,$Tred,$Tred
	 vpunpckhqdq	$Ii,$Ii,$T2
	 vpxor		$Xhi,$Zhi,$Zhi
	 vpclmulqdq	\$0x10,$HK,  $T1,$Zmi
	  vmovdqu	0xb0-0x40($Htbl),$HK
	 vpxor		$Ii,$T2,$T2
	 vpxor		$Xmi,$Zmi,$Zmi

	  vmovdqu	($inp),$Ij		# I[0]
	 vpclmulqdq	\$0x00,$Hkey,$Ii,$Xlo
	  vpshufb	$bswap,$Ij,$Ij
	 vpclmulqdq	\$0x11,$Hkey,$Ii,$Xhi
	  vmovdqu	0xa0-0x40($Htbl),$Hkey	# $Hkey^8
	vpxor		$Tred,$Ij,$Ij
	 vpclmulqdq	\$0x10,$HK,  $T2,$Xmi
	vpxor		$Xi,$Ij,$Ij		# accumulate $Xi

	lea		0x80($inp),$inp
	sub		\$0x80,$len
	jnc		.Loop8x_avx

	add		\$0x80,$len
	jmp		.Ltail_no_xor_avx

.align	32
.Lshort_avx:
	vmovdqu		-0x10($inp,$len),$Ii	# very last word
	lea		($inp,$len),$inp
	vmovdqu		0x00-0x40($Htbl),$Hkey	# $Hkey^1
	vmovdqu		0x20-0x40($Htbl),$HK
	vpshufb		$bswap,$Ii,$Ij

	vmovdqa		$Xlo,$Zlo		# subtle way to zero $Zlo,
	vmovdqa		$Xhi,$Zhi		# $Zhi and
	vmovdqa		$Xmi,$Zmi		# $Zmi
	sub		\$0x10,$len
	jz		.Ltail_avx

	vpunpckhqdq	$Ij,$Ij,$T1
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xlo
	vpxor		$Ij,$T1,$T1
	 vmovdqu	-0x20($inp),$Ii
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xhi
	vmovdqu		0x10-0x40($Htbl),$Hkey	# $Hkey^2
	 vpshufb	$bswap,$Ii,$Ij
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Xmi
	vpsrldq		\$8,$HK,$HK
	sub		\$0x10,$len
	jz		.Ltail_avx

	vpunpckhqdq	$Ij,$Ij,$T1
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xlo
	vpxor		$Ij,$T1,$T1
	 vmovdqu	-0x30($inp),$Ii
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xhi
	vmovdqu		0x30-0x40($Htbl),$Hkey	# $Hkey^3
	 vpshufb	$bswap,$Ii,$Ij
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Xmi
	vmovdqu		0x50-0x40($Htbl),$HK
	sub		\$0x10,$len
	jz		.Ltail_avx

	vpunpckhqdq	$Ij,$Ij,$T1
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xlo
	vpxor		$Ij,$T1,$T1
	 vmovdqu	-0x40($inp),$Ii
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xhi
	vmovdqu		0x40-0x40($Htbl),$Hkey	# $Hkey^4
	 vpshufb	$bswap,$Ii,$Ij
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Xmi
	vpsrldq		\$8,$HK,$HK
	sub		\$0x10,$len
	jz		.Ltail_avx

	vpunpckhqdq	$Ij,$Ij,$T1
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xlo
	vpxor		$Ij,$T1,$T1
	 vmovdqu	-0x50($inp),$Ii
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xhi
	vmovdqu		0x60-0x40($Htbl),$Hkey	# $Hkey^5
	 vpshufb	$bswap,$Ii,$Ij
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Xmi
	vmovdqu		0x80-0x40($Htbl),$HK
	sub		\$0x10,$len
	jz		.Ltail_avx

	vpunpckhqdq	$Ij,$Ij,$T1
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xlo
	vpxor		$Ij,$T1,$T1
	 vmovdqu	-0x60($inp),$Ii
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xhi
	vmovdqu		0x70-0x40($Htbl),$Hkey	# $Hkey^6
	 vpshufb	$bswap,$Ii,$Ij
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Xmi
	vpsrldq		\$8,$HK,$HK
	sub		\$0x10,$len
	jz		.Ltail_avx

	vpunpckhqdq	$Ij,$Ij,$T1
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xlo
	vpxor		$Ij,$T1,$T1
	 vmovdqu	-0x70($inp),$Ii
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xhi
	vmovdqu		0x90-0x40($Htbl),$Hkey	# $Hkey^7
	 vpshufb	$bswap,$Ii,$Ij
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Xmi
	vmovq		0xb8-0x40($Htbl),$HK
	sub		\$0x10,$len
	jmp		.Ltail_avx

.align	32
.Ltail_avx:
	vpxor		$Xi,$Ij,$Ij		# accumulate $Xi
.Ltail_no_xor_avx:
	vpunpckhqdq	$Ij,$Ij,$T1
	vpxor		$Xlo,$Zlo,$Zlo
	vpclmulqdq	\$0x00,$Hkey,$Ij,$Xlo
	vpxor		$Ij,$T1,$T1
	vpxor		$Xhi,$Zhi,$Zhi
	vpclmulqdq	\$0x11,$Hkey,$Ij,$Xhi
	vpxor		$Xmi,$Zmi,$Zmi
	vpclmulqdq	\$0x00,$HK,$T1,$Xmi

	vmovdqu		(%r10),$Tred

	vpxor		$Xlo,$Zlo,$Xi
	vpxor		$Xhi,$Zhi,$Xo
	vpxor		$Xmi,$Zmi,$Zmi

	vpxor		$Xi, $Zmi,$Zmi		# aggregated Karatsuba post-processing
	vpxor		$Xo, $Zmi,$Zmi
	vpslldq		\$8, $Zmi,$T2
	vpsrldq		\$8, $Zmi,$Zmi
	vpxor		$T2, $Xi, $Xi
	vpxor		$Zmi,$Xo, $Xo

	vpclmulqdq	\$0x10,$Tred,$Xi,$T2	# 1st phase
	vpalignr	\$8,$Xi,$Xi,$Xi
	vpxor		$T2,$Xi,$Xi

	vpclmulqdq	\$0x10,$Tred,$Xi,$T2	# 2nd phase
	vpalignr	\$8,$Xi,$Xi,$Xi
	vpxor		$Xo,$Xi,$Xi
	vpxor		$T2,$Xi,$Xi

	cmp		\$0,$len
	jne		.Lshort_avx

	vpshufb		$bswap,$Xi,$Xi
	vmovdqu		$Xi,($Xip)
	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	(%rsp),%xmm6
	movaps	0x10(%rsp),%xmm7
	movaps	0x20(%rsp),%xmm8
	movaps	0x30(%rsp),%xmm9
	movaps	0x40(%rsp),%xmm10
	movaps	0x50(%rsp),%xmm11
	movaps	0x60(%rsp),%xmm12
	movaps	0x70(%rsp),%xmm13
	movaps	0x80(%rsp),%xmm14
	movaps	0x90(%rsp),%xmm15
	lea	0xa8(%rsp),%rsp
___
$code.=<<___;
	ret
.cfi_endproc
.seh_endproc
.size	gcm_ghash_avx,.-gcm_ghash_avx
___
} else {
$code.=<<___;
	jmp	.L_ghash_clmul
.cfi_endproc
.size	gcm_ghash_avx,.-gcm_ghash_avx
___
}

$code.=<<___;
.section .rodata
.align	64
.Lbswap_mask:
	.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
.L0x1c2_polynomial:
	.byte	1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
.L7_mask:
	.long	7,0,7,0
.align	64

.asciz	"GHASH for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
.align	64
.text
___

$code =~ s/\`([^\`]*)\`/eval($1)/gem;

print $code;

close STDOUT or die "error closing STDOUT: $!";