#! /usr/bin/env perl
# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. Rights for redistribution and usage in source and binary
# forms are granted according to the OpenSSL license.
# ====================================================================
#
# Version 1.1
#
# The major reason for undertaken effort was to mitigate the hazard of
# cache-timing attack. This is [currently and initially!] addressed in
# two ways. 1. S-boxes are compressed from 5KB to 2KB+256B size each.
# 2. References to them are scheduled for L2 cache latency, meaning
# that the tables don't have to reside in L1 cache. Once again, this
# is an initial draft and one should expect more countermeasures to
# be implemented...
#
# Version 1.1 prefetches T[ed]4 in order to mitigate attack on last
# round.
#
# Even though performance was not the primary goal [on the contrary,
# extra shifts "induced" by compressed S-box and longer loop epilogue
# "induced" by scheduling for L2 have negative effect on performance],
# the code turned out to run in ~23 cycles per processed byte en-/
# decrypted with 128-bit key. This is pretty good result for code
# with mentioned qualities and UltraSPARC core. Compared to Sun C
# generated code my encrypt procedure runs just few percents faster,
# while decrypt one - whole 50% faster [yes, Sun C failed to generate
# optimal decrypt procedure]. Compared to GNU C generated code both
# procedures are more than 60% faster:-)

$output = pop;
open STDOUT,">$output";

$frame="STACK_FRAME";
$bias="STACK_BIAS";
$locals=16;

$acc0="%l0";
$acc1="%o0";
$acc2="%o1";
$acc3="%o2";

$acc4="%l1";
$acc5="%o3";
$acc6="%o4";
$acc7="%o5";

$acc8="%l2";
$acc9="%o7";
$acc10="%g1";
$acc11="%g2";

$acc12="%l3";
$acc13="%g3";
$acc14="%g4";
$acc15="%g5";

$t0="%l4";
$t1="%l5";
$t2="%l6";
$t3="%l7";

$s0="%i0";
$s1="%i1";
$s2="%i2";
$s3="%i3";
$tbl="%i4";
$key="%i5";
$rounds="%i7";	# aliases with return address, which is off-loaded to stack

sub _data_word()
{ my $i;
    while(defined($i=shift)) { $code.=sprintf"\t.long\t0x%08x,0x%08x\n",$i,$i; }
}

$code.=<<___;
#include "sparc_arch.h"

#ifdef  __arch64__
.register	%g2,#scratch
.register	%g3,#scratch
#endif
.section	".text",#alloc,#execinstr

.align	256
AES_Te:
___
&_data_word(
	0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d,
	0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554,
	0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d,
	0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a,
	0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87,
	0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b,
	0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea,
	0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b,
	0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a,
	0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f,
	0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108,
	0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f,
	0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e,
	0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5,
	0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d,
	0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f,
	0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e,
	0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb,
	0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce,
	0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497,
	0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c,
	0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed,
	0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b,
	0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a,
	0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16,
	0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594,
	0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81,
	0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3,
	0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a,
	0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504,
	0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163,
	0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d,
	0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f,
	0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739,
	0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47,
	0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395,
	0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f,
	0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883,
	0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c,
	0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76,
	0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e,
	0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4,
	0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6,
	0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b,
	0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7,
	0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0,
	0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25,
	0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818,
	0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72,
	0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651,
	0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21,
	0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85,
	0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa,
	0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12,
	0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0,
	0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9,
	0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133,
	0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7,
	0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920,
	0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a,
	0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17,
	0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8,
	0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11,
	0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a);
$code.=<<___;
	.byte	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
	.byte	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
	.byte	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0
	.byte	0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0
	.byte	0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc
	.byte	0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15
	.byte	0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a
	.byte	0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75
	.byte	0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0
	.byte	0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84
	.byte	0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b
	.byte	0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf
	.byte	0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85
	.byte	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8
	.byte	0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5
	.byte	0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2
	.byte	0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17
	.byte	0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73
	.byte	0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88
	.byte	0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb
	.byte	0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c
	.byte	0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79
	.byte	0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9
	.byte	0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08
	.byte	0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6
	.byte	0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a
	.byte	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e
	.byte	0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e
	.byte	0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94
	.byte	0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf
	.byte	0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68
	.byte	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
.type	AES_Te,#object
.size	AES_Te,(.-AES_Te)

.align	64
.skip	16
_sparcv9_AES_encrypt:
	save	%sp,-$frame-$locals,%sp
	stx	%i7,[%sp+$bias+$frame+0]	! off-load return address
	ld	[$key+240],$rounds
	ld	[$key+0],$t0
	ld	[$key+4],$t1			!
	ld	[$key+8],$t2
	srl	$rounds,1,$rounds
	xor	$t0,$s0,$s0
	ld	[$key+12],$t3
	srl	$s0,21,$acc0
	xor	$t1,$s1,$s1
	ld	[$key+16],$t0
	srl	$s1,13,$acc1			!
	xor	$t2,$s2,$s2
	ld	[$key+20],$t1
	xor	$t3,$s3,$s3
	ld	[$key+24],$t2
	and	$acc0,2040,$acc0
	ld	[$key+28],$t3
	nop
.Lenc_loop:
	srl	$s2,5,$acc2			!
	and	$acc1,2040,$acc1
	ldx	[$tbl+$acc0],$acc0
	sll	$s3,3,$acc3
	and	$acc2,2040,$acc2
	ldx	[$tbl+$acc1],$acc1
	srl	$s1,21,$acc4
	and	$acc3,2040,$acc3
	ldx	[$tbl+$acc2],$acc2		!
	srl	$s2,13,$acc5
	and	$acc4,2040,$acc4
	ldx	[$tbl+$acc3],$acc3
	srl	$s3,5,$acc6
	and	$acc5,2040,$acc5
	ldx	[$tbl+$acc4],$acc4
	fmovs	%f0,%f0
	sll	$s0,3,$acc7			!
	and	$acc6,2040,$acc6
	ldx	[$tbl+$acc5],$acc5
	srl	$s2,21,$acc8
	and	$acc7,2040,$acc7
	ldx	[$tbl+$acc6],$acc6
	srl	$s3,13,$acc9
	and	$acc8,2040,$acc8
	ldx	[$tbl+$acc7],$acc7		!
	srl	$s0,5,$acc10
	and	$acc9,2040,$acc9
	ldx	[$tbl+$acc8],$acc8
	sll	$s1,3,$acc11
	and	$acc10,2040,$acc10
	ldx	[$tbl+$acc9],$acc9
	fmovs	%f0,%f0
	srl	$s3,21,$acc12			!
	and	$acc11,2040,$acc11
	ldx	[$tbl+$acc10],$acc10
	srl	$s0,13,$acc13
	and	$acc12,2040,$acc12
	ldx	[$tbl+$acc11],$acc11
	srl	$s1,5,$acc14
	and	$acc13,2040,$acc13
	ldx	[$tbl+$acc12],$acc12		!
	sll	$s2,3,$acc15
	and	$acc14,2040,$acc14
	ldx	[$tbl+$acc13],$acc13
	and	$acc15,2040,$acc15
	add	$key,32,$key
	ldx	[$tbl+$acc14],$acc14
	fmovs	%f0,%f0
	subcc	$rounds,1,$rounds		!
	ldx	[$tbl+$acc15],$acc15
	bz,a,pn	%icc,.Lenc_last
	add	$tbl,2048,$rounds

		srlx	$acc1,8,$acc1
		xor	$acc0,$t0,$t0
	ld	[$key+0],$s0
	fmovs	%f0,%f0
		srlx	$acc2,16,$acc2		!
		xor	$acc1,$t0,$t0
	ld	[$key+4],$s1
		srlx	$acc3,24,$acc3
		xor	$acc2,$t0,$t0
	ld	[$key+8],$s2
		srlx	$acc5,8,$acc5
		xor	$acc3,$t0,$t0
	ld	[$key+12],$s3			!
		srlx	$acc6,16,$acc6
		xor	$acc4,$t1,$t1
	fmovs	%f0,%f0
		srlx	$acc7,24,$acc7
		xor	$acc5,$t1,$t1
		srlx	$acc9,8,$acc9
		xor	$acc6,$t1,$t1
		srlx	$acc10,16,$acc10	!
		xor	$acc7,$t1,$t1
		srlx	$acc11,24,$acc11
		xor	$acc8,$t2,$t2
		srlx	$acc13,8,$acc13
		xor	$acc9,$t2,$t2
		srlx	$acc14,16,$acc14
		xor	$acc10,$t2,$t2
		srlx	$acc15,24,$acc15	!
		xor	$acc11,$t2,$t2
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$t3,$t3
	srl	$t0,21,$acc0
		xor	$acc14,$t3,$t3
	srl	$t1,13,$acc1
		xor	$acc15,$t3,$t3

	and	$acc0,2040,$acc0		!
	srl	$t2,5,$acc2
	and	$acc1,2040,$acc1
	ldx	[$tbl+$acc0],$acc0
	sll	$t3,3,$acc3
	and	$acc2,2040,$acc2
	ldx	[$tbl+$acc1],$acc1
	fmovs	%f0,%f0
	srl	$t1,21,$acc4			!
	and	$acc3,2040,$acc3
	ldx	[$tbl+$acc2],$acc2
	srl	$t2,13,$acc5
	and	$acc4,2040,$acc4
	ldx	[$tbl+$acc3],$acc3
	srl	$t3,5,$acc6
	and	$acc5,2040,$acc5
	ldx	[$tbl+$acc4],$acc4		!
	sll	$t0,3,$acc7
	and	$acc6,2040,$acc6
	ldx	[$tbl+$acc5],$acc5
	srl	$t2,21,$acc8
	and	$acc7,2040,$acc7
	ldx	[$tbl+$acc6],$acc6
	fmovs	%f0,%f0
	srl	$t3,13,$acc9			!
	and	$acc8,2040,$acc8
	ldx	[$tbl+$acc7],$acc7
	srl	$t0,5,$acc10
	and	$acc9,2040,$acc9
	ldx	[$tbl+$acc8],$acc8
	sll	$t1,3,$acc11
	and	$acc10,2040,$acc10
	ldx	[$tbl+$acc9],$acc9		!
	srl	$t3,21,$acc12
	and	$acc11,2040,$acc11
	ldx	[$tbl+$acc10],$acc10
	srl	$t0,13,$acc13
	and	$acc12,2040,$acc12
	ldx	[$tbl+$acc11],$acc11
	fmovs	%f0,%f0
	srl	$t1,5,$acc14			!
	and	$acc13,2040,$acc13
	ldx	[$tbl+$acc12],$acc12
	sll	$t2,3,$acc15
	and	$acc14,2040,$acc14
	ldx	[$tbl+$acc13],$acc13
		srlx	$acc1,8,$acc1
	and	$acc15,2040,$acc15
	ldx	[$tbl+$acc14],$acc14		!

		srlx	$acc2,16,$acc2
		xor	$acc0,$s0,$s0
	ldx	[$tbl+$acc15],$acc15
		srlx	$acc3,24,$acc3
		xor	$acc1,$s0,$s0
	ld	[$key+16],$t0
	fmovs	%f0,%f0
		srlx	$acc5,8,$acc5		!
		xor	$acc2,$s0,$s0
	ld	[$key+20],$t1
		srlx	$acc6,16,$acc6
		xor	$acc3,$s0,$s0
	ld	[$key+24],$t2
		srlx	$acc7,24,$acc7
		xor	$acc4,$s1,$s1
	ld	[$key+28],$t3			!
		srlx	$acc9,8,$acc9
		xor	$acc5,$s1,$s1
	ldx	[$tbl+2048+0],%g0		! prefetch te4
		srlx	$acc10,16,$acc10
		xor	$acc6,$s1,$s1
	ldx	[$tbl+2048+32],%g0		! prefetch te4
		srlx	$acc11,24,$acc11
		xor	$acc7,$s1,$s1
	ldx	[$tbl+2048+64],%g0		! prefetch te4
		srlx	$acc13,8,$acc13
		xor	$acc8,$s2,$s2
	ldx	[$tbl+2048+96],%g0		! prefetch te4
		srlx	$acc14,16,$acc14	!
		xor	$acc9,$s2,$s2
	ldx	[$tbl+2048+128],%g0		! prefetch te4
		srlx	$acc15,24,$acc15
		xor	$acc10,$s2,$s2
	ldx	[$tbl+2048+160],%g0		! prefetch te4
	srl	$s0,21,$acc0
		xor	$acc11,$s2,$s2
	ldx	[$tbl+2048+192],%g0		! prefetch te4
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$s3,$s3
	ldx	[$tbl+2048+224],%g0		! prefetch te4
	srl	$s1,13,$acc1			!
		xor	$acc14,$s3,$s3
		xor	$acc15,$s3,$s3
	ba	.Lenc_loop
	and	$acc0,2040,$acc0

.align	32
.Lenc_last:
		srlx	$acc1,8,$acc1		!
		xor	$acc0,$t0,$t0
	ld	[$key+0],$s0
		srlx	$acc2,16,$acc2
		xor	$acc1,$t0,$t0
	ld	[$key+4],$s1
		srlx	$acc3,24,$acc3
		xor	$acc2,$t0,$t0
	ld	[$key+8],$s2			!
		srlx	$acc5,8,$acc5
		xor	$acc3,$t0,$t0
	ld	[$key+12],$s3
		srlx	$acc6,16,$acc6
		xor	$acc4,$t1,$t1
		srlx	$acc7,24,$acc7
		xor	$acc5,$t1,$t1
		srlx	$acc9,8,$acc9		!
		xor	$acc6,$t1,$t1
		srlx	$acc10,16,$acc10
		xor	$acc7,$t1,$t1
		srlx	$acc11,24,$acc11
		xor	$acc8,$t2,$t2
		srlx	$acc13,8,$acc13
		xor	$acc9,$t2,$t2
		srlx	$acc14,16,$acc14	!
		xor	$acc10,$t2,$t2
		srlx	$acc15,24,$acc15
		xor	$acc11,$t2,$t2
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$t3,$t3
	srl	$t0,24,$acc0
		xor	$acc14,$t3,$t3
	srl	$t1,16,$acc1			!
		xor	$acc15,$t3,$t3

	srl	$t2,8,$acc2
	and	$acc1,255,$acc1
	ldub	[$rounds+$acc0],$acc0
	srl	$t1,24,$acc4
	and	$acc2,255,$acc2
	ldub	[$rounds+$acc1],$acc1
	srl	$t2,16,$acc5			!
	and	$t3,255,$acc3
	ldub	[$rounds+$acc2],$acc2
	ldub	[$rounds+$acc3],$acc3
	srl	$t3,8,$acc6
	and	$acc5,255,$acc5
	ldub	[$rounds+$acc4],$acc4
	fmovs	%f0,%f0
	srl	$t2,24,$acc8			!
	and	$acc6,255,$acc6
	ldub	[$rounds+$acc5],$acc5
	srl	$t3,16,$acc9
	and	$t0,255,$acc7
	ldub	[$rounds+$acc6],$acc6
	ldub	[$rounds+$acc7],$acc7
	fmovs	%f0,%f0
	srl	$t0,8,$acc10			!
	and	$acc9,255,$acc9
	ldub	[$rounds+$acc8],$acc8
	srl	$t3,24,$acc12
	and	$acc10,255,$acc10
	ldub	[$rounds+$acc9],$acc9
	srl	$t0,16,$acc13
	and	$t1,255,$acc11
	ldub	[$rounds+$acc10],$acc10		!
	srl	$t1,8,$acc14
	and	$acc13,255,$acc13
	ldub	[$rounds+$acc11],$acc11
	ldub	[$rounds+$acc12],$acc12
	and	$acc14,255,$acc14
	ldub	[$rounds+$acc13],$acc13
	and	$t2,255,$acc15
	ldub	[$rounds+$acc14],$acc14		!

		sll	$acc0,24,$acc0
		xor	$acc3,$s0,$s0
	ldub	[$rounds+$acc15],$acc15
		sll	$acc1,16,$acc1
		xor	$acc0,$s0,$s0
	ldx	[%sp+$bias+$frame+0],%i7	! restore return address
	fmovs	%f0,%f0
		sll	$acc2,8,$acc2		!
		xor	$acc1,$s0,$s0
		sll	$acc4,24,$acc4
		xor	$acc2,$s0,$s0
		sll	$acc5,16,$acc5
		xor	$acc7,$s1,$s1
		sll	$acc6,8,$acc6
		xor	$acc4,$s1,$s1
		sll	$acc8,24,$acc8		!
		xor	$acc5,$s1,$s1
		sll	$acc9,16,$acc9
		xor	$acc11,$s2,$s2
		sll	$acc10,8,$acc10
		xor	$acc6,$s1,$s1
		sll	$acc12,24,$acc12
		xor	$acc8,$s2,$s2
		sll	$acc13,16,$acc13	!
		xor	$acc9,$s2,$s2
		sll	$acc14,8,$acc14
		xor	$acc10,$s2,$s2
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$s3,$s3
		xor	$acc14,$s3,$s3
		xor	$acc15,$s3,$s3

	ret
	restore
.type	_sparcv9_AES_encrypt,#function
.size	_sparcv9_AES_encrypt,(.-_sparcv9_AES_encrypt)

.align	32
.globl	AES_encrypt
AES_encrypt:
	or	%o0,%o1,%g1
	andcc	%g1,3,%g0
	bnz,pn	%xcc,.Lunaligned_enc
	save	%sp,-$frame,%sp

	ld	[%i0+0],%o0
	ld	[%i0+4],%o1
	ld	[%i0+8],%o2
	ld	[%i0+12],%o3

1:	call	.+8
	add	%o7,AES_Te-1b,%o4
	call	_sparcv9_AES_encrypt
	mov	%i2,%o5

	st	%o0,[%i1+0]
	st	%o1,[%i1+4]
	st	%o2,[%i1+8]
	st	%o3,[%i1+12]

	ret
	restore

.align	32
.Lunaligned_enc:
	ldub	[%i0+0],%l0
	ldub	[%i0+1],%l1
	ldub	[%i0+2],%l2

	sll	%l0,24,%l0
	ldub	[%i0+3],%l3
	sll	%l1,16,%l1
	ldub	[%i0+4],%l4
	sll	%l2,8,%l2
	or	%l1,%l0,%l0
	ldub	[%i0+5],%l5
	sll	%l4,24,%l4
	or	%l3,%l2,%l2
	ldub	[%i0+6],%l6
	sll	%l5,16,%l5
	or	%l0,%l2,%o0
	ldub	[%i0+7],%l7

	sll	%l6,8,%l6
	or	%l5,%l4,%l4
	ldub	[%i0+8],%l0
	or	%l7,%l6,%l6
	ldub	[%i0+9],%l1
	or	%l4,%l6,%o1
	ldub	[%i0+10],%l2

	sll	%l0,24,%l0
	ldub	[%i0+11],%l3
	sll	%l1,16,%l1
	ldub	[%i0+12],%l4
	sll	%l2,8,%l2
	or	%l1,%l0,%l0
	ldub	[%i0+13],%l5
	sll	%l4,24,%l4
	or	%l3,%l2,%l2
	ldub	[%i0+14],%l6
	sll	%l5,16,%l5
	or	%l0,%l2,%o2
	ldub	[%i0+15],%l7

	sll	%l6,8,%l6
	or	%l5,%l4,%l4
	or	%l7,%l6,%l6
	or	%l4,%l6,%o3

1:	call	.+8
	add	%o7,AES_Te-1b,%o4
	call	_sparcv9_AES_encrypt
	mov	%i2,%o5

	srl	%o0,24,%l0
	srl	%o0,16,%l1
	stb	%l0,[%i1+0]
	srl	%o0,8,%l2
	stb	%l1,[%i1+1]
	stb	%l2,[%i1+2]
	srl	%o1,24,%l4
	stb	%o0,[%i1+3]

	srl	%o1,16,%l5
	stb	%l4,[%i1+4]
	srl	%o1,8,%l6
	stb	%l5,[%i1+5]
	stb	%l6,[%i1+6]
	srl	%o2,24,%l0
	stb	%o1,[%i1+7]

	srl	%o2,16,%l1
	stb	%l0,[%i1+8]
	srl	%o2,8,%l2
	stb	%l1,[%i1+9]
	stb	%l2,[%i1+10]
	srl	%o3,24,%l4
	stb	%o2,[%i1+11]

	srl	%o3,16,%l5
	stb	%l4,[%i1+12]
	srl	%o3,8,%l6
	stb	%l5,[%i1+13]
	stb	%l6,[%i1+14]
	stb	%o3,[%i1+15]

	ret
	restore
.type	AES_encrypt,#function
.size	AES_encrypt,(.-AES_encrypt)

___

$code.=<<___;
.align	256
AES_Td:
___
&_data_word(
	0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96,
	0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393,
	0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25,
	0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f,
	0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1,
	0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6,
	0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da,
	0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844,
	0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd,
	0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4,
	0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45,
	0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94,
	0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7,
	0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a,
	0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5,
	0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c,
	0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1,
	0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a,
	0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75,
	0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051,
	0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46,
	0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff,
	0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77,
	0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb,
	0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000,
	0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e,
	0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927,
	0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a,
	0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e,
	0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16,
	0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d,
	0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8,
	0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd,
	0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34,
	0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163,
	0xd731dcca, 0x42638510, 0x13972240, 0x84c61120,
	0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d,
	0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0,
	0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422,
	0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef,
	0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36,
	0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4,
	0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662,
	0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5,
	0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3,
	0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b,
	0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8,
	0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6,
	0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6,
	0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0,
	0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815,
	0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f,
	0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df,
	0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f,
	0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e,
	0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713,
	0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89,
	0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c,
	0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf,
	0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86,
	0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f,
	0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541,
	0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190,
	0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742);
$code.=<<___;
	.byte	0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
	.byte	0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
	.byte	0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
	.byte	0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
	.byte	0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
	.byte	0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
	.byte	0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
	.byte	0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
	.byte	0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
	.byte	0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
	.byte	0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
	.byte	0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
	.byte	0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
	.byte	0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
	.byte	0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
	.byte	0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
	.byte	0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
	.byte	0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
	.byte	0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
	.byte	0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
	.byte	0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
	.byte	0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
	.byte	0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
	.byte	0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
	.byte	0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
	.byte	0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
	.byte	0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
	.byte	0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
	.byte	0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
	.byte	0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
	.byte	0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
	.byte	0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
.type	AES_Td,#object
.size	AES_Td,(.-AES_Td)

.align	64
.skip	16
_sparcv9_AES_decrypt:
	save	%sp,-$frame-$locals,%sp
	stx	%i7,[%sp+$bias+$frame+0]	! off-load return address
	ld	[$key+240],$rounds
	ld	[$key+0],$t0
	ld	[$key+4],$t1			!
	ld	[$key+8],$t2
	ld	[$key+12],$t3
	srl	$rounds,1,$rounds
	xor	$t0,$s0,$s0
	ld	[$key+16],$t0
	xor	$t1,$s1,$s1
	ld	[$key+20],$t1
	srl	$s0,21,$acc0			!
	xor	$t2,$s2,$s2
	ld	[$key+24],$t2
	xor	$t3,$s3,$s3
	and	$acc0,2040,$acc0
	ld	[$key+28],$t3
	srl	$s3,13,$acc1
	nop
.Ldec_loop:
	srl	$s2,5,$acc2			!
	and	$acc1,2040,$acc1
	ldx	[$tbl+$acc0],$acc0
	sll	$s1,3,$acc3
	and	$acc2,2040,$acc2
	ldx	[$tbl+$acc1],$acc1
	srl	$s1,21,$acc4
	and	$acc3,2040,$acc3
	ldx	[$tbl+$acc2],$acc2		!
	srl	$s0,13,$acc5
	and	$acc4,2040,$acc4
	ldx	[$tbl+$acc3],$acc3
	srl	$s3,5,$acc6
	and	$acc5,2040,$acc5
	ldx	[$tbl+$acc4],$acc4
	fmovs	%f0,%f0
	sll	$s2,3,$acc7			!
	and	$acc6,2040,$acc6
	ldx	[$tbl+$acc5],$acc5
	srl	$s2,21,$acc8
	and	$acc7,2040,$acc7
	ldx	[$tbl+$acc6],$acc6
	srl	$s1,13,$acc9
	and	$acc8,2040,$acc8
	ldx	[$tbl+$acc7],$acc7		!
	srl	$s0,5,$acc10
	and	$acc9,2040,$acc9
	ldx	[$tbl+$acc8],$acc8
	sll	$s3,3,$acc11
	and	$acc10,2040,$acc10
	ldx	[$tbl+$acc9],$acc9
	fmovs	%f0,%f0
	srl	$s3,21,$acc12			!
	and	$acc11,2040,$acc11
	ldx	[$tbl+$acc10],$acc10
	srl	$s2,13,$acc13
	and	$acc12,2040,$acc12
	ldx	[$tbl+$acc11],$acc11
	srl	$s1,5,$acc14
	and	$acc13,2040,$acc13
	ldx	[$tbl+$acc12],$acc12		!
	sll	$s0,3,$acc15
	and	$acc14,2040,$acc14
	ldx	[$tbl+$acc13],$acc13
	and	$acc15,2040,$acc15
	add	$key,32,$key
	ldx	[$tbl+$acc14],$acc14
	fmovs	%f0,%f0
	subcc	$rounds,1,$rounds		!
	ldx	[$tbl+$acc15],$acc15
	bz,a,pn	%icc,.Ldec_last
	add	$tbl,2048,$rounds

		srlx	$acc1,8,$acc1
		xor	$acc0,$t0,$t0
	ld	[$key+0],$s0
	fmovs	%f0,%f0
		srlx	$acc2,16,$acc2		!
		xor	$acc1,$t0,$t0
	ld	[$key+4],$s1
		srlx	$acc3,24,$acc3
		xor	$acc2,$t0,$t0
	ld	[$key+8],$s2
		srlx	$acc5,8,$acc5
		xor	$acc3,$t0,$t0
	ld	[$key+12],$s3			!
		srlx	$acc6,16,$acc6
		xor	$acc4,$t1,$t1
	fmovs	%f0,%f0
		srlx	$acc7,24,$acc7
		xor	$acc5,$t1,$t1
		srlx	$acc9,8,$acc9
		xor	$acc6,$t1,$t1
		srlx	$acc10,16,$acc10	!
		xor	$acc7,$t1,$t1
		srlx	$acc11,24,$acc11
		xor	$acc8,$t2,$t2
		srlx	$acc13,8,$acc13
		xor	$acc9,$t2,$t2
		srlx	$acc14,16,$acc14
		xor	$acc10,$t2,$t2
		srlx	$acc15,24,$acc15	!
		xor	$acc11,$t2,$t2
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$t3,$t3
	srl	$t0,21,$acc0
		xor	$acc14,$t3,$t3
		xor	$acc15,$t3,$t3
	srl	$t3,13,$acc1

	and	$acc0,2040,$acc0		!
	srl	$t2,5,$acc2
	and	$acc1,2040,$acc1
	ldx	[$tbl+$acc0],$acc0
	sll	$t1,3,$acc3
	and	$acc2,2040,$acc2
	ldx	[$tbl+$acc1],$acc1
	fmovs	%f0,%f0
	srl	$t1,21,$acc4			!
	and	$acc3,2040,$acc3
	ldx	[$tbl+$acc2],$acc2
	srl	$t0,13,$acc5
	and	$acc4,2040,$acc4
	ldx	[$tbl+$acc3],$acc3
	srl	$t3,5,$acc6
	and	$acc5,2040,$acc5
	ldx	[$tbl+$acc4],$acc4		!
	sll	$t2,3,$acc7
	and	$acc6,2040,$acc6
	ldx	[$tbl+$acc5],$acc5
	srl	$t2,21,$acc8
	and	$acc7,2040,$acc7
	ldx	[$tbl+$acc6],$acc6
	fmovs	%f0,%f0
	srl	$t1,13,$acc9			!
	and	$acc8,2040,$acc8
	ldx	[$tbl+$acc7],$acc7
	srl	$t0,5,$acc10
	and	$acc9,2040,$acc9
	ldx	[$tbl+$acc8],$acc8
	sll	$t3,3,$acc11
	and	$acc10,2040,$acc10
	ldx	[$tbl+$acc9],$acc9		!
	srl	$t3,21,$acc12
	and	$acc11,2040,$acc11
	ldx	[$tbl+$acc10],$acc10
	srl	$t2,13,$acc13
	and	$acc12,2040,$acc12
	ldx	[$tbl+$acc11],$acc11
	fmovs	%f0,%f0
	srl	$t1,5,$acc14			!
	and	$acc13,2040,$acc13
	ldx	[$tbl+$acc12],$acc12
	sll	$t0,3,$acc15
	and	$acc14,2040,$acc14
	ldx	[$tbl+$acc13],$acc13
		srlx	$acc1,8,$acc1
	and	$acc15,2040,$acc15
	ldx	[$tbl+$acc14],$acc14		!

		srlx	$acc2,16,$acc2
		xor	$acc0,$s0,$s0
	ldx	[$tbl+$acc15],$acc15
		srlx	$acc3,24,$acc3
		xor	$acc1,$s0,$s0
	ld	[$key+16],$t0
	fmovs	%f0,%f0
		srlx	$acc5,8,$acc5		!
		xor	$acc2,$s0,$s0
	ld	[$key+20],$t1
		srlx	$acc6,16,$acc6
		xor	$acc3,$s0,$s0
	ld	[$key+24],$t2
		srlx	$acc7,24,$acc7
		xor	$acc4,$s1,$s1
	ld	[$key+28],$t3			!
		srlx	$acc9,8,$acc9
		xor	$acc5,$s1,$s1
	ldx	[$tbl+2048+0],%g0		! prefetch td4
		srlx	$acc10,16,$acc10
		xor	$acc6,$s1,$s1
	ldx	[$tbl+2048+32],%g0		! prefetch td4
		srlx	$acc11,24,$acc11
		xor	$acc7,$s1,$s1
	ldx	[$tbl+2048+64],%g0		! prefetch td4
		srlx	$acc13,8,$acc13
		xor	$acc8,$s2,$s2
	ldx	[$tbl+2048+96],%g0		! prefetch td4
		srlx	$acc14,16,$acc14	!
		xor	$acc9,$s2,$s2
	ldx	[$tbl+2048+128],%g0		! prefetch td4
		srlx	$acc15,24,$acc15
		xor	$acc10,$s2,$s2
	ldx	[$tbl+2048+160],%g0		! prefetch td4
	srl	$s0,21,$acc0
		xor	$acc11,$s2,$s2
	ldx	[$tbl+2048+192],%g0		! prefetch td4
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$s3,$s3
	ldx	[$tbl+2048+224],%g0		! prefetch td4
	and	$acc0,2040,$acc0		!
		xor	$acc14,$s3,$s3
		xor	$acc15,$s3,$s3
	ba	.Ldec_loop
	srl	$s3,13,$acc1

.align	32
.Ldec_last:
		srlx	$acc1,8,$acc1		!
		xor	$acc0,$t0,$t0
	ld	[$key+0],$s0
		srlx	$acc2,16,$acc2
		xor	$acc1,$t0,$t0
	ld	[$key+4],$s1
		srlx	$acc3,24,$acc3
		xor	$acc2,$t0,$t0
	ld	[$key+8],$s2			!
		srlx	$acc5,8,$acc5
		xor	$acc3,$t0,$t0
	ld	[$key+12],$s3
		srlx	$acc6,16,$acc6
		xor	$acc4,$t1,$t1
		srlx	$acc7,24,$acc7
		xor	$acc5,$t1,$t1
		srlx	$acc9,8,$acc9		!
		xor	$acc6,$t1,$t1
		srlx	$acc10,16,$acc10
		xor	$acc7,$t1,$t1
		srlx	$acc11,24,$acc11
		xor	$acc8,$t2,$t2
		srlx	$acc13,8,$acc13
		xor	$acc9,$t2,$t2
		srlx	$acc14,16,$acc14	!
		xor	$acc10,$t2,$t2
		srlx	$acc15,24,$acc15
		xor	$acc11,$t2,$t2
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$t3,$t3
	srl	$t0,24,$acc0
		xor	$acc14,$t3,$t3
		xor	$acc15,$t3,$t3		!
	srl	$t3,16,$acc1

	srl	$t2,8,$acc2
	and	$acc1,255,$acc1
	ldub	[$rounds+$acc0],$acc0
	srl	$t1,24,$acc4
	and	$acc2,255,$acc2
	ldub	[$rounds+$acc1],$acc1
	srl	$t0,16,$acc5			!
	and	$t1,255,$acc3
	ldub	[$rounds+$acc2],$acc2
	ldub	[$rounds+$acc3],$acc3
	srl	$t3,8,$acc6
	and	$acc5,255,$acc5
	ldub	[$rounds+$acc4],$acc4
	fmovs	%f0,%f0
	srl	$t2,24,$acc8			!
	and	$acc6,255,$acc6
	ldub	[$rounds+$acc5],$acc5
	srl	$t1,16,$acc9
	and	$t2,255,$acc7
	ldub	[$rounds+$acc6],$acc6
	ldub	[$rounds+$acc7],$acc7
	fmovs	%f0,%f0
	srl	$t0,8,$acc10			!
	and	$acc9,255,$acc9
	ldub	[$rounds+$acc8],$acc8
	srl	$t3,24,$acc12
	and	$acc10,255,$acc10
	ldub	[$rounds+$acc9],$acc9
	srl	$t2,16,$acc13
	and	$t3,255,$acc11
	ldub	[$rounds+$acc10],$acc10		!
	srl	$t1,8,$acc14
	and	$acc13,255,$acc13
	ldub	[$rounds+$acc11],$acc11
	ldub	[$rounds+$acc12],$acc12
	and	$acc14,255,$acc14
	ldub	[$rounds+$acc13],$acc13
	and	$t0,255,$acc15
	ldub	[$rounds+$acc14],$acc14		!

		sll	$acc0,24,$acc0
		xor	$acc3,$s0,$s0
	ldub	[$rounds+$acc15],$acc15
		sll	$acc1,16,$acc1
		xor	$acc0,$s0,$s0
	ldx	[%sp+$bias+$frame+0],%i7	! restore return address
	fmovs	%f0,%f0
		sll	$acc2,8,$acc2		!
		xor	$acc1,$s0,$s0
		sll	$acc4,24,$acc4
		xor	$acc2,$s0,$s0
		sll	$acc5,16,$acc5
		xor	$acc7,$s1,$s1
		sll	$acc6,8,$acc6
		xor	$acc4,$s1,$s1
		sll	$acc8,24,$acc8		!
		xor	$acc5,$s1,$s1
		sll	$acc9,16,$acc9
		xor	$acc11,$s2,$s2
		sll	$acc10,8,$acc10
		xor	$acc6,$s1,$s1
		sll	$acc12,24,$acc12
		xor	$acc8,$s2,$s2
		sll	$acc13,16,$acc13	!
		xor	$acc9,$s2,$s2
		sll	$acc14,8,$acc14
		xor	$acc10,$s2,$s2
		xor	$acc12,$acc14,$acc14
		xor	$acc13,$s3,$s3
		xor	$acc14,$s3,$s3
		xor	$acc15,$s3,$s3

	ret
	restore
.type	_sparcv9_AES_decrypt,#function
.size	_sparcv9_AES_decrypt,(.-_sparcv9_AES_decrypt)

.align	32
.globl	AES_decrypt
AES_decrypt:
	or	%o0,%o1,%g1
	andcc	%g1,3,%g0
	bnz,pn	%xcc,.Lunaligned_dec
	save	%sp,-$frame,%sp

	ld	[%i0+0],%o0
	ld	[%i0+4],%o1
	ld	[%i0+8],%o2
	ld	[%i0+12],%o3

1:	call	.+8
	add	%o7,AES_Td-1b,%o4
	call	_sparcv9_AES_decrypt
	mov	%i2,%o5

	st	%o0,[%i1+0]
	st	%o1,[%i1+4]
	st	%o2,[%i1+8]
	st	%o3,[%i1+12]

	ret
	restore

.align	32
.Lunaligned_dec:
	ldub	[%i0+0],%l0
	ldub	[%i0+1],%l1
	ldub	[%i0+2],%l2

	sll	%l0,24,%l0
	ldub	[%i0+3],%l3
	sll	%l1,16,%l1
	ldub	[%i0+4],%l4
	sll	%l2,8,%l2
	or	%l1,%l0,%l0
	ldub	[%i0+5],%l5
	sll	%l4,24,%l4
	or	%l3,%l2,%l2
	ldub	[%i0+6],%l6
	sll	%l5,16,%l5
	or	%l0,%l2,%o0
	ldub	[%i0+7],%l7

	sll	%l6,8,%l6
	or	%l5,%l4,%l4
	ldub	[%i0+8],%l0
	or	%l7,%l6,%l6
	ldub	[%i0+9],%l1
	or	%l4,%l6,%o1
	ldub	[%i0+10],%l2

	sll	%l0,24,%l0
	ldub	[%i0+11],%l3
	sll	%l1,16,%l1
	ldub	[%i0+12],%l4
	sll	%l2,8,%l2
	or	%l1,%l0,%l0
	ldub	[%i0+13],%l5
	sll	%l4,24,%l4
	or	%l3,%l2,%l2
	ldub	[%i0+14],%l6
	sll	%l5,16,%l5
	or	%l0,%l2,%o2
	ldub	[%i0+15],%l7

	sll	%l6,8,%l6
	or	%l5,%l4,%l4
	or	%l7,%l6,%l6
	or	%l4,%l6,%o3

1:	call	.+8
	add	%o7,AES_Td-1b,%o4
	call	_sparcv9_AES_decrypt
	mov	%i2,%o5

	srl	%o0,24,%l0
	srl	%o0,16,%l1
	stb	%l0,[%i1+0]
	srl	%o0,8,%l2
	stb	%l1,[%i1+1]
	stb	%l2,[%i1+2]
	srl	%o1,24,%l4
	stb	%o0,[%i1+3]

	srl	%o1,16,%l5
	stb	%l4,[%i1+4]
	srl	%o1,8,%l6
	stb	%l5,[%i1+5]
	stb	%l6,[%i1+6]
	srl	%o2,24,%l0
	stb	%o1,[%i1+7]

	srl	%o2,16,%l1
	stb	%l0,[%i1+8]
	srl	%o2,8,%l2
	stb	%l1,[%i1+9]
	stb	%l2,[%i1+10]
	srl	%o3,24,%l4
	stb	%o2,[%i1+11]

	srl	%o3,16,%l5
	stb	%l4,[%i1+12]
	srl	%o3,8,%l6
	stb	%l5,[%i1+13]
	stb	%l6,[%i1+14]
	stb	%o3,[%i1+15]

	ret
	restore
.type	AES_decrypt,#function
.size	AES_decrypt,(.-AES_decrypt)
___

# fmovs instructions substituting for FP nops were originally added
# to meet specific instruction alignment requirements to maximize ILP.
# As UltraSPARC T1, a.k.a. Niagara, has shared FPU, FP nops can have
# undesired effect, so just omit them and sacrifice some portion of
# percent in performance...
$code =~ s/fmovs.*$//gm;

print $code;
close STDOUT or die "error closing STDOUT: $!";	# ensure flush


#! /usr/bin/env perl
# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# March 2016
#
# Initial support for Fujitsu SPARC64 X/X+ comprises minimally
# required key setup and single-block procedures.
#
# April 2016
#
# Add "teaser" CBC and CTR mode-specific subroutines. "Teaser" means
# that parallelizable nature of CBC decrypt and CTR is not utilized
# yet. CBC encrypt on the other hand is as good as it can possibly
# get processing one byte in 4.1 cycles with 128-bit key on SPARC64 X.
# This is ~6x faster than pure software implementation...
#
# July 2016
#
# Switch from faligndata to fshiftorx, which allows to omit alignaddr
# instructions and improve single-block and short-input performance
# with misaligned data.

$output = pop;
open STDOUT,">$output";

{
my ($inp,$out,$key,$rounds,$tmp,$mask) = map("%o$_",(0..5));

$code.=<<___;
#include "sparc_arch.h"

#define LOCALS (STACK_BIAS+STACK_FRAME)

.text

.globl	aes_fx_encrypt
.align	32
aes_fx_encrypt:
	and		$inp, 7, $tmp		! is input aligned?
	andn		$inp, 7, $inp
	ldd		[$key +  0], %f6	! round[0]
	ldd		[$key +  8], %f8
	mov		%o7, %g1
	ld		[$key + 240], $rounds

1:	call		.+8
	add		%o7, .Linp_align-1b, %o7

	sll		$tmp, 3, $tmp
	ldd		[$inp + 0], %f0		! load input
	brz,pt		$tmp, .Lenc_inp_aligned
	ldd		[$inp + 8], %f2

	ldd		[%o7 + $tmp], %f14	! shift left params
	ldd		[$inp + 16], %f4
	fshiftorx	%f0, %f2, %f14, %f0
	fshiftorx	%f2, %f4, %f14, %f2

.Lenc_inp_aligned:
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fxor		%f0, %f6, %f0		! ^=round[0]
	fxor		%f2, %f8, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8
	add		$key, 32, $key
	sub		$rounds, 4, $rounds

.Loop_enc:
	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$key + 16], %f10
	ldd		[$key + 24], %f12
	add		$key, 32, $key

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	ldd		[$key +  0], %f6
	ldd		[$key +  8], %f8

	brnz,a		$rounds, .Loop_enc
	sub		$rounds, 2, $rounds

	andcc		$out, 7, $tmp		! is output aligned?
	andn		$out, 7, $out
	mov		0xff, $mask
	srl		$mask, $tmp, $mask
	add		%o7, 64, %o7
	sll		$tmp, 3, $tmp

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[%o7 + $tmp], %f14	! shift right params

	fmovd		%f0, %f4
	faesenclx	%f2, %f6, %f0
	faesenclx	%f4, %f8, %f2

	bnz,pn		%icc, .Lenc_out_unaligned
	mov		%g1, %o7

	std		%f0, [$out + 0]
	retl
	std		%f2, [$out + 8]

.align	16
.Lenc_out_unaligned:
	add		$out, 16, $inp
	orn		%g0, $mask, $tmp
	fshiftorx	%f0, %f0, %f14, %f4
	fshiftorx	%f0, %f2, %f14, %f6
	fshiftorx	%f2, %f2, %f14, %f8

	stda		%f4, [$out + $mask]0xc0	! partial store
	std		%f6, [$out + 8]
	stda		%f8, [$inp + $tmp]0xc0	! partial store
	retl
	nop
.type	aes_fx_encrypt,#function
.size	aes_fx_encrypt,.-aes_fx_encrypt

.globl	aes_fx_decrypt
.align	32
aes_fx_decrypt:
	and		$inp, 7, $tmp		! is input aligned?
	andn		$inp, 7, $inp
	ldd		[$key +  0], %f6	! round[0]
	ldd		[$key +  8], %f8
	mov		%o7, %g1
	ld		[$key + 240], $rounds

1:	call		.+8
	add		%o7, .Linp_align-1b, %o7

	sll		$tmp, 3, $tmp
	ldd		[$inp + 0], %f0		! load input
	brz,pt		$tmp, .Ldec_inp_aligned
	ldd		[$inp + 8], %f2

	ldd		[%o7 + $tmp], %f14	! shift left params
	ldd		[$inp + 16], %f4
	fshiftorx	%f0, %f2, %f14, %f0
	fshiftorx	%f2, %f4, %f14, %f2

.Ldec_inp_aligned:
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fxor		%f0, %f6, %f0		! ^=round[0]
	fxor		%f2, %f8, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8
	add		$key, 32, $key
	sub		$rounds, 4, $rounds

.Loop_dec:
	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$key + 16], %f10
	ldd		[$key + 24], %f12
	add		$key, 32, $key

	fmovd		%f0, %f4
	faesdecx	%f2, %f6, %f0
	faesdecx	%f4, %f8, %f2
	ldd		[$key +  0], %f6
	ldd		[$key +  8], %f8

	brnz,a		$rounds, .Loop_dec
	sub		$rounds, 2, $rounds

	andcc		$out, 7, $tmp		! is output aligned?
	andn		$out, 7, $out
	mov		0xff, $mask
	srl		$mask, $tmp, $mask
	add		%o7, 64, %o7
	sll		$tmp, 3, $tmp

	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[%o7 + $tmp], %f14	! shift right params

	fmovd		%f0, %f4
	faesdeclx	%f2, %f6, %f0
	faesdeclx	%f4, %f8, %f2

	bnz,pn		%icc, .Ldec_out_unaligned
	mov		%g1, %o7

	std		%f0, [$out + 0]
	retl
	std		%f2, [$out + 8]

.align	16
.Ldec_out_unaligned:
	add		$out, 16, $inp
	orn		%g0, $mask, $tmp
	fshiftorx	%f0, %f0, %f14, %f4
	fshiftorx	%f0, %f2, %f14, %f6
	fshiftorx	%f2, %f2, %f14, %f8

	stda		%f4, [$out + $mask]0xc0	! partial store
	std		%f6, [$out + 8]
	stda		%f8, [$inp + $tmp]0xc0	! partial store
	retl
	nop
.type	aes_fx_decrypt,#function
.size	aes_fx_decrypt,.-aes_fx_decrypt
___
}
{
my ($inp,$bits,$out,$tmp,$inc) = map("%o$_",(0..5));
$code.=<<___;
.globl	aes_fx_set_decrypt_key
.align	32
aes_fx_set_decrypt_key:
	b		.Lset_encrypt_key
	mov		-1, $inc
	retl
	nop
.type	aes_fx_set_decrypt_key,#function
.size	aes_fx_set_decrypt_key,.-aes_fx_set_decrypt_key

.globl	aes_fx_set_encrypt_key
.align	32
aes_fx_set_encrypt_key:
	mov		1, $inc
	nop
.Lset_encrypt_key:
	and		$inp, 7, $tmp
	andn		$inp, 7, $inp
	sll		$tmp, 3, $tmp
	mov		%o7, %g1

1:	call		.+8
	add		%o7, .Linp_align-1b, %o7

	ldd		[%o7 + $tmp], %f10	! shift left params
	mov		%g1, %o7

	cmp		$bits, 192
	ldd		[$inp + 0], %f0
	bl,pt		%icc, .L128
	ldd		[$inp + 8], %f2

	be,pt		%icc, .L192
	ldd		[$inp + 16], %f4
	brz,pt		$tmp, .L256aligned
	ldd		[$inp + 24], %f6

	ldd		[$inp + 32], %f8
	fshiftorx	%f0, %f2, %f10, %f0
	fshiftorx	%f2, %f4, %f10, %f2
	fshiftorx	%f4, %f6, %f10, %f4
	fshiftorx	%f6, %f8, %f10, %f6

.L256aligned:
	mov		14, $bits
	and		$inc, `14*16`, $tmp
	st		$bits, [$out + 240]	! store rounds
	add		$out, $tmp, $out	! start or end of key schedule
	sllx		$inc, 4, $inc		! 16 or -16
___
for ($i=0; $i<6; $i++) {
    $code.=<<___;
	std		%f0, [$out + 0]
	faeskeyx	%f6, `0x10+$i`, %f0
	std		%f2, [$out + 8]
	add		$out, $inc, $out
	faeskeyx	%f0, 0x00, %f2
	std		%f4, [$out + 0]
	faeskeyx	%f2, 0x01, %f4
	std		%f6, [$out + 8]
	add		$out, $inc, $out
	faeskeyx	%f4, 0x00, %f6
___
}
$code.=<<___;
	std		%f0, [$out + 0]
	faeskeyx	%f6, `0x10+$i`, %f0
	std		%f2, [$out + 8]
	add		$out, $inc, $out
	faeskeyx	%f0, 0x00, %f2
	std		%f4,[$out + 0]
	std		%f6,[$out + 8]
	add		$out, $inc, $out
	std		%f0,[$out + 0]
	std		%f2,[$out + 8]
	retl
	xor		%o0, %o0, %o0		! return 0

.align	16
.L192:
	brz,pt		$tmp, .L192aligned
	nop

	ldd		[$inp + 24], %f6
	fshiftorx	%f0, %f2, %f10, %f0
	fshiftorx	%f2, %f4, %f10, %f2
	fshiftorx	%f4, %f6, %f10, %f4

.L192aligned:
	mov		12, $bits
	and		$inc, `12*16`, $tmp
	st		$bits, [$out + 240]	! store rounds
	add		$out, $tmp, $out	! start or end of key schedule
	sllx		$inc, 4, $inc		! 16 or -16
___
for ($i=0; $i<8; $i+=2) {
    $code.=<<___;
	std		%f0, [$out + 0]
	faeskeyx	%f4, `0x10+$i`, %f0
	std		%f2, [$out + 8]
	add		$out, $inc, $out
	faeskeyx	%f0, 0x00, %f2
	std		%f4, [$out + 0]
	faeskeyx	%f2, 0x00, %f4
	std		%f0, [$out + 8]
	add		$out, $inc, $out
	faeskeyx	%f4, `0x10+$i+1`, %f0
	std		%f2, [$out + 0]
	faeskeyx	%f0, 0x00, %f2
	std		%f4, [$out + 8]
	add		$out, $inc, $out
___
$code.=<<___		if ($i<6);
	faeskeyx	%f2, 0x00, %f4
___
}
$code.=<<___;
	std		%f0, [$out + 0]
	std		%f2, [$out + 8]
	retl
	xor		%o0, %o0, %o0		! return 0

.align	16
.L128:
	brz,pt		$tmp, .L128aligned
	nop

	ldd		[$inp + 16], %f4
	fshiftorx	%f0, %f2, %f10, %f0
	fshiftorx	%f2, %f4, %f10, %f2

.L128aligned:
	mov		10, $bits
	and		$inc, `10*16`, $tmp
	st		$bits, [$out + 240]	! store rounds
	add		$out, $tmp, $out	! start or end of key schedule
	sllx		$inc, 4, $inc		! 16 or -16
___
for ($i=0; $i<10; $i++) {
    $code.=<<___;
	std		%f0, [$out + 0]
	faeskeyx	%f2, `0x10+$i`, %f0
	std		%f2, [$out + 8]
	add		$out, $inc, $out
	faeskeyx	%f0, 0x00, %f2
___
}
$code.=<<___;
	std		%f0, [$out + 0]
	std		%f2, [$out + 8]
	retl
	xor		%o0, %o0, %o0		! return 0
.type	aes_fx_set_encrypt_key,#function
.size	aes_fx_set_encrypt_key,.-aes_fx_set_encrypt_key
___
}
{
my ($inp,$out,$len,$key,$ivp,$dir) = map("%i$_",(0..5));
my ($rounds,$inner,$end,$inc,$ialign,$oalign,$mask) = map("%l$_",(0..7));
my ($iv0,$iv1,$r0hi,$r0lo,$rlhi,$rllo,$in0,$in1,$intail,$outhead,$fshift)
   = map("%f$_",grep { !($_ & 1) } (16 .. 62));
my ($ileft,$iright) = ($ialign,$oalign);

$code.=<<___;
.globl	aes_fx_cbc_encrypt
.align	32
aes_fx_cbc_encrypt:
	save		%sp, -STACK_FRAME-16, %sp
	srln		$len, 4, $len
	and		$inp, 7, $ialign
	andn		$inp, 7, $inp
	brz,pn		$len, .Lcbc_no_data
	sll		$ialign, 3, $ileft

1:	call		.+8
	add		%o7, .Linp_align-1b, %o7

	ld		[$key + 240], $rounds
	and		$out, 7, $oalign
	ld		[$ivp + 0], %f0		! load ivec
	andn		$out, 7, $out
	ld		[$ivp + 4], %f1
	sll		$oalign, 3, $mask
	ld		[$ivp + 8], %f2
	ld		[$ivp + 12], %f3

	sll		$rounds, 4, $rounds
	add		$rounds, $key, $end
	ldd		[$key + 0], $r0hi	! round[0]
	ldd		[$key + 8], $r0lo

	add		$inp, 16, $inp
	sub		$len,  1, $len
	ldd		[$end + 0], $rlhi	! round[last]
	ldd		[$end + 8], $rllo

	mov		16, $inc
	movrz		$len, 0, $inc
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	ldd		[%o7 + $ileft], $fshift	! shift left params
	add		%o7, 64, %o7
	ldd		[$inp - 16], $in0	! load input
	ldd		[$inp -  8], $in1
	ldda		[$inp]0x82, $intail	! non-faulting load
	brz		$dir, .Lcbc_decrypt
	add		$inp, $inc, $inp	! inp+=16

	fxor		$r0hi, %f0, %f0		! ivec^=round[0]
	fxor		$r0lo, %f2, %f2
	fshiftorx	$in0, $in1, $fshift, $in0
	fshiftorx	$in1, $intail, $fshift, $in1
	nop

.Loop_cbc_enc:
	fxor		$in0, %f0, %f0		! inp^ivec^round[0]
	fxor		$in1, %f2, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8
	add		$key, 32, $end
	sub		$rounds, 16*6, $inner

.Lcbc_enc:
	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10
	ldd		[$end + 24], %f12
	add		$end, 32, $end

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	ldd		[$end + 0], %f6
	ldd		[$end + 8], %f8

	brnz,a		$inner, .Lcbc_enc
	sub		$inner, 16*2, $inner

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10	! round[last-1]
	ldd		[$end + 24], %f12

	movrz		$len, 0, $inc
	fmovd		$intail, $in0
	ldd		[$inp - 8], $in1	! load next input block
	ldda		[$inp]0x82, $intail	! non-faulting load
	add		$inp, $inc, $inp	! inp+=16

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2

	fshiftorx	$in0, $in1, $fshift, $in0
	fshiftorx	$in1, $intail, $fshift, $in1

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fxor		$r0hi, $in0, $in0	! inp^=round[0]
	fxor		$r0lo, $in1, $in1

	fmovd		%f0, %f4
	faesenclx	%f2, $rlhi, %f0
	faesenclx	%f4, $rllo, %f2

	brnz,pn		$oalign, .Lcbc_enc_unaligned_out
	nop

	std		%f0, [$out + 0]
	std		%f2, [$out + 8]
	add		$out, 16, $out

	brnz,a		$len, .Loop_cbc_enc
	sub		$len, 1, $len

	st		%f0, [$ivp + 0]		! output ivec
	st		%f1, [$ivp + 4]
	st		%f2, [$ivp + 8]
	st		%f3, [$ivp + 12]

.Lcbc_no_data:
	ret
	restore

.align	32
.Lcbc_enc_unaligned_out:
	ldd		[%o7 + $mask], $fshift	! shift right params
	mov		0xff, $mask
	srl		$mask, $oalign, $mask
	sub		%g0, $ileft, $iright

	fshiftorx	%f0, %f0, $fshift, %f6
	fshiftorx	%f0, %f2, $fshift, %f8

	stda		%f6, [$out + $mask]0xc0	! partial store
	orn		%g0, $mask, $mask
	std		%f8, [$out + 8]
	add		$out, 16, $out
	brz		$len, .Lcbc_enc_unaligned_out_done
	sub		$len, 1, $len
	b		.Loop_cbc_enc_unaligned_out
	nop

.align	32
.Loop_cbc_enc_unaligned_out:
	fmovd		%f2, $outhead
	fxor		$in0, %f0, %f0		! inp^ivec^round[0]
	fxor		$in1, %f2, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$key + 48], %f10	! round[3]
	ldd		[$key + 56], %f12

	ldx		[$inp - 16], %o0
	ldx		[$inp -  8], %o1
	brz		$ileft, .Lcbc_enc_aligned_inp
	movrz		$len, 0, $inc

	ldx		[$inp], %o2
	sllx		%o0, $ileft, %o0
	srlx		%o1, $iright, %g1
	sllx		%o1, $ileft, %o1
	or		%g1, %o0, %o0
	srlx		%o2, $iright, %o2
	or		%o2, %o1, %o1

.Lcbc_enc_aligned_inp:
	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	ldd		[$key + 64], %f6	! round[4]
	ldd		[$key + 72], %f8
	add		$key, 64, $end
	sub		$rounds, 16*8, $inner

	stx		%o0, [%sp + LOCALS + 0]
	stx		%o1, [%sp + LOCALS + 8]
	add		$inp, $inc, $inp	! inp+=16
	nop

.Lcbc_enc_unaligned:
	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10
	ldd		[$end + 24], %f12
	add		$end, 32, $end

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	ldd		[$end + 0], %f6
	ldd		[$end + 8], %f8

	brnz,a		$inner, .Lcbc_enc_unaligned
	sub		$inner, 16*2, $inner

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10	! round[last-1]
	ldd		[$end + 24], %f12

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2

	ldd		[%sp + LOCALS + 0], $in0
	ldd		[%sp + LOCALS + 8], $in1

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fxor		$r0hi, $in0, $in0	! inp^=round[0]
	fxor		$r0lo, $in1, $in1

	fmovd		%f0, %f4
	faesenclx	%f2, $rlhi, %f0
	faesenclx	%f4, $rllo, %f2

	fshiftorx	$outhead, %f0, $fshift, %f6
	fshiftorx	%f0, %f2, $fshift, %f8
	std		%f6, [$out + 0]
	std		%f8, [$out + 8]
	add		$out, 16, $out

	brnz,a		$len, .Loop_cbc_enc_unaligned_out
	sub		$len, 1, $len

.Lcbc_enc_unaligned_out_done:
	fshiftorx	%f2, %f2, $fshift, %f8
	stda		%f8, [$out + $mask]0xc0	! partial store

	st		%f0, [$ivp + 0]		! output ivec
	st		%f1, [$ivp + 4]
	st		%f2, [$ivp + 8]
	st		%f3, [$ivp + 12]

	ret
	restore

.align	32
.Lcbc_decrypt:
	fshiftorx	$in0, $in1, $fshift, $in0
	fshiftorx	$in1, $intail, $fshift, $in1
	fmovd		%f0, $iv0
	fmovd		%f2, $iv1

.Loop_cbc_dec:
	fxor		$in0, $r0hi, %f0	! inp^round[0]
	fxor		$in1, $r0lo, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8
	add		$key, 32, $end
	sub		$rounds, 16*6, $inner

.Lcbc_dec:
	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$end + 16], %f10
	ldd		[$end + 24], %f12
	add		$end, 32, $end

	fmovd		%f0, %f4
	faesdecx	%f2, %f6, %f0
	faesdecx	%f4, %f8, %f2
	ldd		[$end + 0], %f6
	ldd		[$end + 8], %f8

	brnz,a		$inner, .Lcbc_dec
	sub		$inner, 16*2, $inner

	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$end + 16], %f10	! round[last-1]
	ldd		[$end + 24], %f12

	fmovd		%f0, %f4
	faesdecx	%f2, %f6, %f0
	faesdecx	%f4, %f8, %f2
	fxor		$iv0, $rlhi, %f6	! ivec^round[last]
	fxor		$iv1, $rllo, %f8
	fmovd		$in0, $iv0
	fmovd		$in1, $iv1

	movrz		$len, 0, $inc
	fmovd		$intail, $in0
	ldd		[$inp - 8], $in1	! load next input block
	ldda		[$inp]0x82, $intail	! non-faulting load
	add		$inp, $inc, $inp	! inp+=16

	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fshiftorx	$in0, $in1, $fshift, $in0
	fshiftorx	$in1, $intail, $fshift, $in1

	fmovd		%f0, %f4
	faesdeclx	%f2, %f6, %f0
	faesdeclx	%f4, %f8, %f2

	brnz,pn		$oalign, .Lcbc_dec_unaligned_out
	nop

	std		%f0, [$out + 0]
	std		%f2, [$out + 8]
	add		$out, 16, $out

	brnz,a		$len, .Loop_cbc_dec
	sub		$len, 1, $len

	st		$iv0,    [$ivp + 0]	! output ivec
	st		$iv0#lo, [$ivp + 4]
	st		$iv1,    [$ivp + 8]
	st		$iv1#lo, [$ivp + 12]

	ret
	restore

.align	32
.Lcbc_dec_unaligned_out:
	ldd		[%o7 + $mask], $fshift	! shift right params
	mov		0xff, $mask
	srl		$mask, $oalign, $mask
	sub		%g0, $ileft, $iright

	fshiftorx	%f0, %f0, $fshift, %f6
	fshiftorx	%f0, %f2, $fshift, %f8

	stda		%f6, [$out + $mask]0xc0	! partial store
	orn		%g0, $mask, $mask
	std		%f8, [$out + 8]
	add		$out, 16, $out
	brz		$len, .Lcbc_dec_unaligned_out_done
	sub		$len, 1, $len
	b		.Loop_cbc_dec_unaligned_out
	nop

.align	32
.Loop_cbc_dec_unaligned_out:
	fmovd		%f2, $outhead
	fxor		$in0, $r0hi, %f0	! inp^round[0]
	fxor		$in1, $r0lo, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8

	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$key + 48], %f10	! round[3]
	ldd		[$key + 56], %f12

	ldx		[$inp - 16], %o0
	ldx		[$inp - 8], %o1
	brz		$ileft, .Lcbc_dec_aligned_inp
	movrz		$len, 0, $inc

	ldx		[$inp], %o2
	sllx		%o0, $ileft, %o0
	srlx		%o1, $iright, %g1
	sllx		%o1, $ileft, %o1
	or		%g1, %o0, %o0
	srlx		%o2, $iright, %o2
	or		%o2, %o1, %o1

.Lcbc_dec_aligned_inp:
	fmovd		%f0, %f4
	faesdecx	%f2, %f6, %f0
	faesdecx	%f4, %f8, %f2
	ldd		[$key + 64], %f6	! round[4]
	ldd		[$key + 72], %f8
	add		$key, 64, $end
	sub		$rounds, 16*8, $inner

	stx		%o0, [%sp + LOCALS + 0]
	stx		%o1, [%sp + LOCALS + 8]
	add		$inp, $inc, $inp	! inp+=16
	nop

.Lcbc_dec_unaligned:
	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$end + 16], %f10
	ldd		[$end + 24], %f12
	add		$end, 32, $end

	fmovd		%f0, %f4
	faesdecx	%f2, %f6, %f0
	faesdecx	%f4, %f8, %f2
	ldd		[$end + 0], %f6
	ldd		[$end + 8], %f8

	brnz,a		$inner, .Lcbc_dec_unaligned
	sub		$inner, 16*2, $inner

	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$end + 16], %f10	! round[last-1]
	ldd		[$end + 24], %f12

	fmovd		%f0, %f4
	faesdecx	%f2, %f6, %f0
	faesdecx	%f4, %f8, %f2

	fxor		$iv0, $rlhi, %f6	! ivec^round[last]
	fxor		$iv1, $rllo, %f8
	fmovd		$in0, $iv0
	fmovd		$in1, $iv1
	ldd		[%sp + LOCALS + 0], $in0
	ldd		[%sp + LOCALS + 8], $in1

	fmovd		%f0, %f4
	faesdecx	%f2, %f10, %f0
	faesdecx	%f4, %f12, %f2
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fmovd		%f0, %f4
	faesdeclx	%f2, %f6, %f0
	faesdeclx	%f4, %f8, %f2

	fshiftorx	$outhead, %f0, $fshift, %f6
	fshiftorx	%f0, %f2, $fshift, %f8
	std		%f6, [$out + 0]
	std		%f8, [$out + 8]
	add		$out, 16, $out

	brnz,a		$len, .Loop_cbc_dec_unaligned_out
	sub		$len, 1, $len

.Lcbc_dec_unaligned_out_done:
	fshiftorx	%f2, %f2, $fshift, %f8
	stda		%f8, [$out + $mask]0xc0	! partial store

	st		$iv0,    [$ivp + 0]	! output ivec
	st		$iv0#lo, [$ivp + 4]
	st		$iv1,    [$ivp + 8]
	st		$iv1#lo, [$ivp + 12]

	ret
	restore
.type	aes_fx_cbc_encrypt,#function
.size	aes_fx_cbc_encrypt,.-aes_fx_cbc_encrypt
___
}
{
my ($inp,$out,$len,$key,$ivp) = map("%i$_",(0..5));
my ($rounds,$inner,$end,$inc,$ialign,$oalign,$mask) = map("%l$_",(0..7));
my ($ctr0,$ctr1,$r0hi,$r0lo,$rlhi,$rllo,$in0,$in1,$intail,$outhead,$fshift)
   = map("%f$_",grep { !($_ & 1) } (16 .. 62));
my ($ileft,$iright) = ($ialign, $oalign);
my $one = "%f14";

$code.=<<___;
.globl	aes_fx_ctr32_encrypt_blocks
.align	32
aes_fx_ctr32_encrypt_blocks:
	save		%sp, -STACK_FRAME-16, %sp
	srln		$len, 0, $len
	and		$inp, 7, $ialign
	andn		$inp, 7, $inp
	brz,pn		$len, .Lctr32_no_data
	sll		$ialign, 3, $ileft

.Lpic:	call		.+8
	add		%o7, .Linp_align - .Lpic, %o7

	ld		[$key + 240], $rounds
	and		$out, 7, $oalign
	ld		[$ivp +  0], $ctr0	! load counter
	andn		$out, 7, $out
	ld		[$ivp +  4], $ctr0#lo
	sll		$oalign, 3, $mask
	ld		[$ivp +  8], $ctr1
	ld		[$ivp + 12], $ctr1#lo
	ldd		[%o7 + 128], $one

	sll		$rounds, 4, $rounds
	add		$rounds, $key, $end
	ldd		[$key + 0], $r0hi	! round[0]
	ldd		[$key + 8], $r0lo

	add		$inp, 16, $inp
	sub		$len, 1, $len
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	mov		16, $inc
	movrz		$len, 0, $inc
	ldd		[$end + 0], $rlhi	! round[last]
	ldd		[$end + 8], $rllo

	ldd		[%o7 + $ileft], $fshift	! shiftleft params
	add		%o7, 64, %o7
	ldd		[$inp - 16], $in0	! load input
	ldd		[$inp -  8], $in1
	ldda		[$inp]0x82, $intail	! non-faulting load
	add		$inp, $inc, $inp	! inp+=16

	fshiftorx	$in0, $in1, $fshift, $in0
	fshiftorx	$in1, $intail, $fshift, $in1

.Loop_ctr32:
	fxor		$ctr0, $r0hi, %f0	! counter^round[0]
	fxor		$ctr1, $r0lo, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8
	add		$key, 32, $end
	sub		$rounds, 16*6, $inner

.Lctr32_enc:
	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10
	ldd		[$end + 24], %f12
	add		$end, 32, $end

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	ldd		[$end + 0], %f6
	ldd		[$end + 8], %f8

	brnz,a		$inner, .Lctr32_enc
	sub		$inner, 16*2, $inner

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10	! round[last-1]
	ldd		[$end + 24], %f12

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	fxor		$in0, $rlhi, %f6	! inp^round[last]
	fxor		$in1, $rllo, %f8

	movrz		$len, 0, $inc
	fmovd		$intail, $in0
	ldd		[$inp - 8], $in1	! load next input block
	ldda		[$inp]0x82, $intail	! non-faulting load
	add		$inp, $inc, $inp	! inp+=16

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fshiftorx	$in0, $in1, $fshift, $in0
	fshiftorx	$in1, $intail, $fshift, $in1
	fpadd32		$ctr1, $one, $ctr1	! increment counter

	fmovd		%f0, %f4
	faesenclx	%f2, %f6, %f0
	faesenclx	%f4, %f8, %f2

	brnz,pn		$oalign, .Lctr32_unaligned_out
	nop

	std		%f0, [$out + 0]
	std		%f2, [$out + 8]
	add		$out, 16, $out

	brnz,a		$len, .Loop_ctr32
	sub		$len, 1, $len

.Lctr32_no_data:
	ret
	restore

.align	32
.Lctr32_unaligned_out:
	ldd		[%o7 + $mask], $fshift	! shift right params
	mov		0xff, $mask
	srl		$mask, $oalign, $mask
	sub		%g0, $ileft, $iright

	fshiftorx	%f0, %f0, $fshift, %f6
	fshiftorx	%f0, %f2, $fshift, %f8

	stda		%f6, [$out + $mask]0xc0	! partial store
	orn		%g0, $mask, $mask
	std		%f8, [$out + 8]
	add		$out, 16, $out
	brz		$len, .Lctr32_unaligned_out_done
	sub		$len, 1, $len
	b		.Loop_ctr32_unaligned_out
	nop

.align	32
.Loop_ctr32_unaligned_out:
	fmovd		%f2, $outhead
	fxor		$ctr0, $r0hi, %f0	! counter^round[0]
	fxor		$ctr1, $r0lo, %f2
	ldd		[$key + 32], %f6	! round[2]
	ldd		[$key + 40], %f8

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$key + 48], %f10	! round[3]
	ldd		[$key + 56], %f12

	ldx		[$inp - 16], %o0
	ldx		[$inp -  8], %o1
	brz		$ileft, .Lctr32_aligned_inp
	movrz		$len, 0, $inc

	ldx		[$inp], %o2
	sllx		%o0, $ileft, %o0
	srlx		%o1, $iright, %g1
	sllx		%o1, $ileft, %o1
	or		%g1, %o0, %o0
	srlx		%o2, $iright, %o2
	or		%o2, %o1, %o1

.Lctr32_aligned_inp:
	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	ldd		[$key + 64], %f6	! round[4]
	ldd		[$key + 72], %f8
	add		$key, 64, $end
	sub		$rounds, 16*8, $inner

	stx		%o0, [%sp + LOCALS + 0]
	stx		%o1, [%sp + LOCALS + 8]
	add		$inp, $inc, $inp	! inp+=16
	nop

.Lctr32_enc_unaligned:
	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10
	ldd		[$end + 24], %f12
	add		$end, 32, $end

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	ldd		[$end + 0], %f6
	ldd		[$end + 8], %f8

	brnz,a		$inner, .Lctr32_enc_unaligned
	sub		$inner, 16*2, $inner

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$end + 16], %f10	! round[last-1]
	ldd		[$end + 24], %f12
	fpadd32		$ctr1, $one, $ctr1	! increment counter

	fmovd		%f0, %f4
	faesencx	%f2, %f6, %f0
	faesencx	%f4, %f8, %f2
	fxor		$in0, $rlhi, %f6	! inp^round[last]
	fxor		$in1, $rllo, %f8
	ldd		[%sp + LOCALS + 0], $in0
	ldd		[%sp + LOCALS + 8], $in1

	fmovd		%f0, %f4
	faesencx	%f2, %f10, %f0
	faesencx	%f4, %f12, %f2
	ldd		[$key + 16], %f10	! round[1]
	ldd		[$key + 24], %f12

	fmovd		%f0, %f4
	faesenclx	%f2, %f6, %f0
	faesenclx	%f4, %f8, %f2

	fshiftorx	$outhead, %f0, $fshift, %f6
	fshiftorx	%f0, %f2, $fshift, %f8
	std		%f6, [$out + 0]
	std		%f8, [$out + 8]
	add		$out, 16, $out

	brnz,a		$len, .Loop_ctr32_unaligned_out
	sub		$len, 1, $len

.Lctr32_unaligned_out_done:
	fshiftorx	%f2, %f2, $fshift, %f8
	stda		%f8, [$out + $mask]0xc0	! partial store

	ret
	restore
.type	aes_fx_ctr32_encrypt_blocks,#function
.size	aes_fx_ctr32_encrypt_blocks,.-aes_fx_ctr32_encrypt_blocks

.align	32
.Linp_align:		! fshiftorx parameters for left shift toward %rs1
	.byte	0, 0, 64,  0,	0, 64,  0, -64
	.byte	0, 0, 56,  8,	0, 56,  8, -56
	.byte	0, 0, 48, 16,	0, 48, 16, -48
	.byte	0, 0, 40, 24,	0, 40, 24, -40
	.byte	0, 0, 32, 32,	0, 32, 32, -32
	.byte	0, 0, 24, 40,	0, 24, 40, -24
	.byte	0, 0, 16, 48,	0, 16, 48, -16
	.byte	0, 0,  8, 56,	0,  8, 56, -8
.Lout_align:		! fshiftorx parameters for right shift toward %rs2
	.byte	0, 0,  0, 64,	0,  0, 64,   0
	.byte	0, 0,  8, 56,	0,  8, 56,  -8
	.byte	0, 0, 16, 48,	0, 16, 48, -16
	.byte	0, 0, 24, 40,	0, 24, 40, -24
	.byte	0, 0, 32, 32,	0, 32, 32, -32
	.byte	0, 0, 40, 24,	0, 40, 24, -40
	.byte	0, 0, 48, 16,	0, 48, 16, -48
	.byte	0, 0, 56,  8,	0, 56,  8, -56
.Lone:
	.word	0, 1
.asciz	"AES for Fujitsu SPARC64 X, CRYPTOGAMS by <appro\@openssl.org>"
.align	4
___
}
# Purpose of these subroutines is to explicitly encode VIS instructions,
# so that one can compile the module without having to specify VIS
# extensions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
# Idea is to reserve for option to produce "universal" binary and let
# programmer detect if current CPU is VIS capable at run-time.
sub unvis {
my ($mnemonic,$rs1,$rs2,$rd)=@_;
my ($ref,$opf);
my %visopf = (	"faligndata"	=> 0x048,
		"bshuffle"	=> 0x04c,
		"fpadd32"	=> 0x052,
		"fxor"		=> 0x06c,
		"fsrc2"		=> 0x078	);

    $ref = "$mnemonic\t$rs1,$rs2,$rd";

    if ($opf=$visopf{$mnemonic}) {
	foreach ($rs1,$rs2,$rd) {
	    return $ref if (!/%f([0-9]{1,2})/);
	    $_=$1;
	    if ($1>=32) {
		return $ref if ($1&1);
		# re-encode for upper double register addressing
		$_=($1|$1>>5)&31;
	    }
	}

	return	sprintf ".word\t0x%08x !%s",
			0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
			$ref;
    } else {
	return $ref;
    }
}

sub unvis3 {
my ($mnemonic,$rs1,$rs2,$rd)=@_;
my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
my ($ref,$opf);
my %visopf = (	"alignaddr"	=> 0x018,
		"bmask"		=> 0x019,
		"alignaddrl"	=> 0x01a	);

    $ref = "$mnemonic\t$rs1,$rs2,$rd";

    if ($opf=$visopf{$mnemonic}) {
	foreach ($rs1,$rs2,$rd) {
	    return $ref if (!/%([goli])([0-9])/);
	    $_=$bias{$1}+$2;
	}

	return	sprintf ".word\t0x%08x !%s",
			0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
			$ref;
    } else {
	return $ref;
    }
}

sub unfx {
my ($mnemonic,$rs1,$rs2,$rd)=@_;
my ($ref,$opf);
my %aesopf = (	"faesencx"	=> 0x90,
		"faesdecx"	=> 0x91,
		"faesenclx"	=> 0x92,
		"faesdeclx"	=> 0x93,
		"faeskeyx"	=> 0x94	);

    $ref = "$mnemonic\t$rs1,$rs2,$rd";

    if (defined($opf=$aesopf{$mnemonic})) {
	$rs2 = ($rs2 =~ /%f([0-6]*[02468])/) ? (($1|$1>>5)&31) : $rs2;
	$rs2 = oct($rs2) if ($rs2 =~ /^0/);

	foreach ($rs1,$rd) {
	    return $ref if (!/%f([0-9]{1,2})/);
	    $_=$1;
	    if ($1>=32) {
		return $ref if ($1&1);
		# re-encode for upper double register addressing
		$_=($1|$1>>5)&31;
	    }
	}

	return	sprintf ".word\t0x%08x !%s",
			2<<30|$rd<<25|0x36<<19|$rs1<<14|$opf<<5|$rs2,
			$ref;
    } else {
	return $ref;
    }
}

sub unfx3src {
my ($mnemonic,$rs1,$rs2,$rs3,$rd)=@_;
my ($ref,$opf);
my %aesopf = (	"fshiftorx"	=> 0x0b	);

    $ref = "$mnemonic\t$rs1,$rs2,$rs3,$rd";

    if (defined($opf=$aesopf{$mnemonic})) {
	foreach ($rs1,$rs2,$rs3,$rd) {
	    return $ref if (!/%f([0-9]{1,2})/);
	    $_=$1;
	    if ($1>=32) {
		return $ref if ($1&1);
		# re-encode for upper double register addressing
		$_=($1|$1>>5)&31;
	    }
	}

	return	sprintf ".word\t0x%08x !%s",
			2<<30|$rd<<25|0x37<<19|$rs1<<14|$rs3<<9|$opf<<5|$rs2,
			$ref;
    } else {
	return $ref;
    }
}

foreach (split("\n",$code)) {
    s/\`([^\`]*)\`/eval $1/ge;

    s/%f([0-9]+)#lo/sprintf "%%f%d",$1+1/ge;

    s/\b(faes[^x]{3,4}x)\s+(%f[0-9]{1,2}),\s*([%fx0-9]+),\s*(%f[0-9]{1,2})/
		&unfx($1,$2,$3,$4)
     /ge or
    s/\b([f][^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/
		&unfx3src($1,$2,$3,$4,$5)
     /ge or
    s/\b([fb][^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/
		&unvis($1,$2,$3,$4)
     /ge or
    s/\b(alignaddr[l]*)\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/
		&unvis3($1,$2,$3,$4)
     /ge;
    print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# Multi-buffer AES-NI procedures process several independent buffers
# in parallel by interleaving independent instructions.
#
# Cycles per byte for interleave factor 4:
#
#			asymptotic	measured
#			---------------------------
# Westmere		5.00/4=1.25	5.13/4=1.28
# Atom			15.0/4=3.75	?15.7/4=3.93
# Sandy Bridge		5.06/4=1.27	5.18/4=1.29
# Ivy Bridge		5.06/4=1.27	5.14/4=1.29
# Haswell		4.44/4=1.11	4.44/4=1.11
# Bulldozer		5.75/4=1.44	5.76/4=1.44
#
# Cycles per byte for interleave factor 8 (not implemented for
# pre-AVX processors, where higher interleave factor incidentally
# doesn't result in improvement):
#
#			asymptotic	measured
#			---------------------------
# Sandy Bridge		5.06/8=0.64	7.10/8=0.89(*)
# Ivy Bridge		5.06/8=0.64	7.14/8=0.89(*)
# Haswell		5.00/8=0.63	5.00/8=0.63
# Bulldozer		5.75/8=0.72	5.77/8=0.72
#
# (*)	Sandy/Ivy Bridge are known to handle high interleave factors
#	suboptimally;

$flavour = shift;
$output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

$avx=0;

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.19) + ($1>=2.22);
}

if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
	   `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
	$avx = ($1>=2.09) + ($1>=2.10);
}

if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
	   `ml64 2>&1` =~ /Version ([0-9]+)\./) {
	$avx = ($1>=10) + ($1>=11);
}

if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
	$avx = ($2>=3.0) + ($2>3.0);
}

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

# void aesni_multi_cbc_encrypt (
#     struct {	void *inp,*out; int blocks; double iv[2]; } inp[8];
#     const AES_KEY *key,
#     int num);		/* 1 or 2 */
#
$inp="%rdi";	# 1st arg
$key="%rsi";	# 2nd arg
$num="%edx";

@inptr=map("%r$_",(8..11));
@outptr=map("%r$_",(12..15));

($rndkey0,$rndkey1)=("%xmm0","%xmm1");
@out=map("%xmm$_",(2..5));
@inp=map("%xmm$_",(6..9));
($counters,$mask,$zero)=map("%xmm$_",(10..12));

($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx");

$code.=<<___;
.text

.extern	OPENSSL_ia32cap_P

.globl	aesni_multi_cbc_encrypt
.type	aesni_multi_cbc_encrypt,\@function,3
.align	32
aesni_multi_cbc_encrypt:
.cfi_startproc
___
$code.=<<___ if ($avx);
	cmp	\$2,$num
	jb	.Lenc_non_avx
	mov	OPENSSL_ia32cap_P+4(%rip),%ecx
	test	\$`1<<28`,%ecx			# AVX bit
	jnz	_avx_cbc_enc_shortcut
	jmp	.Lenc_non_avx
.align	16
.Lenc_non_avx:
___
$code.=<<___;
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,0x40(%rsp)
	movaps	%xmm11,0x50(%rsp)
	movaps	%xmm12,0x60(%rsp)
	movaps	%xmm13,-0x68(%rax)	# not used, saved to share se_handler
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
___
$code.=<<___;
	# stack layout
	#
	# +0	output sink
	# +16	input sink [original %rsp and $num]
	# +32	counters

	sub	\$48,%rsp
	and	\$-64,%rsp
	mov	%rax,16(%rsp)			# original %rsp
.cfi_cfa_expression	%rsp+16,deref,+8

.Lenc4x_body:
	movdqu	($key),$zero			# 0-round key
	lea	0x78($key),$key			# size optimization
	lea	40*2($inp),$inp

.Lenc4x_loop_grande:
	mov	$num,24(%rsp)			# original $num
	xor	$num,$num
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	mov	`40*$i+16-40*2`($inp),$one	# borrow $one for number of blocks
	mov	`40*$i+0-40*2`($inp),@inptr[$i]
	cmp	$num,$one
	mov	`40*$i+8-40*2`($inp),@outptr[$i]
	cmovg	$one,$num			# find maximum
	test	$one,$one
	movdqu	`40*$i+24-40*2`($inp),@out[$i]	# load IV
	mov	$one,`32+4*$i`(%rsp)		# initialize counters
	cmovle	%rsp,@inptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Lenc4x_done

	movups	0x10-0x78($key),$rndkey1
	 pxor	$zero,@out[0]
	movups	0x20-0x78($key),$rndkey0
	 pxor	$zero,@out[1]
	mov	0xf0-0x78($key),$rounds
	 pxor	$zero,@out[2]
	movdqu	(@inptr[0]),@inp[0]		# load inputs
	 pxor	$zero,@out[3]
	movdqu	(@inptr[1]),@inp[1]
	 pxor	@inp[0],@out[0]
	movdqu	(@inptr[2]),@inp[2]
	 pxor	@inp[1],@out[1]
	movdqu	(@inptr[3]),@inp[3]
	 pxor	@inp[2],@out[2]
	 pxor	@inp[3],@out[3]
	movdqa	32(%rsp),$counters		# load counters
	xor	$offset,$offset
	jmp	.Loop_enc4x

.align	32
.Loop_enc4x:
	add	\$16,$offset
	lea	16(%rsp),$sink			# sink pointer
	mov	\$1,$one			# constant of 1
	sub	$offset,$sink

	aesenc		$rndkey1,@out[0]
	prefetcht0	31(@inptr[0],$offset)	# prefetch input
	prefetcht0	31(@inptr[1],$offset)
	aesenc		$rndkey1,@out[1]
	prefetcht0	31(@inptr[2],$offset)
	prefetcht0	31(@inptr[2],$offset)
	aesenc		$rndkey1,@out[2]
	aesenc		$rndkey1,@out[3]
	movups		0x30-0x78($key),$rndkey1
___
for($i=0;$i<4;$i++) {
my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
$code.=<<___;
	 cmp		`32+4*$i`(%rsp),$one
	aesenc		$rndkey,@out[0]
	aesenc		$rndkey,@out[1]
	aesenc		$rndkey,@out[2]
	 cmovge		$sink,@inptr[$i]	# cancel input
	 cmovg		$sink,@outptr[$i]	# sink output
	aesenc		$rndkey,@out[3]
	movups		`0x40+16*$i-0x78`($key),$rndkey
___
}
$code.=<<___;
	 movdqa		$counters,$mask
	aesenc		$rndkey0,@out[0]
	prefetcht0	15(@outptr[0],$offset)	# prefetch output
	prefetcht0	15(@outptr[1],$offset)
	aesenc		$rndkey0,@out[1]
	prefetcht0	15(@outptr[2],$offset)
	prefetcht0	15(@outptr[3],$offset)
	aesenc		$rndkey0,@out[2]
	aesenc		$rndkey0,@out[3]
	movups		0x80-0x78($key),$rndkey0
	 pxor		$zero,$zero

	aesenc		$rndkey1,@out[0]
	 pcmpgtd	$zero,$mask
	 movdqu		-0x78($key),$zero	# reload 0-round key
	aesenc		$rndkey1,@out[1]
	 paddd		$mask,$counters		# decrement counters
	 movdqa		$counters,32(%rsp)	# update counters
	aesenc		$rndkey1,@out[2]
	aesenc		$rndkey1,@out[3]
	movups		0x90-0x78($key),$rndkey1

	cmp	\$11,$rounds

	aesenc		$rndkey0,@out[0]
	aesenc		$rndkey0,@out[1]
	aesenc		$rndkey0,@out[2]
	aesenc		$rndkey0,@out[3]
	movups		0xa0-0x78($key),$rndkey0

	jb	.Lenc4x_tail

	aesenc		$rndkey1,@out[0]
	aesenc		$rndkey1,@out[1]
	aesenc		$rndkey1,@out[2]
	aesenc		$rndkey1,@out[3]
	movups		0xb0-0x78($key),$rndkey1

	aesenc		$rndkey0,@out[0]
	aesenc		$rndkey0,@out[1]
	aesenc		$rndkey0,@out[2]
	aesenc		$rndkey0,@out[3]
	movups		0xc0-0x78($key),$rndkey0

	je	.Lenc4x_tail

	aesenc		$rndkey1,@out[0]
	aesenc		$rndkey1,@out[1]
	aesenc		$rndkey1,@out[2]
	aesenc		$rndkey1,@out[3]
	movups		0xd0-0x78($key),$rndkey1

	aesenc		$rndkey0,@out[0]
	aesenc		$rndkey0,@out[1]
	aesenc		$rndkey0,@out[2]
	aesenc		$rndkey0,@out[3]
	movups		0xe0-0x78($key),$rndkey0
	jmp	.Lenc4x_tail

.align	32
.Lenc4x_tail:
	aesenc		$rndkey1,@out[0]
	aesenc		$rndkey1,@out[1]
	aesenc		$rndkey1,@out[2]
	aesenc		$rndkey1,@out[3]
	 movdqu		(@inptr[0],$offset),@inp[0]
	movdqu		0x10-0x78($key),$rndkey1

	aesenclast	$rndkey0,@out[0]
	 movdqu		(@inptr[1],$offset),@inp[1]
	 pxor		$zero,@inp[0]
	aesenclast	$rndkey0,@out[1]
	 movdqu		(@inptr[2],$offset),@inp[2]
	 pxor		$zero,@inp[1]
	aesenclast	$rndkey0,@out[2]
	 movdqu		(@inptr[3],$offset),@inp[3]
	 pxor		$zero,@inp[2]
	aesenclast	$rndkey0,@out[3]
	movdqu		0x20-0x78($key),$rndkey0
	 pxor		$zero,@inp[3]

	movups		@out[0],-16(@outptr[0],$offset)
	 pxor		@inp[0],@out[0]
	movups		@out[1],-16(@outptr[1],$offset)
	 pxor		@inp[1],@out[1]
	movups		@out[2],-16(@outptr[2],$offset)
	 pxor		@inp[2],@out[2]
	movups		@out[3],-16(@outptr[3],$offset)
	 pxor		@inp[3],@out[3]

	dec	$num
	jnz	.Loop_enc4x

	mov	16(%rsp),%rax			# original %rsp
.cfi_def_cfa	%rax,8
	mov	24(%rsp),$num

	#pxor	@inp[0],@out[0]
	#pxor	@inp[1],@out[1]
	#movdqu	@out[0],`40*0+24-40*2`($inp)	# output iv FIX ME!
	#pxor	@inp[2],@out[2]
	#movdqu	@out[1],`40*1+24-40*2`($inp)
	#pxor	@inp[3],@out[3]
	#movdqu	@out[2],`40*2+24-40*2`($inp)	# won't fix, let caller
	#movdqu	@out[3],`40*3+24-40*2`($inp)	# figure this out...

	lea	`40*4`($inp),$inp
	dec	$num
	jnz	.Lenc4x_loop_grande

.Lenc4x_done:
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	#movaps	-0x68(%rax),%xmm13
	#movaps	-0x58(%rax),%xmm14
	#movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lenc4x_epilogue:
	ret
.cfi_endproc
.size	aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt

.globl	aesni_multi_cbc_decrypt
.type	aesni_multi_cbc_decrypt,\@function,3
.align	32
aesni_multi_cbc_decrypt:
.cfi_startproc
___
$code.=<<___ if ($avx);
	cmp	\$2,$num
	jb	.Ldec_non_avx
	mov	OPENSSL_ia32cap_P+4(%rip),%ecx
	test	\$`1<<28`,%ecx			# AVX bit
	jnz	_avx_cbc_dec_shortcut
	jmp	.Ldec_non_avx
.align	16
.Ldec_non_avx:
___
$code.=<<___;
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,0x40(%rsp)
	movaps	%xmm11,0x50(%rsp)
	movaps	%xmm12,0x60(%rsp)
	movaps	%xmm13,-0x68(%rax)	# not used, saved to share se_handler
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
___
$code.=<<___;
	# stack layout
	#
	# +0	output sink
	# +16	input sink [original %rsp and $num]
	# +32	counters

	sub	\$48,%rsp
	and	\$-64,%rsp
	mov	%rax,16(%rsp)			# original %rsp
.cfi_cfa_expression	%rsp+16,deref,+8

.Ldec4x_body:
	movdqu	($key),$zero			# 0-round key
	lea	0x78($key),$key			# size optimization
	lea	40*2($inp),$inp

.Ldec4x_loop_grande:
	mov	$num,24(%rsp)			# original $num
	xor	$num,$num
___
for($i=0;$i<4;$i++) {
    $code.=<<___;
	mov	`40*$i+16-40*2`($inp),$one	# borrow $one for number of blocks
	mov	`40*$i+0-40*2`($inp),@inptr[$i]
	cmp	$num,$one
	mov	`40*$i+8-40*2`($inp),@outptr[$i]
	cmovg	$one,$num			# find maximum
	test	$one,$one
	movdqu	`40*$i+24-40*2`($inp),@inp[$i]	# load IV
	mov	$one,`32+4*$i`(%rsp)		# initialize counters
	cmovle	%rsp,@inptr[$i]			# cancel input
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldec4x_done

	movups	0x10-0x78($key),$rndkey1
	movups	0x20-0x78($key),$rndkey0
	mov	0xf0-0x78($key),$rounds
	movdqu	(@inptr[0]),@out[0]		# load inputs
	movdqu	(@inptr[1]),@out[1]
	 pxor	$zero,@out[0]
	movdqu	(@inptr[2]),@out[2]
	 pxor	$zero,@out[1]
	movdqu	(@inptr[3]),@out[3]
	 pxor	$zero,@out[2]
	 pxor	$zero,@out[3]
	movdqa	32(%rsp),$counters		# load counters
	xor	$offset,$offset
	jmp	.Loop_dec4x

.align	32
.Loop_dec4x:
	add	\$16,$offset
	lea	16(%rsp),$sink			# sink pointer
	mov	\$1,$one			# constant of 1
	sub	$offset,$sink

	aesdec		$rndkey1,@out[0]
	prefetcht0	31(@inptr[0],$offset)	# prefetch input
	prefetcht0	31(@inptr[1],$offset)
	aesdec		$rndkey1,@out[1]
	prefetcht0	31(@inptr[2],$offset)
	prefetcht0	31(@inptr[3],$offset)
	aesdec		$rndkey1,@out[2]
	aesdec		$rndkey1,@out[3]
	movups		0x30-0x78($key),$rndkey1
___
for($i=0;$i<4;$i++) {
my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
$code.=<<___;
	 cmp		`32+4*$i`(%rsp),$one
	aesdec		$rndkey,@out[0]
	aesdec		$rndkey,@out[1]
	aesdec		$rndkey,@out[2]
	 cmovge		$sink,@inptr[$i]	# cancel input
	 cmovg		$sink,@outptr[$i]	# sink output
	aesdec		$rndkey,@out[3]
	movups		`0x40+16*$i-0x78`($key),$rndkey
___
}
$code.=<<___;
	 movdqa		$counters,$mask
	aesdec		$rndkey0,@out[0]
	prefetcht0	15(@outptr[0],$offset)	# prefetch output
	prefetcht0	15(@outptr[1],$offset)
	aesdec		$rndkey0,@out[1]
	prefetcht0	15(@outptr[2],$offset)
	prefetcht0	15(@outptr[3],$offset)
	aesdec		$rndkey0,@out[2]
	aesdec		$rndkey0,@out[3]
	movups		0x80-0x78($key),$rndkey0
	 pxor		$zero,$zero

	aesdec		$rndkey1,@out[0]
	 pcmpgtd	$zero,$mask
	 movdqu		-0x78($key),$zero	# reload 0-round key
	aesdec		$rndkey1,@out[1]
	 paddd		$mask,$counters		# decrement counters
	 movdqa		$counters,32(%rsp)	# update counters
	aesdec		$rndkey1,@out[2]
	aesdec		$rndkey1,@out[3]
	movups		0x90-0x78($key),$rndkey1

	cmp	\$11,$rounds

	aesdec		$rndkey0,@out[0]
	aesdec		$rndkey0,@out[1]
	aesdec		$rndkey0,@out[2]
	aesdec		$rndkey0,@out[3]
	movups		0xa0-0x78($key),$rndkey0

	jb	.Ldec4x_tail

	aesdec		$rndkey1,@out[0]
	aesdec		$rndkey1,@out[1]
	aesdec		$rndkey1,@out[2]
	aesdec		$rndkey1,@out[3]
	movups		0xb0-0x78($key),$rndkey1

	aesdec		$rndkey0,@out[0]
	aesdec		$rndkey0,@out[1]
	aesdec		$rndkey0,@out[2]
	aesdec		$rndkey0,@out[3]
	movups		0xc0-0x78($key),$rndkey0

	je	.Ldec4x_tail

	aesdec		$rndkey1,@out[0]
	aesdec		$rndkey1,@out[1]
	aesdec		$rndkey1,@out[2]
	aesdec		$rndkey1,@out[3]
	movups		0xd0-0x78($key),$rndkey1

	aesdec		$rndkey0,@out[0]
	aesdec		$rndkey0,@out[1]
	aesdec		$rndkey0,@out[2]
	aesdec		$rndkey0,@out[3]
	movups		0xe0-0x78($key),$rndkey0
	jmp	.Ldec4x_tail

.align	32
.Ldec4x_tail:
	aesdec		$rndkey1,@out[0]
	aesdec		$rndkey1,@out[1]
	aesdec		$rndkey1,@out[2]
	 pxor		$rndkey0,@inp[0]
	 pxor		$rndkey0,@inp[1]
	aesdec		$rndkey1,@out[3]
	movdqu		0x10-0x78($key),$rndkey1
	 pxor		$rndkey0,@inp[2]
	 pxor		$rndkey0,@inp[3]
	movdqu		0x20-0x78($key),$rndkey0

	aesdeclast	@inp[0],@out[0]
	aesdeclast	@inp[1],@out[1]
	 movdqu		-16(@inptr[0],$offset),@inp[0]	# load next IV
	 movdqu		-16(@inptr[1],$offset),@inp[1]
	aesdeclast	@inp[2],@out[2]
	aesdeclast	@inp[3],@out[3]
	 movdqu		-16(@inptr[2],$offset),@inp[2]
	 movdqu		-16(@inptr[3],$offset),@inp[3]

	movups		@out[0],-16(@outptr[0],$offset)
	 movdqu		(@inptr[0],$offset),@out[0]
	movups		@out[1],-16(@outptr[1],$offset)
	 movdqu		(@inptr[1],$offset),@out[1]
	 pxor		$zero,@out[0]
	movups		@out[2],-16(@outptr[2],$offset)
	 movdqu		(@inptr[2],$offset),@out[2]
	 pxor		$zero,@out[1]
	movups		@out[3],-16(@outptr[3],$offset)
	 movdqu		(@inptr[3],$offset),@out[3]
	 pxor		$zero,@out[2]
	 pxor		$zero,@out[3]

	dec	$num
	jnz	.Loop_dec4x

	mov	16(%rsp),%rax			# original %rsp
.cfi_def_cfa	%rax,8
	mov	24(%rsp),$num

	lea	`40*4`($inp),$inp
	dec	$num
	jnz	.Ldec4x_loop_grande

.Ldec4x_done:
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	#movaps	-0x68(%rax),%xmm13
	#movaps	-0x58(%rax),%xmm14
	#movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Ldec4x_epilogue:
	ret
.cfi_endproc
.size	aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt
___

						if ($avx) {{{
my @ptr=map("%r$_",(8..15));
my $offload=$sink;

my @out=map("%xmm$_",(2..9));
my @inp=map("%xmm$_",(10..13));
my ($counters,$zero)=("%xmm14","%xmm15");

$code.=<<___;
.type	aesni_multi_cbc_encrypt_avx,\@function,3
.align	32
aesni_multi_cbc_encrypt_avx:
.cfi_startproc
_avx_cbc_enc_shortcut:
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,0x40(%rsp)
	movaps	%xmm11,0x50(%rsp)
	movaps	%xmm12,-0x78(%rax)
	movaps	%xmm13,-0x68(%rax)
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
___
$code.=<<___;
	# stack layout
	#
	# +0	output sink
	# +16	input sink [original %rsp and $num]
	# +32	counters
	# +64	distances between inputs and outputs
	# +128	off-load area for @inp[0..3]

	sub	\$192,%rsp
	and	\$-128,%rsp
	mov	%rax,16(%rsp)			# original %rsp
.cfi_cfa_expression	%rsp+16,deref,+8

.Lenc8x_body:
	vzeroupper
	vmovdqu	($key),$zero			# 0-round key
	lea	0x78($key),$key			# size optimization
	lea	40*4($inp),$inp
	shr	\$1,$num

.Lenc8x_loop_grande:
	#mov	$num,24(%rsp)			# original $num
	xor	$num,$num
___
for($i=0;$i<8;$i++) {
  my $temp = $i ? $offload : $offset;
    $code.=<<___;
	mov	`40*$i+16-40*4`($inp),$one	# borrow $one for number of blocks
	mov	`40*$i+0-40*4`($inp),@ptr[$i]	# input pointer
	cmp	$num,$one
	mov	`40*$i+8-40*4`($inp),$temp	# output pointer
	cmovg	$one,$num			# find maximum
	test	$one,$one
	vmovdqu	`40*$i+24-40*4`($inp),@out[$i]	# load IV
	mov	$one,`32+4*$i`(%rsp)		# initialize counters
	cmovle	%rsp,@ptr[$i]			# cancel input
	sub	@ptr[$i],$temp			# distance between input and output
	mov	$temp,`64+8*$i`(%rsp)		# initialize distances
___
}
$code.=<<___;
	test	$num,$num
	jz	.Lenc8x_done

	vmovups	0x10-0x78($key),$rndkey1
	vmovups	0x20-0x78($key),$rndkey0
	mov	0xf0-0x78($key),$rounds

	vpxor	(@ptr[0]),$zero,@inp[0]		# load inputs and xor with 0-round
	 lea	128(%rsp),$offload		# offload area
	vpxor	(@ptr[1]),$zero,@inp[1]
	vpxor	(@ptr[2]),$zero,@inp[2]
	vpxor	(@ptr[3]),$zero,@inp[3]
	 vpxor	@inp[0],@out[0],@out[0]
	vpxor	(@ptr[4]),$zero,@inp[0]
	 vpxor	@inp[1],@out[1],@out[1]
	vpxor	(@ptr[5]),$zero,@inp[1]
	 vpxor	@inp[2],@out[2],@out[2]
	vpxor	(@ptr[6]),$zero,@inp[2]
	 vpxor	@inp[3],@out[3],@out[3]
	vpxor	(@ptr[7]),$zero,@inp[3]
	 vpxor	@inp[0],@out[4],@out[4]
	mov	\$1,$one			# constant of 1
	 vpxor	@inp[1],@out[5],@out[5]
	 vpxor	@inp[2],@out[6],@out[6]
	 vpxor	@inp[3],@out[7],@out[7]
	jmp	.Loop_enc8x

.align	32
.Loop_enc8x:
___
for($i=0;$i<8;$i++) {
my $rndkey=($i&1)?$rndkey0:$rndkey1;
$code.=<<___;
	vaesenc		$rndkey,@out[0],@out[0]
	 cmp		32+4*$i(%rsp),$one
___
$code.=<<___ if ($i);
	 mov		64+8*$i(%rsp),$offset
___
$code.=<<___;
	vaesenc		$rndkey,@out[1],@out[1]
	prefetcht0	31(@ptr[$i])			# prefetch input
	vaesenc		$rndkey,@out[2],@out[2]
___
$code.=<<___ if ($i>1);
	prefetcht0	15(@ptr[$i-2])			# prefetch output
___
$code.=<<___;
	vaesenc		$rndkey,@out[3],@out[3]
	 lea		(@ptr[$i],$offset),$offset
	 cmovge		%rsp,@ptr[$i]			# cancel input
	vaesenc		$rndkey,@out[4],@out[4]
	 cmovg		%rsp,$offset			# sink output
	vaesenc		$rndkey,@out[5],@out[5]
	 sub		@ptr[$i],$offset
	vaesenc		$rndkey,@out[6],@out[6]
	 vpxor		16(@ptr[$i]),$zero,@inp[$i%4]	# load input and xor with 0-round
	 mov		$offset,64+8*$i(%rsp)
	vaesenc		$rndkey,@out[7],@out[7]
	vmovups		`16*(3+$i)-0x78`($key),$rndkey
	 lea		16(@ptr[$i],$offset),@ptr[$i]	# switch to output
___
$code.=<<___ if ($i<4)
	 vmovdqu	@inp[$i%4],`16*$i`($offload)	# off-load
___
}
$code.=<<___;
	 vmovdqu	32(%rsp),$counters
	prefetcht0	15(@ptr[$i-2])			# prefetch output
	prefetcht0	15(@ptr[$i-1])
	cmp	\$11,$rounds
	jb	.Lenc8x_tail

	vaesenc		$rndkey1,@out[0],@out[0]
	vaesenc		$rndkey1,@out[1],@out[1]
	vaesenc		$rndkey1,@out[2],@out[2]
	vaesenc		$rndkey1,@out[3],@out[3]
	vaesenc		$rndkey1,@out[4],@out[4]
	vaesenc		$rndkey1,@out[5],@out[5]
	vaesenc		$rndkey1,@out[6],@out[6]
	vaesenc		$rndkey1,@out[7],@out[7]
	vmovups		0xb0-0x78($key),$rndkey1

	vaesenc		$rndkey0,@out[0],@out[0]
	vaesenc		$rndkey0,@out[1],@out[1]
	vaesenc		$rndkey0,@out[2],@out[2]
	vaesenc		$rndkey0,@out[3],@out[3]
	vaesenc		$rndkey0,@out[4],@out[4]
	vaesenc		$rndkey0,@out[5],@out[5]
	vaesenc		$rndkey0,@out[6],@out[6]
	vaesenc		$rndkey0,@out[7],@out[7]
	vmovups		0xc0-0x78($key),$rndkey0
	je	.Lenc8x_tail

	vaesenc		$rndkey1,@out[0],@out[0]
	vaesenc		$rndkey1,@out[1],@out[1]
	vaesenc		$rndkey1,@out[2],@out[2]
	vaesenc		$rndkey1,@out[3],@out[3]
	vaesenc		$rndkey1,@out[4],@out[4]
	vaesenc		$rndkey1,@out[5],@out[5]
	vaesenc		$rndkey1,@out[6],@out[6]
	vaesenc		$rndkey1,@out[7],@out[7]
	vmovups		0xd0-0x78($key),$rndkey1

	vaesenc		$rndkey0,@out[0],@out[0]
	vaesenc		$rndkey0,@out[1],@out[1]
	vaesenc		$rndkey0,@out[2],@out[2]
	vaesenc		$rndkey0,@out[3],@out[3]
	vaesenc		$rndkey0,@out[4],@out[4]
	vaesenc		$rndkey0,@out[5],@out[5]
	vaesenc		$rndkey0,@out[6],@out[6]
	vaesenc		$rndkey0,@out[7],@out[7]
	vmovups		0xe0-0x78($key),$rndkey0

.Lenc8x_tail:
	vaesenc		$rndkey1,@out[0],@out[0]
	 vpxor		$zero,$zero,$zero
	vaesenc		$rndkey1,@out[1],@out[1]
	vaesenc		$rndkey1,@out[2],@out[2]
	 vpcmpgtd	$zero,$counters,$zero
	vaesenc		$rndkey1,@out[3],@out[3]
	vaesenc		$rndkey1,@out[4],@out[4]
	 vpaddd		$counters,$zero,$zero		# decrement counters
	 vmovdqu	48(%rsp),$counters
	vaesenc		$rndkey1,@out[5],@out[5]
	 mov		64(%rsp),$offset		# pre-load 1st offset
	vaesenc		$rndkey1,@out[6],@out[6]
	vaesenc		$rndkey1,@out[7],@out[7]
	vmovups		0x10-0x78($key),$rndkey1

	vaesenclast	$rndkey0,@out[0],@out[0]
	 vmovdqa	$zero,32(%rsp)			# update counters
	 vpxor		$zero,$zero,$zero
	vaesenclast	$rndkey0,@out[1],@out[1]
	vaesenclast	$rndkey0,@out[2],@out[2]
	 vpcmpgtd	$zero,$counters,$zero
	vaesenclast	$rndkey0,@out[3],@out[3]
	vaesenclast	$rndkey0,@out[4],@out[4]
	 vpaddd		$zero,$counters,$counters	# decrement counters
	 vmovdqu	-0x78($key),$zero		# 0-round
	vaesenclast	$rndkey0,@out[5],@out[5]
	vaesenclast	$rndkey0,@out[6],@out[6]
	 vmovdqa	$counters,48(%rsp)		# update counters
	vaesenclast	$rndkey0,@out[7],@out[7]
	vmovups		0x20-0x78($key),$rndkey0

	vmovups		@out[0],-16(@ptr[0])		# write output
	 sub		$offset,@ptr[0]			# switch to input
	 vpxor		0x00($offload),@out[0],@out[0]
	vmovups		@out[1],-16(@ptr[1])
	 sub		`64+1*8`(%rsp),@ptr[1]
	 vpxor		0x10($offload),@out[1],@out[1]
	vmovups		@out[2],-16(@ptr[2])
	 sub		`64+2*8`(%rsp),@ptr[2]
	 vpxor		0x20($offload),@out[2],@out[2]
	vmovups		@out[3],-16(@ptr[3])
	 sub		`64+3*8`(%rsp),@ptr[3]
	 vpxor		0x30($offload),@out[3],@out[3]
	vmovups		@out[4],-16(@ptr[4])
	 sub		`64+4*8`(%rsp),@ptr[4]
	 vpxor		@inp[0],@out[4],@out[4]
	vmovups		@out[5],-16(@ptr[5])
	 sub		`64+5*8`(%rsp),@ptr[5]
	 vpxor		@inp[1],@out[5],@out[5]
	vmovups		@out[6],-16(@ptr[6])
	 sub		`64+6*8`(%rsp),@ptr[6]
	 vpxor		@inp[2],@out[6],@out[6]
	vmovups		@out[7],-16(@ptr[7])
	 sub		`64+7*8`(%rsp),@ptr[7]
	 vpxor		@inp[3],@out[7],@out[7]

	dec	$num
	jnz	.Loop_enc8x

	mov	16(%rsp),%rax			# original %rsp
.cfi_def_cfa	%rax,8
	#mov	24(%rsp),$num
	#lea	`40*8`($inp),$inp
	#dec	$num
	#jnz	.Lenc8x_loop_grande

.Lenc8x_done:
	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	movaps	-0x68(%rax),%xmm13
	movaps	-0x58(%rax),%xmm14
	movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Lenc8x_epilogue:
	ret
.cfi_endproc
.size	aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx

.type	aesni_multi_cbc_decrypt_avx,\@function,3
.align	32
aesni_multi_cbc_decrypt_avx:
.cfi_startproc
_avx_cbc_dec_shortcut:
	mov	%rsp,%rax
.cfi_def_cfa_register	%rax
	push	%rbx
.cfi_push	%rbx
	push	%rbp
.cfi_push	%rbp
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
___
$code.=<<___ if ($win64);
	lea	-0xa8(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
	movaps	%xmm10,0x40(%rsp)
	movaps	%xmm11,0x50(%rsp)
	movaps	%xmm12,-0x78(%rax)
	movaps	%xmm13,-0x68(%rax)
	movaps	%xmm14,-0x58(%rax)
	movaps	%xmm15,-0x48(%rax)
___
$code.=<<___;
	# stack layout
	#
	# +0	output sink
	# +16	input sink [original %rsp and $num]
	# +32	counters
	# +64	distances between inputs and outputs
	# +128	off-load area for @inp[0..3]
	# +192	IV/input offload

	sub	\$256,%rsp
	and	\$-256,%rsp
	sub	\$192,%rsp
	mov	%rax,16(%rsp)			# original %rsp
.cfi_cfa_expression	%rsp+16,deref,+8

.Ldec8x_body:
	vzeroupper
	vmovdqu	($key),$zero			# 0-round key
	lea	0x78($key),$key			# size optimization
	lea	40*4($inp),$inp
	shr	\$1,$num

.Ldec8x_loop_grande:
	#mov	$num,24(%rsp)			# original $num
	xor	$num,$num
___
for($i=0;$i<8;$i++) {
  my $temp = $i ? $offload : $offset;
    $code.=<<___;
	mov	`40*$i+16-40*4`($inp),$one	# borrow $one for number of blocks
	mov	`40*$i+0-40*4`($inp),@ptr[$i]	# input pointer
	cmp	$num,$one
	mov	`40*$i+8-40*4`($inp),$temp	# output pointer
	cmovg	$one,$num			# find maximum
	test	$one,$one
	vmovdqu	`40*$i+24-40*4`($inp),@out[$i]	# load IV
	mov	$one,`32+4*$i`(%rsp)		# initialize counters
	cmovle	%rsp,@ptr[$i]			# cancel input
	sub	@ptr[$i],$temp			# distance between input and output
	mov	$temp,`64+8*$i`(%rsp)		# initialize distances
	vmovdqu	@out[$i],`192+16*$i`(%rsp)	# offload IV
___
}
$code.=<<___;
	test	$num,$num
	jz	.Ldec8x_done

	vmovups	0x10-0x78($key),$rndkey1
	vmovups	0x20-0x78($key),$rndkey0
	mov	0xf0-0x78($key),$rounds
	 lea	192+128(%rsp),$offload		# offload area

	vmovdqu	(@ptr[0]),@out[0]		# load inputs
	vmovdqu	(@ptr[1]),@out[1]
	vmovdqu	(@ptr[2]),@out[2]
	vmovdqu	(@ptr[3]),@out[3]
	vmovdqu	(@ptr[4]),@out[4]
	vmovdqu	(@ptr[5]),@out[5]
	vmovdqu	(@ptr[6]),@out[6]
	vmovdqu	(@ptr[7]),@out[7]
	vmovdqu	@out[0],0x00($offload)		# offload inputs
	vpxor	$zero,@out[0],@out[0]		# xor inputs with 0-round
	vmovdqu	@out[1],0x10($offload)
	vpxor	$zero,@out[1],@out[1]
	vmovdqu	@out[2],0x20($offload)
	vpxor	$zero,@out[2],@out[2]
	vmovdqu	@out[3],0x30($offload)
	vpxor	$zero,@out[3],@out[3]
	vmovdqu	@out[4],0x40($offload)
	vpxor	$zero,@out[4],@out[4]
	vmovdqu	@out[5],0x50($offload)
	vpxor	$zero,@out[5],@out[5]
	vmovdqu	@out[6],0x60($offload)
	vpxor	$zero,@out[6],@out[6]
	vmovdqu	@out[7],0x70($offload)
	vpxor	$zero,@out[7],@out[7]
	xor	\$0x80,$offload
	mov	\$1,$one			# constant of 1
	jmp	.Loop_dec8x

.align	32
.Loop_dec8x:
___
for($i=0;$i<8;$i++) {
my $rndkey=($i&1)?$rndkey0:$rndkey1;
$code.=<<___;
	vaesdec		$rndkey,@out[0],@out[0]
	 cmp		32+4*$i(%rsp),$one
___
$code.=<<___ if ($i);
	 mov		64+8*$i(%rsp),$offset
___
$code.=<<___;
	vaesdec		$rndkey,@out[1],@out[1]
	prefetcht0	31(@ptr[$i])			# prefetch input
	vaesdec		$rndkey,@out[2],@out[2]
___
$code.=<<___ if ($i>1);
	prefetcht0	15(@ptr[$i-2])			# prefetch output
___
$code.=<<___;
	vaesdec		$rndkey,@out[3],@out[3]
	 lea		(@ptr[$i],$offset),$offset
	 cmovge		%rsp,@ptr[$i]			# cancel input
	vaesdec		$rndkey,@out[4],@out[4]
	 cmovg		%rsp,$offset			# sink output
	vaesdec		$rndkey,@out[5],@out[5]
	 sub		@ptr[$i],$offset
	vaesdec		$rndkey,@out[6],@out[6]
	 vmovdqu	16(@ptr[$i]),@inp[$i%4]		# load input
	 mov		$offset,64+8*$i(%rsp)
	vaesdec		$rndkey,@out[7],@out[7]
	vmovups		`16*(3+$i)-0x78`($key),$rndkey
	 lea		16(@ptr[$i],$offset),@ptr[$i]	# switch to output
___
$code.=<<___ if ($i<4);
	 vmovdqu	@inp[$i%4],`128+16*$i`(%rsp)	# off-load
___
}
$code.=<<___;
	 vmovdqu	32(%rsp),$counters
	prefetcht0	15(@ptr[$i-2])			# prefetch output
	prefetcht0	15(@ptr[$i-1])
	cmp	\$11,$rounds
	jb	.Ldec8x_tail

	vaesdec		$rndkey1,@out[0],@out[0]
	vaesdec		$rndkey1,@out[1],@out[1]
	vaesdec		$rndkey1,@out[2],@out[2]
	vaesdec		$rndkey1,@out[3],@out[3]
	vaesdec		$rndkey1,@out[4],@out[4]
	vaesdec		$rndkey1,@out[5],@out[5]
	vaesdec		$rndkey1,@out[6],@out[6]
	vaesdec		$rndkey1,@out[7],@out[7]
	vmovups		0xb0-0x78($key),$rndkey1

	vaesdec		$rndkey0,@out[0],@out[0]
	vaesdec		$rndkey0,@out[1],@out[1]
	vaesdec		$rndkey0,@out[2],@out[2]
	vaesdec		$rndkey0,@out[3],@out[3]
	vaesdec		$rndkey0,@out[4],@out[4]
	vaesdec		$rndkey0,@out[5],@out[5]
	vaesdec		$rndkey0,@out[6],@out[6]
	vaesdec		$rndkey0,@out[7],@out[7]
	vmovups		0xc0-0x78($key),$rndkey0
	je	.Ldec8x_tail

	vaesdec		$rndkey1,@out[0],@out[0]
	vaesdec		$rndkey1,@out[1],@out[1]
	vaesdec		$rndkey1,@out[2],@out[2]
	vaesdec		$rndkey1,@out[3],@out[3]
	vaesdec		$rndkey1,@out[4],@out[4]
	vaesdec		$rndkey1,@out[5],@out[5]
	vaesdec		$rndkey1,@out[6],@out[6]
	vaesdec		$rndkey1,@out[7],@out[7]
	vmovups		0xd0-0x78($key),$rndkey1

	vaesdec		$rndkey0,@out[0],@out[0]
	vaesdec		$rndkey0,@out[1],@out[1]
	vaesdec		$rndkey0,@out[2],@out[2]
	vaesdec		$rndkey0,@out[3],@out[3]
	vaesdec		$rndkey0,@out[4],@out[4]
	vaesdec		$rndkey0,@out[5],@out[5]
	vaesdec		$rndkey0,@out[6],@out[6]
	vaesdec		$rndkey0,@out[7],@out[7]
	vmovups		0xe0-0x78($key),$rndkey0

.Ldec8x_tail:
	vaesdec		$rndkey1,@out[0],@out[0]
	 vpxor		$zero,$zero,$zero
	vaesdec		$rndkey1,@out[1],@out[1]
	vaesdec		$rndkey1,@out[2],@out[2]
	 vpcmpgtd	$zero,$counters,$zero
	vaesdec		$rndkey1,@out[3],@out[3]
	vaesdec		$rndkey1,@out[4],@out[4]
	 vpaddd		$counters,$zero,$zero		# decrement counters
	 vmovdqu	48(%rsp),$counters
	vaesdec		$rndkey1,@out[5],@out[5]
	 mov		64(%rsp),$offset		# pre-load 1st offset
	vaesdec		$rndkey1,@out[6],@out[6]
	vaesdec		$rndkey1,@out[7],@out[7]
	vmovups		0x10-0x78($key),$rndkey1

	vaesdeclast	$rndkey0,@out[0],@out[0]
	 vmovdqa	$zero,32(%rsp)			# update counters
	 vpxor		$zero,$zero,$zero
	vaesdeclast	$rndkey0,@out[1],@out[1]
	vpxor		0x00($offload),@out[0],@out[0]	# xor with IV
	vaesdeclast	$rndkey0,@out[2],@out[2]
	vpxor		0x10($offload),@out[1],@out[1]
	 vpcmpgtd	$zero,$counters,$zero
	vaesdeclast	$rndkey0,@out[3],@out[3]
	vpxor		0x20($offload),@out[2],@out[2]
	vaesdeclast	$rndkey0,@out[4],@out[4]
	vpxor		0x30($offload),@out[3],@out[3]
	 vpaddd		$zero,$counters,$counters	# decrement counters
	 vmovdqu	-0x78($key),$zero		# 0-round
	vaesdeclast	$rndkey0,@out[5],@out[5]
	vpxor		0x40($offload),@out[4],@out[4]
	vaesdeclast	$rndkey0,@out[6],@out[6]
	vpxor		0x50($offload),@out[5],@out[5]
	 vmovdqa	$counters,48(%rsp)		# update counters
	vaesdeclast	$rndkey0,@out[7],@out[7]
	vpxor		0x60($offload),@out[6],@out[6]
	vmovups		0x20-0x78($key),$rndkey0

	vmovups		@out[0],-16(@ptr[0])		# write output
	 sub		$offset,@ptr[0]			# switch to input
	 vmovdqu	128+0(%rsp),@out[0]
	vpxor		0x70($offload),@out[7],@out[7]
	vmovups		@out[1],-16(@ptr[1])
	 sub		`64+1*8`(%rsp),@ptr[1]
	 vmovdqu	@out[0],0x00($offload)
	 vpxor		$zero,@out[0],@out[0]
	 vmovdqu	128+16(%rsp),@out[1]
	vmovups		@out[2],-16(@ptr[2])
	 sub		`64+2*8`(%rsp),@ptr[2]
	 vmovdqu	@out[1],0x10($offload)
	 vpxor		$zero,@out[1],@out[1]
	 vmovdqu	128+32(%rsp),@out[2]
	vmovups		@out[3],-16(@ptr[3])
	 sub		`64+3*8`(%rsp),@ptr[3]
	 vmovdqu	@out[2],0x20($offload)
	 vpxor		$zero,@out[2],@out[2]
	 vmovdqu	128+48(%rsp),@out[3]
	vmovups		@out[4],-16(@ptr[4])
	 sub		`64+4*8`(%rsp),@ptr[4]
	 vmovdqu	@out[3],0x30($offload)
	 vpxor		$zero,@out[3],@out[3]
	 vmovdqu	@inp[0],0x40($offload)
	 vpxor		@inp[0],$zero,@out[4]
	vmovups		@out[5],-16(@ptr[5])
	 sub		`64+5*8`(%rsp),@ptr[5]
	 vmovdqu	@inp[1],0x50($offload)
	 vpxor		@inp[1],$zero,@out[5]
	vmovups		@out[6],-16(@ptr[6])
	 sub		`64+6*8`(%rsp),@ptr[6]
	 vmovdqu	@inp[2],0x60($offload)
	 vpxor		@inp[2],$zero,@out[6]
	vmovups		@out[7],-16(@ptr[7])
	 sub		`64+7*8`(%rsp),@ptr[7]
	 vmovdqu	@inp[3],0x70($offload)
	 vpxor		@inp[3],$zero,@out[7]

	xor	\$128,$offload
	dec	$num
	jnz	.Loop_dec8x

	mov	16(%rsp),%rax			# original %rsp
.cfi_def_cfa	%rax,8
	#mov	24(%rsp),$num
	#lea	`40*8`($inp),$inp
	#dec	$num
	#jnz	.Ldec8x_loop_grande

.Ldec8x_done:
	vzeroupper
___
$code.=<<___ if ($win64);
	movaps	-0xd8(%rax),%xmm6
	movaps	-0xc8(%rax),%xmm7
	movaps	-0xb8(%rax),%xmm8
	movaps	-0xa8(%rax),%xmm9
	movaps	-0x98(%rax),%xmm10
	movaps	-0x88(%rax),%xmm11
	movaps	-0x78(%rax),%xmm12
	movaps	-0x68(%rax),%xmm13
	movaps	-0x58(%rax),%xmm14
	movaps	-0x48(%rax),%xmm15
___
$code.=<<___;
	mov	-48(%rax),%r15
.cfi_restore	%r15
	mov	-40(%rax),%r14
.cfi_restore	%r14
	mov	-32(%rax),%r13
.cfi_restore	%r13
	mov	-24(%rax),%r12
.cfi_restore	%r12
	mov	-16(%rax),%rbp
.cfi_restore	%rbp
	mov	-8(%rax),%rbx
.cfi_restore	%rbx
	lea	(%rax),%rsp
.cfi_def_cfa_register	%rsp
.Ldec8x_epilogue:
	ret
.cfi_endproc
.size	aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx
___
						}}}

if ($win64) {
# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
$rec="%rcx";
$frame="%rdx";
$context="%r8";
$disp="%r9";

$code.=<<___;
.extern	__imp_RtlVirtualUnwind
.type	se_handler,\@abi-omnipotent
.align	16
se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# prologue label
	cmp	%r10,%rbx		# context->Rip<.Lprologue
	jb	.Lin_prologue

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
	jae	.Lin_prologue

	mov	16(%rax),%rax		# pull saved stack pointer

	mov	-8(%rax),%rbx
	mov	-16(%rax),%rbp
	mov	-24(%rax),%r12
	mov	-32(%rax),%r13
	mov	-40(%rax),%r14
	mov	-48(%rax),%r15
	mov	%rbx,144($context)	# restore context->Rbx
	mov	%rbp,160($context)	# restore context->Rbp
	mov	%r12,216($context)	# restore context->R12
	mov	%r13,224($context)	# restore context->R13
	mov	%r14,232($context)	# restore context->R14
	mov	%r15,240($context)	# restore context->R15

	lea	-56-10*16(%rax),%rsi
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$20,%ecx
	.long	0xa548f3fc		# cld; rep movsq

.Lin_prologue:
	mov	8(%rax),%rdi
	mov	16(%rax),%rsi
	mov	%rax,152($context)	# restore context->Rsp
	mov	%rsi,168($context)	# restore context->Rsi
	mov	%rdi,176($context)	# restore context->Rdi

	mov	40($disp),%rdi		# disp->ContextRecord
	mov	$context,%rsi		# context
	mov	\$154,%ecx		# sizeof(CONTEXT)
	.long	0xa548f3fc		# cld; rep movsq

	mov	$disp,%rsi
	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
	mov	0(%rsi),%r8		# arg3, disp->ControlPc
	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
	mov	40(%rsi),%r10		# disp->ContextRecord
	lea	56(%rsi),%r11		# &disp->HandlerData
	lea	24(%rsi),%r12		# &disp->EstablisherFrame
	mov	%r10,32(%rsp)		# arg5
	mov	%r11,40(%rsp)		# arg6
	mov	%r12,48(%rsp)		# arg7
	mov	%rcx,56(%rsp)		# arg8, (NULL)
	call	*__imp_RtlVirtualUnwind(%rip)

	mov	\$1,%eax		# ExceptionContinueSearch
	add	\$64,%rsp
	popfq
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx
	pop	%rdi
	pop	%rsi
	ret
.size	se_handler,.-se_handler

.section	.pdata
.align	4
	.rva	.LSEH_begin_aesni_multi_cbc_encrypt
	.rva	.LSEH_end_aesni_multi_cbc_encrypt
	.rva	.LSEH_info_aesni_multi_cbc_encrypt
	.rva	.LSEH_begin_aesni_multi_cbc_decrypt
	.rva	.LSEH_end_aesni_multi_cbc_decrypt
	.rva	.LSEH_info_aesni_multi_cbc_decrypt
___
$code.=<<___ if ($avx);
	.rva	.LSEH_begin_aesni_multi_cbc_encrypt_avx
	.rva	.LSEH_end_aesni_multi_cbc_encrypt_avx
	.rva	.LSEH_info_aesni_multi_cbc_encrypt_avx
	.rva	.LSEH_begin_aesni_multi_cbc_decrypt_avx
	.rva	.LSEH_end_aesni_multi_cbc_decrypt_avx
	.rva	.LSEH_info_aesni_multi_cbc_decrypt_avx
___
$code.=<<___;
.section	.xdata
.align	8
.LSEH_info_aesni_multi_cbc_encrypt:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lenc4x_body,.Lenc4x_epilogue		# HandlerData[]
.LSEH_info_aesni_multi_cbc_decrypt:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Ldec4x_body,.Ldec4x_epilogue		# HandlerData[]
___
$code.=<<___ if ($avx);
.LSEH_info_aesni_multi_cbc_encrypt_avx:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Lenc8x_body,.Lenc8x_epilogue		# HandlerData[]
.LSEH_info_aesni_multi_cbc_decrypt_avx:
	.byte	9,0,0,0
	.rva	se_handler
	.rva	.Ldec8x_body,.Ldec8x_epilogue		# HandlerData[]
___
}
####################################################################

sub rex {
  local *opcode=shift;
  my ($dst,$src)=@_;
  my $rex=0;

    $rex|=0x04			if($dst>=8);
    $rex|=0x01			if($src>=8);
    push @opcode,$rex|0x40	if($rex);
}

sub aesni {
  my $line=shift;
  my @opcode=(0x66);

    if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
	rex(\@opcode,$4,$3);
	push @opcode,0x0f,0x3a,0xdf;
	push @opcode,0xc0|($3&7)|(($4&7)<<3);	# ModR/M
	my $c=$2;
	push @opcode,$c=~/^0/?oct($c):$c;
	return ".byte\t".join(',',@opcode);
    }
    elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
	my %opcodelet = (
		"aesimc" => 0xdb,
		"aesenc" => 0xdc,	"aesenclast" => 0xdd,
		"aesdec" => 0xde,	"aesdeclast" => 0xdf
	);
	return undef if (!defined($opcodelet{$1}));
	rex(\@opcode,$3,$2);
	push @opcode,0x0f,0x38,$opcodelet{$1};
	push @opcode,0xc0|($2&7)|(($3&7)<<3);	# ModR/M
	return ".byte\t".join(',',@opcode);
    }
    elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) {
	my %opcodelet = (
		"aesenc" => 0xdc,	"aesenclast" => 0xdd,
		"aesdec" => 0xde,	"aesdeclast" => 0xdf
	);
	return undef if (!defined($opcodelet{$1}));
	my $off = $2;
	push @opcode,0x44 if ($3>=8);
	push @opcode,0x0f,0x38,$opcodelet{$1};
	push @opcode,0x44|(($3&7)<<3),0x24;	# ModR/M
	push @opcode,($off=~/^0/?oct($off):$off)&0xff;
	return ".byte\t".join(',',@opcode);
    }
    return $line;
}

$code =~ s/\`([^\`]*)\`/eval($1)/gem;
$code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;

print $code;
close STDOUT or die "error closing STDOUT: $!";