#! /usr/bin/env perl
# Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Poly1305 hash for C64x+.
#
# October 2015
#
# Performance is [incredible for a 32-bit processor] 1.82 cycles per
# processed byte. Comparison to compiler-generated code is problematic,
# because results were observed to vary from 2.1 to 7.6 cpb depending
# on compiler's ability to inline small functions. Compiler also
# disables interrupts for some reason, thus making interrupt response
# time dependent on input length. This module on the other hand is free
# from such limitation.

$output=pop;
open STDOUT,">$output";

($CTXA,$INPB,$LEN,$PADBIT)=("A4","B4","A6","B6");
($H0,$H1,$H2,$H3,$H4,$H4a)=("A8","B8","A10","B10","B2",$LEN);
($D0,$D1,$D2,$D3)=         ("A9","B9","A11","B11");
($R0,$R1,$R2,$R3,$S1,$S2,$S3,$S3b)=("A0","B0","A1","B1","A12","B12","A13","B13");
($THREE,$R0b,$S2a)=("B7","B5","A5");

$code.=<<___;
	.text

	.if	.ASSEMBLER_VERSION<7000000
	.asg	0,__TI_EABI__
	.endif
	.if	__TI_EABI__
	.asg	poly1305_init,_poly1305_init
	.asg	poly1305_blocks,_poly1305_blocks
	.asg	poly1305_emit,_poly1305_emit
	.endif

	.asg	B3,RA
	.asg	A15,FP
	.asg	B15,SP

	.if	.LITTLE_ENDIAN
	.asg	MV,SWAP2
	.asg	MV.L,SWAP4
	.endif

	.global	_poly1305_init
_poly1305_init:
	.asmfunc
	LDNDW	*${INPB}[0],B17:B16	; load key material
	LDNDW	*${INPB}[1],A17:A16

||	ZERO	B9:B8
||	MVK	-1,B0
	STDW	B9:B8,*${CTXA}[0]	; initialize h1:h0
||	SHRU	B0,4,B0			; 0x0fffffff
||	MVK	-4,B1
	STDW	B9:B8,*${CTXA}[1]	; initialize h3:h2
||	AND	B0,B1,B1		; 0x0ffffffc
	STW	B8,*${CTXA}[4]		; initialize h4

	.if	.BIG_ENDIAN
	SWAP2	B16,B17
||	SWAP2	B17,B16
	SWAP2	A16,A17
||	SWAP2	A17,A16
	SWAP4	B16,B16
||	SWAP4	A16,A16
	SWAP4	B17,B17
||	SWAP4	A17,A17
	.endif

	AND	B16,B0,B20		; r0 = key[0] & 0x0fffffff
||	AND	B17,B1,B22		; r1 = key[1] & 0x0ffffffc
||	EXTU	B17,4,6,B16		; r1>>2
	AND	A16,B1,B21		; r2 = key[2] & 0x0ffffffc
||	AND	A17,B1,A23		; r3 = key[3] & 0x0ffffffc
||	BNOP	RA
	SHRU	B21,2,B18
||	ADD	B22,B16,B16		; s1 = r1 + r1>>2

	STDW	B21:B20,*${CTXA}[3]	; save r2:r0
||	ADD	B21,B18,B18		; s2 = r2 + r2>>2
||	SHRU	A23,2,B17
||	MV	A23,B23
	STDW	B23:B22,*${CTXA}[4]	; save r3:r1
||	ADD	B23,B17,B19		; s3 = r3 + r3>>2
||	ADD	B23,B17,B17		; s3 = r3 + r3>>2
	STDW	B17:B16,*${CTXA}[5]	; save s3:s1
	STDW	B19:B18,*${CTXA}[6]	; save s3:s2
||	ZERO	A4			; return 0
	.endasmfunc

	.global	_poly1305_blocks
	.align	32
_poly1305_blocks:
	.asmfunc	stack_usage(40)
	SHRU	$LEN,4,A2		; A2 is loop counter, number of blocks
  [!A2]	BNOP	RA			; no data
|| [A2]	STW	FP,*SP--(40)		; save frame pointer and alloca(40)
|| [A2]	MV	SP,FP
   [A2]	STDW	B13:B12,*SP[4]		; ABI says so
|| [A2]	MV	$CTXA,$S3b		; borrow $S3b
   [A2]	STDW	B11:B10,*SP[3]
|| [A2]	STDW	A13:A12,*FP[-3]
   [A2]	STDW	A11:A10,*FP[-4]

|| [A2]	LDDW	*${S3b}[0],B25:B24	; load h1:h0
   [A2]	LDNW	*${INPB}++[4],$D0	; load inp[0]
   [A2]	LDNW	*${INPB}[-3],$D1	; load inp[1]

	LDDW	*${CTXA}[1],B29:B28	; load h3:h2, B28 is h2
	LDNW	*${INPB}[-2],$D2	; load inp[2]
	LDNW	*${INPB}[-1],$D3	; load inp[3]

	LDDW	*${CTXA}[3],$R2:$R0	; load r2:r0
||	LDDW	*${S3b}[4],$R3:$R1	; load r3:r1
||	SWAP2	$D0,$D0

	LDDW	*${CTXA}[5],$S3:$S1	; load s3:s1
||	LDDW	*${S3b}[6],$S3b:$S2	; load s3:s2
||	SWAP4	$D0,$D0
||	SWAP2	$D1,$D1

	ADDU	$D0,B24,$D0:$H0		; h0+=inp[0]
||	ADD	$D0,B24,B27		; B-copy of h0+inp[0]
||	SWAP4	$D1,$D1
	ADDU	$D1,B25,$D1:$H1		; h1+=inp[1]
||	MVK	3,$THREE
||	SWAP2	$D2,$D2
	LDW	*${CTXA}[4],$H4		; load h4
||	SWAP4	$D2,$D2
||	MV	B29,B30			; B30 is h3
	MV	$R0,$R0b

loop?:
	MPY32U	$H0,$R0,A17:A16
||	MPY32U	B27,$R1,B17:B16		; MPY32U	$H0,$R1,B17:B16
||	ADDU	$D0,$D1:$H1,B25:B24	; ADDU		$D0,$D1:$H1,$D1:$H1
||	ADDU	$D2,B28,$D2:$H2		; h2+=inp[2]
||	SWAP2	$D3,$D3
	MPY32U	$H0,$R2,A19:A18
||	MPY32U	B27,$R3,B19:B18		; MPY32U	$H0,$R3,B19:B18
||	ADD	$D0,$H1,A24		; A-copy of B24
||	SWAP4	$D3,$D3
|| [A2]	SUB	A2,1,A2			; decrement loop counter

	MPY32U	A24,$S3,A21:A20		; MPY32U	$H1,$S3,A21:A20
||	MPY32U	B24,$R0b,B21:B20	; MPY32U	$H1,$R0,B21:B20
||	ADDU	B25,$D2:$H2,$D2:$H2	; ADDU		$D1,$D2:$H2,$D2:$H2
||	ADDU	$D3,B30,$D3:$H3		; h3+=inp[3]
||	ADD	B25,$H2,B25		; B-copy of $H2
	MPY32U	A24,$R1,A23:A22		; MPY32U	$H1,$R1,A23:A22
||	MPY32U	B24,$R2,B23:B22		; MPY32U	$H1,$R2,B23:B22

	MPY32U	$H2,$S2,A25:A24
||	MPY32U	B25,$S3b,B25:B24	; MPY32U	$H2,$S3,B25:B24
||	ADDU	$D2,$D3:$H3,$D3:$H3
||	ADD	$PADBIT,$H4,$H4		; h4+=padbit
	MPY32U	$H2,$R0,A27:A26
||	MPY32U	$H2,$R1,B27:B26
||	ADD	$D3,$H4,$H4
||	MV	$S2,$S2a

	MPY32U	$H3,$S1,A29:A28
||	MPY32U	$H3,$S2,B29:B28
||	ADD	A21,A17,A21		; start accumulating "d3:d0"
||	ADD	B21,B17,B21
||	ADDU	A20,A16,A17:A16
||	ADDU	B20,B16,B17:B16
|| [A2]	LDNW	*${INPB}++[4],$D0	; load inp[0]
	MPY32U	$H3,$S3,A31:A30
||	MPY32U	$H3,$R0b,B31:B30
||	ADD	A23,A19,A23
||	ADD	B23,B19,B23
||	ADDU	A22,A18,A19:A18
||	ADDU	B22,B18,B19:B18
|| [A2]	LDNW	*${INPB}[-3],$D1	; load inp[1]

	MPY32	$H4,$S1,B20
||	MPY32	$H4,$S2a,A20
||	ADD	A25,A21,A21
||	ADD	B25,B21,B21
||	ADDU	A24,A17:A16,A17:A16
||	ADDU	B24,B17:B16,B17:B16
|| [A2]	LDNW	*${INPB}[-2],$D2	; load inp[2]
	MPY32	$H4,$S3b,B22
||	ADD	A27,A23,A23
||	ADD	B27,B23,B23
||	ADDU	A26,A19:A18,A19:A18
||	ADDU	B26,B19:B18,B19:B18
|| [A2]	LDNW	*${INPB}[-1],$D3	; load inp[3]

	MPY32	$H4,$R0b,$H4
||	ADD	A29,A21,A21		; final hi("d0")
||	ADD	B29,B21,B21		; final hi("d1")
||	ADDU	A28,A17:A16,A17:A16	; final lo("d0")
||	ADDU	B28,B17:B16,B17:B16
	ADD	A31,A23,A23		; final hi("d2")
||	ADD	B31,B23,B23		; final hi("d3")
||	ADDU	A30,A19:A18,A19:A18
||	ADDU	B30,B19:B18,B19:B18
	ADDU	B20,B17:B16,B17:B16	; final lo("d1")
||	ADDU	A20,A19:A18,A19:A18	; final lo("d2")
	ADDU	B22,B19:B18,B19:B18	; final lo("d3")

||	ADD	A17,A21,A21		; "flatten" "d3:d0"
	MV	A19,B29			; move to avoid cross-path stalls
	ADDU	A21,B17:B16,B27:B26	; B26 is h1
	ADD	B21,B27,B27
||	DMV	B29,A18,B29:B28		; move to avoid cross-path stalls
	ADDU	B27,B29:B28,B29:B28	; B28 is h2
|| [A2]	SWAP2	$D0,$D0
	ADD	A23,B29,B29
|| [A2]	SWAP4	$D0,$D0
	ADDU	B29,B19:B18,B31:B30	; B30 is h3
	ADD	B23,B31,B31
||	MV	A16,B24			; B24 is h0
|| [A2]	SWAP2	$D1,$D1
	ADD	B31,$H4,$H4
|| [A2]	SWAP4	$D1,$D1

	SHRU	$H4,2,B16		; last reduction step
||	AND	$H4,$THREE,$H4
	ADDAW	B16,B16,B16		; 5*(h4>>2)
|| [A2]	BNOP	loop?

	ADDU	B24,B16,B25:B24		; B24 is h0
|| [A2]	SWAP2	$D2,$D2
	ADDU	B26,B25,B27:B26		; B26 is h1
|| [A2]	SWAP4	$D2,$D2
	ADDU	B28,B27,B29:B28		; B28 is h2
|| [A2]	ADDU	$D0,B24,$D0:$H0		; h0+=inp[0]
|| [A2]	ADD	$D0,B24,B27		; B-copy of h0+inp[0]
	ADDU	B30,B29,B31:B30		; B30 is h3
	ADD	B31,$H4,$H4
|| [A2]	ADDU	$D1,B26,$D1:$H1		; h1+=inp[1]
;;===== branch to loop? is taken here

	LDDW	*FP[-4],A11:A10		; ABI says so
	LDDW	*FP[-3],A13:A12
||	LDDW	*SP[3],B11:B10
	LDDW	*SP[4],B13:B12
||	MV	B26,B25
||	BNOP	RA
	LDW	*++SP(40),FP		; restore frame pointer
||	MV	B30,B29
	STDW	B25:B24,*${CTXA}[0]	; save h1:h0
	STDW	B29:B28,*${CTXA}[1]	; save h3:h2
	STW	$H4,*${CTXA}[4]		; save h4
	NOP	1
	.endasmfunc
___
{
my ($MAC,$NONCEA,$NONCEB)=($INPB,$LEN,$PADBIT);

$code.=<<___;
	.global	_poly1305_emit
	.align	32
_poly1305_emit:
	.asmfunc
	LDDW	*${CTXA}[0],A17:A16	; load h1:h0
	LDDW	*${CTXA}[1],A19:A18	; load h3:h2
	LDW	*${CTXA}[4],A20		; load h4
	MV	$NONCEA,$NONCEB

	MVK	5,A22			; compare to modulus
	ADDU	A16,A22,A23:A22
||	LDW	*${NONCEA}[0],A8
||	LDW	*${NONCEB}[1],B8
	ADDU	A17,A23,A25:A24
||	LDW	*${NONCEA}[2],A9
||	LDW	*${NONCEB}[3],B9
	ADDU	A19,A25,A27:A26
	ADDU	A19,A27,A29:A28
	ADD	A20,A29,A29

	SHRU	A29,2,A2		; check for overflow in 130-th bit

   [A2]	MV	A22,A16			; select
|| [A2]	MV	A24,A17
   [A2]	MV	A26,A18
|| [A2]	MV	A28,A19

||	ADDU	A8,A16,A23:A22		; accumulate nonce
	ADDU	B8,A17,A25:A24
||	SWAP2	A22,A22
	ADDU	A23,A25:A24,A25:A24
	ADDU	A9,A18,A27:A26
||	SWAP2	A24,A24
	ADDU	A25,A27:A26,A27:A26
||	ADD	B9,A19,A28
	ADD	A27,A28,A28
||	SWAP2	A26,A26

	.if	.BIG_ENDIAN
	SWAP2	A28,A28
||	SWAP4	A22,A22
||	SWAP4	A24,B24
	SWAP4	A26,A26
	SWAP4	A28,A28
||	MV	B24,A24
	.endif

	BNOP	RA,1
	STNW	A22,*${MAC}[0]		; write the result
	STNW	A24,*${MAC}[1]
	STNW	A26,*${MAC}[2]
	STNW	A28,*${MAC}[3]
	.endasmfunc
___
}
$code.=<<___;
	.sect	.const
	.cstring "Poly1305 for C64x+, CRYPTOGAMS by <appro\@openssl.org>"
	.align	4
___

print $code;


#! /usr/bin/env perl
# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# Poly1305 hash for MIPS64.
#
# May 2016
#
# Numbers are cycles per processed byte with poly1305_blocks alone.
#
#		IALU/gcc
# R1x000	5.64/+120%	(big-endian)
# Octeon II	3.80/+280%	(little-endian)

######################################################################
# There is a number of MIPS ABI in use, O32 and N32/64 are most
# widely used. Then there is a new contender: NUBI. It appears that if
# one picks the latter, it's possible to arrange code in ABI neutral
# manner. Therefore let's stick to NUBI register layout:
#
($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25));
($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23));
($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31));
#
# The return value is placed in $a0. Following coding rules facilitate
# interoperability:
#
# - never ever touch $tp, "thread pointer", former $gp [o32 can be
#   excluded from the rule, because it's specified volatile];
# - copy return value to $t0, former $v0 [or to $a0 if you're adapting
#   old code];
# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary;
#
# For reference here is register layout for N32/64 MIPS ABIs:
#
# ($zero,$at,$v0,$v1)=map("\$$_",(0..3));
# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25));
# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23));
# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31));
#
# <appro@openssl.org>
#
######################################################################

$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64

die "MIPS64 only" unless ($flavour =~ /64|n32/i);

$v0 = ($flavour =~ /nubi/i) ? $a0 : $t0;
$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0x0003f000" : "0x00030000";

($ctx,$inp,$len,$padbit) = ($a0,$a1,$a2,$a3);
($in0,$in1,$tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = ($a4,$a5,$a6,$a7,$at,$t0,$t1);

$code.=<<___;
#include "mips_arch.h"

#ifdef MIPSEB
# define MSB 0
# define LSB 7
#else
# define MSB 7
# define LSB 0
#endif

.text
.set	noat
.set	noreorder

.align	5
.globl	poly1305_init
.ent	poly1305_init
poly1305_init:
	.frame	$sp,0,$ra
	.set	reorder

	sd	$zero,0($ctx)
	sd	$zero,8($ctx)
	sd	$zero,16($ctx)

	beqz	$inp,.Lno_key

#if defined(_MIPS_ARCH_MIPS64R6)
	ld	$in0,0($inp)
	ld	$in1,8($inp)
#else
	ldl	$in0,0+MSB($inp)
	ldl	$in1,8+MSB($inp)
	ldr	$in0,0+LSB($inp)
	ldr	$in1,8+LSB($inp)
#endif
#ifdef	MIPSEB
# if defined(_MIPS_ARCH_MIPS64R2)
	dsbh	$in0,$in0		# byte swap
	 dsbh	$in1,$in1
	dshd	$in0,$in0
	 dshd	$in1,$in1
# else
	ori	$tmp0,$zero,0xFF
	dsll	$tmp2,$tmp0,32
	or	$tmp0,$tmp2		# 0x000000FF000000FF

	and	$tmp1,$in0,$tmp0	# byte swap
	 and	$tmp3,$in1,$tmp0
	dsrl	$tmp2,$in0,24
	 dsrl	$tmp4,$in1,24
	dsll	$tmp1,24
	 dsll	$tmp3,24
	and	$tmp2,$tmp0
	 and	$tmp4,$tmp0
	dsll	$tmp0,8			# 0x0000FF000000FF00
	or	$tmp1,$tmp2
	 or	$tmp3,$tmp4
	and	$tmp2,$in0,$tmp0
	 and	$tmp4,$in1,$tmp0
	dsrl	$in0,8
	 dsrl	$in1,8
	dsll	$tmp2,8
	 dsll	$tmp4,8
	and	$in0,$tmp0
	 and	$in1,$tmp0
	or	$tmp1,$tmp2
	 or	$tmp3,$tmp4
	or	$in0,$tmp1
	 or	$in1,$tmp3
	dsrl	$tmp1,$in0,32
	 dsrl	$tmp3,$in1,32
	dsll	$in0,32
	 dsll	$in1,32
	or	$in0,$tmp1
	 or	$in1,$tmp3
# endif
#endif
	li	$tmp0,1
	dsll	$tmp0,32
	daddiu	$tmp0,-63
	dsll	$tmp0,28
	daddiu	$tmp0,-1		# 0ffffffc0fffffff

	and	$in0,$tmp0
	daddiu	$tmp0,-3		# 0ffffffc0ffffffc
	and	$in1,$tmp0

	sd	$in0,24($ctx)
	dsrl	$tmp0,$in1,2
	sd	$in1,32($ctx)
	daddu	$tmp0,$in1		# s1 = r1 + (r1 >> 2)
	sd	$tmp0,40($ctx)

.Lno_key:
	li	$v0,0			# return 0
	jr	$ra
.end	poly1305_init
___
{
my ($h0,$h1,$h2,$r0,$r1,$s1,$d0,$d1,$d2) =
   ($s0,$s1,$s2,$s3,$s4,$s5,$in0,$in1,$t2);

$code.=<<___;
.align	5
.globl	poly1305_blocks
.ent	poly1305_blocks
poly1305_blocks:
	.set	noreorder
	dsrl	$len,4			# number of complete blocks
	bnez	$len,poly1305_blocks_internal
	nop
	jr	$ra
	nop
.end	poly1305_blocks

.align	5
.ent	poly1305_blocks_internal
poly1305_blocks_internal:
	.frame	$sp,6*8,$ra
	.mask	$SAVED_REGS_MASK,-8
	.set	noreorder
	dsubu	$sp,6*8
	sd	$s5,40($sp)
	sd	$s4,32($sp)
___
$code.=<<___ if ($flavour =~ /nubi/i);	# optimize non-nubi prologue
	sd	$s3,24($sp)
	sd	$s2,16($sp)
	sd	$s1,8($sp)
	sd	$s0,0($sp)
___
$code.=<<___;
	.set	reorder

	ld	$h0,0($ctx)		# load hash value
	ld	$h1,8($ctx)
	ld	$h2,16($ctx)

	ld	$r0,24($ctx)		# load key
	ld	$r1,32($ctx)
	ld	$s1,40($ctx)

.Loop:
#if defined(_MIPS_ARCH_MIPS64R6)
	ld	$in0,0($inp)		# load input
	ld	$in1,8($inp)
#else
	ldl	$in0,0+MSB($inp)	# load input
	ldl	$in1,8+MSB($inp)
	ldr	$in0,0+LSB($inp)
	ldr	$in1,8+LSB($inp)
#endif
	daddiu	$len,-1
	daddiu	$inp,16
#ifdef	MIPSEB
# if defined(_MIPS_ARCH_MIPS64R2)
	dsbh	$in0,$in0		# byte swap
	 dsbh	$in1,$in1
	dshd	$in0,$in0
	 dshd	$in1,$in1
# else
	ori	$tmp0,$zero,0xFF
	dsll	$tmp2,$tmp0,32
	or	$tmp0,$tmp2		# 0x000000FF000000FF

	and	$tmp1,$in0,$tmp0	# byte swap
	 and	$tmp3,$in1,$tmp0
	dsrl	$tmp2,$in0,24
	 dsrl	$tmp4,$in1,24
	dsll	$tmp1,24
	 dsll	$tmp3,24
	and	$tmp2,$tmp0
	 and	$tmp4,$tmp0
	dsll	$tmp0,8			# 0x0000FF000000FF00
	or	$tmp1,$tmp2
	 or	$tmp3,$tmp4
	and	$tmp2,$in0,$tmp0
	 and	$tmp4,$in1,$tmp0
	dsrl	$in0,8
	 dsrl	$in1,8
	dsll	$tmp2,8
	 dsll	$tmp4,8
	and	$in0,$tmp0
	 and	$in1,$tmp0
	or	$tmp1,$tmp2
	 or	$tmp3,$tmp4
	or	$in0,$tmp1
	 or	$in1,$tmp3
	dsrl	$tmp1,$in0,32
	 dsrl	$tmp3,$in1,32
	dsll	$in0,32
	 dsll	$in1,32
	or	$in0,$tmp1
	 or	$in1,$tmp3
# endif
#endif
	daddu	$h0,$in0		# accumulate input
	daddu	$h1,$in1
	sltu	$tmp0,$h0,$in0
	sltu	$tmp1,$h1,$in1
	daddu	$h1,$tmp0

	dmultu	($r0,$h0)		# h0*r0
	 daddu	$h2,$padbit
	 sltu	$tmp0,$h1,$tmp0
	mflo	($d0,$r0,$h0)
	mfhi	($d1,$r0,$h0)

	dmultu	($s1,$h1)		# h1*5*r1
	 daddu	$tmp0,$tmp1
	 daddu	$h2,$tmp0
	mflo	($tmp0,$s1,$h1)
	mfhi	($tmp1,$s1,$h1)

	dmultu	($r1,$h0)		# h0*r1
	 daddu	$d0,$tmp0
	 daddu	$d1,$tmp1
	mflo	($tmp2,$r1,$h0)
	mfhi	($d2,$r1,$h0)
	 sltu	$tmp0,$d0,$tmp0
	 daddu	$d1,$tmp0

	dmultu	($r0,$h1)		# h1*r0
	 daddu	$d1,$tmp2
	 sltu	$tmp2,$d1,$tmp2
	mflo	($tmp0,$r0,$h1)
	mfhi	($tmp1,$r0,$h1)
	 daddu	$d2,$tmp2

	dmultu	($s1,$h2)		# h2*5*r1
	 daddu	$d1,$tmp0
	 daddu	$d2,$tmp1
	mflo	($tmp2,$s1,$h2)

	dmultu	($r0,$h2)		# h2*r0
	 sltu	$tmp0,$d1,$tmp0
	 daddu	$d2,$tmp0
	mflo	($tmp3,$r0,$h2)

	daddu	$d1,$tmp2
	daddu	$d2,$tmp3
	sltu	$tmp2,$d1,$tmp2
	daddu	$d2,$tmp2

	li	$tmp0,-4		# final reduction
	and	$tmp0,$d2
	dsrl	$tmp1,$d2,2
	andi	$h2,$d2,3
	daddu	$tmp0,$tmp1
	daddu	$h0,$d0,$tmp0
	sltu	$tmp0,$h0,$tmp0
	daddu	$h1,$d1,$tmp0
	sltu	$tmp0,$h1,$tmp0
	daddu	$h2,$h2,$tmp0

	bnez	$len,.Loop

	sd	$h0,0($ctx)		# store hash value
	sd	$h1,8($ctx)
	sd	$h2,16($ctx)

	.set	noreorder
	ld	$s5,40($sp)		# epilogue
	ld	$s4,32($sp)
___
$code.=<<___ if ($flavour =~ /nubi/i);	# optimize non-nubi epilogue
	ld	$s3,24($sp)
	ld	$s2,16($sp)
	ld	$s1,8($sp)
	ld	$s0,0($sp)
___
$code.=<<___;
	jr	$ra
	daddu	$sp,6*8
.end	poly1305_blocks_internal
___
}
{
my ($ctx,$mac,$nonce) = ($a0,$a1,$a2);

$code.=<<___;
.align	5
.globl	poly1305_emit
.ent	poly1305_emit
poly1305_emit:
	.frame	$sp,0,$ra
	.set	reorder

	ld	$tmp0,0($ctx)
	ld	$tmp1,8($ctx)
	ld	$tmp2,16($ctx)

	daddiu	$in0,$tmp0,5		# compare to modulus
	sltiu	$tmp3,$in0,5
	daddu	$in1,$tmp1,$tmp3
	sltu	$tmp3,$in1,$tmp3
	daddu	$tmp2,$tmp2,$tmp3

	dsrl	$tmp2,2			# see if it carried/borrowed
	dsubu	$tmp2,$zero,$tmp2
	nor	$tmp3,$zero,$tmp2

	and	$in0,$tmp2
	and	$tmp0,$tmp3
	and	$in1,$tmp2
	and	$tmp1,$tmp3
	or	$in0,$tmp0
	or	$in1,$tmp1

	lwu	$tmp0,0($nonce)		# load nonce
	lwu	$tmp1,4($nonce)
	lwu	$tmp2,8($nonce)
	lwu	$tmp3,12($nonce)
	dsll	$tmp1,32
	dsll	$tmp3,32
	or	$tmp0,$tmp1
	or	$tmp2,$tmp3

	daddu	$in0,$tmp0		# accumulate nonce
	daddu	$in1,$tmp2
	sltu	$tmp0,$in0,$tmp0
	daddu	$in1,$tmp0

	dsrl	$tmp0,$in0,8		# write mac value
	dsrl	$tmp1,$in0,16
	dsrl	$tmp2,$in0,24
	sb	$in0,0($mac)
	dsrl	$tmp3,$in0,32
	sb	$tmp0,1($mac)
	dsrl	$tmp0,$in0,40
	sb	$tmp1,2($mac)
	dsrl	$tmp1,$in0,48
	sb	$tmp2,3($mac)
	dsrl	$tmp2,$in0,56
	sb	$tmp3,4($mac)
	dsrl	$tmp3,$in1,8
	sb	$tmp0,5($mac)
	dsrl	$tmp0,$in1,16
	sb	$tmp1,6($mac)
	dsrl	$tmp1,$in1,24
	sb	$tmp2,7($mac)

	sb	$in1,8($mac)
	dsrl	$tmp2,$in1,32
	sb	$tmp3,9($mac)
	dsrl	$tmp3,$in1,40
	sb	$tmp0,10($mac)
	dsrl	$tmp0,$in1,48
	sb	$tmp1,11($mac)
	dsrl	$tmp1,$in1,56
	sb	$tmp2,12($mac)
	sb	$tmp3,13($mac)
	sb	$tmp0,14($mac)
	sb	$tmp1,15($mac)

	jr	$ra
.end	poly1305_emit
.rdata
.asciiz	"Poly1305 for MIPS64, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
___
}

$output=pop and open STDOUT,">$output";
print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements Poly1305 hash for PowerPC.
#
# June 2015
#
# Numbers are cycles per processed byte with poly1305_blocks alone,
# and improvement coefficients relative to gcc-generated code.
#
#			-m32		-m64
#
# Freescale e300	14.8/+80%	-
# PPC74x0		7.60/+60%	-
# PPC970		7.00/+114%	3.51/+205%
# POWER7		3.75/+260%	1.93/+100%
# POWER8		-		2.03/+200%
# POWER9		-		2.00/+150%
#
# Do we need floating-point implementation for PPC? Results presented
# in poly1305_ieee754.c are tricky to compare to, because they are for
# compiler-generated code. On the other hand it's known that floating-
# point performance can be dominated by FPU latency, which means that
# there is limit even for ideally optimized (and even vectorized) code.
# And this limit is estimated to be higher than above -m64 results. Or
# in other words floating-point implementation can be meaningful to
# consider only in 32-bit application context. We probably have to
# recognize that 32-bit builds are getting less popular on high-end
# systems and therefore tend to target embedded ones, which might not
# even have FPU...
#
# On side note, Power ISA 2.07 enables vector base 2^26 implementation,
# and POWER8 might have capacity to break 1.0 cycle per byte barrier...

$flavour = shift;

if ($flavour =~ /64/) {
	$SIZE_T	=8;
	$LRSAVE	=2*$SIZE_T;
	$UCMP	="cmpld";
	$STU	="stdu";
	$POP	="ld";
	$PUSH	="std";
} elsif ($flavour =~ /32/) {
	$SIZE_T	=4;
	$LRSAVE	=$SIZE_T;
	$UCMP	="cmplw";
	$STU	="stwu";
	$POP	="lwz";
	$PUSH	="stw";
} else { die "nonsense $flavour"; }

# Define endianness based on flavour
# i.e.: linux64le
$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";

open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";

$FRAME=24*$SIZE_T;

$sp="r1";
my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
my ($mac,$nonce)=($inp,$len);
my $mask = "r0";

$code=<<___;
.machine	"any"
.text
___
							if ($flavour =~ /64/) {
###############################################################################
# base 2^64 implementation

my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(7..12,27..31));

$code.=<<___;
.globl	.poly1305_init_int
.align	4
.poly1305_init_int:
	xor	r0,r0,r0
	std	r0,0($ctx)		# zero hash value
	std	r0,8($ctx)
	std	r0,16($ctx)

	$UCMP	$inp,r0
	beq-	Lno_key
___
$code.=<<___	if ($LITTLE_ENDIAN);
	ld	$d0,0($inp)		# load key material
	ld	$d1,8($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$h0,4
	lwbrx	$d0,0,$inp		# load key material
	li	$d1,8
	lwbrx	$h0,$h0,$inp
	li	$h1,12
	lwbrx	$d1,$d1,$inp
	lwbrx	$h1,$h1,$inp
	insrdi	$d0,$h0,32,0
	insrdi	$d1,$h1,32,0
___
$code.=<<___;
	lis	$h1,0xfff		# 0x0fff0000
	ori	$h1,$h1,0xfffc		# 0x0ffffffc
	insrdi	$h1,$h1,32,0		# 0x0ffffffc0ffffffc
	ori	$h0,$h1,3		# 0x0ffffffc0fffffff

	and	$d0,$d0,$h0
	and	$d1,$d1,$h1

	std	$d0,32($ctx)		# store key
	std	$d1,40($ctx)

Lno_key:
	xor	r3,r3,r3
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,2,0
.size	.poly1305_init_int,.-.poly1305_init_int

.globl	.poly1305_blocks
.align	4
.poly1305_blocks:
	srdi.	$len,$len,4
	beq-	Labort

	$STU	$sp,-$FRAME($sp)
	mflr	r0
	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$FRAME+$LRSAVE`($sp)

	ld	$r0,32($ctx)		# load key
	ld	$r1,40($ctx)

	ld	$h0,0($ctx)		# load hash value
	ld	$h1,8($ctx)
	ld	$h2,16($ctx)

	srdi	$s1,$r1,2
	mtctr	$len
	add	$s1,$s1,$r1		# s1 = r1 + r1>>2
	li	$mask,3
	b	Loop

.align	4
Loop:
___
$code.=<<___	if ($LITTLE_ENDIAN);
	ld	$t0,0($inp)		# load input
	ld	$t1,8($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$d0,4
	lwbrx	$t0,0,$inp		# load input
	li	$t1,8
	lwbrx	$d0,$d0,$inp
	li	$d1,12
	lwbrx	$t1,$t1,$inp
	lwbrx	$d1,$d1,$inp
	insrdi	$t0,$d0,32,0
	insrdi	$t1,$d1,32,0
___
$code.=<<___;
	addi	$inp,$inp,16

	addc	$h0,$h0,$t0		# accumulate input
	adde	$h1,$h1,$t1

	mulld	$d0,$h0,$r0		# h0*r0
	mulhdu	$d1,$h0,$r0
	adde	$h2,$h2,$padbit

	mulld	$t0,$h1,$s1		# h1*5*r1
	mulhdu	$t1,$h1,$s1
	addc	$d0,$d0,$t0
	adde	$d1,$d1,$t1

	mulld	$t0,$h0,$r1		# h0*r1
	mulhdu	$d2,$h0,$r1
	addc	$d1,$d1,$t0
	addze	$d2,$d2

	mulld	$t0,$h1,$r0		# h1*r0
	mulhdu	$t1,$h1,$r0
	addc	$d1,$d1,$t0
	adde	$d2,$d2,$t1

	mulld	$t0,$h2,$s1		# h2*5*r1
	mulld	$t1,$h2,$r0		# h2*r0
	addc	$d1,$d1,$t0
	adde	$d2,$d2,$t1

	andc	$t0,$d2,$mask		# final reduction step
	and	$h2,$d2,$mask
	srdi	$t1,$t0,2
	add	$t0,$t0,$t1
	addc	$h0,$d0,$t0
	addze	$h1,$d1
	addze	$h2,$h2

	bdnz	Loop

	std	$h0,0($ctx)		# store hash value
	std	$h1,8($ctx)
	std	$h2,16($ctx)

	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
	addi	$sp,$sp,$FRAME
Labort:
	blr
	.long	0
	.byte	0,12,4,1,0x80,5,4,0
.size	.poly1305_blocks,.-.poly1305_blocks

.globl	.poly1305_emit
.align	4
.poly1305_emit:
	ld	$h0,0($ctx)		# load hash
	ld	$h1,8($ctx)
	ld	$h2,16($ctx)
	ld	$padbit,0($nonce)	# load nonce
	ld	$nonce,8($nonce)

	addic	$d0,$h0,5		# compare to modulus
	addze	$d1,$h1
	addze	$d2,$h2

	srdi	$mask,$d2,2		# did it carry/borrow?
	neg	$mask,$mask

	andc	$h0,$h0,$mask
	and	$d0,$d0,$mask
	andc	$h1,$h1,$mask
	and	$d1,$d1,$mask
	or	$h0,$h0,$d0
	or	$h1,$h1,$d1
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	rotldi	$padbit,$padbit,32	# flip nonce words
	rotldi	$nonce,$nonce,32
___
$code.=<<___;
	addc	$h0,$h0,$padbit		# accumulate nonce
	adde	$h1,$h1,$nonce
___
$code.=<<___	if ($LITTLE_ENDIAN);
	std	$h0,0($mac)		# write result
	std	$h1,8($mac)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	extrdi	r0,$h0,32,0
	li	$d0,4
	stwbrx	$h0,0,$mac		# write result
	extrdi	$h0,$h1,32,0
	li	$d1,8
	stwbrx	r0,$d0,$mac
	li	$d2,12
	stwbrx	$h1,$d1,$mac
	stwbrx	$h0,$d2,$mac
___
$code.=<<___;
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,3,0
.size	.poly1305_emit,.-.poly1305_emit
___
							} else {
###############################################################################
# base 2^32 implementation

my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $s1,$s2,$s3,
    $t0,$t1,$t2,$t3, $D0,$D1,$D2,$D3, $d0,$d1,$d2,$d3
   ) = map("r$_",(7..12,14..31));

$code.=<<___;
.globl	.poly1305_init_int
.align	4
.poly1305_init_int:
	xor	r0,r0,r0
	stw	r0,0($ctx)		# zero hash value
	stw	r0,4($ctx)
	stw	r0,8($ctx)
	stw	r0,12($ctx)
	stw	r0,16($ctx)

	$UCMP	$inp,r0
	beq-	Lno_key
___
$code.=<<___	if ($LITTLE_ENDIAN);
	lw	$h0,0($inp)		# load key material
	lw	$h1,4($inp)
	lw	$h2,8($inp)
	lw	$h3,12($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$h1,4
	lwbrx	$h0,0,$inp		# load key material
	li	$h2,8
	lwbrx	$h1,$h1,$inp
	li	$h3,12
	lwbrx	$h2,$h2,$inp
	lwbrx	$h3,$h3,$inp
___
$code.=<<___;
	lis	$mask,0xf000		# 0xf0000000
	li	$r0,-4
	andc	$r0,$r0,$mask		# 0x0ffffffc

	andc	$h0,$h0,$mask
	and	$h1,$h1,$r0
	and	$h2,$h2,$r0
	and	$h3,$h3,$r0

	stw	$h0,32($ctx)		# store key
	stw	$h1,36($ctx)
	stw	$h2,40($ctx)
	stw	$h3,44($ctx)

Lno_key:
	xor	r3,r3,r3
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,2,0
.size	.poly1305_init_int,.-.poly1305_init_int

.globl	.poly1305_blocks
.align	4
.poly1305_blocks:
	srwi.	$len,$len,4
	beq-	Labort

	$STU	$sp,-$FRAME($sp)
	mflr	r0
	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$FRAME+$LRSAVE`($sp)

	lwz	$r0,32($ctx)		# load key
	lwz	$r1,36($ctx)
	lwz	$r2,40($ctx)
	lwz	$r3,44($ctx)

	lwz	$h0,0($ctx)		# load hash value
	lwz	$h1,4($ctx)
	lwz	$h2,8($ctx)
	lwz	$h3,12($ctx)
	lwz	$h4,16($ctx)

	srwi	$s1,$r1,2
	srwi	$s2,$r2,2
	srwi	$s3,$r3,2
	add	$s1,$s1,$r1		# si = ri + ri>>2
	add	$s2,$s2,$r2
	add	$s3,$s3,$r3
	mtctr	$len
	li	$mask,3
	b	Loop

.align	4
Loop:
___
$code.=<<___	if ($LITTLE_ENDIAN);
	lwz	$d0,0($inp)		# load input
	lwz	$d1,4($inp)
	lwz	$d2,8($inp)
	lwz	$d3,12($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$d1,4
	lwbrx	$d0,0,$inp		# load input
	li	$d2,8
	lwbrx	$d1,$d1,$inp
	li	$d3,12
	lwbrx	$d2,$d2,$inp
	lwbrx	$d3,$d3,$inp
___
$code.=<<___;
	addi	$inp,$inp,16

	addc	$h0,$h0,$d0		# accumulate input
	adde	$h1,$h1,$d1
	adde	$h2,$h2,$d2

	mullw	$d0,$h0,$r0		# h0*r0
	mulhwu	$D0,$h0,$r0

	mullw	$d1,$h0,$r1		# h0*r1
	mulhwu	$D1,$h0,$r1

	mullw	$d2,$h0,$r2		# h0*r2
	mulhwu	$D2,$h0,$r2

	 adde	$h3,$h3,$d3
	 adde	$h4,$h4,$padbit

	mullw	$d3,$h0,$r3		# h0*r3
	mulhwu	$D3,$h0,$r3

	mullw	$t0,$h1,$s3		# h1*s3
	mulhwu	$t1,$h1,$s3

	mullw	$t2,$h1,$r0		# h1*r0
	mulhwu	$t3,$h1,$r0
	 addc	$d0,$d0,$t0
	 adde	$D0,$D0,$t1

	mullw	$t0,$h1,$r1		# h1*r1
	mulhwu	$t1,$h1,$r1
	 addc	$d1,$d1,$t2
	 adde	$D1,$D1,$t3

	mullw	$t2,$h1,$r2		# h1*r2
	mulhwu	$t3,$h1,$r2
	 addc	$d2,$d2,$t0
	 adde	$D2,$D2,$t1

	mullw	$t0,$h2,$s2		# h2*s2
	mulhwu	$t1,$h2,$s2
	 addc	$d3,$d3,$t2
	 adde	$D3,$D3,$t3

	mullw	$t2,$h2,$s3		# h2*s3
	mulhwu	$t3,$h2,$s3
	 addc	$d0,$d0,$t0
	 adde	$D0,$D0,$t1

	mullw	$t0,$h2,$r0		# h2*r0
	mulhwu	$t1,$h2,$r0
	 addc	$d1,$d1,$t2
	 adde	$D1,$D1,$t3

	mullw	$t2,$h2,$r1		# h2*r1
	mulhwu	$t3,$h2,$r1
	 addc	$d2,$d2,$t0
	 adde	$D2,$D2,$t1

	mullw	$t0,$h3,$s1		# h3*s1
	mulhwu	$t1,$h3,$s1
	 addc	$d3,$d3,$t2
	 adde	$D3,$D3,$t3

	mullw	$t2,$h3,$s2		# h3*s2
	mulhwu	$t3,$h3,$s2
	 addc	$d0,$d0,$t0
	 adde	$D0,$D0,$t1

	mullw	$t0,$h3,$s3		# h3*s3
	mulhwu	$t1,$h3,$s3
	 addc	$d1,$d1,$t2
	 adde	$D1,$D1,$t3

	mullw	$t2,$h3,$r0		# h3*r0
	mulhwu	$t3,$h3,$r0
	 addc	$d2,$d2,$t0
	 adde	$D2,$D2,$t1

	mullw	$t0,$h4,$s1		# h4*s1
	 addc	$d3,$d3,$t2
	 adde	$D3,$D3,$t3
	addc	$d1,$d1,$t0

	mullw	$t1,$h4,$s2		# h4*s2
	 addze	$D1,$D1
	addc	$d2,$d2,$t1
	addze	$D2,$D2

	mullw	$t2,$h4,$s3		# h4*s3
	addc	$d3,$d3,$t2
	addze	$D3,$D3

	mullw	$h4,$h4,$r0		# h4*r0

	addc	$h1,$d1,$D0
	adde	$h2,$d2,$D1
	adde	$h3,$d3,$D2
	adde	$h4,$h4,$D3

	andc	$D0,$h4,$mask		# final reduction step
	and	$h4,$h4,$mask
	srwi	$D1,$D0,2
	add	$D0,$D0,$D1
	addc	$h0,$d0,$D0
	addze	$h1,$h1
	addze	$h2,$h2
	addze	$h3,$h3
	addze	$h4,$h4

	bdnz	Loop

	stw	$h0,0($ctx)		# store hash value
	stw	$h1,4($ctx)
	stw	$h2,8($ctx)
	stw	$h3,12($ctx)
	stw	$h4,16($ctx)

	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
	addi	$sp,$sp,$FRAME
Labort:
	blr
	.long	0
	.byte	0,12,4,1,0x80,18,4,0
.size	.poly1305_blocks,.-.poly1305_blocks

.globl	.poly1305_emit
.align	4
.poly1305_emit:
	$STU	$sp,-$FRAME($sp)
	mflr	r0
	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$FRAME+$LRSAVE`($sp)

	lwz	$h0,0($ctx)		# load hash
	lwz	$h1,4($ctx)
	lwz	$h2,8($ctx)
	lwz	$h3,12($ctx)
	lwz	$h4,16($ctx)

	addic	$d0,$h0,5		# compare to modulus
	addze	$d1,$h1
	addze	$d2,$h2
	addze	$d3,$h3
	addze	$mask,$h4

	srwi	$mask,$mask,2		# did it carry/borrow?
	neg	$mask,$mask

	andc	$h0,$h0,$mask
	and	$d0,$d0,$mask
	andc	$h1,$h1,$mask
	and	$d1,$d1,$mask
	or	$h0,$h0,$d0
	lwz	$d0,0($nonce)		# load nonce
	andc	$h2,$h2,$mask
	and	$d2,$d2,$mask
	or	$h1,$h1,$d1
	lwz	$d1,4($nonce)
	andc	$h3,$h3,$mask
	and	$d3,$d3,$mask
	or	$h2,$h2,$d2
	lwz	$d2,8($nonce)
	or	$h3,$h3,$d3
	lwz	$d3,12($nonce)

	addc	$h0,$h0,$d0		# accumulate nonce
	adde	$h1,$h1,$d1
	adde	$h2,$h2,$d2
	adde	$h3,$h3,$d3
___
$code.=<<___	if ($LITTLE_ENDIAN);
	stw	$h0,0($mac)		# write result
	stw	$h1,4($mac)
	stw	$h2,8($mac)
	stw	$h3,12($mac)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$d1,4
	stwbrx	$h0,0,$mac		# write result
	li	$d2,8
	stwbrx	$h1,$d1,$mac
	li	$d3,12
	stwbrx	$h2,$d2,$mac
	stwbrx	$h3,$d3,$mac
___
$code.=<<___;
	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
	addi	$sp,$sp,$FRAME
	blr
	.long	0
	.byte	0,12,4,1,0x80,4,3,0
.size	.poly1305_emit,.-.poly1305_emit
___
							}
$code.=<<___;
.asciz	"Poly1305 for PPC, CRYPTOGAMS by <appro\@openssl.org>"
___

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements Poly1305 hash for PowerPC FPU.
#
# June 2015
#
# Numbers are cycles per processed byte with poly1305_blocks alone,
# and improvement coefficients relative to gcc-generated code.
#
# Freescale e300	9.78/+30%
# PPC74x0		6.92/+50%
# PPC970		6.03/+80%
# POWER7		3.50/+30%
# POWER8		3.75/+10%

$flavour = shift;

if ($flavour =~ /64/) {
	$SIZE_T	=8;
	$LRSAVE	=2*$SIZE_T;
	$UCMP	="cmpld";
	$STU	="stdu";
	$POP	="ld";
	$PUSH	="std";
} elsif ($flavour =~ /32/) {
	$SIZE_T	=4;
	$LRSAVE	=$SIZE_T;
	$UCMP	="cmplw";
	$STU	="stwu";
	$POP	="lwz";
	$PUSH	="stw";
} else { die "nonsense $flavour"; }

$LITTLE_ENDIAN = ($flavour=~/le$/) ? 4 : 0;

$LWXLE = $LITTLE_ENDIAN ? "lwzx" : "lwbrx";

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";

open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";

$LOCALS=6*$SIZE_T;
$FRAME=$LOCALS+6*8+18*8;

my $sp="r1";

my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
my ($in0,$in1,$in2,$in3,$i1,$i2,$i3) = map("r$_",(7..12,6));

my ($h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,
    $two0,$two32,$two64,$two96,$two130,$five_two130,
    $r0lo,$r0hi,$r1lo,$r1hi,$r2lo,$r2hi,
    $s2lo,$s2hi,$s3lo,$s3hi,
    $c0lo,$c0hi,$c1lo,$c1hi,$c2lo,$c2hi,$c3lo,$c3hi) = map("f$_",(0..31));
# borrowings
my ($r3lo,$r3hi,$s1lo,$s1hi) = ($c0lo,$c0hi,$c1lo,$c1hi);
my ($x0,$x1,$x2,$x3) = ($c2lo,$c2hi,$c3lo,$c3hi);
my ($y0,$y1,$y2,$y3) = ($c3lo,$c3hi,$c1lo,$c1hi);

$code.=<<___;
.machine	"any"
.text

.globl	.poly1305_init_fpu
.align	6
.poly1305_init_fpu:
	$STU	$sp,-$LOCALS($sp)		# minimal frame
	mflr	$padbit
	$PUSH	$padbit,`$LOCALS+$LRSAVE`($sp)

	bl	LPICmeup

	xor	r0,r0,r0
	mtlr	$padbit				# restore lr

	lfd	$two0,8*0($len)			# load constants
	lfd	$two32,8*1($len)
	lfd	$two64,8*2($len)
	lfd	$two96,8*3($len)
	lfd	$two130,8*4($len)
	lfd	$five_two130,8*5($len)

	stfd	$two0,8*0($ctx)			# initial hash value, biased 0
	stfd	$two32,8*1($ctx)
	stfd	$two64,8*2($ctx)
	stfd	$two96,8*3($ctx)

	$UCMP	$inp,r0
	beq-	Lno_key

	lfd	$h3lo,8*13($len)		# new fpscr
	mffs	$h3hi				# old fpscr

	stfd	$two0,8*4($ctx)			# key "template"
	stfd	$two32,8*5($ctx)
	stfd	$two64,8*6($ctx)
	stfd	$two96,8*7($ctx)

	li	$in1,4
	li	$in2,8
	li	$in3,12
	$LWXLE	$in0,0,$inp			# load key
	$LWXLE	$in1,$in1,$inp
	$LWXLE	$in2,$in2,$inp
	$LWXLE	$in3,$in3,$inp

	lis	$i1,0xf000			#   0xf0000000
	ori	$i2,$i1,3			#   0xf0000003
	andc	$in0,$in0,$i1			# &=0x0fffffff
	andc	$in1,$in1,$i2			# &=0x0ffffffc
	andc	$in2,$in2,$i2
	andc	$in3,$in3,$i2

	stw	$in0,`8*4+(4^$LITTLE_ENDIAN)`($ctx)	# fill "template"
	stw	$in1,`8*5+(4^$LITTLE_ENDIAN)`($ctx)
	stw	$in2,`8*6+(4^$LITTLE_ENDIAN)`($ctx)
	stw	$in3,`8*7+(4^$LITTLE_ENDIAN)`($ctx)

	mtfsf	255,$h3lo			# fpscr
	stfd	$two0,8*18($ctx)		# copy constants to context
	stfd	$two32,8*19($ctx)
	stfd	$two64,8*20($ctx)
	stfd	$two96,8*21($ctx)
	stfd	$two130,8*22($ctx)
	stfd	$five_two130,8*23($ctx)

	lfd	$h0lo,8*4($ctx)			# load [biased] key
	lfd	$h1lo,8*5($ctx)
	lfd	$h2lo,8*6($ctx)
	lfd	$h3lo,8*7($ctx)

	fsub	$h0lo,$h0lo,$two0		# r0
	fsub	$h1lo,$h1lo,$two32		# r1
	fsub	$h2lo,$h2lo,$two64		# r2
	fsub	$h3lo,$h3lo,$two96		# r3

	lfd	$two0,8*6($len)			# more constants
	lfd	$two32,8*7($len)
	lfd	$two64,8*8($len)
	lfd	$two96,8*9($len)

	fmul	$h1hi,$h1lo,$five_two130	# s1
	fmul	$h2hi,$h2lo,$five_two130	# s2
	 stfd	$h3hi,8*15($ctx)		# borrow slot for original fpscr
	fmul	$h3hi,$h3lo,$five_two130	# s3

	fadd	$h0hi,$h0lo,$two0
	 stfd	$h1hi,8*12($ctx)		# put aside for now
	fadd	$h1hi,$h1lo,$two32
	 stfd	$h2hi,8*13($ctx)
	fadd	$h2hi,$h2lo,$two64
	 stfd	$h3hi,8*14($ctx)
	fadd	$h3hi,$h3lo,$two96

	fsub	$h0hi,$h0hi,$two0
	fsub	$h1hi,$h1hi,$two32
	fsub	$h2hi,$h2hi,$two64
	fsub	$h3hi,$h3hi,$two96

	lfd	$two0,8*10($len)		# more constants
	lfd	$two32,8*11($len)
	lfd	$two64,8*12($len)

	fsub	$h0lo,$h0lo,$h0hi
	fsub	$h1lo,$h1lo,$h1hi
	fsub	$h2lo,$h2lo,$h2hi
	fsub	$h3lo,$h3lo,$h3hi

	stfd	$h0hi,8*5($ctx)			# r0hi
	stfd	$h1hi,8*7($ctx)			# r1hi
	stfd	$h2hi,8*9($ctx)			# r2hi
	stfd	$h3hi,8*11($ctx)		# r3hi

	stfd	$h0lo,8*4($ctx)			# r0lo
	stfd	$h1lo,8*6($ctx)			# r1lo
	stfd	$h2lo,8*8($ctx)			# r2lo
	stfd	$h3lo,8*10($ctx)		# r3lo

	lfd	$h1lo,8*12($ctx)		# s1
	lfd	$h2lo,8*13($ctx)		# s2
	lfd	$h3lo,8*14($ctx)		# s3
	lfd	$h0lo,8*15($ctx)		# pull original fpscr

	fadd	$h1hi,$h1lo,$two0
	fadd	$h2hi,$h2lo,$two32
	fadd	$h3hi,$h3lo,$two64

	fsub	$h1hi,$h1hi,$two0
	fsub	$h2hi,$h2hi,$two32
	fsub	$h3hi,$h3hi,$two64

	fsub	$h1lo,$h1lo,$h1hi
	fsub	$h2lo,$h2lo,$h2hi
	fsub	$h3lo,$h3lo,$h3hi

	stfd	$h1hi,8*13($ctx)		# s1hi
	stfd	$h2hi,8*15($ctx)		# s2hi
	stfd	$h3hi,8*17($ctx)		# s3hi

	stfd	$h1lo,8*12($ctx)		# s1lo
	stfd	$h2lo,8*14($ctx)		# s2lo
	stfd	$h3lo,8*16($ctx)		# s3lo

	mtfsf	255,$h0lo			# restore fpscr
Lno_key:
	xor	r3,r3,r3
	addi	$sp,$sp,$LOCALS
	blr
	.long	0
	.byte	0,12,4,1,0x80,0,2,0
.size	.poly1305_init_fpu,.-.poly1305_init_fpu

.globl	.poly1305_blocks_fpu
.align	4
.poly1305_blocks_fpu:
	srwi.	$len,$len,4
	beq-	Labort

	$STU	$sp,-$FRAME($sp)
	mflr	r0
	stfd	f14,`$FRAME-8*18`($sp)
	stfd	f15,`$FRAME-8*17`($sp)
	stfd	f16,`$FRAME-8*16`($sp)
	stfd	f17,`$FRAME-8*15`($sp)
	stfd	f18,`$FRAME-8*14`($sp)
	stfd	f19,`$FRAME-8*13`($sp)
	stfd	f20,`$FRAME-8*12`($sp)
	stfd	f21,`$FRAME-8*11`($sp)
	stfd	f22,`$FRAME-8*10`($sp)
	stfd	f23,`$FRAME-8*9`($sp)
	stfd	f24,`$FRAME-8*8`($sp)
	stfd	f25,`$FRAME-8*7`($sp)
	stfd	f26,`$FRAME-8*6`($sp)
	stfd	f27,`$FRAME-8*5`($sp)
	stfd	f28,`$FRAME-8*4`($sp)
	stfd	f29,`$FRAME-8*3`($sp)
	stfd	f30,`$FRAME-8*2`($sp)
	stfd	f31,`$FRAME-8*1`($sp)
	$PUSH	r0,`$FRAME+$LRSAVE`($sp)

	xor	r0,r0,r0
	li	$in3,1
	mtctr	$len
	neg	$len,$len
	stw	r0,`$LOCALS+8*4+(0^$LITTLE_ENDIAN)`($sp)
	stw	$in3,`$LOCALS+8*4+(4^$LITTLE_ENDIAN)`($sp)

	lfd	$two0,8*18($ctx)		# load constants
	lfd	$two32,8*19($ctx)
	lfd	$two64,8*20($ctx)
	lfd	$two96,8*21($ctx)
	lfd	$two130,8*22($ctx)
	lfd	$five_two130,8*23($ctx)

	lfd	$h0lo,8*0($ctx)			# load [biased] hash value
	lfd	$h1lo,8*1($ctx)
	lfd	$h2lo,8*2($ctx)
	lfd	$h3lo,8*3($ctx)

	stfd	$two0,`$LOCALS+8*0`($sp)	# input "template"
	oris	$in3,$padbit,`(1023+52+96)<<4`
	stfd	$two32,`$LOCALS+8*1`($sp)
	stfd	$two64,`$LOCALS+8*2`($sp)
	stw	$in3,`$LOCALS+8*3+(0^$LITTLE_ENDIAN)`($sp)

	li	$i1,4
	li	$i2,8
	li	$i3,12
	$LWXLE	$in0,0,$inp			# load input
	$LWXLE	$in1,$i1,$inp
	$LWXLE	$in2,$i2,$inp
	$LWXLE	$in3,$i3,$inp
	addi	$inp,$inp,16

	stw	$in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)	# fill "template"
	stw	$in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
	stw	$in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
	stw	$in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)

	mffs	$x0				# original fpscr
	lfd	$x1,`$LOCALS+8*4`($sp)		# new fpscr
	lfd	$r0lo,8*4($ctx)			# load key
	lfd	$r0hi,8*5($ctx)
	lfd	$r1lo,8*6($ctx)
	lfd	$r1hi,8*7($ctx)
	lfd	$r2lo,8*8($ctx)
	lfd	$r2hi,8*9($ctx)
	lfd	$r3lo,8*10($ctx)
	lfd	$r3hi,8*11($ctx)
	lfd	$s1lo,8*12($ctx)
	lfd	$s1hi,8*13($ctx)
	lfd	$s2lo,8*14($ctx)
	lfd	$s2hi,8*15($ctx)
	lfd	$s3lo,8*16($ctx)
	lfd	$s3hi,8*17($ctx)

	stfd	$x0,`$LOCALS+8*4`($sp)		# save original fpscr
	mtfsf	255,$x1

	addic	$len,$len,1
	addze	r0,r0
	slwi.	r0,r0,4
	sub	$inp,$inp,r0			# conditional rewind

	lfd	$x0,`$LOCALS+8*0`($sp)
	lfd	$x1,`$LOCALS+8*1`($sp)
	lfd	$x2,`$LOCALS+8*2`($sp)
	lfd	$x3,`$LOCALS+8*3`($sp)

	fsub	$h0lo,$h0lo,$two0		# de-bias hash value
	 $LWXLE	$in0,0,$inp			# modulo-scheduled input load
	fsub	$h1lo,$h1lo,$two32
	 $LWXLE	$in1,$i1,$inp
	fsub	$h2lo,$h2lo,$two64
	 $LWXLE	$in2,$i2,$inp
	fsub	$h3lo,$h3lo,$two96
	 $LWXLE	$in3,$i3,$inp

	fsub	$x0,$x0,$two0			# de-bias input
	 addi	$inp,$inp,16
	fsub	$x1,$x1,$two32
	fsub	$x2,$x2,$two64
	fsub	$x3,$x3,$two96

	fadd	$x0,$x0,$h0lo			# accumulate input
	 stw	$in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)
	fadd	$x1,$x1,$h1lo
	 stw	$in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
	fadd	$x2,$x2,$h2lo
	 stw	$in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
	fadd	$x3,$x3,$h3lo
	 stw	$in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)

	b	Lentry

.align	4
Loop:
	fsub	$y0,$y0,$two0			# de-bias input
	 addic	$len,$len,1
	fsub	$y1,$y1,$two32
	 addze	r0,r0
	fsub	$y2,$y2,$two64
	 slwi.	r0,r0,4
	fsub	$y3,$y3,$two96
	 sub	$inp,$inp,r0			# conditional rewind

	fadd	$h0lo,$h0lo,$y0			# accumulate input
	fadd	$h0hi,$h0hi,$y1
	fadd	$h2lo,$h2lo,$y2
	fadd	$h2hi,$h2hi,$y3

	######################################### base 2^48 -> base 2^32
	fadd	$c1lo,$h1lo,$two64
	 $LWXLE	$in0,0,$inp			# modulo-scheduled input load
	fadd	$c1hi,$h1hi,$two64
	 $LWXLE	$in1,$i1,$inp
	fadd	$c3lo,$h3lo,$two130
	 $LWXLE	$in2,$i2,$inp
	fadd	$c3hi,$h3hi,$two130
	 $LWXLE	$in3,$i3,$inp
	fadd	$c0lo,$h0lo,$two32
	 addi	$inp,$inp,16
	fadd	$c0hi,$h0hi,$two32
	fadd	$c2lo,$h2lo,$two96
	fadd	$c2hi,$h2hi,$two96

	fsub	$c1lo,$c1lo,$two64
	 stw	$in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)	# fill "template"
	fsub	$c1hi,$c1hi,$two64
	 stw	$in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
	fsub	$c3lo,$c3lo,$two130
	 stw	$in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
	fsub	$c3hi,$c3hi,$two130
	 stw	$in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
	fsub	$c0lo,$c0lo,$two32
	fsub	$c0hi,$c0hi,$two32
	fsub	$c2lo,$c2lo,$two96
	fsub	$c2hi,$c2hi,$two96

	fsub	$h1lo,$h1lo,$c1lo
	fsub	$h1hi,$h1hi,$c1hi
	fsub	$h3lo,$h3lo,$c3lo
	fsub	$h3hi,$h3hi,$c3hi
	fsub	$h2lo,$h2lo,$c2lo
	fsub	$h2hi,$h2hi,$c2hi
	fsub	$h0lo,$h0lo,$c0lo
	fsub	$h0hi,$h0hi,$c0hi

	fadd	$h1lo,$h1lo,$c0lo
	fadd	$h1hi,$h1hi,$c0hi
	fadd	$h3lo,$h3lo,$c2lo
	fadd	$h3hi,$h3hi,$c2hi
	fadd	$h2lo,$h2lo,$c1lo
	fadd	$h2hi,$h2hi,$c1hi
	fmadd	$h0lo,$c3lo,$five_two130,$h0lo
	fmadd	$h0hi,$c3hi,$five_two130,$h0hi

	fadd	$x1,$h1lo,$h1hi
	 lfd	$s1lo,8*12($ctx)		# reload constants
	fadd	$x3,$h3lo,$h3hi
	 lfd	$s1hi,8*13($ctx)
	fadd	$x2,$h2lo,$h2hi
	 lfd	$r3lo,8*10($ctx)
	fadd	$x0,$h0lo,$h0hi
	 lfd	$r3hi,8*11($ctx)
Lentry:
	fmul	$h0lo,$s3lo,$x1
	fmul	$h0hi,$s3hi,$x1
	fmul	$h2lo,$r1lo,$x1
	fmul	$h2hi,$r1hi,$x1
	fmul	$h1lo,$r0lo,$x1
	fmul	$h1hi,$r0hi,$x1
	fmul	$h3lo,$r2lo,$x1
	fmul	$h3hi,$r2hi,$x1

	fmadd	$h0lo,$s1lo,$x3,$h0lo
	fmadd	$h0hi,$s1hi,$x3,$h0hi
	fmadd	$h2lo,$s3lo,$x3,$h2lo
	fmadd	$h2hi,$s3hi,$x3,$h2hi
	fmadd	$h1lo,$s2lo,$x3,$h1lo
	fmadd	$h1hi,$s2hi,$x3,$h1hi
	fmadd	$h3lo,$r0lo,$x3,$h3lo
	fmadd	$h3hi,$r0hi,$x3,$h3hi

	fmadd	$h0lo,$s2lo,$x2,$h0lo
	fmadd	$h0hi,$s2hi,$x2,$h0hi
	fmadd	$h2lo,$r0lo,$x2,$h2lo
	fmadd	$h2hi,$r0hi,$x2,$h2hi
	fmadd	$h1lo,$s3lo,$x2,$h1lo
	fmadd	$h1hi,$s3hi,$x2,$h1hi
	fmadd	$h3lo,$r1lo,$x2,$h3lo
	fmadd	$h3hi,$r1hi,$x2,$h3hi

	fmadd	$h0lo,$r0lo,$x0,$h0lo
	 lfd	$y0,`$LOCALS+8*0`($sp)		# load [biased] input
	fmadd	$h0hi,$r0hi,$x0,$h0hi
	 lfd	$y1,`$LOCALS+8*1`($sp)
	fmadd	$h2lo,$r2lo,$x0,$h2lo
	 lfd	$y2,`$LOCALS+8*2`($sp)
	fmadd	$h2hi,$r2hi,$x0,$h2hi
	 lfd	$y3,`$LOCALS+8*3`($sp)
	fmadd	$h1lo,$r1lo,$x0,$h1lo
	fmadd	$h1hi,$r1hi,$x0,$h1hi
	fmadd	$h3lo,$r3lo,$x0,$h3lo
	fmadd	$h3hi,$r3hi,$x0,$h3hi

	bdnz	Loop

	######################################### base 2^48 -> base 2^32
	fadd	$c0lo,$h0lo,$two32
	fadd	$c0hi,$h0hi,$two32
	fadd	$c2lo,$h2lo,$two96
	fadd	$c2hi,$h2hi,$two96
	fadd	$c1lo,$h1lo,$two64
	fadd	$c1hi,$h1hi,$two64
	fadd	$c3lo,$h3lo,$two130
	fadd	$c3hi,$h3hi,$two130

	fsub	$c0lo,$c0lo,$two32
	fsub	$c0hi,$c0hi,$two32
	fsub	$c2lo,$c2lo,$two96
	fsub	$c2hi,$c2hi,$two96
	fsub	$c1lo,$c1lo,$two64
	fsub	$c1hi,$c1hi,$two64
	fsub	$c3lo,$c3lo,$two130
	fsub	$c3hi,$c3hi,$two130

	fsub	$h1lo,$h1lo,$c1lo
	fsub	$h1hi,$h1hi,$c1hi
	fsub	$h3lo,$h3lo,$c3lo
	fsub	$h3hi,$h3hi,$c3hi
	fsub	$h2lo,$h2lo,$c2lo
	fsub	$h2hi,$h2hi,$c2hi
	fsub	$h0lo,$h0lo,$c0lo
	fsub	$h0hi,$h0hi,$c0hi

	fadd	$h1lo,$h1lo,$c0lo
	fadd	$h1hi,$h1hi,$c0hi
	fadd	$h3lo,$h3lo,$c2lo
	fadd	$h3hi,$h3hi,$c2hi
	fadd	$h2lo,$h2lo,$c1lo
	fadd	$h2hi,$h2hi,$c1hi
	fmadd	$h0lo,$c3lo,$five_two130,$h0lo
	fmadd	$h0hi,$c3hi,$five_two130,$h0hi

	fadd	$x1,$h1lo,$h1hi
	fadd	$x3,$h3lo,$h3hi
	fadd	$x2,$h2lo,$h2hi
	fadd	$x0,$h0lo,$h0hi

	lfd	$h0lo,`$LOCALS+8*4`($sp)	# pull saved fpscr
	fadd	$x1,$x1,$two32			# bias
	fadd	$x3,$x3,$two96
	fadd	$x2,$x2,$two64
	fadd	$x0,$x0,$two0

	stfd	$x1,8*1($ctx)			# store [biased] hash value
	stfd	$x3,8*3($ctx)
	stfd	$x2,8*2($ctx)
	stfd	$x0,8*0($ctx)

	mtfsf	255,$h0lo			# restore original fpscr
	lfd	f14,`$FRAME-8*18`($sp)
	lfd	f15,`$FRAME-8*17`($sp)
	lfd	f16,`$FRAME-8*16`($sp)
	lfd	f17,`$FRAME-8*15`($sp)
	lfd	f18,`$FRAME-8*14`($sp)
	lfd	f19,`$FRAME-8*13`($sp)
	lfd	f20,`$FRAME-8*12`($sp)
	lfd	f21,`$FRAME-8*11`($sp)
	lfd	f22,`$FRAME-8*10`($sp)
	lfd	f23,`$FRAME-8*9`($sp)
	lfd	f24,`$FRAME-8*8`($sp)
	lfd	f25,`$FRAME-8*7`($sp)
	lfd	f26,`$FRAME-8*6`($sp)
	lfd	f27,`$FRAME-8*5`($sp)
	lfd	f28,`$FRAME-8*4`($sp)
	lfd	f29,`$FRAME-8*3`($sp)
	lfd	f30,`$FRAME-8*2`($sp)
	lfd	f31,`$FRAME-8*1`($sp)
	addi	$sp,$sp,$FRAME
Labort:
	blr
	.long	0
	.byte	0,12,4,1,0x80,0,4,0
.size	.poly1305_blocks_fpu,.-.poly1305_blocks_fpu
___
{
my ($mac,$nonce)=($inp,$len);

my ($h0,$h1,$h2,$h3,$h4, $d0,$d1,$d2,$d3
   ) = map("r$_",(7..11,28..31));
my $mask = "r0";
my $FRAME = (6+4)*$SIZE_T;

$code.=<<___;
.globl	.poly1305_emit_fpu
.align	4
.poly1305_emit_fpu:
	$STU	$sp,-$FRAME($sp)
	mflr	r0
	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$FRAME+$LRSAVE`($sp)

	lwz	$d0,`8*0+(0^$LITTLE_ENDIAN)`($ctx)	# load hash
	lwz	$h0,`8*0+(4^$LITTLE_ENDIAN)`($ctx)
	lwz	$d1,`8*1+(0^$LITTLE_ENDIAN)`($ctx)
	lwz	$h1,`8*1+(4^$LITTLE_ENDIAN)`($ctx)
	lwz	$d2,`8*2+(0^$LITTLE_ENDIAN)`($ctx)
	lwz	$h2,`8*2+(4^$LITTLE_ENDIAN)`($ctx)
	lwz	$d3,`8*3+(0^$LITTLE_ENDIAN)`($ctx)
	lwz	$h3,`8*3+(4^$LITTLE_ENDIAN)`($ctx)

	lis	$mask,0xfff0
	andc	$d0,$d0,$mask			# mask exponent
	andc	$d1,$d1,$mask
	andc	$d2,$d2,$mask
	andc	$d3,$d3,$mask			# can be partially reduced...
	li	$mask,3

	srwi	$padbit,$d3,2			# ... so reduce
	and	$h4,$d3,$mask
	andc	$d3,$d3,$mask
	add	$d3,$d3,$padbit
___
						if ($SIZE_T==4) {
$code.=<<___;
	addc	$h0,$h0,$d3
	adde	$h1,$h1,$d0
	adde	$h2,$h2,$d1
	adde	$h3,$h3,$d2
	addze	$h4,$h4

	addic	$d0,$h0,5			# compare to modulus
	addze	$d1,$h1
	addze	$d2,$h2
	addze	$d3,$h3
	addze	$mask,$h4

	srwi	$mask,$mask,2			# did it carry/borrow?
	neg	$mask,$mask
	srawi	$mask,$mask,31			# mask

	andc	$h0,$h0,$mask
	and	$d0,$d0,$mask
	andc	$h1,$h1,$mask
	and	$d1,$d1,$mask
	or	$h0,$h0,$d0
	lwz	$d0,0($nonce)			# load nonce
	andc	$h2,$h2,$mask
	and	$d2,$d2,$mask
	or	$h1,$h1,$d1
	lwz	$d1,4($nonce)
	andc	$h3,$h3,$mask
	and	$d3,$d3,$mask
	or	$h2,$h2,$d2
	lwz	$d2,8($nonce)
	or	$h3,$h3,$d3
	lwz	$d3,12($nonce)

	addc	$h0,$h0,$d0			# accumulate nonce
	adde	$h1,$h1,$d1
	adde	$h2,$h2,$d2
	adde	$h3,$h3,$d3
___
						} else {
$code.=<<___;
	add	$h0,$h0,$d3
	add	$h1,$h1,$d0
	add	$h2,$h2,$d1
	add	$h3,$h3,$d2

	srdi	$d0,$h0,32
	add	$h1,$h1,$d0
	srdi	$d1,$h1,32
	add	$h2,$h2,$d1
	srdi	$d2,$h2,32
	add	$h3,$h3,$d2
	srdi	$d3,$h3,32
	add	$h4,$h4,$d3

	insrdi	$h0,$h1,32,0
	insrdi	$h2,$h3,32,0

	addic	$d0,$h0,5			# compare to modulus
	addze	$d1,$h2
	addze	$d2,$h4

	srdi	$mask,$d2,2			# did it carry/borrow?
	neg	$mask,$mask
	sradi	$mask,$mask,63			# mask
	ld	$d2,0($nonce)			# load nonce
	ld	$d3,8($nonce)

	andc	$h0,$h0,$mask
	and	$d0,$d0,$mask
	andc	$h2,$h2,$mask
	and	$d1,$d1,$mask
	or	$h0,$h0,$d0
	or	$h2,$h2,$d1
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	rotldi	$d2,$d2,32			# flip nonce words
	rotldi	$d3,$d3,32
___
$code.=<<___;
	addc	$h0,$h0,$d2			# accumulate nonce
	adde	$h2,$h2,$d3

	srdi	$h1,$h0,32
	srdi	$h3,$h2,32
___
						}
$code.=<<___	if ($LITTLE_ENDIAN);
	stw	$h0,0($mac)			# write result
	stw	$h1,4($mac)
	stw	$h2,8($mac)
	stw	$h3,12($mac)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$d1,4
	stwbrx	$h0,0,$mac			# write result
	li	$d2,8
	stwbrx	$h1,$d1,$mac
	li	$d3,12
	stwbrx	$h2,$d2,$mac
	stwbrx	$h3,$d3,$mac
___
$code.=<<___;
	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
	addi	$sp,$sp,$FRAME
	blr
	.long	0
	.byte	0,12,4,1,0x80,4,3,0
.size	.poly1305_emit_fpu,.-.poly1305_emit_fpu
___
}
# Ugly hack here, because PPC assembler syntax seem to vary too
# much from platforms to platform...
$code.=<<___;
.align	6
LPICmeup:
	mflr	r0
	bcl	20,31,\$+4
	mflr	$len	# vvvvvv "distance" between . and 1st data entry
	addi	$len,$len,`64-8`	# borrow $len
	mtlr	r0
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,0,0
	.space	`64-9*4`

.quad	0x4330000000000000		# 2^(52+0)
.quad	0x4530000000000000		# 2^(52+32)
.quad	0x4730000000000000		# 2^(52+64)
.quad	0x4930000000000000		# 2^(52+96)
.quad	0x4b50000000000000		# 2^(52+130)

.quad	0x37f4000000000000		# 5/2^130

.quad	0x4430000000000000		# 2^(52+16+0)
.quad	0x4630000000000000		# 2^(52+16+32)
.quad	0x4830000000000000		# 2^(52+16+64)
.quad	0x4a30000000000000		# 2^(52+16+96)
.quad	0x3e30000000000000		# 2^(52+16+0-96)
.quad	0x4030000000000000		# 2^(52+16+32-96)
.quad	0x4230000000000000		# 2^(52+16+64-96)

.quad	0x0000000000000001		# fpscr: truncate, no exceptions
.asciz	"Poly1305 for PPC FPU, CRYPTOGAMS by <appro\@openssl.org>"
.align	4
___

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements Poly1305 hash for s390x.
#
# June 2015
#
# ~6.6/2.3 cpb on z10/z196+, >2x improvement over compiler-generated
# code. For older compiler improvement coefficient is >3x, because
# then base 2^64 and base 2^32 implementations are compared.
#
# On side note, z13 enables vector base 2^26 implementation...

$flavour = shift;

if ($flavour =~ /3[12]/) {
	$SIZE_T=4;
	$g="";
} else {
	$SIZE_T=8;
	$g="g";
}

while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

$sp="%r15";

my ($ctx,$inp,$len,$padbit) = map("%r$_",(2..5));

$code.=<<___;
.text

.globl	poly1305_init
.type	poly1305_init,\@function
.align	16
poly1305_init:
	lghi	%r0,0
	lghi	%r1,-1
	stg	%r0,0($ctx)		# zero hash value
	stg	%r0,8($ctx)
	stg	%r0,16($ctx)

	cl${g}r	$inp,%r0
	je	.Lno_key

	lrvg	%r4,0($inp)		# load little-endian key
	lrvg	%r5,8($inp)

	nihl	%r1,0xffc0		# 0xffffffc0ffffffff
	srlg	%r0,%r1,4		# 0x0ffffffc0fffffff
	srlg	%r1,%r1,4
	nill	%r1,0xfffc		# 0x0ffffffc0ffffffc

	ngr	%r4,%r0
	ngr	%r5,%r1

	stg	%r4,32($ctx)
	stg	%r5,40($ctx)

.Lno_key:
	lghi	%r2,0
	br	%r14
.size	poly1305_init,.-poly1305_init
___
{
my ($d0hi,$d0lo,$d1hi,$d1lo,$t0,$h0,$t1,$h1,$h2) = map("%r$_",(6..14));
my ($r0,$r1,$s1) = map("%r$_",(0..2));

$code.=<<___;
.globl	poly1305_blocks
.type	poly1305_blocks,\@function
.align	16
poly1305_blocks:
	srl${g}	$len,4			# fixed-up in 64-bit build
	lghi	%r0,0
	cl${g}r	$len,%r0
	je	.Lno_data

	stm${g}	%r6,%r14,`6*$SIZE_T`($sp)

	llgfr   $padbit,$padbit		# clear upper half, much needed with
					# non-64-bit ABI
	lg	$r0,32($ctx)		# load key
	lg	$r1,40($ctx)

	lg	$h0,0($ctx)		# load hash value
	lg	$h1,8($ctx)
	lg	$h2,16($ctx)

	st$g	$ctx,`2*$SIZE_T`($sp)	# off-load $ctx
	srlg	$s1,$r1,2
	algr	$s1,$r1			# s1 = r1 + r1>>2
	j	.Loop

.align	16
.Loop:
	lrvg	$d0lo,0($inp)		# load little-endian input
	lrvg	$d1lo,8($inp)
	la	$inp,16($inp)

	algr	$d0lo,$h0		# accumulate input
	alcgr	$d1lo,$h1

	lgr	$h0,$d0lo
	mlgr	$d0hi,$r0		# h0*r0	  -> $d0hi:$d0lo
	lgr	$h1,$d1lo
	mlgr	$d1hi,$s1		# h1*5*r1 -> $d1hi:$d1lo

	mlgr	$t0,$r1			# h0*r1   -> $t0:$h0
	mlgr	$t1,$r0			# h1*r0   -> $t1:$h1
	alcgr	$h2,$padbit

	algr	$d0lo,$d1lo
	lgr	$d1lo,$h2
	alcgr	$d0hi,$d1hi
	lghi	$d1hi,0

	algr	$h1,$h0
	alcgr	$t1,$t0

	msgr	$d1lo,$s1		# h2*s1
	msgr	$h2,$r0			# h2*r0

	algr	$h1,$d1lo
	alcgr	$t1,$d1hi		# $d1hi is zero

	algr	$h1,$d0hi
	alcgr	$h2,$t1

	lghi	$h0,-4			# final reduction step
	ngr	$h0,$h2
	srlg	$t0,$h2,2
	algr	$h0,$t0
	lghi	$t1,3
	ngr	$h2,$t1

	algr	$h0,$d0lo
	alcgr	$h1,$d1hi		# $d1hi is still zero
	alcgr	$h2,$d1hi		# $d1hi is still zero

	brct$g	$len,.Loop

	l$g	$ctx,`2*$SIZE_T`($sp)	# restore $ctx

	stg	$h0,0($ctx)		# store hash value
	stg	$h1,8($ctx)
	stg	$h2,16($ctx)

	lm${g}	%r6,%r14,`6*$SIZE_T`($sp)
.Lno_data:
	br	%r14
.size	poly1305_blocks,.-poly1305_blocks
___
}
{
my ($mac,$nonce)=($inp,$len);
my ($h0,$h1,$h2,$d0,$d1)=map("%r$_",(5..9));

$code.=<<___;
.globl	poly1305_emit
.type	poly1305_emit,\@function
.align	16
poly1305_emit:
	stm${g}	%r6,%r9,`6*$SIZE_T`($sp)

	lg	$h0,0($ctx)
	lg	$h1,8($ctx)
	lg	$h2,16($ctx)

	lghi	%r0,5
	lghi	%r1,0
	lgr	$d0,$h0
	lgr	$d1,$h1

	algr	$h0,%r0			# compare to modulus
	alcgr	$h1,%r1
	alcgr	$h2,%r1

	srlg	$h2,$h2,2		# did it borrow/carry?
	slgr	%r1,$h2			# 0-$h2>>2
	lg	$h2,0($nonce)		# load nonce
	lghi	%r0,-1
	lg	$ctx,8($nonce)
	xgr	%r0,%r1			# ~%r1

	ngr	$h0,%r1
	ngr	$d0,%r0
	ngr	$h1,%r1
	ngr	$d1,%r0
	ogr	$h0,$d0
	rllg	$d0,$h2,32		# flip nonce words
	ogr	$h1,$d1
	rllg	$d1,$ctx,32

	algr	$h0,$d0			# accumulate nonce
	alcgr	$h1,$d1

	strvg	$h0,0($mac)		# write little-endian result
	strvg	$h1,8($mac)

	lm${g}	%r6,%r9,`6*$SIZE_T`($sp)
	br	%r14
.size	poly1305_emit,.-poly1305_emit

.string	"Poly1305 for s390x, CRYPTOGAMS by <appro\@openssl.org>"
___
}

$code =~ s/\`([^\`]*)\`/eval $1/gem;
$code =~ s/\b(srlg\s+)(%r[0-9]+\s*,)\s*([0-9]+)/$1$2$2$3/gm;

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements Poly1305 hash for SPARCv9, vanilla, as well
# as VIS3 and FMA extensions.
#
# May, August 2015
#
# Numbers are cycles per processed byte with poly1305_blocks alone.
#
#			IALU(*)		FMA
#
# UltraSPARC III	12.3(**)
# SPARC T3		7.92
# SPARC T4		1.70(***)	6.55
# SPARC64 X		5.60		3.64
#
# (*)	Comparison to compiler-generated code is really problematic,
#	because latter's performance varies too much depending on too
#	many variables. For example, one can measure from 5x to 15x
#	improvement on T4 for gcc-4.6. Well, in T4 case it's a bit
#	unfair comparison, because compiler doesn't use VIS3, but
#	given same initial conditions coefficient varies from 3x to 9x.
# (**)	Pre-III performance should be even worse; floating-point
#	performance for UltraSPARC I-IV on the other hand is reported
#	to be 4.25 for hand-coded assembly, but they are just too old
#	to care about.
# (***)	Multi-process benchmark saturates at ~12.5x single-process
#	result on 8-core processor, or ~21GBps per 2.85GHz socket.

my $output = pop;
open STDOUT,">$output";

my ($ctx,$inp,$len,$padbit,$shl,$shr)	= map("%i$_",(0..5));
my ($r0,$r1,$r2,$r3,$s1,$s2,$s3,$h4)	= map("%l$_",(0..7));
my ($h0,$h1,$h2,$h3, $t0,$t1,$t2)	= map("%o$_",(0..5,7));
my ($d0,$d1,$d2,$d3)			= map("%g$_",(1..4));

my $output = pop;
open STDOUT,">$stdout";

$code.=<<___;
#include "sparc_arch.h"

#ifdef	__arch64__
.register	%g2,#scratch
.register	%g3,#scratch
# define	STPTR	stx
# define	SIZE_T	8
#else
# define	STPTR	st
# define	SIZE_T	4
#endif
#define	LOCALS	(STACK_BIAS+STACK_FRAME)

.section	".text",#alloc,#execinstr

#ifdef __PIC__
SPARC_PIC_THUNK(%g1)
#endif

.globl	poly1305_init
.align	32
poly1305_init:
	save	%sp,-STACK_FRAME-16,%sp
	nop

	SPARC_LOAD_ADDRESS(OPENSSL_sparcv9cap_P,%g1)
	ld	[%g1],%g1

	and	%g1,SPARCV9_FMADD|SPARCV9_VIS3,%g1
	cmp	%g1,SPARCV9_FMADD
	be	.Lpoly1305_init_fma
	nop

	stx	%g0,[$ctx+0]
	stx	%g0,[$ctx+8]		! zero hash value
	brz,pn	$inp,.Lno_key
	stx	%g0,[$ctx+16]

	and	$inp,7,$shr		! alignment factor
	andn	$inp,7,$inp
	sll	$shr,3,$shr		! *8
	neg	$shr,$shl

	sethi	%hi(0x0ffffffc),$t0
	set	8,$h1
	or	$t0,%lo(0x0ffffffc),$t0
	set	16,$h2
	sllx	$t0,32,$t1
	or	$t0,$t1,$t1		! 0x0ffffffc0ffffffc
	or	$t1,3,$t0		! 0x0ffffffc0fffffff

	ldxa	[$inp+%g0]0x88,$h0	! load little-endian key
	brz,pt	$shr,.Lkey_aligned
	ldxa	[$inp+$h1]0x88,$h1

	ldxa	[$inp+$h2]0x88,$h2
	srlx	$h0,$shr,$h0
	sllx	$h1,$shl,$t2
	srlx	$h1,$shr,$h1
	or	$t2,$h0,$h0
	sllx	$h2,$shl,$h2
	or	$h2,$h1,$h1

.Lkey_aligned:
	and	$t0,$h0,$h0
	and	$t1,$h1,$h1
	stx	$h0,[$ctx+32+0]		! store key
	stx	$h1,[$ctx+32+8]

	andcc	%g1,SPARCV9_VIS3,%g0
	be	.Lno_key
	nop

1:	call	.+8
	add	%o7,poly1305_blocks_vis3-1b,%o7

	add	%o7,poly1305_emit-poly1305_blocks_vis3,%o5
	STPTR	%o7,[%i2]
	STPTR	%o5,[%i2+SIZE_T]

	ret
	restore	%g0,1,%o0		! return 1

.Lno_key:
	ret
	restore	%g0,%g0,%o0		! return 0
.type	poly1305_init,#function
.size	poly1305_init,.-poly1305_init

.globl	poly1305_blocks
.align	32
poly1305_blocks:
	save	%sp,-STACK_FRAME,%sp
	srln	$len,4,$len

	brz,pn	$len,.Lno_data
	nop

	ld	[$ctx+32+0],$r1		! load key
	ld	[$ctx+32+4],$r0
	ld	[$ctx+32+8],$r3
	ld	[$ctx+32+12],$r2

	ld	[$ctx+0],$h1		! load hash value
	ld	[$ctx+4],$h0
	ld	[$ctx+8],$h3
	ld	[$ctx+12],$h2
	ld	[$ctx+16],$h4

	and	$inp,7,$shr		! alignment factor
	andn	$inp,7,$inp
	set	8,$d1
	sll	$shr,3,$shr		! *8
	set	16,$d2
	neg	$shr,$shl

	srl	$r1,2,$s1
	srl	$r2,2,$s2
	add	$r1,$s1,$s1
	srl	$r3,2,$s3
	add	$r2,$s2,$s2
	add	$r3,$s3,$s3

.Loop:
	ldxa	[$inp+%g0]0x88,$d0	! load little-endian input
	brz,pt	$shr,.Linp_aligned
	ldxa	[$inp+$d1]0x88,$d1

	ldxa	[$inp+$d2]0x88,$d2
	srlx	$d0,$shr,$d0
	sllx	$d1,$shl,$t1
	srlx	$d1,$shr,$d1
	or	$t1,$d0,$d0
	sllx	$d2,$shl,$d2
	or	$d2,$d1,$d1

.Linp_aligned:
	srlx	$d0,32,$t0
	addcc	$d0,$h0,$h0		! accumulate input
	srlx	$d1,32,$t1
	addccc	$t0,$h1,$h1
	addccc	$d1,$h2,$h2
	addccc	$t1,$h3,$h3
	addc	$padbit,$h4,$h4

	umul	$r0,$h0,$d0
	umul	$r1,$h0,$d1
	umul	$r2,$h0,$d2
	umul	$r3,$h0,$d3
	 sub	$len,1,$len
	 add	$inp,16,$inp

	umul	$s3,$h1,$t0
	umul	$r0,$h1,$t1
	umul	$r1,$h1,$t2
	add	$t0,$d0,$d0
	add	$t1,$d1,$d1
	umul	$r2,$h1,$t0
	add	$t2,$d2,$d2
	add	$t0,$d3,$d3

	umul	$s2,$h2,$t1
	umul	$s3,$h2,$t2
	umul	$r0,$h2,$t0
	add	$t1,$d0,$d0
	add	$t2,$d1,$d1
	umul	$r1,$h2,$t1
	add	$t0,$d2,$d2
	add	$t1,$d3,$d3

	umul	$s1,$h3,$t2
	umul	$s2,$h3,$t0
	umul	$s3,$h3,$t1
	add	$t2,$d0,$d0
	add	$t0,$d1,$d1
	umul	$r0,$h3,$t2
	add	$t1,$d2,$d2
	add	$t2,$d3,$d3

	umul	$s1,$h4,$t0
	umul	$s2,$h4,$t1
	umul	$s3,$h4,$t2
	umul	$r0,$h4,$h4
	add	$t0,$d1,$d1
	add	$t1,$d2,$d2
	srlx	$d0,32,$h1
	add	$t2,$d3,$d3
	srlx	$d1,32,$h2

	addcc	$d1,$h1,$h1
	srlx	$d2,32,$h3
	 set	8,$d1
	addccc	$d2,$h2,$h2
	srlx	$d3,32,$t0
	 set	16,$d2
	addccc	$d3,$h3,$h3
	addc	$t0,$h4,$h4

	srl	$h4,2,$t0		! final reduction step
	andn	$h4,3,$t1
	and	$h4,3,$h4
	add	$t1,$t0,$t0

	addcc	$t0,$d0,$h0
	addccc	%g0,$h1,$h1
	addccc	%g0,$h2,$h2
	addccc	%g0,$h3,$h3
	brnz,pt	$len,.Loop
	addc	%g0,$h4,$h4

	st	$h1,[$ctx+0]		! store hash value
	st	$h0,[$ctx+4]
	st	$h3,[$ctx+8]
	st	$h2,[$ctx+12]
	st	$h4,[$ctx+16]

.Lno_data:
	ret
	restore
.type	poly1305_blocks,#function
.size	poly1305_blocks,.-poly1305_blocks
___
########################################################################
# VIS3 has umulxhi and addxc...
{
my ($H0,$H1,$H2,$R0,$R1,$S1,$T1) = map("%o$_",(0..5,7));
my ($D0,$D1,$D2,$T0) = map("%g$_",(1..4));

$code.=<<___;
.align	32
poly1305_blocks_vis3:
	save	%sp,-STACK_FRAME,%sp
	srln	$len,4,$len

	brz,pn	$len,.Lno_data
	nop

	ldx	[$ctx+32+0],$R0		! load key
	ldx	[$ctx+32+8],$R1

	ldx	[$ctx+0],$H0		! load hash value
	ldx	[$ctx+8],$H1
	ld	[$ctx+16],$H2

	and	$inp,7,$shr		! alignment factor
	andn	$inp,7,$inp
	set	8,$r1
	sll	$shr,3,$shr		! *8
	set	16,$r2
	neg	$shr,$shl

	srlx	$R1,2,$S1
	b	.Loop_vis3
	add	$R1,$S1,$S1

.Loop_vis3:
	ldxa	[$inp+%g0]0x88,$D0	! load little-endian input
	brz,pt	$shr,.Linp_aligned_vis3
	ldxa	[$inp+$r1]0x88,$D1

	ldxa	[$inp+$r2]0x88,$D2
	srlx	$D0,$shr,$D0
	sllx	$D1,$shl,$T1
	srlx	$D1,$shr,$D1
	or	$T1,$D0,$D0
	sllx	$D2,$shl,$D2
	or	$D2,$D1,$D1

.Linp_aligned_vis3:
	addcc	$D0,$H0,$H0		! accumulate input
	 sub	$len,1,$len
	addxccc	$D1,$H1,$H1
	 add	$inp,16,$inp

	mulx	$R0,$H0,$D0		! r0*h0
	addxc	$padbit,$H2,$H2
	umulxhi	$R0,$H0,$D1
	mulx	$S1,$H1,$T0		! s1*h1
	umulxhi	$S1,$H1,$T1
	addcc	$T0,$D0,$D0
	mulx	$R1,$H0,$T0		! r1*h0
	addxc	$T1,$D1,$D1
	umulxhi	$R1,$H0,$D2
	addcc	$T0,$D1,$D1
	mulx	$R0,$H1,$T0		! r0*h1
	addxc	%g0,$D2,$D2
	umulxhi	$R0,$H1,$T1
	addcc	$T0,$D1,$D1
	mulx	$S1,$H2,$T0		! s1*h2
	addxc	$T1,$D2,$D2
	mulx	$R0,$H2,$T1		! r0*h2
	addcc	$T0,$D1,$D1
	addxc	$T1,$D2,$D2

	srlx	$D2,2,$T0		! final reduction step
	andn	$D2,3,$T1
	and	$D2,3,$H2
	add	$T1,$T0,$T0

	addcc	$T0,$D0,$H0
	addxccc	%g0,$D1,$H1
	brnz,pt	$len,.Loop_vis3
	addxc	%g0,$H2,$H2

	stx	$H0,[$ctx+0]		! store hash value
	stx	$H1,[$ctx+8]
	st	$H2,[$ctx+16]

	ret
	restore
.type	poly1305_blocks_vis3,#function
.size	poly1305_blocks_vis3,.-poly1305_blocks_vis3
___
}
my ($mac,$nonce) = ($inp,$len);

$code.=<<___;
.globl	poly1305_emit
.align	32
poly1305_emit:
	save	%sp,-STACK_FRAME,%sp

	ld	[$ctx+0],$h1		! load hash value
	ld	[$ctx+4],$h0
	ld	[$ctx+8],$h3
	ld	[$ctx+12],$h2
	ld	[$ctx+16],$h4

	addcc	$h0,5,$r0		! compare to modulus
	addccc	$h1,0,$r1
	addccc	$h2,0,$r2
	addccc	$h3,0,$r3
	addc	$h4,0,$h4
	andcc	$h4,4,%g0		! did it carry/borrow?

	movnz	%icc,$r0,$h0
	ld	[$nonce+0],$r0		! load nonce
	movnz	%icc,$r1,$h1
	ld	[$nonce+4],$r1
	movnz	%icc,$r2,$h2
	ld	[$nonce+8],$r2
	movnz	%icc,$r3,$h3
	ld	[$nonce+12],$r3

	addcc	$r0,$h0,$h0		! accumulate nonce
	addccc	$r1,$h1,$h1
	addccc	$r2,$h2,$h2
	addc	$r3,$h3,$h3

	srl	$h0,8,$r0
	stb	$h0,[$mac+0]		! store little-endian result
	srl	$h0,16,$r1
	stb	$r0,[$mac+1]
	srl	$h0,24,$r2
	stb	$r1,[$mac+2]
	stb	$r2,[$mac+3]

	srl	$h1,8,$r0
	stb	$h1,[$mac+4]
	srl	$h1,16,$r1
	stb	$r0,[$mac+5]
	srl	$h1,24,$r2
	stb	$r1,[$mac+6]
	stb	$r2,[$mac+7]

	srl	$h2,8,$r0
	stb	$h2,[$mac+8]
	srl	$h2,16,$r1
	stb	$r0,[$mac+9]
	srl	$h2,24,$r2
	stb	$r1,[$mac+10]
	stb	$r2,[$mac+11]

	srl	$h3,8,$r0
	stb	$h3,[$mac+12]
	srl	$h3,16,$r1
	stb	$r0,[$mac+13]
	srl	$h3,24,$r2
	stb	$r1,[$mac+14]
	stb	$r2,[$mac+15]

	ret
	restore
.type	poly1305_emit,#function
.size	poly1305_emit,.-poly1305_emit
___

{
my ($ctx,$inp,$len,$padbit) = map("%i$_",(0..3));
my ($in0,$in1,$in2,$in3,$in4) = map("%o$_",(0..4));
my ($i1,$step,$shr,$shl) = map("%l$_",(0..7));
my $i2=$step;

my ($h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,
    $two0,$two32,$two64,$two96,$two130,$five_two130,
    $r0lo,$r0hi,$r1lo,$r1hi,$r2lo,$r2hi,
    $s2lo,$s2hi,$s3lo,$s3hi,
    $c0lo,$c0hi,$c1lo,$c1hi,$c2lo,$c2hi,$c3lo,$c3hi) = map("%f".2*$_,(0..31));
# borrowings
my ($r3lo,$r3hi,$s1lo,$s1hi) = ($c0lo,$c0hi,$c1lo,$c1hi);
my ($x0,$x1,$x2,$x3) = ($c2lo,$c2hi,$c3lo,$c3hi);
my ($y0,$y1,$y2,$y3) = ($c1lo,$c1hi,$c3hi,$c3lo);

$code.=<<___;
.align	32
poly1305_init_fma:
	save	%sp,-STACK_FRAME-16,%sp
	nop

.Lpoly1305_init_fma:
1:	call	.+8
	add	%o7,.Lconsts_fma-1b,%o7

	ldd	[%o7+8*0],$two0			! load constants
	ldd	[%o7+8*1],$two32
	ldd	[%o7+8*2],$two64
	ldd	[%o7+8*3],$two96
	ldd	[%o7+8*5],$five_two130

	std	$two0,[$ctx+8*0]		! initial hash value, biased 0
	std	$two32,[$ctx+8*1]
	std	$two64,[$ctx+8*2]
	std	$two96,[$ctx+8*3]

	brz,pn	$inp,.Lno_key_fma
	nop

	stx	%fsr,[%sp+LOCALS]		! save original %fsr
	ldx	[%o7+8*6],%fsr			! load new %fsr

	std	$two0,[$ctx+8*4] 		! key "template"
	std	$two32,[$ctx+8*5]
	std	$two64,[$ctx+8*6]
	std	$two96,[$ctx+8*7]

	and	$inp,7,$shr
	andn	$inp,7,$inp			! align pointer
	mov	8,$i1
	sll	$shr,3,$shr
	mov	16,$i2
	neg	$shr,$shl

	ldxa	[$inp+%g0]0x88,$in0		! load little-endian key
	ldxa	[$inp+$i1]0x88,$in2

	brz	$shr,.Lkey_aligned_fma
	sethi	%hi(0xf0000000),$i1		!   0xf0000000

	ldxa	[$inp+$i2]0x88,$in4

	srlx	$in0,$shr,$in0			! align data
	sllx	$in2,$shl,$in1
	srlx	$in2,$shr,$in2
	or	$in1,$in0,$in0
	sllx	$in4,$shl,$in3
	or	$in3,$in2,$in2

.Lkey_aligned_fma:
	or	$i1,3,$i2			!   0xf0000003
	srlx	$in0,32,$in1
	andn	$in0,$i1,$in0			! &=0x0fffffff
	andn	$in1,$i2,$in1			! &=0x0ffffffc
	srlx	$in2,32,$in3
	andn	$in2,$i2,$in2
	andn	$in3,$i2,$in3

	st	$in0,[$ctx+`8*4+4`]		! fill "template"
	st	$in1,[$ctx+`8*5+4`]
	st	$in2,[$ctx+`8*6+4`]
	st	$in3,[$ctx+`8*7+4`]

	ldd	[$ctx+8*4],$h0lo 		! load [biased] key
	ldd	[$ctx+8*5],$h1lo
	ldd	[$ctx+8*6],$h2lo
	ldd	[$ctx+8*7],$h3lo

	fsubd	$h0lo,$two0, $h0lo		! r0
	 ldd	[%o7+8*7],$two0 		! more constants
	fsubd	$h1lo,$two32,$h1lo		! r1
	 ldd	[%o7+8*8],$two32
	fsubd	$h2lo,$two64,$h2lo		! r2
	 ldd	[%o7+8*9],$two64
	fsubd	$h3lo,$two96,$h3lo		! r3
	 ldd	[%o7+8*10],$two96

	fmuld	$five_two130,$h1lo,$s1lo	! s1
	fmuld	$five_two130,$h2lo,$s2lo	! s2
	fmuld	$five_two130,$h3lo,$s3lo	! s3

	faddd	$h0lo,$two0, $h0hi
	faddd	$h1lo,$two32,$h1hi
	faddd	$h2lo,$two64,$h2hi
	faddd	$h3lo,$two96,$h3hi

	fsubd	$h0hi,$two0, $h0hi
	 ldd	[%o7+8*11],$two0		! more constants
	fsubd	$h1hi,$two32,$h1hi
	 ldd	[%o7+8*12],$two32
	fsubd	$h2hi,$two64,$h2hi
	 ldd	[%o7+8*13],$two64
	fsubd	$h3hi,$two96,$h3hi

	fsubd	$h0lo,$h0hi,$h0lo
	 std	$h0hi,[$ctx+8*5] 		! r0hi
	fsubd	$h1lo,$h1hi,$h1lo
	 std	$h1hi,[$ctx+8*7] 		! r1hi
	fsubd	$h2lo,$h2hi,$h2lo
	 std	$h2hi,[$ctx+8*9] 		! r2hi
	fsubd	$h3lo,$h3hi,$h3lo
	 std	$h3hi,[$ctx+8*11]		! r3hi

	faddd	$s1lo,$two0, $s1hi
	faddd	$s2lo,$two32,$s2hi
	faddd	$s3lo,$two64,$s3hi

	fsubd	$s1hi,$two0, $s1hi
	fsubd	$s2hi,$two32,$s2hi
	fsubd	$s3hi,$two64,$s3hi

	fsubd	$s1lo,$s1hi,$s1lo
	fsubd	$s2lo,$s2hi,$s2lo
	fsubd	$s3lo,$s3hi,$s3lo

	ldx	[%sp+LOCALS],%fsr		! restore %fsr

	std	$h0lo,[$ctx+8*4] 		! r0lo
	std	$h1lo,[$ctx+8*6] 		! r1lo
	std	$h2lo,[$ctx+8*8] 		! r2lo
	std	$h3lo,[$ctx+8*10]		! r3lo

	std	$s1hi,[$ctx+8*13]
	std	$s2hi,[$ctx+8*15]
	std	$s3hi,[$ctx+8*17]

	std	$s1lo,[$ctx+8*12]
	std	$s2lo,[$ctx+8*14]
	std	$s3lo,[$ctx+8*16]

	add	%o7,poly1305_blocks_fma-.Lconsts_fma,%o0
	add	%o7,poly1305_emit_fma-.Lconsts_fma,%o1
	STPTR	%o0,[%i2]
	STPTR	%o1,[%i2+SIZE_T]

	ret
	restore	%g0,1,%o0			! return 1

.Lno_key_fma:
	ret
	restore	%g0,%g0,%o0			! return 0
.type	poly1305_init_fma,#function
.size	poly1305_init_fma,.-poly1305_init_fma

.align	32
poly1305_blocks_fma:
	save	%sp,-STACK_FRAME-48,%sp
	srln	$len,4,$len

	brz,pn	$len,.Labort
	sub	$len,1,$len

1:	call	.+8
	add	%o7,.Lconsts_fma-1b,%o7

	ldd	[%o7+8*0],$two0			! load constants
	ldd	[%o7+8*1],$two32
	ldd	[%o7+8*2],$two64
	ldd	[%o7+8*3],$two96
	ldd	[%o7+8*4],$two130
	ldd	[%o7+8*5],$five_two130

	ldd	[$ctx+8*0],$h0lo 		! load [biased] hash value
	ldd	[$ctx+8*1],$h1lo
	ldd	[$ctx+8*2],$h2lo
	ldd	[$ctx+8*3],$h3lo

	std	$two0,[%sp+LOCALS+8*0]		! input "template"
	sethi	%hi((1023+52+96)<<20),$in3
	std	$two32,[%sp+LOCALS+8*1]
	or	$padbit,$in3,$in3
	std	$two64,[%sp+LOCALS+8*2]
	st	$in3,[%sp+LOCALS+8*3]

	and	$inp,7,$shr
	andn	$inp,7,$inp			! align pointer
	mov	8,$i1
	sll	$shr,3,$shr
	mov	16,$step
	neg	$shr,$shl

	ldxa	[$inp+%g0]0x88,$in0		! load little-endian input
	brz	$shr,.Linp_aligned_fma
	ldxa	[$inp+$i1]0x88,$in2

	ldxa	[$inp+$step]0x88,$in4
	add	$inp,8,$inp

	srlx	$in0,$shr,$in0			! align data
	sllx	$in2,$shl,$in1
	srlx	$in2,$shr,$in2
	or	$in1,$in0,$in0
	sllx	$in4,$shl,$in3
	srlx	$in4,$shr,$in4			! pre-shift
	or	$in3,$in2,$in2

.Linp_aligned_fma:
	srlx	$in0,32,$in1
	movrz	$len,0,$step
	srlx	$in2,32,$in3
	add	$step,$inp,$inp			! conditional advance

	st	$in0,[%sp+LOCALS+8*0+4]		! fill "template"
	st	$in1,[%sp+LOCALS+8*1+4]
	st	$in2,[%sp+LOCALS+8*2+4]
	st	$in3,[%sp+LOCALS+8*3+4]

	ldd	[$ctx+8*4],$r0lo 		! load key
	ldd	[$ctx+8*5],$r0hi
	ldd	[$ctx+8*6],$r1lo
	ldd	[$ctx+8*7],$r1hi
	ldd	[$ctx+8*8],$r2lo
	ldd	[$ctx+8*9],$r2hi
	ldd	[$ctx+8*10],$r3lo
	ldd	[$ctx+8*11],$r3hi
	ldd	[$ctx+8*12],$s1lo
	ldd	[$ctx+8*13],$s1hi
	ldd	[$ctx+8*14],$s2lo
	ldd	[$ctx+8*15],$s2hi
	ldd	[$ctx+8*16],$s3lo
	ldd	[$ctx+8*17],$s3hi

	stx	%fsr,[%sp+LOCALS+8*4]		! save original %fsr
	ldx	[%o7+8*6],%fsr			! load new %fsr

	subcc	$len,1,$len
	movrz	$len,0,$step

	ldd	[%sp+LOCALS+8*0],$x0		! load biased input
	ldd	[%sp+LOCALS+8*1],$x1
	ldd	[%sp+LOCALS+8*2],$x2
	ldd	[%sp+LOCALS+8*3],$x3

	fsubd	$h0lo,$two0, $h0lo		! de-bias hash value
	fsubd	$h1lo,$two32,$h1lo
	 ldxa	[$inp+%g0]0x88,$in0		! modulo-scheduled input load
	fsubd	$h2lo,$two64,$h2lo
	fsubd	$h3lo,$two96,$h3lo
	 ldxa	[$inp+$i1]0x88,$in2

	fsubd	$x0,$two0, $x0  		! de-bias input
	fsubd	$x1,$two32,$x1
	fsubd	$x2,$two64,$x2
	fsubd	$x3,$two96,$x3

	brz	$shr,.Linp_aligned_fma2
	add	$step,$inp,$inp			! conditional advance

	sllx	$in0,$shl,$in1			! align data
	srlx	$in0,$shr,$in3
	or	$in1,$in4,$in0
	sllx	$in2,$shl,$in1
	srlx	$in2,$shr,$in4			! pre-shift
	or	$in3,$in1,$in2
.Linp_aligned_fma2:
	srlx	$in0,32,$in1
	srlx	$in2,32,$in3

	faddd	$h0lo,$x0,$x0			! accumulate input
	 stw	$in0,[%sp+LOCALS+8*0+4]
	faddd	$h1lo,$x1,$x1
	 stw	$in1,[%sp+LOCALS+8*1+4]
	faddd	$h2lo,$x2,$x2
	 stw	$in2,[%sp+LOCALS+8*2+4]
	faddd	$h3lo,$x3,$x3
	 stw	$in3,[%sp+LOCALS+8*3+4]

	b	.Lentry_fma
	nop

.align	16
.Loop_fma:
	ldxa	[$inp+%g0]0x88,$in0		! modulo-scheduled input load
	ldxa	[$inp+$i1]0x88,$in2
	movrz	$len,0,$step

	faddd	$y0,$h0lo,$h0lo 		! accumulate input
	faddd	$y1,$h0hi,$h0hi
	faddd	$y2,$h2lo,$h2lo
	faddd	$y3,$h2hi,$h2hi

	brz,pn	$shr,.Linp_aligned_fma3
	add	$step,$inp,$inp			! conditional advance

	sllx	$in0,$shl,$in1			! align data
	srlx	$in0,$shr,$in3
	or	$in1,$in4,$in0
	sllx	$in2,$shl,$in1
	srlx	$in2,$shr,$in4			! pre-shift
	or	$in3,$in1,$in2

.Linp_aligned_fma3:
	!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! base 2^48 -> base 2^32
	faddd	$two64,$h1lo,$c1lo
	 srlx	$in0,32,$in1
	faddd	$two64,$h1hi,$c1hi
	 srlx	$in2,32,$in3
	faddd	$two130,$h3lo,$c3lo
	 st	$in0,[%sp+LOCALS+8*0+4]		! fill "template"
	faddd	$two130,$h3hi,$c3hi
	 st	$in1,[%sp+LOCALS+8*1+4]
	faddd	$two32,$h0lo,$c0lo
	 st	$in2,[%sp+LOCALS+8*2+4]
	faddd	$two32,$h0hi,$c0hi
	 st	$in3,[%sp+LOCALS+8*3+4]
	faddd	$two96,$h2lo,$c2lo
	faddd	$two96,$h2hi,$c2hi

	fsubd	$c1lo,$two64,$c1lo
	fsubd	$c1hi,$two64,$c1hi
	fsubd	$c3lo,$two130,$c3lo
	fsubd	$c3hi,$two130,$c3hi
	fsubd	$c0lo,$two32,$c0lo
	fsubd	$c0hi,$two32,$c0hi
	fsubd	$c2lo,$two96,$c2lo
	fsubd	$c2hi,$two96,$c2hi

	fsubd	$h1lo,$c1lo,$h1lo
	fsubd	$h1hi,$c1hi,$h1hi
	fsubd	$h3lo,$c3lo,$h3lo
	fsubd	$h3hi,$c3hi,$h3hi
	fsubd	$h2lo,$c2lo,$h2lo
	fsubd	$h2hi,$c2hi,$h2hi
	fsubd	$h0lo,$c0lo,$h0lo
	fsubd	$h0hi,$c0hi,$h0hi

	faddd	$h1lo,$c0lo,$h1lo
	faddd	$h1hi,$c0hi,$h1hi
	faddd	$h3lo,$c2lo,$h3lo
	faddd	$h3hi,$c2hi,$h3hi
	faddd	$h2lo,$c1lo,$h2lo
	faddd	$h2hi,$c1hi,$h2hi
	fmaddd	$five_two130,$c3lo,$h0lo,$h0lo
	fmaddd	$five_two130,$c3hi,$h0hi,$h0hi

	faddd	$h1lo,$h1hi,$x1
	 ldd	[$ctx+8*12],$s1lo		! reload constants
	faddd	$h3lo,$h3hi,$x3
	 ldd	[$ctx+8*13],$s1hi
	faddd	$h2lo,$h2hi,$x2
	 ldd	[$ctx+8*10],$r3lo
	faddd	$h0lo,$h0hi,$x0
	 ldd	[$ctx+8*11],$r3hi

.Lentry_fma:
	fmuld	$x1,$s3lo,$h0lo
	fmuld	$x1,$s3hi,$h0hi
	fmuld	$x1,$r1lo,$h2lo
	fmuld	$x1,$r1hi,$h2hi
	fmuld	$x1,$r0lo,$h1lo
	fmuld	$x1,$r0hi,$h1hi
	fmuld	$x1,$r2lo,$h3lo
	fmuld	$x1,$r2hi,$h3hi

	fmaddd	$x3,$s1lo,$h0lo,$h0lo
	fmaddd	$x3,$s1hi,$h0hi,$h0hi
	fmaddd	$x3,$s3lo,$h2lo,$h2lo
	fmaddd	$x3,$s3hi,$h2hi,$h2hi
	fmaddd	$x3,$s2lo,$h1lo,$h1lo
	fmaddd	$x3,$s2hi,$h1hi,$h1hi
	fmaddd	$x3,$r0lo,$h3lo,$h3lo
	fmaddd	$x3,$r0hi,$h3hi,$h3hi

	fmaddd	$x2,$s2lo,$h0lo,$h0lo
	fmaddd	$x2,$s2hi,$h0hi,$h0hi
	fmaddd	$x2,$r0lo,$h2lo,$h2lo
	fmaddd	$x2,$r0hi,$h2hi,$h2hi
	fmaddd	$x2,$s3lo,$h1lo,$h1lo
	 ldd	[%sp+LOCALS+8*0],$y0		! load [biased] input
	fmaddd	$x2,$s3hi,$h1hi,$h1hi
	 ldd	[%sp+LOCALS+8*1],$y1
	fmaddd	$x2,$r1lo,$h3lo,$h3lo
	 ldd	[%sp+LOCALS+8*2],$y2
	fmaddd	$x2,$r1hi,$h3hi,$h3hi
	 ldd	[%sp+LOCALS+8*3],$y3

	fmaddd	$x0,$r0lo,$h0lo,$h0lo
	 fsubd	$y0,$two0, $y0  		! de-bias input
	fmaddd	$x0,$r0hi,$h0hi,$h0hi
	 fsubd	$y1,$two32,$y1
	fmaddd	$x0,$r2lo,$h2lo,$h2lo
	 fsubd	$y2,$two64,$y2
	fmaddd	$x0,$r2hi,$h2hi,$h2hi
	 fsubd	$y3,$two96,$y3
	fmaddd	$x0,$r1lo,$h1lo,$h1lo
	fmaddd	$x0,$r1hi,$h1hi,$h1hi
	fmaddd	$x0,$r3lo,$h3lo,$h3lo
	fmaddd	$x0,$r3hi,$h3hi,$h3hi

	bcc	SIZE_T_CC,.Loop_fma
	subcc	$len,1,$len

	!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! base 2^48 -> base 2^32
	faddd	$h0lo,$two32,$c0lo
	faddd	$h0hi,$two32,$c0hi
	faddd	$h2lo,$two96,$c2lo
	faddd	$h2hi,$two96,$c2hi
	faddd	$h1lo,$two64,$c1lo
	faddd	$h1hi,$two64,$c1hi
	faddd	$h3lo,$two130,$c3lo
	faddd	$h3hi,$two130,$c3hi

	fsubd	$c0lo,$two32,$c0lo
	fsubd	$c0hi,$two32,$c0hi
	fsubd	$c2lo,$two96,$c2lo
	fsubd	$c2hi,$two96,$c2hi
	fsubd	$c1lo,$two64,$c1lo
	fsubd	$c1hi,$two64,$c1hi
	fsubd	$c3lo,$two130,$c3lo
	fsubd	$c3hi,$two130,$c3hi

	fsubd	$h1lo,$c1lo,$h1lo
	fsubd	$h1hi,$c1hi,$h1hi
	fsubd	$h3lo,$c3lo,$h3lo
	fsubd	$h3hi,$c3hi,$h3hi
	fsubd	$h2lo,$c2lo,$h2lo
	fsubd	$h2hi,$c2hi,$h2hi
	fsubd	$h0lo,$c0lo,$h0lo
	fsubd	$h0hi,$c0hi,$h0hi

	faddd	$h1lo,$c0lo,$h1lo
	faddd	$h1hi,$c0hi,$h1hi
	faddd	$h3lo,$c2lo,$h3lo
	faddd	$h3hi,$c2hi,$h3hi
	faddd	$h2lo,$c1lo,$h2lo
	faddd	$h2hi,$c1hi,$h2hi
	fmaddd	$five_two130,$c3lo,$h0lo,$h0lo
	fmaddd	$five_two130,$c3hi,$h0hi,$h0hi

	faddd	$h1lo,$h1hi,$x1
	faddd	$h3lo,$h3hi,$x3
	faddd	$h2lo,$h2hi,$x2
	faddd	$h0lo,$h0hi,$x0

	faddd	$x1,$two32,$x1  		! bias
	faddd	$x3,$two96,$x3
	faddd	$x2,$two64,$x2
	faddd	$x0,$two0, $x0

	ldx	[%sp+LOCALS+8*4],%fsr		! restore saved %fsr

	std	$x1,[$ctx+8*1]			! store [biased] hash value
	std	$x3,[$ctx+8*3]
	std	$x2,[$ctx+8*2]
	std	$x0,[$ctx+8*0]

.Labort:
	ret
	restore
.type	poly1305_blocks_fma,#function
.size	poly1305_blocks_fma,.-poly1305_blocks_fma
___
{
my ($mac,$nonce)=($inp,$len);

my ($h0,$h1,$h2,$h3,$h4, $d0,$d1,$d2,$d3, $mask
   ) = (map("%l$_",(0..5)),map("%o$_",(0..4)));

$code.=<<___;
.align	32
poly1305_emit_fma:
	save	%sp,-STACK_FRAME,%sp

	ld	[$ctx+8*0+0],$d0		! load hash
	ld	[$ctx+8*0+4],$h0
	ld	[$ctx+8*1+0],$d1
	ld	[$ctx+8*1+4],$h1
	ld	[$ctx+8*2+0],$d2
	ld	[$ctx+8*2+4],$h2
	ld	[$ctx+8*3+0],$d3
	ld	[$ctx+8*3+4],$h3

	sethi	%hi(0xfff00000),$mask
	andn	$d0,$mask,$d0			! mask exponent
	andn	$d1,$mask,$d1
	andn	$d2,$mask,$d2
	andn	$d3,$mask,$d3			! can be partially reduced...
	mov	3,$mask

	srl	$d3,2,$padbit			! ... so reduce
	and	$d3,$mask,$h4
	andn	$d3,$mask,$d3
	add	$padbit,$d3,$d3

	addcc	$d3,$h0,$h0
	addccc	$d0,$h1,$h1
	addccc	$d1,$h2,$h2
	addccc	$d2,$h3,$h3
	addc	%g0,$h4,$h4

	addcc	$h0,5,$d0			! compare to modulus
	addccc	$h1,0,$d1
	addccc	$h2,0,$d2
	addccc	$h3,0,$d3
	addc	$h4,0,$mask

	srl	$mask,2,$mask			! did it carry/borrow?
	neg	$mask,$mask
	sra	$mask,31,$mask			! mask

	andn	$h0,$mask,$h0
	and	$d0,$mask,$d0
	andn	$h1,$mask,$h1
	and	$d1,$mask,$d1
	or	$d0,$h0,$h0
	ld	[$nonce+0],$d0			! load nonce
	andn	$h2,$mask,$h2
	and	$d2,$mask,$d2
	or	$d1,$h1,$h1
	ld	[$nonce+4],$d1
	andn	$h3,$mask,$h3
	and	$d3,$mask,$d3
	or	$d2,$h2,$h2
	ld	[$nonce+8],$d2
	or	$d3,$h3,$h3
	ld	[$nonce+12],$d3

	addcc	$d0,$h0,$h0			! accumulate nonce
	addccc	$d1,$h1,$h1
	addccc	$d2,$h2,$h2
	addc	$d3,$h3,$h3

	stb	$h0,[$mac+0]			! write little-endian result
	srl	$h0,8,$h0
	stb	$h1,[$mac+4]
	srl	$h1,8,$h1
	stb	$h2,[$mac+8]
	srl	$h2,8,$h2
	stb	$h3,[$mac+12]
	srl	$h3,8,$h3

	stb	$h0,[$mac+1]
	srl	$h0,8,$h0
	stb	$h1,[$mac+5]
	srl	$h1,8,$h1
	stb	$h2,[$mac+9]
	srl	$h2,8,$h2
	stb	$h3,[$mac+13]
	srl	$h3,8,$h3

	stb	$h0,[$mac+2]
	srl	$h0,8,$h0
	stb	$h1,[$mac+6]
	srl	$h1,8,$h1
	stb	$h2,[$mac+10]
	srl	$h2,8,$h2
	stb	$h3,[$mac+14]
	srl	$h3,8,$h3

	stb	$h0,[$mac+3]
	stb	$h1,[$mac+7]
	stb	$h2,[$mac+11]
	stb	$h3,[$mac+15]

	ret
	restore
.type	poly1305_emit_fma,#function
.size	poly1305_emit_fma,.-poly1305_emit_fma
___
}

$code.=<<___;
.align	64
.Lconsts_fma:
.word	0x43300000,0x00000000		! 2^(52+0)
.word	0x45300000,0x00000000		! 2^(52+32)
.word	0x47300000,0x00000000		! 2^(52+64)
.word	0x49300000,0x00000000		! 2^(52+96)
.word	0x4b500000,0x00000000		! 2^(52+130)

.word	0x37f40000,0x00000000		! 5/2^130
.word	0,1<<30				! fsr: truncate, no exceptions

.word	0x44300000,0x00000000		! 2^(52+16+0)
.word	0x46300000,0x00000000		! 2^(52+16+32)
.word	0x48300000,0x00000000		! 2^(52+16+64)
.word	0x4a300000,0x00000000		! 2^(52+16+96)
.word	0x3e300000,0x00000000		! 2^(52+16+0-96)
.word	0x40300000,0x00000000		! 2^(52+16+32-96)
.word	0x42300000,0x00000000		! 2^(52+16+64-96)
.asciz	"Poly1305 for SPARCv9/VIS3/FMA, CRYPTOGAMS by <appro\@openssl.org>"
.align	4
___
}

# Purpose of these subroutines is to explicitly encode VIS instructions,
# so that one can compile the module without having to specify VIS
# extensions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
# Idea is to reserve for option to produce "universal" binary and let
# programmer detect if current CPU is VIS capable at run-time.
sub unvis3 {
my ($mnemonic,$rs1,$rs2,$rd)=@_;
my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
my ($ref,$opf);
my %visopf = (	"addxc"		=> 0x011,
		"addxccc"	=> 0x013,
		"umulxhi"	=> 0x016	);

    $ref = "$mnemonic\t$rs1,$rs2,$rd";

    if ($opf=$visopf{$mnemonic}) {
	foreach ($rs1,$rs2,$rd) {
	    return $ref if (!/%([goli])([0-9])/);
	    $_=$bias{$1}+$2;
	}

	return	sprintf ".word\t0x%08x !%s",
			0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
			$ref;
    } else {
	return $ref;
    }
}

sub unfma {
my ($mnemonic,$rs1,$rs2,$rs3,$rd)=@_;
my ($ref,$opf);
my %fmaopf = (	"fmadds"	=> 0x1,
		"fmaddd"	=> 0x2,
		"fmsubs"	=> 0x5,
		"fmsubd"	=> 0x6		);

    $ref = "$mnemonic\t$rs1,$rs2,$rs3,$rd";

    if ($opf=$fmaopf{$mnemonic}) {
	foreach ($rs1,$rs2,$rs3,$rd) {
	    return $ref if (!/%f([0-9]{1,2})/);
	    $_=$1;
	    if ($1>=32) {
		return $ref if ($1&1);
		# re-encode for upper double register addressing
		$_=($1|$1>>5)&31;
	    }
	}

	return	sprintf ".word\t0x%08x !%s",
			0x81b80000|$rd<<25|$rs1<<14|$rs3<<9|$opf<<5|$rs2,
			$ref;
    } else {
	return $ref;
    }
}

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/ge;

	s/\b(umulxhi|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/
		&unvis3($1,$2,$3,$4)
	 /ge	or
	s/\b(fmadd[sd])\s+(%f[0-9]+),\s*(%f[0-9]+),\s*(%f[0-9]+),\s*(%f[0-9]+)/
		&unfma($1,$2,$3,$4,$5)
	 /ge;

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";