#! /usr/bin/env perl
# Copyright 2018-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# X25519 lower-level primitives for PPC64.
#
# July 2018.
#
# Base 2^64 is faster than base 2^51 on pre-POWER8, most notably ~15%
# faster on PPC970/G5. POWER8 on the other hand seems to trip on own
# shoelaces when handling longer carry chains. As base 2^51 has just
# single-carry pairs, it's 25% faster than base 2^64. Since PPC970 is
# pretty old, base 2^64 implementation is not engaged. Comparison to
# compiler-generated code is complicated by the fact that not all
# compilers support 128-bit integers. When compiler doesn't, like xlc,
# this module delivers more than 2x improvement, and when it does,
# from 12% to 30% improvement was measured...

$flavour = shift;
while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;

my $sp = "r1";
my ($rp,$ap,$bp) = map("r$_",3..5);

####################################################### base 2^64
if (0) {
my ($bi,$a0,$a1,$a2,$a3,$t0,$t1, $t2,$t3,
    $acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7) =
    map("r$_",(6..12,22..31));
my $zero = "r0";
my $FRAME = 16*8;

$code.=<<___;
.text

.globl	x25519_fe64_mul
.type	x25519_fe64_mul,\@function
.align	5
x25519_fe64_mul:
	stdu	$sp,-$FRAME($sp)
	std	r22,`$FRAME-8*10`($sp)
	std	r23,`$FRAME-8*9`($sp)
	std	r24,`$FRAME-8*8`($sp)
	std	r25,`$FRAME-8*7`($sp)
	std	r26,`$FRAME-8*6`($sp)
	std	r27,`$FRAME-8*5`($sp)
	std	r28,`$FRAME-8*4`($sp)
	std	r29,`$FRAME-8*3`($sp)
	std	r30,`$FRAME-8*2`($sp)
	std	r31,`$FRAME-8*1`($sp)

	ld	$bi,0($bp)
	ld	$a0,0($ap)
	xor	$zero,$zero,$zero
	ld	$a1,8($ap)
	ld	$a2,16($ap)
	ld	$a3,24($ap)

	mulld	$acc0,$a0,$bi		# a[0]*b[0]
	mulhdu	$t0,$a0,$bi
	mulld	$acc1,$a1,$bi		# a[1]*b[0]
	mulhdu	$t1,$a1,$bi
	mulld	$acc2,$a2,$bi		# a[2]*b[0]
	mulhdu	$t2,$a2,$bi
	mulld	$acc3,$a3,$bi		# a[3]*b[0]
	mulhdu	$t3,$a3,$bi
___
for(my @acc=($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7),
    my $i=1; $i<4; shift(@acc), $i++) {
my $acc4 = $i==1? $zero : @acc[4];

$code.=<<___;
	ld	$bi,`8*$i`($bp)
	addc	@acc[1],@acc[1],$t0	# accumulate high parts
	mulld	$t0,$a0,$bi
	adde	@acc[2],@acc[2],$t1
	mulld	$t1,$a1,$bi
	adde	@acc[3],@acc[3],$t2
	mulld	$t2,$a2,$bi
	adde	@acc[4],$acc4,$t3
	mulld	$t3,$a3,$bi
	addc	@acc[1],@acc[1],$t0	# accumulate low parts
	mulhdu	$t0,$a0,$bi
	adde	@acc[2],@acc[2],$t1
	mulhdu	$t1,$a1,$bi
	adde	@acc[3],@acc[3],$t2
	mulhdu	$t2,$a2,$bi
	adde	@acc[4],@acc[4],$t3
	mulhdu	$t3,$a3,$bi
	adde	@acc[5],$zero,$zero
___
}
$code.=<<___;
	li	$bi,38
	addc	$acc4,$acc4,$t0
	mulld	$t0,$acc4,$bi
	adde	$acc5,$acc5,$t1
	mulld	$t1,$acc5,$bi
	adde	$acc6,$acc6,$t2
	mulld	$t2,$acc6,$bi
	adde	$acc7,$acc7,$t3
	mulld	$t3,$acc7,$bi

	addc	$acc0,$acc0,$t0
	mulhdu	$t0,$acc4,$bi
	adde	$acc1,$acc1,$t1
	mulhdu	$t1,$acc5,$bi
	adde	$acc2,$acc2,$t2
	mulhdu	$t2,$acc6,$bi
	adde	$acc3,$acc3,$t3
	mulhdu	$t3,$acc7,$bi
	adde	$acc4,$zero,$zero

	addc	$acc1,$acc1,$t0
	adde	$acc2,$acc2,$t1
	adde	$acc3,$acc3,$t2
	adde	$acc4,$acc4,$t3

	mulld	$acc4,$acc4,$bi

	addc	$acc0,$acc0,$acc4
	addze	$acc1,$acc1
	addze	$acc2,$acc2
	addze	$acc3,$acc3

	subfe	$acc4,$acc4,$acc4	# carry -> ~mask
	std	$acc1,8($rp)
	andc	$acc4,$bi,$acc4
	std	$acc2,16($rp)
	add	$acc0,$acc0,$acc4
	std	$acc3,24($rp)
	std	$acc0,0($rp)

	ld	r22,`$FRAME-8*10`($sp)
	ld	r23,`$FRAME-8*9`($sp)
	ld	r24,`$FRAME-8*8`($sp)
	ld	r25,`$FRAME-8*7`($sp)
	ld	r26,`$FRAME-8*6`($sp)
	ld	r27,`$FRAME-8*5`($sp)
	ld	r28,`$FRAME-8*4`($sp)
	ld	r29,`$FRAME-8*3`($sp)
	ld	r30,`$FRAME-8*2`($sp)
	ld	r31,`$FRAME-8*1`($sp)
	addi	$sp,$sp,$FRAME
	blr
	.long	0
	.byte	0,12,4,0,0x80,10,3,0
	.long	0
.size	x25519_fe64_mul,.-x25519_fe64_mul

.globl	x25519_fe64_sqr
.type	x25519_fe64_sqr,\@function
.align	5
x25519_fe64_sqr:
	stdu	$sp,-$FRAME($sp)
	std	r22,`$FRAME-8*10`($sp)
	std	r23,`$FRAME-8*9`($sp)
	std	r24,`$FRAME-8*8`($sp)
	std	r25,`$FRAME-8*7`($sp)
	std	r26,`$FRAME-8*6`($sp)
	std	r27,`$FRAME-8*5`($sp)
	std	r28,`$FRAME-8*4`($sp)
	std	r29,`$FRAME-8*3`($sp)
	std	r30,`$FRAME-8*2`($sp)
	std	r31,`$FRAME-8*1`($sp)

	ld	$a0,0($ap)
	xor	$zero,$zero,$zero
	ld	$a1,8($ap)
	ld	$a2,16($ap)
	ld	$a3,24($ap)

	################################
	#  |  |  |  |  |  |a1*a0|  |
	#  |  |  |  |  |a2*a0|  |  |
	#  |  |a3*a2|a3*a0|  |  |  |
	#  |  |  |  |a2*a1|  |  |  |
	#  |  |  |a3*a1|  |  |  |  |
	# *|  |  |  |  |  |  |  | 2|
	# +|a3*a3|a2*a2|a1*a1|a0*a0|
	#  |--+--+--+--+--+--+--+--|
	#  |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
	#
	#  "can't overflow" below mark carrying into high part of
	#  multiplication result, which can't overflow, because it
	#  can never be all ones.

	mulld	$acc1,$a1,$a0		# a[1]*a[0]
	mulhdu	$t1,$a1,$a0
	mulld	$acc2,$a2,$a0		# a[2]*a[0]
	mulhdu	$t2,$a2,$a0
	mulld	$acc3,$a3,$a0		# a[3]*a[0]
	mulhdu	$acc4,$a3,$a0

	addc	$acc2,$acc2,$t1		# accumulate high parts of multiplication
	 mulld	$t0,$a2,$a1		# a[2]*a[1]
	 mulhdu	$t1,$a2,$a1
	adde	$acc3,$acc3,$t2
	 mulld	$t2,$a3,$a1		# a[3]*a[1]
	 mulhdu	$t3,$a3,$a1
	addze	$acc4,$acc4		# can't overflow

	mulld	$acc5,$a3,$a2		# a[3]*a[2]
	mulhdu	$acc6,$a3,$a2

	addc	$t1,$t1,$t2		# accumulate high parts of multiplication
	 mulld	$acc0,$a0,$a0		# a[0]*a[0]
	addze	$t2,$t3			# can't overflow

	addc	$acc3,$acc3,$t0		# accumulate low parts of multiplication
	 mulhdu	$a0,$a0,$a0
	adde	$acc4,$acc4,$t1
	 mulld	$t1,$a1,$a1		# a[1]*a[1]
	adde	$acc5,$acc5,$t2
	 mulhdu	$a1,$a1,$a1
	addze	$acc6,$acc6		# can't overflow

	addc	$acc1,$acc1,$acc1	# acc[1-6]*=2
	 mulld	$t2,$a2,$a2		# a[2]*a[2]
	adde	$acc2,$acc2,$acc2
	 mulhdu	$a2,$a2,$a2
	adde	$acc3,$acc3,$acc3
	 mulld	$t3,$a3,$a3		# a[3]*a[3]
	adde	$acc4,$acc4,$acc4
	 mulhdu	$a3,$a3,$a3
	adde	$acc5,$acc5,$acc5
	adde	$acc6,$acc6,$acc6
	addze	$acc7,$zero

	addc	$acc1,$acc1,$a0		# +a[i]*a[i]
	 li	$bi,38
	adde	$acc2,$acc2,$t1
	adde	$acc3,$acc3,$a1
	adde	$acc4,$acc4,$t2
	adde	$acc5,$acc5,$a2
	adde	$acc6,$acc6,$t3
	adde	$acc7,$acc7,$a3

	mulld	$t0,$acc4,$bi
	mulld	$t1,$acc5,$bi
	mulld	$t2,$acc6,$bi
	mulld	$t3,$acc7,$bi

	addc	$acc0,$acc0,$t0
	mulhdu	$t0,$acc4,$bi
	adde	$acc1,$acc1,$t1
	mulhdu	$t1,$acc5,$bi
	adde	$acc2,$acc2,$t2
	mulhdu	$t2,$acc6,$bi
	adde	$acc3,$acc3,$t3
	mulhdu	$t3,$acc7,$bi
	addze	$acc4,$zero

	addc	$acc1,$acc1,$t0
	adde	$acc2,$acc2,$t1
	adde	$acc3,$acc3,$t2
	adde	$acc4,$acc4,$t3

	mulld	$acc4,$acc4,$bi

	addc	$acc0,$acc0,$acc4
	addze	$acc1,$acc1
	addze	$acc2,$acc2
	addze	$acc3,$acc3

	subfe	$acc4,$acc4,$acc4	# carry -> ~mask
	std	$acc1,8($rp)
	andc	$acc4,$bi,$acc4
	std	$acc2,16($rp)
	add	$acc0,$acc0,$acc4
	std	$acc3,24($rp)
	std	$acc0,0($rp)

	ld	r22,`$FRAME-8*10`($sp)
	ld	r23,`$FRAME-8*9`($sp)
	ld	r24,`$FRAME-8*8`($sp)
	ld	r25,`$FRAME-8*7`($sp)
	ld	r26,`$FRAME-8*6`($sp)
	ld	r27,`$FRAME-8*5`($sp)
	ld	r28,`$FRAME-8*4`($sp)
	ld	r29,`$FRAME-8*3`($sp)
	ld	r30,`$FRAME-8*2`($sp)
	ld	r31,`$FRAME-8*1`($sp)
	addi	$sp,$sp,$FRAME
	blr
	.long	0
	.byte	0,12,4,0,0x80,10,2,0
	.long	0
.size	x25519_fe64_sqr,.-x25519_fe64_sqr

.globl	x25519_fe64_mul121666
.type	x25519_fe64_mul121666,\@function
.align	5
x25519_fe64_mul121666:
	lis	$bi,`65536>>16`
	ori	$bi,$bi,`121666-65536`

	ld	$t0,0($ap)
	ld	$t1,8($ap)
	ld	$bp,16($ap)
	ld	$ap,24($ap)

	mulld	$a0,$t0,$bi
	mulhdu	$t0,$t0,$bi
	mulld	$a1,$t1,$bi
	mulhdu	$t1,$t1,$bi
	mulld	$a2,$bp,$bi
	mulhdu	$bp,$bp,$bi
	mulld	$a3,$ap,$bi
	mulhdu	$ap,$ap,$bi

	addc	$a1,$a1,$t0
	adde	$a2,$a2,$t1
	adde	$a3,$a3,$bp
	addze	$ap,    $ap

	mulli	$ap,$ap,38

	addc	$a0,$a0,$ap
	addze	$a1,$a1
	addze	$a2,$a2
	addze	$a3,$a3

	subfe	$t1,$t1,$t1		# carry -> ~mask
	std	$a1,8($rp)
	andc	$t0,$t0,$t1
	std	$a2,16($rp)
	add	$a0,$a0,$t0
	std	$a3,24($rp)
	std	$a0,0($rp)

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,2,0
	.long	0
.size	x25519_fe64_mul121666,.-x25519_fe64_mul121666

.globl	x25519_fe64_add
.type	x25519_fe64_add,\@function
.align	5
x25519_fe64_add:
	ld	$a0,0($ap)
	ld	$t0,0($bp)
	ld	$a1,8($ap)
	ld	$t1,8($bp)
	ld	$a2,16($ap)
	ld	$bi,16($bp)
	ld	$a3,24($ap)
	ld	$bp,24($bp)

	addc	$a0,$a0,$t0
	adde	$a1,$a1,$t1
	adde	$a2,$a2,$bi
	adde	$a3,$a3,$bp

	li	$t0,38
	subfe	$t1,$t1,$t1		# carry -> ~mask
	andc	$t1,$t0,$t1

	addc	$a0,$a0,$t1
	addze	$a1,$a1
	addze	$a2,$a2
	addze	$a3,$a3

	subfe	$t1,$t1,$t1		# carry -> ~mask
	std	$a1,8($rp)
	andc	$t0,$t0,$t1
	std	$a2,16($rp)
	add	$a0,$a0,$t0
	std	$a3,24($rp)
	std	$a0,0($rp)

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,3,0
	.long	0
.size	x25519_fe64_add,.-x25519_fe64_add

.globl	x25519_fe64_sub
.type	x25519_fe64_sub,\@function
.align	5
x25519_fe64_sub:
	ld	$a0,0($ap)
	ld	$t0,0($bp)
	ld	$a1,8($ap)
	ld	$t1,8($bp)
	ld	$a2,16($ap)
	ld	$bi,16($bp)
	ld	$a3,24($ap)
	ld	$bp,24($bp)

	subfc	$a0,$t0,$a0
	subfe	$a1,$t1,$a1
	subfe	$a2,$bi,$a2
	subfe	$a3,$bp,$a3

	li	$t0,38
	subfe	$t1,$t1,$t1		# borrow -> mask
	xor	$zero,$zero,$zero
	and	$t1,$t0,$t1

	subfc	$a0,$t1,$a0
	subfe	$a1,$zero,$a1
	subfe	$a2,$zero,$a2
	subfe	$a3,$zero,$a3

	subfe	$t1,$t1,$t1		# borrow -> mask
	std	$a1,8($rp)
	and	$t0,$t0,$t1
	std	$a2,16($rp)
	subf	$a0,$t0,$a0
	std	$a3,24($rp)
	std	$a0,0($rp)

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,3,0
	.long	0
.size	x25519_fe64_sub,.-x25519_fe64_sub

.globl	x25519_fe64_tobytes
.type	x25519_fe64_tobytes,\@function
.align	5
x25519_fe64_tobytes:
	ld	$a3,24($ap)
	ld	$a0,0($ap)
	ld	$a1,8($ap)
	ld	$a2,16($ap)

	sradi	$t0,$a3,63		# most significant bit -> mask
	li	$t1,19
	and	$t0,$t0,$t1
	sldi	$a3,$a3,1
	add	$t0,$t0,$t1		# compare to modulus in the same go
	srdi	$a3,$a3,1		# most significant bit cleared

	addc	$a0,$a0,$t0
	addze	$a1,$a1
	addze	$a2,$a2
	addze	$a3,$a3

	xor	$zero,$zero,$zero
	sradi	$t0,$a3,63		# most significant bit -> mask
	sldi	$a3,$a3,1
	andc	$t0,$t1,$t0
	srdi	$a3,$a3,1		# most significant bit cleared

	subi	$rp,$rp,1
	subfc	$a0,$t0,$a0
	subfe	$a1,$zero,$a1
	subfe	$a2,$zero,$a2
	subfe	$a3,$zero,$a3

___
for (my @a=($a0,$a1,$a2,$a3), my $i=0; $i<4; shift(@a), $i++) {
$code.=<<___;
	srdi	$t0,@a[0],8
	stbu	@a[0],1($rp)
	srdi	@a[0],@a[0],16
	stbu	$t0,1($rp)
	srdi	$t0,@a[0],8
	stbu	@a[0],1($rp)
	srdi	@a[0],@a[0],16
	stbu	$t0,1($rp)
	srdi	$t0,@a[0],8
	stbu	@a[0],1($rp)
	srdi	@a[0],@a[0],16
	stbu	$t0,1($rp)
	srdi	$t0,@a[0],8
	stbu	@a[0],1($rp)
	stbu	$t0,1($rp)
___
}
$code.=<<___;
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,2,0
	.long	0
.size	x25519_fe64_tobytes,.-x25519_fe64_tobytes
___
}
####################################################### base 2^51
{
my ($bi,$a0,$a1,$a2,$a3,$a4,$t0, $t1,
    $h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,$h4lo,$h4hi) =
    map("r$_",(6..12,21..31));
my $mask = "r0";
my $FRAME = 18*8;

$code.=<<___;
.text

.globl	x25519_fe51_mul
.type	x25519_fe51_mul,\@function
.align	5
x25519_fe51_mul:
	stdu	$sp,-$FRAME($sp)
	std	r21,`$FRAME-8*11`($sp)
	std	r22,`$FRAME-8*10`($sp)
	std	r23,`$FRAME-8*9`($sp)
	std	r24,`$FRAME-8*8`($sp)
	std	r25,`$FRAME-8*7`($sp)
	std	r26,`$FRAME-8*6`($sp)
	std	r27,`$FRAME-8*5`($sp)
	std	r28,`$FRAME-8*4`($sp)
	std	r29,`$FRAME-8*3`($sp)
	std	r30,`$FRAME-8*2`($sp)
	std	r31,`$FRAME-8*1`($sp)

	ld	$bi,0($bp)
	ld	$a0,0($ap)
	ld	$a1,8($ap)
	ld	$a2,16($ap)
	ld	$a3,24($ap)
	ld	$a4,32($ap)

	mulld	$h0lo,$a0,$bi		# a[0]*b[0]
	mulhdu	$h0hi,$a0,$bi

	mulld	$h1lo,$a1,$bi		# a[1]*b[0]
	mulhdu	$h1hi,$a1,$bi

	 mulld	$h4lo,$a4,$bi		# a[4]*b[0]
	 mulhdu	$h4hi,$a4,$bi
	 ld	$ap,8($bp)
	 mulli	$a4,$a4,19

	mulld	$h2lo,$a2,$bi		# a[2]*b[0]
	mulhdu	$h2hi,$a2,$bi

	mulld	$h3lo,$a3,$bi		# a[3]*b[0]
	mulhdu	$h3hi,$a3,$bi
___
for(my @a=($a0,$a1,$a2,$a3,$a4),
    my $i=1; $i<4; $i++) {
	($ap,$bi) = ($bi,$ap);
$code.=<<___;
	mulld	$t0,@a[4],$bi
	mulhdu	$t1,@a[4],$bi
	addc	$h0lo,$h0lo,$t0
	adde	$h0hi,$h0hi,$t1

	mulld	$t0,@a[0],$bi
	mulhdu	$t1,@a[0],$bi
	addc	$h1lo,$h1lo,$t0
	adde	$h1hi,$h1hi,$t1

	 mulld	$t0,@a[3],$bi
	 mulhdu	$t1,@a[3],$bi
	 ld	$ap,`8*($i+1)`($bp)
	 mulli	@a[3],@a[3],19
	 addc	$h4lo,$h4lo,$t0
	 adde	$h4hi,$h4hi,$t1

	mulld	$t0,@a[1],$bi
	mulhdu	$t1,@a[1],$bi
	addc	$h2lo,$h2lo,$t0
	adde	$h2hi,$h2hi,$t1

	mulld	$t0,@a[2],$bi
	mulhdu	$t1,@a[2],$bi
	addc	$h3lo,$h3lo,$t0
	adde	$h3hi,$h3hi,$t1
___
	unshift(@a,pop(@a));
}
	($ap,$bi) = ($bi,$ap);
$code.=<<___;
	mulld	$t0,$a1,$bi
	mulhdu	$t1,$a1,$bi
	addc	$h0lo,$h0lo,$t0
	adde	$h0hi,$h0hi,$t1

	mulld	$t0,$a2,$bi
	mulhdu	$t1,$a2,$bi
	addc	$h1lo,$h1lo,$t0
	adde	$h1hi,$h1hi,$t1

	mulld	$t0,$a3,$bi
	mulhdu	$t1,$a3,$bi
	addc	$h2lo,$h2lo,$t0
	adde	$h2hi,$h2hi,$t1

	mulld	$t0,$a4,$bi
	mulhdu	$t1,$a4,$bi
	addc	$h3lo,$h3lo,$t0
	adde	$h3hi,$h3hi,$t1

	mulld	$t0,$a0,$bi
	mulhdu	$t1,$a0,$bi
	addc	$h4lo,$h4lo,$t0
	adde	$h4hi,$h4hi,$t1

.Lfe51_reduce:
	li	$mask,-1
	srdi	$mask,$mask,13		# 0x7ffffffffffff

	srdi	$t0,$h2lo,51
	and	$a2,$h2lo,$mask
	insrdi	$t0,$h2hi,51,0		# h2>>51
	 srdi	$t1,$h0lo,51
	 and	$a0,$h0lo,$mask
	 insrdi	$t1,$h0hi,51,0		# h0>>51
	addc	$h3lo,$h3lo,$t0
	addze	$h3hi,$h3hi
	 addc	$h1lo,$h1lo,$t1
	 addze	$h1hi,$h1hi

	srdi	$t0,$h3lo,51
	and	$a3,$h3lo,$mask
	insrdi	$t0,$h3hi,51,0		# h3>>51
	 srdi	$t1,$h1lo,51
	 and	$a1,$h1lo,$mask
	 insrdi	$t1,$h1hi,51,0		# h1>>51
	addc	$h4lo,$h4lo,$t0
	addze	$h4hi,$h4hi
	 add	$a2,$a2,$t1

	srdi	$t0,$h4lo,51
	and	$a4,$h4lo,$mask
	insrdi	$t0,$h4hi,51,0
	mulli	$t0,$t0,19		# (h4 >> 51) * 19

	add	$a0,$a0,$t0

	srdi	$t1,$a2,51
	and	$a2,$a2,$mask
	add	$a3,$a3,$t1

	srdi	$t0,$a0,51
	and	$a0,$a0,$mask
	add	$a1,$a1,$t0

	std	$a2,16($rp)
	std	$a3,24($rp)
	std	$a4,32($rp)
	std	$a0,0($rp)
	std	$a1,8($rp)

	ld	r21,`$FRAME-8*11`($sp)
	ld	r22,`$FRAME-8*10`($sp)
	ld	r23,`$FRAME-8*9`($sp)
	ld	r24,`$FRAME-8*8`($sp)
	ld	r25,`$FRAME-8*7`($sp)
	ld	r26,`$FRAME-8*6`($sp)
	ld	r27,`$FRAME-8*5`($sp)
	ld	r28,`$FRAME-8*4`($sp)
	ld	r29,`$FRAME-8*3`($sp)
	ld	r30,`$FRAME-8*2`($sp)
	ld	r31,`$FRAME-8*1`($sp)
	addi	$sp,$sp,$FRAME
	blr
	.long	0
	.byte	0,12,4,0,0x80,11,3,0
	.long	0
.size	x25519_fe51_mul,.-x25519_fe51_mul
___
{
my ($a0,$a1,$a2,$a3,$a4,$t0,$t1) = ($a0,$a1,$a2,$a3,$a4,$t0,$t1);
$code.=<<___;
.globl	x25519_fe51_sqr
.type	x25519_fe51_sqr,\@function
.align	5
x25519_fe51_sqr:
	stdu	$sp,-$FRAME($sp)
	std	r21,`$FRAME-8*11`($sp)
	std	r22,`$FRAME-8*10`($sp)
	std	r23,`$FRAME-8*9`($sp)
	std	r24,`$FRAME-8*8`($sp)
	std	r25,`$FRAME-8*7`($sp)
	std	r26,`$FRAME-8*6`($sp)
	std	r27,`$FRAME-8*5`($sp)
	std	r28,`$FRAME-8*4`($sp)
	std	r29,`$FRAME-8*3`($sp)
	std	r30,`$FRAME-8*2`($sp)
	std	r31,`$FRAME-8*1`($sp)

	ld	$a0,0($ap)
	ld	$a1,8($ap)
	ld	$a2,16($ap)
	ld	$a3,24($ap)
	ld	$a4,32($ap)

	add	$bi,$a0,$a0		# a[0]*2
	mulli	$t1,$a4,19		# a[4]*19

	mulld	$h0lo,$a0,$a0
	mulhdu	$h0hi,$a0,$a0
	mulld	$h1lo,$a1,$bi
	mulhdu	$h1hi,$a1,$bi
	mulld	$h2lo,$a2,$bi
	mulhdu	$h2hi,$a2,$bi
	mulld	$h3lo,$a3,$bi
	mulhdu	$h3hi,$a3,$bi
	mulld	$h4lo,$a4,$bi
	mulhdu	$h4hi,$a4,$bi
	add	$bi,$a1,$a1		# a[1]*2
___
	($a4,$t1) = ($t1,$a4);
$code.=<<___;
	mulld	$t0,$t1,$a4
	mulhdu	$t1,$t1,$a4
	addc	$h3lo,$h3lo,$t0
	adde	$h3hi,$h3hi,$t1

	mulli	$bp,$a3,19		# a[3]*19

	mulld	$t0,$a1,$a1
	mulhdu	$t1,$a1,$a1
	addc	$h2lo,$h2lo,$t0
	adde	$h2hi,$h2hi,$t1
	mulld	$t0,$a2,$bi
	mulhdu	$t1,$a2,$bi
	addc	$h3lo,$h3lo,$t0
	adde	$h3hi,$h3hi,$t1
	mulld	$t0,$a3,$bi
	mulhdu	$t1,$a3,$bi
	addc	$h4lo,$h4lo,$t0
	adde	$h4hi,$h4hi,$t1
	mulld	$t0,$a4,$bi
	mulhdu	$t1,$a4,$bi
	add	$bi,$a3,$a3		# a[3]*2
	addc	$h0lo,$h0lo,$t0
	adde	$h0hi,$h0hi,$t1
___
	($a3,$t1) = ($bp,$a3);
$code.=<<___;
	mulld	$t0,$t1,$a3
	mulhdu	$t1,$t1,$a3
	addc	$h1lo,$h1lo,$t0
	adde	$h1hi,$h1hi,$t1
	mulld	$t0,$bi,$a4
	mulhdu	$t1,$bi,$a4
	add	$bi,$a2,$a2		# a[2]*2
	addc	$h2lo,$h2lo,$t0
	adde	$h2hi,$h2hi,$t1

	mulld	$t0,$a2,$a2
	mulhdu	$t1,$a2,$a2
	addc	$h4lo,$h4lo,$t0
	adde	$h4hi,$h4hi,$t1
	mulld	$t0,$a3,$bi
	mulhdu	$t1,$a3,$bi
	addc	$h0lo,$h0lo,$t0
	adde	$h0hi,$h0hi,$t1
	mulld	$t0,$a4,$bi
	mulhdu	$t1,$a4,$bi
	addc	$h1lo,$h1lo,$t0
	adde	$h1hi,$h1hi,$t1

	b	.Lfe51_reduce
	.long	0
	.byte	0,12,4,0,0x80,11,2,0
	.long	0
.size	x25519_fe51_sqr,.-x25519_fe51_sqr
___
}
$code.=<<___;
.globl	x25519_fe51_mul121666
.type	x25519_fe51_mul121666,\@function
.align	5
x25519_fe51_mul121666:
	stdu	$sp,-$FRAME($sp)
	std	r21,`$FRAME-8*11`($sp)
	std	r22,`$FRAME-8*10`($sp)
	std	r23,`$FRAME-8*9`($sp)
	std	r24,`$FRAME-8*8`($sp)
	std	r25,`$FRAME-8*7`($sp)
	std	r26,`$FRAME-8*6`($sp)
	std	r27,`$FRAME-8*5`($sp)
	std	r28,`$FRAME-8*4`($sp)
	std	r29,`$FRAME-8*3`($sp)
	std	r30,`$FRAME-8*2`($sp)
	std	r31,`$FRAME-8*1`($sp)

	lis	$bi,`65536>>16`
	ori	$bi,$bi,`121666-65536`
	ld	$a0,0($ap)
	ld	$a1,8($ap)
	ld	$a2,16($ap)
	ld	$a3,24($ap)
	ld	$a4,32($ap)

	mulld	$h0lo,$a0,$bi		# a[0]*121666
	mulhdu	$h0hi,$a0,$bi
	mulld	$h1lo,$a1,$bi		# a[1]*121666
	mulhdu	$h1hi,$a1,$bi
	mulld	$h2lo,$a2,$bi		# a[2]*121666
	mulhdu	$h2hi,$a2,$bi
	mulld	$h3lo,$a3,$bi		# a[3]*121666
	mulhdu	$h3hi,$a3,$bi
	mulld	$h4lo,$a4,$bi		# a[4]*121666
	mulhdu	$h4hi,$a4,$bi

	b	.Lfe51_reduce
	.long	0
	.byte	0,12,4,0,0x80,11,2,0
	.long	0
.size	x25519_fe51_mul121666,.-x25519_fe51_mul121666
___
}

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT or die "error closing STDOUT: $!";


#!/usr/bin/env perl
# Copyright 2018-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# X25519 lower-level primitives for x86_64.
#
# February 2018.
#
# This module implements radix 2^51 multiplication and squaring, and
# radix 2^64 multiplication, squaring, addition, subtraction and final
# reduction. Latter radix is used on ADCX/ADOX-capable processors such
# as Broadwell. On related note one should mention that there are
# vector implementations that provide significantly better performance
# on some processors(*), but they are large and overly complex. Which
# in combination with them being effectively processor-specific makes
# the undertaking hard to justify. The goal for this implementation
# is rather versatility and simplicity [and ultimately formal
# verification].
#
# (*)	For example sandy2x should provide ~30% improvement on Sandy
#	Bridge, but only nominal ~5% on Haswell [and big loss on
#	Broadwell and successors].
#
######################################################################
# Improvement coefficients:
#
#			amd64-51(*)	gcc-5.x(**)
#
# P4			+22%		+40%
# Sandy Bridge		-3%		+11%
# Haswell		-1%		+13%
# Broadwell(***)	+30%		+35%
# Skylake(***)		+33%		+47%
# Silvermont		+20%		+26%
# Goldmont		+40%		+50%
# Bulldozer		+20%		+9%
# Ryzen(***)		+43%		+40%
# VIA			+170%		+120%
#
# (*)	amd64-51 is popular assembly implementation with 2^51 radix,
#	only multiplication and squaring subroutines were linked
#	for comparison, but not complete ladder step; gain on most
#	processors is because this module refrains from shld, and
#	minor regression on others is because this does result in
#	higher instruction count;
# (**)	compiler is free to inline functions, in assembly one would
#	need to implement ladder step to do that, and it will improve
#	performance by several percent;
# (***)	ADCX/ADOX result for 2^64 radix, there is no corresponding
#	C implementation, so that comparison is always against
#	2^51 radix;

$flavour = shift;
$output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
	$addx = ($1>=2.23);
}

if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
	$addx = ($1>=2.10);
}

if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
	$addx = ($1>=12);
}

if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)/) {
	my $ver = $2 + $3/100.0;	# 3.1->3.01, 3.10->3.10
	$addx = ($ver>=3.03);
}

$code.=<<___;
.text

.globl	x25519_fe51_mul
.type	x25519_fe51_mul,\@function,3
.align	32
x25519_fe51_mul:
.cfi_startproc
	push	%rbp
.cfi_push	%rbp
	push	%rbx
.cfi_push	%rbx
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
	lea	-8*5(%rsp),%rsp
.cfi_adjust_cfa_offset	40
.Lfe51_mul_body:

	mov	8*0(%rsi),%rax		# f[0]
	mov	8*0(%rdx),%r11		# load g[0-4]
	mov	8*1(%rdx),%r12
	mov	8*2(%rdx),%r13
	mov	8*3(%rdx),%rbp
	mov	8*4(%rdx),%r14

	mov	%rdi,8*4(%rsp)		# offload 1st argument
	mov	%rax,%rdi
	mulq	%r11			# f[0]*g[0]
	mov	%r11,8*0(%rsp)		# offload g[0]
	mov	%rax,%rbx		# %rbx:%rcx = h0
	mov	%rdi,%rax
	mov	%rdx,%rcx
	mulq	%r12			# f[0]*g[1]
	mov	%r12,8*1(%rsp)		# offload g[1]
	mov	%rax,%r8		# %r8:%r9 = h1
	mov	%rdi,%rax
	lea	(%r14,%r14,8),%r15
	mov	%rdx,%r9
	mulq	%r13			# f[0]*g[2]
	mov	%r13,8*2(%rsp)		# offload g[2]
	mov	%rax,%r10		# %r10:%r11 = h2
	mov	%rdi,%rax
	lea	(%r14,%r15,2),%rdi	# g[4]*19
	mov	%rdx,%r11
	mulq	%rbp			# f[0]*g[3]
	mov	%rax,%r12		# %r12:%r13 = h3
	mov	8*0(%rsi),%rax		# f[0]
	mov	%rdx,%r13
	mulq	%r14			# f[0]*g[4]
	mov	%rax,%r14		# %r14:%r15 = h4
	mov	8*1(%rsi),%rax		# f[1]
	mov	%rdx,%r15

	mulq	%rdi			# f[1]*g[4]*19
	add	%rax,%rbx
	mov	8*2(%rsi),%rax		# f[2]
	adc	%rdx,%rcx
	mulq	%rdi			# f[2]*g[4]*19
	add	%rax,%r8
	mov	8*3(%rsi),%rax		# f[3]
	adc	%rdx,%r9
	mulq	%rdi			# f[3]*g[4]*19
	add	%rax,%r10
	mov	8*4(%rsi),%rax		# f[4]
	adc	%rdx,%r11
	mulq	%rdi			# f[4]*g[4]*19
	imulq	\$19,%rbp,%rdi		# g[3]*19
	add	%rax,%r12
	mov	8*1(%rsi),%rax		# f[1]
	adc	%rdx,%r13
	mulq	%rbp			# f[1]*g[3]
	mov	8*2(%rsp),%rbp		# g[2]
	add	%rax,%r14
	mov	8*2(%rsi),%rax		# f[2]
	adc	%rdx,%r15

	mulq	%rdi			# f[2]*g[3]*19
	add	%rax,%rbx
	mov	8*3(%rsi),%rax		# f[3]
	adc	%rdx,%rcx
	mulq	%rdi			# f[3]*g[3]*19
	add	%rax,%r8
	mov	8*4(%rsi),%rax		# f[4]
	adc	%rdx,%r9
	mulq	%rdi			# f[4]*g[3]*19
	imulq	\$19,%rbp,%rdi		# g[2]*19
	add	%rax,%r10
	mov	8*1(%rsi),%rax		# f[1]
	adc	%rdx,%r11
	mulq	%rbp			# f[1]*g[2]
	add	%rax,%r12
	mov	8*2(%rsi),%rax		# f[2]
	adc	%rdx,%r13
	mulq	%rbp			# f[2]*g[2]
	mov	8*1(%rsp),%rbp		# g[1]
	add	%rax,%r14
	mov	8*3(%rsi),%rax		# f[3]
	adc	%rdx,%r15

	mulq	%rdi			# f[3]*g[2]*19
	add	%rax,%rbx
	mov	8*4(%rsi),%rax		# f[3]
	adc	%rdx,%rcx
	mulq	%rdi			# f[4]*g[2]*19
	add	%rax,%r8
	mov	8*1(%rsi),%rax		# f[1]
	adc	%rdx,%r9
	mulq	%rbp			# f[1]*g[1]
	imulq	\$19,%rbp,%rdi
	add	%rax,%r10
	mov	8*2(%rsi),%rax		# f[2]
	adc	%rdx,%r11
	mulq	%rbp			# f[2]*g[1]
	add	%rax,%r12
	mov	8*3(%rsi),%rax		# f[3]
	adc	%rdx,%r13
	mulq	%rbp			# f[3]*g[1]
	mov	8*0(%rsp),%rbp		# g[0]
	add	%rax,%r14
	mov	8*4(%rsi),%rax		# f[4]
	adc	%rdx,%r15

	mulq	%rdi			# f[4]*g[1]*19
	add	%rax,%rbx
	mov	8*1(%rsi),%rax		# f[1]
	adc	%rdx,%rcx
	mul	%rbp			# f[1]*g[0]
	add	%rax,%r8
	mov	8*2(%rsi),%rax		# f[2]
	adc	%rdx,%r9
	mul	%rbp			# f[2]*g[0]
	add	%rax,%r10
	mov	8*3(%rsi),%rax		# f[3]
	adc	%rdx,%r11
	mul	%rbp			# f[3]*g[0]
	add	%rax,%r12
	mov	8*4(%rsi),%rax		# f[4]
	adc	%rdx,%r13
	mulq	%rbp			# f[4]*g[0]
	add	%rax,%r14
	adc	%rdx,%r15

	mov	8*4(%rsp),%rdi		# restore 1st argument
	jmp	.Lreduce51
.Lfe51_mul_epilogue:
.cfi_endproc
.size	x25519_fe51_mul,.-x25519_fe51_mul

.globl	x25519_fe51_sqr
.type	x25519_fe51_sqr,\@function,2
.align	32
x25519_fe51_sqr:
.cfi_startproc
	push	%rbp
.cfi_push	%rbp
	push	%rbx
.cfi_push	%rbx
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
	lea	-8*5(%rsp),%rsp
.cfi_adjust_cfa_offset	40
.Lfe51_sqr_body:

	mov	8*0(%rsi),%rax		# g[0]
	mov	8*2(%rsi),%r15		# g[2]
	mov	8*4(%rsi),%rbp		# g[4]

	mov	%rdi,8*4(%rsp)		# offload 1st argument
	lea	(%rax,%rax),%r14
	mulq	%rax			# g[0]*g[0]
	mov	%rax,%rbx
	mov	8*1(%rsi),%rax		# g[1]
	mov	%rdx,%rcx
	mulq	%r14			# 2*g[0]*g[1]
	mov	%rax,%r8
	mov	%r15,%rax
	mov	%r15,8*0(%rsp)		# offload g[2]
	mov	%rdx,%r9
	mulq	%r14			# 2*g[0]*g[2]
	mov	%rax,%r10
	mov	8*3(%rsi),%rax
	mov	%rdx,%r11
	imulq	\$19,%rbp,%rdi		# g[4]*19
	mulq	%r14			# 2*g[0]*g[3]
	mov	%rax,%r12
	mov	%rbp,%rax
	mov	%rdx,%r13
	mulq	%r14			# 2*g[0]*g[4]
	mov	%rax,%r14
	mov	%rbp,%rax
	mov	%rdx,%r15

	mulq	%rdi			# g[4]*g[4]*19
	add	%rax,%r12
	mov	8*1(%rsi),%rax		# g[1]
	adc	%rdx,%r13

	mov	8*3(%rsi),%rsi		# g[3]
	lea	(%rax,%rax),%rbp
	mulq	%rax			# g[1]*g[1]
	add	%rax,%r10
	mov	8*0(%rsp),%rax		# g[2]
	adc	%rdx,%r11
	mulq	%rbp			# 2*g[1]*g[2]
	add	%rax,%r12
	mov	%rbp,%rax
	adc	%rdx,%r13
	mulq	%rsi			# 2*g[1]*g[3]
	add	%rax,%r14
	mov	%rbp,%rax
	adc	%rdx,%r15
	imulq	\$19,%rsi,%rbp		# g[3]*19
	mulq	%rdi			# 2*g[1]*g[4]*19
	add	%rax,%rbx
	lea	(%rsi,%rsi),%rax
	adc	%rdx,%rcx

	mulq	%rdi			# 2*g[3]*g[4]*19
	add	%rax,%r10
	mov	%rsi,%rax
	adc	%rdx,%r11
	mulq	%rbp			# g[3]*g[3]*19
	add	%rax,%r8
	mov	8*0(%rsp),%rax		# g[2]
	adc	%rdx,%r9

	lea	(%rax,%rax),%rsi
	mulq	%rax			# g[2]*g[2]
	add	%rax,%r14
	mov	%rbp,%rax
	adc	%rdx,%r15
	mulq	%rsi			# 2*g[2]*g[3]*19
	add	%rax,%rbx
	mov	%rsi,%rax
	adc	%rdx,%rcx
	mulq	%rdi			# 2*g[2]*g[4]*19
	add	%rax,%r8
	adc	%rdx,%r9

	mov	8*4(%rsp),%rdi		# restore 1st argument
	jmp	.Lreduce51

.align	32
.Lreduce51:
	mov	\$0x7ffffffffffff,%rbp

	mov	%r10,%rdx
	shr	\$51,%r10
	shl	\$13,%r11
	and	%rbp,%rdx		# %rdx = g2 = h2 & mask
	or	%r10,%r11		# h2>>51
	add	%r11,%r12
	adc	\$0,%r13		# h3 += h2>>51

	mov	%rbx,%rax
	shr	\$51,%rbx
	shl	\$13,%rcx
	and	%rbp,%rax		# %rax = g0 = h0 & mask
	or	%rbx,%rcx		# h0>>51
	add	%rcx,%r8		# h1 += h0>>51
	adc	\$0,%r9

	mov	%r12,%rbx
	shr	\$51,%r12
	shl	\$13,%r13
	and	%rbp,%rbx		# %rbx = g3 = h3 & mask
	or	%r12,%r13		# h3>>51
	add	%r13,%r14		# h4 += h3>>51
	adc	\$0,%r15

	mov	%r8,%rcx
	shr	\$51,%r8
	shl	\$13,%r9
	and	%rbp,%rcx		# %rcx = g1 = h1 & mask
	or	%r8,%r9
	add	%r9,%rdx		# g2 += h1>>51

	mov	%r14,%r10
	shr	\$51,%r14
	shl	\$13,%r15
	and	%rbp,%r10		# %r10 = g4 = h0 & mask
	or	%r14,%r15		# h0>>51

	lea	(%r15,%r15,8),%r14
	lea	(%r15,%r14,2),%r15
	add	%r15,%rax		# g0 += (h0>>51)*19

	mov	%rdx,%r8
	and	%rbp,%rdx		# g2 &= mask
	shr	\$51,%r8
	add	%r8,%rbx		# g3 += g2>>51

	mov	%rax,%r9
	and	%rbp,%rax		# g0 &= mask
	shr	\$51,%r9
	add	%r9,%rcx		# g1 += g0>>51

	mov	%rax,8*0(%rdi)		# save the result
	mov	%rcx,8*1(%rdi)
	mov	%rdx,8*2(%rdi)
	mov	%rbx,8*3(%rdi)
	mov	%r10,8*4(%rdi)

	mov	8*5(%rsp),%r15
.cfi_restore	%r15
	mov	8*6(%rsp),%r14
.cfi_restore	%r14
	mov	8*7(%rsp),%r13
.cfi_restore	%r13
	mov	8*8(%rsp),%r12
.cfi_restore	%r12
	mov	8*9(%rsp),%rbx
.cfi_restore	%rbx
	mov	8*10(%rsp),%rbp
.cfi_restore	%rbp
	lea	8*11(%rsp),%rsp
.cfi_adjust_cfa_offset	88
.Lfe51_sqr_epilogue:
	ret
.cfi_endproc
.size	x25519_fe51_sqr,.-x25519_fe51_sqr

.globl	x25519_fe51_mul121666
.type	x25519_fe51_mul121666,\@function,2
.align	32
x25519_fe51_mul121666:
.cfi_startproc
	push	%rbp
.cfi_push	%rbp
	push	%rbx
.cfi_push	%rbx
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
	lea	-8*5(%rsp),%rsp
.cfi_adjust_cfa_offset	40
.Lfe51_mul121666_body:
	mov	\$121666,%eax

	mulq	8*0(%rsi)
	mov	%rax,%rbx		# %rbx:%rcx = h0
	mov	\$121666,%eax
	mov	%rdx,%rcx
	mulq	8*1(%rsi)
	mov	%rax,%r8		# %r8:%r9 = h1
	mov	\$121666,%eax
	mov	%rdx,%r9
	mulq	8*2(%rsi)
	mov	%rax,%r10		# %r10:%r11 = h2
	mov	\$121666,%eax
	mov	%rdx,%r11
	mulq	8*3(%rsi)
	mov	%rax,%r12		# %r12:%r13 = h3
	mov	\$121666,%eax		# f[0]
	mov	%rdx,%r13
	mulq	8*4(%rsi)
	mov	%rax,%r14		# %r14:%r15 = h4
	mov	%rdx,%r15

	jmp	.Lreduce51
.Lfe51_mul121666_epilogue:
.cfi_endproc
.size	x25519_fe51_mul121666,.-x25519_fe51_mul121666
___
########################################################################
# Base 2^64 subroutines modulo 2*(2^255-19)
#
if ($addx) {
my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7) = map("%r$_",(8..15));

$code.=<<___;
.extern	OPENSSL_ia32cap_P
.globl	x25519_fe64_eligible
.type	x25519_fe64_eligible,\@abi-omnipotent
.align	32
x25519_fe64_eligible:
.cfi_startproc
	mov	OPENSSL_ia32cap_P+8(%rip),%ecx
	xor	%eax,%eax
	and	\$0x80100,%ecx
	cmp	\$0x80100,%ecx
	cmove	%ecx,%eax
	ret
.cfi_endproc
.size	x25519_fe64_eligible,.-x25519_fe64_eligible

.globl	x25519_fe64_mul
.type	x25519_fe64_mul,\@function,3
.align	32
x25519_fe64_mul:
.cfi_startproc
	push	%rbp
.cfi_push	%rbp
	push	%rbx
.cfi_push	%rbx
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
	push	%rdi			# offload dst
.cfi_push	%rdi
	lea	-8*2(%rsp),%rsp
.cfi_adjust_cfa_offset	16
.Lfe64_mul_body:

	mov	%rdx,%rax
	mov	8*0(%rdx),%rbp		# b[0]
	mov	8*0(%rsi),%rdx		# a[0]
	mov	8*1(%rax),%rcx		# b[1]
	mov	8*2(%rax),$acc6		# b[2]
	mov	8*3(%rax),$acc7		# b[3]

	mulx	%rbp,$acc0,%rax		# a[0]*b[0]
	xor	%edi,%edi		# cf=0,of=0
	mulx	%rcx,$acc1,%rbx		# a[0]*b[1]
	adcx	%rax,$acc1
	mulx	$acc6,$acc2,%rax	# a[0]*b[2]
	adcx	%rbx,$acc2
	mulx	$acc7,$acc3,$acc4	# a[0]*b[3]
	 mov	8*1(%rsi),%rdx		# a[1]
	adcx	%rax,$acc3
	mov	$acc6,(%rsp)		# offload b[2]
	adcx	%rdi,$acc4		# cf=0

	mulx	%rbp,%rax,%rbx		# a[1]*b[0]
	adox	%rax,$acc1
	adcx	%rbx,$acc2
	mulx	%rcx,%rax,%rbx		# a[1]*b[1]
	adox	%rax,$acc2
	adcx	%rbx,$acc3
	mulx	$acc6,%rax,%rbx		# a[1]*b[2]
	adox	%rax,$acc3
	adcx	%rbx,$acc4
	mulx	$acc7,%rax,$acc5	# a[1]*b[3]
	 mov	8*2(%rsi),%rdx		# a[2]
	adox	%rax,$acc4
	adcx	%rdi,$acc5		# cf=0
	adox	%rdi,$acc5		# of=0

	mulx	%rbp,%rax,%rbx		# a[2]*b[0]
	adcx	%rax,$acc2
	adox	%rbx,$acc3
	mulx	%rcx,%rax,%rbx		# a[2]*b[1]
	adcx	%rax,$acc3
	adox	%rbx,$acc4
	mulx	$acc6,%rax,%rbx		# a[2]*b[2]
	adcx	%rax,$acc4
	adox	%rbx,$acc5
	mulx	$acc7,%rax,$acc6	# a[2]*b[3]
	 mov	8*3(%rsi),%rdx		# a[3]
	adcx	%rax,$acc5
	adox	%rdi,$acc6		# of=0
	adcx	%rdi,$acc6		# cf=0

	mulx	%rbp,%rax,%rbx		# a[3]*b[0]
	adox	%rax,$acc3
	adcx	%rbx,$acc4
	mulx	%rcx,%rax,%rbx		# a[3]*b[1]
	adox	%rax,$acc4
	adcx	%rbx,$acc5
	mulx	(%rsp),%rax,%rbx	# a[3]*b[2]
	adox	%rax,$acc5
	adcx	%rbx,$acc6
	mulx	$acc7,%rax,$acc7	# a[3]*b[3]
	 mov	\$38,%edx
	adox	%rax,$acc6
	adcx	%rdi,$acc7		# cf=0
	adox	%rdi,$acc7		# of=0

	jmp	.Lreduce64
.Lfe64_mul_epilogue:
.cfi_endproc
.size	x25519_fe64_mul,.-x25519_fe64_mul

.globl	x25519_fe64_sqr
.type	x25519_fe64_sqr,\@function,2
.align	32
x25519_fe64_sqr:
.cfi_startproc
	push	%rbp
.cfi_push	%rbp
	push	%rbx
.cfi_push	%rbx
	push	%r12
.cfi_push	%r12
	push	%r13
.cfi_push	%r13
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
	push	%rdi			# offload dst
.cfi_push	%rdi
	lea	-8*2(%rsp),%rsp
.cfi_adjust_cfa_offset	16
.Lfe64_sqr_body:

	mov	8*0(%rsi),%rdx		# a[0]
	mov	8*1(%rsi),%rcx		# a[1]
	mov	8*2(%rsi),%rbp		# a[2]
	mov	8*3(%rsi),%rsi		# a[3]

	################################################################
	mulx	%rdx,$acc0,$acc7	# a[0]*a[0]
	mulx	%rcx,$acc1,%rax		# a[0]*a[1]
	xor	%edi,%edi		# cf=0,of=0
	mulx	%rbp,$acc2,%rbx		# a[0]*a[2]
	adcx	%rax,$acc2
	mulx	%rsi,$acc3,$acc4	# a[0]*a[3]
	 mov	%rcx,%rdx		# a[1]
	adcx	%rbx,$acc3
	adcx	%rdi,$acc4		# cf=0

	################################################################
	mulx	%rbp,%rax,%rbx		# a[1]*a[2]
	adox	%rax,$acc3
	adcx	%rbx,$acc4
	mulx	%rsi,%rax,$acc5		# a[1]*a[3]
	 mov	%rbp,%rdx		# a[2]
	adox	%rax,$acc4
	adcx	%rdi,$acc5

	################################################################
	mulx	%rsi,%rax,$acc6		# a[2]*a[3]
	 mov	%rcx,%rdx		# a[1]
	adox	%rax,$acc5
	adcx	%rdi,$acc6		# cf=0
	adox	%rdi,$acc6		# of=0

	 adcx	$acc1,$acc1		# acc1:6<<1
	adox	$acc7,$acc1
	 adcx	$acc2,$acc2
	mulx	%rdx,%rax,%rbx		# a[1]*a[1]
	 mov	%rbp,%rdx		# a[2]
	 adcx	$acc3,$acc3
	adox	%rax,$acc2
	 adcx	$acc4,$acc4
	adox	%rbx,$acc3
	mulx	%rdx,%rax,%rbx		# a[2]*a[2]
	 mov	%rsi,%rdx		# a[3]
	 adcx	$acc5,$acc5
	adox	%rax,$acc4
	 adcx	$acc6,$acc6
	adox	%rbx,$acc5
	mulx	%rdx,%rax,$acc7		# a[3]*a[3]
	 mov	\$38,%edx
	adox	%rax,$acc6
	adcx	%rdi,$acc7		# cf=0
	adox	%rdi,$acc7		# of=0
	jmp	.Lreduce64

.align	32
.Lreduce64:
	mulx	$acc4,%rax,%rbx
	adcx	%rax,$acc0
	adox	%rbx,$acc1
	mulx	$acc5,%rax,%rbx
	adcx	%rax,$acc1
	adox	%rbx,$acc2
	mulx	$acc6,%rax,%rbx
	adcx	%rax,$acc2
	adox	%rbx,$acc3
	mulx	$acc7,%rax,$acc4
	adcx	%rax,$acc3
	adox	%rdi,$acc4
	adcx	%rdi,$acc4

	mov	8*2(%rsp),%rdi		# restore dst
	imulq	%rdx,$acc4

	add	$acc4,$acc0
	adc	\$0,$acc1
	adc	\$0,$acc2
	adc	\$0,$acc3

	sbb	%rax,%rax		# cf -> mask
	and	\$38,%rax

	add	%rax,$acc0
	mov	$acc1,8*1(%rdi)
	mov	$acc2,8*2(%rdi)
	mov	$acc3,8*3(%rdi)
	mov	$acc0,8*0(%rdi)

	mov	8*3(%rsp),%r15
.cfi_restore	%r15
	mov	8*4(%rsp),%r14
.cfi_restore	%r14
	mov	8*5(%rsp),%r13
.cfi_restore	%r13
	mov	8*6(%rsp),%r12
.cfi_restore	%r12
	mov	8*7(%rsp),%rbx
.cfi_restore	%rbx
	mov	8*8(%rsp),%rbp
.cfi_restore	%rbp
	lea	8*9(%rsp),%rsp
.cfi_adjust_cfa_offset	88
.Lfe64_sqr_epilogue:
	ret
.cfi_endproc
.size	x25519_fe64_sqr,.-x25519_fe64_sqr

.globl	x25519_fe64_mul121666
.type	x25519_fe64_mul121666,\@function,2
.align	32
x25519_fe64_mul121666:
.Lfe64_mul121666_body:
.cfi_startproc
	mov	\$121666,%edx
	mulx	8*0(%rsi),$acc0,%rcx
	mulx	8*1(%rsi),$acc1,%rax
	add	%rcx,$acc1
	mulx	8*2(%rsi),$acc2,%rcx
	adc	%rax,$acc2
	mulx	8*3(%rsi),$acc3,%rax
	adc	%rcx,$acc3
	adc	\$0,%rax

	imulq	\$38,%rax,%rax

	add	%rax,$acc0
	adc	\$0,$acc1
	adc	\$0,$acc2
	adc	\$0,$acc3

	sbb	%rax,%rax		# cf -> mask
	and	\$38,%rax

	add	%rax,$acc0
	mov	$acc1,8*1(%rdi)
	mov	$acc2,8*2(%rdi)
	mov	$acc3,8*3(%rdi)
	mov	$acc0,8*0(%rdi)

.Lfe64_mul121666_epilogue:
	ret
.cfi_endproc
.size	x25519_fe64_mul121666,.-x25519_fe64_mul121666

.globl	x25519_fe64_add
.type	x25519_fe64_add,\@function,3
.align	32
x25519_fe64_add:
.Lfe64_add_body:
.cfi_startproc
	mov	8*0(%rsi),$acc0
	mov	8*1(%rsi),$acc1
	mov	8*2(%rsi),$acc2
	mov	8*3(%rsi),$acc3

	add	8*0(%rdx),$acc0
	adc	8*1(%rdx),$acc1
	adc	8*2(%rdx),$acc2
	adc	8*3(%rdx),$acc3

	sbb	%rax,%rax		# cf -> mask
	and	\$38,%rax

	add	%rax,$acc0
	adc	\$0,$acc1
	adc	\$0,$acc2
	mov	$acc1,8*1(%rdi)
	adc	\$0,$acc3
	mov	$acc2,8*2(%rdi)
	sbb	%rax,%rax		# cf -> mask
	mov	$acc3,8*3(%rdi)
	and	\$38,%rax

	add	%rax,$acc0
	mov	$acc0,8*0(%rdi)

.Lfe64_add_epilogue:
	ret
.cfi_endproc
.size	x25519_fe64_add,.-x25519_fe64_add

.globl	x25519_fe64_sub
.type	x25519_fe64_sub,\@function,3
.align	32
x25519_fe64_sub:
.Lfe64_sub_body:
.cfi_startproc
	mov	8*0(%rsi),$acc0
	mov	8*1(%rsi),$acc1
	mov	8*2(%rsi),$acc2
	mov	8*3(%rsi),$acc3

	sub	8*0(%rdx),$acc0
	sbb	8*1(%rdx),$acc1
	sbb	8*2(%rdx),$acc2
	sbb	8*3(%rdx),$acc3

	sbb	%rax,%rax		# cf -> mask
	and	\$38,%rax

	sub	%rax,$acc0
	sbb	\$0,$acc1
	sbb	\$0,$acc2
	mov	$acc1,8*1(%rdi)
	sbb	\$0,$acc3
	mov	$acc2,8*2(%rdi)
	sbb	%rax,%rax		# cf -> mask
	mov	$acc3,8*3(%rdi)
	and	\$38,%rax

	sub	%rax,$acc0
	mov	$acc0,8*0(%rdi)

.Lfe64_sub_epilogue:
	ret
.cfi_endproc
.size	x25519_fe64_sub,.-x25519_fe64_sub

.globl	x25519_fe64_tobytes
.type	x25519_fe64_tobytes,\@function,2
.align	32
x25519_fe64_tobytes:
.Lfe64_to_body:
.cfi_startproc
	mov	8*0(%rsi),$acc0
	mov	8*1(%rsi),$acc1
	mov	8*2(%rsi),$acc2
	mov	8*3(%rsi),$acc3

	################################# reduction modulo 2^255-19
	lea	($acc3,$acc3),%rax
	sar	\$63,$acc3		# most significant bit -> mask
	shr	\$1,%rax		# most significant bit cleared
	and	\$19,$acc3
	add	\$19,$acc3		# compare to modulus in the same go

	add	$acc3,$acc0
	adc	\$0,$acc1
	adc	\$0,$acc2
	adc	\$0,%rax

	lea	(%rax,%rax),$acc3
	sar	\$63,%rax		# most significant bit -> mask
	shr	\$1,$acc3		# most significant bit cleared
	not	%rax
	and	\$19,%rax

	sub	%rax,$acc0
	sbb	\$0,$acc1
	sbb	\$0,$acc2
	sbb	\$0,$acc3

	mov	$acc0,8*0(%rdi)
	mov	$acc1,8*1(%rdi)
	mov	$acc2,8*2(%rdi)
	mov	$acc3,8*3(%rdi)

.Lfe64_to_epilogue:
	ret
.cfi_endproc
.size	x25519_fe64_tobytes,.-x25519_fe64_tobytes
___
} else {
$code.=<<___;
.globl	x25519_fe64_eligible
.type	x25519_fe64_eligible,\@abi-omnipotent
.align	32
x25519_fe64_eligible:
.cfi_startproc
	xor	%eax,%eax
	ret
.cfi_endproc
.size	x25519_fe64_eligible,.-x25519_fe64_eligible

.globl	x25519_fe64_mul
.type	x25519_fe64_mul,\@abi-omnipotent
.globl	x25519_fe64_sqr
.globl	x25519_fe64_mul121666
.globl	x25519_fe64_add
.globl	x25519_fe64_sub
.globl	x25519_fe64_tobytes
x25519_fe64_mul:
x25519_fe64_sqr:
x25519_fe64_mul121666:
x25519_fe64_add:
x25519_fe64_sub:
x25519_fe64_tobytes:
.cfi_startproc
	.byte	0x0f,0x0b	# ud2
	ret
.cfi_endproc
.size	x25519_fe64_mul,.-x25519_fe64_mul
___
}
$code.=<<___;
.asciz	"X25519 primitives for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
___

# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
if ($win64) {
$rec="%rcx";
$frame="%rdx";
$context="%r8";
$disp="%r9";

$code.=<<___;
.extern	__imp_RtlVirtualUnwind

.type	short_handler,\@abi-omnipotent
.align	16
short_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# end of prologue label
	cmp	%r10,%rbx		# context->Rip<end of prologue label
	jb	.Lcommon_seh_tail

	mov	152($context),%rax	# pull context->Rsp
	jmp	.Lcommon_seh_tail
.size	short_handler,.-short_handler

.type	full_handler,\@abi-omnipotent
.align	16
full_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# end of prologue label
	cmp	%r10,%rbx		# context->Rip<end of prologue label
	jb	.Lcommon_seh_tail

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=epilogue label
	jae	.Lcommon_seh_tail

	mov	8(%r11),%r10d		# HandlerData[2]
	lea	(%rax,%r10),%rax

	mov	-8(%rax),%rbp
	mov	-16(%rax),%rbx
	mov	-24(%rax),%r12
	mov	-32(%rax),%r13
	mov	-40(%rax),%r14
	mov	-48(%rax),%r15
	mov	%rbx,144($context)	# restore context->Rbx
	mov	%rbp,160($context)	# restore context->Rbp
	mov	%r12,216($context)	# restore context->R12
	mov	%r13,224($context)	# restore context->R13
	mov	%r14,232($context)	# restore context->R14
	mov	%r15,240($context)	# restore context->R15

.Lcommon_seh_tail:
	mov	8(%rax),%rdi
	mov	16(%rax),%rsi
	mov	%rax,152($context)	# restore context->Rsp
	mov	%rsi,168($context)	# restore context->Rsi
	mov	%rdi,176($context)	# restore context->Rdi

	mov	40($disp),%rdi		# disp->ContextRecord
	mov	$context,%rsi		# context
	mov	\$154,%ecx		# sizeof(CONTEXT)
	.long	0xa548f3fc		# cld; rep movsq

	mov	$disp,%rsi
	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
	mov	0(%rsi),%r8		# arg3, disp->ControlPc
	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
	mov	40(%rsi),%r10		# disp->ContextRecord
	lea	56(%rsi),%r11		# &disp->HandlerData
	lea	24(%rsi),%r12		# &disp->EstablisherFrame
	mov	%r10,32(%rsp)		# arg5
	mov	%r11,40(%rsp)		# arg6
	mov	%r12,48(%rsp)		# arg7
	mov	%rcx,56(%rsp)		# arg8, (NULL)
	call	*__imp_RtlVirtualUnwind(%rip)

	mov	\$1,%eax		# ExceptionContinueSearch
	add	\$64,%rsp
	popfq
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx
	pop	%rdi
	pop	%rsi
	ret
.size	full_handler,.-full_handler

.section	.pdata
.align	4
	.rva	.LSEH_begin_x25519_fe51_mul
	.rva	.LSEH_end_x25519_fe51_mul
	.rva	.LSEH_info_x25519_fe51_mul

	.rva	.LSEH_begin_x25519_fe51_sqr
	.rva	.LSEH_end_x25519_fe51_sqr
	.rva	.LSEH_info_x25519_fe51_sqr

	.rva	.LSEH_begin_x25519_fe51_mul121666
	.rva	.LSEH_end_x25519_fe51_mul121666
	.rva	.LSEH_info_x25519_fe51_mul121666
___
$code.=<<___	if ($addx);
	.rva	.LSEH_begin_x25519_fe64_mul
	.rva	.LSEH_end_x25519_fe64_mul
	.rva	.LSEH_info_x25519_fe64_mul

	.rva	.LSEH_begin_x25519_fe64_sqr
	.rva	.LSEH_end_x25519_fe64_sqr
	.rva	.LSEH_info_x25519_fe64_sqr

	.rva	.LSEH_begin_x25519_fe64_mul121666
	.rva	.LSEH_end_x25519_fe64_mul121666
	.rva	.LSEH_info_x25519_fe64_mul121666

	.rva	.LSEH_begin_x25519_fe64_add
	.rva	.LSEH_end_x25519_fe64_add
	.rva	.LSEH_info_x25519_fe64_add

	.rva	.LSEH_begin_x25519_fe64_sub
	.rva	.LSEH_end_x25519_fe64_sub
	.rva	.LSEH_info_x25519_fe64_sub

	.rva	.LSEH_begin_x25519_fe64_tobytes
	.rva	.LSEH_end_x25519_fe64_tobytes
	.rva	.LSEH_info_x25519_fe64_tobytes
___
$code.=<<___;
.section	.xdata
.align	8
.LSEH_info_x25519_fe51_mul:
	.byte	9,0,0,0
	.rva	full_handler
	.rva	.Lfe51_mul_body,.Lfe51_mul_epilogue	# HandlerData[]
	.long	88,0
.LSEH_info_x25519_fe51_sqr:
	.byte	9,0,0,0
	.rva	full_handler
	.rva	.Lfe51_sqr_body,.Lfe51_sqr_epilogue	# HandlerData[]
	.long	88,0
.LSEH_info_x25519_fe51_mul121666:
	.byte	9,0,0,0
	.rva	full_handler
	.rva	.Lfe51_mul121666_body,.Lfe51_mul121666_epilogue	# HandlerData[]
	.long	88,0
___
$code.=<<___	if ($addx);
.LSEH_info_x25519_fe64_mul:
	.byte	9,0,0,0
	.rva	full_handler
	.rva	.Lfe64_mul_body,.Lfe64_mul_epilogue	# HandlerData[]
	.long	72,0
.LSEH_info_x25519_fe64_sqr:
	.byte	9,0,0,0
	.rva	full_handler
	.rva	.Lfe64_sqr_body,.Lfe64_sqr_epilogue	# HandlerData[]
	.long	72,0
.LSEH_info_x25519_fe64_mul121666:
	.byte	9,0,0,0
	.rva	short_handler
	.rva	.Lfe64_mul121666_body,.Lfe64_mul121666_epilogue	# HandlerData[]
.LSEH_info_x25519_fe64_add:
	.byte	9,0,0,0
	.rva	short_handler
	.rva	.Lfe64_add_body,.Lfe64_add_epilogue	# HandlerData[]
.LSEH_info_x25519_fe64_sub:
	.byte	9,0,0,0
	.rva	short_handler
	.rva	.Lfe64_sub_body,.Lfe64_sub_epilogue	# HandlerData[]
.LSEH_info_x25519_fe64_tobytes:
	.byte	9,0,0,0
	.rva	short_handler
	.rva	.Lfe64_to_body,.Lfe64_to_epilogue	# HandlerData[]
___
}

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# Normal is the
# md5_block_x86(MD5_CTX *c, ULONG *X);
# version, non-normal is the
# md5_block_x86(MD5_CTX *c, ULONG *X,int blocks);

$normal=0;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";

$output=pop;
open STDOUT,">$output";

&asm_init($ARGV[0]);

$A="eax";
$B="ebx";
$C="ecx";
$D="edx";
$tmp1="edi";
$tmp2="ebp";
$X="esi";

# What we need to load into $tmp for the next round
%Ltmp1=("R0",&Np($C), "R1",&Np($C), "R2",&Np($C), "R3",&Np($D));
@xo=(
 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,	# R0
 1, 6, 11, 0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12,	# R1
 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2,	# R2
 0, 7, 14, 5, 12, 3, 10, 1, 8, 15, 6, 13, 4, 11, 2, 9,	# R3
 );

&md5_block("md5_block_asm_data_order");
&asm_finish();

close STDOUT or die "error closing STDOUT: $!";

sub Np
	{
	local($p)=@_;
	local(%n)=($A,$D,$B,$A,$C,$B,$D,$C);
	return($n{$p});
	}

sub R0
	{
	local($pos,$a,$b,$c,$d,$K,$ki,$s,$t)=@_;

	&mov($tmp1,$C)  if $pos < 0;
	&mov($tmp2,&DWP($xo[$ki]*4,$K,"",0)) if $pos < 0; # very first one

	# body proper

	&comment("R0 $ki");
	&xor($tmp1,$d); # F function - part 2

	&and($tmp1,$b); # F function - part 3
	&lea($a,&DWP($t,$a,$tmp2,1));

	&xor($tmp1,$d); # F function - part 4
	&mov($tmp2,&DWP($xo[$ki+1]*4,$K,"",0)) if ($pos != 2);

	&add($a,$tmp1);

	&rotl($a,$s);

	&mov($tmp1,&Np($c)) if $pos < 1;	# next tmp1 for R0
	&mov($tmp1,&Np($c)) if $pos == 1;	# next tmp1 for R1

	&add($a,$b);
	}

sub R1
	{
	local($pos,$a,$b,$c,$d,$K,$ki,$s,$t)=@_;

	&comment("R1 $ki");

	&xor($tmp1,$b); # G function - part 2
	&and($tmp1,$d); # G function - part 3
	&lea($a,&DWP($t,$a,$tmp2,1));

	&xor($tmp1,$c);			# G function - part 4
	&mov($tmp2,&DWP($xo[$ki+1]*4,$K,"",0)) if ($pos != 2);

	&add($a,$tmp1);
	&mov($tmp1,&Np($c)) if $pos < 1;	# G function - part 1
	&mov($tmp1,&Np($c)) if $pos == 1;	# G function - part 1

	&rotl($a,$s);

	&add($a,$b);
	}

sub R2
	{
	local($n,$pos,$a,$b,$c,$d,$K,$ki,$s,$t)=@_;
	# This one is different, only 3 logical operations

if (($n & 1) == 0)
	{
	&comment("R2 $ki");
	# make sure to do 'D' first, not 'B', else we clash with
	# the last add from the previous round.

	&xor($tmp1,$d); # H function - part 2

	&xor($tmp1,$b); # H function - part 3
	&lea($a,&DWP($t,$a,$tmp2,1));

	&add($a,$tmp1);
	&mov($tmp2,&DWP($xo[$ki+1]*4,$K,"",0));

	&rotl($a,$s);

	&mov($tmp1,&Np($c));
	}
else
	{
	&comment("R2 $ki");
	# make sure to do 'D' first, not 'B', else we clash with
	# the last add from the previous round.

	&add($b,$c);			# MOVED FORWARD
	&xor($tmp1,$d); # H function - part 2

	&lea($a,&DWP($t,$a,$tmp2,1));

	&xor($tmp1,$b); # H function - part 3
	&mov($tmp2,&DWP($xo[$ki+1]*4,$K,"",0)) if ($pos != 2);

	&add($a,$tmp1);
	&mov($tmp1,&Np($c)) if $pos < 1;	# H function - part 1
	&mov($tmp1,-1) if $pos == 1;		# I function - part 1

	&rotl($a,$s);

	&add($a,$b);
	}
	}

sub R3
	{
	local($pos,$a,$b,$c,$d,$K,$ki,$s,$t)=@_;

	&comment("R3 $ki");

	# &not($tmp1)
	&xor($tmp1,$d) if $pos < 0; 	# I function - part 2

	&or($tmp1,$b);				# I function - part 3
	&lea($a,&DWP($t,$a,$tmp2,1));

	&xor($tmp1,$c); 			# I function - part 4
	&mov($tmp2,&DWP($xo[$ki+1]*4,$K,"",0))	if $pos != 2; # load X/k value
	&mov($tmp2,&wparam(0)) if $pos == 2;

	&add($a,$tmp1);
	&mov($tmp1,-1) if $pos < 1;	# H function - part 1
	&add($K,64) if $pos >=1 && !$normal;

	&rotl($a,$s);

	&xor($tmp1,&Np($d)) if $pos <= 0; 	# I function - part = first time
	&mov($tmp1,&DWP( 0,$tmp2,"",0)) if $pos > 0;
	&add($a,$b);
	}

sub md5_block
	{
	local($name)=@_;

	&function_begin_B($name,"",3);

	# parameter 1 is the MD5_CTX structure.
	# A	0
	# B	4
	# C	8
	# D 	12

	&push("esi");
	 &push("edi");
	&mov($tmp1,	&wparam(0)); # edi
	 &mov($X,	&wparam(1)); # esi
	&mov($C,	&wparam(2));
	 &push("ebp");
	&shl($C,	6);
	&push("ebx");
	 &add($C,	$X); # offset we end at
	&sub($C,	64);
	 &mov($A,	&DWP( 0,$tmp1,"",0));
	&push($C);	# Put on the TOS
	 &mov($B,	&DWP( 4,$tmp1,"",0));
	&mov($C,	&DWP( 8,$tmp1,"",0));
	 &mov($D,	&DWP(12,$tmp1,"",0));

	&set_label("start") unless $normal;
	&comment("");
	&comment("R0 section");

	&R0(-2,$A,$B,$C,$D,$X, 0, 7,0xd76aa478);
	&R0( 0,$D,$A,$B,$C,$X, 1,12,0xe8c7b756);
	&R0( 0,$C,$D,$A,$B,$X, 2,17,0x242070db);
	&R0( 0,$B,$C,$D,$A,$X, 3,22,0xc1bdceee);
	&R0( 0,$A,$B,$C,$D,$X, 4, 7,0xf57c0faf);
	&R0( 0,$D,$A,$B,$C,$X, 5,12,0x4787c62a);
	&R0( 0,$C,$D,$A,$B,$X, 6,17,0xa8304613);
	&R0( 0,$B,$C,$D,$A,$X, 7,22,0xfd469501);
	&R0( 0,$A,$B,$C,$D,$X, 8, 7,0x698098d8);
	&R0( 0,$D,$A,$B,$C,$X, 9,12,0x8b44f7af);
	&R0( 0,$C,$D,$A,$B,$X,10,17,0xffff5bb1);
	&R0( 0,$B,$C,$D,$A,$X,11,22,0x895cd7be);
	&R0( 0,$A,$B,$C,$D,$X,12, 7,0x6b901122);
	&R0( 0,$D,$A,$B,$C,$X,13,12,0xfd987193);
	&R0( 0,$C,$D,$A,$B,$X,14,17,0xa679438e);
	&R0( 1,$B,$C,$D,$A,$X,15,22,0x49b40821);

	&comment("");
	&comment("R1 section");
	&R1(-1,$A,$B,$C,$D,$X,16, 5,0xf61e2562);
	&R1( 0,$D,$A,$B,$C,$X,17, 9,0xc040b340);
	&R1( 0,$C,$D,$A,$B,$X,18,14,0x265e5a51);
	&R1( 0,$B,$C,$D,$A,$X,19,20,0xe9b6c7aa);
	&R1( 0,$A,$B,$C,$D,$X,20, 5,0xd62f105d);
	&R1( 0,$D,$A,$B,$C,$X,21, 9,0x02441453);
	&R1( 0,$C,$D,$A,$B,$X,22,14,0xd8a1e681);
	&R1( 0,$B,$C,$D,$A,$X,23,20,0xe7d3fbc8);
	&R1( 0,$A,$B,$C,$D,$X,24, 5,0x21e1cde6);
	&R1( 0,$D,$A,$B,$C,$X,25, 9,0xc33707d6);
	&R1( 0,$C,$D,$A,$B,$X,26,14,0xf4d50d87);
	&R1( 0,$B,$C,$D,$A,$X,27,20,0x455a14ed);
	&R1( 0,$A,$B,$C,$D,$X,28, 5,0xa9e3e905);
	&R1( 0,$D,$A,$B,$C,$X,29, 9,0xfcefa3f8);
	&R1( 0,$C,$D,$A,$B,$X,30,14,0x676f02d9);
	&R1( 1,$B,$C,$D,$A,$X,31,20,0x8d2a4c8a);

	&comment("");
	&comment("R2 section");
	&R2( 0,-1,$A,$B,$C,$D,$X,32, 4,0xfffa3942);
	&R2( 1, 0,$D,$A,$B,$C,$X,33,11,0x8771f681);
	&R2( 2, 0,$C,$D,$A,$B,$X,34,16,0x6d9d6122);
	&R2( 3, 0,$B,$C,$D,$A,$X,35,23,0xfde5380c);
	&R2( 4, 0,$A,$B,$C,$D,$X,36, 4,0xa4beea44);
	&R2( 5, 0,$D,$A,$B,$C,$X,37,11,0x4bdecfa9);
	&R2( 6, 0,$C,$D,$A,$B,$X,38,16,0xf6bb4b60);
	&R2( 7, 0,$B,$C,$D,$A,$X,39,23,0xbebfbc70);
	&R2( 8, 0,$A,$B,$C,$D,$X,40, 4,0x289b7ec6);
	&R2( 9, 0,$D,$A,$B,$C,$X,41,11,0xeaa127fa);
	&R2(10, 0,$C,$D,$A,$B,$X,42,16,0xd4ef3085);
	&R2(11, 0,$B,$C,$D,$A,$X,43,23,0x04881d05);
	&R2(12, 0,$A,$B,$C,$D,$X,44, 4,0xd9d4d039);
	&R2(13, 0,$D,$A,$B,$C,$X,45,11,0xe6db99e5);
	&R2(14, 0,$C,$D,$A,$B,$X,46,16,0x1fa27cf8);
	&R2(15, 1,$B,$C,$D,$A,$X,47,23,0xc4ac5665);

	&comment("");
	&comment("R3 section");
	&R3(-1,$A,$B,$C,$D,$X,48, 6,0xf4292244);
	&R3( 0,$D,$A,$B,$C,$X,49,10,0x432aff97);
	&R3( 0,$C,$D,$A,$B,$X,50,15,0xab9423a7);
	&R3( 0,$B,$C,$D,$A,$X,51,21,0xfc93a039);
	&R3( 0,$A,$B,$C,$D,$X,52, 6,0x655b59c3);
	&R3( 0,$D,$A,$B,$C,$X,53,10,0x8f0ccc92);
	&R3( 0,$C,$D,$A,$B,$X,54,15,0xffeff47d);
	&R3( 0,$B,$C,$D,$A,$X,55,21,0x85845dd1);
	&R3( 0,$A,$B,$C,$D,$X,56, 6,0x6fa87e4f);
	&R3( 0,$D,$A,$B,$C,$X,57,10,0xfe2ce6e0);
	&R3( 0,$C,$D,$A,$B,$X,58,15,0xa3014314);
	&R3( 0,$B,$C,$D,$A,$X,59,21,0x4e0811a1);
	&R3( 0,$A,$B,$C,$D,$X,60, 6,0xf7537e82);
	&R3( 0,$D,$A,$B,$C,$X,61,10,0xbd3af235);
	&R3( 0,$C,$D,$A,$B,$X,62,15,0x2ad7d2bb);
	&R3( 2,$B,$C,$D,$A,$X,63,21,0xeb86d391);

	# &mov($tmp2,&wparam(0));	# done in the last R3
	# &mov($tmp1,	&DWP( 0,$tmp2,"",0)); # done is the last R3

	&add($A,$tmp1);
	 &mov($tmp1,	&DWP( 4,$tmp2,"",0));

	&add($B,$tmp1);
	&mov($tmp1,	&DWP( 8,$tmp2,"",0));

	&add($C,$tmp1);
	&mov($tmp1,	&DWP(12,$tmp2,"",0));

	&add($D,$tmp1);
	&mov(&DWP( 0,$tmp2,"",0),$A);

	&mov(&DWP( 4,$tmp2,"",0),$B);
	&mov($tmp1,&swtmp(0)) unless $normal;

	&mov(&DWP( 8,$tmp2,"",0),$C);
	 &mov(&DWP(12,$tmp2,"",0),$D);

	&cmp($tmp1,$X) unless $normal;			# check count
	 &jae(&label("start")) unless $normal;

	&pop("eax"); # pop the temp variable off the stack
	 &pop("ebx");
	&pop("ebp");
	 &pop("edi");
	&pop("esi");
	 &ret();
	&function_end_B($name);
	}


#! /usr/bin/env perl
# Copyright 2012-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
#
# Hardware SPARC T4 support by David S. Miller.
# ====================================================================

# MD5 for SPARCv9, 6.9 cycles per byte on UltraSPARC, >40% faster than
# code generated by Sun C 5.2.

# SPARC T4 MD5 hardware achieves 3.20 cycles per byte, which is 2.1x
# faster than software. Multi-process benchmark saturates at 12x
# single-process result on 8-core processor, or ~11GBps per 2.85GHz
# socket.

$output=pop;
open STDOUT,">$output";

use integer;

($ctx,$inp,$len)=("%i0","%i1","%i2");	# input arguments

# 64-bit values
@X=("%o0","%o1","%o2","%o3","%o4","%o5","%o7","%g1","%g2");
$tx="%g3";
($AB,$CD)=("%g4","%g5");

# 32-bit values
@V=($A,$B,$C,$D)=map("%l$_",(0..3));
($t1,$t2,$t3,$saved_asi)=map("%l$_",(4..7));
($shr,$shl1,$shl2)=("%i3","%i4","%i5");

my @K=(	0xd76aa478,0xe8c7b756,0x242070db,0xc1bdceee,
	0xf57c0faf,0x4787c62a,0xa8304613,0xfd469501,
	0x698098d8,0x8b44f7af,0xffff5bb1,0x895cd7be,
	0x6b901122,0xfd987193,0xa679438e,0x49b40821,

	0xf61e2562,0xc040b340,0x265e5a51,0xe9b6c7aa,
	0xd62f105d,0x02441453,0xd8a1e681,0xe7d3fbc8,
	0x21e1cde6,0xc33707d6,0xf4d50d87,0x455a14ed,
	0xa9e3e905,0xfcefa3f8,0x676f02d9,0x8d2a4c8a,

	0xfffa3942,0x8771f681,0x6d9d6122,0xfde5380c,
	0xa4beea44,0x4bdecfa9,0xf6bb4b60,0xbebfbc70,
	0x289b7ec6,0xeaa127fa,0xd4ef3085,0x04881d05,
	0xd9d4d039,0xe6db99e5,0x1fa27cf8,0xc4ac5665,

	0xf4292244,0x432aff97,0xab9423a7,0xfc93a039,
	0x655b59c3,0x8f0ccc92,0xffeff47d,0x85845dd1,
	0x6fa87e4f,0xfe2ce6e0,0xa3014314,0x4e0811a1,
	0xf7537e82,0xbd3af235,0x2ad7d2bb,0xeb86d391, 0	);

sub R0 {
  my ($i,$a,$b,$c,$d) = @_;
  my $rot = (7,12,17,22)[$i%4];
  my $j   = ($i+1)/2;

  if ($i&1) {
    $code.=<<___;
	 srlx	@X[$j],$shr,@X[$j]	! align X[`$i+1`]
	and	$b,$t1,$t1		! round $i
	 sllx	@X[$j+1],$shl1,$tx
	add	$t2,$a,$a
	 sllx	$tx,$shl2,$tx
	xor	$d,$t1,$t1
	 or	$tx,@X[$j],@X[$j]
	 sethi	%hi(@K[$i+1]),$t2
	add	$t1,$a,$a
	 or	$t2,%lo(@K[$i+1]),$t2
	sll	$a,$rot,$t3
	 add	@X[$j],$t2,$t2		! X[`$i+1`]+K[`$i+1`]
	srl	$a,32-$rot,$a
	add	$b,$t3,$t3
	 xor	 $b,$c,$t1
	add	$t3,$a,$a
___
  } else {
    $code.=<<___;
	 srlx	@X[$j],32,$tx		! extract X[`2*$j+1`]
	and	$b,$t1,$t1		! round $i
	add	$t2,$a,$a
	xor	$d,$t1,$t1
	 sethi	%hi(@K[$i+1]),$t2
	add	$t1,$a,$a
	 or	$t2,%lo(@K[$i+1]),$t2
	sll	$a,$rot,$t3
	 add	$tx,$t2,$t2		! X[`2*$j+1`]+K[`$i+1`]
	srl	$a,32-$rot,$a
	add	$b,$t3,$t3
	 xor	 $b,$c,$t1
	add	$t3,$a,$a
___
  }
}

sub R0_1 {
  my ($i,$a,$b,$c,$d) = @_;
  my $rot = (7,12,17,22)[$i%4];

$code.=<<___;
	 srlx	@X[0],32,$tx		! extract X[1]
	and	$b,$t1,$t1		! round $i
	add	$t2,$a,$a
	xor	$d,$t1,$t1
	 sethi	%hi(@K[$i+1]),$t2
	add	$t1,$a,$a
	 or	$t2,%lo(@K[$i+1]),$t2
	sll	$a,$rot,$t3
	 add	$tx,$t2,$t2		! X[1]+K[`$i+1`]
	srl	$a,32-$rot,$a
	add	$b,$t3,$t3
	 andn	 $b,$c,$t1
	add	$t3,$a,$a
___
}

sub R1 {
  my ($i,$a,$b,$c,$d) = @_;
  my $rot = (5,9,14,20)[$i%4];
  my $j   = $i<31 ? (1+5*($i+1))%16 : (5+3*($i+1))%16;
  my $xi  = @X[$j/2];

$code.=<<___ if ($j&1 && ($xi=$tx));
	 srlx	@X[$j/2],32,$xi		! extract X[$j]
___
$code.=<<___;
	and	$b,$d,$t3		! round $i
	add	$t2,$a,$a
	or	$t3,$t1,$t1
	 sethi	%hi(@K[$i+1]),$t2
	add	$t1,$a,$a
	 or	$t2,%lo(@K[$i+1]),$t2
	sll	$a,$rot,$t3
	 add	$xi,$t2,$t2		! X[$j]+K[`$i+1`]
	srl	$a,32-$rot,$a
	add	$b,$t3,$t3
	 `$i<31?"andn":"xor"`	 $b,$c,$t1
	add	$t3,$a,$a
___
}

sub R2 {
  my ($i,$a,$b,$c,$d) = @_;
  my $rot = (4,11,16,23)[$i%4];
  my $j   = $i<47 ? (5+3*($i+1))%16 : (0+7*($i+1))%16;
  my $xi  = @X[$j/2];

$code.=<<___ if ($j&1 && ($xi=$tx));
	 srlx	@X[$j/2],32,$xi		! extract X[$j]
___
$code.=<<___;
	add	$t2,$a,$a		! round $i
	xor	$b,$t1,$t1
	 sethi	%hi(@K[$i+1]),$t2
	add	$t1,$a,$a
	 or	$t2,%lo(@K[$i+1]),$t2
	sll	$a,$rot,$t3
	 add	$xi,$t2,$t2		! X[$j]+K[`$i+1`]
	srl	$a,32-$rot,$a
	add	$b,$t3,$t3
	 xor	 $b,$c,$t1
	add	$t3,$a,$a
___
}

sub R3 {
  my ($i,$a,$b,$c,$d) = @_;
  my $rot = (6,10,15,21)[$i%4];
  my $j   = (0+7*($i+1))%16;
  my $xi  = @X[$j/2];

$code.=<<___;
	add	$t2,$a,$a		! round $i
___
$code.=<<___ if ($j&1 && ($xi=$tx));
	 srlx	@X[$j/2],32,$xi		! extract X[$j]
___
$code.=<<___;
	orn	$b,$d,$t1
	 sethi	%hi(@K[$i+1]),$t2
	xor	$c,$t1,$t1
	 or	$t2,%lo(@K[$i+1]),$t2
	add	$t1,$a,$a
	sll	$a,$rot,$t3
	 add	$xi,$t2,$t2		! X[$j]+K[`$i+1`]
	srl	$a,32-$rot,$a
	add	$b,$t3,$t3
	add	$t3,$a,$a
___
}

$code.=<<___;
#include "sparc_arch.h"

#ifdef __arch64__
.register	%g2,#scratch
.register	%g3,#scratch
#endif

.section	".text",#alloc,#execinstr

#ifdef __PIC__
SPARC_PIC_THUNK(%g1)
#endif

.globl	md5_block_asm_data_order
.align	32
md5_block_asm_data_order:
	SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)
	ld	[%g1+4],%g1		! OPENSSL_sparcv9cap_P[1]

	andcc	%g1, CFR_MD5, %g0
	be	.Lsoftware
	nop

	mov	4, %g1
	andcc	%o1, 0x7, %g0
	lda	[%o0 + %g0]0x88, %f0		! load context
	lda	[%o0 + %g1]0x88, %f1
	add	%o0, 8, %o0
	lda	[%o0 + %g0]0x88, %f2
	lda	[%o0 + %g1]0x88, %f3
	bne,pn	%icc, .Lhwunaligned
	sub	%o0, 8, %o0

.Lhw_loop:
	ldd	[%o1 + 0x00], %f8
	ldd	[%o1 + 0x08], %f10
	ldd	[%o1 + 0x10], %f12
	ldd	[%o1 + 0x18], %f14
	ldd	[%o1 + 0x20], %f16
	ldd	[%o1 + 0x28], %f18
	ldd	[%o1 + 0x30], %f20
	subcc	%o2, 1, %o2		! done yet?
	ldd	[%o1 + 0x38], %f22
	add	%o1, 0x40, %o1
	prefetch [%o1 + 63], 20

	.word	0x81b02800		! MD5

	bne,pt	SIZE_T_CC, .Lhw_loop
	nop

.Lhwfinish:
	sta	%f0, [%o0 + %g0]0x88	! store context
	sta	%f1, [%o0 + %g1]0x88
	add	%o0, 8, %o0
	sta	%f2, [%o0 + %g0]0x88
	sta	%f3, [%o0 + %g1]0x88
	retl
	nop

.align	8
.Lhwunaligned:
	alignaddr %o1, %g0, %o1

	ldd	[%o1 + 0x00], %f10
.Lhwunaligned_loop:
	ldd	[%o1 + 0x08], %f12
	ldd	[%o1 + 0x10], %f14
	ldd	[%o1 + 0x18], %f16
	ldd	[%o1 + 0x20], %f18
	ldd	[%o1 + 0x28], %f20
	ldd	[%o1 + 0x30], %f22
	ldd	[%o1 + 0x38], %f24
	subcc	%o2, 1, %o2		! done yet?
	ldd	[%o1 + 0x40], %f26
	add	%o1, 0x40, %o1
	prefetch [%o1 + 63], 20

	faligndata %f10, %f12, %f8
	faligndata %f12, %f14, %f10
	faligndata %f14, %f16, %f12
	faligndata %f16, %f18, %f14
	faligndata %f18, %f20, %f16
	faligndata %f20, %f22, %f18
	faligndata %f22, %f24, %f20
	faligndata %f24, %f26, %f22

	.word	0x81b02800		! MD5

	bne,pt	SIZE_T_CC, .Lhwunaligned_loop
	for	%f26, %f26, %f10	! %f10=%f26

	ba	.Lhwfinish
	nop

.align	16
.Lsoftware:
	save	%sp,-STACK_FRAME,%sp

	rd	%asi,$saved_asi
	wr	%g0,0x88,%asi		! ASI_PRIMARY_LITTLE
	and	$inp,7,$shr
	andn	$inp,7,$inp

	sll	$shr,3,$shr		! *=8
	mov	56,$shl2
	ld	[$ctx+0],$A
	sub	$shl2,$shr,$shl2
	ld	[$ctx+4],$B
	and	$shl2,32,$shl1
	add	$shl2,8,$shl2
	ld	[$ctx+8],$C
	sub	$shl2,$shl1,$shl2	! shr+shl1+shl2==64
	ld	[$ctx+12],$D
	nop

.Loop:
	 cmp	$shr,0			! was inp aligned?
	ldxa	[$inp+0]%asi,@X[0]	! load little-endian input
	ldxa	[$inp+8]%asi,@X[1]
	ldxa	[$inp+16]%asi,@X[2]
	ldxa	[$inp+24]%asi,@X[3]
	ldxa	[$inp+32]%asi,@X[4]
	 sllx	$A,32,$AB		! pack A,B
	ldxa	[$inp+40]%asi,@X[5]
	 sllx	$C,32,$CD		! pack C,D
	ldxa	[$inp+48]%asi,@X[6]
	 or	$B,$AB,$AB
	ldxa	[$inp+56]%asi,@X[7]
	 or	$D,$CD,$CD
	bnz,a,pn	%icc,.+8
	ldxa	[$inp+64]%asi,@X[8]

	srlx	@X[0],$shr,@X[0]	! align X[0]
	sllx	@X[1],$shl1,$tx
	 sethi	%hi(@K[0]),$t2
	sllx	$tx,$shl2,$tx
	 or	$t2,%lo(@K[0]),$t2
	or	$tx,@X[0],@X[0]
	 xor	$C,$D,$t1
	 add	@X[0],$t2,$t2		! X[0]+K[0]
___
	for ($i=0;$i<15;$i++)	{ &R0($i,@V);	unshift(@V,pop(@V)); }
	for (;$i<16;$i++)	{ &R0_1($i,@V);	unshift(@V,pop(@V)); }
	for (;$i<32;$i++)	{ &R1($i,@V);	unshift(@V,pop(@V)); }
	for (;$i<48;$i++)	{ &R2($i,@V);	unshift(@V,pop(@V)); }
	for (;$i<64;$i++)	{ &R3($i,@V);	unshift(@V,pop(@V)); }
$code.=<<___;
	srlx	$AB,32,$t1		! unpack A,B,C,D and accumulate
	add	$inp,64,$inp		! advance inp
	srlx	$CD,32,$t2
	add	$t1,$A,$A
	subcc	$len,1,$len		! done yet?
	add	$AB,$B,$B
	add	$t2,$C,$C
	add	$CD,$D,$D
	srl	$B,0,$B			! clruw	$B
	bne	SIZE_T_CC,.Loop
	srl	$D,0,$D			! clruw	$D

	st	$A,[$ctx+0]		! write out ctx
	st	$B,[$ctx+4]
	st	$C,[$ctx+8]
	st	$D,[$ctx+12]

	wr	%g0,$saved_asi,%asi
	ret
	restore
.type	md5_block_asm_data_order,#function
.size	md5_block_asm_data_order,(.-md5_block_asm_data_order)

.asciz	"MD5 block transform for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>"
.align	4
___

# Purpose of these subroutines is to explicitly encode VIS instructions,
# so that one can compile the module without having to specify VIS
# extensions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
# Idea is to reserve for option to produce "universal" binary and let
# programmer detect if current CPU is VIS capable at run-time.
sub unvis {
my ($mnemonic,$rs1,$rs2,$rd)=@_;
my $ref,$opf;
my %visopf = (	"faligndata"	=> 0x048,
		"for"		=> 0x07c	);

    $ref = "$mnemonic\t$rs1,$rs2,$rd";

    if ($opf=$visopf{$mnemonic}) {
	foreach ($rs1,$rs2,$rd) {
	    return $ref if (!/%f([0-9]{1,2})/);
	    $_=$1;
	    if ($1>=32) {
		return $ref if ($1&1);
		# re-encode for upper double register addressing
		$_=($1|$1>>5)&31;
	    }
	}

	return	sprintf ".word\t0x%08x !%s",
			0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
			$ref;
    } else {
	return $ref;
    }
}
sub unalignaddr {
my ($mnemonic,$rs1,$rs2,$rd)=@_;
my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
my $ref="$mnemonic\t$rs1,$rs2,$rd";

    foreach ($rs1,$rs2,$rd) {
	if (/%([goli])([0-7])/)	{ $_=$bias{$1}+$2; }
	else			{ return $ref; }
    }
    return  sprintf ".word\t0x%08x !%s",
		    0x81b00300|$rd<<25|$rs1<<14|$rs2,
		    $ref;
}

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/ge;

	s/\b(f[^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/
		&unvis($1,$2,$3,$4)
	 /ge;
	s/\b(alignaddr)\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/
		&unalignaddr($1,$2,$3,$4)
	 /ge;

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Author: Marc Bevand <bevand_m (at) epita.fr>
# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# MD5 optimized for AMD64.

use strict;

my $code;

# round1_step() does:
#   dst = x + ((dst + F(x,y,z) + X[k] + T_i) <<< s)
#   %r10d = X[k_next]
#   %r11d = z' (copy of z for the next step)
# Each round1_step() takes about 5.3 clocks (9 instructions, 1.7 IPC)
sub round1_step
{
    my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
    $code .= " mov	0*4(%rsi),	%r10d		/* (NEXT STEP) X[0] */\n" if ($pos == -1);
    $code .= " mov	%edx,		%r11d		/* (NEXT STEP) z' = %edx */\n" if ($pos == -1);
    $code .= <<EOF;
	xor	$y,		%r11d		/* y ^ ... */
	lea	$T_i($dst,%r10d),$dst		/* Const + dst + ... */
	and	$x,		%r11d		/* x & ... */
	mov	$k_next*4(%rsi),%r10d		/* (NEXT STEP) X[$k_next] */
	xor	$z,		%r11d		/* z ^ ... */
	add	%r11d,		$dst		/* dst += ... */
	rol	\$$s,		$dst		/* dst <<< s */
	mov	$y,		%r11d		/* (NEXT STEP) z' = $y */
	add	$x,		$dst		/* dst += x */
EOF
}

# round2_step() does:
#   dst = x + ((dst + G(x,y,z) + X[k] + T_i) <<< s)
#   %r10d = X[k_next]
#   %r11d = z' (copy of z for the next step)
#   %r12d = z' (copy of z for the next step)
# Each round2_step() takes about 5.4 clocks (11 instructions, 2.0 IPC)
sub round2_step
{
    my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
    $code .= " mov	%edx,		%r11d		/* (NEXT STEP) z' = %edx */\n" if ($pos == -1);
    $code .= " mov	%edx,		%r12d		/* (NEXT STEP) z' = %edx */\n" if ($pos == -1);
    $code .= <<EOF;
	not	%r11d				/* not z */
	and	$x,		%r12d		/* x & z */
	lea	$T_i($dst,%r10d),$dst		/* Const + dst + ... */
	and	$y,		%r11d		/* y & (not z) */
	mov	$k_next*4(%rsi),%r10d		/* (NEXT STEP) X[$k_next] */
	or	%r11d,		%r12d		/* (y & (not z)) | (x & z) */
	mov	$y,		%r11d		/* (NEXT STEP) z' = $y */
	add	%r12d,		$dst		/* dst += ... */
	mov	$y,		%r12d		/* (NEXT STEP) z' = $y */
	rol	\$$s,		$dst		/* dst <<< s */
	add	$x,		$dst		/* dst += x */
EOF
}

# round3_step() does:
#   dst = x + ((dst + H(x,y,z) + X[k] + T_i) <<< s)
#   %r10d = X[k_next]
#   %r11d = y' (copy of y for the next step)
# Each round3_step() takes about 4.2 clocks (8 instructions, 1.9 IPC)
{ my $round3_alter=0;
sub round3_step
{
    my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
    $code .= " mov	%ecx,		%r11d		/* (NEXT STEP) y' = %ecx */\n" if ($pos == -1);
    $code .= <<EOF;
	lea	$T_i($dst,%r10d),$dst		/* Const + dst + ... */
	xor	$z,		%r11d		/* z ^ ... */
	mov	$k_next*4(%rsi),%r10d		/* (NEXT STEP) X[$k_next] */
	xor	$x,		%r11d		/* x ^ ... */
	add	%r11d,		$dst		/* dst += ... */
EOF
    $code .= <<EOF if ($round3_alter);
	rol	\$$s,		$dst		/* dst <<< s */
	mov	$x,		%r11d		/* (NEXT STEP) y' = $x */
EOF
    $code .= <<EOF if (!$round3_alter);
	mov	$x,		%r11d		/* (NEXT STEP) y' = $x */
	rol	\$$s,		$dst		/* dst <<< s */
EOF
    $code .= <<EOF;
	add	$x,		$dst		/* dst += x */
EOF
    $round3_alter^=1;
}
}

# round4_step() does:
#   dst = x + ((dst + I(x,y,z) + X[k] + T_i) <<< s)
#   %r10d = X[k_next]
#   %r11d = not z' (copy of not z for the next step)
# Each round4_step() takes about 5.2 clocks (9 instructions, 1.7 IPC)
sub round4_step
{
    my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_;
    $code .= " mov	\$0xffffffff,	%r11d\n" if ($pos == -1);
    $code .= " xor	%edx,		%r11d		/* (NEXT STEP) not z' = not %edx*/\n"
    if ($pos == -1);
    $code .= <<EOF;
	lea	$T_i($dst,%r10d),$dst		/* Const + dst + ... */
	or	$x,		%r11d		/* x | ... */
	mov	$k_next*4(%rsi),%r10d		/* (NEXT STEP) X[$k_next] */
	xor	$y,		%r11d		/* y ^ ... */
	add	%r11d,		$dst		/* dst += ... */
	mov	\$0xffffffff,	%r11d
	rol	\$$s,		$dst		/* dst <<< s */
	xor	$y,		%r11d		/* (NEXT STEP) not z' = not $y */
	add	$x,		$dst		/* dst += x */
EOF
}

no warnings qw(uninitialized);
my $flavour = shift;
my $output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1; my $xlate;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";

open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

$code .= <<EOF;
.text
.align 16

.globl md5_block_asm_data_order
.type md5_block_asm_data_order,\@function,3
md5_block_asm_data_order:
.cfi_startproc
	push	%rbp
.cfi_push	%rbp
	push	%rbx
.cfi_push	%rbx
	push	%r12
.cfi_push	%r12
	push	%r14
.cfi_push	%r14
	push	%r15
.cfi_push	%r15
.Lprologue:

	# rdi = arg #1 (ctx, MD5_CTX pointer)
	# rsi = arg #2 (ptr, data pointer)
	# rdx = arg #3 (nbr, number of 16-word blocks to process)
	mov	%rdi,		%rbp	# rbp = ctx
	shl	\$6,		%rdx	# rdx = nbr in bytes
	lea	(%rsi,%rdx),	%rdi	# rdi = end
	mov	0*4(%rbp),	%eax	# eax = ctx->A
	mov	1*4(%rbp),	%ebx	# ebx = ctx->B
	mov	2*4(%rbp),	%ecx	# ecx = ctx->C
	mov	3*4(%rbp),	%edx	# edx = ctx->D
	# end is 'rdi'
	# ptr is 'rsi'
	# A is 'eax'
	# B is 'ebx'
	# C is 'ecx'
	# D is 'edx'

	cmp	%rdi,		%rsi		# cmp end with ptr
	je	.Lend				# jmp if ptr == end

	# BEGIN of loop over 16-word blocks
.Lloop:	# save old values of A, B, C, D
	mov	%eax,		%r8d
	mov	%ebx,		%r9d
	mov	%ecx,		%r14d
	mov	%edx,		%r15d
EOF
round1_step(-1,'%eax','%ebx','%ecx','%edx', '1','0xd76aa478', '7');
round1_step( 0,'%edx','%eax','%ebx','%ecx', '2','0xe8c7b756','12');
round1_step( 0,'%ecx','%edx','%eax','%ebx', '3','0x242070db','17');
round1_step( 0,'%ebx','%ecx','%edx','%eax', '4','0xc1bdceee','22');
round1_step( 0,'%eax','%ebx','%ecx','%edx', '5','0xf57c0faf', '7');
round1_step( 0,'%edx','%eax','%ebx','%ecx', '6','0x4787c62a','12');
round1_step( 0,'%ecx','%edx','%eax','%ebx', '7','0xa8304613','17');
round1_step( 0,'%ebx','%ecx','%edx','%eax', '8','0xfd469501','22');
round1_step( 0,'%eax','%ebx','%ecx','%edx', '9','0x698098d8', '7');
round1_step( 0,'%edx','%eax','%ebx','%ecx','10','0x8b44f7af','12');
round1_step( 0,'%ecx','%edx','%eax','%ebx','11','0xffff5bb1','17');
round1_step( 0,'%ebx','%ecx','%edx','%eax','12','0x895cd7be','22');
round1_step( 0,'%eax','%ebx','%ecx','%edx','13','0x6b901122', '7');
round1_step( 0,'%edx','%eax','%ebx','%ecx','14','0xfd987193','12');
round1_step( 0,'%ecx','%edx','%eax','%ebx','15','0xa679438e','17');
round1_step( 1,'%ebx','%ecx','%edx','%eax', '1','0x49b40821','22');

round2_step(-1,'%eax','%ebx','%ecx','%edx', '6','0xf61e2562', '5');
round2_step( 0,'%edx','%eax','%ebx','%ecx','11','0xc040b340', '9');
round2_step( 0,'%ecx','%edx','%eax','%ebx', '0','0x265e5a51','14');
round2_step( 0,'%ebx','%ecx','%edx','%eax', '5','0xe9b6c7aa','20');
round2_step( 0,'%eax','%ebx','%ecx','%edx','10','0xd62f105d', '5');
round2_step( 0,'%edx','%eax','%ebx','%ecx','15', '0x2441453', '9');
round2_step( 0,'%ecx','%edx','%eax','%ebx', '4','0xd8a1e681','14');
round2_step( 0,'%ebx','%ecx','%edx','%eax', '9','0xe7d3fbc8','20');
round2_step( 0,'%eax','%ebx','%ecx','%edx','14','0x21e1cde6', '5');
round2_step( 0,'%edx','%eax','%ebx','%ecx', '3','0xc33707d6', '9');
round2_step( 0,'%ecx','%edx','%eax','%ebx', '8','0xf4d50d87','14');
round2_step( 0,'%ebx','%ecx','%edx','%eax','13','0x455a14ed','20');
round2_step( 0,'%eax','%ebx','%ecx','%edx', '2','0xa9e3e905', '5');
round2_step( 0,'%edx','%eax','%ebx','%ecx', '7','0xfcefa3f8', '9');
round2_step( 0,'%ecx','%edx','%eax','%ebx','12','0x676f02d9','14');
round2_step( 1,'%ebx','%ecx','%edx','%eax', '5','0x8d2a4c8a','20');

round3_step(-1,'%eax','%ebx','%ecx','%edx', '8','0xfffa3942', '4');
round3_step( 0,'%edx','%eax','%ebx','%ecx','11','0x8771f681','11');
round3_step( 0,'%ecx','%edx','%eax','%ebx','14','0x6d9d6122','16');
round3_step( 0,'%ebx','%ecx','%edx','%eax', '1','0xfde5380c','23');
round3_step( 0,'%eax','%ebx','%ecx','%edx', '4','0xa4beea44', '4');
round3_step( 0,'%edx','%eax','%ebx','%ecx', '7','0x4bdecfa9','11');
round3_step( 0,'%ecx','%edx','%eax','%ebx','10','0xf6bb4b60','16');
round3_step( 0,'%ebx','%ecx','%edx','%eax','13','0xbebfbc70','23');
round3_step( 0,'%eax','%ebx','%ecx','%edx', '0','0x289b7ec6', '4');
round3_step( 0,'%edx','%eax','%ebx','%ecx', '3','0xeaa127fa','11');
round3_step( 0,'%ecx','%edx','%eax','%ebx', '6','0xd4ef3085','16');
round3_step( 0,'%ebx','%ecx','%edx','%eax', '9', '0x4881d05','23');
round3_step( 0,'%eax','%ebx','%ecx','%edx','12','0xd9d4d039', '4');
round3_step( 0,'%edx','%eax','%ebx','%ecx','15','0xe6db99e5','11');
round3_step( 0,'%ecx','%edx','%eax','%ebx', '2','0x1fa27cf8','16');
round3_step( 1,'%ebx','%ecx','%edx','%eax', '0','0xc4ac5665','23');

round4_step(-1,'%eax','%ebx','%ecx','%edx', '7','0xf4292244', '6');
round4_step( 0,'%edx','%eax','%ebx','%ecx','14','0x432aff97','10');
round4_step( 0,'%ecx','%edx','%eax','%ebx', '5','0xab9423a7','15');
round4_step( 0,'%ebx','%ecx','%edx','%eax','12','0xfc93a039','21');
round4_step( 0,'%eax','%ebx','%ecx','%edx', '3','0x655b59c3', '6');
round4_step( 0,'%edx','%eax','%ebx','%ecx','10','0x8f0ccc92','10');
round4_step( 0,'%ecx','%edx','%eax','%ebx', '1','0xffeff47d','15');
round4_step( 0,'%ebx','%ecx','%edx','%eax', '8','0x85845dd1','21');
round4_step( 0,'%eax','%ebx','%ecx','%edx','15','0x6fa87e4f', '6');
round4_step( 0,'%edx','%eax','%ebx','%ecx', '6','0xfe2ce6e0','10');
round4_step( 0,'%ecx','%edx','%eax','%ebx','13','0xa3014314','15');
round4_step( 0,'%ebx','%ecx','%edx','%eax', '4','0x4e0811a1','21');
round4_step( 0,'%eax','%ebx','%ecx','%edx','11','0xf7537e82', '6');
round4_step( 0,'%edx','%eax','%ebx','%ecx', '2','0xbd3af235','10');
round4_step( 0,'%ecx','%edx','%eax','%ebx', '9','0x2ad7d2bb','15');
round4_step( 1,'%ebx','%ecx','%edx','%eax', '0','0xeb86d391','21');
$code .= <<EOF;
	# add old values of A, B, C, D
	add	%r8d,	%eax
	add	%r9d,	%ebx
	add	%r14d,	%ecx
	add	%r15d,	%edx

	# loop control
	add	\$64,		%rsi		# ptr += 64
	cmp	%rdi,		%rsi		# cmp end with ptr
	jb	.Lloop				# jmp if ptr < end
	# END of loop over 16-word blocks

.Lend:
	mov	%eax,		0*4(%rbp)	# ctx->A = A
	mov	%ebx,		1*4(%rbp)	# ctx->B = B
	mov	%ecx,		2*4(%rbp)	# ctx->C = C
	mov	%edx,		3*4(%rbp)	# ctx->D = D

	mov	(%rsp),%r15
.cfi_restore	%r15
	mov	8(%rsp),%r14
.cfi_restore	%r14
	mov	16(%rsp),%r12
.cfi_restore	%r12
	mov	24(%rsp),%rbx
.cfi_restore	%rbx
	mov	32(%rsp),%rbp
.cfi_restore	%rbp
	add	\$40,%rsp
.cfi_adjust_cfa_offset	-40
.Lepilogue:
	ret
.cfi_endproc
.size md5_block_asm_data_order,.-md5_block_asm_data_order
EOF

# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
if ($win64) {
my $rec="%rcx";
my $frame="%rdx";
my $context="%r8";
my $disp="%r9";

$code.=<<___;
.extern	__imp_RtlVirtualUnwind
.type	se_handler,\@abi-omnipotent
.align	16
se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	lea	.Lprologue(%rip),%r10
	cmp	%r10,%rbx		# context->Rip<.Lprologue
	jb	.Lin_prologue

	mov	152($context),%rax	# pull context->Rsp

	lea	.Lepilogue(%rip),%r10
	cmp	%r10,%rbx		# context->Rip>=.Lepilogue
	jae	.Lin_prologue

	lea	40(%rax),%rax

	mov	-8(%rax),%rbp
	mov	-16(%rax),%rbx
	mov	-24(%rax),%r12
	mov	-32(%rax),%r14
	mov	-40(%rax),%r15
	mov	%rbx,144($context)	# restore context->Rbx
	mov	%rbp,160($context)	# restore context->Rbp
	mov	%r12,216($context)	# restore context->R12
	mov	%r14,232($context)	# restore context->R14
	mov	%r15,240($context)	# restore context->R15

.Lin_prologue:
	mov	8(%rax),%rdi
	mov	16(%rax),%rsi
	mov	%rax,152($context)	# restore context->Rsp
	mov	%rsi,168($context)	# restore context->Rsi
	mov	%rdi,176($context)	# restore context->Rdi

	mov	40($disp),%rdi		# disp->ContextRecord
	mov	$context,%rsi		# context
	mov	\$154,%ecx		# sizeof(CONTEXT)
	.long	0xa548f3fc		# cld; rep movsq

	mov	$disp,%rsi
	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
	mov	0(%rsi),%r8		# arg3, disp->ControlPc
	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
	mov	40(%rsi),%r10		# disp->ContextRecord
	lea	56(%rsi),%r11		# &disp->HandlerData
	lea	24(%rsi),%r12		# &disp->EstablisherFrame
	mov	%r10,32(%rsp)		# arg5
	mov	%r11,40(%rsp)		# arg6
	mov	%r12,48(%rsp)		# arg7
	mov	%rcx,56(%rsp)		# arg8, (NULL)
	call	*__imp_RtlVirtualUnwind(%rip)

	mov	\$1,%eax		# ExceptionContinueSearch
	add	\$64,%rsp
	popfq
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx
	pop	%rdi
	pop	%rsi
	ret
.size	se_handler,.-se_handler

.section	.pdata
.align	4
	.rva	.LSEH_begin_md5_block_asm_data_order
	.rva	.LSEH_end_md5_block_asm_data_order
	.rva	.LSEH_info_md5_block_asm_data_order

.section	.xdata
.align	8
.LSEH_info_md5_block_asm_data_order:
	.byte	9,0,0,0
	.rva	se_handler
___
}

print $code;

close STDOUT or die "error closing STDOUT: $!";