#! /usr/bin/env perl
# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# January 2007.

# Montgomery multiplication for ARMv4.
#
# Performance improvement naturally varies among CPU implementations
# and compilers. The code was observed to provide +65-35% improvement
# [depending on key length, less for longer keys] on ARM920T, and
# +115-80% on Intel IXP425. This is compared to pre-bn_mul_mont code
# base and compiler generated code with in-lined umull and even umlal
# instructions. The latter means that this code didn't really have an
# "advantage" of utilizing some "secret" instruction.
#
# The code is interoperable with Thumb ISA and is rather compact, less
# than 1/2KB. Windows CE port would be trivial, as it's exclusively
# about decorations, ABI and instruction syntax are identical.

# November 2013
#
# Add NEON code path, which handles lengths divisible by 8. RSA/DSA
# performance improvement on Cortex-A8 is ~45-100% depending on key
# length, more for longer keys. On Cortex-A15 the span is ~10-105%.
# On Snapdragon S4 improvement was measured to vary from ~70% to
# incredible ~380%, yes, 4.8x faster, for RSA4096 sign. But this is
# rather because original integer-only code seems to perform
# suboptimally on S4. Situation on Cortex-A9 is unfortunately
# different. It's being looked into, but the trouble is that
# performance for vectors longer than 256 bits is actually couple
# of percent worse than for integer-only code. The code is chosen
# for execution on all NEON-capable processors, because gain on
# others outweighs the marginal loss on Cortex-A9.

# September 2015
#
# Align Cortex-A9 performance with November 2013 improvements, i.e.
# NEON code is now ~20-105% faster than integer-only one on this
# processor. But this optimization further improved performance even
# on other processors: NEON code path is ~45-180% faster than original
# integer-only on Cortex-A8, ~10-210% on Cortex-A15, ~70-450% on
# Snapdragon S4.

$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
    open STDOUT,">$output";
}

$num="r0";	# starts as num argument, but holds &tp[num-1]
$ap="r1";
$bp="r2"; $bi="r2"; $rp="r2";
$np="r3";
$tp="r4";
$aj="r5";
$nj="r6";
$tj="r7";
$n0="r8";
###########	# r9 is reserved by ELF as platform specific, e.g. TLS pointer
$alo="r10";	# sl, gcc uses it to keep @GOT
$ahi="r11";	# fp
$nlo="r12";	# ip
###########	# r13 is stack pointer
$nhi="r14";	# lr
###########	# r15 is program counter

#### argument block layout relative to &tp[num-1], a.k.a. $num
$_rp="$num,#12*4";
# ap permanently resides in r1
$_bp="$num,#13*4";
# np permanently resides in r3
$_n0="$num,#14*4";
$_num="$num,#15*4";	$_bpend=$_num;

$code=<<___;
#include "arm_arch.h"

.text
#if defined(__thumb2__)
.syntax	unified
.thumb
#else
.code	32
#endif

#if __ARM_MAX_ARCH__>=7
.align	5
.LOPENSSL_armcap:
.word	OPENSSL_armcap_P-.Lbn_mul_mont
#endif

.global	bn_mul_mont
.type	bn_mul_mont,%function

.align	5
bn_mul_mont:
.Lbn_mul_mont:
	ldr	ip,[sp,#4]		@ load num
	stmdb	sp!,{r0,r2}		@ sp points at argument block
#if __ARM_MAX_ARCH__>=7
	tst	ip,#7
	bne	.Lialu
	adr	r0,.Lbn_mul_mont
	ldr	r2,.LOPENSSL_armcap
	ldr	r0,[r0,r2]
#ifdef	__APPLE__
	ldr	r0,[r0]
#endif
	tst	r0,#ARMV7_NEON		@ NEON available?
	ldmia	sp, {r0,r2}
	beq	.Lialu
	add	sp,sp,#8
	b	bn_mul8x_mont_neon
.align	4
.Lialu:
#endif
	cmp	ip,#2
	mov	$num,ip			@ load num
#ifdef	__thumb2__
	ittt	lt
#endif
	movlt	r0,#0
	addlt	sp,sp,#2*4
	blt	.Labrt

	stmdb	sp!,{r4-r12,lr}		@ save 10 registers

	mov	$num,$num,lsl#2		@ rescale $num for byte count
	sub	sp,sp,$num		@ alloca(4*num)
	sub	sp,sp,#4		@ +extra dword
	sub	$num,$num,#4		@ "num=num-1"
	add	$tp,$bp,$num		@ &bp[num-1]

	add	$num,sp,$num		@ $num to point at &tp[num-1]
	ldr	$n0,[$_n0]		@ &n0
	ldr	$bi,[$bp]		@ bp[0]
	ldr	$aj,[$ap],#4		@ ap[0],ap++
	ldr	$nj,[$np],#4		@ np[0],np++
	ldr	$n0,[$n0]		@ *n0
	str	$tp,[$_bpend]		@ save &bp[num]

	umull	$alo,$ahi,$aj,$bi	@ ap[0]*bp[0]
	str	$n0,[$_n0]		@ save n0 value
	mul	$n0,$alo,$n0		@ "tp[0]"*n0
	mov	$nlo,#0
	umlal	$alo,$nlo,$nj,$n0	@ np[0]*n0+"t[0]"
	mov	$tp,sp

.L1st:
	ldr	$aj,[$ap],#4		@ ap[j],ap++
	mov	$alo,$ahi
	ldr	$nj,[$np],#4		@ np[j],np++
	mov	$ahi,#0
	umlal	$alo,$ahi,$aj,$bi	@ ap[j]*bp[0]
	mov	$nhi,#0
	umlal	$nlo,$nhi,$nj,$n0	@ np[j]*n0
	adds	$nlo,$nlo,$alo
	str	$nlo,[$tp],#4		@ tp[j-1]=,tp++
	adc	$nlo,$nhi,#0
	cmp	$tp,$num
	bne	.L1st

	adds	$nlo,$nlo,$ahi
	ldr	$tp,[$_bp]		@ restore bp
	mov	$nhi,#0
	ldr	$n0,[$_n0]		@ restore n0
	adc	$nhi,$nhi,#0
	str	$nlo,[$num]		@ tp[num-1]=
	mov	$tj,sp
	str	$nhi,[$num,#4]		@ tp[num]=

.Louter:
	sub	$tj,$num,$tj		@ "original" $num-1 value
	sub	$ap,$ap,$tj		@ "rewind" ap to &ap[1]
	ldr	$bi,[$tp,#4]!		@ *(++bp)
	sub	$np,$np,$tj		@ "rewind" np to &np[1]
	ldr	$aj,[$ap,#-4]		@ ap[0]
	ldr	$alo,[sp]		@ tp[0]
	ldr	$nj,[$np,#-4]		@ np[0]
	ldr	$tj,[sp,#4]		@ tp[1]

	mov	$ahi,#0
	umlal	$alo,$ahi,$aj,$bi	@ ap[0]*bp[i]+tp[0]
	str	$tp,[$_bp]		@ save bp
	mul	$n0,$alo,$n0
	mov	$nlo,#0
	umlal	$alo,$nlo,$nj,$n0	@ np[0]*n0+"tp[0]"
	mov	$tp,sp

.Linner:
	ldr	$aj,[$ap],#4		@ ap[j],ap++
	adds	$alo,$ahi,$tj		@ +=tp[j]
	ldr	$nj,[$np],#4		@ np[j],np++
	mov	$ahi,#0
	umlal	$alo,$ahi,$aj,$bi	@ ap[j]*bp[i]
	mov	$nhi,#0
	umlal	$nlo,$nhi,$nj,$n0	@ np[j]*n0
	adc	$ahi,$ahi,#0
	ldr	$tj,[$tp,#8]		@ tp[j+1]
	adds	$nlo,$nlo,$alo
	str	$nlo,[$tp],#4		@ tp[j-1]=,tp++
	adc	$nlo,$nhi,#0
	cmp	$tp,$num
	bne	.Linner

	adds	$nlo,$nlo,$ahi
	mov	$nhi,#0
	ldr	$tp,[$_bp]		@ restore bp
	adc	$nhi,$nhi,#0
	ldr	$n0,[$_n0]		@ restore n0
	adds	$nlo,$nlo,$tj
	ldr	$tj,[$_bpend]		@ restore &bp[num]
	adc	$nhi,$nhi,#0
	str	$nlo,[$num]		@ tp[num-1]=
	str	$nhi,[$num,#4]		@ tp[num]=

	cmp	$tp,$tj
#ifdef	__thumb2__
	itt	ne
#endif
	movne	$tj,sp
	bne	.Louter

	ldr	$rp,[$_rp]		@ pull rp
	mov	$aj,sp
	add	$num,$num,#4		@ $num to point at &tp[num]
	sub	$aj,$num,$aj		@ "original" num value
	mov	$tp,sp			@ "rewind" $tp
	mov	$ap,$tp			@ "borrow" $ap
	sub	$np,$np,$aj		@ "rewind" $np to &np[0]

	subs	$tj,$tj,$tj		@ "clear" carry flag
.Lsub:	ldr	$tj,[$tp],#4
	ldr	$nj,[$np],#4
	sbcs	$tj,$tj,$nj		@ tp[j]-np[j]
	str	$tj,[$rp],#4		@ rp[j]=
	teq	$tp,$num		@ preserve carry
	bne	.Lsub
	sbcs	$nhi,$nhi,#0		@ upmost carry
	mov	$tp,sp			@ "rewind" $tp
	sub	$rp,$rp,$aj		@ "rewind" $rp

.Lcopy:	ldr	$tj,[$tp]		@ conditional copy
	ldr	$aj,[$rp]
	str	sp,[$tp],#4		@ zap tp
#ifdef	__thumb2__
	it	cc
#endif
	movcc	$aj,$tj
	str	$aj,[$rp],#4
	teq	$tp,$num		@ preserve carry
	bne	.Lcopy

	mov	sp,$num
	add	sp,sp,#4		@ skip over tp[num+1]
	ldmia	sp!,{r4-r12,lr}		@ restore registers
	add	sp,sp,#2*4		@ skip over {r0,r2}
	mov	r0,#1
.Labrt:
#if __ARM_ARCH__>=5
	ret				@ bx lr
#else
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	bn_mul_mont,.-bn_mul_mont
___
{
my ($A0,$A1,$A2,$A3)=map("d$_",(0..3));
my ($N0,$N1,$N2,$N3)=map("d$_",(4..7));
my ($Z,$Temp)=("q4","q5");
my @ACC=map("q$_",(6..13));
my ($Bi,$Ni,$M0)=map("d$_",(28..31));
my $zero="$Z#lo";
my $temp="$Temp#lo";

my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("r$_",(0..5));
my ($tinptr,$toutptr,$inner,$outer,$bnptr)=map("r$_",(6..11));

$code.=<<___;
#if __ARM_MAX_ARCH__>=7
.arch	armv7-a
.fpu	neon

.type	bn_mul8x_mont_neon,%function
.align	5
bn_mul8x_mont_neon:
	mov	ip,sp
	stmdb	sp!,{r4-r11}
	vstmdb	sp!,{d8-d15}		@ ABI specification says so
	ldmia	ip,{r4-r5}		@ load rest of parameter block
	mov	ip,sp

	cmp	$num,#8
	bhi	.LNEON_8n

	@ special case for $num==8, everything is in register bank...

	vld1.32		{${Bi}[0]}, [$bptr,:32]!
	veor		$zero,$zero,$zero
	sub		$toutptr,sp,$num,lsl#4
	vld1.32		{$A0-$A3},  [$aptr]!		@ can't specify :32 :-(
	and		$toutptr,$toutptr,#-64
	vld1.32		{${M0}[0]}, [$n0,:32]
	mov		sp,$toutptr			@ alloca
	vzip.16		$Bi,$zero

	vmull.u32	@ACC[0],$Bi,${A0}[0]
	vmull.u32	@ACC[1],$Bi,${A0}[1]
	vmull.u32	@ACC[2],$Bi,${A1}[0]
	vshl.i64	$Ni,@ACC[0]#hi,#16
	vmull.u32	@ACC[3],$Bi,${A1}[1]

	vadd.u64	$Ni,$Ni,@ACC[0]#lo
	veor		$zero,$zero,$zero
	vmul.u32	$Ni,$Ni,$M0

	vmull.u32	@ACC[4],$Bi,${A2}[0]
	 vld1.32	{$N0-$N3}, [$nptr]!
	vmull.u32	@ACC[5],$Bi,${A2}[1]
	vmull.u32	@ACC[6],$Bi,${A3}[0]
	vzip.16		$Ni,$zero
	vmull.u32	@ACC[7],$Bi,${A3}[1]

	vmlal.u32	@ACC[0],$Ni,${N0}[0]
	sub		$outer,$num,#1
	vmlal.u32	@ACC[1],$Ni,${N0}[1]
	vmlal.u32	@ACC[2],$Ni,${N1}[0]
	vmlal.u32	@ACC[3],$Ni,${N1}[1]

	vmlal.u32	@ACC[4],$Ni,${N2}[0]
	vmov		$Temp,@ACC[0]
	vmlal.u32	@ACC[5],$Ni,${N2}[1]
	vmov		@ACC[0],@ACC[1]
	vmlal.u32	@ACC[6],$Ni,${N3}[0]
	vmov		@ACC[1],@ACC[2]
	vmlal.u32	@ACC[7],$Ni,${N3}[1]
	vmov		@ACC[2],@ACC[3]
	vmov		@ACC[3],@ACC[4]
	vshr.u64	$temp,$temp,#16
	vmov		@ACC[4],@ACC[5]
	vmov		@ACC[5],@ACC[6]
	vadd.u64	$temp,$temp,$Temp#hi
	vmov		@ACC[6],@ACC[7]
	veor		@ACC[7],@ACC[7]
	vshr.u64	$temp,$temp,#16

	b	.LNEON_outer8

.align	4
.LNEON_outer8:
	vld1.32		{${Bi}[0]}, [$bptr,:32]!
	veor		$zero,$zero,$zero
	vzip.16		$Bi,$zero
	vadd.u64	@ACC[0]#lo,@ACC[0]#lo,$temp

	vmlal.u32	@ACC[0],$Bi,${A0}[0]
	vmlal.u32	@ACC[1],$Bi,${A0}[1]
	vmlal.u32	@ACC[2],$Bi,${A1}[0]
	vshl.i64	$Ni,@ACC[0]#hi,#16
	vmlal.u32	@ACC[3],$Bi,${A1}[1]

	vadd.u64	$Ni,$Ni,@ACC[0]#lo
	veor		$zero,$zero,$zero
	subs		$outer,$outer,#1
	vmul.u32	$Ni,$Ni,$M0

	vmlal.u32	@ACC[4],$Bi,${A2}[0]
	vmlal.u32	@ACC[5],$Bi,${A2}[1]
	vmlal.u32	@ACC[6],$Bi,${A3}[0]
	vzip.16		$Ni,$zero
	vmlal.u32	@ACC[7],$Bi,${A3}[1]

	vmlal.u32	@ACC[0],$Ni,${N0}[0]
	vmlal.u32	@ACC[1],$Ni,${N0}[1]
	vmlal.u32	@ACC[2],$Ni,${N1}[0]
	vmlal.u32	@ACC[3],$Ni,${N1}[1]

	vmlal.u32	@ACC[4],$Ni,${N2}[0]
	vmov		$Temp,@ACC[0]
	vmlal.u32	@ACC[5],$Ni,${N2}[1]
	vmov		@ACC[0],@ACC[1]
	vmlal.u32	@ACC[6],$Ni,${N3}[0]
	vmov		@ACC[1],@ACC[2]
	vmlal.u32	@ACC[7],$Ni,${N3}[1]
	vmov		@ACC[2],@ACC[3]
	vmov		@ACC[3],@ACC[4]
	vshr.u64	$temp,$temp,#16
	vmov		@ACC[4],@ACC[5]
	vmov		@ACC[5],@ACC[6]
	vadd.u64	$temp,$temp,$Temp#hi
	vmov		@ACC[6],@ACC[7]
	veor		@ACC[7],@ACC[7]
	vshr.u64	$temp,$temp,#16

	bne	.LNEON_outer8

	vadd.u64	@ACC[0]#lo,@ACC[0]#lo,$temp
	mov		$toutptr,sp
	vshr.u64	$temp,@ACC[0]#lo,#16
	mov		$inner,$num
	vadd.u64	@ACC[0]#hi,@ACC[0]#hi,$temp
	add		$tinptr,sp,#96
	vshr.u64	$temp,@ACC[0]#hi,#16
	vzip.16		@ACC[0]#lo,@ACC[0]#hi

	b	.LNEON_tail_entry

.align	4
.LNEON_8n:
	veor		@ACC[0],@ACC[0],@ACC[0]
	 sub		$toutptr,sp,#128
	veor		@ACC[1],@ACC[1],@ACC[1]
	 sub		$toutptr,$toutptr,$num,lsl#4
	veor		@ACC[2],@ACC[2],@ACC[2]
	 and		$toutptr,$toutptr,#-64
	veor		@ACC[3],@ACC[3],@ACC[3]
	 mov		sp,$toutptr			@ alloca
	veor		@ACC[4],@ACC[4],@ACC[4]
	 add		$toutptr,$toutptr,#256
	veor		@ACC[5],@ACC[5],@ACC[5]
	 sub		$inner,$num,#8
	veor		@ACC[6],@ACC[6],@ACC[6]
	veor		@ACC[7],@ACC[7],@ACC[7]

.LNEON_8n_init:
	vst1.64		{@ACC[0]-@ACC[1]},[$toutptr,:256]!
	subs		$inner,$inner,#8
	vst1.64		{@ACC[2]-@ACC[3]},[$toutptr,:256]!
	vst1.64		{@ACC[4]-@ACC[5]},[$toutptr,:256]!
	vst1.64		{@ACC[6]-@ACC[7]},[$toutptr,:256]!
	bne		.LNEON_8n_init

	add		$tinptr,sp,#256
	vld1.32		{$A0-$A3},[$aptr]!
	add		$bnptr,sp,#8
	vld1.32		{${M0}[0]},[$n0,:32]
	mov		$outer,$num
	b		.LNEON_8n_outer

.align	4
.LNEON_8n_outer:
	vld1.32		{${Bi}[0]},[$bptr,:32]!	@ *b++
	veor		$zero,$zero,$zero
	vzip.16		$Bi,$zero
	add		$toutptr,sp,#128
	vld1.32		{$N0-$N3},[$nptr]!

	vmlal.u32	@ACC[0],$Bi,${A0}[0]
	vmlal.u32	@ACC[1],$Bi,${A0}[1]
	 veor		$zero,$zero,$zero
	vmlal.u32	@ACC[2],$Bi,${A1}[0]
	 vshl.i64	$Ni,@ACC[0]#hi,#16
	vmlal.u32	@ACC[3],$Bi,${A1}[1]
	 vadd.u64	$Ni,$Ni,@ACC[0]#lo
	vmlal.u32	@ACC[4],$Bi,${A2}[0]
	 vmul.u32	$Ni,$Ni,$M0
	vmlal.u32	@ACC[5],$Bi,${A2}[1]
	vst1.32		{$Bi},[sp,:64]		@ put aside smashed b[8*i+0]
	vmlal.u32	@ACC[6],$Bi,${A3}[0]
	 vzip.16	$Ni,$zero
	vmlal.u32	@ACC[7],$Bi,${A3}[1]
___
for ($i=0; $i<7;) {
$code.=<<___;
	vld1.32		{${Bi}[0]},[$bptr,:32]!	@ *b++
	vmlal.u32	@ACC[0],$Ni,${N0}[0]
	veor		$temp,$temp,$temp
	vmlal.u32	@ACC[1],$Ni,${N0}[1]
	vzip.16		$Bi,$temp
	vmlal.u32	@ACC[2],$Ni,${N1}[0]
	 vshr.u64	@ACC[0]#lo,@ACC[0]#lo,#16
	vmlal.u32	@ACC[3],$Ni,${N1}[1]
	vmlal.u32	@ACC[4],$Ni,${N2}[0]
	 vadd.u64	@ACC[0]#lo,@ACC[0]#lo,@ACC[0]#hi
	vmlal.u32	@ACC[5],$Ni,${N2}[1]
	 vshr.u64	@ACC[0]#lo,@ACC[0]#lo,#16
	vmlal.u32	@ACC[6],$Ni,${N3}[0]
	vmlal.u32	@ACC[7],$Ni,${N3}[1]
	 vadd.u64	@ACC[1]#lo,@ACC[1]#lo,@ACC[0]#lo
	vst1.32		{$Ni},[$bnptr,:64]!	@ put aside smashed m[8*i+$i]
___
	push(@ACC,shift(@ACC));	$i++;
$code.=<<___;
	vmlal.u32	@ACC[0],$Bi,${A0}[0]
	vld1.64		{@ACC[7]},[$tinptr,:128]!
	vmlal.u32	@ACC[1],$Bi,${A0}[1]
	 veor		$zero,$zero,$zero
	vmlal.u32	@ACC[2],$Bi,${A1}[0]
	 vshl.i64	$Ni,@ACC[0]#hi,#16
	vmlal.u32	@ACC[3],$Bi,${A1}[1]
	 vadd.u64	$Ni,$Ni,@ACC[0]#lo
	vmlal.u32	@ACC[4],$Bi,${A2}[0]
	 vmul.u32	$Ni,$Ni,$M0
	vmlal.u32	@ACC[5],$Bi,${A2}[1]
	vst1.32		{$Bi},[$bnptr,:64]!	@ put aside smashed b[8*i+$i]
	vmlal.u32	@ACC[6],$Bi,${A3}[0]
	 vzip.16	$Ni,$zero
	vmlal.u32	@ACC[7],$Bi,${A3}[1]
___
}
$code.=<<___;
	vld1.32		{$Bi},[sp,:64]		@ pull smashed b[8*i+0]
	vmlal.u32	@ACC[0],$Ni,${N0}[0]
	vld1.32		{$A0-$A3},[$aptr]!
	vmlal.u32	@ACC[1],$Ni,${N0}[1]
	vmlal.u32	@ACC[2],$Ni,${N1}[0]
	 vshr.u64	@ACC[0]#lo,@ACC[0]#lo,#16
	vmlal.u32	@ACC[3],$Ni,${N1}[1]
	vmlal.u32	@ACC[4],$Ni,${N2}[0]
	 vadd.u64	@ACC[0]#lo,@ACC[0]#lo,@ACC[0]#hi
	vmlal.u32	@ACC[5],$Ni,${N2}[1]
	 vshr.u64	@ACC[0]#lo,@ACC[0]#lo,#16
	vmlal.u32	@ACC[6],$Ni,${N3}[0]
	vmlal.u32	@ACC[7],$Ni,${N3}[1]
	 vadd.u64	@ACC[1]#lo,@ACC[1]#lo,@ACC[0]#lo
	vst1.32		{$Ni},[$bnptr,:64]	@ put aside smashed m[8*i+$i]
	add		$bnptr,sp,#8		@ rewind
___
	push(@ACC,shift(@ACC));
$code.=<<___;
	sub		$inner,$num,#8
	b		.LNEON_8n_inner

.align	4
.LNEON_8n_inner:
	subs		$inner,$inner,#8
	vmlal.u32	@ACC[0],$Bi,${A0}[0]
	vld1.64		{@ACC[7]},[$tinptr,:128]
	vmlal.u32	@ACC[1],$Bi,${A0}[1]
	vld1.32		{$Ni},[$bnptr,:64]!	@ pull smashed m[8*i+0]
	vmlal.u32	@ACC[2],$Bi,${A1}[0]
	vld1.32		{$N0-$N3},[$nptr]!
	vmlal.u32	@ACC[3],$Bi,${A1}[1]
	it		ne
	addne		$tinptr,$tinptr,#16	@ don't advance in last iteration
	vmlal.u32	@ACC[4],$Bi,${A2}[0]
	vmlal.u32	@ACC[5],$Bi,${A2}[1]
	vmlal.u32	@ACC[6],$Bi,${A3}[0]
	vmlal.u32	@ACC[7],$Bi,${A3}[1]
___
for ($i=1; $i<8; $i++) {
$code.=<<___;
	vld1.32		{$Bi},[$bnptr,:64]!	@ pull smashed b[8*i+$i]
	vmlal.u32	@ACC[0],$Ni,${N0}[0]
	vmlal.u32	@ACC[1],$Ni,${N0}[1]
	vmlal.u32	@ACC[2],$Ni,${N1}[0]
	vmlal.u32	@ACC[3],$Ni,${N1}[1]
	vmlal.u32	@ACC[4],$Ni,${N2}[0]
	vmlal.u32	@ACC[5],$Ni,${N2}[1]
	vmlal.u32	@ACC[6],$Ni,${N3}[0]
	vmlal.u32	@ACC[7],$Ni,${N3}[1]
	vst1.64		{@ACC[0]},[$toutptr,:128]!
___
	push(@ACC,shift(@ACC));
$code.=<<___;
	vmlal.u32	@ACC[0],$Bi,${A0}[0]
	vld1.64		{@ACC[7]},[$tinptr,:128]
	vmlal.u32	@ACC[1],$Bi,${A0}[1]
	vld1.32		{$Ni},[$bnptr,:64]!	@ pull smashed m[8*i+$i]
	vmlal.u32	@ACC[2],$Bi,${A1}[0]
	it		ne
	addne		$tinptr,$tinptr,#16	@ don't advance in last iteration
	vmlal.u32	@ACC[3],$Bi,${A1}[1]
	vmlal.u32	@ACC[4],$Bi,${A2}[0]
	vmlal.u32	@ACC[5],$Bi,${A2}[1]
	vmlal.u32	@ACC[6],$Bi,${A3}[0]
	vmlal.u32	@ACC[7],$Bi,${A3}[1]
___
}
$code.=<<___;
	it		eq
	subeq		$aptr,$aptr,$num,lsl#2	@ rewind
	vmlal.u32	@ACC[0],$Ni,${N0}[0]
	vld1.32		{$Bi},[sp,:64]		@ pull smashed b[8*i+0]
	vmlal.u32	@ACC[1],$Ni,${N0}[1]
	vld1.32		{$A0-$A3},[$aptr]!
	vmlal.u32	@ACC[2],$Ni,${N1}[0]
	add		$bnptr,sp,#8		@ rewind
	vmlal.u32	@ACC[3],$Ni,${N1}[1]
	vmlal.u32	@ACC[4],$Ni,${N2}[0]
	vmlal.u32	@ACC[5],$Ni,${N2}[1]
	vmlal.u32	@ACC[6],$Ni,${N3}[0]
	vst1.64		{@ACC[0]},[$toutptr,:128]!
	vmlal.u32	@ACC[7],$Ni,${N3}[1]

	bne		.LNEON_8n_inner
___
	push(@ACC,shift(@ACC));
$code.=<<___;
	add		$tinptr,sp,#128
	vst1.64		{@ACC[0]-@ACC[1]},[$toutptr,:256]!
	veor		q2,q2,q2		@ $N0-$N1
	vst1.64		{@ACC[2]-@ACC[3]},[$toutptr,:256]!
	veor		q3,q3,q3		@ $N2-$N3
	vst1.64		{@ACC[4]-@ACC[5]},[$toutptr,:256]!
	vst1.64		{@ACC[6]},[$toutptr,:128]

	subs		$outer,$outer,#8
	vld1.64		{@ACC[0]-@ACC[1]},[$tinptr,:256]!
	vld1.64		{@ACC[2]-@ACC[3]},[$tinptr,:256]!
	vld1.64		{@ACC[4]-@ACC[5]},[$tinptr,:256]!
	vld1.64		{@ACC[6]-@ACC[7]},[$tinptr,:256]!

	itt		ne
	subne		$nptr,$nptr,$num,lsl#2	@ rewind
	bne		.LNEON_8n_outer

	add		$toutptr,sp,#128
	vst1.64		{q2-q3}, [sp,:256]!	@ start wiping stack frame
	vshr.u64	$temp,@ACC[0]#lo,#16
	vst1.64		{q2-q3},[sp,:256]!
	vadd.u64	@ACC[0]#hi,@ACC[0]#hi,$temp
	vst1.64		{q2-q3}, [sp,:256]!
	vshr.u64	$temp,@ACC[0]#hi,#16
	vst1.64		{q2-q3}, [sp,:256]!
	vzip.16		@ACC[0]#lo,@ACC[0]#hi

	mov		$inner,$num
	b		.LNEON_tail_entry

.align	4
.LNEON_tail:
	vadd.u64	@ACC[0]#lo,@ACC[0]#lo,$temp
	vshr.u64	$temp,@ACC[0]#lo,#16
	vld1.64		{@ACC[2]-@ACC[3]}, [$tinptr, :256]!
	vadd.u64	@ACC[0]#hi,@ACC[0]#hi,$temp
	vld1.64		{@ACC[4]-@ACC[5]}, [$tinptr, :256]!
	vshr.u64	$temp,@ACC[0]#hi,#16
	vld1.64		{@ACC[6]-@ACC[7]}, [$tinptr, :256]!
	vzip.16		@ACC[0]#lo,@ACC[0]#hi

.LNEON_tail_entry:
___
for ($i=1; $i<8; $i++) {
$code.=<<___;
	vadd.u64	@ACC[1]#lo,@ACC[1]#lo,$temp
	vst1.32		{@ACC[0]#lo[0]}, [$toutptr, :32]!
	vshr.u64	$temp,@ACC[1]#lo,#16
	vadd.u64	@ACC[1]#hi,@ACC[1]#hi,$temp
	vshr.u64	$temp,@ACC[1]#hi,#16
	vzip.16		@ACC[1]#lo,@ACC[1]#hi
___
	push(@ACC,shift(@ACC));
}
	push(@ACC,shift(@ACC));
$code.=<<___;
	vld1.64		{@ACC[0]-@ACC[1]}, [$tinptr, :256]!
	subs		$inner,$inner,#8
	vst1.32		{@ACC[7]#lo[0]},   [$toutptr, :32]!
	bne	.LNEON_tail

	vst1.32	{${temp}[0]}, [$toutptr, :32]		@ top-most bit
	sub	$nptr,$nptr,$num,lsl#2			@ rewind $nptr
	subs	$aptr,sp,#0				@ clear carry flag
	add	$bptr,sp,$num,lsl#2

.LNEON_sub:
	ldmia	$aptr!, {r4-r7}
	ldmia	$nptr!, {r8-r11}
	sbcs	r8, r4,r8
	sbcs	r9, r5,r9
	sbcs	r10,r6,r10
	sbcs	r11,r7,r11
	teq	$aptr,$bptr				@ preserves carry
	stmia	$rptr!, {r8-r11}
	bne	.LNEON_sub

	ldr	r10, [$aptr]				@ load top-most bit
	mov	r11,sp
	veor	q0,q0,q0
	sub	r11,$bptr,r11				@ this is num*4
	veor	q1,q1,q1
	mov	$aptr,sp
	sub	$rptr,$rptr,r11				@ rewind $rptr
	mov	$nptr,$bptr				@ second 3/4th of frame
	sbcs	r10,r10,#0				@ result is carry flag

.LNEON_copy_n_zap:
	ldmia	$aptr!, {r4-r7}
	ldmia	$rptr,  {r8-r11}
	it	cc
	movcc	r8, r4
	vst1.64	{q0-q1}, [$nptr,:256]!			@ wipe
	itt	cc
	movcc	r9, r5
	movcc	r10,r6
	vst1.64	{q0-q1}, [$nptr,:256]!			@ wipe
	it	cc
	movcc	r11,r7
	ldmia	$aptr, {r4-r7}
	stmia	$rptr!, {r8-r11}
	sub	$aptr,$aptr,#16
	ldmia	$rptr, {r8-r11}
	it	cc
	movcc	r8, r4
	vst1.64	{q0-q1}, [$aptr,:256]!			@ wipe
	itt	cc
	movcc	r9, r5
	movcc	r10,r6
	vst1.64	{q0-q1}, [$nptr,:256]!			@ wipe
	it	cc
	movcc	r11,r7
	teq	$aptr,$bptr				@ preserves carry
	stmia	$rptr!, {r8-r11}
	bne	.LNEON_copy_n_zap

	mov	sp,ip
        vldmia  sp!,{d8-d15}
        ldmia   sp!,{r4-r11}
	ret						@ bx lr
.size	bn_mul8x_mont_neon,.-bn_mul8x_mont_neon
#endif
___
}
$code.=<<___;
.asciz	"Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
#if __ARM_MAX_ARCH__>=7
.comm	OPENSSL_armcap_P,4,4
#endif
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/ge;

	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/ge	or
	s/\bret\b/bx    lr/g						or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/g;	# make it possible to compile with -march=armv4

	print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# March 2015
#
# "Teaser" Montgomery multiplication module for ARMv8. Needs more
# work. While it does improve RSA sign performance by 20-30% (less for
# longer keys) on most processors, for some reason RSA2048 is not
# faster and RSA4096 goes 15-20% slower on Cortex-A57. Multiplication
# instruction issue rate is limited on processor in question, meaning
# that dedicated squaring procedure is a must. Well, actually all
# contemporary AArch64 processors seem to have limited multiplication
# issue rate, i.e. they can't issue multiplication every cycle, which
# explains moderate improvement coefficients in comparison to
# compiler-generated code. Recall that compiler is instructed to use
# umulh and therefore uses same amount of multiplication instructions
# to do the job. Assembly's edge is to minimize number of "collateral"
# instructions and of course instruction scheduling.
#
# April 2015
#
# Squaring procedure that handles lengths divisible by 8 improves
# RSA/DSA performance by 25-40-60% depending on processor and key
# length. Overall improvement coefficients are always positive in
# comparison to compiler-generated code. On Cortex-A57 improvement
# is still modest on longest key lengths, while others exhibit e.g.
# 50-70% improvement for RSA4096 sign. RSA2048 sign is ~25% faster
# on Cortex-A57 and ~60-100% faster on others.

$flavour = shift;
$output  = shift;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;

($lo0,$hi0,$aj,$m0,$alo,$ahi,
 $lo1,$hi1,$nj,$m1,$nlo,$nhi,
 $ovf, $i,$j,$tp,$tj) = map("x$_",6..17,19..24);

# int bn_mul_mont(
$rp="x0";	# BN_ULONG *rp,
$ap="x1";	# const BN_ULONG *ap,
$bp="x2";	# const BN_ULONG *bp,
$np="x3";	# const BN_ULONG *np,
$n0="x4";	# const BN_ULONG *n0,
$num="x5";	# int num);

$code.=<<___;
.text

.globl	bn_mul_mont
.type	bn_mul_mont,%function
.align	5
bn_mul_mont:
	tst	$num,#7
	b.eq	__bn_sqr8x_mont
	tst	$num,#3
	b.eq	__bn_mul4x_mont
.Lmul_mont:
	stp	x29,x30,[sp,#-64]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]

	ldr	$m0,[$bp],#8		// bp[0]
	sub	$tp,sp,$num,lsl#3
	ldp	$hi0,$aj,[$ap],#16	// ap[0..1]
	lsl	$num,$num,#3
	ldr	$n0,[$n0]		// *n0
	and	$tp,$tp,#-16		// ABI says so
	ldp	$hi1,$nj,[$np],#16	// np[0..1]

	mul	$lo0,$hi0,$m0		// ap[0]*bp[0]
	sub	$j,$num,#16		// j=num-2
	umulh	$hi0,$hi0,$m0
	mul	$alo,$aj,$m0		// ap[1]*bp[0]
	umulh	$ahi,$aj,$m0

	mul	$m1,$lo0,$n0		// "tp[0]"*n0
	mov	sp,$tp			// alloca

	// (*)	mul	$lo1,$hi1,$m1	// np[0]*m1
	umulh	$hi1,$hi1,$m1
	mul	$nlo,$nj,$m1		// np[1]*m1
	// (*)	adds	$lo1,$lo1,$lo0	// discarded
	// (*)	As for removal of first multiplication and addition
	//	instructions. The outcome of first addition is
	//	guaranteed to be zero, which leaves two computationally
	//	significant outcomes: it either carries or not. Then
	//	question is when does it carry? Is there alternative
	//	way to deduce it? If you follow operations, you can
	//	observe that condition for carry is quite simple:
	//	$lo0 being non-zero. So that carry can be calculated
	//	by adding -1 to $lo0. That's what next instruction does.
	subs	xzr,$lo0,#1		// (*)
	umulh	$nhi,$nj,$m1
	adc	$hi1,$hi1,xzr
	cbz	$j,.L1st_skip

.L1st:
	ldr	$aj,[$ap],#8
	adds	$lo0,$alo,$hi0
	sub	$j,$j,#8		// j--
	adc	$hi0,$ahi,xzr

	ldr	$nj,[$np],#8
	adds	$lo1,$nlo,$hi1
	mul	$alo,$aj,$m0		// ap[j]*bp[0]
	adc	$hi1,$nhi,xzr
	umulh	$ahi,$aj,$m0

	adds	$lo1,$lo1,$lo0
	mul	$nlo,$nj,$m1		// np[j]*m1
	adc	$hi1,$hi1,xzr
	umulh	$nhi,$nj,$m1
	str	$lo1,[$tp],#8		// tp[j-1]
	cbnz	$j,.L1st

.L1st_skip:
	adds	$lo0,$alo,$hi0
	sub	$ap,$ap,$num		// rewind $ap
	adc	$hi0,$ahi,xzr

	adds	$lo1,$nlo,$hi1
	sub	$np,$np,$num		// rewind $np
	adc	$hi1,$nhi,xzr

	adds	$lo1,$lo1,$lo0
	sub	$i,$num,#8		// i=num-1
	adcs	$hi1,$hi1,$hi0

	adc	$ovf,xzr,xzr		// upmost overflow bit
	stp	$lo1,$hi1,[$tp]

.Louter:
	ldr	$m0,[$bp],#8		// bp[i]
	ldp	$hi0,$aj,[$ap],#16
	ldr	$tj,[sp]		// tp[0]
	add	$tp,sp,#8

	mul	$lo0,$hi0,$m0		// ap[0]*bp[i]
	sub	$j,$num,#16		// j=num-2
	umulh	$hi0,$hi0,$m0
	ldp	$hi1,$nj,[$np],#16
	mul	$alo,$aj,$m0		// ap[1]*bp[i]
	adds	$lo0,$lo0,$tj
	umulh	$ahi,$aj,$m0
	adc	$hi0,$hi0,xzr

	mul	$m1,$lo0,$n0
	sub	$i,$i,#8		// i--

	// (*)	mul	$lo1,$hi1,$m1	// np[0]*m1
	umulh	$hi1,$hi1,$m1
	mul	$nlo,$nj,$m1		// np[1]*m1
	// (*)	adds	$lo1,$lo1,$lo0
	subs	xzr,$lo0,#1		// (*)
	umulh	$nhi,$nj,$m1
	cbz	$j,.Linner_skip

.Linner:
	ldr	$aj,[$ap],#8
	adc	$hi1,$hi1,xzr
	ldr	$tj,[$tp],#8		// tp[j]
	adds	$lo0,$alo,$hi0
	sub	$j,$j,#8		// j--
	adc	$hi0,$ahi,xzr

	adds	$lo1,$nlo,$hi1
	ldr	$nj,[$np],#8
	adc	$hi1,$nhi,xzr

	mul	$alo,$aj,$m0		// ap[j]*bp[i]
	adds	$lo0,$lo0,$tj
	umulh	$ahi,$aj,$m0
	adc	$hi0,$hi0,xzr

	mul	$nlo,$nj,$m1		// np[j]*m1
	adds	$lo1,$lo1,$lo0
	umulh	$nhi,$nj,$m1
	str	$lo1,[$tp,#-16]		// tp[j-1]
	cbnz	$j,.Linner

.Linner_skip:
	ldr	$tj,[$tp],#8		// tp[j]
	adc	$hi1,$hi1,xzr
	adds	$lo0,$alo,$hi0
	sub	$ap,$ap,$num		// rewind $ap
	adc	$hi0,$ahi,xzr

	adds	$lo1,$nlo,$hi1
	sub	$np,$np,$num		// rewind $np
	adcs	$hi1,$nhi,$ovf
	adc	$ovf,xzr,xzr

	adds	$lo0,$lo0,$tj
	adc	$hi0,$hi0,xzr

	adds	$lo1,$lo1,$lo0
	adcs	$hi1,$hi1,$hi0
	adc	$ovf,$ovf,xzr		// upmost overflow bit
	stp	$lo1,$hi1,[$tp,#-16]

	cbnz	$i,.Louter

	// Final step. We see if result is larger than modulus, and
	// if it is, subtract the modulus. But comparison implies
	// subtraction. So we subtract modulus, see if it borrowed,
	// and conditionally copy original value.
	ldr	$tj,[sp]		// tp[0]
	add	$tp,sp,#8
	ldr	$nj,[$np],#8		// np[0]
	subs	$j,$num,#8		// j=num-1 and clear borrow
	mov	$ap,$rp
.Lsub:
	sbcs	$aj,$tj,$nj		// tp[j]-np[j]
	ldr	$tj,[$tp],#8
	sub	$j,$j,#8		// j--
	ldr	$nj,[$np],#8
	str	$aj,[$ap],#8		// rp[j]=tp[j]-np[j]
	cbnz	$j,.Lsub

	sbcs	$aj,$tj,$nj
	sbcs	$ovf,$ovf,xzr		// did it borrow?
	str	$aj,[$ap],#8		// rp[num-1]

	ldr	$tj,[sp]		// tp[0]
	add	$tp,sp,#8
	ldr	$aj,[$rp],#8		// rp[0]
	sub	$num,$num,#8		// num--
	nop
.Lcond_copy:
	sub	$num,$num,#8		// num--
	csel	$nj,$tj,$aj,lo		// did it borrow?
	ldr	$tj,[$tp],#8
	ldr	$aj,[$rp],#8
	str	xzr,[$tp,#-16]		// wipe tp
	str	$nj,[$rp,#-16]
	cbnz	$num,.Lcond_copy

	csel	$nj,$tj,$aj,lo
	str	xzr,[$tp,#-8]		// wipe tp
	str	$nj,[$rp,#-8]

	ldp	x19,x20,[x29,#16]
	mov	sp,x29
	ldp	x21,x22,[x29,#32]
	mov	x0,#1
	ldp	x23,x24,[x29,#48]
	ldr	x29,[sp],#64
	ret
.size	bn_mul_mont,.-bn_mul_mont
___
{
########################################################################
# Following is ARMv8 adaptation of sqrx8x_mont from x86_64-mont5 module.

my ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("x$_",(6..13));
my ($t0,$t1,$t2,$t3)=map("x$_",(14..17));
my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("x$_",(19..26));
my ($cnt,$carry,$topmost)=("x27","x28","x30");
my ($tp,$ap_end,$na0)=($bp,$np,$carry);

$code.=<<___;
.type	__bn_sqr8x_mont,%function
.align	5
__bn_sqr8x_mont:
	cmp	$ap,$bp
	b.ne	__bn_mul4x_mont
.Lsqr8x_mont:
	.inst	0xd503233f		// paciasp
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]
	stp	$rp,$np,[sp,#96]	// offload rp and np

	ldp	$a0,$a1,[$ap,#8*0]
	ldp	$a2,$a3,[$ap,#8*2]
	ldp	$a4,$a5,[$ap,#8*4]
	ldp	$a6,$a7,[$ap,#8*6]

	sub	$tp,sp,$num,lsl#4
	lsl	$num,$num,#3
	ldr	$n0,[$n0]		// *n0
	mov	sp,$tp			// alloca
	sub	$cnt,$num,#8*8
	b	.Lsqr8x_zero_start

.Lsqr8x_zero:
	sub	$cnt,$cnt,#8*8
	stp	xzr,xzr,[$tp,#8*0]
	stp	xzr,xzr,[$tp,#8*2]
	stp	xzr,xzr,[$tp,#8*4]
	stp	xzr,xzr,[$tp,#8*6]
.Lsqr8x_zero_start:
	stp	xzr,xzr,[$tp,#8*8]
	stp	xzr,xzr,[$tp,#8*10]
	stp	xzr,xzr,[$tp,#8*12]
	stp	xzr,xzr,[$tp,#8*14]
	add	$tp,$tp,#8*16
	cbnz	$cnt,.Lsqr8x_zero

	add	$ap_end,$ap,$num
	add	$ap,$ap,#8*8
	mov	$acc0,xzr
	mov	$acc1,xzr
	mov	$acc2,xzr
	mov	$acc3,xzr
	mov	$acc4,xzr
	mov	$acc5,xzr
	mov	$acc6,xzr
	mov	$acc7,xzr
	mov	$tp,sp
	str	$n0,[x29,#112]		// offload n0

	// Multiply everything but a[i]*a[i]
.align	4
.Lsqr8x_outer_loop:
        //                                                 a[1]a[0]	(i)
        //                                             a[2]a[0]
        //                                         a[3]a[0]
        //                                     a[4]a[0]
        //                                 a[5]a[0]
        //                             a[6]a[0]
        //                         a[7]a[0]
        //                                         a[2]a[1]		(ii)
        //                                     a[3]a[1]
        //                                 a[4]a[1]
        //                             a[5]a[1]
        //                         a[6]a[1]
        //                     a[7]a[1]
        //                                 a[3]a[2]			(iii)
        //                             a[4]a[2]
        //                         a[5]a[2]
        //                     a[6]a[2]
        //                 a[7]a[2]
        //                         a[4]a[3]				(iv)
        //                     a[5]a[3]
        //                 a[6]a[3]
        //             a[7]a[3]
        //                 a[5]a[4]					(v)
        //             a[6]a[4]
        //         a[7]a[4]
        //         a[6]a[5]						(vi)
        //     a[7]a[5]
        // a[7]a[6]							(vii)

	mul	$t0,$a1,$a0		// lo(a[1..7]*a[0])		(i)
	mul	$t1,$a2,$a0
	mul	$t2,$a3,$a0
	mul	$t3,$a4,$a0
	adds	$acc1,$acc1,$t0		// t[1]+lo(a[1]*a[0])
	mul	$t0,$a5,$a0
	adcs	$acc2,$acc2,$t1
	mul	$t1,$a6,$a0
	adcs	$acc3,$acc3,$t2
	mul	$t2,$a7,$a0
	adcs	$acc4,$acc4,$t3
	umulh	$t3,$a1,$a0		// hi(a[1..7]*a[0])
	adcs	$acc5,$acc5,$t0
	umulh	$t0,$a2,$a0
	adcs	$acc6,$acc6,$t1
	umulh	$t1,$a3,$a0
	adcs	$acc7,$acc7,$t2
	umulh	$t2,$a4,$a0
	stp	$acc0,$acc1,[$tp],#8*2	// t[0..1]
	adc	$acc0,xzr,xzr		// t[8]
	adds	$acc2,$acc2,$t3		// t[2]+lo(a[1]*a[0])
	umulh	$t3,$a5,$a0
	adcs	$acc3,$acc3,$t0
	umulh	$t0,$a6,$a0
	adcs	$acc4,$acc4,$t1
	umulh	$t1,$a7,$a0
	adcs	$acc5,$acc5,$t2
	 mul	$t2,$a2,$a1		// lo(a[2..7]*a[1])		(ii)
	adcs	$acc6,$acc6,$t3
	 mul	$t3,$a3,$a1
	adcs	$acc7,$acc7,$t0
	 mul	$t0,$a4,$a1
	adc	$acc0,$acc0,$t1

	mul	$t1,$a5,$a1
	adds	$acc3,$acc3,$t2
	mul	$t2,$a6,$a1
	adcs	$acc4,$acc4,$t3
	mul	$t3,$a7,$a1
	adcs	$acc5,$acc5,$t0
	umulh	$t0,$a2,$a1		// hi(a[2..7]*a[1])
	adcs	$acc6,$acc6,$t1
	umulh	$t1,$a3,$a1
	adcs	$acc7,$acc7,$t2
	umulh	$t2,$a4,$a1
	adcs	$acc0,$acc0,$t3
	umulh	$t3,$a5,$a1
	stp	$acc2,$acc3,[$tp],#8*2	// t[2..3]
	adc	$acc1,xzr,xzr		// t[9]
	adds	$acc4,$acc4,$t0
	umulh	$t0,$a6,$a1
	adcs	$acc5,$acc5,$t1
	umulh	$t1,$a7,$a1
	adcs	$acc6,$acc6,$t2
	 mul	$t2,$a3,$a2		// lo(a[3..7]*a[2])		(iii)
	adcs	$acc7,$acc7,$t3
	 mul	$t3,$a4,$a2
	adcs	$acc0,$acc0,$t0
	 mul	$t0,$a5,$a2
	adc	$acc1,$acc1,$t1

	mul	$t1,$a6,$a2
	adds	$acc5,$acc5,$t2
	mul	$t2,$a7,$a2
	adcs	$acc6,$acc6,$t3
	umulh	$t3,$a3,$a2		// hi(a[3..7]*a[2])
	adcs	$acc7,$acc7,$t0
	umulh	$t0,$a4,$a2
	adcs	$acc0,$acc0,$t1
	umulh	$t1,$a5,$a2
	adcs	$acc1,$acc1,$t2
	umulh	$t2,$a6,$a2
	stp	$acc4,$acc5,[$tp],#8*2	// t[4..5]
	adc	$acc2,xzr,xzr		// t[10]
	adds	$acc6,$acc6,$t3
	umulh	$t3,$a7,$a2
	adcs	$acc7,$acc7,$t0
	 mul	$t0,$a4,$a3		// lo(a[4..7]*a[3])		(iv)
	adcs	$acc0,$acc0,$t1
	 mul	$t1,$a5,$a3
	adcs	$acc1,$acc1,$t2
	 mul	$t2,$a6,$a3
	adc	$acc2,$acc2,$t3

	mul	$t3,$a7,$a3
	adds	$acc7,$acc7,$t0
	umulh	$t0,$a4,$a3		// hi(a[4..7]*a[3])
	adcs	$acc0,$acc0,$t1
	umulh	$t1,$a5,$a3
	adcs	$acc1,$acc1,$t2
	umulh	$t2,$a6,$a3
	adcs	$acc2,$acc2,$t3
	umulh	$t3,$a7,$a3
	stp	$acc6,$acc7,[$tp],#8*2	// t[6..7]
	adc	$acc3,xzr,xzr		// t[11]
	adds	$acc0,$acc0,$t0
	 mul	$t0,$a5,$a4		// lo(a[5..7]*a[4])		(v)
	adcs	$acc1,$acc1,$t1
	 mul	$t1,$a6,$a4
	adcs	$acc2,$acc2,$t2
	 mul	$t2,$a7,$a4
	adc	$acc3,$acc3,$t3

	umulh	$t3,$a5,$a4		// hi(a[5..7]*a[4])
	adds	$acc1,$acc1,$t0
	umulh	$t0,$a6,$a4
	adcs	$acc2,$acc2,$t1
	umulh	$t1,$a7,$a4
	adcs	$acc3,$acc3,$t2
	 mul	$t2,$a6,$a5		// lo(a[6..7]*a[5])		(vi)
	adc	$acc4,xzr,xzr		// t[12]
	adds	$acc2,$acc2,$t3
	 mul	$t3,$a7,$a5
	adcs	$acc3,$acc3,$t0
	 umulh	$t0,$a6,$a5		// hi(a[6..7]*a[5])
	adc	$acc4,$acc4,$t1

	umulh	$t1,$a7,$a5
	adds	$acc3,$acc3,$t2
	 mul	$t2,$a7,$a6		// lo(a[7]*a[6])		(vii)
	adcs	$acc4,$acc4,$t3
	 umulh	$t3,$a7,$a6		// hi(a[7]*a[6])
	adc	$acc5,xzr,xzr		// t[13]
	adds	$acc4,$acc4,$t0
	sub	$cnt,$ap_end,$ap	// done yet?
	adc	$acc5,$acc5,$t1

	adds	$acc5,$acc5,$t2
	sub	$t0,$ap_end,$num	// rewinded ap
	adc	$acc6,xzr,xzr		// t[14]
	add	$acc6,$acc6,$t3

	cbz	$cnt,.Lsqr8x_outer_break

	mov	$n0,$a0
	ldp	$a0,$a1,[$tp,#8*0]
	ldp	$a2,$a3,[$tp,#8*2]
	ldp	$a4,$a5,[$tp,#8*4]
	ldp	$a6,$a7,[$tp,#8*6]
	adds	$acc0,$acc0,$a0
	adcs	$acc1,$acc1,$a1
	ldp	$a0,$a1,[$ap,#8*0]
	adcs	$acc2,$acc2,$a2
	adcs	$acc3,$acc3,$a3
	ldp	$a2,$a3,[$ap,#8*2]
	adcs	$acc4,$acc4,$a4
	adcs	$acc5,$acc5,$a5
	ldp	$a4,$a5,[$ap,#8*4]
	adcs	$acc6,$acc6,$a6
	mov	$rp,$ap
	adcs	$acc7,xzr,$a7
	ldp	$a6,$a7,[$ap,#8*6]
	add	$ap,$ap,#8*8
	//adc	$carry,xzr,xzr		// moved below
	mov	$cnt,#-8*8

	//                                                         a[8]a[0]
	//                                                     a[9]a[0]
	//                                                 a[a]a[0]
	//                                             a[b]a[0]
	//                                         a[c]a[0]
	//                                     a[d]a[0]
	//                                 a[e]a[0]
	//                             a[f]a[0]
	//                                                     a[8]a[1]
	//                         a[f]a[1]........................
	//                                                 a[8]a[2]
	//                     a[f]a[2]........................
	//                                             a[8]a[3]
	//                 a[f]a[3]........................
	//                                         a[8]a[4]
	//             a[f]a[4]........................
	//                                     a[8]a[5]
	//         a[f]a[5]........................
	//                                 a[8]a[6]
	//     a[f]a[6]........................
	//                             a[8]a[7]
	// a[f]a[7]........................
.Lsqr8x_mul:
	mul	$t0,$a0,$n0
	adc	$carry,xzr,xzr		// carry bit, modulo-scheduled
	mul	$t1,$a1,$n0
	add	$cnt,$cnt,#8
	mul	$t2,$a2,$n0
	mul	$t3,$a3,$n0
	adds	$acc0,$acc0,$t0
	mul	$t0,$a4,$n0
	adcs	$acc1,$acc1,$t1
	mul	$t1,$a5,$n0
	adcs	$acc2,$acc2,$t2
	mul	$t2,$a6,$n0
	adcs	$acc3,$acc3,$t3
	mul	$t3,$a7,$n0
	adcs	$acc4,$acc4,$t0
	umulh	$t0,$a0,$n0
	adcs	$acc5,$acc5,$t1
	umulh	$t1,$a1,$n0
	adcs	$acc6,$acc6,$t2
	umulh	$t2,$a2,$n0
	adcs	$acc7,$acc7,$t3
	umulh	$t3,$a3,$n0
	adc	$carry,$carry,xzr
	str	$acc0,[$tp],#8
	adds	$acc0,$acc1,$t0
	umulh	$t0,$a4,$n0
	adcs	$acc1,$acc2,$t1
	umulh	$t1,$a5,$n0
	adcs	$acc2,$acc3,$t2
	umulh	$t2,$a6,$n0
	adcs	$acc3,$acc4,$t3
	umulh	$t3,$a7,$n0
	ldr	$n0,[$rp,$cnt]
	adcs	$acc4,$acc5,$t0
	adcs	$acc5,$acc6,$t1
	adcs	$acc6,$acc7,$t2
	adcs	$acc7,$carry,$t3
	//adc	$carry,xzr,xzr		// moved above
	cbnz	$cnt,.Lsqr8x_mul
					// note that carry flag is guaranteed
					// to be zero at this point
	cmp	$ap,$ap_end		// done yet?
	b.eq	.Lsqr8x_break

	ldp	$a0,$a1,[$tp,#8*0]
	ldp	$a2,$a3,[$tp,#8*2]
	ldp	$a4,$a5,[$tp,#8*4]
	ldp	$a6,$a7,[$tp,#8*6]
	adds	$acc0,$acc0,$a0
	ldr	$n0,[$rp,#-8*8]
	adcs	$acc1,$acc1,$a1
	ldp	$a0,$a1,[$ap,#8*0]
	adcs	$acc2,$acc2,$a2
	adcs	$acc3,$acc3,$a3
	ldp	$a2,$a3,[$ap,#8*2]
	adcs	$acc4,$acc4,$a4
	adcs	$acc5,$acc5,$a5
	ldp	$a4,$a5,[$ap,#8*4]
	adcs	$acc6,$acc6,$a6
	mov	$cnt,#-8*8
	adcs	$acc7,$acc7,$a7
	ldp	$a6,$a7,[$ap,#8*6]
	add	$ap,$ap,#8*8
	//adc	$carry,xzr,xzr		// moved above
	b	.Lsqr8x_mul

.align	4
.Lsqr8x_break:
	ldp	$a0,$a1,[$rp,#8*0]
	add	$ap,$rp,#8*8
	ldp	$a2,$a3,[$rp,#8*2]
	sub	$t0,$ap_end,$ap		// is it last iteration?
	ldp	$a4,$a5,[$rp,#8*4]
	sub	$t1,$tp,$t0
	ldp	$a6,$a7,[$rp,#8*6]
	cbz	$t0,.Lsqr8x_outer_loop

	stp	$acc0,$acc1,[$tp,#8*0]
	ldp	$acc0,$acc1,[$t1,#8*0]
	stp	$acc2,$acc3,[$tp,#8*2]
	ldp	$acc2,$acc3,[$t1,#8*2]
	stp	$acc4,$acc5,[$tp,#8*4]
	ldp	$acc4,$acc5,[$t1,#8*4]
	stp	$acc6,$acc7,[$tp,#8*6]
	mov	$tp,$t1
	ldp	$acc6,$acc7,[$t1,#8*6]
	b	.Lsqr8x_outer_loop

.align	4
.Lsqr8x_outer_break:
	// Now multiply above result by 2 and add a[n-1]*a[n-1]|...|a[0]*a[0]
	ldp	$a1,$a3,[$t0,#8*0]	// recall that $t0 is &a[0]
	ldp	$t1,$t2,[sp,#8*1]
	ldp	$a5,$a7,[$t0,#8*2]
	add	$ap,$t0,#8*4
	ldp	$t3,$t0,[sp,#8*3]

	stp	$acc0,$acc1,[$tp,#8*0]
	mul	$acc0,$a1,$a1
	stp	$acc2,$acc3,[$tp,#8*2]
	umulh	$a1,$a1,$a1
	stp	$acc4,$acc5,[$tp,#8*4]
	mul	$a2,$a3,$a3
	stp	$acc6,$acc7,[$tp,#8*6]
	mov	$tp,sp
	umulh	$a3,$a3,$a3
	adds	$acc1,$a1,$t1,lsl#1
	extr	$t1,$t2,$t1,#63
	sub	$cnt,$num,#8*4

.Lsqr4x_shift_n_add:
	adcs	$acc2,$a2,$t1
	extr	$t2,$t3,$t2,#63
	sub	$cnt,$cnt,#8*4
	adcs	$acc3,$a3,$t2
	ldp	$t1,$t2,[$tp,#8*5]
	mul	$a4,$a5,$a5
	ldp	$a1,$a3,[$ap],#8*2
	umulh	$a5,$a5,$a5
	mul	$a6,$a7,$a7
	umulh	$a7,$a7,$a7
	extr	$t3,$t0,$t3,#63
	stp	$acc0,$acc1,[$tp,#8*0]
	adcs	$acc4,$a4,$t3
	extr	$t0,$t1,$t0,#63
	stp	$acc2,$acc3,[$tp,#8*2]
	adcs	$acc5,$a5,$t0
	ldp	$t3,$t0,[$tp,#8*7]
	extr	$t1,$t2,$t1,#63
	adcs	$acc6,$a6,$t1
	extr	$t2,$t3,$t2,#63
	adcs	$acc7,$a7,$t2
	ldp	$t1,$t2,[$tp,#8*9]
	mul	$a0,$a1,$a1
	ldp	$a5,$a7,[$ap],#8*2
	umulh	$a1,$a1,$a1
	mul	$a2,$a3,$a3
	umulh	$a3,$a3,$a3
	stp	$acc4,$acc5,[$tp,#8*4]
	extr	$t3,$t0,$t3,#63
	stp	$acc6,$acc7,[$tp,#8*6]
	add	$tp,$tp,#8*8
	adcs	$acc0,$a0,$t3
	extr	$t0,$t1,$t0,#63
	adcs	$acc1,$a1,$t0
	ldp	$t3,$t0,[$tp,#8*3]
	extr	$t1,$t2,$t1,#63
	cbnz	$cnt,.Lsqr4x_shift_n_add
___
my ($np,$np_end)=($ap,$ap_end);
$code.=<<___;
	 ldp	$np,$n0,[x29,#104]	// pull np and n0

	adcs	$acc2,$a2,$t1
	extr	$t2,$t3,$t2,#63
	adcs	$acc3,$a3,$t2
	ldp	$t1,$t2,[$tp,#8*5]
	mul	$a4,$a5,$a5
	umulh	$a5,$a5,$a5
	stp	$acc0,$acc1,[$tp,#8*0]
	mul	$a6,$a7,$a7
	umulh	$a7,$a7,$a7
	stp	$acc2,$acc3,[$tp,#8*2]
	extr	$t3,$t0,$t3,#63
	adcs	$acc4,$a4,$t3
	extr	$t0,$t1,$t0,#63
	 ldp	$acc0,$acc1,[sp,#8*0]
	adcs	$acc5,$a5,$t0
	extr	$t1,$t2,$t1,#63
	 ldp	$a0,$a1,[$np,#8*0]
	adcs	$acc6,$a6,$t1
	extr	$t2,xzr,$t2,#63
	 ldp	$a2,$a3,[$np,#8*2]
	adc	$acc7,$a7,$t2
	 ldp	$a4,$a5,[$np,#8*4]

	// Reduce by 512 bits per iteration
	mul	$na0,$n0,$acc0		// t[0]*n0
	ldp	$a6,$a7,[$np,#8*6]
	add	$np_end,$np,$num
	ldp	$acc2,$acc3,[sp,#8*2]
	stp	$acc4,$acc5,[$tp,#8*4]
	ldp	$acc4,$acc5,[sp,#8*4]
	stp	$acc6,$acc7,[$tp,#8*6]
	ldp	$acc6,$acc7,[sp,#8*6]
	add	$np,$np,#8*8
	mov	$topmost,xzr		// initial top-most carry
	mov	$tp,sp
	mov	$cnt,#8

.Lsqr8x_reduction:
	// (*)	mul	$t0,$a0,$na0	// lo(n[0-7])*lo(t[0]*n0)
	mul	$t1,$a1,$na0
	sub	$cnt,$cnt,#1
	mul	$t2,$a2,$na0
	str	$na0,[$tp],#8		// put aside t[0]*n0 for tail processing
	mul	$t3,$a3,$na0
	// (*)	adds	xzr,$acc0,$t0
	subs	xzr,$acc0,#1		// (*)
	mul	$t0,$a4,$na0
	adcs	$acc0,$acc1,$t1
	mul	$t1,$a5,$na0
	adcs	$acc1,$acc2,$t2
	mul	$t2,$a6,$na0
	adcs	$acc2,$acc3,$t3
	mul	$t3,$a7,$na0
	adcs	$acc3,$acc4,$t0
	umulh	$t0,$a0,$na0		// hi(n[0-7])*lo(t[0]*n0)
	adcs	$acc4,$acc5,$t1
	umulh	$t1,$a1,$na0
	adcs	$acc5,$acc6,$t2
	umulh	$t2,$a2,$na0
	adcs	$acc6,$acc7,$t3
	umulh	$t3,$a3,$na0
	adc	$acc7,xzr,xzr
	adds	$acc0,$acc0,$t0
	umulh	$t0,$a4,$na0
	adcs	$acc1,$acc1,$t1
	umulh	$t1,$a5,$na0
	adcs	$acc2,$acc2,$t2
	umulh	$t2,$a6,$na0
	adcs	$acc3,$acc3,$t3
	umulh	$t3,$a7,$na0
	mul	$na0,$n0,$acc0		// next t[0]*n0
	adcs	$acc4,$acc4,$t0
	adcs	$acc5,$acc5,$t1
	adcs	$acc6,$acc6,$t2
	adc	$acc7,$acc7,$t3
	cbnz	$cnt,.Lsqr8x_reduction

	ldp	$t0,$t1,[$tp,#8*0]
	ldp	$t2,$t3,[$tp,#8*2]
	mov	$rp,$tp
	sub	$cnt,$np_end,$np	// done yet?
	adds	$acc0,$acc0,$t0
	adcs	$acc1,$acc1,$t1
	ldp	$t0,$t1,[$tp,#8*4]
	adcs	$acc2,$acc2,$t2
	adcs	$acc3,$acc3,$t3
	ldp	$t2,$t3,[$tp,#8*6]
	adcs	$acc4,$acc4,$t0
	adcs	$acc5,$acc5,$t1
	adcs	$acc6,$acc6,$t2
	adcs	$acc7,$acc7,$t3
	//adc	$carry,xzr,xzr		// moved below
	cbz	$cnt,.Lsqr8x8_post_condition

	ldr	$n0,[$tp,#-8*8]
	ldp	$a0,$a1,[$np,#8*0]
	ldp	$a2,$a3,[$np,#8*2]
	ldp	$a4,$a5,[$np,#8*4]
	mov	$cnt,#-8*8
	ldp	$a6,$a7,[$np,#8*6]
	add	$np,$np,#8*8

.Lsqr8x_tail:
	mul	$t0,$a0,$n0
	adc	$carry,xzr,xzr		// carry bit, modulo-scheduled
	mul	$t1,$a1,$n0
	add	$cnt,$cnt,#8
	mul	$t2,$a2,$n0
	mul	$t3,$a3,$n0
	adds	$acc0,$acc0,$t0
	mul	$t0,$a4,$n0
	adcs	$acc1,$acc1,$t1
	mul	$t1,$a5,$n0
	adcs	$acc2,$acc2,$t2
	mul	$t2,$a6,$n0
	adcs	$acc3,$acc3,$t3
	mul	$t3,$a7,$n0
	adcs	$acc4,$acc4,$t0
	umulh	$t0,$a0,$n0
	adcs	$acc5,$acc5,$t1
	umulh	$t1,$a1,$n0
	adcs	$acc6,$acc6,$t2
	umulh	$t2,$a2,$n0
	adcs	$acc7,$acc7,$t3
	umulh	$t3,$a3,$n0
	adc	$carry,$carry,xzr
	str	$acc0,[$tp],#8
	adds	$acc0,$acc1,$t0
	umulh	$t0,$a4,$n0
	adcs	$acc1,$acc2,$t1
	umulh	$t1,$a5,$n0
	adcs	$acc2,$acc3,$t2
	umulh	$t2,$a6,$n0
	adcs	$acc3,$acc4,$t3
	umulh	$t3,$a7,$n0
	ldr	$n0,[$rp,$cnt]
	adcs	$acc4,$acc5,$t0
	adcs	$acc5,$acc6,$t1
	adcs	$acc6,$acc7,$t2
	adcs	$acc7,$carry,$t3
	//adc	$carry,xzr,xzr		// moved above
	cbnz	$cnt,.Lsqr8x_tail
					// note that carry flag is guaranteed
					// to be zero at this point
	ldp	$a0,$a1,[$tp,#8*0]
	sub	$cnt,$np_end,$np	// done yet?
	sub	$t2,$np_end,$num	// rewinded np
	ldp	$a2,$a3,[$tp,#8*2]
	ldp	$a4,$a5,[$tp,#8*4]
	ldp	$a6,$a7,[$tp,#8*6]
	cbz	$cnt,.Lsqr8x_tail_break

	ldr	$n0,[$rp,#-8*8]
	adds	$acc0,$acc0,$a0
	adcs	$acc1,$acc1,$a1
	ldp	$a0,$a1,[$np,#8*0]
	adcs	$acc2,$acc2,$a2
	adcs	$acc3,$acc3,$a3
	ldp	$a2,$a3,[$np,#8*2]
	adcs	$acc4,$acc4,$a4
	adcs	$acc5,$acc5,$a5
	ldp	$a4,$a5,[$np,#8*4]
	adcs	$acc6,$acc6,$a6
	mov	$cnt,#-8*8
	adcs	$acc7,$acc7,$a7
	ldp	$a6,$a7,[$np,#8*6]
	add	$np,$np,#8*8
	//adc	$carry,xzr,xzr		// moved above
	b	.Lsqr8x_tail

.align	4
.Lsqr8x_tail_break:
	ldr	$n0,[x29,#112]		// pull n0
	add	$cnt,$tp,#8*8		// end of current t[num] window

	subs	xzr,$topmost,#1		// "move" top-most carry to carry bit
	adcs	$t0,$acc0,$a0
	adcs	$t1,$acc1,$a1
	ldp	$acc0,$acc1,[$rp,#8*0]
	adcs	$acc2,$acc2,$a2
	ldp	$a0,$a1,[$t2,#8*0]	// recall that $t2 is &n[0]
	adcs	$acc3,$acc3,$a3
	ldp	$a2,$a3,[$t2,#8*2]
	adcs	$acc4,$acc4,$a4
	adcs	$acc5,$acc5,$a5
	ldp	$a4,$a5,[$t2,#8*4]
	adcs	$acc6,$acc6,$a6
	adcs	$acc7,$acc7,$a7
	ldp	$a6,$a7,[$t2,#8*6]
	add	$np,$t2,#8*8
	adc	$topmost,xzr,xzr	// top-most carry
	mul	$na0,$n0,$acc0
	stp	$t0,$t1,[$tp,#8*0]
	stp	$acc2,$acc3,[$tp,#8*2]
	ldp	$acc2,$acc3,[$rp,#8*2]
	stp	$acc4,$acc5,[$tp,#8*4]
	ldp	$acc4,$acc5,[$rp,#8*4]
	cmp	$cnt,x29		// did we hit the bottom?
	stp	$acc6,$acc7,[$tp,#8*6]
	mov	$tp,$rp			// slide the window
	ldp	$acc6,$acc7,[$rp,#8*6]
	mov	$cnt,#8
	b.ne	.Lsqr8x_reduction

	// Final step. We see if result is larger than modulus, and
	// if it is, subtract the modulus. But comparison implies
	// subtraction. So we subtract modulus, see if it borrowed,
	// and conditionally copy original value.
	ldr	$rp,[x29,#96]		// pull rp
	add	$tp,$tp,#8*8
	subs	$t0,$acc0,$a0
	sbcs	$t1,$acc1,$a1
	sub	$cnt,$num,#8*8
	mov	$ap_end,$rp		// $rp copy

.Lsqr8x_sub:
	sbcs	$t2,$acc2,$a2
	ldp	$a0,$a1,[$np,#8*0]
	sbcs	$t3,$acc3,$a3
	stp	$t0,$t1,[$rp,#8*0]
	sbcs	$t0,$acc4,$a4
	ldp	$a2,$a3,[$np,#8*2]
	sbcs	$t1,$acc5,$a5
	stp	$t2,$t3,[$rp,#8*2]
	sbcs	$t2,$acc6,$a6
	ldp	$a4,$a5,[$np,#8*4]
	sbcs	$t3,$acc7,$a7
	ldp	$a6,$a7,[$np,#8*6]
	add	$np,$np,#8*8
	ldp	$acc0,$acc1,[$tp,#8*0]
	sub	$cnt,$cnt,#8*8
	ldp	$acc2,$acc3,[$tp,#8*2]
	ldp	$acc4,$acc5,[$tp,#8*4]
	ldp	$acc6,$acc7,[$tp,#8*6]
	add	$tp,$tp,#8*8
	stp	$t0,$t1,[$rp,#8*4]
	sbcs	$t0,$acc0,$a0
	stp	$t2,$t3,[$rp,#8*6]
	add	$rp,$rp,#8*8
	sbcs	$t1,$acc1,$a1
	cbnz	$cnt,.Lsqr8x_sub

	sbcs	$t2,$acc2,$a2
	 mov	$tp,sp
	 add	$ap,sp,$num
	 ldp	$a0,$a1,[$ap_end,#8*0]
	sbcs	$t3,$acc3,$a3
	stp	$t0,$t1,[$rp,#8*0]
	sbcs	$t0,$acc4,$a4
	 ldp	$a2,$a3,[$ap_end,#8*2]
	sbcs	$t1,$acc5,$a5
	stp	$t2,$t3,[$rp,#8*2]
	sbcs	$t2,$acc6,$a6
	 ldp	$acc0,$acc1,[$ap,#8*0]
	sbcs	$t3,$acc7,$a7
	 ldp	$acc2,$acc3,[$ap,#8*2]
	sbcs	xzr,$topmost,xzr	// did it borrow?
	ldr	x30,[x29,#8]		// pull return address
	stp	$t0,$t1,[$rp,#8*4]
	stp	$t2,$t3,[$rp,#8*6]

	sub	$cnt,$num,#8*4
.Lsqr4x_cond_copy:
	sub	$cnt,$cnt,#8*4
	csel	$t0,$acc0,$a0,lo
	 stp	xzr,xzr,[$tp,#8*0]
	csel	$t1,$acc1,$a1,lo
	ldp	$a0,$a1,[$ap_end,#8*4]
	ldp	$acc0,$acc1,[$ap,#8*4]
	csel	$t2,$acc2,$a2,lo
	 stp	xzr,xzr,[$tp,#8*2]
	 add	$tp,$tp,#8*4
	csel	$t3,$acc3,$a3,lo
	ldp	$a2,$a3,[$ap_end,#8*6]
	ldp	$acc2,$acc3,[$ap,#8*6]
	add	$ap,$ap,#8*4
	stp	$t0,$t1,[$ap_end,#8*0]
	stp	$t2,$t3,[$ap_end,#8*2]
	add	$ap_end,$ap_end,#8*4
	 stp	xzr,xzr,[$ap,#8*0]
	 stp	xzr,xzr,[$ap,#8*2]
	cbnz	$cnt,.Lsqr4x_cond_copy

	csel	$t0,$acc0,$a0,lo
	 stp	xzr,xzr,[$tp,#8*0]
	csel	$t1,$acc1,$a1,lo
	 stp	xzr,xzr,[$tp,#8*2]
	csel	$t2,$acc2,$a2,lo
	csel	$t3,$acc3,$a3,lo
	stp	$t0,$t1,[$ap_end,#8*0]
	stp	$t2,$t3,[$ap_end,#8*2]

	b	.Lsqr8x_done

.align	4
.Lsqr8x8_post_condition:
	adc	$carry,xzr,xzr
	ldr	x30,[x29,#8]		// pull return address
	// $acc0-7,$carry hold result, $a0-7 hold modulus
	subs	$a0,$acc0,$a0
	ldr	$ap,[x29,#96]		// pull rp
	sbcs	$a1,$acc1,$a1
	 stp	xzr,xzr,[sp,#8*0]
	sbcs	$a2,$acc2,$a2
	 stp	xzr,xzr,[sp,#8*2]
	sbcs	$a3,$acc3,$a3
	 stp	xzr,xzr,[sp,#8*4]
	sbcs	$a4,$acc4,$a4
	 stp	xzr,xzr,[sp,#8*6]
	sbcs	$a5,$acc5,$a5
	 stp	xzr,xzr,[sp,#8*8]
	sbcs	$a6,$acc6,$a6
	 stp	xzr,xzr,[sp,#8*10]
	sbcs	$a7,$acc7,$a7
	 stp	xzr,xzr,[sp,#8*12]
	sbcs	$carry,$carry,xzr	// did it borrow?
	 stp	xzr,xzr,[sp,#8*14]

	// $a0-7 hold result-modulus
	csel	$a0,$acc0,$a0,lo
	csel	$a1,$acc1,$a1,lo
	csel	$a2,$acc2,$a2,lo
	csel	$a3,$acc3,$a3,lo
	stp	$a0,$a1,[$ap,#8*0]
	csel	$a4,$acc4,$a4,lo
	csel	$a5,$acc5,$a5,lo
	stp	$a2,$a3,[$ap,#8*2]
	csel	$a6,$acc6,$a6,lo
	csel	$a7,$acc7,$a7,lo
	stp	$a4,$a5,[$ap,#8*4]
	stp	$a6,$a7,[$ap,#8*6]

.Lsqr8x_done:
	ldp	x19,x20,[x29,#16]
	mov	sp,x29
	ldp	x21,x22,[x29,#32]
	mov	x0,#1
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldr	x29,[sp],#128
	.inst	0xd50323bf		// autiasp
	ret
.size	__bn_sqr8x_mont,.-__bn_sqr8x_mont
___
}

{
########################################################################
# Even though this might look as ARMv8 adaptation of mulx4x_mont from
# x86_64-mont5 module, it's different in sense that it performs
# reduction 256 bits at a time.

my ($a0,$a1,$a2,$a3,
    $t0,$t1,$t2,$t3,
    $m0,$m1,$m2,$m3,
    $acc0,$acc1,$acc2,$acc3,$acc4,
    $bi,$mi,$tp,$ap_end,$cnt) = map("x$_",(6..17,19..28));
my  $bp_end=$rp;
my  ($carry,$topmost) = ($rp,"x30");

$code.=<<___;
.type	__bn_mul4x_mont,%function
.align	5
__bn_mul4x_mont:
	.inst	0xd503233f		// paciasp
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]

	sub	$tp,sp,$num,lsl#3
	lsl	$num,$num,#3
	ldr	$n0,[$n0]		// *n0
	sub	sp,$tp,#8*4		// alloca

	add	$t0,$bp,$num
	add	$ap_end,$ap,$num
	stp	$rp,$t0,[x29,#96]	// offload rp and &b[num]

	ldr	$bi,[$bp,#8*0]		// b[0]
	ldp	$a0,$a1,[$ap,#8*0]	// a[0..3]
	ldp	$a2,$a3,[$ap,#8*2]
	add	$ap,$ap,#8*4
	mov	$acc0,xzr
	mov	$acc1,xzr
	mov	$acc2,xzr
	mov	$acc3,xzr
	ldp	$m0,$m1,[$np,#8*0]	// n[0..3]
	ldp	$m2,$m3,[$np,#8*2]
	adds	$np,$np,#8*4		// clear carry bit
	mov	$carry,xzr
	mov	$cnt,#0
	mov	$tp,sp

.Loop_mul4x_1st_reduction:
	mul	$t0,$a0,$bi		// lo(a[0..3]*b[0])
	adc	$carry,$carry,xzr	// modulo-scheduled
	mul	$t1,$a1,$bi
	add	$cnt,$cnt,#8
	mul	$t2,$a2,$bi
	and	$cnt,$cnt,#31
	mul	$t3,$a3,$bi
	adds	$acc0,$acc0,$t0
	umulh	$t0,$a0,$bi		// hi(a[0..3]*b[0])
	adcs	$acc1,$acc1,$t1
	mul	$mi,$acc0,$n0		// t[0]*n0
	adcs	$acc2,$acc2,$t2
	umulh	$t1,$a1,$bi
	adcs	$acc3,$acc3,$t3
	umulh	$t2,$a2,$bi
	adc	$acc4,xzr,xzr
	umulh	$t3,$a3,$bi
	ldr	$bi,[$bp,$cnt]		// next b[i] (or b[0])
	adds	$acc1,$acc1,$t0
	// (*)	mul	$t0,$m0,$mi	// lo(n[0..3]*t[0]*n0)
	str	$mi,[$tp],#8		// put aside t[0]*n0 for tail processing
	adcs	$acc2,$acc2,$t1
	mul	$t1,$m1,$mi
	adcs	$acc3,$acc3,$t2
	mul	$t2,$m2,$mi
	adc	$acc4,$acc4,$t3		// can't overflow
	mul	$t3,$m3,$mi
	// (*)	adds	xzr,$acc0,$t0
	subs	xzr,$acc0,#1		// (*)
	umulh	$t0,$m0,$mi		// hi(n[0..3]*t[0]*n0)
	adcs	$acc0,$acc1,$t1
	umulh	$t1,$m1,$mi
	adcs	$acc1,$acc2,$t2
	umulh	$t2,$m2,$mi
	adcs	$acc2,$acc3,$t3
	umulh	$t3,$m3,$mi
	adcs	$acc3,$acc4,$carry
	adc	$carry,xzr,xzr
	adds	$acc0,$acc0,$t0
	sub	$t0,$ap_end,$ap
	adcs	$acc1,$acc1,$t1
	adcs	$acc2,$acc2,$t2
	adcs	$acc3,$acc3,$t3
	//adc	$carry,$carry,xzr
	cbnz	$cnt,.Loop_mul4x_1st_reduction

	cbz	$t0,.Lmul4x4_post_condition

	ldp	$a0,$a1,[$ap,#8*0]	// a[4..7]
	ldp	$a2,$a3,[$ap,#8*2]
	add	$ap,$ap,#8*4
	ldr	$mi,[sp]		// a[0]*n0
	ldp	$m0,$m1,[$np,#8*0]	// n[4..7]
	ldp	$m2,$m3,[$np,#8*2]
	add	$np,$np,#8*4

.Loop_mul4x_1st_tail:
	mul	$t0,$a0,$bi		// lo(a[4..7]*b[i])
	adc	$carry,$carry,xzr	// modulo-scheduled
	mul	$t1,$a1,$bi
	add	$cnt,$cnt,#8
	mul	$t2,$a2,$bi
	and	$cnt,$cnt,#31
	mul	$t3,$a3,$bi
	adds	$acc0,$acc0,$t0
	umulh	$t0,$a0,$bi		// hi(a[4..7]*b[i])
	adcs	$acc1,$acc1,$t1
	umulh	$t1,$a1,$bi
	adcs	$acc2,$acc2,$t2
	umulh	$t2,$a2,$bi
	adcs	$acc3,$acc3,$t3
	umulh	$t3,$a3,$bi
	adc	$acc4,xzr,xzr
	ldr	$bi,[$bp,$cnt]		// next b[i] (or b[0])
	adds	$acc1,$acc1,$t0
	mul	$t0,$m0,$mi		// lo(n[4..7]*a[0]*n0)
	adcs	$acc2,$acc2,$t1
	mul	$t1,$m1,$mi
	adcs	$acc3,$acc3,$t2
	mul	$t2,$m2,$mi
	adc	$acc4,$acc4,$t3		// can't overflow
	mul	$t3,$m3,$mi
	adds	$acc0,$acc0,$t0
	umulh	$t0,$m0,$mi		// hi(n[4..7]*a[0]*n0)
	adcs	$acc1,$acc1,$t1
	umulh	$t1,$m1,$mi
	adcs	$acc2,$acc2,$t2
	umulh	$t2,$m2,$mi
	adcs	$acc3,$acc3,$t3
	adcs	$acc4,$acc4,$carry
	umulh	$t3,$m3,$mi
	adc	$carry,xzr,xzr
	ldr	$mi,[sp,$cnt]		// next t[0]*n0
	str	$acc0,[$tp],#8		// result!!!
	adds	$acc0,$acc1,$t0
	sub	$t0,$ap_end,$ap		// done yet?
	adcs	$acc1,$acc2,$t1
	adcs	$acc2,$acc3,$t2
	adcs	$acc3,$acc4,$t3
	//adc	$carry,$carry,xzr
	cbnz	$cnt,.Loop_mul4x_1st_tail

	sub	$t1,$ap_end,$num	// rewinded $ap
	cbz	$t0,.Lmul4x_proceed

	ldp	$a0,$a1,[$ap,#8*0]
	ldp	$a2,$a3,[$ap,#8*2]
	add	$ap,$ap,#8*4
	ldp	$m0,$m1,[$np,#8*0]
	ldp	$m2,$m3,[$np,#8*2]
	add	$np,$np,#8*4
	b	.Loop_mul4x_1st_tail

.align	5
.Lmul4x_proceed:
	ldr	$bi,[$bp,#8*4]!		// *++b
	adc	$topmost,$carry,xzr
	ldp	$a0,$a1,[$t1,#8*0]	// a[0..3]
	sub	$np,$np,$num		// rewind np
	ldp	$a2,$a3,[$t1,#8*2]
	add	$ap,$t1,#8*4

	stp	$acc0,$acc1,[$tp,#8*0]	// result!!!
	ldp	$acc0,$acc1,[sp,#8*4]	// t[0..3]
	stp	$acc2,$acc3,[$tp,#8*2]	// result!!!
	ldp	$acc2,$acc3,[sp,#8*6]

	ldp	$m0,$m1,[$np,#8*0]	// n[0..3]
	mov	$tp,sp
	ldp	$m2,$m3,[$np,#8*2]
	adds	$np,$np,#8*4		// clear carry bit
	mov	$carry,xzr

.align	4
.Loop_mul4x_reduction:
	mul	$t0,$a0,$bi		// lo(a[0..3]*b[4])
	adc	$carry,$carry,xzr	// modulo-scheduled
	mul	$t1,$a1,$bi
	add	$cnt,$cnt,#8
	mul	$t2,$a2,$bi
	and	$cnt,$cnt,#31
	mul	$t3,$a3,$bi
	adds	$acc0,$acc0,$t0
	umulh	$t0,$a0,$bi		// hi(a[0..3]*b[4])
	adcs	$acc1,$acc1,$t1
	mul	$mi,$acc0,$n0		// t[0]*n0
	adcs	$acc2,$acc2,$t2
	umulh	$t1,$a1,$bi
	adcs	$acc3,$acc3,$t3
	umulh	$t2,$a2,$bi
	adc	$acc4,xzr,xzr
	umulh	$t3,$a3,$bi
	ldr	$bi,[$bp,$cnt]		// next b[i]
	adds	$acc1,$acc1,$t0
	// (*)	mul	$t0,$m0,$mi
	str	$mi,[$tp],#8		// put aside t[0]*n0 for tail processing
	adcs	$acc2,$acc2,$t1
	mul	$t1,$m1,$mi		// lo(n[0..3]*t[0]*n0
	adcs	$acc3,$acc3,$t2
	mul	$t2,$m2,$mi
	adc	$acc4,$acc4,$t3		// can't overflow
	mul	$t3,$m3,$mi
	// (*)	adds	xzr,$acc0,$t0
	subs	xzr,$acc0,#1		// (*)
	umulh	$t0,$m0,$mi		// hi(n[0..3]*t[0]*n0
	adcs	$acc0,$acc1,$t1
	umulh	$t1,$m1,$mi
	adcs	$acc1,$acc2,$t2
	umulh	$t2,$m2,$mi
	adcs	$acc2,$acc3,$t3
	umulh	$t3,$m3,$mi
	adcs	$acc3,$acc4,$carry
	adc	$carry,xzr,xzr
	adds	$acc0,$acc0,$t0
	adcs	$acc1,$acc1,$t1
	adcs	$acc2,$acc2,$t2
	adcs	$acc3,$acc3,$t3
	//adc	$carry,$carry,xzr
	cbnz	$cnt,.Loop_mul4x_reduction

	adc	$carry,$carry,xzr
	ldp	$t0,$t1,[$tp,#8*4]	// t[4..7]
	ldp	$t2,$t3,[$tp,#8*6]
	ldp	$a0,$a1,[$ap,#8*0]	// a[4..7]
	ldp	$a2,$a3,[$ap,#8*2]
	add	$ap,$ap,#8*4
	adds	$acc0,$acc0,$t0
	adcs	$acc1,$acc1,$t1
	adcs	$acc2,$acc2,$t2
	adcs	$acc3,$acc3,$t3
	//adc	$carry,$carry,xzr

	ldr	$mi,[sp]		// t[0]*n0
	ldp	$m0,$m1,[$np,#8*0]	// n[4..7]
	ldp	$m2,$m3,[$np,#8*2]
	add	$np,$np,#8*4

.align	4
.Loop_mul4x_tail:
	mul	$t0,$a0,$bi		// lo(a[4..7]*b[4])
	adc	$carry,$carry,xzr	// modulo-scheduled
	mul	$t1,$a1,$bi
	add	$cnt,$cnt,#8
	mul	$t2,$a2,$bi
	and	$cnt,$cnt,#31
	mul	$t3,$a3,$bi
	adds	$acc0,$acc0,$t0
	umulh	$t0,$a0,$bi		// hi(a[4..7]*b[4])
	adcs	$acc1,$acc1,$t1
	umulh	$t1,$a1,$bi
	adcs	$acc2,$acc2,$t2
	umulh	$t2,$a2,$bi
	adcs	$acc3,$acc3,$t3
	umulh	$t3,$a3,$bi
	adc	$acc4,xzr,xzr
	ldr	$bi,[$bp,$cnt]		// next b[i]
	adds	$acc1,$acc1,$t0
	mul	$t0,$m0,$mi		// lo(n[4..7]*t[0]*n0)
	adcs	$acc2,$acc2,$t1
	mul	$t1,$m1,$mi
	adcs	$acc3,$acc3,$t2
	mul	$t2,$m2,$mi
	adc	$acc4,$acc4,$t3		// can't overflow
	mul	$t3,$m3,$mi
	adds	$acc0,$acc0,$t0
	umulh	$t0,$m0,$mi		// hi(n[4..7]*t[0]*n0)
	adcs	$acc1,$acc1,$t1
	umulh	$t1,$m1,$mi
	adcs	$acc2,$acc2,$t2
	umulh	$t2,$m2,$mi
	adcs	$acc3,$acc3,$t3
	umulh	$t3,$m3,$mi
	adcs	$acc4,$acc4,$carry
	ldr	$mi,[sp,$cnt]		// next a[0]*n0
	adc	$carry,xzr,xzr
	str	$acc0,[$tp],#8		// result!!!
	adds	$acc0,$acc1,$t0
	sub	$t0,$ap_end,$ap		// done yet?
	adcs	$acc1,$acc2,$t1
	adcs	$acc2,$acc3,$t2
	adcs	$acc3,$acc4,$t3
	//adc	$carry,$carry,xzr
	cbnz	$cnt,.Loop_mul4x_tail

	sub	$t1,$np,$num		// rewinded np?
	adc	$carry,$carry,xzr
	cbz	$t0,.Loop_mul4x_break

	ldp	$t0,$t1,[$tp,#8*4]
	ldp	$t2,$t3,[$tp,#8*6]
	ldp	$a0,$a1,[$ap,#8*0]
	ldp	$a2,$a3,[$ap,#8*2]
	add	$ap,$ap,#8*4
	adds	$acc0,$acc0,$t0
	adcs	$acc1,$acc1,$t1
	adcs	$acc2,$acc2,$t2
	adcs	$acc3,$acc3,$t3
	//adc	$carry,$carry,xzr
	ldp	$m0,$m1,[$np,#8*0]
	ldp	$m2,$m3,[$np,#8*2]
	add	$np,$np,#8*4
	b	.Loop_mul4x_tail

.align	4
.Loop_mul4x_break:
	ldp	$t2,$t3,[x29,#96]	// pull rp and &b[num]
	adds	$acc0,$acc0,$topmost
	add	$bp,$bp,#8*4		// bp++
	adcs	$acc1,$acc1,xzr
	sub	$ap,$ap,$num		// rewind ap
	adcs	$acc2,$acc2,xzr
	stp	$acc0,$acc1,[$tp,#8*0]	// result!!!
	adcs	$acc3,$acc3,xzr
	ldp	$acc0,$acc1,[sp,#8*4]	// t[0..3]
	adc	$topmost,$carry,xzr
	stp	$acc2,$acc3,[$tp,#8*2]	// result!!!
	cmp	$bp,$t3			// done yet?
	ldp	$acc2,$acc3,[sp,#8*6]
	ldp	$m0,$m1,[$t1,#8*0]	// n[0..3]
	ldp	$m2,$m3,[$t1,#8*2]
	add	$np,$t1,#8*4
	b.eq	.Lmul4x_post

	ldr	$bi,[$bp]
	ldp	$a0,$a1,[$ap,#8*0]	// a[0..3]
	ldp	$a2,$a3,[$ap,#8*2]
	adds	$ap,$ap,#8*4		// clear carry bit
	mov	$carry,xzr
	mov	$tp,sp
	b	.Loop_mul4x_reduction

.align	4
.Lmul4x_post:
	// Final step. We see if result is larger than modulus, and
	// if it is, subtract the modulus. But comparison implies
	// subtraction. So we subtract modulus, see if it borrowed,
	// and conditionally copy original value.
	mov	$rp,$t2
	mov	$ap_end,$t2		// $rp copy
	subs	$t0,$acc0,$m0
	add	$tp,sp,#8*8
	sbcs	$t1,$acc1,$m1
	sub	$cnt,$num,#8*4

.Lmul4x_sub:
	sbcs	$t2,$acc2,$m2
	ldp	$m0,$m1,[$np,#8*0]
	sub	$cnt,$cnt,#8*4
	ldp	$acc0,$acc1,[$tp,#8*0]
	sbcs	$t3,$acc3,$m3
	ldp	$m2,$m3,[$np,#8*2]
	add	$np,$np,#8*4
	ldp	$acc2,$acc3,[$tp,#8*2]
	add	$tp,$tp,#8*4
	stp	$t0,$t1,[$rp,#8*0]
	sbcs	$t0,$acc0,$m0
	stp	$t2,$t3,[$rp,#8*2]
	add	$rp,$rp,#8*4
	sbcs	$t1,$acc1,$m1
	cbnz	$cnt,.Lmul4x_sub

	sbcs	$t2,$acc2,$m2
	 mov	$tp,sp
	 add	$ap,sp,#8*4
	 ldp	$a0,$a1,[$ap_end,#8*0]
	sbcs	$t3,$acc3,$m3
	stp	$t0,$t1,[$rp,#8*0]
	 ldp	$a2,$a3,[$ap_end,#8*2]
	stp	$t2,$t3,[$rp,#8*2]
	 ldp	$acc0,$acc1,[$ap,#8*0]
	 ldp	$acc2,$acc3,[$ap,#8*2]
	sbcs	xzr,$topmost,xzr	// did it borrow?
	ldr	x30,[x29,#8]		// pull return address

	sub	$cnt,$num,#8*4
.Lmul4x_cond_copy:
	sub	$cnt,$cnt,#8*4
	csel	$t0,$acc0,$a0,lo
	 stp	xzr,xzr,[$tp,#8*0]
	csel	$t1,$acc1,$a1,lo
	ldp	$a0,$a1,[$ap_end,#8*4]
	ldp	$acc0,$acc1,[$ap,#8*4]
	csel	$t2,$acc2,$a2,lo
	 stp	xzr,xzr,[$tp,#8*2]
	 add	$tp,$tp,#8*4
	csel	$t3,$acc3,$a3,lo
	ldp	$a2,$a3,[$ap_end,#8*6]
	ldp	$acc2,$acc3,[$ap,#8*6]
	add	$ap,$ap,#8*4
	stp	$t0,$t1,[$ap_end,#8*0]
	stp	$t2,$t3,[$ap_end,#8*2]
	add	$ap_end,$ap_end,#8*4
	cbnz	$cnt,.Lmul4x_cond_copy

	csel	$t0,$acc0,$a0,lo
	 stp	xzr,xzr,[$tp,#8*0]
	csel	$t1,$acc1,$a1,lo
	 stp	xzr,xzr,[$tp,#8*2]
	csel	$t2,$acc2,$a2,lo
	 stp	xzr,xzr,[$tp,#8*3]
	csel	$t3,$acc3,$a3,lo
	 stp	xzr,xzr,[$tp,#8*4]
	stp	$t0,$t1,[$ap_end,#8*0]
	stp	$t2,$t3,[$ap_end,#8*2]

	b	.Lmul4x_done

.align	4
.Lmul4x4_post_condition:
	adc	$carry,$carry,xzr
	ldr	$ap,[x29,#96]		// pull rp
	// $acc0-3,$carry hold result, $m0-7 hold modulus
	subs	$a0,$acc0,$m0
	ldr	x30,[x29,#8]		// pull return address
	sbcs	$a1,$acc1,$m1
	 stp	xzr,xzr,[sp,#8*0]
	sbcs	$a2,$acc2,$m2
	 stp	xzr,xzr,[sp,#8*2]
	sbcs	$a3,$acc3,$m3
	 stp	xzr,xzr,[sp,#8*4]
	sbcs	xzr,$carry,xzr		// did it borrow?
	 stp	xzr,xzr,[sp,#8*6]

	// $a0-3 hold result-modulus
	csel	$a0,$acc0,$a0,lo
	csel	$a1,$acc1,$a1,lo
	csel	$a2,$acc2,$a2,lo
	csel	$a3,$acc3,$a3,lo
	stp	$a0,$a1,[$ap,#8*0]
	stp	$a2,$a3,[$ap,#8*2]

.Lmul4x_done:
	ldp	x19,x20,[x29,#16]
	mov	sp,x29
	ldp	x21,x22,[x29,#32]
	mov	x0,#1
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldr	x29,[sp],#128
	.inst	0xd50323bf		// autiasp
	ret
.size	__bn_mul4x_mont,.-__bn_mul4x_mont
___
}
$code.=<<___;
.asciz	"Montgomery Multiplication for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align	4
___

print $code;

close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";

$output = pop;
open STDOUT,">$output";

&asm_init($ARGV[0]);

$sse2=0;
for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }

&external_label("OPENSSL_ia32cap_P") if ($sse2);

&bn_mul_add_words("bn_mul_add_words");
&bn_mul_words("bn_mul_words");
&bn_sqr_words("bn_sqr_words");
&bn_div_words("bn_div_words");
&bn_add_words("bn_add_words");
&bn_sub_words("bn_sub_words");
&bn_sub_part_words("bn_sub_part_words");

&asm_finish();

close STDOUT or die "error closing STDOUT: $!";

sub bn_mul_add_words
	{
	local($name)=@_;

	&function_begin_B($name,$sse2?"EXTRN\t_OPENSSL_ia32cap_P:DWORD":"");

	$r="eax";
	$a="edx";
	$c="ecx";

	if ($sse2) {
		&picmeup("eax","OPENSSL_ia32cap_P");
		&bt(&DWP(0,"eax"),26);
		&jnc(&label("maw_non_sse2"));

		&mov($r,&wparam(0));
		&mov($a,&wparam(1));
		&mov($c,&wparam(2));
		&movd("mm0",&wparam(3));	# mm0 = w
		&pxor("mm1","mm1");		# mm1 = carry_in
		&jmp(&label("maw_sse2_entry"));

	&set_label("maw_sse2_unrolled",16);
		&movd("mm3",&DWP(0,$r,"",0));	# mm3 = r[0]
		&paddq("mm1","mm3");		# mm1 = carry_in + r[0]
		&movd("mm2",&DWP(0,$a,"",0));	# mm2 = a[0]
		&pmuludq("mm2","mm0");		# mm2 = w*a[0]
		&movd("mm4",&DWP(4,$a,"",0));	# mm4 = a[1]
		&pmuludq("mm4","mm0");		# mm4 = w*a[1]
		&movd("mm6",&DWP(8,$a,"",0));	# mm6 = a[2]
		&pmuludq("mm6","mm0");		# mm6 = w*a[2]
		&movd("mm7",&DWP(12,$a,"",0));	# mm7 = a[3]
		&pmuludq("mm7","mm0");		# mm7 = w*a[3]
		&paddq("mm1","mm2");		# mm1 = carry_in + r[0] + w*a[0]
		&movd("mm3",&DWP(4,$r,"",0));	# mm3 = r[1]
		&paddq("mm3","mm4");		# mm3 = r[1] + w*a[1]
		&movd("mm5",&DWP(8,$r,"",0));	# mm5 = r[2]
		&paddq("mm5","mm6");		# mm5 = r[2] + w*a[2]
		&movd("mm4",&DWP(12,$r,"",0));	# mm4 = r[3]
		&paddq("mm7","mm4");		# mm7 = r[3] + w*a[3]
		&movd(&DWP(0,$r,"",0),"mm1");
		&movd("mm2",&DWP(16,$a,"",0));	# mm2 = a[4]
		&pmuludq("mm2","mm0");		# mm2 = w*a[4]
		&psrlq("mm1",32);		# mm1 = carry0
		&movd("mm4",&DWP(20,$a,"",0));	# mm4 = a[5]
		&pmuludq("mm4","mm0");		# mm4 = w*a[5]
		&paddq("mm1","mm3");		# mm1 = carry0 + r[1] + w*a[1]
		&movd("mm6",&DWP(24,$a,"",0));	# mm6 = a[6]
		&pmuludq("mm6","mm0");		# mm6 = w*a[6]
		&movd(&DWP(4,$r,"",0),"mm1");
		&psrlq("mm1",32);		# mm1 = carry1
		&movd("mm3",&DWP(28,$a,"",0));	# mm3 = a[7]
		&add($a,32);
		&pmuludq("mm3","mm0");		# mm3 = w*a[7]
		&paddq("mm1","mm5");		# mm1 = carry1 + r[2] + w*a[2]
		&movd("mm5",&DWP(16,$r,"",0));	# mm5 = r[4]
		&paddq("mm2","mm5");		# mm2 = r[4] + w*a[4]
		&movd(&DWP(8,$r,"",0),"mm1");
		&psrlq("mm1",32);		# mm1 = carry2
		&paddq("mm1","mm7");		# mm1 = carry2 + r[3] + w*a[3]
		&movd("mm5",&DWP(20,$r,"",0));	# mm5 = r[5]
		&paddq("mm4","mm5");		# mm4 = r[5] + w*a[5]
		&movd(&DWP(12,$r,"",0),"mm1");
		&psrlq("mm1",32);		# mm1 = carry3
		&paddq("mm1","mm2");		# mm1 = carry3 + r[4] + w*a[4]
		&movd("mm5",&DWP(24,$r,"",0));	# mm5 = r[6]
		&paddq("mm6","mm5");		# mm6 = r[6] + w*a[6]
		&movd(&DWP(16,$r,"",0),"mm1");
		&psrlq("mm1",32);		# mm1 = carry4
		&paddq("mm1","mm4");		# mm1 = carry4 + r[5] + w*a[5]
		&movd("mm5",&DWP(28,$r,"",0));	# mm5 = r[7]
		&paddq("mm3","mm5");		# mm3 = r[7] + w*a[7]
		&movd(&DWP(20,$r,"",0),"mm1");
		&psrlq("mm1",32);		# mm1 = carry5
		&paddq("mm1","mm6");		# mm1 = carry5 + r[6] + w*a[6]
		&movd(&DWP(24,$r,"",0),"mm1");
		&psrlq("mm1",32);		# mm1 = carry6
		&paddq("mm1","mm3");		# mm1 = carry6 + r[7] + w*a[7]
		&movd(&DWP(28,$r,"",0),"mm1");
		&lea($r,&DWP(32,$r));
		&psrlq("mm1",32);		# mm1 = carry_out

		&sub($c,8);
		&jz(&label("maw_sse2_exit"));
	&set_label("maw_sse2_entry");
		&test($c,0xfffffff8);
		&jnz(&label("maw_sse2_unrolled"));

	&set_label("maw_sse2_loop",4);
		&movd("mm2",&DWP(0,$a));	# mm2 = a[i]
		&movd("mm3",&DWP(0,$r));	# mm3 = r[i]
		&pmuludq("mm2","mm0");		# a[i] *= w
		&lea($a,&DWP(4,$a));
		&paddq("mm1","mm3");		# carry += r[i]
		&paddq("mm1","mm2");		# carry += a[i]*w
		&movd(&DWP(0,$r),"mm1");	# r[i] = carry_low
		&sub($c,1);
		&psrlq("mm1",32);		# carry = carry_high
		&lea($r,&DWP(4,$r));
		&jnz(&label("maw_sse2_loop"));
	&set_label("maw_sse2_exit");
		&movd("eax","mm1");		# c = carry_out
		&emms();
		&ret();

	&set_label("maw_non_sse2",16);
	}

	# function_begin prologue
	&push("ebp");
	&push("ebx");
	&push("esi");
	&push("edi");

	&comment("");
	$Low="eax";
	$High="edx";
	$a="ebx";
	$w="ebp";
	$r="edi";
	$c="esi";

	&xor($c,$c);		# clear carry
	&mov($r,&wparam(0));	#

	&mov("ecx",&wparam(2));	#
	&mov($a,&wparam(1));	#

	&and("ecx",0xfffffff8);	# num / 8
	&mov($w,&wparam(3));	#

	&push("ecx");		# Up the stack for a tmp variable

	&jz(&label("maw_finish"));

	&set_label("maw_loop",16);

	for ($i=0; $i<32; $i+=4)
		{
		&comment("Round $i");

		 &mov("eax",&DWP($i,$a)); 	# *a
		&mul($w);			# *a * w
		&add("eax",$c);			# L(t)+= c
		&adc("edx",0);			# H(t)+=carry
		 &add("eax",&DWP($i,$r));	# L(t)+= *r
		&adc("edx",0);			# H(t)+=carry
		 &mov(&DWP($i,$r),"eax");	# *r= L(t);
		&mov($c,"edx");			# c=  H(t);
		}

	&comment("");
	&sub("ecx",8);
	&lea($a,&DWP(32,$a));
	&lea($r,&DWP(32,$r));
	&jnz(&label("maw_loop"));

	&set_label("maw_finish",0);
	&mov("ecx",&wparam(2));	# get num
	&and("ecx",7);
	&jnz(&label("maw_finish2"));	# helps branch prediction
	&jmp(&label("maw_end"));

	&set_label("maw_finish2",1);
	for ($i=0; $i<7; $i++)
		{
		&comment("Tail Round $i");
		 &mov("eax",&DWP($i*4,$a));	# *a
		&mul($w);			# *a * w
		&add("eax",$c);			# L(t)+=c
		&adc("edx",0);			# H(t)+=carry
		 &add("eax",&DWP($i*4,$r));	# L(t)+= *r
		&adc("edx",0);			# H(t)+=carry
		 &dec("ecx") if ($i != 7-1);
		&mov(&DWP($i*4,$r),"eax");	# *r= L(t);
		 &mov($c,"edx");		# c=  H(t);
		&jz(&label("maw_end")) if ($i != 7-1);
		}
	&set_label("maw_end",0);
	&mov("eax",$c);

	&pop("ecx");	# clear variable from

	&function_end($name);
	}

sub bn_mul_words
	{
	local($name)=@_;

	&function_begin_B($name,$sse2?"EXTRN\t_OPENSSL_ia32cap_P:DWORD":"");

	$r="eax";
	$a="edx";
	$c="ecx";

	if ($sse2) {
		&picmeup("eax","OPENSSL_ia32cap_P");
		&bt(&DWP(0,"eax"),26);
		&jnc(&label("mw_non_sse2"));

		&mov($r,&wparam(0));
		&mov($a,&wparam(1));
		&mov($c,&wparam(2));
		&movd("mm0",&wparam(3));	# mm0 = w
		&pxor("mm1","mm1");		# mm1 = carry = 0

	&set_label("mw_sse2_loop",16);
		&movd("mm2",&DWP(0,$a));	# mm2 = a[i]
		&pmuludq("mm2","mm0");		# a[i] *= w
		&lea($a,&DWP(4,$a));
		&paddq("mm1","mm2");		# carry += a[i]*w
		&movd(&DWP(0,$r),"mm1");	# r[i] = carry_low
		&sub($c,1);
		&psrlq("mm1",32);		# carry = carry_high
		&lea($r,&DWP(4,$r));
		&jnz(&label("mw_sse2_loop"));

		&movd("eax","mm1");		# return carry
		&emms();
		&ret();
	&set_label("mw_non_sse2",16);
	}

	# function_begin prologue
	&push("ebp");
	&push("ebx");
	&push("esi");
	&push("edi");

	&comment("");
	$Low="eax";
	$High="edx";
	$a="ebx";
	$w="ecx";
	$r="edi";
	$c="esi";
	$num="ebp";

	&xor($c,$c);		# clear carry
	&mov($r,&wparam(0));	#
	&mov($a,&wparam(1));	#
	&mov($num,&wparam(2));	#
	&mov($w,&wparam(3));	#

	&and($num,0xfffffff8);	# num / 8
	&jz(&label("mw_finish"));

	&set_label("mw_loop",0);
	for ($i=0; $i<32; $i+=4)
		{
		&comment("Round $i");

		 &mov("eax",&DWP($i,$a,"",0)); 	# *a
		&mul($w);			# *a * w
		&add("eax",$c);			# L(t)+=c
		 # XXX

		&adc("edx",0);			# H(t)+=carry
		 &mov(&DWP($i,$r,"",0),"eax");	# *r= L(t);

		&mov($c,"edx");			# c=  H(t);
		}

	&comment("");
	&add($a,32);
	&add($r,32);
	&sub($num,8);
	&jz(&label("mw_finish"));
	&jmp(&label("mw_loop"));

	&set_label("mw_finish",0);
	&mov($num,&wparam(2));	# get num
	&and($num,7);
	&jnz(&label("mw_finish2"));
	&jmp(&label("mw_end"));

	&set_label("mw_finish2",1);
	for ($i=0; $i<7; $i++)
		{
		&comment("Tail Round $i");
		 &mov("eax",&DWP($i*4,$a,"",0));# *a
		&mul($w);			# *a * w
		&add("eax",$c);			# L(t)+=c
		 # XXX
		&adc("edx",0);			# H(t)+=carry
		 &mov(&DWP($i*4,$r,"",0),"eax");# *r= L(t);
		&mov($c,"edx");			# c=  H(t);
		 &dec($num) if ($i != 7-1);
		&jz(&label("mw_end")) if ($i != 7-1);
		}
	&set_label("mw_end",0);
	&mov("eax",$c);

	&function_end($name);
	}

sub bn_sqr_words
	{
	local($name)=@_;

	&function_begin_B($name,$sse2?"EXTRN\t_OPENSSL_ia32cap_P:DWORD":"");

	$r="eax";
	$a="edx";
	$c="ecx";

	if ($sse2) {
		&picmeup("eax","OPENSSL_ia32cap_P");
		&bt(&DWP(0,"eax"),26);
		&jnc(&label("sqr_non_sse2"));

		&mov($r,&wparam(0));
		&mov($a,&wparam(1));
		&mov($c,&wparam(2));

	&set_label("sqr_sse2_loop",16);
		&movd("mm0",&DWP(0,$a));	# mm0 = a[i]
		&pmuludq("mm0","mm0");		# a[i] *= a[i]
		&lea($a,&DWP(4,$a));		# a++
		&movq(&QWP(0,$r),"mm0");	# r[i] = a[i]*a[i]
		&sub($c,1);
		&lea($r,&DWP(8,$r));		# r += 2
		&jnz(&label("sqr_sse2_loop"));

		&emms();
		&ret();
	&set_label("sqr_non_sse2",16);
	}

	# function_begin prologue
	&push("ebp");
	&push("ebx");
	&push("esi");
	&push("edi");

	&comment("");
	$r="esi";
	$a="edi";
	$num="ebx";

	&mov($r,&wparam(0));	#
	&mov($a,&wparam(1));	#
	&mov($num,&wparam(2));	#

	&and($num,0xfffffff8);	# num / 8
	&jz(&label("sw_finish"));

	&set_label("sw_loop",0);
	for ($i=0; $i<32; $i+=4)
		{
		&comment("Round $i");
		&mov("eax",&DWP($i,$a,"",0)); 	# *a
		 # XXX
		&mul("eax");			# *a * *a
		&mov(&DWP($i*2,$r,"",0),"eax");	#
		 &mov(&DWP($i*2+4,$r,"",0),"edx");#
		}

	&comment("");
	&add($a,32);
	&add($r,64);
	&sub($num,8);
	&jnz(&label("sw_loop"));

	&set_label("sw_finish",0);
	&mov($num,&wparam(2));	# get num
	&and($num,7);
	&jz(&label("sw_end"));

	for ($i=0; $i<7; $i++)
		{
		&comment("Tail Round $i");
		&mov("eax",&DWP($i*4,$a,"",0));	# *a
		 # XXX
		&mul("eax");			# *a * *a
		&mov(&DWP($i*8,$r,"",0),"eax");	#
		 &dec($num) if ($i != 7-1);
		&mov(&DWP($i*8+4,$r,"",0),"edx");
		 &jz(&label("sw_end")) if ($i != 7-1);
		}
	&set_label("sw_end",0);

	&function_end($name);
	}

sub bn_div_words
	{
	local($name)=@_;

	&function_begin_B($name,"");
	&mov("edx",&wparam(0));	#
	&mov("eax",&wparam(1));	#
	&mov("ecx",&wparam(2));	#
	&div("ecx");
	&ret();
	&function_end_B($name);
	}

sub bn_add_words
	{
	local($name)=@_;

	&function_begin($name,"");

	&comment("");
	$a="esi";
	$b="edi";
	$c="eax";
	$r="ebx";
	$tmp1="ecx";
	$tmp2="edx";
	$num="ebp";

	&mov($r,&wparam(0));	# get r
	 &mov($a,&wparam(1));	# get a
	&mov($b,&wparam(2));	# get b
	 &mov($num,&wparam(3));	# get num
	&xor($c,$c);		# clear carry
	 &and($num,0xfffffff8);	# num / 8

	&jz(&label("aw_finish"));

	&set_label("aw_loop",0);
	for ($i=0; $i<8; $i++)
		{
		&comment("Round $i");

		&mov($tmp1,&DWP($i*4,$a,"",0)); 	# *a
		 &mov($tmp2,&DWP($i*4,$b,"",0)); 	# *b
		&add($tmp1,$c);
		 &mov($c,0);
		&adc($c,$c);
		 &add($tmp1,$tmp2);
		&adc($c,0);
		 &mov(&DWP($i*4,$r,"",0),$tmp1); 	# *r
		}

	&comment("");
	&add($a,32);
	 &add($b,32);
	&add($r,32);
	 &sub($num,8);
	&jnz(&label("aw_loop"));

	&set_label("aw_finish",0);
	&mov($num,&wparam(3));	# get num
	&and($num,7);
	 &jz(&label("aw_end"));

	for ($i=0; $i<7; $i++)
		{
		&comment("Tail Round $i");
		&mov($tmp1,&DWP($i*4,$a,"",0));	# *a
		 &mov($tmp2,&DWP($i*4,$b,"",0));# *b
		&add($tmp1,$c);
		 &mov($c,0);
		&adc($c,$c);
		 &add($tmp1,$tmp2);
		&adc($c,0);
		 &dec($num) if ($i != 6);
		&mov(&DWP($i*4,$r,"",0),$tmp1);	# *r
		 &jz(&label("aw_end")) if ($i != 6);
		}
	&set_label("aw_end",0);

#	&mov("eax",$c);		# $c is "eax"

	&function_end($name);
	}

sub bn_sub_words
	{
	local($name)=@_;

	&function_begin($name,"");

	&comment("");
	$a="esi";
	$b="edi";
	$c="eax";
	$r="ebx";
	$tmp1="ecx";
	$tmp2="edx";
	$num="ebp";

	&mov($r,&wparam(0));	# get r
	 &mov($a,&wparam(1));	# get a
	&mov($b,&wparam(2));	# get b
	 &mov($num,&wparam(3));	# get num
	&xor($c,$c);		# clear carry
	 &and($num,0xfffffff8);	# num / 8

	&jz(&label("aw_finish"));

	&set_label("aw_loop",0);
	for ($i=0; $i<8; $i++)
		{
		&comment("Round $i");

		&mov($tmp1,&DWP($i*4,$a,"",0)); 	# *a
		 &mov($tmp2,&DWP($i*4,$b,"",0)); 	# *b
		&sub($tmp1,$c);
		 &mov($c,0);
		&adc($c,$c);
		 &sub($tmp1,$tmp2);
		&adc($c,0);
		 &mov(&DWP($i*4,$r,"",0),$tmp1); 	# *r
		}

	&comment("");
	&add($a,32);
	 &add($b,32);
	&add($r,32);
	 &sub($num,8);
	&jnz(&label("aw_loop"));

	&set_label("aw_finish",0);
	&mov($num,&wparam(3));	# get num
	&and($num,7);
	 &jz(&label("aw_end"));

	for ($i=0; $i<7; $i++)
		{
		&comment("Tail Round $i");
		&mov($tmp1,&DWP($i*4,$a,"",0));	# *a
		 &mov($tmp2,&DWP($i*4,$b,"",0));# *b
		&sub($tmp1,$c);
		 &mov($c,0);
		&adc($c,$c);
		 &sub($tmp1,$tmp2);
		&adc($c,0);
		 &dec($num) if ($i != 6);
		&mov(&DWP($i*4,$r,"",0),$tmp1);	# *r
		 &jz(&label("aw_end")) if ($i != 6);
		}
	&set_label("aw_end",0);

#	&mov("eax",$c);		# $c is "eax"

	&function_end($name);
	}

sub bn_sub_part_words
	{
	local($name)=@_;

	&function_begin($name,"");

	&comment("");
	$a="esi";
	$b="edi";
	$c="eax";
	$r="ebx";
	$tmp1="ecx";
	$tmp2="edx";
	$num="ebp";

	&mov($r,&wparam(0));	# get r
	 &mov($a,&wparam(1));	# get a
	&mov($b,&wparam(2));	# get b
	 &mov($num,&wparam(3));	# get num
	&xor($c,$c);		# clear carry
	 &and($num,0xfffffff8);	# num / 8

	&jz(&label("aw_finish"));

	&set_label("aw_loop",0);
	for ($i=0; $i<8; $i++)
		{
		&comment("Round $i");

		&mov($tmp1,&DWP($i*4,$a,"",0)); 	# *a
		 &mov($tmp2,&DWP($i*4,$b,"",0)); 	# *b
		&sub($tmp1,$c);
		 &mov($c,0);
		&adc($c,$c);
		 &sub($tmp1,$tmp2);
		&adc($c,0);
		 &mov(&DWP($i*4,$r,"",0),$tmp1); 	# *r
		}

	&comment("");
	&add($a,32);
	 &add($b,32);
	&add($r,32);
	 &sub($num,8);
	&jnz(&label("aw_loop"));

	&set_label("aw_finish",0);
	&mov($num,&wparam(3));	# get num
	&and($num,7);
	 &jz(&label("aw_end"));

	for ($i=0; $i<7; $i++)
		{
		&comment("Tail Round $i");
		&mov($tmp1,&DWP(0,$a,"",0));	# *a
		 &mov($tmp2,&DWP(0,$b,"",0));# *b
		&sub($tmp1,$c);
		 &mov($c,0);
		&adc($c,$c);
		 &sub($tmp1,$tmp2);
		&adc($c,0);
		&mov(&DWP(0,$r,"",0),$tmp1);	# *r
		&add($a, 4);
		&add($b, 4);
		&add($r, 4);
		 &dec($num) if ($i != 6);
		 &jz(&label("aw_end")) if ($i != 6);
		}
	&set_label("aw_end",0);

	&cmp(&wparam(4),0);
	&je(&label("pw_end"));

	&mov($num,&wparam(4));	# get dl
	&cmp($num,0);
	&je(&label("pw_end"));
	&jge(&label("pw_pos"));

	&comment("pw_neg");
	&mov($tmp2,0);
	&sub($tmp2,$num);
	&mov($num,$tmp2);
	&and($num,0xfffffff8);	# num / 8
	&jz(&label("pw_neg_finish"));

	&set_label("pw_neg_loop",0);
	for ($i=0; $i<8; $i++)
	{
	    &comment("dl<0 Round $i");

	    &mov($tmp1,0);
	    &mov($tmp2,&DWP($i*4,$b,"",0)); 	# *b
	    &sub($tmp1,$c);
	    &mov($c,0);
	    &adc($c,$c);
	    &sub($tmp1,$tmp2);
	    &adc($c,0);
	    &mov(&DWP($i*4,$r,"",0),$tmp1); 	# *r
	}

	&comment("");
	&add($b,32);
	&add($r,32);
	&sub($num,8);
	&jnz(&label("pw_neg_loop"));

	&set_label("pw_neg_finish",0);
	&mov($tmp2,&wparam(4));	# get dl
	&mov($num,0);
	&sub($num,$tmp2);
	&and($num,7);
	&jz(&label("pw_end"));

	for ($i=0; $i<7; $i++)
	{
	    &comment("dl<0 Tail Round $i");
	    &mov($tmp1,0);
	    &mov($tmp2,&DWP($i*4,$b,"",0));# *b
	    &sub($tmp1,$c);
	    &mov($c,0);
	    &adc($c,$c);
	    &sub($tmp1,$tmp2);
	    &adc($c,0);
	    &dec($num) if ($i != 6);
	    &mov(&DWP($i*4,$r,"",0),$tmp1);	# *r
	    &jz(&label("pw_end")) if ($i != 6);
	}

	&jmp(&label("pw_end"));

	&set_label("pw_pos",0);

	&and($num,0xfffffff8);	# num / 8
	&jz(&label("pw_pos_finish"));

	&set_label("pw_pos_loop",0);

	for ($i=0; $i<8; $i++)
	{
	    &comment("dl>0 Round $i");

	    &mov($tmp1,&DWP($i*4,$a,"",0));	# *a
	    &sub($tmp1,$c);
	    &mov(&DWP($i*4,$r,"",0),$tmp1);	# *r
	    &jnc(&label("pw_nc".$i));
	}

	&comment("");
	&add($a,32);
	&add($r,32);
	&sub($num,8);
	&jnz(&label("pw_pos_loop"));

	&set_label("pw_pos_finish",0);
	&mov($num,&wparam(4));	# get dl
	&and($num,7);
	&jz(&label("pw_end"));

	for ($i=0; $i<7; $i++)
	{
	    &comment("dl>0 Tail Round $i");
	    &mov($tmp1,&DWP($i*4,$a,"",0));	# *a
	    &sub($tmp1,$c);
	    &mov(&DWP($i*4,$r,"",0),$tmp1);	# *r
	    &jnc(&label("pw_tail_nc".$i));
	    &dec($num) if ($i != 6);
	    &jz(&label("pw_end")) if ($i != 6);
	}
	&mov($c,1);
	&jmp(&label("pw_end"));

	&set_label("pw_nc_loop",0);
	for ($i=0; $i<8; $i++)
	{
	    &mov($tmp1,&DWP($i*4,$a,"",0));	# *a
	    &mov(&DWP($i*4,$r,"",0),$tmp1);	# *r
	    &set_label("pw_nc".$i,0);
	}

	&comment("");
	&add($a,32);
	&add($r,32);
	&sub($num,8);
	&jnz(&label("pw_nc_loop"));

	&mov($num,&wparam(4));	# get dl
	&and($num,7);
	&jz(&label("pw_nc_end"));

	for ($i=0; $i<7; $i++)
	{
	    &mov($tmp1,&DWP($i*4,$a,"",0));	# *a
	    &mov(&DWP($i*4,$r,"",0),$tmp1);	# *r
	    &set_label("pw_tail_nc".$i,0);
	    &dec($num) if ($i != 6);
	    &jz(&label("pw_nc_end")) if ($i != 6);
	}

	&set_label("pw_nc_end",0);
	&mov($c,0);

	&set_label("pw_end",0);

#	&mov("eax",$c);		# $c is "eax"

	&function_end($name);
	}


#! /usr/bin/env perl
# Copyright 2012-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# February 2012
#
# The module implements bn_GF2m_mul_2x2 polynomial multiplication
# used in bn_gf2m.c. It's kind of low-hanging mechanical port from
# C for the time being... The subroutine runs in 37 cycles, which is
# 4.5x faster than compiler-generated code. Though comparison is
# totally unfair, because this module utilizes Galois Field Multiply
# instruction.

while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

($rp,$a1,$a0,$b1,$b0)=("A4","B4","A6","B6","A8");   # argument vector

($Alo,$Alox0,$Alox1,$Alox2,$Alox3)=map("A$_",(16..20));
($Ahi,$Ahix0,$Ahix1,$Ahix2,$Ahix3)=map("B$_",(16..20));
($B_0,$B_1,$B_2,$B_3)=("B5","A5","A7","B7");
($A,$B)=($Alo,$B_1);
$xFF="B1";

sub mul_1x1_upper {
my ($A,$B)=@_;
$code.=<<___;
	EXTU	$B,8,24,$B_2		; smash $B to 4 bytes
||	AND	$B,$xFF,$B_0
||	SHRU	$B,24,$B_3
	SHRU	$A,16,   $Ahi		; smash $A to two halfwords
||	EXTU	$A,16,16,$Alo

	XORMPY	$Alo,$B_2,$Alox2	; 16x8 bits multiplication
||	XORMPY	$Ahi,$B_2,$Ahix2
||	EXTU	$B,16,24,$B_1
	XORMPY	$Alo,$B_0,$Alox0
||	XORMPY	$Ahi,$B_0,$Ahix0
	XORMPY	$Alo,$B_3,$Alox3
||	XORMPY	$Ahi,$B_3,$Ahix3
	XORMPY	$Alo,$B_1,$Alox1
||	XORMPY	$Ahi,$B_1,$Ahix1
___
}
sub mul_1x1_merged {
my ($OUTlo,$OUThi,$A,$B)=@_;
$code.=<<___;
	 EXTU	$B,8,24,$B_2		; smash $B to 4 bytes
||	 AND	$B,$xFF,$B_0
||	 SHRU	$B,24,$B_3
	 SHRU	$A,16,   $Ahi		; smash $A to two halfwords
||	 EXTU	$A,16,16,$Alo

	XOR	$Ahix0,$Alox2,$Ahix0
||	MV	$Ahix2,$OUThi
||	 XORMPY	$Alo,$B_2,$Alox2
	 XORMPY	$Ahi,$B_2,$Ahix2
||	 EXTU	$B,16,24,$B_1
||	 XORMPY	$Alo,$B_0,A1		; $Alox0
	XOR	$Ahix1,$Alox3,$Ahix1
||	SHL	$Ahix0,16,$OUTlo
||	SHRU	$Ahix0,16,$Ahix0
	XOR	$Alox0,$OUTlo,$OUTlo
||	XOR	$Ahix0,$OUThi,$OUThi
||	 XORMPY	$Ahi,$B_0,$Ahix0
||	 XORMPY	$Alo,$B_3,$Alox3
||	SHL	$Alox1,8,$Alox1
||	SHL	$Ahix3,8,$Ahix3
	XOR	$Alox1,$OUTlo,$OUTlo
||	XOR	$Ahix3,$OUThi,$OUThi
||	 XORMPY	$Ahi,$B_3,$Ahix3
||	SHL	$Ahix1,24,$Alox1
||	SHRU	$Ahix1,8, $Ahix1
	XOR	$Alox1,$OUTlo,$OUTlo
||	XOR	$Ahix1,$OUThi,$OUThi
||	 XORMPY	$Alo,$B_1,$Alox1
||	 XORMPY	$Ahi,$B_1,$Ahix1
||	 MV	A1,$Alox0
___
}
sub mul_1x1_lower {
my ($OUTlo,$OUThi)=@_;
$code.=<<___;
	;NOP
	XOR	$Ahix0,$Alox2,$Ahix0
||	MV	$Ahix2,$OUThi
	NOP
	XOR	$Ahix1,$Alox3,$Ahix1
||	SHL	$Ahix0,16,$OUTlo
||	SHRU	$Ahix0,16,$Ahix0
	XOR	$Alox0,$OUTlo,$OUTlo
||	XOR	$Ahix0,$OUThi,$OUThi
||	SHL	$Alox1,8,$Alox1
||	SHL	$Ahix3,8,$Ahix3
	XOR	$Alox1,$OUTlo,$OUTlo
||	XOR	$Ahix3,$OUThi,$OUThi
||	SHL	$Ahix1,24,$Alox1
||	SHRU	$Ahix1,8, $Ahix1
	XOR	$Alox1,$OUTlo,$OUTlo
||	XOR	$Ahix1,$OUThi,$OUThi
___
}
$code.=<<___;
	.text

	.if	.ASSEMBLER_VERSION<7000000
	.asg	0,__TI_EABI__
	.endif
	.if	__TI_EABI__
	.asg	bn_GF2m_mul_2x2,_bn_GF2m_mul_2x2
	.endif

	.global	_bn_GF2m_mul_2x2
_bn_GF2m_mul_2x2:
	.asmfunc
	MVK	0xFF,$xFF
___
	&mul_1x1_upper($a0,$b0);		# a0Â·b0
$code.=<<___;
||	MV	$b1,$B
	MV	$a1,$A
___
	&mul_1x1_merged("A28","B28",$A,$B);	# a0Â·b0/a1Â·b1
$code.=<<___;
||	XOR	$b0,$b1,$B
	XOR	$a0,$a1,$A
___
	&mul_1x1_merged("A31","B31",$A,$B);	# a1Â·b1/(a0+a1)Â·(b0+b1)
$code.=<<___;
	XOR	A28,A31,A29
||	XOR	B28,B31,B29			; a0Â·b0+a1Â·b1
___
	&mul_1x1_lower("A30","B30");		# (a0+a1)Â·(b0+b1)
$code.=<<___;
||	BNOP	B3
	XOR	A29,A30,A30
||	XOR	B29,B30,B30			; (a0+a1)Â·(b0+b1)-a0Â·b0-a1Â·b1
	XOR	B28,A30,A30
||	STW	A28,*${rp}[0]
	XOR	B30,A31,A31
||	STW	A30,*${rp}[1]
	STW	A31,*${rp}[2]
	STW	B31,*${rp}[3]
	.endasmfunc
___

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";

$output = pop;
open STDOUT,">$output";

&asm_init($ARGV[0]);

&bn_mul_comba("bn_mul_comba8",8);
&bn_mul_comba("bn_mul_comba4",4);
&bn_sqr_comba("bn_sqr_comba8",8);
&bn_sqr_comba("bn_sqr_comba4",4);

&asm_finish();

close STDOUT or die "error closing STDOUT: $!";

sub mul_add_c
	{
	local($a,$ai,$b,$bi,$c0,$c1,$c2,$pos,$i,$na,$nb)=@_;

	# pos == -1 if eax and edx are pre-loaded, 0 to load from next
	# words, and 1 if load return value

	&comment("mul a[$ai]*b[$bi]");

	# "eax" and "edx" will always be pre-loaded.
	# &mov("eax",&DWP($ai*4,$a,"",0)) ;
	# &mov("edx",&DWP($bi*4,$b,"",0));

	&mul("edx");
	&add($c0,"eax");
	 &mov("eax",&DWP(($na)*4,$a,"",0)) if $pos == 0;	# load next a
	 &mov("eax",&wparam(0)) if $pos > 0;			# load r[]
	 ###
	&adc($c1,"edx");
	 &mov("edx",&DWP(($nb)*4,$b,"",0)) if $pos == 0;	# load next b
	 &mov("edx",&DWP(($nb)*4,$b,"",0)) if $pos == 1;	# load next b
	 ###
	&adc($c2,0);
	 # is pos > 1, it means it is the last loop
	 &mov(&DWP($i*4,"eax","",0),$c0) if $pos > 0;		# save r[];
	&mov("eax",&DWP(($na)*4,$a,"",0)) if $pos == 1;		# load next a
	}

sub sqr_add_c
	{
	local($r,$a,$ai,$bi,$c0,$c1,$c2,$pos,$i,$na,$nb)=@_;

	# pos == -1 if eax and edx are pre-loaded, 0 to load from next
	# words, and 1 if load return value

	&comment("sqr a[$ai]*a[$bi]");

	# "eax" and "edx" will always be pre-loaded.
	# &mov("eax",&DWP($ai*4,$a,"",0)) ;
	# &mov("edx",&DWP($bi*4,$b,"",0));

	if ($ai == $bi)
		{ &mul("eax");}
	else
		{ &mul("edx");}
	&add($c0,"eax");
	 &mov("eax",&DWP(($na)*4,$a,"",0)) if $pos == 0;	# load next a
	 ###
	&adc($c1,"edx");
	 &mov("edx",&DWP(($nb)*4,$a,"",0)) if ($pos == 1) && ($na != $nb);
	 ###
	&adc($c2,0);
	 # is pos > 1, it means it is the last loop
	 &mov(&DWP($i*4,$r,"",0),$c0) if $pos > 0;		# save r[];
	&mov("eax",&DWP(($na)*4,$a,"",0)) if $pos == 1;		# load next b
	}

sub sqr_add_c2
	{
	local($r,$a,$ai,$bi,$c0,$c1,$c2,$pos,$i,$na,$nb)=@_;

	# pos == -1 if eax and edx are pre-loaded, 0 to load from next
	# words, and 1 if load return value

	&comment("sqr a[$ai]*a[$bi]");

	# "eax" and "edx" will always be pre-loaded.
	# &mov("eax",&DWP($ai*4,$a,"",0)) ;
	# &mov("edx",&DWP($bi*4,$a,"",0));

	if ($ai == $bi)
		{ &mul("eax");}
	else
		{ &mul("edx");}
	&add("eax","eax");
	 ###
	&adc("edx","edx");
	 ###
	&adc($c2,0);
	 &add($c0,"eax");
	&adc($c1,"edx");
	 &mov("eax",&DWP(($na)*4,$a,"",0)) if $pos == 0;	# load next a
	 &mov("eax",&DWP(($na)*4,$a,"",0)) if $pos == 1;	# load next b
	&adc($c2,0);
	&mov(&DWP($i*4,$r,"",0),$c0) if $pos > 0;		# save r[];
	 &mov("edx",&DWP(($nb)*4,$a,"",0)) if ($pos <= 1) && ($na != $nb);
	 ###
	}

sub bn_mul_comba
	{
	local($name,$num)=@_;
	local($a,$b,$c0,$c1,$c2);
	local($i,$as,$ae,$bs,$be,$ai,$bi);
	local($tot,$end);

	&function_begin_B($name,"");

	$c0="ebx";
	$c1="ecx";
	$c2="ebp";
	$a="esi";
	$b="edi";

	$as=0;
	$ae=0;
	$bs=0;
	$be=0;
	$tot=$num+$num-1;

	&push("esi");
	 &mov($a,&wparam(1));
	&push("edi");
	 &mov($b,&wparam(2));
	&push("ebp");
	 &push("ebx");

	&xor($c0,$c0);
	 &mov("eax",&DWP(0,$a,"",0));	# load the first word
	&xor($c1,$c1);
	 &mov("edx",&DWP(0,$b,"",0));	# load the first second

	for ($i=0; $i<$tot; $i++)
		{
		$ai=$as;
		$bi=$bs;
		$end=$be+1;

		&comment("################## Calculate word $i");

		for ($j=$bs; $j<$end; $j++)
			{
			&xor($c2,$c2) if ($j == $bs);
			if (($j+1) == $end)
				{
				$v=1;
				$v=2 if (($i+1) == $tot);
				}
			else
				{ $v=0; }
			if (($j+1) != $end)
				{
				$na=($ai-1);
				$nb=($bi+1);
				}
			else
				{
				$na=$as+($i < ($num-1));
				$nb=$bs+($i >= ($num-1));
				}
#printf STDERR "[$ai,$bi] -> [$na,$nb]\n";
			&mul_add_c($a,$ai,$b,$bi,$c0,$c1,$c2,$v,$i,$na,$nb);
			if ($v)
				{
				&comment("saved r[$i]");
				# &mov("eax",&wparam(0));
				# &mov(&DWP($i*4,"eax","",0),$c0);
				($c0,$c1,$c2)=($c1,$c2,$c0);
				}
			$ai--;
			$bi++;
			}
		$as++ if ($i < ($num-1));
		$ae++ if ($i >= ($num-1));

		$bs++ if ($i >= ($num-1));
		$be++ if ($i < ($num-1));
		}
	&comment("save r[$i]");
	# &mov("eax",&wparam(0));
	&mov(&DWP($i*4,"eax","",0),$c0);

	&pop("ebx");
	&pop("ebp");
	&pop("edi");
	&pop("esi");
	&ret();
	&function_end_B($name);
	}

sub bn_sqr_comba
	{
	local($name,$num)=@_;
	local($r,$a,$c0,$c1,$c2)=@_;
	local($i,$as,$ae,$bs,$be,$ai,$bi);
	local($b,$tot,$end,$half);

	&function_begin_B($name,"");

	$c0="ebx";
	$c1="ecx";
	$c2="ebp";
	$a="esi";
	$r="edi";

	&push("esi");
	 &push("edi");
	&push("ebp");
	 &push("ebx");
	&mov($r,&wparam(0));
	 &mov($a,&wparam(1));
	&xor($c0,$c0);
	 &xor($c1,$c1);
	&mov("eax",&DWP(0,$a,"",0)); # load the first word

	$as=0;
	$ae=0;
	$bs=0;
	$be=0;
	$tot=$num+$num-1;

	for ($i=0; $i<$tot; $i++)
		{
		$ai=$as;
		$bi=$bs;
		$end=$be+1;

		&comment("############### Calculate word $i");
		for ($j=$bs; $j<$end; $j++)
			{
			&xor($c2,$c2) if ($j == $bs);
			if (($ai-1) < ($bi+1))
				{
				$v=1;
				$v=2 if ($i+1) == $tot;
				}
			else
				{ $v=0; }
			if (!$v)
				{
				$na=$ai-1;
				$nb=$bi+1;
				}
			else
				{
				$na=$as+($i < ($num-1));
				$nb=$bs+($i >= ($num-1));
				}
			if ($ai == $bi)
				{
				&sqr_add_c($r,$a,$ai,$bi,
					$c0,$c1,$c2,$v,$i,$na,$nb);
				}
			else
				{
				&sqr_add_c2($r,$a,$ai,$bi,
					$c0,$c1,$c2,$v,$i,$na,$nb);
				}
			if ($v)
				{
				&comment("saved r[$i]");
				#&mov(&DWP($i*4,$r,"",0),$c0);
				($c0,$c1,$c2)=($c1,$c2,$c0);
				last;
				}
			$ai--;
			$bi++;
			}
		$as++ if ($i < ($num-1));
		$ae++ if ($i >= ($num-1));

		$bs++ if ($i >= ($num-1));
		$be++ if ($i < ($num-1));
		}
	&mov(&DWP($i*4,$r,"",0),$c0);
	&pop("ebx");
	&pop("ebp");
	&pop("edi");
	&pop("esi");
	&ret();
	&function_end_B($name);
	}