#! /usr/bin/env perl
# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# January 2010
#
# "Teaser" Montgomery multiplication module for IA-64. There are
# several possibilities for improvement:
#
# - modulo-scheduling outer loop would eliminate quite a number of
#   stalls after ldf8, xma and getf.sig outside inner loop and
#   improve shorter key performance;
# - shorter vector support [with input vectors being fetched only
#   once] should be added;
# - 2x unroll with help of n0[1] would make the code scalable on
#   "wider" IA-64, "wider" than Itanium 2 that is, which is not of
#   acute interest, because upcoming Tukwila's individual cores are
#   reportedly based on Itanium 2 design;
# - dedicated squaring procedure(?);
#
# January 2010
#
# Shorter vector support is implemented by zero-padding ap and np
# vectors up to 8 elements, or 512 bits. This means that 256-bit
# inputs will be processed only 2 times faster than 512-bit inputs,
# not 4 [as one would expect, because algorithm complexity is n^2].
# The reason for padding is that inputs shorter than 512 bits won't
# be processed faster anyway, because minimal critical path of the
# core loop happens to match 512-bit timing. Either way, it resulted
# in >100% improvement of 512-bit RSA sign benchmark and 50% - of
# 1024-bit one [in comparison to original version of *this* module].
#
# So far 'openssl speed rsa dsa' output on 900MHz Itanium 2 *with*
# this module is:
#                   sign    verify    sign/s verify/s
# rsa  512 bits 0.000290s 0.000024s   3452.8  42031.4
# rsa 1024 bits 0.000793s 0.000058s   1261.7  17172.0
# rsa 2048 bits 0.005908s 0.000148s    169.3   6754.0
# rsa 4096 bits 0.033456s 0.000469s     29.9   2133.6
# dsa  512 bits 0.000253s 0.000198s   3949.9   5057.0
# dsa 1024 bits 0.000585s 0.000607s   1708.4   1647.4
# dsa 2048 bits 0.001453s 0.001703s    688.1    587.4
#
# ... and *without* (but still with ia64.S):
#
# rsa  512 bits 0.000670s 0.000041s   1491.8  24145.5
# rsa 1024 bits 0.001988s 0.000080s    502.9  12499.3
# rsa 2048 bits 0.008702s 0.000189s    114.9   5293.9
# rsa 4096 bits 0.043860s 0.000533s     22.8   1875.9
# dsa  512 bits 0.000441s 0.000427s   2265.3   2340.6
# dsa 1024 bits 0.000823s 0.000867s   1215.6   1153.2
# dsa 2048 bits 0.001894s 0.002179s    528.1    458.9
#
# As it can be seen, RSA sign performance improves by 130-30%,
# hereafter less for longer keys, while verify - by 74-13%.
# DSA performance improves by 115-30%.

$output=pop;

if ($^O eq "hpux") {
    $ADDP="addp4";
    for (@ARGV) { $ADDP="add" if (/[\+DD|\-mlp]64/); }
} else { $ADDP="add"; }

$code=<<___;
.explicit
.text

// int bn_mul_mont (BN_ULONG *rp,const BN_ULONG *ap,
//		    const BN_ULONG *bp,const BN_ULONG *np,
//		    const BN_ULONG *n0p,int num);
.align	64
.global	bn_mul_mont#
.proc	bn_mul_mont#
bn_mul_mont:
	.prologue
	.body
{ .mmi;	cmp4.le		p6,p7=2,r37;;
(p6)	cmp4.lt.unc	p8,p9=8,r37
	mov		ret0=r0		};;
{ .bbb;
(p9)	br.cond.dptk.many	bn_mul_mont_8
(p8)	br.cond.dpnt.many	bn_mul_mont_general
(p7)	br.ret.spnt.many	b0	};;
.endp	bn_mul_mont#

prevfs=r2;	prevpr=r3;	prevlc=r10;	prevsp=r11;

rptr=r8;	aptr=r9;	bptr=r14;	nptr=r15;
tptr=r16;	// &tp[0]
tp_1=r17;	// &tp[-1]
num=r18;	len=r19;	lc=r20;
topbit=r21;	// carry bit from tmp[num]

n0=f6;
m0=f7;
bi=f8;

.align	64
.local	bn_mul_mont_general#
.proc	bn_mul_mont_general#
bn_mul_mont_general:
	.prologue
{ .mmi;	.save	ar.pfs,prevfs
	alloc	prevfs=ar.pfs,6,2,0,8
	$ADDP	aptr=0,in1
	.save	ar.lc,prevlc
	mov	prevlc=ar.lc		}
{ .mmi;	.vframe	prevsp
	mov	prevsp=sp
	$ADDP	bptr=0,in2
	.save	pr,prevpr
	mov	prevpr=pr		};;

	.body
	.rotf		alo[6],nlo[4],ahi[8],nhi[6]
	.rotr		a[3],n[3],t[2]

{ .mmi;	ldf8		bi=[bptr],8		// (*bp++)
	ldf8		alo[4]=[aptr],16	// ap[0]
	$ADDP		r30=8,in1	};;
{ .mmi;	ldf8		alo[3]=[r30],16		// ap[1]
	ldf8		alo[2]=[aptr],16	// ap[2]
	$ADDP		in4=0,in4	};;
{ .mmi;	ldf8		alo[1]=[r30]		// ap[3]
	ldf8		n0=[in4]		// n0
	$ADDP		rptr=0,in0		}
{ .mmi;	$ADDP		nptr=0,in3
	mov		r31=16
	zxt4		num=in5		};;
{ .mmi;	ldf8		nlo[2]=[nptr],8		// np[0]
	shladd		len=num,3,r0
	shladd		r31=num,3,r31	};;
{ .mmi;	ldf8		nlo[1]=[nptr],8		// np[1]
	add		lc=-5,num
	sub		r31=sp,r31	};;
{ .mfb;	and		sp=-16,r31		// alloca
	xmpy.hu		ahi[2]=alo[4],bi	// ap[0]*bp[0]
	nop.b		0		}
{ .mfb;	nop.m		0
	xmpy.lu		alo[4]=alo[4],bi
	brp.loop.imp	.L1st_ctop,.L1st_cend-16
					};;
{ .mfi;	nop.m		0
	xma.hu		ahi[1]=alo[3],bi,ahi[2]	// ap[1]*bp[0]
	add		tp_1=8,sp	}
{ .mfi;	nop.m		0
	xma.lu		alo[3]=alo[3],bi,ahi[2]
	mov		pr.rot=0x20001f<<16
			// ------^----- (p40) at first (p23)
			// ----------^^ p[16:20]=1
					};;
{ .mfi;	nop.m		0
	xmpy.lu		m0=alo[4],n0		// (ap[0]*bp[0])*n0
	mov		ar.lc=lc	}
{ .mfi;	nop.m		0
	fcvt.fxu.s1	nhi[1]=f0
	mov		ar.ec=8		};;

.align	32
.L1st_ctop:
.pred.rel	"mutex",p40,p42
{ .mfi;	(p16)	ldf8		alo[0]=[aptr],8		    // *(aptr++)
	(p18)	xma.hu		ahi[0]=alo[2],bi,ahi[1]
	(p40)	add		n[2]=n[2],a[2]		}   // (p23)					}
{ .mfi;	(p18)	ldf8		nlo[0]=[nptr],8		    // *(nptr++)(p16)
	(p18)	xma.lu		alo[2]=alo[2],bi,ahi[1]
	(p42)	add		n[2]=n[2],a[2],1	};; // (p23)
{ .mfi;	(p21)	getf.sig	a[0]=alo[5]
	(p20)	xma.hu		nhi[0]=nlo[2],m0,nhi[1]
	(p42)	cmp.leu		p41,p39=n[2],a[2]   	}   // (p23)
{ .mfi;	(p23)	st8		[tp_1]=n[2],8
	(p20)	xma.lu		nlo[2]=nlo[2],m0,nhi[1]
	(p40)	cmp.ltu		p41,p39=n[2],a[2]	}   // (p23)
{ .mmb;	(p21)	getf.sig	n[0]=nlo[3]
	(p16)	nop.m		0
	br.ctop.sptk	.L1st_ctop			};;
.L1st_cend:

{ .mmi;	getf.sig	a[0]=ahi[6]		// (p24)
	getf.sig	n[0]=nhi[4]
	add		num=-1,num	};;	// num--
{ .mmi;	.pred.rel	"mutex",p40,p42
(p40)	add		n[0]=n[0],a[0]
(p42)	add		n[0]=n[0],a[0],1
	sub		aptr=aptr,len	};;	// rewind
{ .mmi;	.pred.rel	"mutex",p40,p42
(p40)	cmp.ltu		p41,p39=n[0],a[0]
(p42)	cmp.leu		p41,p39=n[0],a[0]
	sub		nptr=nptr,len	};;
{ .mmi;	.pred.rel	"mutex",p39,p41
(p39)	add		topbit=r0,r0
(p41)	add		topbit=r0,r0,1
	nop.i		0		}
{ .mmi;	st8		[tp_1]=n[0]
	add		tptr=16,sp
	add		tp_1=8,sp	};;

.Louter:
{ .mmi;	ldf8		bi=[bptr],8		// (*bp++)
	ldf8		ahi[3]=[tptr]		// tp[0]
	add		r30=8,aptr	};;
{ .mmi;	ldf8		alo[4]=[aptr],16	// ap[0]
	ldf8		alo[3]=[r30],16		// ap[1]
	add		r31=8,nptr	};;
{ .mfb;	ldf8		alo[2]=[aptr],16	// ap[2]
	xma.hu		ahi[2]=alo[4],bi,ahi[3]	// ap[0]*bp[i]+tp[0]
	brp.loop.imp	.Linner_ctop,.Linner_cend-16
					}
{ .mfb;	ldf8		alo[1]=[r30]		// ap[3]
	xma.lu		alo[4]=alo[4],bi,ahi[3]
	clrrrb.pr			};;
{ .mfi;	ldf8		nlo[2]=[nptr],16	// np[0]
	xma.hu		ahi[1]=alo[3],bi,ahi[2]	// ap[1]*bp[i]
	nop.i		0		}
{ .mfi;	ldf8		nlo[1]=[r31]		// np[1]
	xma.lu		alo[3]=alo[3],bi,ahi[2]
	mov		pr.rot=0x20101f<<16
			// ------^----- (p40) at first (p23)
			// --------^--- (p30) at first (p22)
			// ----------^^ p[16:20]=1
					};;
{ .mfi;	st8		[tptr]=r0		// tp[0] is already accounted
	xmpy.lu		m0=alo[4],n0		// (ap[0]*bp[i]+tp[0])*n0
	mov		ar.lc=lc	}
{ .mfi;
	fcvt.fxu.s1	nhi[1]=f0
	mov		ar.ec=8		};;

// This loop spins in 4*(n+7) ticks on Itanium 2 and should spin in
// 7*(n+7) ticks on Itanium (the one codenamed Merced). Factor of 7
// in latter case accounts for two-tick pipeline stall, which means
// that its performance would be ~20% lower than optimal one. No
// attempt was made to address this, because original Itanium is
// hardly represented out in the wild...
.align	32
.Linner_ctop:
.pred.rel	"mutex",p40,p42
.pred.rel	"mutex",p30,p32
{ .mfi;	(p16)	ldf8		alo[0]=[aptr],8		    // *(aptr++)
	(p18)	xma.hu		ahi[0]=alo[2],bi,ahi[1]
	(p40)	add		n[2]=n[2],a[2]		}   // (p23)
{ .mfi;	(p16)	nop.m		0
	(p18)	xma.lu		alo[2]=alo[2],bi,ahi[1]
	(p42)	add		n[2]=n[2],a[2],1	};; // (p23)
{ .mfi;	(p21)	getf.sig	a[0]=alo[5]
	(p16)	nop.f		0
	(p40)	cmp.ltu		p41,p39=n[2],a[2]	}   // (p23)
{ .mfi;	(p21)	ld8		t[0]=[tptr],8
	(p16)	nop.f		0
	(p42)	cmp.leu		p41,p39=n[2],a[2]	};; // (p23)
{ .mfi;	(p18)	ldf8		nlo[0]=[nptr],8		    // *(nptr++)
	(p20)	xma.hu		nhi[0]=nlo[2],m0,nhi[1]
	(p30)	add		a[1]=a[1],t[1]		}   // (p22)
{ .mfi;	(p16)	nop.m		0
	(p20)	xma.lu		nlo[2]=nlo[2],m0,nhi[1]
	(p32)	add		a[1]=a[1],t[1],1	};; // (p22)
{ .mmi;	(p21)	getf.sig	n[0]=nlo[3]
	(p16)	nop.m		0
	(p30)	cmp.ltu		p31,p29=a[1],t[1]	}   // (p22)
{ .mmb;	(p23)	st8		[tp_1]=n[2],8
	(p32)	cmp.leu		p31,p29=a[1],t[1]	    // (p22)
	br.ctop.sptk	.Linner_ctop			};;
.Linner_cend:

{ .mmi;	getf.sig	a[0]=ahi[6]		// (p24)
	getf.sig	n[0]=nhi[4]
	nop.i		0		};;

{ .mmi;	.pred.rel	"mutex",p31,p33
(p31)	add		a[0]=a[0],topbit
(p33)	add		a[0]=a[0],topbit,1
	mov		topbit=r0	};;
{ .mfi; .pred.rel	"mutex",p31,p33
(p31)	cmp.ltu		p32,p30=a[0],topbit
(p33)	cmp.leu		p32,p30=a[0],topbit
					}
{ .mfi;	.pred.rel	"mutex",p40,p42
(p40)	add		n[0]=n[0],a[0]
(p42)	add		n[0]=n[0],a[0],1
					};;
{ .mmi;	.pred.rel	"mutex",p44,p46
(p40)	cmp.ltu		p41,p39=n[0],a[0]
(p42)	cmp.leu		p41,p39=n[0],a[0]
(p32)	add		topbit=r0,r0,1	}

{ .mmi;	st8		[tp_1]=n[0],8
	cmp4.ne		p6,p0=1,num
	sub		aptr=aptr,len	};;	// rewind
{ .mmi;	sub		nptr=nptr,len
(p41)	add		topbit=r0,r0,1
	add		tptr=16,sp	}
{ .mmb;	add		tp_1=8,sp
	add		num=-1,num		// num--
(p6)	br.cond.sptk.many	.Louter	};;

{ .mbb;	add		lc=4,lc
	brp.loop.imp	.Lsub_ctop,.Lsub_cend-16
	clrrrb.pr			};;
{ .mii;	nop.m		0
	mov		pr.rot=0x10001<<16
			// ------^---- (p33) at first (p17)
	mov		ar.lc=lc	}
{ .mii;	nop.m		0
	mov		ar.ec=3
	nop.i		0		};;

.Lsub_ctop:
.pred.rel	"mutex",p33,p35
{ .mfi;	(p16)	ld8		t[0]=[tptr],8		    // t=*(tp++)
	(p16)	nop.f		0
	(p33)	sub		n[1]=t[1],n[1]		}   // (p17)
{ .mfi;	(p16)	ld8		n[0]=[nptr],8		    // n=*(np++)
	(p16)	nop.f		0
	(p35)	sub		n[1]=t[1],n[1],1	};; // (p17)
{ .mib;	(p18)	st8		[rptr]=n[2],8		    // *(rp++)=r
	(p33)	cmp.gtu		p34,p32=n[1],t[1]	    // (p17)
	(p18)	nop.b		0			}
{ .mib;	(p18)	nop.m		0
	(p35)	cmp.geu		p34,p32=n[1],t[1]	    // (p17)
	br.ctop.sptk	.Lsub_ctop			};;
.Lsub_cend:

{ .mmb;	.pred.rel	"mutex",p34,p36
(p34)	sub	topbit=topbit,r0	// (p19)
(p36)	sub	topbit=topbit,r0,1
	brp.loop.imp	.Lcopy_ctop,.Lcopy_cend-16
					}
{ .mmb;	sub	rptr=rptr,len		// rewind
	sub	tptr=tptr,len
	clrrrb.pr			};;
{ .mmi;	mov	aptr=rptr
	mov	bptr=tptr
	mov	pr.rot=1<<16		};;
{ .mii;	cmp.eq	p0,p6=topbit,r0
	mov	ar.lc=lc
	mov	ar.ec=2			};;

.Lcopy_ctop:
{ .mmi;	(p16)	ld8	a[0]=[aptr],8
	(p16)	ld8	t[0]=[bptr],8
	(p6)	mov	a[1]=t[1]	};;	// (p17)
{ .mmb;	(p17)	st8	[rptr]=a[1],8
	(p17)	st8	[tptr]=r0,8
	br.ctop.sptk	.Lcopy_ctop	};;
.Lcopy_cend:

{ .mmi;	mov		ret0=1			// signal "handled"
	rum		1<<5			// clear um.mfh
	mov		ar.lc=prevlc	}
{ .mib;	.restore	sp
	mov		sp=prevsp
	mov		pr=prevpr,0x1ffff
	br.ret.sptk.many	b0	};;
.endp	bn_mul_mont_general#

a1=r16;  a2=r17;  a3=r18;  a4=r19;  a5=r20;  a6=r21;  a7=r22;  a8=r23;
n1=r24;  n2=r25;  n3=r26;  n4=r27;  n5=r28;  n6=r29;  n7=r30;  n8=r31;
t0=r15;

ai0=f8;  ai1=f9;  ai2=f10; ai3=f11; ai4=f12; ai5=f13; ai6=f14; ai7=f15;
ni0=f16; ni1=f17; ni2=f18; ni3=f19; ni4=f20; ni5=f21; ni6=f22; ni7=f23;

.align	64
.skip	48		// aligns loop body
.local	bn_mul_mont_8#
.proc	bn_mul_mont_8#
bn_mul_mont_8:
	.prologue
{ .mmi;	.save		ar.pfs,prevfs
	alloc		prevfs=ar.pfs,6,2,0,8
	.vframe		prevsp
	mov		prevsp=sp
	.save		ar.lc,prevlc
	mov		prevlc=ar.lc	}
{ .mmi;	add		r17=-6*16,sp
	add		sp=-7*16,sp
	.save		pr,prevpr
	mov		prevpr=pr	};;

{ .mmi;	.save.gf	0,0x10
	stf.spill	[sp]=f16,-16
	.save.gf	0,0x20
	stf.spill	[r17]=f17,32
	add		r16=-5*16,prevsp};;
{ .mmi;	.save.gf	0,0x40
	stf.spill	[r16]=f18,32
	.save.gf	0,0x80
	stf.spill	[r17]=f19,32
	$ADDP		aptr=0,in1	};;
{ .mmi;	.save.gf	0,0x100
	stf.spill	[r16]=f20,32
	.save.gf	0,0x200
	stf.spill	[r17]=f21,32
	$ADDP		r29=8,in1	};;
{ .mmi;	.save.gf	0,0x400
	stf.spill	[r16]=f22
	.save.gf	0,0x800
	stf.spill	[r17]=f23
	$ADDP		rptr=0,in0	};;

	.body
	.rotf		bj[8],mj[2],tf[2],alo[10],ahi[10],nlo[10],nhi[10]
	.rotr		t[8]

// load input vectors padding them to 8 elements
{ .mmi;	ldf8		ai0=[aptr],16		// ap[0]
	ldf8		ai1=[r29],16		// ap[1]
	$ADDP		bptr=0,in2	}
{ .mmi;	$ADDP		r30=8,in2
	$ADDP		nptr=0,in3
	$ADDP		r31=8,in3	};;
{ .mmi;	ldf8		bj[7]=[bptr],16		// bp[0]
	ldf8		bj[6]=[r30],16		// bp[1]
	cmp4.le		p4,p5=3,in5	}
{ .mmi;	ldf8		ni0=[nptr],16		// np[0]
	ldf8		ni1=[r31],16		// np[1]
	cmp4.le		p6,p7=4,in5	};;

{ .mfi;	(p4)ldf8	ai2=[aptr],16		// ap[2]
	(p5)fcvt.fxu	ai2=f0
	cmp4.le		p8,p9=5,in5	}
{ .mfi;	(p6)ldf8	ai3=[r29],16		// ap[3]
	(p7)fcvt.fxu	ai3=f0
	cmp4.le		p10,p11=6,in5	}
{ .mfi;	(p4)ldf8	bj[5]=[bptr],16		// bp[2]
	(p5)fcvt.fxu	bj[5]=f0
	cmp4.le		p12,p13=7,in5	}
{ .mfi;	(p6)ldf8	bj[4]=[r30],16		// bp[3]
	(p7)fcvt.fxu	bj[4]=f0
	cmp4.le		p14,p15=8,in5	}
{ .mfi;	(p4)ldf8	ni2=[nptr],16		// np[2]
	(p5)fcvt.fxu	ni2=f0
	addp4		r28=-1,in5	}
{ .mfi;	(p6)ldf8	ni3=[r31],16		// np[3]
	(p7)fcvt.fxu	ni3=f0
	$ADDP		in4=0,in4	};;

{ .mfi;	ldf8		n0=[in4]
	fcvt.fxu	tf[1]=f0
	nop.i		0		}

{ .mfi;	(p8)ldf8	ai4=[aptr],16		// ap[4]
	(p9)fcvt.fxu	ai4=f0
	mov		t[0]=r0		}
{ .mfi;	(p10)ldf8	ai5=[r29],16		// ap[5]
	(p11)fcvt.fxu	ai5=f0
	mov		t[1]=r0		}
{ .mfi;	(p8)ldf8	bj[3]=[bptr],16		// bp[4]
	(p9)fcvt.fxu	bj[3]=f0
	mov		t[2]=r0		}
{ .mfi;	(p10)ldf8	bj[2]=[r30],16		// bp[5]
	(p11)fcvt.fxu	bj[2]=f0
	mov		t[3]=r0		}
{ .mfi;	(p8)ldf8	ni4=[nptr],16		// np[4]
	(p9)fcvt.fxu	ni4=f0
	mov		t[4]=r0		}
{ .mfi;	(p10)ldf8	ni5=[r31],16		// np[5]
	(p11)fcvt.fxu	ni5=f0
	mov		t[5]=r0		};;

{ .mfi;	(p12)ldf8	ai6=[aptr],16		// ap[6]
	(p13)fcvt.fxu	ai6=f0
	mov		t[6]=r0		}
{ .mfi;	(p14)ldf8	ai7=[r29],16		// ap[7]
	(p15)fcvt.fxu	ai7=f0
	mov		t[7]=r0		}
{ .mfi;	(p12)ldf8	bj[1]=[bptr],16		// bp[6]
	(p13)fcvt.fxu	bj[1]=f0
	mov		ar.lc=r28	}
{ .mfi;	(p14)ldf8	bj[0]=[r30],16		// bp[7]
	(p15)fcvt.fxu	bj[0]=f0
	mov		ar.ec=1		}
{ .mfi;	(p12)ldf8	ni6=[nptr],16		// np[6]
	(p13)fcvt.fxu	ni6=f0
	mov		pr.rot=1<<16	}
{ .mfb;	(p14)ldf8	ni7=[r31],16		// np[7]
	(p15)fcvt.fxu	ni7=f0
	brp.loop.imp	.Louter_8_ctop,.Louter_8_cend-16
					};;

// The loop is scheduled for 32*n ticks on Itanium 2. Actual attempt
// to measure with help of Interval Time Counter indicated that the
// factor is a tad higher: 33 or 34, if not 35. Exact measurement and
// addressing the issue is problematic, because I don't have access
// to platform-specific instruction-level profiler. On Itanium it
// should run in 56*n ticks, because of higher xma latency...
.Louter_8_ctop:
	.pred.rel		"mutex",p40,p42
	.pred.rel		"mutex",p48,p50
{ .mfi;	(p16)	nop.m		0			// 0:
	(p16)	xma.hu		ahi[0]=ai0,bj[7],tf[1]	//	ap[0]*b[i]+t[0]
	(p40)	add		a3=a3,n3	}	//	(p17) a3+=n3
{ .mfi;	(p42)	add		a3=a3,n3,1
	(p16)	xma.lu		alo[0]=ai0,bj[7],tf[1]
	(p16)	nop.i		0		};;
{ .mii;	(p17)	getf.sig	a7=alo[8]		// 1:
	(p48)	add		t[6]=t[6],a3		//	(p17) t[6]+=a3
	(p50)	add		t[6]=t[6],a3,1	};;
{ .mfi;	(p17)	getf.sig	a8=ahi[8]		// 2:
	(p17)	xma.hu		nhi[7]=ni6,mj[1],nhi[6]	//	np[6]*m0
	(p40)	cmp.ltu		p43,p41=a3,n3	}
{ .mfi;	(p42)	cmp.leu		p43,p41=a3,n3
	(p17)	xma.lu		nlo[7]=ni6,mj[1],nhi[6]
	(p16)	nop.i		0		};;
{ .mii;	(p17)	getf.sig	n5=nlo[6]		// 3:
	(p48)	cmp.ltu		p51,p49=t[6],a3
	(p50)	cmp.leu		p51,p49=t[6],a3	};;
	.pred.rel		"mutex",p41,p43
	.pred.rel		"mutex",p49,p51
{ .mfi;	(p16)	nop.m		0			// 4:
	(p16)	xma.hu		ahi[1]=ai1,bj[7],ahi[0]	//	ap[1]*b[i]
	(p41)	add		a4=a4,n4	}	//	(p17) a4+=n4
{ .mfi;	(p43)	add		a4=a4,n4,1
	(p16)	xma.lu		alo[1]=ai1,bj[7],ahi[0]
	(p16)	nop.i		0		};;
{ .mfi;	(p49)	add		t[5]=t[5],a4		// 5:	(p17) t[5]+=a4
	(p16)	xmpy.lu		mj[0]=alo[0],n0		//	(ap[0]*b[i]+t[0])*n0
	(p51)	add		t[5]=t[5],a4,1	};;
{ .mfi;	(p16)	nop.m		0			// 6:
	(p17)	xma.hu		nhi[8]=ni7,mj[1],nhi[7]	//	np[7]*m0
	(p41)	cmp.ltu		p42,p40=a4,n4	}
{ .mfi;	(p43)	cmp.leu		p42,p40=a4,n4
	(p17)	xma.lu		nlo[8]=ni7,mj[1],nhi[7]
	(p16)	nop.i		0		};;
{ .mii;	(p17)	getf.sig	n6=nlo[7]		// 7:
	(p49)	cmp.ltu		p50,p48=t[5],a4
	(p51)	cmp.leu		p50,p48=t[5],a4	};;
	.pred.rel		"mutex",p40,p42
	.pred.rel		"mutex",p48,p50
{ .mfi;	(p16)	nop.m		0			// 8:
	(p16)	xma.hu		ahi[2]=ai2,bj[7],ahi[1]	//	ap[2]*b[i]
	(p40)	add		a5=a5,n5	}	//	(p17) a5+=n5
{ .mfi;	(p42)	add		a5=a5,n5,1
	(p16)	xma.lu		alo[2]=ai2,bj[7],ahi[1]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	a1=alo[1]		// 9:
	(p48)	add		t[4]=t[4],a5		//	p(17) t[4]+=a5
	(p50)	add		t[4]=t[4],a5,1	};;
{ .mfi;	(p16)	nop.m		0			// 10:
	(p16)	xma.hu		nhi[0]=ni0,mj[0],alo[0]	//	np[0]*m0
	(p40)	cmp.ltu		p43,p41=a5,n5	}
{ .mfi;	(p42)	cmp.leu		p43,p41=a5,n5
	(p16)	xma.lu		nlo[0]=ni0,mj[0],alo[0]
	(p16)	nop.i		0		};;
{ .mii;	(p17)	getf.sig	n7=nlo[8]		// 11:
	(p48)	cmp.ltu		p51,p49=t[4],a5
	(p50)	cmp.leu		p51,p49=t[4],a5	};;
	.pred.rel		"mutex",p41,p43
	.pred.rel		"mutex",p49,p51
{ .mfi;	(p17)	getf.sig	n8=nhi[8]		// 12:
	(p16)	xma.hu		ahi[3]=ai3,bj[7],ahi[2]	//	ap[3]*b[i]
	(p41)	add		a6=a6,n6	}	//	(p17) a6+=n6
{ .mfi;	(p43)	add		a6=a6,n6,1
	(p16)	xma.lu		alo[3]=ai3,bj[7],ahi[2]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	a2=alo[2]		// 13:
	(p49)	add		t[3]=t[3],a6		//	(p17) t[3]+=a6
	(p51)	add		t[3]=t[3],a6,1	};;
{ .mfi;	(p16)	nop.m		0			// 14:
	(p16)	xma.hu		nhi[1]=ni1,mj[0],nhi[0]	//	np[1]*m0
	(p41)	cmp.ltu		p42,p40=a6,n6	}
{ .mfi;	(p43)	cmp.leu		p42,p40=a6,n6
	(p16)	xma.lu		nlo[1]=ni1,mj[0],nhi[0]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	nop.m		0			// 15:
	(p49)	cmp.ltu		p50,p48=t[3],a6
	(p51)	cmp.leu		p50,p48=t[3],a6	};;
	.pred.rel		"mutex",p40,p42
	.pred.rel		"mutex",p48,p50
{ .mfi;	(p16)	nop.m		0			// 16:
	(p16)	xma.hu		ahi[4]=ai4,bj[7],ahi[3]	//	ap[4]*b[i]
	(p40)	add		a7=a7,n7	}	//	(p17) a7+=n7
{ .mfi;	(p42)	add		a7=a7,n7,1
	(p16)	xma.lu		alo[4]=ai4,bj[7],ahi[3]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	a3=alo[3]		// 17:
	(p48)	add		t[2]=t[2],a7		//	(p17) t[2]+=a7
	(p50)	add		t[2]=t[2],a7,1	};;
{ .mfi;	(p16)	nop.m		0			// 18:
	(p16)	xma.hu		nhi[2]=ni2,mj[0],nhi[1]	//	np[2]*m0
	(p40)	cmp.ltu		p43,p41=a7,n7	}
{ .mfi;	(p42)	cmp.leu		p43,p41=a7,n7
	(p16)	xma.lu		nlo[2]=ni2,mj[0],nhi[1]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	n1=nlo[1]		// 19:
	(p48)	cmp.ltu		p51,p49=t[2],a7
	(p50)	cmp.leu		p51,p49=t[2],a7	};;
	.pred.rel		"mutex",p41,p43
	.pred.rel		"mutex",p49,p51
{ .mfi;	(p16)	nop.m		0			// 20:
	(p16)	xma.hu		ahi[5]=ai5,bj[7],ahi[4]	//	ap[5]*b[i]
	(p41)	add		a8=a8,n8	}	//	(p17) a8+=n8
{ .mfi;	(p43)	add		a8=a8,n8,1
	(p16)	xma.lu		alo[5]=ai5,bj[7],ahi[4]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	a4=alo[4]		// 21:
	(p49)	add		t[1]=t[1],a8		//	(p17) t[1]+=a8
	(p51)	add		t[1]=t[1],a8,1	};;
{ .mfi;	(p16)	nop.m		0			// 22:
	(p16)	xma.hu		nhi[3]=ni3,mj[0],nhi[2]	//	np[3]*m0
	(p41)	cmp.ltu		p42,p40=a8,n8	}
{ .mfi;	(p43)	cmp.leu		p42,p40=a8,n8
	(p16)	xma.lu		nlo[3]=ni3,mj[0],nhi[2]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	n2=nlo[2]		// 23:
	(p49)	cmp.ltu		p50,p48=t[1],a8
	(p51)	cmp.leu		p50,p48=t[1],a8	};;
{ .mfi;	(p16)	nop.m		0			// 24:
	(p16)	xma.hu		ahi[6]=ai6,bj[7],ahi[5]	//	ap[6]*b[i]
	(p16)	add		a1=a1,n1	}	//	(p16) a1+=n1
{ .mfi;	(p16)	nop.m		0
	(p16)	xma.lu		alo[6]=ai6,bj[7],ahi[5]
	(p17)	mov		t[0]=r0		};;
{ .mii;	(p16)	getf.sig	a5=alo[5]		// 25:
	(p16)	add		t0=t[7],a1		//	(p16) t[7]+=a1
	(p42)	add		t[0]=t[0],r0,1	};;
{ .mfi;	(p16)	setf.sig	tf[0]=t0		// 26:
	(p16)	xma.hu		nhi[4]=ni4,mj[0],nhi[3]	//	np[4]*m0
	(p50)	add		t[0]=t[0],r0,1	}
{ .mfi;	(p16)	cmp.ltu.unc	p42,p40=a1,n1
	(p16)	xma.lu		nlo[4]=ni4,mj[0],nhi[3]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	n3=nlo[3]		// 27:
	(p16)	cmp.ltu.unc	p50,p48=t0,a1
	(p16)	nop.i		0		};;
	.pred.rel		"mutex",p40,p42
	.pred.rel		"mutex",p48,p50
{ .mfi;	(p16)	nop.m		0			// 28:
	(p16)	xma.hu		ahi[7]=ai7,bj[7],ahi[6]	//	ap[7]*b[i]
	(p40)	add		a2=a2,n2	}	//	(p16) a2+=n2
{ .mfi;	(p42)	add		a2=a2,n2,1
	(p16)	xma.lu		alo[7]=ai7,bj[7],ahi[6]
	(p16)	nop.i		0		};;
{ .mii;	(p16)	getf.sig	a6=alo[6]		// 29:
	(p48)	add		t[6]=t[6],a2		//	(p16) t[6]+=a2
	(p50)	add		t[6]=t[6],a2,1	};;
{ .mfi;	(p16)	nop.m		0			// 30:
	(p16)	xma.hu		nhi[5]=ni5,mj[0],nhi[4]	//	np[5]*m0
	(p40)	cmp.ltu		p41,p39=a2,n2	}
{ .mfi;	(p42)	cmp.leu		p41,p39=a2,n2
	(p16)	xma.lu		nlo[5]=ni5,mj[0],nhi[4]
	(p16)	nop.i		0		};;
{ .mfi;	(p16)	getf.sig	n4=nlo[4]		// 31:
	(p16)	nop.f		0
	(p48)	cmp.ltu		p49,p47=t[6],a2	}
{ .mfb;	(p50)	cmp.leu		p49,p47=t[6],a2
	(p16)	nop.f		0
	br.ctop.sptk.many	.Louter_8_ctop	};;
.Louter_8_cend:

// above loop has to execute one more time, without (p16), which is
// replaced with merged move of np[8] to GPR bank
	.pred.rel		"mutex",p40,p42
	.pred.rel		"mutex",p48,p50
{ .mmi;	(p0)	getf.sig	n1=ni0			// 0:
	(p40)	add		a3=a3,n3		//	(p17) a3+=n3
	(p42)	add		a3=a3,n3,1	};;
{ .mii;	(p17)	getf.sig	a7=alo[8]		// 1:
	(p48)	add		t[6]=t[6],a3		//	(p17) t[6]+=a3
	(p50)	add		t[6]=t[6],a3,1	};;
{ .mfi;	(p17)	getf.sig	a8=ahi[8]		// 2:
	(p17)	xma.hu		nhi[7]=ni6,mj[1],nhi[6]	//	np[6]*m0
	(p40)	cmp.ltu		p43,p41=a3,n3	}
{ .mfi;	(p42)	cmp.leu		p43,p41=a3,n3
	(p17)	xma.lu		nlo[7]=ni6,mj[1],nhi[6]
	(p0)	nop.i		0		};;
{ .mii;	(p17)	getf.sig	n5=nlo[6]		// 3:
	(p48)	cmp.ltu		p51,p49=t[6],a3
	(p50)	cmp.leu		p51,p49=t[6],a3	};;
	.pred.rel		"mutex",p41,p43
	.pred.rel		"mutex",p49,p51
{ .mmi;	(p0)	getf.sig	n2=ni1			// 4:
	(p41)	add		a4=a4,n4		//	(p17) a4+=n4
	(p43)	add		a4=a4,n4,1	};;
{ .mfi;	(p49)	add		t[5]=t[5],a4		// 5:	(p17) t[5]+=a4
	(p0)	nop.f		0
	(p51)	add		t[5]=t[5],a4,1	};;
{ .mfi;	(p0)	getf.sig	n3=ni2			// 6:
	(p17)	xma.hu		nhi[8]=ni7,mj[1],nhi[7]	//	np[7]*m0
	(p41)	cmp.ltu		p42,p40=a4,n4	}
{ .mfi;	(p43)	cmp.leu		p42,p40=a4,n4
	(p17)	xma.lu		nlo[8]=ni7,mj[1],nhi[7]
	(p0)	nop.i		0		};;
{ .mii;	(p17)	getf.sig	n6=nlo[7]		// 7:
	(p49)	cmp.ltu		p50,p48=t[5],a4
	(p51)	cmp.leu		p50,p48=t[5],a4	};;
	.pred.rel		"mutex",p40,p42
	.pred.rel		"mutex",p48,p50
{ .mii;	(p0)	getf.sig	n4=ni3			// 8:
	(p40)	add		a5=a5,n5		//	(p17) a5+=n5
	(p42)	add		a5=a5,n5,1	};;
{ .mii;	(p0)	nop.m		0			// 9:
	(p48)	add		t[4]=t[4],a5		//	p(17) t[4]+=a5
	(p50)	add		t[4]=t[4],a5,1	};;
{ .mii;	(p0)	nop.m		0			// 10:
	(p40)	cmp.ltu		p43,p41=a5,n5
	(p42)	cmp.leu		p43,p41=a5,n5	};;
{ .mii;	(p17)	getf.sig	n7=nlo[8]		// 11:
	(p48)	cmp.ltu		p51,p49=t[4],a5
	(p50)	cmp.leu		p51,p49=t[4],a5	};;
	.pred.rel		"mutex",p41,p43
	.pred.rel		"mutex",p49,p51
{ .mii;	(p17)	getf.sig	n8=nhi[8]		// 12:
	(p41)	add		a6=a6,n6		//	(p17) a6+=n6
	(p43)	add		a6=a6,n6,1	};;
{ .mii;	(p0)	getf.sig	n5=ni4			// 13:
	(p49)	add		t[3]=t[3],a6		//	(p17) t[3]+=a6
	(p51)	add		t[3]=t[3],a6,1	};;
{ .mii;	(p0)	nop.m		0			// 14:
	(p41)	cmp.ltu		p42,p40=a6,n6
	(p43)	cmp.leu		p42,p40=a6,n6	};;
{ .mii;	(p0)	getf.sig	n6=ni5			// 15:
	(p49)	cmp.ltu		p50,p48=t[3],a6
	(p51)	cmp.leu		p50,p48=t[3],a6	};;
	.pred.rel		"mutex",p40,p42
	.pred.rel		"mutex",p48,p50
{ .mii;	(p0)	nop.m		0			// 16:
	(p40)	add		a7=a7,n7		//	(p17) a7+=n7
	(p42)	add		a7=a7,n7,1	};;
{ .mii;	(p0)	nop.m		0			// 17:
	(p48)	add		t[2]=t[2],a7		//	(p17) t[2]+=a7
	(p50)	add		t[2]=t[2],a7,1	};;
{ .mii;	(p0)	nop.m		0			// 18:
	(p40)	cmp.ltu		p43,p41=a7,n7
	(p42)	cmp.leu		p43,p41=a7,n7	};;
{ .mii;	(p0)	getf.sig	n7=ni6			// 19:
	(p48)	cmp.ltu		p51,p49=t[2],a7
	(p50)	cmp.leu		p51,p49=t[2],a7	};;
	.pred.rel		"mutex",p41,p43
	.pred.rel		"mutex",p49,p51
{ .mii;	(p0)	nop.m		0			// 20:
	(p41)	add		a8=a8,n8		//	(p17) a8+=n8
	(p43)	add		a8=a8,n8,1	};;
{ .mmi;	(p0)	nop.m		0			// 21:
	(p49)	add		t[1]=t[1],a8		//	(p17) t[1]+=a8
	(p51)	add		t[1]=t[1],a8,1	}
{ .mmi;	(p17)	mov		t[0]=r0
	(p41)	cmp.ltu		p42,p40=a8,n8
	(p43)	cmp.leu		p42,p40=a8,n8	};;
{ .mmi;	(p0)	getf.sig	n8=ni7			// 22:
	(p49)	cmp.ltu		p50,p48=t[1],a8
	(p51)	cmp.leu		p50,p48=t[1],a8	}
{ .mmi;	(p42)	add		t[0]=t[0],r0,1
	(p0)	add		r16=-7*16,prevsp
	(p0)	add		r17=-6*16,prevsp	};;

// subtract np[8] from carrybit|tmp[8]
// carrybit|tmp[8] layout upon exit from above loop is:
//	t[0]|t[1]|t[2]|t[3]|t[4]|t[5]|t[6]|t[7]|t0 (least significant)
{ .mmi;	(p50)add	t[0]=t[0],r0,1
	add		r18=-5*16,prevsp
	sub		n1=t0,n1	};;
{ .mmi;	cmp.gtu		p34,p32=n1,t0;;
	.pred.rel	"mutex",p32,p34
	(p32)sub	n2=t[7],n2
	(p34)sub	n2=t[7],n2,1	};;
{ .mii;	(p32)cmp.gtu	p35,p33=n2,t[7]
	(p34)cmp.geu	p35,p33=n2,t[7];;
	.pred.rel	"mutex",p33,p35
	(p33)sub	n3=t[6],n3	}
{ .mmi;	(p35)sub	n3=t[6],n3,1;;
	(p33)cmp.gtu	p34,p32=n3,t[6]
	(p35)cmp.geu	p34,p32=n3,t[6]	};;
	.pred.rel	"mutex",p32,p34
{ .mii;	(p32)sub	n4=t[5],n4
	(p34)sub	n4=t[5],n4,1;;
	(p32)cmp.gtu	p35,p33=n4,t[5]	}
{ .mmi;	(p34)cmp.geu	p35,p33=n4,t[5];;
	.pred.rel	"mutex",p33,p35
	(p33)sub	n5=t[4],n5
	(p35)sub	n5=t[4],n5,1	};;
{ .mii;	(p33)cmp.gtu	p34,p32=n5,t[4]
	(p35)cmp.geu	p34,p32=n5,t[4];;
	.pred.rel	"mutex",p32,p34
	(p32)sub	n6=t[3],n6	}
{ .mmi;	(p34)sub	n6=t[3],n6,1;;
	(p32)cmp.gtu	p35,p33=n6,t[3]
	(p34)cmp.geu	p35,p33=n6,t[3]	};;
	.pred.rel	"mutex",p33,p35
{ .mii;	(p33)sub	n7=t[2],n7
	(p35)sub	n7=t[2],n7,1;;
	(p33)cmp.gtu	p34,p32=n7,t[2]	}
{ .mmi;	(p35)cmp.geu	p34,p32=n7,t[2];;
	.pred.rel	"mutex",p32,p34
	(p32)sub	n8=t[1],n8
	(p34)sub	n8=t[1],n8,1	};;
{ .mii;	(p32)cmp.gtu	p35,p33=n8,t[1]
	(p34)cmp.geu	p35,p33=n8,t[1];;
	.pred.rel	"mutex",p33,p35
	(p33)sub	a8=t[0],r0	}
{ .mmi;	(p35)sub	a8=t[0],r0,1;;
	(p33)cmp.gtu	p34,p32=a8,t[0]
	(p35)cmp.geu	p34,p32=a8,t[0]	};;

// save the result, either tmp[num] or tmp[num]-np[num]
	.pred.rel	"mutex",p32,p34
{ .mmi;	(p32)st8	[rptr]=n1,8
	(p34)st8	[rptr]=t0,8
	add		r19=-4*16,prevsp};;
{ .mmb;	(p32)st8	[rptr]=n2,8
	(p34)st8	[rptr]=t[7],8
	(p5)br.cond.dpnt.few	.Ldone	};;
{ .mmb;	(p32)st8	[rptr]=n3,8
	(p34)st8	[rptr]=t[6],8
	(p7)br.cond.dpnt.few	.Ldone	};;
{ .mmb;	(p32)st8	[rptr]=n4,8
	(p34)st8	[rptr]=t[5],8
	(p9)br.cond.dpnt.few	.Ldone	};;
{ .mmb;	(p32)st8	[rptr]=n5,8
	(p34)st8	[rptr]=t[4],8
	(p11)br.cond.dpnt.few	.Ldone	};;
{ .mmb;	(p32)st8	[rptr]=n6,8
	(p34)st8	[rptr]=t[3],8
	(p13)br.cond.dpnt.few	.Ldone	};;
{ .mmb;	(p32)st8	[rptr]=n7,8
	(p34)st8	[rptr]=t[2],8
	(p15)br.cond.dpnt.few	.Ldone	};;
{ .mmb;	(p32)st8	[rptr]=n8,8
	(p34)st8	[rptr]=t[1],8
	nop.b		0		};;
.Ldone:						// epilogue
{ .mmi;	ldf.fill	f16=[r16],64
	ldf.fill	f17=[r17],64
	nop.i		0		}
{ .mmi;	ldf.fill	f18=[r18],64
	ldf.fill	f19=[r19],64
	mov		pr=prevpr,0x1ffff	};;
{ .mmi;	ldf.fill	f20=[r16]
	ldf.fill	f21=[r17]
	mov		ar.lc=prevlc	}
{ .mmi;	ldf.fill	f22=[r18]
	ldf.fill	f23=[r19]
	mov		ret0=1		}	// signal "handled"
{ .mib;	rum		1<<5
	.restore	sp
	mov		sp=prevsp
	br.ret.sptk.many	b0	};;
.endp	bn_mul_mont_8#

.type	copyright#,\@object
copyright:
stringz	"Montgomery multiplication for IA-64, CRYPTOGAMS by <appro\@openssl.org>"
___

open STDOUT,">$output" if $output;
print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# This module doesn't present direct interest for OpenSSL, because it
# doesn't provide better performance for longer keys, at least not on
# in-order-execution cores. While 512-bit RSA sign operations can be
# 65% faster in 64-bit mode, 1024-bit ones are only 15% faster, and
# 4096-bit ones are up to 15% slower. In 32-bit mode it varies from
# 16% improvement for 512-bit RSA sign to -33% for 4096-bit RSA
# verify:-( All comparisons are against bn_mul_mont-free assembler.
# The module might be of interest to embedded system developers, as
# the code is smaller than 1KB, yet offers >3x improvement on MIPS64
# and 75-30% [less for longer keys] on MIPS32 over compiler-generated
# code.

######################################################################
# There is a number of MIPS ABI in use, O32 and N32/64 are most
# widely used. Then there is a new contender: NUBI. It appears that if
# one picks the latter, it's possible to arrange code in ABI neutral
# manner. Therefore let's stick to NUBI register layout:
#
($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25));
($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23));
($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31));
#
# The return value is placed in $a0. Following coding rules facilitate
# interoperability:
#
# - never ever touch $tp, "thread pointer", former $gp;
# - copy return value to $t0, former $v0 [or to $a0 if you're adapting
#   old code];
# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary;
#
# For reference here is register layout for N32/64 MIPS ABIs:
#
# ($zero,$at,$v0,$v1)=map("\$$_",(0..3));
# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25));
# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23));
# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31));
#
$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64

if ($flavour =~ /64|n32/i) {
	$PTR_ADD="daddu";	# incidentally works even on n32
	$PTR_SUB="dsubu";	# incidentally works even on n32
	$REG_S="sd";
	$REG_L="ld";
	$SZREG=8;
} else {
	$PTR_ADD="addu";
	$PTR_SUB="subu";
	$REG_S="sw";
	$REG_L="lw";
	$SZREG=4;
}
$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? 0x00fff000 : 0x00ff0000;
#
# <appro@openssl.org>
#
######################################################################

while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

if ($flavour =~ /64|n32/i) {
	$LD="ld";
	$ST="sd";
	$MULTU="dmultu";
	$ADDU="daddu";
	$SUBU="dsubu";
	$BNSZ=8;
} else {
	$LD="lw";
	$ST="sw";
	$MULTU="multu";
	$ADDU="addu";
	$SUBU="subu";
	$BNSZ=4;
}

# int bn_mul_mont(
$rp=$a0;	# BN_ULONG *rp,
$ap=$a1;	# const BN_ULONG *ap,
$bp=$a2;	# const BN_ULONG *bp,
$np=$a3;	# const BN_ULONG *np,
$n0=$a4;	# const BN_ULONG *n0,
$num=$a5;	# int num);

$lo0=$a6;
$hi0=$a7;
$lo1=$t1;
$hi1=$t2;
$aj=$s0;
$bi=$s1;
$nj=$s2;
$tp=$s3;
$alo=$s4;
$ahi=$s5;
$nlo=$s6;
$nhi=$s7;
$tj=$s8;
$i=$s9;
$j=$s10;
$m1=$s11;

$FRAMESIZE=14;

$code=<<___;
#include "mips_arch.h"

.text

.set	noat
.set	noreorder

.align	5
.globl	bn_mul_mont
.ent	bn_mul_mont
bn_mul_mont:
___
$code.=<<___ if ($flavour =~ /o32/i);
	lw	$n0,16($sp)
	lw	$num,20($sp)
___
$code.=<<___;
	slt	$at,$num,4
	bnez	$at,1f
	li	$t0,0
	slt	$at,$num,17	# on in-order CPU
	bnez	$at,bn_mul_mont_internal
	nop
1:	jr	$ra
	li	$a0,0
.end	bn_mul_mont

.align	5
.ent	bn_mul_mont_internal
bn_mul_mont_internal:
	.frame	$fp,$FRAMESIZE*$SZREG,$ra
	.mask	0x40000000|$SAVED_REGS_MASK,-$SZREG
	$PTR_SUB $sp,$FRAMESIZE*$SZREG
	$REG_S	$fp,($FRAMESIZE-1)*$SZREG($sp)
	$REG_S	$s11,($FRAMESIZE-2)*$SZREG($sp)
	$REG_S	$s10,($FRAMESIZE-3)*$SZREG($sp)
	$REG_S	$s9,($FRAMESIZE-4)*$SZREG($sp)
	$REG_S	$s8,($FRAMESIZE-5)*$SZREG($sp)
	$REG_S	$s7,($FRAMESIZE-6)*$SZREG($sp)
	$REG_S	$s6,($FRAMESIZE-7)*$SZREG($sp)
	$REG_S	$s5,($FRAMESIZE-8)*$SZREG($sp)
	$REG_S	$s4,($FRAMESIZE-9)*$SZREG($sp)
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_S	$s3,($FRAMESIZE-10)*$SZREG($sp)
	$REG_S	$s2,($FRAMESIZE-11)*$SZREG($sp)
	$REG_S	$s1,($FRAMESIZE-12)*$SZREG($sp)
	$REG_S	$s0,($FRAMESIZE-13)*$SZREG($sp)
___
$code.=<<___;
	move	$fp,$sp

	.set	reorder
	$LD	$n0,0($n0)
	$LD	$bi,0($bp)	# bp[0]
	$LD	$aj,0($ap)	# ap[0]
	$LD	$nj,0($np)	# np[0]

	$PTR_SUB $sp,2*$BNSZ	# place for two extra words
	sll	$num,`log($BNSZ)/log(2)`
	li	$at,-4096
	$PTR_SUB $sp,$num
	and	$sp,$at

	$MULTU	($aj,$bi)
	$LD	$ahi,$BNSZ($ap)
	$LD	$nhi,$BNSZ($np)
	mflo	($lo0,$aj,$bi)
	mfhi	($hi0,$aj,$bi)
	$MULTU	($lo0,$n0)
	mflo	($m1,$lo0,$n0)

	$MULTU	($ahi,$bi)
	mflo	($alo,$ahi,$bi)
	mfhi	($ahi,$ahi,$bi)

	$MULTU	($nj,$m1)
	mflo	($lo1,$nj,$m1)
	mfhi	($hi1,$nj,$m1)
	$MULTU	($nhi,$m1)
	$ADDU	$lo1,$lo0
	sltu	$at,$lo1,$lo0
	$ADDU	$hi1,$at
	mflo	($nlo,$nhi,$m1)
	mfhi	($nhi,$nhi,$m1)

	move	$tp,$sp
	li	$j,2*$BNSZ
.align	4
.L1st:
	.set	noreorder
	$PTR_ADD $aj,$ap,$j
	$PTR_ADD $nj,$np,$j
	$LD	$aj,($aj)
	$LD	$nj,($nj)

	$MULTU	($aj,$bi)
	$ADDU	$lo0,$alo,$hi0
	$ADDU	$lo1,$nlo,$hi1
	sltu	$at,$lo0,$hi0
	sltu	$t0,$lo1,$hi1
	$ADDU	$hi0,$ahi,$at
	$ADDU	$hi1,$nhi,$t0
	mflo	($alo,$aj,$bi)
	mfhi	($ahi,$aj,$bi)

	$ADDU	$lo1,$lo0
	sltu	$at,$lo1,$lo0
	$MULTU	($nj,$m1)
	$ADDU	$hi1,$at
	addu	$j,$BNSZ
	$ST	$lo1,($tp)
	sltu	$t0,$j,$num
	mflo	($nlo,$nj,$m1)
	mfhi	($nhi,$nj,$m1)

	bnez	$t0,.L1st
	$PTR_ADD $tp,$BNSZ
	.set	reorder

	$ADDU	$lo0,$alo,$hi0
	sltu	$at,$lo0,$hi0
	$ADDU	$hi0,$ahi,$at

	$ADDU	$lo1,$nlo,$hi1
	sltu	$t0,$lo1,$hi1
	$ADDU	$hi1,$nhi,$t0
	$ADDU	$lo1,$lo0
	sltu	$at,$lo1,$lo0
	$ADDU	$hi1,$at

	$ST	$lo1,($tp)

	$ADDU	$hi1,$hi0
	sltu	$at,$hi1,$hi0
	$ST	$hi1,$BNSZ($tp)
	$ST	$at,2*$BNSZ($tp)

	li	$i,$BNSZ
.align	4
.Louter:
	$PTR_ADD $bi,$bp,$i
	$LD	$bi,($bi)
	$LD	$aj,($ap)
	$LD	$ahi,$BNSZ($ap)
	$LD	$tj,($sp)

	$MULTU	($aj,$bi)
	$LD	$nj,($np)
	$LD	$nhi,$BNSZ($np)
	mflo	($lo0,$aj,$bi)
	mfhi	($hi0,$aj,$bi)
	$ADDU	$lo0,$tj
	$MULTU	($lo0,$n0)
	sltu	$at,$lo0,$tj
	$ADDU	$hi0,$at
	mflo	($m1,$lo0,$n0)

	$MULTU	($ahi,$bi)
	mflo	($alo,$ahi,$bi)
	mfhi	($ahi,$ahi,$bi)

	$MULTU	($nj,$m1)
	mflo	($lo1,$nj,$m1)
	mfhi	($hi1,$nj,$m1)

	$MULTU	($nhi,$m1)
	$ADDU	$lo1,$lo0
	sltu	$at,$lo1,$lo0
	$ADDU	$hi1,$at
	mflo	($nlo,$nhi,$m1)
	mfhi	($nhi,$nhi,$m1)

	move	$tp,$sp
	li	$j,2*$BNSZ
	$LD	$tj,$BNSZ($tp)
.align	4
.Linner:
	.set	noreorder
	$PTR_ADD $aj,$ap,$j
	$PTR_ADD $nj,$np,$j
	$LD	$aj,($aj)
	$LD	$nj,($nj)

	$MULTU	($aj,$bi)
	$ADDU	$lo0,$alo,$hi0
	$ADDU	$lo1,$nlo,$hi1
	sltu	$at,$lo0,$hi0
	sltu	$t0,$lo1,$hi1
	$ADDU	$hi0,$ahi,$at
	$ADDU	$hi1,$nhi,$t0
	mflo	($alo,$aj,$bi)
	mfhi	($ahi,$aj,$bi)

	$ADDU	$lo0,$tj
	addu	$j,$BNSZ
	$MULTU	($nj,$m1)
	sltu	$at,$lo0,$tj
	$ADDU	$lo1,$lo0
	$ADDU	$hi0,$at
	sltu	$t0,$lo1,$lo0
	$LD	$tj,2*$BNSZ($tp)
	$ADDU	$hi1,$t0
	sltu	$at,$j,$num
	mflo	($nlo,$nj,$m1)
	mfhi	($nhi,$nj,$m1)
	$ST	$lo1,($tp)
	bnez	$at,.Linner
	$PTR_ADD $tp,$BNSZ
	.set	reorder

	$ADDU	$lo0,$alo,$hi0
	sltu	$at,$lo0,$hi0
	$ADDU	$hi0,$ahi,$at
	$ADDU	$lo0,$tj
	sltu	$t0,$lo0,$tj
	$ADDU	$hi0,$t0

	$LD	$tj,2*$BNSZ($tp)
	$ADDU	$lo1,$nlo,$hi1
	sltu	$at,$lo1,$hi1
	$ADDU	$hi1,$nhi,$at
	$ADDU	$lo1,$lo0
	sltu	$t0,$lo1,$lo0
	$ADDU	$hi1,$t0
	$ST	$lo1,($tp)

	$ADDU	$lo1,$hi1,$hi0
	sltu	$hi1,$lo1,$hi0
	$ADDU	$lo1,$tj
	sltu	$at,$lo1,$tj
	$ADDU	$hi1,$at
	$ST	$lo1,$BNSZ($tp)
	$ST	$hi1,2*$BNSZ($tp)

	addu	$i,$BNSZ
	sltu	$t0,$i,$num
	bnez	$t0,.Louter

	.set	noreorder
	$PTR_ADD $tj,$sp,$num	# &tp[num]
	move	$tp,$sp
	move	$ap,$sp
	li	$hi0,0		# clear borrow bit

.align	4
.Lsub:	$LD	$lo0,($tp)
	$LD	$lo1,($np)
	$PTR_ADD $tp,$BNSZ
	$PTR_ADD $np,$BNSZ
	$SUBU	$lo1,$lo0,$lo1	# tp[i]-np[i]
	sgtu	$at,$lo1,$lo0
	$SUBU	$lo0,$lo1,$hi0
	sgtu	$hi0,$lo0,$lo1
	$ST	$lo0,($rp)
	or	$hi0,$at
	sltu	$at,$tp,$tj
	bnez	$at,.Lsub
	$PTR_ADD $rp,$BNSZ

	$SUBU	$hi0,$hi1,$hi0	# handle upmost overflow bit
	move	$tp,$sp
	$PTR_SUB $rp,$num	# restore rp
	not	$hi1,$hi0

.Lcopy:	$LD	$nj,($tp)	# conditional move
	$LD	$aj,($rp)
	$ST	$zero,($tp)
	$PTR_ADD $tp,$BNSZ
	and	$nj,$hi0
	and	$aj,$hi1
	or	$aj,$nj
	sltu	$at,$tp,$tj
	$ST	$aj,($rp)
	bnez	$at,.Lcopy
	$PTR_ADD $rp,$BNSZ

	li	$a0,1
	li	$t0,1

	.set	noreorder
	move	$sp,$fp
	$REG_L	$fp,($FRAMESIZE-1)*$SZREG($sp)
	$REG_L	$s11,($FRAMESIZE-2)*$SZREG($sp)
	$REG_L	$s10,($FRAMESIZE-3)*$SZREG($sp)
	$REG_L	$s9,($FRAMESIZE-4)*$SZREG($sp)
	$REG_L	$s8,($FRAMESIZE-5)*$SZREG($sp)
	$REG_L	$s7,($FRAMESIZE-6)*$SZREG($sp)
	$REG_L	$s6,($FRAMESIZE-7)*$SZREG($sp)
	$REG_L	$s5,($FRAMESIZE-8)*$SZREG($sp)
	$REG_L	$s4,($FRAMESIZE-9)*$SZREG($sp)
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$s3,($FRAMESIZE-10)*$SZREG($sp)
	$REG_L	$s2,($FRAMESIZE-11)*$SZREG($sp)
	$REG_L	$s1,($FRAMESIZE-12)*$SZREG($sp)
	$REG_L	$s0,($FRAMESIZE-13)*$SZREG($sp)
___
$code.=<<___;
	jr	$ra
	$PTR_ADD $sp,$FRAMESIZE*$SZREG
.end	bn_mul_mont_internal
.rdata
.asciiz	"Montgomery Multiplication for MIPS, CRYPTOGAMS by <appro\@openssl.org>"
___

$code =~ s/\`([^\`]*)\`/eval $1/gem;

print $code;
close STDOUT or die "error closing STDOUT: $!";


#! /usr/bin/env perl
# Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project.
#
# Rights for redistribution and usage in source and binary forms are
# granted according to the OpenSSL license. Warranty of any kind is
# disclaimed.
# ====================================================================

# July 1999
#
# This is drop-in MIPS III/IV ISA replacement for crypto/bn/bn_asm.c.
#
# The module is designed to work with either of the "new" MIPS ABI(5),
# namely N32 or N64, offered by IRIX 6.x. It's not meant to work under
# IRIX 5.x not only because it doesn't support new ABIs but also
# because 5.x kernels put R4x00 CPU into 32-bit mode and all those
# 64-bit instructions (daddu, dmultu, etc.) found below gonna only
# cause illegal instruction exception:-(
#
# In addition the code depends on preprocessor flags set up by MIPSpro
# compiler driver (either as or cc) and therefore (probably?) can't be
# compiled by the GNU assembler. GNU C driver manages fine though...
# I mean as long as -mmips-as is specified or is the default option,
# because then it simply invokes /usr/bin/as which in turn takes
# perfect care of the preprocessor definitions. Another neat feature
# offered by the MIPSpro assembler is an optimization pass. This gave
# me the opportunity to have the code looking more regular as all those
# architecture dependent instruction rescheduling details were left to
# the assembler. Cool, huh?
#
# Performance improvement is astonishing! 'apps/openssl speed rsa dsa'
# goes way over 3 times faster!
#
#					<appro@openssl.org>

# October 2010
#
# Adapt the module even for 32-bit ABIs and other OSes. The former was
# achieved by mechanical replacement of 64-bit arithmetic instructions
# such as dmultu, daddu, etc. with their 32-bit counterparts and
# adjusting offsets denoting multiples of BN_ULONG. Above mentioned
# >3x performance improvement naturally does not apply to 32-bit code
# [because there is no instruction 32-bit compiler can't use], one
# has to content with 40-85% improvement depending on benchmark and
# key length, more for longer keys.

$flavour = shift || "o32";
while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

if ($flavour =~ /64|n32/i) {
	$LD="ld";
	$ST="sd";
	$MULTU="dmultu";
	$DIVU="ddivu";
	$ADDU="daddu";
	$SUBU="dsubu";
	$SRL="dsrl";
	$SLL="dsll";
	$BNSZ=8;
	$PTR_ADD="daddu";
	$PTR_SUB="dsubu";
	$SZREG=8;
	$REG_S="sd";
	$REG_L="ld";
} else {
	$LD="lw";
	$ST="sw";
	$MULTU="multu";
	$DIVU="divu";
	$ADDU="addu";
	$SUBU="subu";
	$SRL="srl";
	$SLL="sll";
	$BNSZ=4;
	$PTR_ADD="addu";
	$PTR_SUB="subu";
	$SZREG=4;
	$REG_S="sw";
	$REG_L="lw";
	$code="#if !(defined (__mips_isa_rev) && (__mips_isa_rev >= 6))\n.set     mips2\n#endif\n";
}

# Below is N32/64 register layout used in the original module.
#
($zero,$at,$v0,$v1)=map("\$$_",(0..3));
($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11));
($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25));
($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23));
($gp,$sp,$fp,$ra)=map("\$$_",(28..31));
($ta0,$ta1,$ta2,$ta3)=($a4,$a5,$a6,$a7);
#
# No special adaptation is required for O32. NUBI on the other hand
# is treated by saving/restoring ($v1,$t0..$t3).

$gp=$v1 if ($flavour =~ /nubi/i);

$minus4=$v1;

$code.=<<___;
#include "mips_arch.h"

#if defined(_MIPS_ARCH_MIPS64R6)
# define ddivu(rs,rt)
# define mfqt(rd,rs,rt)	ddivu	rd,rs,rt
# define mfrm(rd,rs,rt)	dmodu	rd,rs,rt
#elif defined(_MIPS_ARCH_MIPS32R6)
# define divu(rs,rt)
# define mfqt(rd,rs,rt)	divu	rd,rs,rt
# define mfrm(rd,rs,rt)	modu	rd,rs,rt
#else
# define $DIVU(rs,rt)	$DIVU	$zero,rs,rt
# define mfqt(rd,rs,rt)	mflo	rd
# define mfrm(rd,rs,rt)	mfhi	rd
#endif

.rdata
.asciiz	"mips3.s, Version 1.2"
.asciiz	"MIPS II/III/IV ISA artwork by Andy Polyakov <appro\@fy.chalmers.se>"

.text
.set	noat

.align	5
.globl	bn_mul_add_words
.ent	bn_mul_add_words
bn_mul_add_words:
	.set	noreorder
	bgtz	$a2,bn_mul_add_words_internal
	move	$v0,$zero
	jr	$ra
	move	$a0,$v0
.end	bn_mul_add_words

.align	5
.ent	bn_mul_add_words_internal
bn_mul_add_words_internal:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	li	$minus4,-4
	and	$ta0,$a2,$minus4
	beqz	$ta0,.L_bn_mul_add_words_tail

.L_bn_mul_add_words_loop:
	$LD	$t0,0($a1)
	$MULTU	($t0,$a3)
	$LD	$t1,0($a0)
	$LD	$t2,$BNSZ($a1)
	$LD	$t3,$BNSZ($a0)
	$LD	$ta0,2*$BNSZ($a1)
	$LD	$ta1,2*$BNSZ($a0)
	$ADDU	$t1,$v0
	sltu	$v0,$t1,$v0	# All manuals say it "compares 32-bit
				# values", but it seems to work fine
				# even on 64-bit registers.
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$t1,$at
	$ADDU	$v0,$t0
	 $MULTU	($t2,$a3)
	sltu	$at,$t1,$at
	$ST	$t1,0($a0)
	$ADDU	$v0,$at

	$LD	$ta2,3*$BNSZ($a1)
	$LD	$ta3,3*$BNSZ($a0)
	$ADDU	$t3,$v0
	sltu	$v0,$t3,$v0
	mflo	($at,$t2,$a3)
	mfhi	($t2,$t2,$a3)
	$ADDU	$t3,$at
	$ADDU	$v0,$t2
	 $MULTU	($ta0,$a3)
	sltu	$at,$t3,$at
	$ST	$t3,$BNSZ($a0)
	$ADDU	$v0,$at

	subu	$a2,4
	$PTR_ADD $a0,4*$BNSZ
	$PTR_ADD $a1,4*$BNSZ
	$ADDU	$ta1,$v0
	sltu	$v0,$ta1,$v0
	mflo	($at,$ta0,$a3)
	mfhi	($ta0,$ta0,$a3)
	$ADDU	$ta1,$at
	$ADDU	$v0,$ta0
	 $MULTU	($ta2,$a3)
	sltu	$at,$ta1,$at
	$ST	$ta1,-2*$BNSZ($a0)
	$ADDU	$v0,$at

	and	$ta0,$a2,$minus4
	$ADDU	$ta3,$v0
	sltu	$v0,$ta3,$v0
	mflo	($at,$ta2,$a3)
	mfhi	($ta2,$ta2,$a3)
	$ADDU	$ta3,$at
	$ADDU	$v0,$ta2
	sltu	$at,$ta3,$at
	$ST	$ta3,-$BNSZ($a0)
	.set	noreorder
	bgtz	$ta0,.L_bn_mul_add_words_loop
	$ADDU	$v0,$at

	beqz	$a2,.L_bn_mul_add_words_return
	nop

.L_bn_mul_add_words_tail:
	.set	reorder
	$LD	$t0,0($a1)
	$MULTU	($t0,$a3)
	$LD	$t1,0($a0)
	subu	$a2,1
	$ADDU	$t1,$v0
	sltu	$v0,$t1,$v0
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$t1,$at
	$ADDU	$v0,$t0
	sltu	$at,$t1,$at
	$ST	$t1,0($a0)
	$ADDU	$v0,$at
	beqz	$a2,.L_bn_mul_add_words_return

	$LD	$t0,$BNSZ($a1)
	$MULTU	($t0,$a3)
	$LD	$t1,$BNSZ($a0)
	subu	$a2,1
	$ADDU	$t1,$v0
	sltu	$v0,$t1,$v0
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$t1,$at
	$ADDU	$v0,$t0
	sltu	$at,$t1,$at
	$ST	$t1,$BNSZ($a0)
	$ADDU	$v0,$at
	beqz	$a2,.L_bn_mul_add_words_return

	$LD	$t0,2*$BNSZ($a1)
	$MULTU	($t0,$a3)
	$LD	$t1,2*$BNSZ($a0)
	$ADDU	$t1,$v0
	sltu	$v0,$t1,$v0
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$t1,$at
	$ADDU	$v0,$t0
	sltu	$at,$t1,$at
	$ST	$t1,2*$BNSZ($a0)
	$ADDU	$v0,$at

.L_bn_mul_add_words_return:
	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	move	$a0,$v0
.end	bn_mul_add_words_internal

.align	5
.globl	bn_mul_words
.ent	bn_mul_words
bn_mul_words:
	.set	noreorder
	bgtz	$a2,bn_mul_words_internal
	move	$v0,$zero
	jr	$ra
	move	$a0,$v0
.end	bn_mul_words

.align	5
.ent	bn_mul_words_internal
bn_mul_words_internal:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	li	$minus4,-4
	and	$ta0,$a2,$minus4
	beqz	$ta0,.L_bn_mul_words_tail

.L_bn_mul_words_loop:
	$LD	$t0,0($a1)
	$MULTU	($t0,$a3)
	$LD	$t2,$BNSZ($a1)
	$LD	$ta0,2*$BNSZ($a1)
	$LD	$ta2,3*$BNSZ($a1)
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$v0,$at
	sltu	$t1,$v0,$at
	 $MULTU	($t2,$a3)
	$ST	$v0,0($a0)
	$ADDU	$v0,$t1,$t0

	subu	$a2,4
	$PTR_ADD $a0,4*$BNSZ
	$PTR_ADD $a1,4*$BNSZ
	mflo	($at,$t2,$a3)
	mfhi	($t2,$t2,$a3)
	$ADDU	$v0,$at
	sltu	$t3,$v0,$at
	 $MULTU	($ta0,$a3)
	$ST	$v0,-3*$BNSZ($a0)
	$ADDU	$v0,$t3,$t2

	mflo	($at,$ta0,$a3)
	mfhi	($ta0,$ta0,$a3)
	$ADDU	$v0,$at
	sltu	$ta1,$v0,$at
	 $MULTU	($ta2,$a3)
	$ST	$v0,-2*$BNSZ($a0)
	$ADDU	$v0,$ta1,$ta0

	and	$ta0,$a2,$minus4
	mflo	($at,$ta2,$a3)
	mfhi	($ta2,$ta2,$a3)
	$ADDU	$v0,$at
	sltu	$ta3,$v0,$at
	$ST	$v0,-$BNSZ($a0)
	.set	noreorder
	bgtz	$ta0,.L_bn_mul_words_loop
	$ADDU	$v0,$ta3,$ta2

	beqz	$a2,.L_bn_mul_words_return
	nop

.L_bn_mul_words_tail:
	.set	reorder
	$LD	$t0,0($a1)
	$MULTU	($t0,$a3)
	subu	$a2,1
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$v0,$at
	sltu	$t1,$v0,$at
	$ST	$v0,0($a0)
	$ADDU	$v0,$t1,$t0
	beqz	$a2,.L_bn_mul_words_return

	$LD	$t0,$BNSZ($a1)
	$MULTU	($t0,$a3)
	subu	$a2,1
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$v0,$at
	sltu	$t1,$v0,$at
	$ST	$v0,$BNSZ($a0)
	$ADDU	$v0,$t1,$t0
	beqz	$a2,.L_bn_mul_words_return

	$LD	$t0,2*$BNSZ($a1)
	$MULTU	($t0,$a3)
	mflo	($at,$t0,$a3)
	mfhi	($t0,$t0,$a3)
	$ADDU	$v0,$at
	sltu	$t1,$v0,$at
	$ST	$v0,2*$BNSZ($a0)
	$ADDU	$v0,$t1,$t0

.L_bn_mul_words_return:
	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	move	$a0,$v0
.end	bn_mul_words_internal

.align	5
.globl	bn_sqr_words
.ent	bn_sqr_words
bn_sqr_words:
	.set	noreorder
	bgtz	$a2,bn_sqr_words_internal
	move	$v0,$zero
	jr	$ra
	move	$a0,$v0
.end	bn_sqr_words

.align	5
.ent	bn_sqr_words_internal
bn_sqr_words_internal:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	li	$minus4,-4
	and	$ta0,$a2,$minus4
	beqz	$ta0,.L_bn_sqr_words_tail

.L_bn_sqr_words_loop:
	$LD	$t0,0($a1)
	$MULTU	($t0,$t0)
	$LD	$t2,$BNSZ($a1)
	$LD	$ta0,2*$BNSZ($a1)
	$LD	$ta2,3*$BNSZ($a1)
	mflo	($t1,$t0,$t0)
	mfhi	($t0,$t0,$t0)
	$ST	$t1,0($a0)
	$ST	$t0,$BNSZ($a0)

	$MULTU	($t2,$t2)
	subu	$a2,4
	$PTR_ADD $a0,8*$BNSZ
	$PTR_ADD $a1,4*$BNSZ
	mflo	($t3,$t2,$t2)
	mfhi	($t2,$t2,$t2)
	$ST	$t3,-6*$BNSZ($a0)
	$ST	$t2,-5*$BNSZ($a0)

	$MULTU	($ta0,$ta0)
	mflo	($ta1,$ta0,$ta0)
	mfhi	($ta0,$ta0,$ta0)
	$ST	$ta1,-4*$BNSZ($a0)
	$ST	$ta0,-3*$BNSZ($a0)

	$MULTU	($ta2,$ta2)
	and	$ta0,$a2,$minus4
	mflo	($ta3,$ta2,$ta2)
	mfhi	($ta2,$ta2,$ta2)
	$ST	$ta3,-2*$BNSZ($a0)

	.set	noreorder
	bgtz	$ta0,.L_bn_sqr_words_loop
	$ST	$ta2,-$BNSZ($a0)

	beqz	$a2,.L_bn_sqr_words_return
	nop

.L_bn_sqr_words_tail:
	.set	reorder
	$LD	$t0,0($a1)
	$MULTU	($t0,$t0)
	subu	$a2,1
	mflo	($t1,$t0,$t0)
	mfhi	($t0,$t0,$t0)
	$ST	$t1,0($a0)
	$ST	$t0,$BNSZ($a0)
	beqz	$a2,.L_bn_sqr_words_return

	$LD	$t0,$BNSZ($a1)
	$MULTU	($t0,$t0)
	subu	$a2,1
	mflo	($t1,$t0,$t0)
	mfhi	($t0,$t0,$t0)
	$ST	$t1,2*$BNSZ($a0)
	$ST	$t0,3*$BNSZ($a0)
	beqz	$a2,.L_bn_sqr_words_return

	$LD	$t0,2*$BNSZ($a1)
	$MULTU	($t0,$t0)
	mflo	($t1,$t0,$t0)
	mfhi	($t0,$t0,$t0)
	$ST	$t1,4*$BNSZ($a0)
	$ST	$t0,5*$BNSZ($a0)

.L_bn_sqr_words_return:
	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	move	$a0,$v0

.end	bn_sqr_words_internal

.align	5
.globl	bn_add_words
.ent	bn_add_words
bn_add_words:
	.set	noreorder
	bgtz	$a3,bn_add_words_internal
	move	$v0,$zero
	jr	$ra
	move	$a0,$v0
.end	bn_add_words

.align	5
.ent	bn_add_words_internal
bn_add_words_internal:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	li	$minus4,-4
	and	$at,$a3,$minus4
	beqz	$at,.L_bn_add_words_tail

.L_bn_add_words_loop:
	$LD	$t0,0($a1)
	$LD	$ta0,0($a2)
	subu	$a3,4
	$LD	$t1,$BNSZ($a1)
	and	$at,$a3,$minus4
	$LD	$t2,2*$BNSZ($a1)
	$PTR_ADD $a2,4*$BNSZ
	$LD	$t3,3*$BNSZ($a1)
	$PTR_ADD $a0,4*$BNSZ
	$LD	$ta1,-3*$BNSZ($a2)
	$PTR_ADD $a1,4*$BNSZ
	$LD	$ta2,-2*$BNSZ($a2)
	$LD	$ta3,-$BNSZ($a2)
	$ADDU	$ta0,$t0
	sltu	$t8,$ta0,$t0
	$ADDU	$t0,$ta0,$v0
	sltu	$v0,$t0,$ta0
	$ST	$t0,-4*$BNSZ($a0)
	$ADDU	$v0,$t8

	$ADDU	$ta1,$t1
	sltu	$t9,$ta1,$t1
	$ADDU	$t1,$ta1,$v0
	sltu	$v0,$t1,$ta1
	$ST	$t1,-3*$BNSZ($a0)
	$ADDU	$v0,$t9

	$ADDU	$ta2,$t2
	sltu	$t8,$ta2,$t2
	$ADDU	$t2,$ta2,$v0
	sltu	$v0,$t2,$ta2
	$ST	$t2,-2*$BNSZ($a0)
	$ADDU	$v0,$t8

	$ADDU	$ta3,$t3
	sltu	$t9,$ta3,$t3
	$ADDU	$t3,$ta3,$v0
	sltu	$v0,$t3,$ta3
	$ST	$t3,-$BNSZ($a0)

	.set	noreorder
	bgtz	$at,.L_bn_add_words_loop
	$ADDU	$v0,$t9

	beqz	$a3,.L_bn_add_words_return
	nop

.L_bn_add_words_tail:
	.set	reorder
	$LD	$t0,0($a1)
	$LD	$ta0,0($a2)
	$ADDU	$ta0,$t0
	subu	$a3,1
	sltu	$t8,$ta0,$t0
	$ADDU	$t0,$ta0,$v0
	sltu	$v0,$t0,$ta0
	$ST	$t0,0($a0)
	$ADDU	$v0,$t8
	beqz	$a3,.L_bn_add_words_return

	$LD	$t1,$BNSZ($a1)
	$LD	$ta1,$BNSZ($a2)
	$ADDU	$ta1,$t1
	subu	$a3,1
	sltu	$t9,$ta1,$t1
	$ADDU	$t1,$ta1,$v0
	sltu	$v0,$t1,$ta1
	$ST	$t1,$BNSZ($a0)
	$ADDU	$v0,$t9
	beqz	$a3,.L_bn_add_words_return

	$LD	$t2,2*$BNSZ($a1)
	$LD	$ta2,2*$BNSZ($a2)
	$ADDU	$ta2,$t2
	sltu	$t8,$ta2,$t2
	$ADDU	$t2,$ta2,$v0
	sltu	$v0,$t2,$ta2
	$ST	$t2,2*$BNSZ($a0)
	$ADDU	$v0,$t8

.L_bn_add_words_return:
	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	move	$a0,$v0

.end	bn_add_words_internal

.align	5
.globl	bn_sub_words
.ent	bn_sub_words
bn_sub_words:
	.set	noreorder
	bgtz	$a3,bn_sub_words_internal
	move	$v0,$zero
	jr	$ra
	move	$a0,$zero
.end	bn_sub_words

.align	5
.ent	bn_sub_words_internal
bn_sub_words_internal:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	li	$minus4,-4
	and	$at,$a3,$minus4
	beqz	$at,.L_bn_sub_words_tail

.L_bn_sub_words_loop:
	$LD	$t0,0($a1)
	$LD	$ta0,0($a2)
	subu	$a3,4
	$LD	$t1,$BNSZ($a1)
	and	$at,$a3,$minus4
	$LD	$t2,2*$BNSZ($a1)
	$PTR_ADD $a2,4*$BNSZ
	$LD	$t3,3*$BNSZ($a1)
	$PTR_ADD $a0,4*$BNSZ
	$LD	$ta1,-3*$BNSZ($a2)
	$PTR_ADD $a1,4*$BNSZ
	$LD	$ta2,-2*$BNSZ($a2)
	$LD	$ta3,-$BNSZ($a2)
	sltu	$t8,$t0,$ta0
	$SUBU	$ta0,$t0,$ta0
	$SUBU	$t0,$ta0,$v0
	sgtu	$v0,$t0,$ta0
	$ST	$t0,-4*$BNSZ($a0)
	$ADDU	$v0,$t8

	sltu	$t9,$t1,$ta1
	$SUBU	$ta1,$t1,$ta1
	$SUBU	$t1,$ta1,$v0
	sgtu	$v0,$t1,$ta1
	$ST	$t1,-3*$BNSZ($a0)
	$ADDU	$v0,$t9

	sltu	$t8,$t2,$ta2
	$SUBU	$ta2,$t2,$ta2
	$SUBU	$t2,$ta2,$v0
	sgtu	$v0,$t2,$ta2
	$ST	$t2,-2*$BNSZ($a0)
	$ADDU	$v0,$t8

	sltu	$t9,$t3,$ta3
	$SUBU	$ta3,$t3,$ta3
	$SUBU	$t3,$ta3,$v0
	sgtu	$v0,$t3,$ta3
	$ST	$t3,-$BNSZ($a0)

	.set	noreorder
	bgtz	$at,.L_bn_sub_words_loop
	$ADDU	$v0,$t9

	beqz	$a3,.L_bn_sub_words_return
	nop

.L_bn_sub_words_tail:
	.set	reorder
	$LD	$t0,0($a1)
	$LD	$ta0,0($a2)
	subu	$a3,1
	sltu	$t8,$t0,$ta0
	$SUBU	$ta0,$t0,$ta0
	$SUBU	$t0,$ta0,$v0
	sgtu	$v0,$t0,$ta0
	$ST	$t0,0($a0)
	$ADDU	$v0,$t8
	beqz	$a3,.L_bn_sub_words_return

	$LD	$t1,$BNSZ($a1)
	subu	$a3,1
	$LD	$ta1,$BNSZ($a2)
	sltu	$t9,$t1,$ta1
	$SUBU	$ta1,$t1,$ta1
	$SUBU	$t1,$ta1,$v0
	sgtu	$v0,$t1,$ta1
	$ST	$t1,$BNSZ($a0)
	$ADDU	$v0,$t9
	beqz	$a3,.L_bn_sub_words_return

	$LD	$t2,2*$BNSZ($a1)
	$LD	$ta2,2*$BNSZ($a2)
	sltu	$t8,$t2,$ta2
	$SUBU	$ta2,$t2,$ta2
	$SUBU	$t2,$ta2,$v0
	sgtu	$v0,$t2,$ta2
	$ST	$t2,2*$BNSZ($a0)
	$ADDU	$v0,$t8

.L_bn_sub_words_return:
	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	move	$a0,$v0
.end	bn_sub_words_internal

#if 0
/*
 * The bn_div_3_words entry point is re-used for constant-time interface.
 * Implementation is retained as historical reference.
 */
.align 5
.globl	bn_div_3_words
.ent	bn_div_3_words
bn_div_3_words:
	.set	noreorder
	move	$a3,$a0		# we know that bn_div_words does not
				# touch $a3, $ta2, $ta3 and preserves $a2
				# so that we can save two arguments
				# and return address in registers
				# instead of stack:-)

	$LD	$a0,($a3)
	move	$ta2,$a1
	bne	$a0,$a2,bn_div_3_words_internal
	$LD	$a1,-$BNSZ($a3)
	li	$v0,-1
	jr	$ra
	move	$a0,$v0
.end	bn_div_3_words

.align	5
.ent	bn_div_3_words_internal
bn_div_3_words_internal:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	move	$ta3,$ra
	bal	bn_div_words_internal
	move	$ra,$ta3
	$MULTU	($ta2,$v0)
	$LD	$t2,-2*$BNSZ($a3)
	move	$ta0,$zero
	mfhi	($t1,$ta2,$v0)
	mflo	($t0,$ta2,$v0)
	sltu	$t8,$t1,$a1
.L_bn_div_3_words_inner_loop:
	bnez	$t8,.L_bn_div_3_words_inner_loop_done
	sgeu	$at,$t2,$t0
	seq	$t9,$t1,$a1
	and	$at,$t9
	sltu	$t3,$t0,$ta2
	$ADDU	$a1,$a2
	$SUBU	$t1,$t3
	$SUBU	$t0,$ta2
	sltu	$t8,$t1,$a1
	sltu	$ta0,$a1,$a2
	or	$t8,$ta0
	.set	noreorder
	beqz	$at,.L_bn_div_3_words_inner_loop
	$SUBU	$v0,1
	$ADDU	$v0,1
	.set	reorder
.L_bn_div_3_words_inner_loop_done:
	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	move	$a0,$v0
.end	bn_div_3_words_internal
#endif

.align	5
.globl	bn_div_words
.ent	bn_div_words
bn_div_words:
	.set	noreorder
	bnez	$a2,bn_div_words_internal
	li	$v0,-1		# I would rather signal div-by-zero
				# which can be done with 'break 7'
	jr	$ra
	move	$a0,$v0
.end	bn_div_words

.align	5
.ent	bn_div_words_internal
bn_div_words_internal:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	move	$v1,$zero
	bltz	$a2,.L_bn_div_words_body
	move	$t9,$v1
	$SLL	$a2,1
	bgtz	$a2,.-4
	addu	$t9,1

	.set	reorder
	negu	$t1,$t9
	li	$t2,-1
	$SLL	$t2,$t1
	and	$t2,$a0
	$SRL	$at,$a1,$t1
	.set	noreorder
	beqz	$t2,.+12
	nop
	break	6		# signal overflow
	.set	reorder
	$SLL	$a0,$t9
	$SLL	$a1,$t9
	or	$a0,$at
___
$QT=$ta0;
$HH=$ta1;
$DH=$v1;
$code.=<<___;
.L_bn_div_words_body:
	$SRL	$DH,$a2,4*$BNSZ	# bits
	sgeu	$at,$a0,$a2
	.set	noreorder
	beqz	$at,.+12
	nop
	$SUBU	$a0,$a2
	.set	reorder

	li	$QT,-1
	$SRL	$HH,$a0,4*$BNSZ	# bits
	$SRL	$QT,4*$BNSZ	# q=0xffffffff
	beq	$DH,$HH,.L_bn_div_words_skip_div1
	$DIVU	($a0,$DH)
	mfqt	($QT,$a0,$DH)
.L_bn_div_words_skip_div1:
	$MULTU	($a2,$QT)
	$SLL	$t3,$a0,4*$BNSZ	# bits
	$SRL	$at,$a1,4*$BNSZ	# bits
	or	$t3,$at
	mflo	($t0,$a2,$QT)
	mfhi	($t1,$a2,$QT)
.L_bn_div_words_inner_loop1:
	sltu	$t2,$t3,$t0
	seq	$t8,$HH,$t1
	sltu	$at,$HH,$t1
	and	$t2,$t8
	sltu	$v0,$t0,$a2
	or	$at,$t2
	.set	noreorder
	beqz	$at,.L_bn_div_words_inner_loop1_done
	$SUBU	$t1,$v0
	$SUBU	$t0,$a2
	b	.L_bn_div_words_inner_loop1
	$SUBU	$QT,1
	.set	reorder
.L_bn_div_words_inner_loop1_done:

	$SLL	$a1,4*$BNSZ	# bits
	$SUBU	$a0,$t3,$t0
	$SLL	$v0,$QT,4*$BNSZ	# bits

	li	$QT,-1
	$SRL	$HH,$a0,4*$BNSZ	# bits
	$SRL	$QT,4*$BNSZ	# q=0xffffffff
	beq	$DH,$HH,.L_bn_div_words_skip_div2
	$DIVU	($a0,$DH)
	mfqt	($QT,$a0,$DH)
.L_bn_div_words_skip_div2:
	$MULTU	($a2,$QT)
	$SLL	$t3,$a0,4*$BNSZ	# bits
	$SRL	$at,$a1,4*$BNSZ	# bits
	or	$t3,$at
	mflo	($t0,$a2,$QT)
	mfhi	($t1,$a2,$QT)
.L_bn_div_words_inner_loop2:
	sltu	$t2,$t3,$t0
	seq	$t8,$HH,$t1
	sltu	$at,$HH,$t1
	and	$t2,$t8
	sltu	$v1,$t0,$a2
	or	$at,$t2
	.set	noreorder
	beqz	$at,.L_bn_div_words_inner_loop2_done
	$SUBU	$t1,$v1
	$SUBU	$t0,$a2
	b	.L_bn_div_words_inner_loop2
	$SUBU	$QT,1
	.set	reorder
.L_bn_div_words_inner_loop2_done:

	$SUBU	$a0,$t3,$t0
	or	$v0,$QT
	$SRL	$v1,$a0,$t9	# $v1 contains remainder if anybody wants it
	$SRL	$a2,$t9		# restore $a2

	.set	noreorder
	move	$a1,$v1
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	move	$a0,$v0
.end	bn_div_words_internal
___
undef $HH; undef $QT; undef $DH;

($a_0,$a_1,$a_2,$a_3)=($t0,$t1,$t2,$t3);
($b_0,$b_1,$b_2,$b_3)=($ta0,$ta1,$ta2,$ta3);

($a_4,$a_5,$a_6,$a_7)=($s0,$s2,$s4,$a1); # once we load a[7], no use for $a1
($b_4,$b_5,$b_6,$b_7)=($s1,$s3,$s5,$a2); # once we load b[7], no use for $a2

($t_1,$t_2,$c_1,$c_2,$c_3)=($t8,$t9,$v0,$v1,$a3);

$code.=<<___;

.align	5
.globl	bn_mul_comba8
.ent	bn_mul_comba8
bn_mul_comba8:
	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,12*$SZREG,$ra
	.mask	0x803ff008,-$SZREG
	$PTR_SUB $sp,12*$SZREG
	$REG_S	$ra,11*$SZREG($sp)
	$REG_S	$s5,10*$SZREG($sp)
	$REG_S	$s4,9*$SZREG($sp)
	$REG_S	$s3,8*$SZREG($sp)
	$REG_S	$s2,7*$SZREG($sp)
	$REG_S	$s1,6*$SZREG($sp)
	$REG_S	$s0,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___ if ($flavour !~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x003f0000,-$SZREG
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$s5,5*$SZREG($sp)
	$REG_S	$s4,4*$SZREG($sp)
	$REG_S	$s3,3*$SZREG($sp)
	$REG_S	$s2,2*$SZREG($sp)
	$REG_S	$s1,1*$SZREG($sp)
	$REG_S	$s0,0*$SZREG($sp)
___
$code.=<<___;

	.set	reorder
	$LD	$a_0,0($a1)	# If compiled with -mips3 option on
				# R5000 box assembler barks on this
				# 1ine with "should not have mult/div
				# as last instruction in bb (R10K
				# bug)" warning. If anybody out there
				# has a clue about how to circumvent
				# this do send me a note.
				#		<appro\@fy.chalmers.se>

	$LD	$b_0,0($a2)
	$LD	$a_1,$BNSZ($a1)
	$LD	$a_2,2*$BNSZ($a1)
	$MULTU	($a_0,$b_0)		# mul_add_c(a[0],b[0],c1,c2,c3);
	$LD	$a_3,3*$BNSZ($a1)
	$LD	$b_1,$BNSZ($a2)
	$LD	$b_2,2*$BNSZ($a2)
	$LD	$b_3,3*$BNSZ($a2)
	mflo	($c_1,$a_0,$b_0)
	mfhi	($c_2,$a_0,$b_0)

	$LD	$a_4,4*$BNSZ($a1)
	$LD	$a_5,5*$BNSZ($a1)
	$MULTU	($a_0,$b_1)		# mul_add_c(a[0],b[1],c2,c3,c1);
	$LD	$a_6,6*$BNSZ($a1)
	$LD	$a_7,7*$BNSZ($a1)
	$LD	$b_4,4*$BNSZ($a2)
	$LD	$b_5,5*$BNSZ($a2)
	mflo	($t_1,$a_0,$b_1)
	mfhi	($t_2,$a_0,$b_1)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_1,$b_0)		# mul_add_c(a[1],b[0],c2,c3,c1);
	$ADDU	$c_3,$t_2,$at
	$LD	$b_6,6*$BNSZ($a2)
	$LD	$b_7,7*$BNSZ($a2)
	$ST	$c_1,0($a0)	# r[0]=c1;
	mflo	($t_1,$a_1,$b_0)
	mfhi	($t_2,$a_1,$b_0)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_2,$b_0)		# mul_add_c(a[2],b[0],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$c_1,$c_3,$t_2
	$ST	$c_2,$BNSZ($a0)	# r[1]=c2;

	mflo	($t_1,$a_2,$b_0)
	mfhi	($t_2,$a_2,$b_0)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_1,$b_1)		# mul_add_c(a[1],b[1],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	mflo	($t_1,$a_1,$b_1)
	mfhi	($t_2,$a_1,$b_1)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_0,$b_2)		# mul_add_c(a[0],b[2],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$c_2,$c_1,$t_2
	mflo	($t_1,$a_0,$b_2)
	mfhi	($t_2,$a_0,$b_2)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_0,$b_3)		# mul_add_c(a[0],b[3],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,2*$BNSZ($a0)	# r[2]=c3;

	mflo	($t_1,$a_0,$b_3)
	mfhi	($t_2,$a_0,$b_3)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_1,$b_2)		# mul_add_c(a[1],b[2],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$c_3,$c_2,$t_2
	mflo	($t_1,$a_1,$b_2)
	mfhi	($t_2,$a_1,$b_2)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_2,$b_1)		# mul_add_c(a[2],b[1],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_2,$b_1)
	mfhi	($t_2,$a_2,$b_1)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_3,$b_0)		# mul_add_c(a[3],b[0],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_3,$b_0)
	mfhi	($t_2,$a_3,$b_0)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	 $MULTU	($a_4,$b_0)		# mul_add_c(a[4],b[0],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	$ST	$c_1,3*$BNSZ($a0)	# r[3]=c1;

	mflo	($t_1,$a_4,$b_0)
	mfhi	($t_2,$a_4,$b_0)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_3,$b_1)		# mul_add_c(a[3],b[1],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$c_1,$c_3,$t_2
	mflo	($t_1,$a_3,$b_1)
	mfhi	($t_2,$a_3,$b_1)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_2,$b_2)		# mul_add_c(a[2],b[2],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_2,$b_2)
	mfhi	($t_2,$a_2,$b_2)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_1,$b_3)		# mul_add_c(a[1],b[3],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_1,$b_3)
	mfhi	($t_2,$a_1,$b_3)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_0,$b_4)		# mul_add_c(a[0],b[4],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_0,$b_4)
	mfhi	($t_2,$a_0,$b_4)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_0,$b_5)		# mul_add_c(a[0],b[5],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,4*$BNSZ($a0)	# r[4]=c2;

	mflo	($t_1,$a_0,$b_5)
	mfhi	($t_2,$a_0,$b_5)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_1,$b_4)		# mul_add_c(a[1],b[4],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$c_2,$c_1,$t_2
	mflo	($t_1,$a_1,$b_4)
	mfhi	($t_2,$a_1,$b_4)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_2,$b_3)		# mul_add_c(a[2],b[3],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_2,$b_3)
	mfhi	($t_2,$a_2,$b_3)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_3,$b_2)		# mul_add_c(a[3],b[2],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_3,$b_2)
	mfhi	($t_2,$a_3,$b_2)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_4,$b_1)		# mul_add_c(a[4],b[1],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_4,$b_1)
	mfhi	($t_2,$a_4,$b_1)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_5,$b_0)		# mul_add_c(a[5],b[0],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_5,$b_0)
	mfhi	($t_2,$a_5,$b_0)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_6,$b_0)		# mul_add_c(a[6],b[0],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,5*$BNSZ($a0)	# r[5]=c3;

	mflo	($t_1,$a_6,$b_0)
	mfhi	($t_2,$a_6,$b_0)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_5,$b_1)		# mul_add_c(a[5],b[1],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$c_3,$c_2,$t_2
	mflo	($t_1,$a_5,$b_1)
	mfhi	($t_2,$a_5,$b_1)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_4,$b_2)		# mul_add_c(a[4],b[2],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_4,$b_2)
	mfhi	($t_2,$a_4,$b_2)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_3,$b_3)		# mul_add_c(a[3],b[3],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_3,$b_3)
	mfhi	($t_2,$a_3,$b_3)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_2,$b_4)		# mul_add_c(a[2],b[4],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_2,$b_4)
	mfhi	($t_2,$a_2,$b_4)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_1,$b_5)		# mul_add_c(a[1],b[5],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_1,$b_5)
	mfhi	($t_2,$a_1,$b_5)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_0,$b_6)		# mul_add_c(a[0],b[6],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_0,$b_6)
	mfhi	($t_2,$a_0,$b_6)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	 $MULTU	($a_0,$b_7)		# mul_add_c(a[0],b[7],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	$ST	$c_1,6*$BNSZ($a0)	# r[6]=c1;

	mflo	($t_1,$a_0,$b_7)
	mfhi	($t_2,$a_0,$b_7)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_1,$b_6)		# mul_add_c(a[1],b[6],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$c_1,$c_3,$t_2
	mflo	($t_1,$a_1,$b_6)
	mfhi	($t_2,$a_1,$b_6)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_2,$b_5)		# mul_add_c(a[2],b[5],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_2,$b_5)
	mfhi	($t_2,$a_2,$b_5)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_3,$b_4)		# mul_add_c(a[3],b[4],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_3,$b_4)
	mfhi	($t_2,$a_3,$b_4)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_4,$b_3)		# mul_add_c(a[4],b[3],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_4,$b_3)
	mfhi	($t_2,$a_4,$b_3)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_5,$b_2)		# mul_add_c(a[5],b[2],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_5,$b_2)
	mfhi	($t_2,$a_5,$b_2)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_6,$b_1)		# mul_add_c(a[6],b[1],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_6,$b_1)
	mfhi	($t_2,$a_6,$b_1)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_7,$b_0)		# mul_add_c(a[7],b[0],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_7,$b_0)
	mfhi	($t_2,$a_7,$b_0)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_7,$b_1)		# mul_add_c(a[7],b[1],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,7*$BNSZ($a0)	# r[7]=c2;

	mflo	($t_1,$a_7,$b_1)
	mfhi	($t_2,$a_7,$b_1)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_6,$b_2)		# mul_add_c(a[6],b[2],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$c_2,$c_1,$t_2
	mflo	($t_1,$a_6,$b_2)
	mfhi	($t_2,$a_6,$b_2)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_5,$b_3)		# mul_add_c(a[5],b[3],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_5,$b_3)
	mfhi	($t_2,$a_5,$b_3)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_4,$b_4)		# mul_add_c(a[4],b[4],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_4,$b_4)
	mfhi	($t_2,$a_4,$b_4)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_3,$b_5)		# mul_add_c(a[3],b[5],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_3,$b_5)
	mfhi	($t_2,$a_3,$b_5)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_2,$b_6)		# mul_add_c(a[2],b[6],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_2,$b_6)
	mfhi	($t_2,$a_2,$b_6)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_1,$b_7)		# mul_add_c(a[1],b[7],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_1,$b_7)
	mfhi	($t_2,$a_1,$b_7)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_2,$b_7)		# mul_add_c(a[2],b[7],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,8*$BNSZ($a0)	# r[8]=c3;

	mflo	($t_1,$a_2,$b_7)
	mfhi	($t_2,$a_2,$b_7)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_3,$b_6)		# mul_add_c(a[3],b[6],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$c_3,$c_2,$t_2
	mflo	($t_1,$a_3,$b_6)
	mfhi	($t_2,$a_3,$b_6)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_4,$b_5)		# mul_add_c(a[4],b[5],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_4,$b_5)
	mfhi	($t_2,$a_4,$b_5)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_5,$b_4)		# mul_add_c(a[5],b[4],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_5,$b_4)
	mfhi	($t_2,$a_5,$b_4)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_6,$b_3)		# mul_add_c(a[6],b[3],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_6,$b_3)
	mfhi	($t_2,$a_6,$b_3)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_7,$b_2)		# mul_add_c(a[7],b[2],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_7,$b_2)
	mfhi	($t_2,$a_7,$b_2)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	 $MULTU	($a_7,$b_3)		# mul_add_c(a[7],b[3],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	$ST	$c_1,9*$BNSZ($a0)	# r[9]=c1;

	mflo	($t_1,$a_7,$b_3)
	mfhi	($t_2,$a_7,$b_3)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_6,$b_4)		# mul_add_c(a[6],b[4],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$c_1,$c_3,$t_2
	mflo	($t_1,$a_6,$b_4)
	mfhi	($t_2,$a_6,$b_4)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_5,$b_5)		# mul_add_c(a[5],b[5],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_5,$b_5)
	mfhi	($t_2,$a_5,$b_5)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_4,$b_6)		# mul_add_c(a[4],b[6],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_4,$b_6)
	mfhi	($t_2,$a_4,$b_6)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_3,$b_7)		# mul_add_c(a[3],b[7],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_3,$b_7)
	mfhi	($t_2,$a_3,$b_7)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_4,$b_7)		# mul_add_c(a[4],b[7],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,10*$BNSZ($a0)	# r[10]=c2;

	mflo	($t_1,$a_4,$b_7)
	mfhi	($t_2,$a_4,$b_7)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_5,$b_6)		# mul_add_c(a[5],b[6],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$c_2,$c_1,$t_2
	mflo	($t_1,$a_5,$b_6)
	mfhi	($t_2,$a_5,$b_6)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_6,$b_5)		# mul_add_c(a[6],b[5],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_6,$b_5)
	mfhi	($t_2,$a_6,$b_5)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_7,$b_4)		# mul_add_c(a[7],b[4],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	mflo	($t_1,$a_7,$b_4)
	mfhi	($t_2,$a_7,$b_4)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_7,$b_5)		# mul_add_c(a[7],b[5],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,11*$BNSZ($a0)	# r[11]=c3;

	mflo	($t_1,$a_7,$b_5)
	mfhi	($t_2,$a_7,$b_5)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_6,$b_6)		# mul_add_c(a[6],b[6],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$c_3,$c_2,$t_2
	mflo	($t_1,$a_6,$b_6)
	mfhi	($t_2,$a_6,$b_6)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_5,$b_7)		# mul_add_c(a[5],b[7],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_5,$b_7)
	mfhi	($t_2,$a_5,$b_7)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	 $MULTU	($a_6,$b_7)		# mul_add_c(a[6],b[7],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	$ST	$c_1,12*$BNSZ($a0)	# r[12]=c1;

	mflo	($t_1,$a_6,$b_7)
	mfhi	($t_2,$a_6,$b_7)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_7,$b_6)		# mul_add_c(a[7],b[6],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$c_1,$c_3,$t_2
	mflo	($t_1,$a_7,$b_6)
	mfhi	($t_2,$a_7,$b_6)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_7,$b_7)		# mul_add_c(a[7],b[7],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,13*$BNSZ($a0)	# r[13]=c2;

	mflo	($t_1,$a_7,$b_7)
	mfhi	($t_2,$a_7,$b_7)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	$ST	$c_3,14*$BNSZ($a0)	# r[14]=c3;
	$ST	$c_1,15*$BNSZ($a0)	# r[15]=c1;

	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$s5,10*$SZREG($sp)
	$REG_L	$s4,9*$SZREG($sp)
	$REG_L	$s3,8*$SZREG($sp)
	$REG_L	$s2,7*$SZREG($sp)
	$REG_L	$s1,6*$SZREG($sp)
	$REG_L	$s0,5*$SZREG($sp)
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	jr	$ra
	$PTR_ADD $sp,12*$SZREG
___
$code.=<<___ if ($flavour !~ /nubi/i);
	$REG_L	$s5,5*$SZREG($sp)
	$REG_L	$s4,4*$SZREG($sp)
	$REG_L	$s3,3*$SZREG($sp)
	$REG_L	$s2,2*$SZREG($sp)
	$REG_L	$s1,1*$SZREG($sp)
	$REG_L	$s0,0*$SZREG($sp)
	jr	$ra
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
.end	bn_mul_comba8

.align	5
.globl	bn_mul_comba4
.ent	bn_mul_comba4
bn_mul_comba4:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	$LD	$a_0,0($a1)
	$LD	$b_0,0($a2)
	$LD	$a_1,$BNSZ($a1)
	$LD	$a_2,2*$BNSZ($a1)
	$MULTU	($a_0,$b_0)		# mul_add_c(a[0],b[0],c1,c2,c3);
	$LD	$a_3,3*$BNSZ($a1)
	$LD	$b_1,$BNSZ($a2)
	$LD	$b_2,2*$BNSZ($a2)
	$LD	$b_3,3*$BNSZ($a2)
	mflo	($c_1,$a_0,$b_0)
	mfhi	($c_2,$a_0,$b_0)
	$ST	$c_1,0($a0)

	$MULTU	($a_0,$b_1)		# mul_add_c(a[0],b[1],c2,c3,c1);
	mflo	($t_1,$a_0,$b_1)
	mfhi	($t_2,$a_0,$b_1)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_1,$b_0)		# mul_add_c(a[1],b[0],c2,c3,c1);
	$ADDU	$c_3,$t_2,$at
	mflo	($t_1,$a_1,$b_0)
	mfhi	($t_2,$a_1,$b_0)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_2,$b_0)		# mul_add_c(a[2],b[0],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$c_1,$c_3,$t_2
	$ST	$c_2,$BNSZ($a0)

	mflo	($t_1,$a_2,$b_0)
	mfhi	($t_2,$a_2,$b_0)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_1,$b_1)		# mul_add_c(a[1],b[1],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	mflo	($t_1,$a_1,$b_1)
	mfhi	($t_2,$a_1,$b_1)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_0,$b_2)		# mul_add_c(a[0],b[2],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$c_2,$c_1,$t_2
	mflo	($t_1,$a_0,$b_2)
	mfhi	($t_2,$a_0,$b_2)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_0,$b_3)		# mul_add_c(a[0],b[3],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,2*$BNSZ($a0)

	mflo	($t_1,$a_0,$b_3)
	mfhi	($t_2,$a_0,$b_3)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_1,$b_2)		# mul_add_c(a[1],b[2],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$c_3,$c_2,$t_2
	mflo	($t_1,$a_1,$b_2)
	mfhi	($t_2,$a_1,$b_2)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_2,$b_1)		# mul_add_c(a[2],b[1],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_2,$b_1)
	mfhi	($t_2,$a_2,$b_1)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$MULTU	($a_3,$b_0)		# mul_add_c(a[3],b[0],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	mflo	($t_1,$a_3,$b_0)
	mfhi	($t_2,$a_3,$b_0)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	 $MULTU	($a_3,$b_1)		# mul_add_c(a[3],b[1],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	$ST	$c_1,3*$BNSZ($a0)

	mflo	($t_1,$a_3,$b_1)
	mfhi	($t_2,$a_3,$b_1)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_2,$b_2)		# mul_add_c(a[2],b[2],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$c_1,$c_3,$t_2
	mflo	($t_1,$a_2,$b_2)
	mfhi	($t_2,$a_2,$b_2)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$MULTU	($a_1,$b_3)		# mul_add_c(a[1],b[3],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_1,$b_3)
	mfhi	($t_2,$a_1,$b_3)
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_2,$b_3)		# mul_add_c(a[2],b[3],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,4*$BNSZ($a0)

	mflo	($t_1,$a_2,$b_3)
	mfhi	($t_2,$a_2,$b_3)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$MULTU	($a_3,$b_2)		# mul_add_c(a[3],b[2],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$c_2,$c_1,$t_2
	mflo	($t_1,$a_3,$b_2)
	mfhi	($t_2,$a_3,$b_2)
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_3,$b_3)		# mul_add_c(a[3],b[3],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,5*$BNSZ($a0)

	mflo	($t_1,$a_3,$b_3)
	mfhi	($t_2,$a_3,$b_3)
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	$ST	$c_1,6*$BNSZ($a0)
	$ST	$c_2,7*$BNSZ($a0)

	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	nop
.end	bn_mul_comba4
___

($a_4,$a_5,$a_6,$a_7)=($b_0,$b_1,$b_2,$b_3);

sub add_c2 () {
my ($hi,$lo,$c0,$c1,$c2,
    $warm,      # !$warm denotes first call with specific sequence of
                # $c_[XYZ] when there is no Z-carry to accumulate yet;
    $an,$bn     # these two are arguments for multiplication which
                # result is used in *next* step [which is why it's
                # commented as "forward multiplication" below];
    )=@_;
$code.=<<___;
	$ADDU	$c0,$lo
	sltu	$at,$c0,$lo
	 $MULTU	($an,$bn)		# forward multiplication
	$ADDU	$c0,$lo
	$ADDU	$at,$hi
	sltu	$lo,$c0,$lo
	$ADDU	$c1,$at
	$ADDU	$hi,$lo
___
$code.=<<___	if (!$warm);
	sltu	$c2,$c1,$at
	$ADDU	$c1,$hi
___
$code.=<<___	if ($warm);
	sltu	$at,$c1,$at
	$ADDU	$c1,$hi
	$ADDU	$c2,$at
___
$code.=<<___;
	sltu	$hi,$c1,$hi
	$ADDU	$c2,$hi
	mflo	($lo,$an,$bn)
	mfhi	($hi,$an,$bn)
___
}

$code.=<<___;

.align	5
.globl	bn_sqr_comba8
.ent	bn_sqr_comba8
bn_sqr_comba8:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	$LD	$a_0,0($a1)
	$LD	$a_1,$BNSZ($a1)
	$LD	$a_2,2*$BNSZ($a1)
	$LD	$a_3,3*$BNSZ($a1)

	$MULTU	($a_0,$a_0)		# mul_add_c(a[0],b[0],c1,c2,c3);
	$LD	$a_4,4*$BNSZ($a1)
	$LD	$a_5,5*$BNSZ($a1)
	$LD	$a_6,6*$BNSZ($a1)
	$LD	$a_7,7*$BNSZ($a1)
	mflo	($c_1,$a_0,$a_0)
	mfhi	($c_2,$a_0,$a_0)
	$ST	$c_1,0($a0)

	$MULTU	($a_0,$a_1)		# mul_add_c2(a[0],b[1],c2,c3,c1);
	mflo	($t_1,$a_0,$a_1)
	mfhi	($t_2,$a_0,$a_1)
	slt	$c_1,$t_2,$zero
	$SLL	$t_2,1
	 $MULTU	($a_2,$a_0)		# mul_add_c2(a[2],b[0],c3,c1,c2);
	slt	$a2,$t_1,$zero
	$ADDU	$t_2,$a2
	$SLL	$t_1,1
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$ADDU	$c_3,$t_2,$at
	$ST	$c_2,$BNSZ($a0)
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_2,$a_0)
	mfhi	($t_2,$a_2,$a_0)
___
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,0,
		$a_1,$a_1);		# mul_add_c(a[1],b[1],c3,c1,c2);
$code.=<<___;
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_0,$a_3)		# mul_add_c2(a[0],b[3],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,2*$BNSZ($a0)
	mflo	($t_1,$a_0,$a_3)
	mfhi	($t_2,$a_0,$a_3)
___
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,0,
		$a_1,$a_2);		# mul_add_c2(a[1],b[2],c1,c2,c3);
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,1,
		$a_4,$a_0);		# mul_add_c2(a[4],b[0],c2,c3,c1);
$code.=<<___;
	$ST	$c_1,3*$BNSZ($a0)
___
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,0,
		$a_3,$a_1);		# mul_add_c2(a[3],b[1],c2,c3,c1);
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,1,
		$a_2,$a_2);		# mul_add_c(a[2],b[2],c2,c3,c1);
$code.=<<___;
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_0,$a_5)		# mul_add_c2(a[0],b[5],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,4*$BNSZ($a0)
	mflo	($t_1,$a_0,$a_5)
	mfhi	($t_2,$a_0,$a_5)
___
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,0,
		$a_1,$a_4);		# mul_add_c2(a[1],b[4],c3,c1,c2);
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,1,
		$a_2,$a_3);		# mul_add_c2(a[2],b[3],c3,c1,c2);
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,1,
		$a_6,$a_0);		# mul_add_c2(a[6],b[0],c1,c2,c3);
$code.=<<___;
	$ST	$c_3,5*$BNSZ($a0)
___
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,0,
		$a_5,$a_1);		# mul_add_c2(a[5],b[1],c1,c2,c3);
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,1,
		$a_4,$a_2);		# mul_add_c2(a[4],b[2],c1,c2,c3);
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,1,
		$a_3,$a_3);		# mul_add_c(a[3],b[3],c1,c2,c3);
$code.=<<___;
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	 $MULTU	($a_0,$a_7)		# mul_add_c2(a[0],b[7],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	$ST	$c_1,6*$BNSZ($a0)
	mflo	($t_1,$a_0,$a_7)
	mfhi	($t_2,$a_0,$a_7)
___
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,0,
		$a_1,$a_6);		# mul_add_c2(a[1],b[6],c2,c3,c1);
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,1,
		$a_2,$a_5);		# mul_add_c2(a[2],b[5],c2,c3,c1);
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,1,
		$a_3,$a_4);		# mul_add_c2(a[3],b[4],c2,c3,c1);
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,1,
		$a_7,$a_1);		# mul_add_c2(a[7],b[1],c3,c1,c2);
$code.=<<___;
	$ST	$c_2,7*$BNSZ($a0)
___
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,0,
		$a_6,$a_2);		# mul_add_c2(a[6],b[2],c3,c1,c2);
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,1,
		$a_5,$a_3);		# mul_add_c2(a[5],b[3],c3,c1,c2);
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,1,
		$a_4,$a_4);		# mul_add_c(a[4],b[4],c3,c1,c2);
$code.=<<___;
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_2,$a_7)		# mul_add_c2(a[2],b[7],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,8*$BNSZ($a0)
	mflo	($t_1,$a_2,$a_7)
	mfhi	($t_2,$a_2,$a_7)
___
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,0,
		$a_3,$a_6);		# mul_add_c2(a[3],b[6],c1,c2,c3);
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,1,
		$a_4,$a_5);		# mul_add_c2(a[4],b[5],c1,c2,c3);
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,1,
		$a_7,$a_3);		# mul_add_c2(a[7],b[3],c2,c3,c1);
$code.=<<___;
	$ST	$c_1,9*$BNSZ($a0)
___
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,0,
		$a_6,$a_4);		# mul_add_c2(a[6],b[4],c2,c3,c1);
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,1,
		$a_5,$a_5);		# mul_add_c(a[5],b[5],c2,c3,c1);
$code.=<<___;
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_4,$a_7)		# mul_add_c2(a[4],b[7],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,10*$BNSZ($a0)
	mflo	($t_1,$a_4,$a_7)
	mfhi	($t_2,$a_4,$a_7)
___
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,0,
		$a_5,$a_6);		# mul_add_c2(a[5],b[6],c3,c1,c2);
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,1,
		$a_7,$a_5);		# mul_add_c2(a[7],b[5],c1,c2,c3);
$code.=<<___;
	$ST	$c_3,11*$BNSZ($a0)
___
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,0,
		$a_6,$a_6);		# mul_add_c(a[6],b[6],c1,c2,c3);
$code.=<<___;
	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	 $MULTU	($a_6,$a_7)		# mul_add_c2(a[6],b[7],c2,c3,c1);
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	sltu	$at,$c_2,$t_2
	$ADDU	$c_3,$at
	$ST	$c_1,12*$BNSZ($a0)
	mflo	($t_1,$a_6,$a_7)
	mfhi	($t_2,$a_6,$a_7)
___
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,0,
		$a_7,$a_7);		# mul_add_c(a[7],b[7],c3,c1,c2);
$code.=<<___;
	$ST	$c_2,13*$BNSZ($a0)

	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	$ST	$c_3,14*$BNSZ($a0)
	$ST	$c_1,15*$BNSZ($a0)

	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	nop
.end	bn_sqr_comba8

.align	5
.globl	bn_sqr_comba4
.ent	bn_sqr_comba4
bn_sqr_comba4:
___
$code.=<<___ if ($flavour =~ /nubi/i);
	.frame	$sp,6*$SZREG,$ra
	.mask	0x8000f008,-$SZREG
	.set	noreorder
	$PTR_SUB $sp,6*$SZREG
	$REG_S	$ra,5*$SZREG($sp)
	$REG_S	$t3,4*$SZREG($sp)
	$REG_S	$t2,3*$SZREG($sp)
	$REG_S	$t1,2*$SZREG($sp)
	$REG_S	$t0,1*$SZREG($sp)
	$REG_S	$gp,0*$SZREG($sp)
___
$code.=<<___;
	.set	reorder
	$LD	$a_0,0($a1)
	$LD	$a_1,$BNSZ($a1)
	$MULTU	($a_0,$a_0)		# mul_add_c(a[0],b[0],c1,c2,c3);
	$LD	$a_2,2*$BNSZ($a1)
	$LD	$a_3,3*$BNSZ($a1)
	mflo	($c_1,$a_0,$a_0)
	mfhi	($c_2,$a_0,$a_0)
	$ST	$c_1,0($a0)

	$MULTU	($a_0,$a_1)		# mul_add_c2(a[0],b[1],c2,c3,c1);
	mflo	($t_1,$a_0,$a_1)
	mfhi	($t_2,$a_0,$a_1)
	slt	$c_1,$t_2,$zero
	$SLL	$t_2,1
	 $MULTU	($a_2,$a_0)		# mul_add_c2(a[2],b[0],c3,c1,c2);
	slt	$a2,$t_1,$zero
	$ADDU	$t_2,$a2
	$SLL	$t_1,1
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	$ADDU	$c_3,$t_2,$at
	$ST	$c_2,$BNSZ($a0)
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	mflo	($t_1,$a_2,$a_0)
	mfhi	($t_2,$a_2,$a_0)
___
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,0,
		$a_1,$a_1);		# mul_add_c(a[1],b[1],c3,c1,c2);
$code.=<<___;
	$ADDU	$c_3,$t_1
	sltu	$at,$c_3,$t_1
	 $MULTU	($a_0,$a_3)		# mul_add_c2(a[0],b[3],c1,c2,c3);
	$ADDU	$t_2,$at
	$ADDU	$c_1,$t_2
	sltu	$at,$c_1,$t_2
	$ADDU	$c_2,$at
	$ST	$c_3,2*$BNSZ($a0)
	mflo	($t_1,$a_0,$a_3)
	mfhi	($t_2,$a_0,$a_3)
___
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,0,
		$a_1,$a_2);		# mul_add_c2(a2[1],b[2],c1,c2,c3);
	&add_c2($t_2,$t_1,$c_1,$c_2,$c_3,1,
		$a_3,$a_1);		# mul_add_c2(a[3],b[1],c2,c3,c1);
$code.=<<___;
	$ST	$c_1,3*$BNSZ($a0)
___
	&add_c2($t_2,$t_1,$c_2,$c_3,$c_1,0,
		$a_2,$a_2);		# mul_add_c(a[2],b[2],c2,c3,c1);
$code.=<<___;
	$ADDU	$c_2,$t_1
	sltu	$at,$c_2,$t_1
	 $MULTU	($a_2,$a_3)		# mul_add_c2(a[2],b[3],c3,c1,c2);
	$ADDU	$t_2,$at
	$ADDU	$c_3,$t_2
	sltu	$at,$c_3,$t_2
	$ADDU	$c_1,$at
	$ST	$c_2,4*$BNSZ($a0)
	mflo	($t_1,$a_2,$a_3)
	mfhi	($t_2,$a_2,$a_3)
___
	&add_c2($t_2,$t_1,$c_3,$c_1,$c_2,0,
		$a_3,$a_3);		# mul_add_c(a[3],b[3],c1,c2,c3);
$code.=<<___;
	$ST	$c_3,5*$BNSZ($a0)

	$ADDU	$c_1,$t_1
	sltu	$at,$c_1,$t_1
	$ADDU	$t_2,$at
	$ADDU	$c_2,$t_2
	$ST	$c_1,6*$BNSZ($a0)
	$ST	$c_2,7*$BNSZ($a0)

	.set	noreorder
___
$code.=<<___ if ($flavour =~ /nubi/i);
	$REG_L	$t3,4*$SZREG($sp)
	$REG_L	$t2,3*$SZREG($sp)
	$REG_L	$t1,2*$SZREG($sp)
	$REG_L	$t0,1*$SZREG($sp)
	$REG_L	$gp,0*$SZREG($sp)
	$PTR_ADD $sp,6*$SZREG
___
$code.=<<___;
	jr	$ra
	nop
.end	bn_sqr_comba4
___
print $code;
close STDOUT or die "error closing STDOUT: $!";