#! /usr/bin/env perl
# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

# Ascetic x86_64 AT&T to MASM/NASM assembler translator by <appro>.
#
# Why AT&T to MASM and not vice versa? Several reasons. Because AT&T
# format is way easier to parse. Because it's simpler to "gear" from
# Unix ABI to Windows one [see cross-reference "card" at the end of
# file]. Because Linux targets were available first...
#
# In addition the script also "distills" code suitable for GNU
# assembler, so that it can be compiled with more rigid assemblers,
# such as Solaris /usr/ccs/bin/as.
#
# This translator is not designed to convert *arbitrary* assembler
# code from AT&T format to MASM one. It's designed to convert just
# enough to provide for dual-ABI OpenSSL modules development...
# There *are* limitations and you might have to modify your assembler
# code or this script to achieve the desired result...
#
# Currently recognized limitations:
#
# - can't use multiple ops per line;
#
# Dual-ABI styling rules.
#
# 1. Adhere to Unix register and stack layout [see cross-reference
#    ABI "card" at the end for explanation].
# 2. Forget about "red zone," stick to more traditional blended
#    stack frame allocation. If volatile storage is actually required
#    that is. If not, just leave the stack as is.
# 3. Functions tagged with ".type name,@function" get crafted with
#    unified Win64 prologue and epilogue automatically. If you want
#    to take care of ABI differences yourself, tag functions as
#    ".type name,@abi-omnipotent" instead.
# 4. To optimize the Win64 prologue you can specify number of input
#    arguments as ".type name,@function,N." Keep in mind that if N is
#    larger than 6, then you *have to* write "abi-omnipotent" code,
#    because >6 cases can't be addressed with unified prologue.
# 5. Name local labels as .L*, do *not* use dynamic labels such as 1:
#    (sorry about latter).
# 6. Don't use [or hand-code with .byte] "rep ret." "ret" mnemonic is
#    required to identify the spots, where to inject Win64 epilogue!
#    But on the pros, it's then prefixed with rep automatically:-)
# 7. Stick to explicit ip-relative addressing. If you have to use
#    GOTPCREL addressing, stick to mov symbol@GOTPCREL(%rip),%r??.
#    Both are recognized and translated to proper Win64 addressing
#    modes.
#
# 8. In order to provide for structured exception handling unified
#    Win64 prologue copies %rsp value to %rax. For further details
#    see SEH paragraph at the end.
# 9. .init segment is allowed to contain calls to functions only.
# a. If function accepts more than 4 arguments *and* >4th argument
#    is declared as non 64-bit value, do clear its upper part.

use strict;

my $flavour = shift;
my $output  = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

open STDOUT,">$output" || die "can't open $output: $!"
	if (defined($output));

my $gas=1;	$gas=0 if ($output =~ /\.asm$/);
my $elf=1;	$elf=0 if (!$gas);
my $win64=0;
my $prefix="";
my $decor=".L";

my $masmref=8 + 50727*2**-32;	# 8.00.50727 shipped with VS2005
my $masm=0;
my $PTR=" PTR";

my $nasmref=2.03;
my $nasm=0;

if    ($flavour eq "mingw64")	{ $gas=1; $elf=0; $win64=1;
				  $prefix=`echo __USER_LABEL_PREFIX__ | $ENV{CC} -E -P -`;
				  $prefix =~ s|\R$||; # Better chomp
				}
elsif ($flavour eq "macosx")	{ $gas=1; $elf=0; $prefix="_"; $decor="L\$"; }
elsif ($flavour eq "masm")	{ $gas=0; $elf=0; $masm=$masmref; $win64=1; $decor="\$L\$"; }
elsif ($flavour eq "nasm")	{ $gas=0; $elf=0; $nasm=$nasmref; $win64=1; $decor="\$L\$"; $PTR=""; }
elsif (!$gas)
{   if ($ENV{ASM} =~ m/nasm/ && `nasm -v` =~ m/version ([0-9]+)\.([0-9]+)/i)
    {	$nasm = $1 + $2*0.01; $PTR="";  }
    elsif (`ml64 2>&1` =~ m/Version ([0-9]+)\.([0-9]+)(\.([0-9]+))?/)
    {	$masm = $1 + $2*2**-16 + $4*2**-32;   }
    die "no assembler found on %PATH%" if (!($nasm || $masm));
    $win64=1;
    $elf=0;
    $decor="\$L\$";
}

my $current_segment;
my $current_function;
my %globals;

{ package opcode;	# pick up opcodes
    sub re {
	my	($class, $line) = @_;
	my	$self = {};
	my	$ret;

	if ($$line =~ /^([a-z][a-z0-9]*)/i) {
	    bless $self,$class;
	    $self->{op} = $1;
	    $ret = $self;
	    $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;

	    undef $self->{sz};
	    if ($self->{op} =~ /^(movz)x?([bw]).*/) {	# movz is pain...
		$self->{op} = $1;
		$self->{sz} = $2;
	    } elsif ($self->{op} =~ /call|jmp/) {
		$self->{sz} = "";
	    } elsif ($self->{op} =~ /^p/ && $' !~ /^(ush|op|insrw)/) { # SSEn
		$self->{sz} = "";
	    } elsif ($self->{op} =~ /^[vk]/) { # VEX or k* such as kmov
		$self->{sz} = "";
	    } elsif ($self->{op} =~ /mov[dq]/ && $$line =~ /%xmm/) {
		$self->{sz} = "";
	    } elsif ($self->{op} =~ /([a-z]{3,})([qlwb])$/) {
		$self->{op} = $1;
		$self->{sz} = $2;
	    }
	}
	$ret;
    }
    sub size {
	my ($self, $sz) = @_;
	$self->{sz} = $sz if (defined($sz) && !defined($self->{sz}));
	$self->{sz};
    }
    sub out {
	my $self = shift;
	if ($gas) {
	    if ($self->{op} eq "movz") {	# movz is pain...
		sprintf "%s%s%s",$self->{op},$self->{sz},shift;
	    } elsif ($self->{op} =~ /^set/) {
		"$self->{op}";
	    } elsif ($self->{op} eq "ret") {
		my $epilogue = "";
		if ($win64 && $current_function->{abi} eq "svr4") {
		    $epilogue = "movq	8(%rsp),%rdi\n\t" .
				"movq	16(%rsp),%rsi\n\t";
		}
	    	$epilogue . ".byte	0xf3,0xc3";
	    } elsif ($self->{op} eq "call" && !$elf && $current_segment eq ".init") {
		".p2align\t3\n\t.quad";
	    } else {
		"$self->{op}$self->{sz}";
	    }
	} else {
	    $self->{op} =~ s/^movz/movzx/;
	    if ($self->{op} eq "ret") {
		$self->{op} = "";
		if ($win64 && $current_function->{abi} eq "svr4") {
		    $self->{op} = "mov	rdi,QWORD$PTR\[8+rsp\]\t;WIN64 epilogue\n\t".
				  "mov	rsi,QWORD$PTR\[16+rsp\]\n\t";
	    	}
		$self->{op} .= "DB\t0F3h,0C3h\t\t;repret";
	    } elsif ($self->{op} =~ /^(pop|push)f/) {
		$self->{op} .= $self->{sz};
	    } elsif ($self->{op} eq "call" && $current_segment eq ".CRT\$XCU") {
		$self->{op} = "\tDQ";
	    }
	    $self->{op};
	}
    }
    sub mnemonic {
	my ($self, $op) = @_;
	$self->{op}=$op if (defined($op));
	$self->{op};
    }
}
{ package const;	# pick up constants, which start with $
    sub re {
	my	($class, $line) = @_;
	my	$self = {};
	my	$ret;

	if ($$line =~ /^\$([^,]+)/) {
	    bless $self, $class;
	    $self->{value} = $1;
	    $ret = $self;
	    $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
	}
	$ret;
    }
    sub out {
    	my $self = shift;

	$self->{value} =~ s/\b(0b[0-1]+)/oct($1)/eig;
	if ($gas) {
	    # Solaris /usr/ccs/bin/as can't handle multiplications
	    # in $self->{value}
	    my $value = $self->{value};
	    no warnings;    # oct might complain about overflow, ignore here...
	    $value =~ s/(?<![\w\$\.])(0x?[0-9a-f]+)/oct($1)/egi;
	    if ($value =~ s/([0-9]+\s*[\*\/\%]\s*[0-9]+)/eval($1)/eg) {
		$self->{value} = $value;
	    }
	    sprintf "\$%s",$self->{value};
	} else {
	    my $value = $self->{value};
	    $value =~ s/0x([0-9a-f]+)/0$1h/ig if ($masm);
	    sprintf "%s",$value;
	}
    }
}
{ package ea;		# pick up effective addresses: expr(%reg,%reg,scale)

    my %szmap = (	b=>"BYTE$PTR",    w=>"WORD$PTR",
			l=>"DWORD$PTR",   d=>"DWORD$PTR",
			q=>"QWORD$PTR",   o=>"OWORD$PTR",
			x=>"XMMWORD$PTR", y=>"YMMWORD$PTR",
			z=>"ZMMWORD$PTR" ) if (!$gas);

    sub re {
	my	($class, $line, $opcode) = @_;
	my	$self = {};
	my	$ret;

	# optional * ----vvv--- appears in indirect jmp/call
	if ($$line =~ /^(\*?)([^\(,]*)\(([%\w,]+)\)((?:{[^}]+})*)/) {
	    bless $self, $class;
	    $self->{asterisk} = $1;
	    $self->{label} = $2;
	    ($self->{base},$self->{index},$self->{scale})=split(/,/,$3);
	    $self->{scale} = 1 if (!defined($self->{scale}));
	    $self->{opmask} = $4;
	    $ret = $self;
	    $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;

	    if ($win64 && $self->{label} =~ s/\@GOTPCREL//) {
		die if ($opcode->mnemonic() ne "mov");
		$opcode->mnemonic("lea");
	    }
	    $self->{base}  =~ s/^%//;
	    $self->{index} =~ s/^%// if (defined($self->{index}));
	    $self->{opcode} = $opcode;
	}
	$ret;
    }
    sub size {}
    sub out {
	my ($self, $sz) = @_;

	$self->{label} =~ s/([_a-z][_a-z0-9]*)/$globals{$1} or $1/gei;
	$self->{label} =~ s/\.L/$decor/g;

	# Silently convert all EAs to 64-bit. This is required for
	# elder GNU assembler and results in more compact code,
	# *but* most importantly AES module depends on this feature!
	$self->{index} =~ s/^[er](.?[0-9xpi])[d]?$/r\1/;
	$self->{base}  =~ s/^[er](.?[0-9xpi])[d]?$/r\1/;

	# Solaris /usr/ccs/bin/as can't handle multiplications
	# in $self->{label}...
	use integer;
	$self->{label} =~ s/(?<![\w\$\.])(0x?[0-9a-f]+)/oct($1)/egi;
	$self->{label} =~ s/\b([0-9]+\s*[\*\/\%]\s*[0-9]+)\b/eval($1)/eg;

	# Some assemblers insist on signed presentation of 32-bit
	# offsets, but sign extension is a tricky business in perl...
	if ((1<<31)<<1) {
	    $self->{label} =~ s/\b([0-9]+)\b/$1<<32>>32/eg;
	} else {
	    $self->{label} =~ s/\b([0-9]+)\b/$1>>0/eg;
	}

	# if base register is %rbp or %r13, see if it's possible to
	# flip base and index registers [for better performance]
	if (!$self->{label} && $self->{index} && $self->{scale}==1 &&
	    $self->{base} =~ /(rbp|r13)/) {
		$self->{base} = $self->{index}; $self->{index} = $1;
	}

	if ($gas) {
	    $self->{label} =~ s/^___imp_/__imp__/   if ($flavour eq "mingw64");

	    if (defined($self->{index})) {
		sprintf "%s%s(%s,%%%s,%d)%s",
					$self->{asterisk},$self->{label},
					$self->{base}?"%$self->{base}":"",
					$self->{index},$self->{scale},
					$self->{opmask};
	    } else {
		sprintf "%s%s(%%%s)%s",	$self->{asterisk},$self->{label},
					$self->{base},$self->{opmask};
	    }
	} else {
	    $self->{label} =~ s/\./\$/g;
	    $self->{label} =~ s/(?<![\w\$\.])0x([0-9a-f]+)/0$1h/ig;
	    $self->{label} = "($self->{label})" if ($self->{label} =~ /[\*\+\-\/]/);

	    my $mnemonic = $self->{opcode}->mnemonic();
	    ($self->{asterisk})				&& ($sz="q") ||
	    ($mnemonic =~ /^v?mov([qd])$/)		&& ($sz=$1)  ||
	    ($mnemonic =~ /^v?pinsr([qdwb])$/)		&& ($sz=$1)  ||
	    ($mnemonic =~ /^vpbroadcast([qdwb])$/)	&& ($sz=$1)  ||
	    ($mnemonic =~ /^v(?!perm)[a-z]+[fi]128$/)	&& ($sz="x");

	    $self->{opmask}  =~ s/%(k[0-7])/$1/;

	    if (defined($self->{index})) {
		sprintf "%s[%s%s*%d%s]%s",$szmap{$sz},
					$self->{label}?"$self->{label}+":"",
					$self->{index},$self->{scale},
					$self->{base}?"+$self->{base}":"",
					$self->{opmask};
	    } elsif ($self->{base} eq "rip") {
		sprintf "%s[%s]",$szmap{$sz},$self->{label};
	    } else {
		sprintf "%s[%s%s]%s",	$szmap{$sz},
					$self->{label}?"$self->{label}+":"",
					$self->{base},$self->{opmask};
	    }
	}
    }
}
{ package register;	# pick up registers, which start with %.
    sub re {
	my	($class, $line, $opcode) = @_;
	my	$self = {};
	my	$ret;

	# optional * ----vvv--- appears in indirect jmp/call
	if ($$line =~ /^(\*?)%(\w+)((?:{[^}]+})*)/) {
	    bless $self,$class;
	    $self->{asterisk} = $1;
	    $self->{value} = $2;
	    $self->{opmask} = $3;
	    $opcode->size($self->size());
	    $ret = $self;
	    $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
	}
	$ret;
    }
    sub size {
	my	$self = shift;
	my	$ret;

	if    ($self->{value} =~ /^r[\d]+b$/i)	{ $ret="b"; }
	elsif ($self->{value} =~ /^r[\d]+w$/i)	{ $ret="w"; }
	elsif ($self->{value} =~ /^r[\d]+d$/i)	{ $ret="l"; }
	elsif ($self->{value} =~ /^r[\w]+$/i)	{ $ret="q"; }
	elsif ($self->{value} =~ /^[a-d][hl]$/i){ $ret="b"; }
	elsif ($self->{value} =~ /^[\w]{2}l$/i)	{ $ret="b"; }
	elsif ($self->{value} =~ /^[\w]{2}$/i)	{ $ret="w"; }
	elsif ($self->{value} =~ /^e[a-z]{2}$/i){ $ret="l"; }

	$ret;
    }
    sub out {
    	my $self = shift;
	if ($gas)	{ sprintf "%s%%%s%s",	$self->{asterisk},
						$self->{value},
						$self->{opmask}; }
	else		{ $self->{opmask} =~ s/%(k[0-7])/$1/;
			  $self->{value}.$self->{opmask}; }
    }
}
{ package label;	# pick up labels, which end with :
    sub re {
	my	($class, $line) = @_;
	my	$self = {};
	my	$ret;

	if ($$line =~ /(^[\.\w]+)\:/) {
	    bless $self,$class;
	    $self->{value} = $1;
	    $ret = $self;
	    $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;

	    $self->{value} =~ s/^\.L/$decor/;
	}
	$ret;
    }
    sub out {
	my $self = shift;

	if ($gas) {
	    my $func = ($globals{$self->{value}} or $self->{value}) . ":";
	    if ($win64	&& $current_function->{name} eq $self->{value}
			&& $current_function->{abi} eq "svr4") {
		$func .= "\n";
		$func .= "	movq	%rdi,8(%rsp)\n";
		$func .= "	movq	%rsi,16(%rsp)\n";
		$func .= "	movq	%rsp,%rax\n";
		$func .= "${decor}SEH_begin_$current_function->{name}:\n";
		my $narg = $current_function->{narg};
		$narg=6 if (!defined($narg));
		$func .= "	movq	%rcx,%rdi\n" if ($narg>0);
		$func .= "	movq	%rdx,%rsi\n" if ($narg>1);
		$func .= "	movq	%r8,%rdx\n"  if ($narg>2);
		$func .= "	movq	%r9,%rcx\n"  if ($narg>3);
		$func .= "	movq	40(%rsp),%r8\n" if ($narg>4);
		$func .= "	movq	48(%rsp),%r9\n" if ($narg>5);
	    }
	    $func;
	} elsif ($self->{value} ne "$current_function->{name}") {
	    # Make all labels in masm global.
	    $self->{value} .= ":" if ($masm);
	    $self->{value} . ":";
	} elsif ($win64 && $current_function->{abi} eq "svr4") {
	    my $func =	"$current_function->{name}" .
			($nasm ? ":" : "\tPROC $current_function->{scope}") .
			"\n";
	    $func .= "	mov	QWORD$PTR\[8+rsp\],rdi\t;WIN64 prologue\n";
	    $func .= "	mov	QWORD$PTR\[16+rsp\],rsi\n";
	    $func .= "	mov	rax,rsp\n";
	    $func .= "${decor}SEH_begin_$current_function->{name}:";
	    $func .= ":" if ($masm);
	    $func .= "\n";
	    my $narg = $current_function->{narg};
	    $narg=6 if (!defined($narg));
	    $func .= "	mov	rdi,rcx\n" if ($narg>0);
	    $func .= "	mov	rsi,rdx\n" if ($narg>1);
	    $func .= "	mov	rdx,r8\n"  if ($narg>2);
	    $func .= "	mov	rcx,r9\n"  if ($narg>3);
	    $func .= "	mov	r8,QWORD$PTR\[40+rsp\]\n" if ($narg>4);
	    $func .= "	mov	r9,QWORD$PTR\[48+rsp\]\n" if ($narg>5);
	    $func .= "\n";
	} else {
	   "$current_function->{name}".
			($nasm ? ":" : "\tPROC $current_function->{scope}");
	}
    }
}
{ package expr;		# pick up expressions
    sub re {
	my	($class, $line, $opcode) = @_;
	my	$self = {};
	my	$ret;

	if ($$line =~ /(^[^,]+)/) {
	    bless $self,$class;
	    $self->{value} = $1;
	    $ret = $self;
	    $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;

	    $self->{value} =~ s/\@PLT// if (!$elf);
	    $self->{value} =~ s/([_a-z][_a-z0-9]*)/$globals{$1} or $1/gei;
	    $self->{value} =~ s/\.L/$decor/g;
	    $self->{opcode} = $opcode;
	}
	$ret;
    }
    sub out {
	my $self = shift;
	if ($nasm && $self->{opcode}->mnemonic()=~m/^j(?![re]cxz)/) {
	    "NEAR ".$self->{value};
	} else {
	    $self->{value};
	}
    }
}
{ package cfi_directive;
    # CFI directives annotate instructions that are significant for
    # stack unwinding procedure compliant with DWARF specification,
    # see http://dwarfstd.org/. Besides naturally expected for this
    # script platform-specific filtering function, this module adds
    # three auxiliary synthetic directives not recognized by [GNU]
    # assembler:
    #
    # - .cfi_push to annotate push instructions in prologue, which
    #   translates to .cfi_adjust_cfa_offset (if needed) and
    #   .cfi_offset;
    # - .cfi_pop to annotate pop instructions in epilogue, which
    #   translates to .cfi_adjust_cfa_offset (if needed) and
    #   .cfi_restore;
    # - [and most notably] .cfi_cfa_expression which encodes
    #   DW_CFA_def_cfa_expression and passes it to .cfi_escape as
    #   byte vector;
    #
    # CFA expressions were introduced in DWARF specification version
    # 3 and describe how to deduce CFA, Canonical Frame Address. This
    # becomes handy if your stack frame is variable and you can't
    # spare register for [previous] frame pointer. Suggested directive
    # syntax is made-up mix of DWARF operator suffixes [subset of]
    # and references to registers with optional bias. Following example
    # describes offloaded *original* stack pointer at specific offset
    # from *current* stack pointer:
    #
    #   .cfi_cfa_expression     %rsp+40,deref,+8
    #
    # Final +8 has everything to do with the fact that CFA is defined
    # as reference to top of caller's stack, and on x86_64 call to
    # subroutine pushes 8-byte return address. In other words original
    # stack pointer upon entry to a subroutine is 8 bytes off from CFA.

    # Below constants are taken from "DWARF Expressions" section of the
    # DWARF specification, section is numbered 7.7 in versions 3 and 4.
    my %DW_OP_simple = (	# no-arg operators, mapped directly
	deref	=> 0x06,	dup	=> 0x12,
	drop	=> 0x13,	over	=> 0x14,
	pick	=> 0x15,	swap	=> 0x16,
	rot	=> 0x17,	xderef	=> 0x18,

	abs	=> 0x19,	and	=> 0x1a,
	div	=> 0x1b,	minus	=> 0x1c,
	mod	=> 0x1d,	mul	=> 0x1e,
	neg	=> 0x1f,	not	=> 0x20,
	or	=> 0x21,	plus	=> 0x22,
	shl	=> 0x24,	shr	=> 0x25,
	shra	=> 0x26,	xor	=> 0x27,
	);

    my %DW_OP_complex = (	# used in specific subroutines
	constu		=> 0x10,	# uleb128
	consts		=> 0x11,	# sleb128
	plus_uconst	=> 0x23,	# uleb128
	lit0 		=> 0x30,	# add 0-31 to opcode
	reg0		=> 0x50,	# add 0-31 to opcode
	breg0		=> 0x70,	# add 0-31 to opcole, sleb128
	regx		=> 0x90,	# uleb28
	fbreg		=> 0x91,	# sleb128
	bregx		=> 0x92,	# uleb128, sleb128
	piece		=> 0x93,	# uleb128
	);

    # Following constants are defined in x86_64 ABI supplement, for
    # example available at https://www.uclibc.org/docs/psABI-x86_64.pdf,
    # see section 3.7 "Stack Unwind Algorithm".
    my %DW_reg_idx = (
	"%rax"=>0,  "%rdx"=>1,  "%rcx"=>2,  "%rbx"=>3,
	"%rsi"=>4,  "%rdi"=>5,  "%rbp"=>6,  "%rsp"=>7,
	"%r8" =>8,  "%r9" =>9,  "%r10"=>10, "%r11"=>11,
	"%r12"=>12, "%r13"=>13, "%r14"=>14, "%r15"=>15
	);

    my ($cfa_reg, $cfa_rsp);
    my @cfa_stack;

    # [us]leb128 format is variable-length integer representation base
    # 2^128, with most significant bit of each byte being 0 denoting
    # *last* most significant digit. See "Variable Length Data" in the
    # DWARF specification, numbered 7.6 at least in versions 3 and 4.
    sub sleb128 {
	use integer;	# get right shift extend sign

	my $val = shift;
	my $sign = ($val < 0) ? -1 : 0;
	my @ret = ();

	while(1) {
	    push @ret, $val&0x7f;

	    # see if remaining bits are same and equal to most
	    # significant bit of the current digit, if so, it's
	    # last digit...
	    last if (($val>>6) == $sign);

	    @ret[-1] |= 0x80;
	    $val >>= 7;
	}

	return @ret;
    }
    sub uleb128 {
	my $val = shift;
	my @ret = ();

	while(1) {
	    push @ret, $val&0x7f;

	    # see if it's last significant digit...
	    last if (($val >>= 7) == 0);

	    @ret[-1] |= 0x80;
	}

	return @ret;
    }
    sub const {
	my $val = shift;

	if ($val >= 0 && $val < 32) {
            return ($DW_OP_complex{lit0}+$val);
	}
	return ($DW_OP_complex{consts}, sleb128($val));
    }
    sub reg {
	my $val = shift;

	return if ($val !~ m/^(%r\w+)(?:([\+\-])((?:0x)?[0-9a-f]+))?/);

	my $reg = $DW_reg_idx{$1};
	my $off = eval ("0 $2 $3");

	return (($DW_OP_complex{breg0} + $reg), sleb128($off));
	# Yes, we use DW_OP_bregX+0 to push register value and not
	# DW_OP_regX, because latter would require even DW_OP_piece,
	# which would be a waste under the circumstances. If you have
	# to use DWP_OP_reg, use "regx:N"...
    }
    sub cfa_expression {
	my $line = shift;
	my @ret;

	foreach my $token (split(/,\s*/,$line)) {
	    if ($token =~ /^%r/) {
		push @ret,reg($token);
	    } elsif ($token =~ /((?:0x)?[0-9a-f]+)\((%r\w+)\)/) {
		push @ret,reg("$2+$1");
	    } elsif ($token =~ /(\w+):(\-?(?:0x)?[0-9a-f]+)(U?)/i) {
		my $i = 1*eval($2);
		push @ret,$DW_OP_complex{$1}, ($3 ? uleb128($i) : sleb128($i));
	    } elsif (my $i = 1*eval($token) or $token eq "0") {
		if ($token =~ /^\+/) {
		    push @ret,$DW_OP_complex{plus_uconst},uleb128($i);
		} else {
		    push @ret,const($i);
		}
	    } else {
		push @ret,$DW_OP_simple{$token};
	    }
	}

	# Finally we return DW_CFA_def_cfa_expression, 15, followed by
	# length of the expression and of course the expression itself.
	return (15,scalar(@ret),@ret);
    }
    sub re {
	my	($class, $line) = @_;
	my	$self = {};
	my	$ret;

	if ($$line =~ s/^\s*\.cfi_(\w+)\s*//) {
	    bless $self,$class;
	    $ret = $self;
	    undef $self->{value};
	    my $dir = $1;

	    SWITCH: for ($dir) {
	    # What is $cfa_rsp? Effectively it's difference between %rsp
	    # value and current CFA, Canonical Frame Address, which is
	    # why it starts with -8. Recall that CFA is top of caller's
	    # stack...
	    /startproc/	&& do {	($cfa_reg, $cfa_rsp) = ("%rsp", -8); last; };
	    /endproc/	&& do {	($cfa_reg, $cfa_rsp) = ("%rsp",  0);
				# .cfi_remember_state directives that are not
				# matched with .cfi_restore_state are
				# unnecessary.
				die "unpaired .cfi_remember_state" if (@cfa_stack);
				last;
			      };
	    /def_cfa_register/
			&& do {	$cfa_reg = $$line; last; };
	    /def_cfa_offset/
			&& do {	$cfa_rsp = -1*eval($$line) if ($cfa_reg eq "%rsp");
				last;
			      };
	    /adjust_cfa_offset/
			&& do {	$cfa_rsp -= 1*eval($$line) if ($cfa_reg eq "%rsp");
				last;
			      };
	    /def_cfa/	&& do {	if ($$line =~ /(%r\w+)\s*,\s*(.+)/) {
				    $cfa_reg = $1;
				    $cfa_rsp = -1*eval($2) if ($cfa_reg eq "%rsp");
				}
				last;
			      };
	    /push/	&& do {	$dir = undef;
				$cfa_rsp -= 8;
				if ($cfa_reg eq "%rsp") {
				    $self->{value} = ".cfi_adjust_cfa_offset\t8\n";
				}
				$self->{value} .= ".cfi_offset\t$$line,$cfa_rsp";
				last;
			      };
	    /pop/	&& do {	$dir = undef;
				$cfa_rsp += 8;
				if ($cfa_reg eq "%rsp") {
				    $self->{value} = ".cfi_adjust_cfa_offset\t-8\n";
				}
				$self->{value} .= ".cfi_restore\t$$line";
				last;
			      };
	    /cfa_expression/
			&& do {	$dir = undef;
				$self->{value} = ".cfi_escape\t" .
					join(",", map(sprintf("0x%02x", $_),
						      cfa_expression($$line)));
				last;
			      };
	    /remember_state/
			&& do {	push @cfa_stack, [$cfa_reg, $cfa_rsp];
				last;
			      };
	    /restore_state/
			&& do {	($cfa_reg, $cfa_rsp) = @{pop @cfa_stack};
				last;
			      };
	    }

	    $self->{value} = ".cfi_$dir\t$$line" if ($dir);

	    $$line = "";
	}

	return $ret;
    }
    sub out {
	my $self = shift;
	return ($elf ? $self->{value} : undef);
    }
}
{ package directive;	# pick up directives, which start with .
    sub re {
	my	($class, $line) = @_;
	my	$self = {};
	my	$ret;
	my	$dir;

	# chain-call to cfi_directive
	$ret = cfi_directive->re($line) and return $ret;

	if ($$line =~ /^\s*(\.\w+)/) {
	    bless $self,$class;
	    $dir = $1;
	    $ret = $self;
	    undef $self->{value};
	    $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;

	    SWITCH: for ($dir) {
		/\.global|\.globl|\.extern/
			    && do { $globals{$$line} = $prefix . $$line;
				    $$line = $globals{$$line} if ($prefix);
				    last;
				  };
		/\.type/    && do { my ($sym,$type,$narg) = split(',',$$line);
				    if ($type eq "\@function") {
					undef $current_function;
					$current_function->{name} = $sym;
					$current_function->{abi}  = "svr4";
					$current_function->{narg} = $narg;
					$current_function->{scope} = defined($globals{$sym})?"PUBLIC":"PRIVATE";
				    } elsif ($type eq "\@abi-omnipotent") {
					undef $current_function;
					$current_function->{name} = $sym;
					$current_function->{scope} = defined($globals{$sym})?"PUBLIC":"PRIVATE";
				    }
				    $$line =~ s/\@abi\-omnipotent/\@function/;
				    $$line =~ s/\@function.*/\@function/;
				    last;
				  };
		/\.asciz/   && do { if ($$line =~ /^"(.*)"$/) {
					$dir  = ".byte";
					$$line = join(",",unpack("C*",$1),0);
				    }
				    last;
				  };
		/\.rva|\.long|\.quad/
			    && do { $$line =~ s/([_a-z][_a-z0-9]*)/$globals{$1} or $1/gei;
				    $$line =~ s/\.L/$decor/g;
				    last;
				  };
	    }

	    if ($gas) {
		$self->{value} = $dir . "\t" . $$line;

		if ($dir =~ /\.extern/) {
		    $self->{value} = ""; # swallow extern
		} elsif (!$elf && $dir =~ /\.type/) {
		    $self->{value} = "";
		    $self->{value} = ".def\t" . ($globals{$1} or $1) . ";\t" .
				(defined($globals{$1})?".scl 2;":".scl 3;") .
				"\t.type 32;\t.endef"
				if ($win64 && $$line =~ /([^,]+),\@function/);
		} elsif (!$elf && $dir =~ /\.size/) {
		    $self->{value} = "";
		    if (defined($current_function)) {
			$self->{value} .= "${decor}SEH_end_$current_function->{name}:"
				if ($win64 && $current_function->{abi} eq "svr4");
			undef $current_function;
		    }
		} elsif (!$elf && $dir =~ /\.align/) {
		    $self->{value} = ".p2align\t" . (log($$line)/log(2));
		} elsif ($dir eq ".section") {
		    $current_segment=$$line;
		    if (!$elf && $current_segment eq ".init") {
			if	($flavour eq "macosx")	{ $self->{value} = ".mod_init_func"; }
			elsif	($flavour eq "mingw64")	{ $self->{value} = ".section\t.ctors"; }
		    }
		} elsif ($dir =~ /\.(text|data)/) {
		    $current_segment=".$1";
		} elsif ($dir =~ /\.hidden/) {
		    if    ($flavour eq "macosx")  { $self->{value} = ".private_extern\t$prefix$$line"; }
		    elsif ($flavour eq "mingw64") { $self->{value} = ""; }
		} elsif ($dir =~ /\.comm/) {
		    $self->{value} = "$dir\t$prefix$$line";
		    $self->{value} =~ s|,([0-9]+),([0-9]+)$|",$1,".log($2)/log(2)|e if ($flavour eq "macosx");
		}
		$$line = "";
		return $self;
	    }

	    # non-gas case or nasm/masm
	    SWITCH: for ($dir) {
		/\.text/    && do { my $v=undef;
				    if ($nasm) {
					$v="section	.text code align=64\n";
				    } else {
					$v="$current_segment\tENDS\n" if ($current_segment);
					$current_segment = ".text\$";
					$v.="$current_segment\tSEGMENT ";
					$v.=$masm>=$masmref ? "ALIGN(256)" : "PAGE";
					$v.=" 'CODE'";
				    }
				    $self->{value} = $v;
				    last;
				  };
		/\.data/    && do { my $v=undef;
				    if ($nasm) {
					$v="section	.data data align=8\n";
				    } else {
					$v="$current_segment\tENDS\n" if ($current_segment);
					$current_segment = "_DATA";
					$v.="$current_segment\tSEGMENT";
				    }
				    $self->{value} = $v;
				    last;
				  };
		/\.section/ && do { my $v=undef;
				    $$line =~ s/([^,]*).*/$1/;
				    $$line = ".CRT\$XCU" if ($$line eq ".init");
				    if ($nasm) {
					$v="section	$$line";
					if ($$line=~/\.([px])data/) {
					    $v.=" rdata align=";
					    $v.=$1 eq "p"? 4 : 8;
					} elsif ($$line=~/\.CRT\$/i) {
					    $v.=" rdata align=8";
					}
				    } else {
					$v="$current_segment\tENDS\n" if ($current_segment);
					$v.="$$line\tSEGMENT";
					if ($$line=~/\.([px])data/) {
					    $v.=" READONLY";
					    $v.=" ALIGN(".($1 eq "p" ? 4 : 8).")" if ($masm>=$masmref);
					} elsif ($$line=~/\.CRT\$/i) {
					    $v.=" READONLY ";
					    $v.=$masm>=$masmref ? "ALIGN(8)" : "DWORD";
					}
				    }
				    $current_segment = $$line;
				    $self->{value} = $v;
				    last;
				  };
		/\.extern/  && do { $self->{value}  = "EXTERN\t".$$line;
				    $self->{value} .= ":NEAR" if ($masm);
				    last;
				  };
		/\.globl|.global/
			    && do { $self->{value}  = $masm?"PUBLIC":"global";
				    $self->{value} .= "\t".$$line;
				    last;
				  };
		/\.size/    && do { if (defined($current_function)) {
					undef $self->{value};
					if ($current_function->{abi} eq "svr4") {
					    $self->{value}="${decor}SEH_end_$current_function->{name}:";
					    $self->{value}.=":\n" if($masm);
					}
					$self->{value}.="$current_function->{name}\tENDP" if($masm && $current_function->{name});
					undef $current_function;
				    }
				    last;
				  };
		/\.align/   && do { my $max = ($masm && $masm>=$masmref) ? 256 : 4096;
				    $self->{value} = "ALIGN\t".($$line>$max?$max:$$line);
				    last;
				  };
		/\.(value|long|rva|quad)/
			    && do { my $sz  = substr($1,0,1);
				    my @arr = split(/,\s*/,$$line);
				    my $last = pop(@arr);
				    my $conv = sub  {	my $var=shift;
							$var=~s/^(0b[0-1]+)/oct($1)/eig;
							$var=~s/^0x([0-9a-f]+)/0$1h/ig if ($masm);
							if ($sz eq "D" && ($current_segment=~/.[px]data/ || $dir eq ".rva"))
							{ $var=~s/^([_a-z\$\@][_a-z0-9\$\@]*)/$nasm?"$1 wrt ..imagebase":"imagerel $1"/egi; }
							$var;
						    };

				    $sz =~ tr/bvlrq/BWDDQ/;
				    $self->{value} = "\tD$sz\t";
				    for (@arr) { $self->{value} .= &$conv($_).","; }
				    $self->{value} .= &$conv($last);
				    last;
				  };
		/\.byte/    && do { my @str=split(/,\s*/,$$line);
				    map(s/(0b[0-1]+)/oct($1)/eig,@str);
				    map(s/0x([0-9a-f]+)/0$1h/ig,@str) if ($masm);
				    while ($#str>15) {
					$self->{value}.="DB\t"
						.join(",",@str[0..15])."\n";
					foreach (0..15) { shift @str; }
				    }
				    $self->{value}.="DB\t"
						.join(",",@str) if (@str);
				    last;
				  };
		/\.comm/    && do { my @str=split(/,\s*/,$$line);
				    my $v=undef;
				    if ($nasm) {
					$v.="common	$prefix@str[0] @str[1]";
				    } else {
					$v="$current_segment\tENDS\n" if ($current_segment);
					$current_segment = "_DATA";
					$v.="$current_segment\tSEGMENT\n";
					$v.="COMM	@str[0]:DWORD:".@str[1]/4;
				    }
				    $self->{value} = $v;
				    last;
				  };
	    }
	    $$line = "";
	}

	$ret;
    }
    sub out {
	my $self = shift;
	$self->{value};
    }
}

# Upon initial x86_64 introduction SSE>2 extensions were not introduced
# yet. In order not to be bothered by tracing exact assembler versions,
# but at the same time to provide a bare security minimum of AES-NI, we
# hard-code some instructions. Extensions past AES-NI on the other hand
# are traced by examining assembler version in individual perlasm
# modules...

my %regrm = (	"%eax"=>0, "%ecx"=>1, "%edx"=>2, "%ebx"=>3,
		"%esp"=>4, "%ebp"=>5, "%esi"=>6, "%edi"=>7	);

sub rex {
 my $opcode=shift;
 my ($dst,$src,$rex)=@_;

   $rex|=0x04 if($dst>=8);
   $rex|=0x01 if($src>=8);
   push @$opcode,($rex|0x40) if ($rex);
}

my $movq = sub {	# elderly gas can't handle inter-register movq
  my $arg = shift;
  my @opcode=(0x66);
    if ($arg =~ /%xmm([0-9]+),\s*%r(\w+)/) {
	my ($src,$dst)=($1,$2);
	if ($dst !~ /[0-9]+/)	{ $dst = $regrm{"%e$dst"}; }
	rex(\@opcode,$src,$dst,0x8);
	push @opcode,0x0f,0x7e;
	push @opcode,0xc0|(($src&7)<<3)|($dst&7);	# ModR/M
	@opcode;
    } elsif ($arg =~ /%r(\w+),\s*%xmm([0-9]+)/) {
	my ($src,$dst)=($2,$1);
	if ($dst !~ /[0-9]+/)	{ $dst = $regrm{"%e$dst"}; }
	rex(\@opcode,$src,$dst,0x8);
	push @opcode,0x0f,0x6e;
	push @opcode,0xc0|(($src&7)<<3)|($dst&7);	# ModR/M
	@opcode;
    } else {
	();
    }
};

my $pextrd = sub {
    if (shift =~ /\$([0-9]+),\s*%xmm([0-9]+),\s*(%\w+)/) {
      my @opcode=(0x66);
	my $imm=$1;
	my $src=$2;
	my $dst=$3;
	if ($dst =~ /%r([0-9]+)d/)	{ $dst = $1; }
	elsif ($dst =~ /%e/)		{ $dst = $regrm{$dst}; }
	rex(\@opcode,$src,$dst);
	push @opcode,0x0f,0x3a,0x16;
	push @opcode,0xc0|(($src&7)<<3)|($dst&7);	# ModR/M
	push @opcode,$imm;
	@opcode;
    } else {
	();
    }
};

my $pinsrd = sub {
    if (shift =~ /\$([0-9]+),\s*(%\w+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x66);
	my $imm=$1;
	my $src=$2;
	my $dst=$3;
	if ($src =~ /%r([0-9]+)/)	{ $src = $1; }
	elsif ($src =~ /%e/)		{ $src = $regrm{$src}; }
	rex(\@opcode,$dst,$src);
	push @opcode,0x0f,0x3a,0x22;
	push @opcode,0xc0|(($dst&7)<<3)|($src&7);	# ModR/M
	push @opcode,$imm;
	@opcode;
    } else {
	();
    }
};

my $pshufb = sub {
    if (shift =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x66);
	rex(\@opcode,$2,$1);
	push @opcode,0x0f,0x38,0x00;
	push @opcode,0xc0|($1&7)|(($2&7)<<3);		# ModR/M
	@opcode;
    } else {
	();
    }
};

my $palignr = sub {
    if (shift =~ /\$([0-9]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x66);
	rex(\@opcode,$3,$2);
	push @opcode,0x0f,0x3a,0x0f;
	push @opcode,0xc0|($2&7)|(($3&7)<<3);		# ModR/M
	push @opcode,$1;
	@opcode;
    } else {
	();
    }
};

my $pclmulqdq = sub {
    if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x66);
	rex(\@opcode,$3,$2);
	push @opcode,0x0f,0x3a,0x44;
	push @opcode,0xc0|($2&7)|(($3&7)<<3);		# ModR/M
	my $c=$1;
	push @opcode,$c=~/^0/?oct($c):$c;
	@opcode;
    } else {
	();
    }
};

my $rdrand = sub {
    if (shift =~ /%[er](\w+)/) {
      my @opcode=();
      my $dst=$1;
	if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; }
	rex(\@opcode,0,$dst,8);
	push @opcode,0x0f,0xc7,0xf0|($dst&7);
	@opcode;
    } else {
	();
    }
};

my $rdseed = sub {
    if (shift =~ /%[er](\w+)/) {
      my @opcode=();
      my $dst=$1;
	if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; }
	rex(\@opcode,0,$dst,8);
	push @opcode,0x0f,0xc7,0xf8|($dst&7);
	@opcode;
    } else {
	();
    }
};

# Not all AVX-capable assemblers recognize AMD XOP extension. Since we
# are using only two instructions hand-code them in order to be excused
# from chasing assembler versions...

sub rxb {
 my $opcode=shift;
 my ($dst,$src1,$src2,$rxb)=@_;

   $rxb|=0x7<<5;
   $rxb&=~(0x04<<5) if($dst>=8);
   $rxb&=~(0x01<<5) if($src1>=8);
   $rxb&=~(0x02<<5) if($src2>=8);
   push @$opcode,$rxb;
}

my $vprotd = sub {
    if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x8f);
	rxb(\@opcode,$3,$2,-1,0x08);
	push @opcode,0x78,0xc2;
	push @opcode,0xc0|($2&7)|(($3&7)<<3);		# ModR/M
	my $c=$1;
	push @opcode,$c=~/^0/?oct($c):$c;
	@opcode;
    } else {
	();
    }
};

my $vprotq = sub {
    if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
      my @opcode=(0x8f);
	rxb(\@opcode,$3,$2,-1,0x08);
	push @opcode,0x78,0xc3;
	push @opcode,0xc0|($2&7)|(($3&7)<<3);		# ModR/M
	my $c=$1;
	push @opcode,$c=~/^0/?oct($c):$c;
	@opcode;
    } else {
	();
    }
};

# Intel Control-flow Enforcement Technology extension. All functions and
# indirect branch targets will have to start with this instruction...

my $endbranch = sub {
    (0xf3,0x0f,0x1e,0xfa);
};

########################################################################

if ($nasm) {
    print <<___;
default	rel
%define XMMWORD
%define YMMWORD
%define ZMMWORD
___
} elsif ($masm) {
    print <<___;
OPTION	DOTNAME
___
}
while(defined(my $line=<>)) {

    $line =~ s|\R$||;           # Better chomp

    $line =~ s|[#!].*$||;	# get rid of asm-style comments...
    $line =~ s|/\*.*\*/||;	# ... and C-style comments...
    $line =~ s|^\s+||;		# ... and skip white spaces in beginning
    $line =~ s|\s+$||;		# ... and at the end

    if (my $label=label->re(\$line))	{ print $label->out(); }

    if (my $directive=directive->re(\$line)) {
	printf "%s",$directive->out();
    } elsif (my $opcode=opcode->re(\$line)) {
	my $asm = eval("\$".$opcode->mnemonic());

	if ((ref($asm) eq 'CODE') && scalar(my @bytes=&$asm($line))) {
	    print $gas?".byte\t":"DB\t",join(',',@bytes),"\n";
	    next;
	}

	my @args;
	ARGUMENT: while (1) {
	    my $arg;

	    ($arg=register->re(\$line, $opcode))||
	    ($arg=const->re(\$line))		||
	    ($arg=ea->re(\$line, $opcode))	||
	    ($arg=expr->re(\$line, $opcode))	||
	    last ARGUMENT;

	    push @args,$arg;

	    last ARGUMENT if ($line !~ /^,/);

	    $line =~ s/^,\s*//;
	} # ARGUMENT:

	if ($#args>=0) {
	    my $insn;
	    my $sz=$opcode->size();

	    if ($gas) {
		$insn = $opcode->out($#args>=1?$args[$#args]->size():$sz);
		@args = map($_->out($sz),@args);
		printf "\t%s\t%s",$insn,join(",",@args);
	    } else {
		$insn = $opcode->out();
		foreach (@args) {
		    my $arg = $_->out();
		    # $insn.=$sz compensates for movq, pinsrw, ...
		    if ($arg =~ /^xmm[0-9]+$/) { $insn.=$sz; $sz="x" if(!$sz); last; }
		    if ($arg =~ /^ymm[0-9]+$/) { $insn.=$sz; $sz="y" if(!$sz); last; }
		    if ($arg =~ /^zmm[0-9]+$/) { $insn.=$sz; $sz="z" if(!$sz); last; }
		    if ($arg =~ /^mm[0-9]+$/)  { $insn.=$sz; $sz="q" if(!$sz); last; }
		}
		@args = reverse(@args);
		undef $sz if ($nasm && $opcode->mnemonic() eq "lea");
		printf "\t%s\t%s",$insn,join(",",map($_->out($sz),@args));
	    }
	} else {
	    printf "\t%s",$opcode->out();
	}
    }

    print $line,"\n";
}

print "\n$current_segment\tENDS\n"	if ($current_segment && $masm);
print "END\n"				if ($masm);

close STDOUT or die "error closing STDOUT: $!";

#################################################
# Cross-reference x86_64 ABI "card"
#
# 		Unix		Win64
# %rax		*		*
# %rbx		-		-
# %rcx		#4		#1
# %rdx		#3		#2
# %rsi		#2		-
# %rdi		#1		-
# %rbp		-		-
# %rsp		-		-
# %r8		#5		#3
# %r9		#6		#4
# %r10		*		*
# %r11		*		*
# %r12		-		-
# %r13		-		-
# %r14		-		-
# %r15		-		-
#
# (*)	volatile register
# (-)	preserved by callee
# (#)	Nth argument, volatile
#
# In Unix terms top of stack is argument transfer area for arguments
# which could not be accommodated in registers. Or in other words 7th
# [integer] argument resides at 8(%rsp) upon function entry point.
# 128 bytes above %rsp constitute a "red zone" which is not touched
# by signal handlers and can be used as temporal storage without
# allocating a frame.
#
# In Win64 terms N*8 bytes on top of stack is argument transfer area,
# which belongs to/can be overwritten by callee. N is the number of
# arguments passed to callee, *but* not less than 4! This means that
# upon function entry point 5th argument resides at 40(%rsp), as well
# as that 32 bytes from 8(%rsp) can always be used as temporal
# storage [without allocating a frame]. One can actually argue that
# one can assume a "red zone" above stack pointer under Win64 as well.
# Point is that at apparently no occasion Windows kernel would alter
# the area above user stack pointer in true asynchronous manner...
#
# All the above means that if assembler programmer adheres to Unix
# register and stack layout, but disregards the "red zone" existence,
# it's possible to use following prologue and epilogue to "gear" from
# Unix to Win64 ABI in leaf functions with not more than 6 arguments.
#
# omnipotent_function:
# ifdef WIN64
#	movq	%rdi,8(%rsp)
#	movq	%rsi,16(%rsp)
#	movq	%rcx,%rdi	; if 1st argument is actually present
#	movq	%rdx,%rsi	; if 2nd argument is actually ...
#	movq	%r8,%rdx	; if 3rd argument is ...
#	movq	%r9,%rcx	; if 4th argument ...
#	movq	40(%rsp),%r8	; if 5th ...
#	movq	48(%rsp),%r9	; if 6th ...
# endif
#	...
# ifdef WIN64
#	movq	8(%rsp),%rdi
#	movq	16(%rsp),%rsi
# endif
#	ret
#
#################################################
# Win64 SEH, Structured Exception Handling.
#
# Unlike on Unix systems(*) lack of Win64 stack unwinding information
# has undesired side-effect at run-time: if an exception is raised in
# assembler subroutine such as those in question (basically we're
# referring to segmentation violations caused by malformed input
# parameters), the application is briskly terminated without invoking
# any exception handlers, most notably without generating memory dump
# or any user notification whatsoever. This poses a problem. It's
# possible to address it by registering custom language-specific
# handler that would restore processor context to the state at
# subroutine entry point and return "exception is not handled, keep
# unwinding" code. Writing such handler can be a challenge... But it's
# doable, though requires certain coding convention. Consider following
# snippet:
#
# .type	function,@function
# function:
#	movq	%rsp,%rax	# copy rsp to volatile register
#	pushq	%r15		# save non-volatile registers
#	pushq	%rbx
#	pushq	%rbp
#	movq	%rsp,%r11
#	subq	%rdi,%r11	# prepare [variable] stack frame
#	andq	$-64,%r11
#	movq	%rax,0(%r11)	# check for exceptions
#	movq	%r11,%rsp	# allocate [variable] stack frame
#	movq	%rax,0(%rsp)	# save original rsp value
# magic_point:
#	...
#	movq	0(%rsp),%rcx	# pull original rsp value
#	movq	-24(%rcx),%rbp	# restore non-volatile registers
#	movq	-16(%rcx),%rbx
#	movq	-8(%rcx),%r15
#	movq	%rcx,%rsp	# restore original rsp
# magic_epilogue:
#	ret
# .size function,.-function
#
# The key is that up to magic_point copy of original rsp value remains
# in chosen volatile register and no non-volatile register, except for
# rsp, is modified. While past magic_point rsp remains constant till
# the very end of the function. In this case custom language-specific
# exception handler would look like this:
#
# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
# {	ULONG64 *rsp = (ULONG64 *)context->Rax;
#	ULONG64  rip = context->Rip;
#
#	if (rip >= magic_point)
#	{   rsp = (ULONG64 *)context->Rsp;
#	    if (rip < magic_epilogue)
#	    {	rsp = (ULONG64 *)rsp[0];
#		context->Rbp = rsp[-3];
#		context->Rbx = rsp[-2];
#		context->R15 = rsp[-1];
#	    }
#	}
#	context->Rsp = (ULONG64)rsp;
#	context->Rdi = rsp[1];
#	context->Rsi = rsp[2];
#
#	memcpy (disp->ContextRecord,context,sizeof(CONTEXT));
#	RtlVirtualUnwind(UNW_FLAG_NHANDLER,disp->ImageBase,
#		dips->ControlPc,disp->FunctionEntry,disp->ContextRecord,
#		&disp->HandlerData,&disp->EstablisherFrame,NULL);
#	return ExceptionContinueSearch;
# }
#
# It's appropriate to implement this handler in assembler, directly in
# function's module. In order to do that one has to know members'
# offsets in CONTEXT and DISPATCHER_CONTEXT structures and some constant
# values. Here they are:
#
#	CONTEXT.Rax				120
#	CONTEXT.Rcx				128
#	CONTEXT.Rdx				136
#	CONTEXT.Rbx				144
#	CONTEXT.Rsp				152
#	CONTEXT.Rbp				160
#	CONTEXT.Rsi				168
#	CONTEXT.Rdi				176
#	CONTEXT.R8				184
#	CONTEXT.R9				192
#	CONTEXT.R10				200
#	CONTEXT.R11				208
#	CONTEXT.R12				216
#	CONTEXT.R13				224
#	CONTEXT.R14				232
#	CONTEXT.R15				240
#	CONTEXT.Rip				248
#	CONTEXT.Xmm6				512
#	sizeof(CONTEXT)				1232
#	DISPATCHER_CONTEXT.ControlPc		0
#	DISPATCHER_CONTEXT.ImageBase		8
#	DISPATCHER_CONTEXT.FunctionEntry	16
#	DISPATCHER_CONTEXT.EstablisherFrame	24
#	DISPATCHER_CONTEXT.TargetIp		32
#	DISPATCHER_CONTEXT.ContextRecord	40
#	DISPATCHER_CONTEXT.LanguageHandler	48
#	DISPATCHER_CONTEXT.HandlerData		56
#	UNW_FLAG_NHANDLER			0
#	ExceptionContinueSearch			1
#
# In order to tie the handler to the function one has to compose
# couple of structures: one for .xdata segment and one for .pdata.
#
# UNWIND_INFO structure for .xdata segment would be
#
# function_unwind_info:
#	.byte	9,0,0,0
#	.rva	handler
#
# This structure designates exception handler for a function with
# zero-length prologue, no stack frame or frame register.
#
# To facilitate composing of .pdata structures, auto-generated "gear"
# prologue copies rsp value to rax and denotes next instruction with
# .LSEH_begin_{function_name} label. This essentially defines the SEH
# styling rule mentioned in the beginning. Position of this label is
# chosen in such manner that possible exceptions raised in the "gear"
# prologue would be accounted to caller and unwound from latter's frame.
# End of function is marked with respective .LSEH_end_{function_name}
# label. To summarize, .pdata segment would contain
#
#	.rva	.LSEH_begin_function
#	.rva	.LSEH_end_function
#	.rva	function_unwind_info
#
# Reference to function_unwind_info from .xdata segment is the anchor.
# In case you wonder why references are 32-bit .rvas and not 64-bit
# .quads. References put into these two segments are required to be
# *relative* to the base address of the current binary module, a.k.a.
# image base. No Win64 module, be it .exe or .dll, can be larger than
# 2GB and thus such relative references can be and are accommodated in
# 32 bits.
#
# Having reviewed the example function code, one can argue that "movq
# %rsp,%rax" above is redundant. It is not! Keep in mind that on Unix
# rax would contain an undefined value. If this "offends" you, use
# another register and refrain from modifying rax till magic_point is
# reached, i.e. as if it was a non-volatile register. If more registers
# are required prior [variable] frame setup is completed, note that
# nobody says that you can have only one "magic point." You can
# "liberate" non-volatile registers by denoting last stack off-load
# instruction and reflecting it in finer grade unwind logic in handler.
# After all, isn't it why it's called *language-specific* handler...
#
# SE handlers are also involved in unwinding stack when executable is
# profiled or debugged. Profiling implies additional limitations that
# are too subtle to discuss here. For now it's sufficient to say that
# in order to simplify handlers one should either a) offload original
# %rsp to stack (like discussed above); or b) if you have a register to
# spare for frame pointer, choose volatile one.
#
# (*)	Note that we're talking about run-time, not debug-time. Lack of
#	unwind information makes debugging hard on both Windows and
#	Unix. "Unlike" refers to the fact that on Unix signal handler
#	will always be invoked, core dumped and appropriate exit code
#	returned to parent (for user notification).


#! /usr/bin/env perl
# Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
#			IALU(*)/gcc-4.4		NEON
#
# ARM11xx(ARMv6)	7.78/+100%		-
# Cortex-A5		6.35/+130%		3.00
# Cortex-A8		6.25/+115%		2.36
# Cortex-A9		5.10/+95%		2.55
# Cortex-A15		3.85/+85%		1.25(**)
# Snapdragon S4		5.70/+100%		1.48(**)
#
# (*)	this is for -march=armv6, i.e. with bunch of ldrb loading data;
# (**)	these are trade-off results, they can be improved by ~8% but at
#	the cost of 15/12% regression on Cortex-A5/A7, it's even possible
#	to improve Cortex-A9 result, but then A5/A7 loose more than 20%;

$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }

if ($flavour && $flavour ne "void") {
    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
    die "can't locate arm-xlate.pl";

    open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
    open STDOUT,">$output";
}

($ctx,$inp,$len,$padbit)=map("r$_",(0..3));

$code.=<<___;
#include "arm_arch.h"

.text
#if defined(__thumb2__)
.syntax	unified
.thumb
#else
.code	32
#endif

.globl	poly1305_emit
.globl	poly1305_blocks
.globl	poly1305_init
.type	poly1305_init,%function
.align	5
poly1305_init:
.Lpoly1305_init:
	stmdb	sp!,{r4-r11}

	eor	r3,r3,r3
	cmp	$inp,#0
	str	r3,[$ctx,#0]		@ zero hash value
	str	r3,[$ctx,#4]
	str	r3,[$ctx,#8]
	str	r3,[$ctx,#12]
	str	r3,[$ctx,#16]
	str	r3,[$ctx,#36]		@ is_base2_26
	add	$ctx,$ctx,#20

#ifdef	__thumb2__
	it	eq
#endif
	moveq	r0,#0
	beq	.Lno_key

#if	__ARM_MAX_ARCH__>=7
	adr	r11,.Lpoly1305_init
	ldr	r12,.LOPENSSL_armcap
#endif
	ldrb	r4,[$inp,#0]
	mov	r10,#0x0fffffff
	ldrb	r5,[$inp,#1]
	and	r3,r10,#-4		@ 0x0ffffffc
	ldrb	r6,[$inp,#2]
	ldrb	r7,[$inp,#3]
	orr	r4,r4,r5,lsl#8
	ldrb	r5,[$inp,#4]
	orr	r4,r4,r6,lsl#16
	ldrb	r6,[$inp,#5]
	orr	r4,r4,r7,lsl#24
	ldrb	r7,[$inp,#6]
	and	r4,r4,r10

#if	__ARM_MAX_ARCH__>=7
	ldr	r12,[r11,r12]		@ OPENSSL_armcap_P
# ifdef	__APPLE__
	ldr	r12,[r12]
# endif
#endif
	ldrb	r8,[$inp,#7]
	orr	r5,r5,r6,lsl#8
	ldrb	r6,[$inp,#8]
	orr	r5,r5,r7,lsl#16
	ldrb	r7,[$inp,#9]
	orr	r5,r5,r8,lsl#24
	ldrb	r8,[$inp,#10]
	and	r5,r5,r3

#if	__ARM_MAX_ARCH__>=7
	tst	r12,#ARMV7_NEON		@ check for NEON
# ifdef	__APPLE__
	adr	r9,poly1305_blocks_neon
	adr	r11,poly1305_blocks
#  ifdef __thumb2__
	it	ne
#  endif
	movne	r11,r9
	adr	r12,poly1305_emit
	adr	r10,poly1305_emit_neon
#  ifdef __thumb2__
	it	ne
#  endif
	movne	r12,r10
# else
#  ifdef __thumb2__
	itete	eq
#  endif
	addeq	r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init)
	addne	r12,r11,#(.Lpoly1305_emit_neon-.Lpoly1305_init)
	addeq	r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init)
	addne	r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init)
# endif
# ifdef	__thumb2__
	orr	r12,r12,#1	@ thumb-ify address
	orr	r11,r11,#1
# endif
#endif
	ldrb	r9,[$inp,#11]
	orr	r6,r6,r7,lsl#8
	ldrb	r7,[$inp,#12]
	orr	r6,r6,r8,lsl#16
	ldrb	r8,[$inp,#13]
	orr	r6,r6,r9,lsl#24
	ldrb	r9,[$inp,#14]
	and	r6,r6,r3

	ldrb	r10,[$inp,#15]
	orr	r7,r7,r8,lsl#8
	str	r4,[$ctx,#0]
	orr	r7,r7,r9,lsl#16
	str	r5,[$ctx,#4]
	orr	r7,r7,r10,lsl#24
	str	r6,[$ctx,#8]
	and	r7,r7,r3
	str	r7,[$ctx,#12]
#if	__ARM_MAX_ARCH__>=7
	stmia	r2,{r11,r12}		@ fill functions table
	mov	r0,#1
#else
	mov	r0,#0
#endif
.Lno_key:
	ldmia	sp!,{r4-r11}
#if	__ARM_ARCH__>=5
	ret				@ bx	lr
#else
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	poly1305_init,.-poly1305_init
___
{
my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
my ($s1,$s2,$s3)=($r1,$r2,$r3);

$code.=<<___;
.type	poly1305_blocks,%function
.align	5
poly1305_blocks:
.Lpoly1305_blocks:
	stmdb	sp!,{r3-r11,lr}

	ands	$len,$len,#-16
	beq	.Lno_data

	cmp	$padbit,#0
	add	$len,$len,$inp		@ end pointer
	sub	sp,sp,#32

	ldmia	$ctx,{$h0-$r3}		@ load context

	str	$ctx,[sp,#12]		@ offload stuff
	mov	lr,$inp
	str	$len,[sp,#16]
	str	$r1,[sp,#20]
	str	$r2,[sp,#24]
	str	$r3,[sp,#28]
	b	.Loop

.Loop:
#if __ARM_ARCH__<7
	ldrb	r0,[lr],#16		@ load input
# ifdef	__thumb2__
	it	hi
# endif
	addhi	$h4,$h4,#1		@ 1<<128
	ldrb	r1,[lr,#-15]
	ldrb	r2,[lr,#-14]
	ldrb	r3,[lr,#-13]
	orr	r1,r0,r1,lsl#8
	ldrb	r0,[lr,#-12]
	orr	r2,r1,r2,lsl#16
	ldrb	r1,[lr,#-11]
	orr	r3,r2,r3,lsl#24
	ldrb	r2,[lr,#-10]
	adds	$h0,$h0,r3		@ accumulate input

	ldrb	r3,[lr,#-9]
	orr	r1,r0,r1,lsl#8
	ldrb	r0,[lr,#-8]
	orr	r2,r1,r2,lsl#16
	ldrb	r1,[lr,#-7]
	orr	r3,r2,r3,lsl#24
	ldrb	r2,[lr,#-6]
	adcs	$h1,$h1,r3

	ldrb	r3,[lr,#-5]
	orr	r1,r0,r1,lsl#8
	ldrb	r0,[lr,#-4]
	orr	r2,r1,r2,lsl#16
	ldrb	r1,[lr,#-3]
	orr	r3,r2,r3,lsl#24
	ldrb	r2,[lr,#-2]
	adcs	$h2,$h2,r3

	ldrb	r3,[lr,#-1]
	orr	r1,r0,r1,lsl#8
	str	lr,[sp,#8]		@ offload input pointer
	orr	r2,r1,r2,lsl#16
	add	$s1,$r1,$r1,lsr#2
	orr	r3,r2,r3,lsl#24
#else
	ldr	r0,[lr],#16		@ load input
# ifdef	__thumb2__
	it	hi
# endif
	addhi	$h4,$h4,#1		@ padbit
	ldr	r1,[lr,#-12]
	ldr	r2,[lr,#-8]
	ldr	r3,[lr,#-4]
# ifdef	__ARMEB__
	rev	r0,r0
	rev	r1,r1
	rev	r2,r2
	rev	r3,r3
# endif
	adds	$h0,$h0,r0		@ accumulate input
	str	lr,[sp,#8]		@ offload input pointer
	adcs	$h1,$h1,r1
	add	$s1,$r1,$r1,lsr#2
	adcs	$h2,$h2,r2
#endif
	add	$s2,$r2,$r2,lsr#2
	adcs	$h3,$h3,r3
	add	$s3,$r3,$r3,lsr#2

	umull	r2,r3,$h1,$r0
	 adc	$h4,$h4,#0
	umull	r0,r1,$h0,$r0
	umlal	r2,r3,$h4,$s1
	umlal	r0,r1,$h3,$s1
	ldr	$r1,[sp,#20]		@ reload $r1
	umlal	r2,r3,$h2,$s3
	umlal	r0,r1,$h1,$s3
	umlal	r2,r3,$h3,$s2
	umlal	r0,r1,$h2,$s2
	umlal	r2,r3,$h0,$r1
	str	r0,[sp,#0]		@ future $h0
	 mul	r0,$s2,$h4
	ldr	$r2,[sp,#24]		@ reload $r2
	adds	r2,r2,r1		@ d1+=d0>>32
	 eor	r1,r1,r1
	adc	lr,r3,#0		@ future $h2
	str	r2,[sp,#4]		@ future $h1

	mul	r2,$s3,$h4
	eor	r3,r3,r3
	umlal	r0,r1,$h3,$s3
	ldr	$r3,[sp,#28]		@ reload $r3
	umlal	r2,r3,$h3,$r0
	umlal	r0,r1,$h2,$r0
	umlal	r2,r3,$h2,$r1
	umlal	r0,r1,$h1,$r1
	umlal	r2,r3,$h1,$r2
	umlal	r0,r1,$h0,$r2
	umlal	r2,r3,$h0,$r3
	ldr	$h0,[sp,#0]
	mul	$h4,$r0,$h4
	ldr	$h1,[sp,#4]

	adds	$h2,lr,r0		@ d2+=d1>>32
	ldr	lr,[sp,#8]		@ reload input pointer
	adc	r1,r1,#0
	adds	$h3,r2,r1		@ d3+=d2>>32
	ldr	r0,[sp,#16]		@ reload end pointer
	adc	r3,r3,#0
	add	$h4,$h4,r3		@ h4+=d3>>32

	and	r1,$h4,#-4
	and	$h4,$h4,#3
	add	r1,r1,r1,lsr#2		@ *=5
	adds	$h0,$h0,r1
	adcs	$h1,$h1,#0
	adcs	$h2,$h2,#0
	adcs	$h3,$h3,#0
	adc	$h4,$h4,#0

	cmp	r0,lr			@ done yet?
	bhi	.Loop

	ldr	$ctx,[sp,#12]
	add	sp,sp,#32
	stmia	$ctx,{$h0-$h4}		@ store the result

.Lno_data:
#if	__ARM_ARCH__>=5
	ldmia	sp!,{r3-r11,pc}
#else
	ldmia	sp!,{r3-r11,lr}
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	poly1305_blocks,.-poly1305_blocks
___
}
{
my ($ctx,$mac,$nonce)=map("r$_",(0..2));
my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
my $g4=$h4;

$code.=<<___;
.type	poly1305_emit,%function
.align	5
poly1305_emit:
.Lpoly1305_emit:
	stmdb	sp!,{r4-r11}
.Lpoly1305_emit_enter:

	ldmia	$ctx,{$h0-$h4}
	adds	$g0,$h0,#5		@ compare to modulus
	adcs	$g1,$h1,#0
	adcs	$g2,$h2,#0
	adcs	$g3,$h3,#0
	adc	$g4,$h4,#0
	tst	$g4,#4			@ did it carry/borrow?

#ifdef	__thumb2__
	it	ne
#endif
	movne	$h0,$g0
	ldr	$g0,[$nonce,#0]
#ifdef	__thumb2__
	it	ne
#endif
	movne	$h1,$g1
	ldr	$g1,[$nonce,#4]
#ifdef	__thumb2__
	it	ne
#endif
	movne	$h2,$g2
	ldr	$g2,[$nonce,#8]
#ifdef	__thumb2__
	it	ne
#endif
	movne	$h3,$g3
	ldr	$g3,[$nonce,#12]

	adds	$h0,$h0,$g0
	adcs	$h1,$h1,$g1
	adcs	$h2,$h2,$g2
	adc	$h3,$h3,$g3

#if __ARM_ARCH__>=7
# ifdef __ARMEB__
	rev	$h0,$h0
	rev	$h1,$h1
	rev	$h2,$h2
	rev	$h3,$h3
# endif
	str	$h0,[$mac,#0]
	str	$h1,[$mac,#4]
	str	$h2,[$mac,#8]
	str	$h3,[$mac,#12]
#else
	strb	$h0,[$mac,#0]
	mov	$h0,$h0,lsr#8
	strb	$h1,[$mac,#4]
	mov	$h1,$h1,lsr#8
	strb	$h2,[$mac,#8]
	mov	$h2,$h2,lsr#8
	strb	$h3,[$mac,#12]
	mov	$h3,$h3,lsr#8

	strb	$h0,[$mac,#1]
	mov	$h0,$h0,lsr#8
	strb	$h1,[$mac,#5]
	mov	$h1,$h1,lsr#8
	strb	$h2,[$mac,#9]
	mov	$h2,$h2,lsr#8
	strb	$h3,[$mac,#13]
	mov	$h3,$h3,lsr#8

	strb	$h0,[$mac,#2]
	mov	$h0,$h0,lsr#8
	strb	$h1,[$mac,#6]
	mov	$h1,$h1,lsr#8
	strb	$h2,[$mac,#10]
	mov	$h2,$h2,lsr#8
	strb	$h3,[$mac,#14]
	mov	$h3,$h3,lsr#8

	strb	$h0,[$mac,#3]
	strb	$h1,[$mac,#7]
	strb	$h2,[$mac,#11]
	strb	$h3,[$mac,#15]
#endif
	ldmia	sp!,{r4-r11}
#if	__ARM_ARCH__>=5
	ret				@ bx	lr
#else
	tst	lr,#1
	moveq	pc,lr			@ be binary compatible with V4, yet
	bx	lr			@ interoperable with Thumb ISA:-)
#endif
.size	poly1305_emit,.-poly1305_emit
___
{
my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
my ($T0,$T1,$MASK) = map("q$_",(15,4,0));

my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));

$code.=<<___;
#if	__ARM_MAX_ARCH__>=7
.fpu	neon

.type	poly1305_init_neon,%function
.align	5
poly1305_init_neon:
	ldr	r4,[$ctx,#20]		@ load key base 2^32
	ldr	r5,[$ctx,#24]
	ldr	r6,[$ctx,#28]
	ldr	r7,[$ctx,#32]

	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
	mov	r3,r4,lsr#26
	mov	r4,r5,lsr#20
	orr	r3,r3,r5,lsl#6
	mov	r5,r6,lsr#14
	orr	r4,r4,r6,lsl#12
	mov	r6,r7,lsr#8
	orr	r5,r5,r7,lsl#18
	and	r3,r3,#0x03ffffff
	and	r4,r4,#0x03ffffff
	and	r5,r5,#0x03ffffff

	vdup.32	$R0,r2			@ r^1 in both lanes
	add	r2,r3,r3,lsl#2		@ *5
	vdup.32	$R1,r3
	add	r3,r4,r4,lsl#2
	vdup.32	$S1,r2
	vdup.32	$R2,r4
	add	r4,r5,r5,lsl#2
	vdup.32	$S2,r3
	vdup.32	$R3,r5
	add	r5,r6,r6,lsl#2
	vdup.32	$S3,r4
	vdup.32	$R4,r6
	vdup.32	$S4,r5

	mov	$zeros,#2		@ counter

.Lsquare_neon:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4

	vmull.u32	$D0,$R0,${R0}[1]
	vmull.u32	$D1,$R1,${R0}[1]
	vmull.u32	$D2,$R2,${R0}[1]
	vmull.u32	$D3,$R3,${R0}[1]
	vmull.u32	$D4,$R4,${R0}[1]

	vmlal.u32	$D0,$R4,${S1}[1]
	vmlal.u32	$D1,$R0,${R1}[1]
	vmlal.u32	$D2,$R1,${R1}[1]
	vmlal.u32	$D3,$R2,${R1}[1]
	vmlal.u32	$D4,$R3,${R1}[1]

	vmlal.u32	$D0,$R3,${S2}[1]
	vmlal.u32	$D1,$R4,${S2}[1]
	vmlal.u32	$D3,$R1,${R2}[1]
	vmlal.u32	$D2,$R0,${R2}[1]
	vmlal.u32	$D4,$R2,${R2}[1]

	vmlal.u32	$D0,$R2,${S3}[1]
	vmlal.u32	$D3,$R0,${R3}[1]
	vmlal.u32	$D1,$R3,${S3}[1]
	vmlal.u32	$D2,$R4,${S3}[1]
	vmlal.u32	$D4,$R1,${R3}[1]

	vmlal.u32	$D3,$R4,${S4}[1]
	vmlal.u32	$D0,$R1,${S4}[1]
	vmlal.u32	$D1,$R2,${S4}[1]
	vmlal.u32	$D2,$R3,${S4}[1]
	vmlal.u32	$D4,$R0,${R4}[1]

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
	@ and P. Schwabe
	@
	@ H0>>+H1>>+H2>>+H3>>+H4
	@ H3>>+H4>>*5+H0>>+H1
	@
	@ Trivia.
	@
	@ Result of multiplication of n-bit number by m-bit number is
	@ n+m bits wide. However! Even though 2^n is a n+1-bit number,
	@ m-bit number multiplied by 2^n is still n+m bits wide.
	@
	@ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
	@ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
	@ one is n+1 bits wide.
	@
	@ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
	@ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
	@ can be 27. However! In cases when their width exceeds 26 bits
	@ they are limited by 2^26+2^6. This in turn means that *sum*
	@ of the products with these values can still be viewed as sum
	@ of 52-bit numbers as long as the amount of addends is not a
	@ power of 2. For example,
	@
	@ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
	@
	@ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
	@ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
	@ 8 * (2^52) or 2^55. However, the value is then multiplied by
	@ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
	@ which is less than 32 * (2^52) or 2^57. And when processing
	@ data we are looking at triple as many addends...
	@
	@ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
	@ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
	@ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
	@ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
	@ instruction accepts 2x32-bit input and writes 2x64-bit result.
	@ This means that result of reduction have to be compressed upon
	@ loop wrap-around. This can be done in the process of reduction
	@ to minimize amount of instructions [as well as amount of
	@ 128-bit instructions, which benefits low-end processors], but
	@ one has to watch for H2 (which is narrower than H0) and 5*H4
	@ not being wider than 58 bits, so that result of right shift
	@ by 26 bits fits in 32 bits. This is also useful on x86,
	@ because it allows to use paddd in place for paddq, which
	@ benefits Atom, where paddq is ridiculously slow.

	vshr.u64	$T0,$D3,#26
	vmovn.i64	$D3#lo,$D3
	 vshr.u64	$T1,$D0,#26
	 vmovn.i64	$D0#lo,$D0
	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
	vbic.i32	$D3#lo,#0xfc000000	@ &=0x03ffffff
	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
	 vbic.i32	$D0#lo,#0xfc000000

	vshrn.u64	$T0#lo,$D4,#26
	vmovn.i64	$D4#lo,$D4
	 vshr.u64	$T1,$D1,#26
	 vmovn.i64	$D1#lo,$D1
	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
	vbic.i32	$D4#lo,#0xfc000000
	 vbic.i32	$D1#lo,#0xfc000000

	vadd.i32	$D0#lo,$D0#lo,$T0#lo
	vshl.u32	$T0#lo,$T0#lo,#2
	 vshrn.u64	$T1#lo,$D2,#26
	 vmovn.i64	$D2#lo,$D2
	vadd.i32	$D0#lo,$D0#lo,$T0#lo	@ h4 -> h0
	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
	 vbic.i32	$D2#lo,#0xfc000000

	vshr.u32	$T0#lo,$D0#lo,#26
	vbic.i32	$D0#lo,#0xfc000000
	 vshr.u32	$T1#lo,$D3#lo,#26
	 vbic.i32	$D3#lo,#0xfc000000
	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4

	subs		$zeros,$zeros,#1
	beq		.Lsquare_break_neon

	add		$tbl0,$ctx,#(48+0*9*4)
	add		$tbl1,$ctx,#(48+1*9*4)

	vtrn.32		$R0,$D0#lo		@ r^2:r^1
	vtrn.32		$R2,$D2#lo
	vtrn.32		$R3,$D3#lo
	vtrn.32		$R1,$D1#lo
	vtrn.32		$R4,$D4#lo

	vshl.u32	$S2,$R2,#2		@ *5
	vshl.u32	$S3,$R3,#2
	vshl.u32	$S1,$R1,#2
	vshl.u32	$S4,$R4,#2
	vadd.i32	$S2,$S2,$R2
	vadd.i32	$S1,$S1,$R1
	vadd.i32	$S3,$S3,$R3
	vadd.i32	$S4,$S4,$R4

	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vst1.32		{${S4}[0]},[$tbl0,:32]
	vst1.32		{${S4}[1]},[$tbl1,:32]

	b		.Lsquare_neon

.align	4
.Lsquare_break_neon:
	add		$tbl0,$ctx,#(48+2*4*9)
	add		$tbl1,$ctx,#(48+3*4*9)

	vmov		$R0,$D0#lo		@ r^4:r^3
	vshl.u32	$S1,$D1#lo,#2		@ *5
	vmov		$R1,$D1#lo
	vshl.u32	$S2,$D2#lo,#2
	vmov		$R2,$D2#lo
	vshl.u32	$S3,$D3#lo,#2
	vmov		$R3,$D3#lo
	vshl.u32	$S4,$D4#lo,#2
	vmov		$R4,$D4#lo
	vadd.i32	$S1,$S1,$D1#lo
	vadd.i32	$S2,$S2,$D2#lo
	vadd.i32	$S3,$S3,$D3#lo
	vadd.i32	$S4,$S4,$D4#lo

	vst4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
	vst4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
	vst4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vst4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vst1.32		{${S4}[0]},[$tbl0]
	vst1.32		{${S4}[1]},[$tbl1]

	ret				@ bx	lr
.size	poly1305_init_neon,.-poly1305_init_neon

.type	poly1305_blocks_neon,%function
.align	5
poly1305_blocks_neon:
.Lpoly1305_blocks_neon:
	ldr	ip,[$ctx,#36]		@ is_base2_26
	ands	$len,$len,#-16
	beq	.Lno_data_neon

	cmp	$len,#64
	bhs	.Lenter_neon
	tst	ip,ip			@ is_base2_26?
	beq	.Lpoly1305_blocks

.Lenter_neon:
	stmdb	sp!,{r4-r7}
	vstmdb	sp!,{d8-d15}		@ ABI specification says so

	tst	ip,ip			@ is_base2_26?
	bne	.Lbase2_26_neon

	stmdb	sp!,{r1-r3,lr}
	bl	poly1305_init_neon

	ldr	r4,[$ctx,#0]		@ load hash value base 2^32
	ldr	r5,[$ctx,#4]
	ldr	r6,[$ctx,#8]
	ldr	r7,[$ctx,#12]
	ldr	ip,[$ctx,#16]

	and	r2,r4,#0x03ffffff	@ base 2^32 -> base 2^26
	mov	r3,r4,lsr#26
	 veor	$D0#lo,$D0#lo,$D0#lo
	mov	r4,r5,lsr#20
	orr	r3,r3,r5,lsl#6
	 veor	$D1#lo,$D1#lo,$D1#lo
	mov	r5,r6,lsr#14
	orr	r4,r4,r6,lsl#12
	 veor	$D2#lo,$D2#lo,$D2#lo
	mov	r6,r7,lsr#8
	orr	r5,r5,r7,lsl#18
	 veor	$D3#lo,$D3#lo,$D3#lo
	and	r3,r3,#0x03ffffff
	orr	r6,r6,ip,lsl#24
	 veor	$D4#lo,$D4#lo,$D4#lo
	and	r4,r4,#0x03ffffff
	mov	r1,#1
	and	r5,r5,#0x03ffffff
	str	r1,[$ctx,#36]		@ is_base2_26

	vmov.32	$D0#lo[0],r2
	vmov.32	$D1#lo[0],r3
	vmov.32	$D2#lo[0],r4
	vmov.32	$D3#lo[0],r5
	vmov.32	$D4#lo[0],r6
	adr	$zeros,.Lzeros

	ldmia	sp!,{r1-r3,lr}
	b	.Lbase2_32_neon

.align	4
.Lbase2_26_neon:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ load hash value

	veor		$D0#lo,$D0#lo,$D0#lo
	veor		$D1#lo,$D1#lo,$D1#lo
	veor		$D2#lo,$D2#lo,$D2#lo
	veor		$D3#lo,$D3#lo,$D3#lo
	veor		$D4#lo,$D4#lo,$D4#lo
	vld4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
	adr		$zeros,.Lzeros
	vld1.32		{$D4#lo[0]},[$ctx]
	sub		$ctx,$ctx,#16		@ rewind

.Lbase2_32_neon:
	add		$in2,$inp,#32
	mov		$padbit,$padbit,lsl#24
	tst		$len,#31
	beq		.Leven

	vld4.32		{$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
	vmov.32		$H4#lo[0],$padbit
	sub		$len,$len,#16
	add		$in2,$inp,#32

# ifdef	__ARMEB__
	vrev32.8	$H0,$H0
	vrev32.8	$H3,$H3
	vrev32.8	$H1,$H1
	vrev32.8	$H2,$H2
# endif
	vsri.u32	$H4#lo,$H3#lo,#8	@ base 2^32 -> base 2^26
	vshl.u32	$H3#lo,$H3#lo,#18

	vsri.u32	$H3#lo,$H2#lo,#14
	vshl.u32	$H2#lo,$H2#lo,#12
	vadd.i32	$H4#hi,$H4#lo,$D4#lo	@ add hash value and move to #hi

	vbic.i32	$H3#lo,#0xfc000000
	vsri.u32	$H2#lo,$H1#lo,#20
	vshl.u32	$H1#lo,$H1#lo,#6

	vbic.i32	$H2#lo,#0xfc000000
	vsri.u32	$H1#lo,$H0#lo,#26
	vadd.i32	$H3#hi,$H3#lo,$D3#lo

	vbic.i32	$H0#lo,#0xfc000000
	vbic.i32	$H1#lo,#0xfc000000
	vadd.i32	$H2#hi,$H2#lo,$D2#lo

	vadd.i32	$H0#hi,$H0#lo,$D0#lo
	vadd.i32	$H1#hi,$H1#lo,$D1#lo

	mov		$tbl1,$zeros
	add		$tbl0,$ctx,#48

	cmp		$len,$len
	b		.Long_tail

.align	4
.Leven:
	subs		$len,$len,#64
	it		lo
	movlo		$in2,$zeros

	vmov.i32	$H4,#1<<24		@ padbit, yes, always
	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
	add		$inp,$inp,#64
	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
	add		$in2,$in2,#64
	itt		hi
	addhi		$tbl1,$ctx,#(48+1*9*4)
	addhi		$tbl0,$ctx,#(48+3*9*4)

# ifdef	__ARMEB__
	vrev32.8	$H0,$H0
	vrev32.8	$H3,$H3
	vrev32.8	$H1,$H1
	vrev32.8	$H2,$H2
# endif
	vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
	vshl.u32	$H3,$H3,#18

	vsri.u32	$H3,$H2,#14
	vshl.u32	$H2,$H2,#12

	vbic.i32	$H3,#0xfc000000
	vsri.u32	$H2,$H1,#20
	vshl.u32	$H1,$H1,#6

	vbic.i32	$H2,#0xfc000000
	vsri.u32	$H1,$H0,#26

	vbic.i32	$H0,#0xfc000000
	vbic.i32	$H1,#0xfc000000

	bls		.Lskip_loop

	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^2
	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4
	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	b		.Loop_neon

.align	5
.Loop_neon:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
	@   \___________________/
	@ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
	@ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
	@   \___________________/ \____________________/
	@
	@ Note that we start with inp[2:3]*r^2. This is because it
	@ doesn't depend on reduction in previous iteration.
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
	@ d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
	@ d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
	@ d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
	@ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ inp[2:3]*r^2

	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ accumulate inp[0:1]
	vmull.u32	$D2,$H2#hi,${R0}[1]
	vadd.i32	$H0#lo,$H0#lo,$D0#lo
	vmull.u32	$D0,$H0#hi,${R0}[1]
	vadd.i32	$H3#lo,$H3#lo,$D3#lo
	vmull.u32	$D3,$H3#hi,${R0}[1]
	vmlal.u32	$D2,$H1#hi,${R1}[1]
	vadd.i32	$H1#lo,$H1#lo,$D1#lo
	vmull.u32	$D1,$H1#hi,${R0}[1]

	vadd.i32	$H4#lo,$H4#lo,$D4#lo
	vmull.u32	$D4,$H4#hi,${R0}[1]
	subs		$len,$len,#64
	vmlal.u32	$D0,$H4#hi,${S1}[1]
	it		lo
	movlo		$in2,$zeros
	vmlal.u32	$D3,$H2#hi,${R1}[1]
	vld1.32		${S4}[1],[$tbl1,:32]
	vmlal.u32	$D1,$H0#hi,${R1}[1]
	vmlal.u32	$D4,$H3#hi,${R1}[1]

	vmlal.u32	$D0,$H3#hi,${S2}[1]
	vmlal.u32	$D3,$H1#hi,${R2}[1]
	vmlal.u32	$D4,$H2#hi,${R2}[1]
	vmlal.u32	$D1,$H4#hi,${S2}[1]
	vmlal.u32	$D2,$H0#hi,${R2}[1]

	vmlal.u32	$D3,$H0#hi,${R3}[1]
	vmlal.u32	$D0,$H2#hi,${S3}[1]
	vmlal.u32	$D4,$H1#hi,${R3}[1]
	vmlal.u32	$D1,$H3#hi,${S3}[1]
	vmlal.u32	$D2,$H4#hi,${S3}[1]

	vmlal.u32	$D3,$H4#hi,${S4}[1]
	vmlal.u32	$D0,$H1#hi,${S4}[1]
	vmlal.u32	$D4,$H0#hi,${R4}[1]
	vmlal.u32	$D1,$H2#hi,${S4}[1]
	vmlal.u32	$D2,$H3#hi,${S4}[1]

	vld4.32		{$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2]	@ inp[2:3] (or 0)
	add		$in2,$in2,#64

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ (hash+inp[0:1])*r^4 and accumulate

	vmlal.u32	$D3,$H3#lo,${R0}[0]
	vmlal.u32	$D0,$H0#lo,${R0}[0]
	vmlal.u32	$D4,$H4#lo,${R0}[0]
	vmlal.u32	$D1,$H1#lo,${R0}[0]
	vmlal.u32	$D2,$H2#lo,${R0}[0]
	vld1.32		${S4}[0],[$tbl0,:32]

	vmlal.u32	$D3,$H2#lo,${R1}[0]
	vmlal.u32	$D0,$H4#lo,${S1}[0]
	vmlal.u32	$D4,$H3#lo,${R1}[0]
	vmlal.u32	$D1,$H0#lo,${R1}[0]
	vmlal.u32	$D2,$H1#lo,${R1}[0]

	vmlal.u32	$D3,$H1#lo,${R2}[0]
	vmlal.u32	$D0,$H3#lo,${S2}[0]
	vmlal.u32	$D4,$H2#lo,${R2}[0]
	vmlal.u32	$D1,$H4#lo,${S2}[0]
	vmlal.u32	$D2,$H0#lo,${R2}[0]

	vmlal.u32	$D3,$H0#lo,${R3}[0]
	vmlal.u32	$D0,$H2#lo,${S3}[0]
	vmlal.u32	$D4,$H1#lo,${R3}[0]
	vmlal.u32	$D1,$H3#lo,${S3}[0]
	vmlal.u32	$D3,$H4#lo,${S4}[0]

	vmlal.u32	$D2,$H4#lo,${S3}[0]
	vmlal.u32	$D0,$H1#lo,${S4}[0]
	vmlal.u32	$D4,$H0#lo,${R4}[0]
	vmov.i32	$H4,#1<<24		@ padbit, yes, always
	vmlal.u32	$D1,$H2#lo,${S4}[0]
	vmlal.u32	$D2,$H3#lo,${S4}[0]

	vld4.32		{$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp]	@ inp[0:1]
	add		$inp,$inp,#64
# ifdef	__ARMEB__
	vrev32.8	$H0,$H0
	vrev32.8	$H1,$H1
	vrev32.8	$H2,$H2
	vrev32.8	$H3,$H3
# endif

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ lazy reduction interleaved with base 2^32 -> base 2^26 of
	@ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.

	vshr.u64	$T0,$D3,#26
	vmovn.i64	$D3#lo,$D3
	 vshr.u64	$T1,$D0,#26
	 vmovn.i64	$D0#lo,$D0
	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
	vbic.i32	$D3#lo,#0xfc000000
	  vsri.u32	$H4,$H3,#8		@ base 2^32 -> base 2^26
	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1
	  vshl.u32	$H3,$H3,#18
	 vbic.i32	$D0#lo,#0xfc000000

	vshrn.u64	$T0#lo,$D4,#26
	vmovn.i64	$D4#lo,$D4
	 vshr.u64	$T1,$D1,#26
	 vmovn.i64	$D1#lo,$D1
	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2
	  vsri.u32	$H3,$H2,#14
	vbic.i32	$D4#lo,#0xfc000000
	  vshl.u32	$H2,$H2,#12
	 vbic.i32	$D1#lo,#0xfc000000

	vadd.i32	$D0#lo,$D0#lo,$T0#lo
	vshl.u32	$T0#lo,$T0#lo,#2
	  vbic.i32	$H3,#0xfc000000
	 vshrn.u64	$T1#lo,$D2,#26
	 vmovn.i64	$D2#lo,$D2
	vaddl.u32	$D0,$D0#lo,$T0#lo	@ h4 -> h0 [widen for a sec]
	  vsri.u32	$H2,$H1,#20
	 vadd.i32	$D3#lo,$D3#lo,$T1#lo	@ h2 -> h3
	  vshl.u32	$H1,$H1,#6
	 vbic.i32	$D2#lo,#0xfc000000
	  vbic.i32	$H2,#0xfc000000

	vshrn.u64	$T0#lo,$D0,#26		@ re-narrow
	vmovn.i64	$D0#lo,$D0
	  vsri.u32	$H1,$H0,#26
	  vbic.i32	$H0,#0xfc000000
	 vshr.u32	$T1#lo,$D3#lo,#26
	 vbic.i32	$D3#lo,#0xfc000000
	vbic.i32	$D0#lo,#0xfc000000
	vadd.i32	$D1#lo,$D1#lo,$T0#lo	@ h0 -> h1
	 vadd.i32	$D4#lo,$D4#lo,$T1#lo	@ h3 -> h4
	  vbic.i32	$H1,#0xfc000000

	bhi		.Loop_neon

.Lskip_loop:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1

	add		$tbl1,$ctx,#(48+0*9*4)
	add		$tbl0,$ctx,#(48+1*9*4)
	adds		$len,$len,#32
	it		ne
	movne		$len,#0
	bne		.Long_tail

	vadd.i32	$H2#hi,$H2#lo,$D2#lo	@ add hash value and move to #hi
	vadd.i32	$H0#hi,$H0#lo,$D0#lo
	vadd.i32	$H3#hi,$H3#lo,$D3#lo
	vadd.i32	$H1#hi,$H1#lo,$D1#lo
	vadd.i32	$H4#hi,$H4#lo,$D4#lo

.Long_tail:
	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^1
	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^2

	vadd.i32	$H2#lo,$H2#lo,$D2#lo	@ can be redundant
	vmull.u32	$D2,$H2#hi,$R0
	vadd.i32	$H0#lo,$H0#lo,$D0#lo
	vmull.u32	$D0,$H0#hi,$R0
	vadd.i32	$H3#lo,$H3#lo,$D3#lo
	vmull.u32	$D3,$H3#hi,$R0
	vadd.i32	$H1#lo,$H1#lo,$D1#lo
	vmull.u32	$D1,$H1#hi,$R0
	vadd.i32	$H4#lo,$H4#lo,$D4#lo
	vmull.u32	$D4,$H4#hi,$R0

	vmlal.u32	$D0,$H4#hi,$S1
	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vmlal.u32	$D3,$H2#hi,$R1
	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vmlal.u32	$D1,$H0#hi,$R1
	vmlal.u32	$D4,$H3#hi,$R1
	vmlal.u32	$D2,$H1#hi,$R1

	vmlal.u32	$D3,$H1#hi,$R2
	vld1.32		${S4}[1],[$tbl1,:32]
	vmlal.u32	$D0,$H3#hi,$S2
	vld1.32		${S4}[0],[$tbl0,:32]
	vmlal.u32	$D4,$H2#hi,$R2
	vmlal.u32	$D1,$H4#hi,$S2
	vmlal.u32	$D2,$H0#hi,$R2

	vmlal.u32	$D3,$H0#hi,$R3
	 it		ne
	 addne		$tbl1,$ctx,#(48+2*9*4)
	vmlal.u32	$D0,$H2#hi,$S3
	 it		ne
	 addne		$tbl0,$ctx,#(48+3*9*4)
	vmlal.u32	$D4,$H1#hi,$R3
	vmlal.u32	$D1,$H3#hi,$S3
	vmlal.u32	$D2,$H4#hi,$S3

	vmlal.u32	$D3,$H4#hi,$S4
	 vorn		$MASK,$MASK,$MASK	@ all-ones, can be redundant
	vmlal.u32	$D0,$H1#hi,$S4
	 vshr.u64	$MASK,$MASK,#38
	vmlal.u32	$D4,$H0#hi,$R4
	vmlal.u32	$D1,$H2#hi,$S4
	vmlal.u32	$D2,$H3#hi,$S4

	beq		.Lshort_tail

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ (hash+inp[0:1])*r^4:r^3 and accumulate

	vld4.32		{${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!	@ load r^3
	vld4.32		{${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!	@ load r^4

	vmlal.u32	$D2,$H2#lo,$R0
	vmlal.u32	$D0,$H0#lo,$R0
	vmlal.u32	$D3,$H3#lo,$R0
	vmlal.u32	$D1,$H1#lo,$R0
	vmlal.u32	$D4,$H4#lo,$R0

	vmlal.u32	$D0,$H4#lo,$S1
	vld4.32		{${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
	vmlal.u32	$D3,$H2#lo,$R1
	vld4.32		{${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
	vmlal.u32	$D1,$H0#lo,$R1
	vmlal.u32	$D4,$H3#lo,$R1
	vmlal.u32	$D2,$H1#lo,$R1

	vmlal.u32	$D3,$H1#lo,$R2
	vld1.32		${S4}[1],[$tbl1,:32]
	vmlal.u32	$D0,$H3#lo,$S2
	vld1.32		${S4}[0],[$tbl0,:32]
	vmlal.u32	$D4,$H2#lo,$R2
	vmlal.u32	$D1,$H4#lo,$S2
	vmlal.u32	$D2,$H0#lo,$R2

	vmlal.u32	$D3,$H0#lo,$R3
	vmlal.u32	$D0,$H2#lo,$S3
	vmlal.u32	$D4,$H1#lo,$R3
	vmlal.u32	$D1,$H3#lo,$S3
	vmlal.u32	$D2,$H4#lo,$S3

	vmlal.u32	$D3,$H4#lo,$S4
	 vorn		$MASK,$MASK,$MASK	@ all-ones
	vmlal.u32	$D0,$H1#lo,$S4
	 vshr.u64	$MASK,$MASK,#38
	vmlal.u32	$D4,$H0#lo,$R4
	vmlal.u32	$D1,$H2#lo,$S4
	vmlal.u32	$D2,$H3#lo,$S4

.Lshort_tail:
	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ horizontal addition

	vadd.i64	$D3#lo,$D3#lo,$D3#hi
	vadd.i64	$D0#lo,$D0#lo,$D0#hi
	vadd.i64	$D4#lo,$D4#lo,$D4#hi
	vadd.i64	$D1#lo,$D1#lo,$D1#hi
	vadd.i64	$D2#lo,$D2#lo,$D2#hi

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ lazy reduction, but without narrowing

	vshr.u64	$T0,$D3,#26
	vand.i64	$D3,$D3,$MASK
	 vshr.u64	$T1,$D0,#26
	 vand.i64	$D0,$D0,$MASK
	vadd.i64	$D4,$D4,$T0		@ h3 -> h4
	 vadd.i64	$D1,$D1,$T1		@ h0 -> h1

	vshr.u64	$T0,$D4,#26
	vand.i64	$D4,$D4,$MASK
	 vshr.u64	$T1,$D1,#26
	 vand.i64	$D1,$D1,$MASK
	 vadd.i64	$D2,$D2,$T1		@ h1 -> h2

	vadd.i64	$D0,$D0,$T0
	vshl.u64	$T0,$T0,#2
	 vshr.u64	$T1,$D2,#26
	 vand.i64	$D2,$D2,$MASK
	vadd.i64	$D0,$D0,$T0		@ h4 -> h0
	 vadd.i64	$D3,$D3,$T1		@ h2 -> h3

	vshr.u64	$T0,$D0,#26
	vand.i64	$D0,$D0,$MASK
	 vshr.u64	$T1,$D3,#26
	 vand.i64	$D3,$D3,$MASK
	vadd.i64	$D1,$D1,$T0		@ h0 -> h1
	 vadd.i64	$D4,$D4,$T1		@ h3 -> h4

	cmp		$len,#0
	bne		.Leven

	@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
	@ store hash value

	vst4.32		{$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
	vst1.32		{$D4#lo[0]},[$ctx]

	vldmia	sp!,{d8-d15}			@ epilogue
	ldmia	sp!,{r4-r7}
.Lno_data_neon:
	ret					@ bx	lr
.size	poly1305_blocks_neon,.-poly1305_blocks_neon

.type	poly1305_emit_neon,%function
.align	5
poly1305_emit_neon:
.Lpoly1305_emit_neon:
	ldr	ip,[$ctx,#36]		@ is_base2_26

	stmdb	sp!,{r4-r11}

	tst	ip,ip
	beq	.Lpoly1305_emit_enter

	ldmia	$ctx,{$h0-$h4}
	eor	$g0,$g0,$g0

	adds	$h0,$h0,$h1,lsl#26	@ base 2^26 -> base 2^32
	mov	$h1,$h1,lsr#6
	adcs	$h1,$h1,$h2,lsl#20
	mov	$h2,$h2,lsr#12
	adcs	$h2,$h2,$h3,lsl#14
	mov	$h3,$h3,lsr#18
	adcs	$h3,$h3,$h4,lsl#8
	adc	$h4,$g0,$h4,lsr#24	@ can be partially reduced ...

	and	$g0,$h4,#-4		@ ... so reduce
	and	$h4,$h3,#3
	add	$g0,$g0,$g0,lsr#2	@ *= 5
	adds	$h0,$h0,$g0
	adcs	$h1,$h1,#0
	adcs	$h2,$h2,#0
	adcs	$h3,$h3,#0
	adc	$h4,$h4,#0

	adds	$g0,$h0,#5		@ compare to modulus
	adcs	$g1,$h1,#0
	adcs	$g2,$h2,#0
	adcs	$g3,$h3,#0
	adc	$g4,$h4,#0
	tst	$g4,#4			@ did it carry/borrow?

	it	ne
	movne	$h0,$g0
	ldr	$g0,[$nonce,#0]
	it	ne
	movne	$h1,$g1
	ldr	$g1,[$nonce,#4]
	it	ne
	movne	$h2,$g2
	ldr	$g2,[$nonce,#8]
	it	ne
	movne	$h3,$g3
	ldr	$g3,[$nonce,#12]

	adds	$h0,$h0,$g0		@ accumulate nonce
	adcs	$h1,$h1,$g1
	adcs	$h2,$h2,$g2
	adc	$h3,$h3,$g3

# ifdef __ARMEB__
	rev	$h0,$h0
	rev	$h1,$h1
	rev	$h2,$h2
	rev	$h3,$h3
# endif
	str	$h0,[$mac,#0]		@ store the result
	str	$h1,[$mac,#4]
	str	$h2,[$mac,#8]
	str	$h3,[$mac,#12]

	ldmia	sp!,{r4-r11}
	ret				@ bx	lr
.size	poly1305_emit_neon,.-poly1305_emit_neon

.align	5
.Lzeros:
.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
.LOPENSSL_armcap:
.word	OPENSSL_armcap_P-.Lpoly1305_init
#endif
___
}	}
$code.=<<___;
.asciz	"Poly1305 for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
#if	__ARM_MAX_ARCH__>=7
.comm   OPENSSL_armcap_P,4,4
#endif
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval $1/geo;

	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
	s/\bret\b/bx	lr/go						or
	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;	# make it possible to compile with -march=armv4

	print $_,"\n";
}
close STDOUT or die "error closing STDOUT: $!"; # enforce flush


#! /usr/bin/env perl
# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements Poly1305 hash for ARMv8.
#
# June 2015
#
# Numbers are cycles per processed byte with poly1305_blocks alone.
#
#		IALU/gcc-4.9	NEON
#
# Apple A7	1.86/+5%	0.72
# Cortex-A53	2.69/+58%	1.47
# Cortex-A57	2.70/+7%	1.14
# Denver	1.64/+50%	1.18(*)
# X-Gene	2.13/+68%	2.27
# Mongoose	1.77/+75%	1.12
# Kryo		2.70/+55%	1.13
#
# (*)	estimate based on resources availability is less than 1.0,
#	i.e. measured result is worse than expected, presumably binary
#	translator is not almighty;

$flavour=shift;
$output=shift;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;

my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
my ($mac,$nonce)=($inp,$len);

my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));

$code.=<<___;
#include "arm_arch.h"

.text

// forward "declarations" are required for Apple
.extern	OPENSSL_armcap_P
.hidden	OPENSSL_armcap_P
.globl	poly1305_init
.hidden	poly1305_init
.globl	poly1305_blocks
.hidden	poly1305_blocks
.globl	poly1305_emit
.hidden	poly1305_emit

.type	poly1305_init,%function
.align	5
poly1305_init:
	cmp	$inp,xzr
	stp	xzr,xzr,[$ctx]		// zero hash value
	stp	xzr,xzr,[$ctx,#16]	// [along with is_base2_26]

	csel	x0,xzr,x0,eq
	b.eq	.Lno_key

#ifdef	__ILP32__
	ldrsw	$t1,.LOPENSSL_armcap_P
#else
	ldr	$t1,.LOPENSSL_armcap_P
#endif
	adr	$t0,.LOPENSSL_armcap_P

	ldp	$r0,$r1,[$inp]		// load key
	mov	$s1,#0xfffffffc0fffffff
	movk	$s1,#0x0fff,lsl#48
	ldr	w17,[$t0,$t1]
#ifdef	__ARMEB__
	rev	$r0,$r0			// flip bytes
	rev	$r1,$r1
#endif
	and	$r0,$r0,$s1		// &=0ffffffc0fffffff
	and	$s1,$s1,#-4
	and	$r1,$r1,$s1		// &=0ffffffc0ffffffc
	stp	$r0,$r1,[$ctx,#32]	// save key value

	tst	w17,#ARMV7_NEON

	adr	$d0,poly1305_blocks
	adr	$r0,poly1305_blocks_neon
	adr	$d1,poly1305_emit
	adr	$r1,poly1305_emit_neon

	csel	$d0,$d0,$r0,eq
	csel	$d1,$d1,$r1,eq

#ifdef	__ILP32__
	stp	w12,w13,[$len]
#else
	stp	$d0,$d1,[$len]
#endif

	mov	x0,#1
.Lno_key:
	ret
.size	poly1305_init,.-poly1305_init

.type	poly1305_blocks,%function
.align	5
poly1305_blocks:
	ands	$len,$len,#-16
	b.eq	.Lno_data

	ldp	$h0,$h1,[$ctx]		// load hash value
	ldp	$r0,$r1,[$ctx,#32]	// load key value
	ldr	$h2,[$ctx,#16]
	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
	b	.Loop

.align	5
.Loop:
	ldp	$t0,$t1,[$inp],#16	// load input
	sub	$len,$len,#16
#ifdef	__ARMEB__
	rev	$t0,$t0
	rev	$t1,$t1
#endif
	adds	$h0,$h0,$t0		// accumulate input
	adcs	$h1,$h1,$t1

	mul	$d0,$h0,$r0		// h0*r0
	adc	$h2,$h2,$padbit
	umulh	$d1,$h0,$r0

	mul	$t0,$h1,$s1		// h1*5*r1
	umulh	$t1,$h1,$s1

	adds	$d0,$d0,$t0
	mul	$t0,$h0,$r1		// h0*r1
	adc	$d1,$d1,$t1
	umulh	$d2,$h0,$r1

	adds	$d1,$d1,$t0
	mul	$t0,$h1,$r0		// h1*r0
	adc	$d2,$d2,xzr
	umulh	$t1,$h1,$r0

	adds	$d1,$d1,$t0
	mul	$t0,$h2,$s1		// h2*5*r1
	adc	$d2,$d2,$t1
	mul	$t1,$h2,$r0		// h2*r0

	adds	$d1,$d1,$t0
	adc	$d2,$d2,$t1

	and	$t0,$d2,#-4		// final reduction
	and	$h2,$d2,#3
	add	$t0,$t0,$d2,lsr#2
	adds	$h0,$d0,$t0
	adcs	$h1,$d1,xzr
	adc	$h2,$h2,xzr

	cbnz	$len,.Loop

	stp	$h0,$h1,[$ctx]		// store hash value
	str	$h2,[$ctx,#16]

.Lno_data:
	ret
.size	poly1305_blocks,.-poly1305_blocks

.type	poly1305_emit,%function
.align	5
poly1305_emit:
	ldp	$h0,$h1,[$ctx]		// load hash base 2^64
	ldr	$h2,[$ctx,#16]
	ldp	$t0,$t1,[$nonce]	// load nonce

	adds	$d0,$h0,#5		// compare to modulus
	adcs	$d1,$h1,xzr
	adc	$d2,$h2,xzr

	tst	$d2,#-4			// see if it's carried/borrowed

	csel	$h0,$h0,$d0,eq
	csel	$h1,$h1,$d1,eq

#ifdef	__ARMEB__
	ror	$t0,$t0,#32		// flip nonce words
	ror	$t1,$t1,#32
#endif
	adds	$h0,$h0,$t0		// accumulate nonce
	adc	$h1,$h1,$t1
#ifdef	__ARMEB__
	rev	$h0,$h0			// flip output bytes
	rev	$h1,$h1
#endif
	stp	$h0,$h1,[$mac]		// write result

	ret
.size	poly1305_emit,.-poly1305_emit
___
my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
my ($T0,$T1,$MASK) = map("v$_",(29..31));

my ($in2,$zeros)=("x16","x17");
my $is_base2_26 = $zeros;		# borrow

$code.=<<___;
.type	poly1305_mult,%function
.align	5
poly1305_mult:
	mul	$d0,$h0,$r0		// h0*r0
	umulh	$d1,$h0,$r0

	mul	$t0,$h1,$s1		// h1*5*r1
	umulh	$t1,$h1,$s1

	adds	$d0,$d0,$t0
	mul	$t0,$h0,$r1		// h0*r1
	adc	$d1,$d1,$t1
	umulh	$d2,$h0,$r1

	adds	$d1,$d1,$t0
	mul	$t0,$h1,$r0		// h1*r0
	adc	$d2,$d2,xzr
	umulh	$t1,$h1,$r0

	adds	$d1,$d1,$t0
	mul	$t0,$h2,$s1		// h2*5*r1
	adc	$d2,$d2,$t1
	mul	$t1,$h2,$r0		// h2*r0

	adds	$d1,$d1,$t0
	adc	$d2,$d2,$t1

	and	$t0,$d2,#-4		// final reduction
	and	$h2,$d2,#3
	add	$t0,$t0,$d2,lsr#2
	adds	$h0,$d0,$t0
	adcs	$h1,$d1,xzr
	adc	$h2,$h2,xzr

	ret
.size	poly1305_mult,.-poly1305_mult

.type	poly1305_splat,%function
.align	5
poly1305_splat:
	and	x12,$h0,#0x03ffffff	// base 2^64 -> base 2^26
	ubfx	x13,$h0,#26,#26
	extr	x14,$h1,$h0,#52
	and	x14,x14,#0x03ffffff
	ubfx	x15,$h1,#14,#26
	extr	x16,$h2,$h1,#40

	str	w12,[$ctx,#16*0]	// r0
	add	w12,w13,w13,lsl#2	// r1*5
	str	w13,[$ctx,#16*1]	// r1
	add	w13,w14,w14,lsl#2	// r2*5
	str	w12,[$ctx,#16*2]	// s1
	str	w14,[$ctx,#16*3]	// r2
	add	w14,w15,w15,lsl#2	// r3*5
	str	w13,[$ctx,#16*4]	// s2
	str	w15,[$ctx,#16*5]	// r3
	add	w15,w16,w16,lsl#2	// r4*5
	str	w14,[$ctx,#16*6]	// s3
	str	w16,[$ctx,#16*7]	// r4
	str	w15,[$ctx,#16*8]	// s4

	ret
.size	poly1305_splat,.-poly1305_splat

.type	poly1305_blocks_neon,%function
.align	5
poly1305_blocks_neon:
	ldr	$is_base2_26,[$ctx,#24]
	cmp	$len,#128
	b.hs	.Lblocks_neon
	cbz	$is_base2_26,poly1305_blocks

.Lblocks_neon:
	.inst	0xd503233f		// paciasp
	stp	x29,x30,[sp,#-80]!
	add	x29,sp,#0

	ands	$len,$len,#-16
	b.eq	.Lno_data_neon

	cbz	$is_base2_26,.Lbase2_64_neon

	ldp	w10,w11,[$ctx]		// load hash value base 2^26
	ldp	w12,w13,[$ctx,#8]
	ldr	w14,[$ctx,#16]

	tst	$len,#31
	b.eq	.Leven_neon

	ldp	$r0,$r1,[$ctx,#32]	// load key value

	add	$h0,x10,x11,lsl#26	// base 2^26 -> base 2^64
	lsr	$h1,x12,#12
	adds	$h0,$h0,x12,lsl#52
	add	$h1,$h1,x13,lsl#14
	adc	$h1,$h1,xzr
	lsr	$h2,x14,#24
	adds	$h1,$h1,x14,lsl#40
	adc	$d2,$h2,xzr		// can be partially reduced...

	ldp	$d0,$d1,[$inp],#16	// load input
	sub	$len,$len,#16
	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)

	and	$t0,$d2,#-4		// ... so reduce
	and	$h2,$d2,#3
	add	$t0,$t0,$d2,lsr#2
	adds	$h0,$h0,$t0
	adcs	$h1,$h1,xzr
	adc	$h2,$h2,xzr

#ifdef	__ARMEB__
	rev	$d0,$d0
	rev	$d1,$d1
#endif
	adds	$h0,$h0,$d0		// accumulate input
	adcs	$h1,$h1,$d1
	adc	$h2,$h2,$padbit

	bl	poly1305_mult
	ldr	x30,[sp,#8]

	cbz	$padbit,.Lstore_base2_64_neon

	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26
	ubfx	x11,$h0,#26,#26
	extr	x12,$h1,$h0,#52
	and	x12,x12,#0x03ffffff
	ubfx	x13,$h1,#14,#26
	extr	x14,$h2,$h1,#40

	cbnz	$len,.Leven_neon

	stp	w10,w11,[$ctx]		// store hash value base 2^26
	stp	w12,w13,[$ctx,#8]
	str	w14,[$ctx,#16]
	b	.Lno_data_neon

.align	4
.Lstore_base2_64_neon:
	stp	$h0,$h1,[$ctx]		// store hash value base 2^64
	stp	$h2,xzr,[$ctx,#16]	// note that is_base2_26 is zeroed
	b	.Lno_data_neon

.align	4
.Lbase2_64_neon:
	ldp	$r0,$r1,[$ctx,#32]	// load key value

	ldp	$h0,$h1,[$ctx]		// load hash value base 2^64
	ldr	$h2,[$ctx,#16]

	tst	$len,#31
	b.eq	.Linit_neon

	ldp	$d0,$d1,[$inp],#16	// load input
	sub	$len,$len,#16
	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
#ifdef	__ARMEB__
	rev	$d0,$d0
	rev	$d1,$d1
#endif
	adds	$h0,$h0,$d0		// accumulate input
	adcs	$h1,$h1,$d1
	adc	$h2,$h2,$padbit

	bl	poly1305_mult

.Linit_neon:
	and	x10,$h0,#0x03ffffff	// base 2^64 -> base 2^26
	ubfx	x11,$h0,#26,#26
	extr	x12,$h1,$h0,#52
	and	x12,x12,#0x03ffffff
	ubfx	x13,$h1,#14,#26
	extr	x14,$h2,$h1,#40

	stp	d8,d9,[sp,#16]		// meet ABI requirements
	stp	d10,d11,[sp,#32]
	stp	d12,d13,[sp,#48]
	stp	d14,d15,[sp,#64]

	fmov	${H0},x10
	fmov	${H1},x11
	fmov	${H2},x12
	fmov	${H3},x13
	fmov	${H4},x14

	////////////////////////////////// initialize r^n table
	mov	$h0,$r0			// r^1
	add	$s1,$r1,$r1,lsr#2	// s1 = r1 + (r1 >> 2)
	mov	$h1,$r1
	mov	$h2,xzr
	add	$ctx,$ctx,#48+12
	bl	poly1305_splat

	bl	poly1305_mult		// r^2
	sub	$ctx,$ctx,#4
	bl	poly1305_splat

	bl	poly1305_mult		// r^3
	sub	$ctx,$ctx,#4
	bl	poly1305_splat

	bl	poly1305_mult		// r^4
	sub	$ctx,$ctx,#4
	bl	poly1305_splat
	ldr	x30,[sp,#8]

	add	$in2,$inp,#32
	adr	$zeros,.Lzeros
	subs	$len,$len,#64
	csel	$in2,$zeros,$in2,lo

	mov	x4,#1
	str	x4,[$ctx,#-24]		// set is_base2_26
	sub	$ctx,$ctx,#48		// restore original $ctx
	b	.Ldo_neon

.align	4
.Leven_neon:
	add	$in2,$inp,#32
	adr	$zeros,.Lzeros
	subs	$len,$len,#64
	csel	$in2,$zeros,$in2,lo

	stp	d8,d9,[sp,#16]		// meet ABI requirements
	stp	d10,d11,[sp,#32]
	stp	d12,d13,[sp,#48]
	stp	d14,d15,[sp,#64]

	fmov	${H0},x10
	fmov	${H1},x11
	fmov	${H2},x12
	fmov	${H3},x13
	fmov	${H4},x14

.Ldo_neon:
	ldp	x8,x12,[$in2],#16	// inp[2:3] (or zero)
	ldp	x9,x13,[$in2],#48

	lsl	$padbit,$padbit,#24
	add	x15,$ctx,#48

#ifdef	__ARMEB__
	rev	x8,x8
	rev	x12,x12
	rev	x9,x9
	rev	x13,x13
#endif
	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
	and	x5,x9,#0x03ffffff
	ubfx	x6,x8,#26,#26
	ubfx	x7,x9,#26,#26
	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
	extr	x8,x12,x8,#52
	extr	x9,x13,x9,#52
	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
	fmov	$IN23_0,x4
	and	x8,x8,#0x03ffffff
	and	x9,x9,#0x03ffffff
	ubfx	x10,x12,#14,#26
	ubfx	x11,x13,#14,#26
	add	x12,$padbit,x12,lsr#40
	add	x13,$padbit,x13,lsr#40
	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
	fmov	$IN23_1,x6
	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
	fmov	$IN23_2,x8
	fmov	$IN23_3,x10
	fmov	$IN23_4,x12

	ldp	x8,x12,[$inp],#16	// inp[0:1]
	ldp	x9,x13,[$inp],#48

	ld1	{$R0,$R1,$S1,$R2},[x15],#64
	ld1	{$S2,$R3,$S3,$R4},[x15],#64
	ld1	{$S4},[x15]

#ifdef	__ARMEB__
	rev	x8,x8
	rev	x12,x12
	rev	x9,x9
	rev	x13,x13
#endif
	and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
	and	x5,x9,#0x03ffffff
	ubfx	x6,x8,#26,#26
	ubfx	x7,x9,#26,#26
	add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
	extr	x8,x12,x8,#52
	extr	x9,x13,x9,#52
	add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
	fmov	$IN01_0,x4
	and	x8,x8,#0x03ffffff
	and	x9,x9,#0x03ffffff
	ubfx	x10,x12,#14,#26
	ubfx	x11,x13,#14,#26
	add	x12,$padbit,x12,lsr#40
	add	x13,$padbit,x13,lsr#40
	add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
	fmov	$IN01_1,x6
	add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
	add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
	movi	$MASK.2d,#-1
	fmov	$IN01_2,x8
	fmov	$IN01_3,x10
	fmov	$IN01_4,x12
	ushr	$MASK.2d,$MASK.2d,#38

	b.ls	.Lskip_loop

.align	4
.Loop_neon:
	////////////////////////////////////////////////////////////////
	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
	//   \___________________/
	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
	//   \___________________/ \____________________/
	//
	// Note that we start with inp[2:3]*r^2. This is because it
	// doesn't depend on reduction in previous iteration.
	////////////////////////////////////////////////////////////////
	// d4 = h0*r4 + h1*r3   + h2*r2   + h3*r1   + h4*r0
	// d3 = h0*r3 + h1*r2   + h2*r1   + h3*r0   + h4*5*r4
	// d2 = h0*r2 + h1*r1   + h2*r0   + h3*5*r4 + h4*5*r3
	// d1 = h0*r1 + h1*r0   + h2*5*r4 + h3*5*r3 + h4*5*r2
	// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1

	subs	$len,$len,#64
	umull	$ACC4,$IN23_0,${R4}[2]
	csel	$in2,$zeros,$in2,lo
	umull	$ACC3,$IN23_0,${R3}[2]
	umull	$ACC2,$IN23_0,${R2}[2]
	 ldp	x8,x12,[$in2],#16	// inp[2:3] (or zero)
	umull	$ACC1,$IN23_0,${R1}[2]
	 ldp	x9,x13,[$in2],#48
	umull	$ACC0,$IN23_0,${R0}[2]
#ifdef	__ARMEB__
	 rev	x8,x8
	 rev	x12,x12
	 rev	x9,x9
	 rev	x13,x13
#endif

	umlal	$ACC4,$IN23_1,${R3}[2]
	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
	umlal	$ACC3,$IN23_1,${R2}[2]
	 and	x5,x9,#0x03ffffff
	umlal	$ACC2,$IN23_1,${R1}[2]
	 ubfx	x6,x8,#26,#26
	umlal	$ACC1,$IN23_1,${R0}[2]
	 ubfx	x7,x9,#26,#26
	umlal	$ACC0,$IN23_1,${S4}[2]
	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32

	umlal	$ACC4,$IN23_2,${R2}[2]
	 extr	x8,x12,x8,#52
	umlal	$ACC3,$IN23_2,${R1}[2]
	 extr	x9,x13,x9,#52
	umlal	$ACC2,$IN23_2,${R0}[2]
	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
	umlal	$ACC1,$IN23_2,${S4}[2]
	 fmov	$IN23_0,x4
	umlal	$ACC0,$IN23_2,${S3}[2]
	 and	x8,x8,#0x03ffffff

	umlal	$ACC4,$IN23_3,${R1}[2]
	 and	x9,x9,#0x03ffffff
	umlal	$ACC3,$IN23_3,${R0}[2]
	 ubfx	x10,x12,#14,#26
	umlal	$ACC2,$IN23_3,${S4}[2]
	 ubfx	x11,x13,#14,#26
	umlal	$ACC1,$IN23_3,${S3}[2]
	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
	umlal	$ACC0,$IN23_3,${S2}[2]
	 fmov	$IN23_1,x6

	add	$IN01_2,$IN01_2,$H2
	 add	x12,$padbit,x12,lsr#40
	umlal	$ACC4,$IN23_4,${R0}[2]
	 add	x13,$padbit,x13,lsr#40
	umlal	$ACC3,$IN23_4,${S4}[2]
	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
	umlal	$ACC2,$IN23_4,${S3}[2]
	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
	umlal	$ACC1,$IN23_4,${S2}[2]
	 fmov	$IN23_2,x8
	umlal	$ACC0,$IN23_4,${S1}[2]
	 fmov	$IN23_3,x10

	////////////////////////////////////////////////////////////////
	// (hash+inp[0:1])*r^4 and accumulate

	add	$IN01_0,$IN01_0,$H0
	 fmov	$IN23_4,x12
	umlal	$ACC3,$IN01_2,${R1}[0]
	 ldp	x8,x12,[$inp],#16	// inp[0:1]
	umlal	$ACC0,$IN01_2,${S3}[0]
	 ldp	x9,x13,[$inp],#48
	umlal	$ACC4,$IN01_2,${R2}[0]
	umlal	$ACC1,$IN01_2,${S4}[0]
	umlal	$ACC2,$IN01_2,${R0}[0]
#ifdef	__ARMEB__
	 rev	x8,x8
	 rev	x12,x12
	 rev	x9,x9
	 rev	x13,x13
#endif

	add	$IN01_1,$IN01_1,$H1
	umlal	$ACC3,$IN01_0,${R3}[0]
	umlal	$ACC4,$IN01_0,${R4}[0]
	 and	x4,x8,#0x03ffffff	// base 2^64 -> base 2^26
	umlal	$ACC2,$IN01_0,${R2}[0]
	 and	x5,x9,#0x03ffffff
	umlal	$ACC0,$IN01_0,${R0}[0]
	 ubfx	x6,x8,#26,#26
	umlal	$ACC1,$IN01_0,${R1}[0]
	 ubfx	x7,x9,#26,#26

	add	$IN01_3,$IN01_3,$H3
	 add	x4,x4,x5,lsl#32		// bfi	x4,x5,#32,#32
	umlal	$ACC3,$IN01_1,${R2}[0]
	 extr	x8,x12,x8,#52
	umlal	$ACC4,$IN01_1,${R3}[0]
	 extr	x9,x13,x9,#52
	umlal	$ACC0,$IN01_1,${S4}[0]
	 add	x6,x6,x7,lsl#32		// bfi	x6,x7,#32,#32
	umlal	$ACC2,$IN01_1,${R1}[0]
	 fmov	$IN01_0,x4
	umlal	$ACC1,$IN01_1,${R0}[0]
	 and	x8,x8,#0x03ffffff

	add	$IN01_4,$IN01_4,$H4
	 and	x9,x9,#0x03ffffff
	umlal	$ACC3,$IN01_3,${R0}[0]
	 ubfx	x10,x12,#14,#26
	umlal	$ACC0,$IN01_3,${S2}[0]
	 ubfx	x11,x13,#14,#26
	umlal	$ACC4,$IN01_3,${R1}[0]
	 add	x8,x8,x9,lsl#32		// bfi	x8,x9,#32,#32
	umlal	$ACC1,$IN01_3,${S3}[0]
	 fmov	$IN01_1,x6
	umlal	$ACC2,$IN01_3,${S4}[0]
	 add	x12,$padbit,x12,lsr#40

	umlal	$ACC3,$IN01_4,${S4}[0]
	 add	x13,$padbit,x13,lsr#40
	umlal	$ACC0,$IN01_4,${S1}[0]
	 add	x10,x10,x11,lsl#32	// bfi	x10,x11,#32,#32
	umlal	$ACC4,$IN01_4,${R0}[0]
	 add	x12,x12,x13,lsl#32	// bfi	x12,x13,#32,#32
	umlal	$ACC1,$IN01_4,${S2}[0]
	 fmov	$IN01_2,x8
	umlal	$ACC2,$IN01_4,${S3}[0]
	 fmov	$IN01_3,x10
	 fmov	$IN01_4,x12

	/////////////////////////////////////////////////////////////////
	// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
	// and P. Schwabe
	//
	// [see discussion in poly1305-armv4 module]

	ushr	$T0.2d,$ACC3,#26
	xtn	$H3,$ACC3
	 ushr	$T1.2d,$ACC0,#26
	 and	$ACC0,$ACC0,$MASK.2d
	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4
	bic	$H3,#0xfc,lsl#24	// &=0x03ffffff
	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1

	ushr	$T0.2d,$ACC4,#26
	xtn	$H4,$ACC4
	 ushr	$T1.2d,$ACC1,#26
	 xtn	$H1,$ACC1
	bic	$H4,#0xfc,lsl#24
	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2

	add	$ACC0,$ACC0,$T0.2d
	shl	$T0.2d,$T0.2d,#2
	 shrn	$T1.2s,$ACC2,#26
	 xtn	$H2,$ACC2
	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0
	 bic	$H1,#0xfc,lsl#24
	 add	$H3,$H3,$T1.2s		// h2 -> h3
	 bic	$H2,#0xfc,lsl#24

	shrn	$T0.2s,$ACC0,#26
	xtn	$H0,$ACC0
	 ushr	$T1.2s,$H3,#26
	 bic	$H3,#0xfc,lsl#24
	 bic	$H0,#0xfc,lsl#24
	add	$H1,$H1,$T0.2s		// h0 -> h1
	 add	$H4,$H4,$T1.2s		// h3 -> h4

	b.hi	.Loop_neon

.Lskip_loop:
	dup	$IN23_2,${IN23_2}[0]
	add	$IN01_2,$IN01_2,$H2

	////////////////////////////////////////////////////////////////
	// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1

	adds	$len,$len,#32
	b.ne	.Long_tail

	dup	$IN23_2,${IN01_2}[0]
	add	$IN23_0,$IN01_0,$H0
	add	$IN23_3,$IN01_3,$H3
	add	$IN23_1,$IN01_1,$H1
	add	$IN23_4,$IN01_4,$H4

.Long_tail:
	dup	$IN23_0,${IN23_0}[0]
	umull2	$ACC0,$IN23_2,${S3}
	umull2	$ACC3,$IN23_2,${R1}
	umull2	$ACC4,$IN23_2,${R2}
	umull2	$ACC2,$IN23_2,${R0}
	umull2	$ACC1,$IN23_2,${S4}

	dup	$IN23_1,${IN23_1}[0]
	umlal2	$ACC0,$IN23_0,${R0}
	umlal2	$ACC2,$IN23_0,${R2}
	umlal2	$ACC3,$IN23_0,${R3}
	umlal2	$ACC4,$IN23_0,${R4}
	umlal2	$ACC1,$IN23_0,${R1}

	dup	$IN23_3,${IN23_3}[0]
	umlal2	$ACC0,$IN23_1,${S4}
	umlal2	$ACC3,$IN23_1,${R2}
	umlal2	$ACC2,$IN23_1,${R1}
	umlal2	$ACC4,$IN23_1,${R3}
	umlal2	$ACC1,$IN23_1,${R0}

	dup	$IN23_4,${IN23_4}[0]
	umlal2	$ACC3,$IN23_3,${R0}
	umlal2	$ACC4,$IN23_3,${R1}
	umlal2	$ACC0,$IN23_3,${S2}
	umlal2	$ACC1,$IN23_3,${S3}
	umlal2	$ACC2,$IN23_3,${S4}

	umlal2	$ACC3,$IN23_4,${S4}
	umlal2	$ACC0,$IN23_4,${S1}
	umlal2	$ACC4,$IN23_4,${R0}
	umlal2	$ACC1,$IN23_4,${S2}
	umlal2	$ACC2,$IN23_4,${S3}

	b.eq	.Lshort_tail

	////////////////////////////////////////////////////////////////
	// (hash+inp[0:1])*r^4:r^3 and accumulate

	add	$IN01_0,$IN01_0,$H0
	umlal	$ACC3,$IN01_2,${R1}
	umlal	$ACC0,$IN01_2,${S3}
	umlal	$ACC4,$IN01_2,${R2}
	umlal	$ACC1,$IN01_2,${S4}
	umlal	$ACC2,$IN01_2,${R0}

	add	$IN01_1,$IN01_1,$H1
	umlal	$ACC3,$IN01_0,${R3}
	umlal	$ACC0,$IN01_0,${R0}
	umlal	$ACC4,$IN01_0,${R4}
	umlal	$ACC1,$IN01_0,${R1}
	umlal	$ACC2,$IN01_0,${R2}

	add	$IN01_3,$IN01_3,$H3
	umlal	$ACC3,$IN01_1,${R2}
	umlal	$ACC0,$IN01_1,${S4}
	umlal	$ACC4,$IN01_1,${R3}
	umlal	$ACC1,$IN01_1,${R0}
	umlal	$ACC2,$IN01_1,${R1}

	add	$IN01_4,$IN01_4,$H4
	umlal	$ACC3,$IN01_3,${R0}
	umlal	$ACC0,$IN01_3,${S2}
	umlal	$ACC4,$IN01_3,${R1}
	umlal	$ACC1,$IN01_3,${S3}
	umlal	$ACC2,$IN01_3,${S4}

	umlal	$ACC3,$IN01_4,${S4}
	umlal	$ACC0,$IN01_4,${S1}
	umlal	$ACC4,$IN01_4,${R0}
	umlal	$ACC1,$IN01_4,${S2}
	umlal	$ACC2,$IN01_4,${S3}

.Lshort_tail:
	////////////////////////////////////////////////////////////////
	// horizontal add

	addp	$ACC3,$ACC3,$ACC3
	 ldp	d8,d9,[sp,#16]		// meet ABI requirements
	addp	$ACC0,$ACC0,$ACC0
	 ldp	d10,d11,[sp,#32]
	addp	$ACC4,$ACC4,$ACC4
	 ldp	d12,d13,[sp,#48]
	addp	$ACC1,$ACC1,$ACC1
	 ldp	d14,d15,[sp,#64]
	addp	$ACC2,$ACC2,$ACC2

	////////////////////////////////////////////////////////////////
	// lazy reduction, but without narrowing

	ushr	$T0.2d,$ACC3,#26
	and	$ACC3,$ACC3,$MASK.2d
	 ushr	$T1.2d,$ACC0,#26
	 and	$ACC0,$ACC0,$MASK.2d

	add	$ACC4,$ACC4,$T0.2d	// h3 -> h4
	 add	$ACC1,$ACC1,$T1.2d	// h0 -> h1

	ushr	$T0.2d,$ACC4,#26
	and	$ACC4,$ACC4,$MASK.2d
	 ushr	$T1.2d,$ACC1,#26
	 and	$ACC1,$ACC1,$MASK.2d
	 add	$ACC2,$ACC2,$T1.2d	// h1 -> h2

	add	$ACC0,$ACC0,$T0.2d
	shl	$T0.2d,$T0.2d,#2
	 ushr	$T1.2d,$ACC2,#26
	 and	$ACC2,$ACC2,$MASK.2d
	add	$ACC0,$ACC0,$T0.2d	// h4 -> h0
	 add	$ACC3,$ACC3,$T1.2d	// h2 -> h3

	ushr	$T0.2d,$ACC0,#26
	and	$ACC0,$ACC0,$MASK.2d
	 ushr	$T1.2d,$ACC3,#26
	 and	$ACC3,$ACC3,$MASK.2d
	add	$ACC1,$ACC1,$T0.2d	// h0 -> h1
	 add	$ACC4,$ACC4,$T1.2d	// h3 -> h4

	////////////////////////////////////////////////////////////////
	// write the result, can be partially reduced

	st4	{$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
	st1	{$ACC4}[0],[$ctx]

.Lno_data_neon:
	ldr	x29,[sp],#80
	.inst	0xd50323bf		// autiasp
	ret
.size	poly1305_blocks_neon,.-poly1305_blocks_neon

.type	poly1305_emit_neon,%function
.align	5
poly1305_emit_neon:
	ldr	$is_base2_26,[$ctx,#24]
	cbz	$is_base2_26,poly1305_emit

	ldp	w10,w11,[$ctx]		// load hash value base 2^26
	ldp	w12,w13,[$ctx,#8]
	ldr	w14,[$ctx,#16]

	add	$h0,x10,x11,lsl#26	// base 2^26 -> base 2^64
	lsr	$h1,x12,#12
	adds	$h0,$h0,x12,lsl#52
	add	$h1,$h1,x13,lsl#14
	adc	$h1,$h1,xzr
	lsr	$h2,x14,#24
	adds	$h1,$h1,x14,lsl#40
	adc	$h2,$h2,xzr		// can be partially reduced...

	ldp	$t0,$t1,[$nonce]	// load nonce

	and	$d0,$h2,#-4		// ... so reduce
	add	$d0,$d0,$h2,lsr#2
	and	$h2,$h2,#3
	adds	$h0,$h0,$d0
	adcs	$h1,$h1,xzr
	adc	$h2,$h2,xzr

	adds	$d0,$h0,#5		// compare to modulus
	adcs	$d1,$h1,xzr
	adc	$d2,$h2,xzr

	tst	$d2,#-4			// see if it's carried/borrowed

	csel	$h0,$h0,$d0,eq
	csel	$h1,$h1,$d1,eq

#ifdef	__ARMEB__
	ror	$t0,$t0,#32		// flip nonce words
	ror	$t1,$t1,#32
#endif
	adds	$h0,$h0,$t0		// accumulate nonce
	adc	$h1,$h1,$t1
#ifdef	__ARMEB__
	rev	$h0,$h0			// flip output bytes
	rev	$h1,$h1
#endif
	stp	$h0,$h1,[$mac]		// write result

	ret
.size	poly1305_emit_neon,.-poly1305_emit_neon

.align	5
.Lzeros:
.long	0,0,0,0,0,0,0,0
.LOPENSSL_armcap_P:
#ifdef	__ILP32__
.long	OPENSSL_armcap_P-.
#else
.quad	OPENSSL_armcap_P-.
#endif
.asciz	"Poly1305 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align	2
___

foreach (split("\n",$code)) {
	s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/			or
	s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/	or
	(m/\bdup\b/ and (s/\.[24]s/.2d/g or 1))			or
	(m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1))	or
	(m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1))		or
	(m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1))		or
	(m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));

	s/\.[124]([sd])\[/.$1\[/;

	print $_,"\n";
}
close STDOUT or die "error closing STDOUT: $!";