text: // Copyright 2018 The Go Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style
  // license that can be found in the LICENSE file.
  
  // +build !gccgo,!purego
  
  #include "go_asm.h"
  #include "textflag.h"
  
  // This is an implementation of the ChaCha20 encryption algorithm as
  // specified in RFC 7539. It uses vector instructions to compute
  // 4 keystream blocks in parallel (256 bytes) which are then XORed
  // with the bytes in the input slice.
  
  GLOBL ·constants<>(SB), RODATA|NOPTR, $32
  // BSWAP: swap bytes in each 4-byte element
  DATA ·constants<>+0x00(SB)/4, $0x03020100
  DATA ·constants<>+0x04(SB)/4, $0x07060504
  DATA ·constants<>+0x08(SB)/4, $0x0b0a0908
  DATA ·constants<>+0x0c(SB)/4, $0x0f0e0d0c
  // J0: [j0, j1, j2, j3]
  DATA ·constants<>+0x10(SB)/4, $0x61707865
  DATA ·constants<>+0x14(SB)/4, $0x3320646e
  DATA ·constants<>+0x18(SB)/4, $0x79622d32
  DATA ·constants<>+0x1c(SB)/4, $0x6b206574
  
  #define BSWAP V5
  #define J0    V6
  #define KEY0  V7
  #define KEY1  V8
  #define NONCE V9
  #define CTR   V10
  #define M0    V11
  #define M1    V12
  #define M2    V13
  #define M3    V14
  #define INC   V15
  #define X0    V16
  #define X1    V17
  #define X2    V18
  #define X3    V19
  #define X4    V20
  #define X5    V21
  #define X6    V22
  #define X7    V23
  #define X8    V24
  #define X9    V25
  #define X10   V26
  #define X11   V27
  #define X12   V28
  #define X13   V29
  #define X14   V30
  #define X15   V31
  
  #define NUM_ROUNDS 20
  
  #define ROUND4(a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3) \
  	VAF    a1, a0, a0  \
  	VAF    b1, b0, b0  \
  	VAF    c1, c0, c0  \
  	VAF    d1, d0, d0  \
  	VX     a0, a2, a2  \
  	VX     b0, b2, b2  \
  	VX     c0, c2, c2  \
  	VX     d0, d2, d2  \
  	VERLLF $16, a2, a2 \
  	VERLLF $16, b2, b2 \
  	VERLLF $16, c2, c2 \
  	VERLLF $16, d2, d2 \
  	VAF    a2, a3, a3  \
  	VAF    b2, b3, b3  \
  	VAF    c2, c3, c3  \
  	VAF    d2, d3, d3  \
  	VX     a3, a1, a1  \
  	VX     b3, b1, b1  \
  	VX     c3, c1, c1  \
  	VX     d3, d1, d1  \
  	VERLLF $12, a1, a1 \
  	VERLLF $12, b1, b1 \
  	VERLLF $12, c1, c1 \
  	VERLLF $12, d1, d1 \
  	VAF    a1, a0, a0  \
  	VAF    b1, b0, b0  \
  	VAF    c1, c0, c0  \
  	VAF    d1, d0, d0  \
  	VX     a0, a2, a2  \
  	VX     b0, b2, b2  \
  	VX     c0, c2, c2  \
  	VX     d0, d2, d2  \
  	VERLLF $8, a2, a2  \
  	VERLLF $8, b2, b2  \
  	VERLLF $8, c2, c2  \
  	VERLLF $8, d2, d2  \
  	VAF    a2, a3, a3  \
  	VAF    b2, b3, b3  \
  	VAF    c2, c3, c3  \
  	VAF    d2, d3, d3  \
  	VX     a3, a1, a1  \
  	VX     b3, b1, b1  \
  	VX     c3, c1, c1  \
  	VX     d3, d1, d1  \
  	VERLLF $7, a1, a1  \
  	VERLLF $7, b1, b1  \
  	VERLLF $7, c1, c1  \
  	VERLLF $7, d1, d1
  
  #define PERMUTE(mask, v0, v1, v2, v3) \
  	VPERM v0, v0, mask, v0 \
  	VPERM v1, v1, mask, v1 \
  	VPERM v2, v2, mask, v2 \
  	VPERM v3, v3, mask, v3
  
  #define ADDV(x, v0, v1, v2, v3) \
  	VAF x, v0, v0 \
  	VAF x, v1, v1 \
  	VAF x, v2, v2 \
  	VAF x, v3, v3
  
  #define XORV(off, dst, src, v0, v1, v2, v3) \
  	VLM  off(src), M0, M3          \
  	PERMUTE(BSWAP, v0, v1, v2, v3) \
  	VX   v0, M0, M0                \
  	VX   v1, M1, M1                \
  	VX   v2, M2, M2                \
  	VX   v3, M3, M3                \
  	VSTM M0, M3, off(dst)
  
  #define SHUFFLE(a, b, c, d, t, u, v, w) \
  	VMRHF a, c, t \ // t = {a[0], c[0], a[1], c[1]}
  	VMRHF b, d, u \ // u = {b[0], d[0], b[1], d[1]}
  	VMRLF a, c, v \ // v = {a[2], c[2], a[3], c[3]}
  	VMRLF b, d, w \ // w = {b[2], d[2], b[3], d[3]}
  	VMRHF t, u, a \ // a = {a[0], b[0], c[0], d[0]}
  	VMRLF t, u, b \ // b = {a[1], b[1], c[1], d[1]}
  	VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]}
  	VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]}
  
  // func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
  TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
  	MOVD $·constants<>(SB), R1
  	MOVD dst+0(FP), R2         // R2=&dst[0]
  	LMG  src+24(FP), R3, R4    // R3=&src[0] R4=len(src)
  	MOVD key+48(FP), R5        // R5=key
  	MOVD nonce+56(FP), R6      // R6=nonce
  	MOVD counter+64(FP), R7    // R7=counter
  
  	// load BSWAP and J0
  	VLM (R1), BSWAP, J0
  
  	// setup
  	MOVD  $95, R0
  	VLM   (R5), KEY0, KEY1
  	VLL   R0, (R6), NONCE
  	VZERO M0
  	VLEIB $7, $32, M0
  	VSRLB M0, NONCE, NONCE
  
  	// initialize counter values
  	VLREPF (R7), CTR
  	VZERO  INC
  	VLEIF  $1, $1, INC
  	VLEIF  $2, $2, INC
  	VLEIF  $3, $3, INC
  	VAF    INC, CTR, CTR
  	VREPIF $4, INC
  
  chacha:
  	VREPF $0, J0, X0
  	VREPF $1, J0, X1
  	VREPF $2, J0, X2
  	VREPF $3, J0, X3
  	VREPF $0, KEY0, X4
  	VREPF $1, KEY0, X5
  	VREPF $2, KEY0, X6
  	VREPF $3, KEY0, X7
  	VREPF $0, KEY1, X8
  	VREPF $1, KEY1, X9
  	VREPF $2, KEY1, X10
  	VREPF $3, KEY1, X11
  	VLR   CTR, X12
  	VREPF $1, NONCE, X13
  	VREPF $2, NONCE, X14
  	VREPF $3, NONCE, X15
  
  	MOVD $(NUM_ROUNDS/2), R1
  
  loop:
  	ROUND4(X0, X4, X12,  X8, X1, X5, X13,  X9, X2, X6, X14, X10, X3, X7, X15, X11)
  	ROUND4(X0, X5, X15, X10, X1, X6, X12, X11, X2, X7, X13, X8,  X3, X4, X14, X9)
  
  	ADD $-1, R1
  	BNE loop
  
  	// decrement length
  	ADD $-256, R4
  
  	// rearrange vectors
  	SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3)
  	ADDV(J0, X0, X1, X2, X3)
  	SHUFFLE(X4, X5, X6, X7, M0, M1, M2, M3)
  	ADDV(KEY0, X4, X5, X6, X7)
  	SHUFFLE(X8, X9, X10, X11, M0, M1, M2, M3)
  	ADDV(KEY1, X8, X9, X10, X11)
  	VAF CTR, X12, X12
  	SHUFFLE(X12, X13, X14, X15, M0, M1, M2, M3)
  	ADDV(NONCE, X12, X13, X14, X15)
  
  	// increment counters
  	VAF INC, CTR, CTR
  
  	// xor keystream with plaintext
  	XORV(0*64, R2, R3, X0, X4,  X8, X12)
  	XORV(1*64, R2, R3, X1, X5,  X9, X13)
  	XORV(2*64, R2, R3, X2, X6, X10, X14)
  	XORV(3*64, R2, R3, X3, X7, X11, X15)
  
  	// increment pointers
  	MOVD $256(R2), R2
  	MOVD $256(R3), R3
  
  	CMPBNE  R4, $0, chacha
  
  	VSTEF $0, CTR, (R7)
  	RET
  