File size: 6,549 Bytes
8b7c501 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 |
// Copyright 2019 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
#include <xnnpack/assembly.h>
# void xnn_f32_dwconv_minmax_ukernel_9p4c__asm_aarch64_neonfma(
# size_t channels, x0, x20
# size_t output_width, x1
# const float** input, x2
# const float* weights, x3, x19
# float* output, x4
# intptr_t input_stride, x5
# size_t output_increment, x6
# size_t input_offset, x7
# const float* zero, [sp + 80] -> x17
# const xnn_f32_minmax_params params [sp + 88] -> (x16)
# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
# inputs
# i0 x8 v21
# i1 x9 v22
# i2 x10 v23
# i3 x11 v24
# i4 x12 v25
# i5 x13 v26
# i6 x14 v27
# i7 x15 v28
# i8 x16 v29
# weights
# x19 v0 (acc) v1 v2 v3 v4 v5 v6 v7 v16 v17
# Clamp v30 v31
# unused v18 v19 v20
BEGIN_FUNCTION xnn_f32_dwconv_minmax_ukernel_9p4c__asm_aarch64_neonfma
# Load zero, params pointer
LDP x17, x16, [sp]
# Save x19,x20 on stack
STP x19, x20, [sp, -16]!
# Load min/max values
LD2R {v30.4s, v31.4s}, [x16]
0:
# Load 9 input pointers
LDP x8, x9, [x2]
LDP x10, x11, [x2, 16]
LDP x12, x13, [x2, 32]
LDP x14, x15, [x2, 48]
LDR x16, [x2, 64]
CMP x8, x17 // if i0 == zero
ADD x8, x8, x7 // i0 += input_offset
CSEL x8, x17, x8, EQ // i0 = zero, else += i0 + input_offset
CMP x9, x17 // if i1 == zero
ADD x9, x9, x7 // i1 += input_offset
CSEL x9, x17, x9, EQ // i1 = zero, else += i1 + input_offset
CMP x10, x17 // if i2 == zero
ADD x10, x10, x7 // i2 += input_offset
CSEL x10, x17, x10, EQ // i2 = zero, else += i2 + input_offset
CMP x11, x17 // if i3 == zero
ADD x11, x11, x7 // i3 += input_offset
CSEL x11, x17, x11, EQ // i3 = zero, else += i3 + input_offset
CMP x12, x17 // if i4 == zero
ADD x12, x12, x7 // i4 += input_offset
CSEL x12, x17, x12, EQ // i4 = zero, else += i4 + input_offset
CMP x13, x17 // if i5 == zero
ADD x13, x13, x7 // i5 += input_offset
CSEL x13, x17, x13, EQ // i5 = zero, else += i5 + input_offset
CMP x14, x17 // if i6 == zero
ADD x14, x14, x7 // i6 += input_offset
CSEL x14, x17, x14, EQ // i6 = zero, else += i6 + input_offset
CMP x15, x17 // if i7 == zero
ADD x15, x15, x7 // i7 += input_offset
CSEL x15, x17, x15, EQ // i7 = zero, else += i7 + input_offset
CMP x16, x17 // if i8 == zero
ADD x16, x16, x7 // i8 += input_offset
CSEL x16, x17, x16, EQ // i8 = zero, else += i8 + input_offset
# input += input_stride
ADD x2, x2, x5
# x20 := c = channels
# c -= 4
SUBS x20, x0, 4
# x19 := w = weights
MOV x19, x3
# skip main loop if c <= 4
B.LO 2f
1:
LDR q21, [x8], 16 // load 9 inputs
LDP q0, q1, [x19], 32 // load bias and 9 weights
LDR q22, [x9], 16
LDR q23, [x10], 16
LDR q24, [x11], 16
LDR q25, [x12], 16
LDR q26, [x13], 16
LDR q27, [x14], 16
LDR q28, [x15], 16
LDR q29, [x16], 16
LDP q2, q3, [x19], 32
LDP q4, q5, [x19], 32
LDP q6, q7, [x19], 32
LDP q16, q17, [x19], 32
FMLA v0.4S, v1.4S, v21.4S
FMLA v0.4S, v2.4S, v22.4S
FMLA v0.4S, v3.4S, v23.4S
FMLA v0.4S, v4.4S, v24.4S
FMLA v0.4S, v5.4S, v25.4S
FMLA v0.4S, v6.4S, v26.4S
FMLA v0.4S, v7.4S, v27.4S
FMLA v0.4S, v16.4S, v28.4S
FMLA v0.4S, v17.4S, v29.4S
SUBS x20, x20, 4
FMAX v0.4S, v0.4S, v30.4S
FMIN v0.4S, v0.4S, v31.4S
STR q0, [x4], 16
B.HS 1b
2:
# Is there a remainder?- 1 to 3 channels
TST x20, 3
B.EQ 4f
LDR q21, [x8], 16 // load 9 inputs
LDP q0, q1, [x19], 32 // load bias and 9 weights
LDR q22, [x9], 16
LDR q23, [x10], 16
LDR q24, [x11], 16
LDR q25, [x12], 16
LDR q26, [x13], 16
LDR q27, [x14], 16
LDR q28, [x15], 16
LDR q29, [x16], 16
LDP q2, q3, [x19], 32
LDP q4, q5, [x19], 32
LDP q6, q7, [x19], 32
LDP q16, q17, [x19], 32
FMLA v0.4S, v1.4S, v21.4S
FMLA v0.4S, v2.4S, v22.4S
FMLA v0.4S, v3.4S, v23.4S
FMLA v0.4S, v4.4S, v24.4S
FMLA v0.4S, v5.4S, v25.4S
FMLA v0.4S, v6.4S, v26.4S
FMLA v0.4S, v7.4S, v27.4S
FMLA v0.4S, v16.4S, v28.4S
FMLA v0.4S, v17.4S, v29.4S
FMAX v0.4S, v0.4S, v30.4S
FMIN v0.4S, v0.4S, v31.4S
TBZ x20, 1, 3f
STR d0, [x4], 8
DUP d0, v0.D[1]
TBZ x20, 0, 4f
3:
STR s0, [x4], 4
4:
# output_width -= 1
SUBS x1, x1, 1
# output += output_increment
ADD x4, x4, x6
# process next pixel if output_width != 0
B.NE 0b
# Restore x19,x20 from stack
LDP x19, x20, [sp], 16
RET
END_FUNCTION xnn_f32_dwconv_minmax_ukernel_9p4c__asm_aarch64_neonfma
#ifdef __ELF__
.section ".note.GNU-stack","",%progbits
#endif
|