File size: 10,626 Bytes
8b7c501 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 |
// Auto-generated file. Do not edit!
// Template: src/f16-vsigmoid/neonfp16arith.c.in
// Generator: tools/xngen
//
// Copyright 2022 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
#include <assert.h>
#include <arm_neon.h>
#include <xnnpack/common.h>
#include <xnnpack/vunary.h>
void xnn_f16_vsigmoid_ukernel__neonfp16arith_rr2_p2_nr1recps_x48(
size_t batch,
const void* input,
void* output,
const union xnn_f16_sigmoid_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
{
assert(batch != 0);
assert(batch % sizeof(uint16_t) == 0);
assert(input != NULL);
assert(output != NULL);
const float16x8_t vmagic_bias = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->fp16arith_rr2_p2.magic_bias));
const float16x8_t vminus_log2e = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->fp16arith_rr2_p2.minus_log2e));
const float16x8_t vln2_hi = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->fp16arith_rr2_p2.ln2_hi));
const float16x8_t vln2_lo = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->fp16arith_rr2_p2.ln2_lo));
const float16x8_t vc2 = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->fp16arith_rr2_p2.c2));
const float16x8_t vc1 = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->fp16arith_rr2_p2.c1));
const float16x8_t vone = vreinterpretq_f16_u16(vmovq_n_u16(UINT16_C(0x3C00))); // 1.0h
const float16x8_t vdenorm_cutoff = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->fp16arith_rr2_p2.denorm_cutoff));
const uint16_t* i = (const uint16_t*) input;
uint16_t* o = (uint16_t*) output;
for (; batch >= 48 * sizeof(uint16_t); batch -= 48 * sizeof(uint16_t)) {
const float16x8_t vx0 = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vx1 = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vx2 = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vx3 = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vx4 = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vx5 = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vz0 = vabsq_f16(vx0);
const float16x8_t vz1 = vabsq_f16(vx1);
const float16x8_t vz2 = vabsq_f16(vx2);
const float16x8_t vz3 = vabsq_f16(vx3);
const float16x8_t vz4 = vabsq_f16(vx4);
const float16x8_t vz5 = vabsq_f16(vx5);
float16x8_t vn0 = vfmaq_f16(vmagic_bias, vz0, vminus_log2e);
float16x8_t vn1 = vfmaq_f16(vmagic_bias, vz1, vminus_log2e);
float16x8_t vn2 = vfmaq_f16(vmagic_bias, vz2, vminus_log2e);
float16x8_t vn3 = vfmaq_f16(vmagic_bias, vz3, vminus_log2e);
float16x8_t vn4 = vfmaq_f16(vmagic_bias, vz4, vminus_log2e);
float16x8_t vn5 = vfmaq_f16(vmagic_bias, vz5, vminus_log2e);
const float16x8_t vs0 = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn0), 10));
const float16x8_t vs1 = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn1), 10));
const float16x8_t vs2 = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn2), 10));
const float16x8_t vs3 = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn3), 10));
const float16x8_t vs4 = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn4), 10));
const float16x8_t vs5 = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn5), 10));
vn0 = vsubq_f16(vn0, vmagic_bias);
vn1 = vsubq_f16(vn1, vmagic_bias);
vn2 = vsubq_f16(vn2, vmagic_bias);
vn3 = vsubq_f16(vn3, vmagic_bias);
vn4 = vsubq_f16(vn4, vmagic_bias);
vn5 = vsubq_f16(vn5, vmagic_bias);
float16x8_t vt0 = vfmaq_f16(vz0, vn0, vln2_hi);
float16x8_t vt1 = vfmaq_f16(vz1, vn1, vln2_hi);
float16x8_t vt2 = vfmaq_f16(vz2, vn2, vln2_hi);
float16x8_t vt3 = vfmaq_f16(vz3, vn3, vln2_hi);
float16x8_t vt4 = vfmaq_f16(vz4, vn4, vln2_hi);
float16x8_t vt5 = vfmaq_f16(vz5, vn5, vln2_hi);
vt0 = vfmaq_f16(vt0, vn0, vln2_lo);
vt1 = vfmaq_f16(vt1, vn1, vln2_lo);
vt2 = vfmaq_f16(vt2, vn2, vln2_lo);
vt3 = vfmaq_f16(vt3, vn3, vln2_lo);
vt4 = vfmaq_f16(vt4, vn4, vln2_lo);
vt5 = vfmaq_f16(vt5, vn5, vln2_lo);
const float16x8_t vp0 = vfmaq_f16(vc1, vc2, vt0);
const float16x8_t vp1 = vfmaq_f16(vc1, vc2, vt1);
const float16x8_t vp2 = vfmaq_f16(vc1, vc2, vt2);
const float16x8_t vp3 = vfmaq_f16(vc1, vc2, vt3);
const float16x8_t vp4 = vfmaq_f16(vc1, vc2, vt4);
const float16x8_t vp5 = vfmaq_f16(vc1, vc2, vt5);
vt0 = vmulq_f16(vt0, vs0);
vt1 = vmulq_f16(vt1, vs1);
vt2 = vmulq_f16(vt2, vs2);
vt3 = vmulq_f16(vt3, vs3);
vt4 = vmulq_f16(vt4, vs4);
vt5 = vmulq_f16(vt5, vs5);
const float16x8_t ve0 = vfmaq_f16(vs0, vp0, vt0);
const float16x8_t ve1 = vfmaq_f16(vs1, vp1, vt1);
const float16x8_t ve2 = vfmaq_f16(vs2, vp2, vt2);
const float16x8_t ve3 = vfmaq_f16(vs3, vp3, vt3);
const float16x8_t ve4 = vfmaq_f16(vs4, vp4, vt4);
const float16x8_t ve5 = vfmaq_f16(vs5, vp5, vt5);
const float16x8_t vd0 = vaddq_f16(ve0, vone);
const float16x8_t vd1 = vaddq_f16(ve1, vone);
const float16x8_t vd2 = vaddq_f16(ve2, vone);
const float16x8_t vd3 = vaddq_f16(ve3, vone);
const float16x8_t vd4 = vaddq_f16(ve4, vone);
const float16x8_t vd5 = vaddq_f16(ve5, vone);
float16x8_t vr0 = vrecpeq_f16(vd0);
float16x8_t vr1 = vrecpeq_f16(vd1);
float16x8_t vr2 = vrecpeq_f16(vd2);
float16x8_t vr3 = vrecpeq_f16(vd3);
float16x8_t vr4 = vrecpeq_f16(vd4);
float16x8_t vr5 = vrecpeq_f16(vd5);
const float16x8_t vadj0 = vrecpsq_f16(vr0, vd0);
const float16x8_t vadj1 = vrecpsq_f16(vr1, vd1);
const float16x8_t vadj2 = vrecpsq_f16(vr2, vd2);
const float16x8_t vadj3 = vrecpsq_f16(vr3, vd3);
const float16x8_t vadj4 = vrecpsq_f16(vr4, vd4);
const float16x8_t vadj5 = vrecpsq_f16(vr5, vd5);
vr0 = vmulq_f16(vr0, vadj0);
vr1 = vmulq_f16(vr1, vadj1);
vr2 = vmulq_f16(vr2, vadj2);
vr3 = vmulq_f16(vr3, vadj3);
vr4 = vmulq_f16(vr4, vadj4);
vr5 = vmulq_f16(vr5, vadj5);
float16x8_t vf0 = vmulq_f16(ve0, vr0);
float16x8_t vf1 = vmulq_f16(ve1, vr1);
float16x8_t vf2 = vmulq_f16(ve2, vr2);
float16x8_t vf3 = vmulq_f16(ve3, vr3);
float16x8_t vf4 = vmulq_f16(ve4, vr4);
float16x8_t vf5 = vmulq_f16(ve5, vr5);
vf0 = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf0), vcagtq_f16(vx0, vdenorm_cutoff)));
vf1 = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf1), vcagtq_f16(vx1, vdenorm_cutoff)));
vf2 = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf2), vcagtq_f16(vx2, vdenorm_cutoff)));
vf3 = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf3), vcagtq_f16(vx3, vdenorm_cutoff)));
vf4 = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf4), vcagtq_f16(vx4, vdenorm_cutoff)));
vf5 = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf5), vcagtq_f16(vx5, vdenorm_cutoff)));
const uint16x8_t vm0 = vcltq_f16(vx0, vreinterpretq_f16_u16(vmovq_n_u16(0)));
const uint16x8_t vm1 = vcltq_f16(vx1, vreinterpretq_f16_u16(vmovq_n_u16(0)));
const uint16x8_t vm2 = vcltq_f16(vx2, vreinterpretq_f16_u16(vmovq_n_u16(0)));
const uint16x8_t vm3 = vcltq_f16(vx3, vreinterpretq_f16_u16(vmovq_n_u16(0)));
const uint16x8_t vm4 = vcltq_f16(vx4, vreinterpretq_f16_u16(vmovq_n_u16(0)));
const uint16x8_t vm5 = vcltq_f16(vx5, vreinterpretq_f16_u16(vmovq_n_u16(0)));
vf0 = vbslq_f16(vm0, vf0, vsubq_f16(vone, vf0));
vf1 = vbslq_f16(vm1, vf1, vsubq_f16(vone, vf1));
vf2 = vbslq_f16(vm2, vf2, vsubq_f16(vone, vf2));
vf3 = vbslq_f16(vm3, vf3, vsubq_f16(vone, vf3));
vf4 = vbslq_f16(vm4, vf4, vsubq_f16(vone, vf4));
vf5 = vbslq_f16(vm5, vf5, vsubq_f16(vone, vf5));
vst1q_u16(o, vreinterpretq_u16_f16(vf0)); o += 8;
vst1q_u16(o, vreinterpretq_u16_f16(vf1)); o += 8;
vst1q_u16(o, vreinterpretq_u16_f16(vf2)); o += 8;
vst1q_u16(o, vreinterpretq_u16_f16(vf3)); o += 8;
vst1q_u16(o, vreinterpretq_u16_f16(vf4)); o += 8;
vst1q_u16(o, vreinterpretq_u16_f16(vf5)); o += 8;
}
for (; batch >= 8 * sizeof(uint16_t); batch -= 8 * sizeof(uint16_t)) {
const float16x8_t vx = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
const float16x8_t vz = vabsq_f16(vx);
float16x8_t vn = vfmaq_f16(vmagic_bias, vz, vminus_log2e);
const float16x8_t vs = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn), 10));
vn = vsubq_f16(vn, vmagic_bias);
float16x8_t vt = vfmaq_f16(vz, vn, vln2_hi);
vt = vfmaq_f16(vt, vn, vln2_lo);
const float16x8_t vp = vfmaq_f16(vc1, vc2, vt);
vt = vmulq_f16(vt, vs);
const float16x8_t ve = vfmaq_f16(vs, vp, vt);
const float16x8_t vd = vaddq_f16(ve, vone);
float16x8_t vr = vrecpeq_f16(vd);
const float16x8_t vadj = vrecpsq_f16(vr, vd);
vr = vmulq_f16(vr, vadj);
float16x8_t vf = vmulq_f16(ve, vr);
vf = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf), vcagtq_f16(vx, vdenorm_cutoff)));
const uint16x8_t vm = vcltq_f16(vx, vreinterpretq_f16_u16(vmovq_n_u16(0)));
vf = vbslq_f16(vm, vf, vsubq_f16(vone, vf));
vst1q_u16(o, vreinterpretq_u16_f16(vf)); o += 8;
}
if XNN_UNLIKELY(batch != 0) {
const float16x8_t vx = vreinterpretq_f16_u16(vld1q_u16(i));
const float16x8_t vz = vabsq_f16(vx);
float16x8_t vn = vfmaq_f16(vmagic_bias, vz, vminus_log2e);
const float16x8_t vs = vreinterpretq_f16_s16(vshlq_n_s16(vreinterpretq_s16_f16(vn), 10));
vn = vsubq_f16(vn, vmagic_bias);
float16x8_t vt = vfmaq_f16(vz, vn, vln2_hi);
vt = vfmaq_f16(vt, vn, vln2_lo);
const float16x8_t vp = vfmaq_f16(vc1, vc2, vt);
vt = vmulq_f16(vt, vs);
const float16x8_t ve = vfmaq_f16(vs, vp, vt);
const float16x8_t vd = vaddq_f16(ve, vone);
float16x8_t vr = vrecpeq_f16(vd);
const float16x8_t vadj = vrecpsq_f16(vr, vd);
vr = vmulq_f16(vr, vadj);
float16x8_t vf = vmulq_f16(ve, vr);
vf = vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(vf), vcagtq_f16(vx, vdenorm_cutoff)));
const uint16x8_t vm = vcltq_f16(vx, vreinterpretq_f16_u16(vmovq_n_u16(0)));
vf = vbslq_f16(vm, vf, vsubq_f16(vone, vf));
float16x4_t vf_lo = vget_low_f16(vf);
if (batch & (4 * sizeof(uint16_t))) {
vst1_u16(o, vreinterpret_u16_f16(vf_lo)); o += 4;
vf_lo = vget_high_f16(vf);
}
if (batch & (2 * sizeof(uint16_t))) {
vst1_lane_u32((void*) o, vreinterpret_u32_f16(vf_lo), 0); o += 2;
vf_lo = vext_f16(vf_lo, vf_lo, 2);
}
if (batch & (1 * sizeof(uint16_t))) {
vst1_lane_u16(o, vreinterpret_u16_f16(vf_lo), 0);
}
}
}
|