|
|
|
|
|
|
|
|
|
|
|
#include <assert.h> |
|
#include <stddef.h> |
|
#include <stdint.h> |
|
|
|
#include <xnnpack/math.h> |
|
#include <xnnpack/fft.h> |
|
|
|
#include <arm_neon.h> |
|
|
|
static const int16_t xnn_table_fft256_samples4_twiddle[24] = { |
|
32767,0, 30273,-12539, 23170,-23170, 12539,-30273, |
|
32767,0, 23170,-23170, 0,-32767, -23170,-23170, |
|
32767,0, 12539,-30273, -23170,-23170, -30273, 12539, |
|
}; |
|
|
|
void xnn_cs16_bfly4_samples4_ukernel__neon( |
|
size_t batch, |
|
size_t samples, |
|
int16_t* data, |
|
const int16_t* twiddle, |
|
size_t stride) |
|
{ |
|
assert(batch != 0); |
|
assert(samples == sizeof(int16_t) * 8); |
|
assert(data != NULL); |
|
assert(stride == sizeof(int16_t) * 2 * 16); |
|
assert(twiddle != NULL); |
|
|
|
const int16x4_t vdiv4 = vdup_n_s16(8191); |
|
int16x4x2_t vtw1 = vld2_s16(xnn_table_fft256_samples4_twiddle); |
|
int16x4x2_t vtw2 = vld2_s16(xnn_table_fft256_samples4_twiddle + 8); |
|
int16x4x2_t vtw3 = vld2_s16(xnn_table_fft256_samples4_twiddle + 16); |
|
|
|
int16_t* data3 = data; |
|
do { |
|
int16_t* data0 = data3; |
|
int16_t* data1 = (int16_t*) ((uintptr_t) data0 + samples); |
|
int16_t* data2 = (int16_t*) ((uintptr_t) data1 + samples); |
|
data3 = (int16_t*) ((uintptr_t) data2 + samples); |
|
|
|
int16x4x2_t vout0 = vld2_s16(data0); |
|
int16x4x2_t vout1 = vld2_s16(data1); |
|
int16x4x2_t vout2 = vld2_s16(data2); |
|
int16x4x2_t vout3 = vld2_s16(data3); |
|
|
|
|
|
vout0.val[0] = vqrdmulh_s16(vout0.val[0], vdiv4); |
|
vout0.val[1] = vqrdmulh_s16(vout0.val[1], vdiv4); |
|
vout1.val[0] = vqrdmulh_s16(vout1.val[0], vdiv4); |
|
vout1.val[1] = vqrdmulh_s16(vout1.val[1], vdiv4); |
|
vout2.val[0] = vqrdmulh_s16(vout2.val[0], vdiv4); |
|
vout2.val[1] = vqrdmulh_s16(vout2.val[1], vdiv4); |
|
vout3.val[0] = vqrdmulh_s16(vout3.val[0], vdiv4); |
|
vout3.val[1] = vqrdmulh_s16(vout3.val[1], vdiv4); |
|
|
|
int32x4_t vacc0r = vmull_s16(vout1.val[0], vtw1.val[0]); |
|
int32x4_t vacc1r = vmull_s16(vout2.val[0], vtw2.val[0]); |
|
int32x4_t vacc2r = vmull_s16(vout3.val[0], vtw3.val[0]); |
|
int32x4_t vacc0i = vmull_s16(vout1.val[0], vtw1.val[1]); |
|
int32x4_t vacc1i = vmull_s16(vout2.val[0], vtw2.val[1]); |
|
int32x4_t vacc2i = vmull_s16(vout3.val[0], vtw3.val[1]); |
|
vacc0r = vmlsl_s16(vacc0r, vout1.val[1], vtw1.val[1]); |
|
vacc1r = vmlsl_s16(vacc1r, vout2.val[1], vtw2.val[1]); |
|
vacc2r = vmlsl_s16(vacc2r, vout3.val[1], vtw3.val[1]); |
|
vacc0i = vmlal_s16(vacc0i, vout1.val[1], vtw1.val[0]); |
|
vacc1i = vmlal_s16(vacc1i, vout2.val[1], vtw2.val[0]); |
|
vacc2i = vmlal_s16(vacc2i, vout3.val[1], vtw3.val[0]); |
|
int16x4_t vtmp0r = vrshrn_n_s32(vacc0r, 15); |
|
int16x4_t vtmp1r = vrshrn_n_s32(vacc1r, 15); |
|
int16x4_t vtmp2r = vrshrn_n_s32(vacc2r, 15); |
|
int16x4_t vtmp0i = vrshrn_n_s32(vacc0i, 15); |
|
int16x4_t vtmp1i = vrshrn_n_s32(vacc1i, 15); |
|
int16x4_t vtmp2i = vrshrn_n_s32(vacc2i, 15); |
|
|
|
const int16x4_t vtmp4r = vsub_s16(vtmp0r, vtmp2r); |
|
const int16x4_t vtmp4i = vsub_s16(vtmp0i, vtmp2i); |
|
const int16x4_t vtmp3r = vadd_s16(vtmp0r, vtmp2r); |
|
const int16x4_t vtmp3i = vadd_s16(vtmp0i, vtmp2i); |
|
|
|
const int16x4_t vtmp5r = vsub_s16(vout0.val[0], vtmp1r); |
|
const int16x4_t vtmp5i = vsub_s16(vout0.val[1], vtmp1i); |
|
vout0.val[0] = vadd_s16(vout0.val[0], vtmp1r); |
|
vout0.val[1] = vadd_s16(vout0.val[1], vtmp1i); |
|
|
|
vout2.val[0] = vsub_s16(vout0.val[0], vtmp3r); |
|
vout2.val[1] = vsub_s16(vout0.val[1], vtmp3i); |
|
vout0.val[0] = vadd_s16(vout0.val[0], vtmp3r); |
|
vout0.val[1] = vadd_s16(vout0.val[1], vtmp3i); |
|
|
|
vout1.val[0] = vadd_s16(vtmp5r, vtmp4i); |
|
vout1.val[1] = vsub_s16(vtmp5i, vtmp4r); |
|
vout3.val[0] = vsub_s16(vtmp5r, vtmp4i); |
|
vout3.val[1] = vadd_s16(vtmp5i, vtmp4r); |
|
|
|
vst2_s16(data0, vout0); |
|
vst2_s16(data1, vout1); |
|
vst2_s16(data2, vout2); |
|
vst2_s16(data3, vout3); data3 += 8; |
|
|
|
} while (--batch != 0); |
|
} |
|
|
