Repository: ggerganov/llama.cpp
File: ggml/src/ggml-cpu/arch/arm/quants.c (C)

Code before:

            if (j == 0) {
                qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);
                qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);
            }

        }
        sum += d * isum;

    }

    *s = sum;

#else
    UNUSED(kmask1);
    UNUSED(kmask2);
    UNUSED(x);
    UNUSED(y);
    UNUSED(nb);
    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
#endif

}

#ifdef __ARM_FEATURE_SVE
static inline svuint32_t ggml_decode_q4scales_and_mins_for_mmla(const uint32_t *vx_scales) {
  const svbool_t pg_all   = svptrue_pat_b32(SV_VL4);
  const svbool_t pg_false = svpfalse_b();            // 0x0000
  const svbool_t pg_lo_8  = svwhilelt_b8_s32(0,  8); // 0x00ff
  const svbool_t pg_odd   = svzip1_b32(pg_false, pg_lo_8);

  svuint32_t vutmp_hi, vutmp_lo;
  svuint32_t vx01 = svld1_u32(pg_lo_8, vx_scales);
  vutmp_hi = svzip1_u32(vx01, vx01);
  vutmp_hi = svlsr_n_u32_m(pg_odd, vutmp_hi, 2);
  vutmp_hi = svreinterpret_u32_u64(svand_n_u64_x(pg_all, svreinterpret_u64_u32(vutmp_hi), UINT64_C(0x303030303f3f3f3f)));
  const svuint32_t vx2 = svdup_u32(vx_scales[2]);
  vutmp_lo = svlsr_u32_x(pg_all, vx2, svreinterpret_u32_s32(svindex_s32(-2, 2)));
  vutmp_lo = svand_n_u32_z(pg_odd, vutmp_lo, UINT32_C(0x0f0f0f0f));
  svuint32_t vutmp = svorr_u32_z(pg_all, vutmp_hi, vutmp_lo);
  return vutmp;
}
#endif

void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
    assert(n % QK_K == 0);
#ifdef __ARM_FEATURE_MATMUL_INT8
    assert((nrc == 2) || (nrc == 1));
#else
    assert(nrc == 1);
#endif

Review comment (suggestion):
```suggestion
static inline svuint32_t ggml_decode_q4scales_and_mins_for_mmla(const uint32_t * vx_scales) {
```

Code after:

            if (j == 0) {
                qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);
                qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);
            }

        }
        sum += d * isum;

    }

    *s = sum;

#else
    UNUSED(kmask1);
    UNUSED(kmask2);
    UNUSED(x);
    UNUSED(y);
    UNUSED(nb);
    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
#endif

}

#ifdef __ARM_FEATURE_SVE
static inline svuint32_t ggml_decode_q4scales_and_mins_for_mmla(const uint32_t * vx_scales) {
    const svbool_t pg_all   = svptrue_pat_b32(SV_VL4);
    const svbool_t pg_false = svpfalse_b();            // 0x0000
    const svbool_t pg_lo_8  = svwhilelt_b8_s32(0,  8); // 0x00ff
    const svbool_t pg_odd   = svzip1_b32(pg_false, pg_lo_8);

    svuint32_t vutmp_hi, vutmp_lo;
    svuint32_t vx01 = svld1_u32(pg_lo_8, vx_scales);
    vutmp_hi = svzip1_u32(vx01, vx01);
    vutmp_hi = svlsr_n_u32_m(pg_odd, vutmp_hi, 2);
    vutmp_hi = svreinterpret_u32_u64(svand_n_u64_x(pg_all, svreinterpret_u64_u32(vutmp_hi), UINT64_C(0x303030303f3f3f3f)));
    const svuint32_t vx2 = svdup_u32(vx_scales[2]);
    vutmp_lo = svlsr_u32_x(pg_all, vx2, svreinterpret_u32_s32(svindex_s32(-2, 2)));
    vutmp_lo = svand_n_u32_z(pg_odd, vutmp_lo, UINT32_C(0x0f0f0f0f));
    svuint32_t vutmp = svorr_u32_z(pg_all, vutmp_hi, vutmp_lo);
    return vutmp;
}
#endif

void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
    assert(n % QK_K == 0);
#ifdef __ARM_FEATURE_MATMUL_INT8
    assert((nrc == 2) || (nrc == 1));
#else
    assert(nrc == 1);
#endif