Repository: ggerganov/llama.cpp
File: ggml/src/ggml-cpu/vec.h (C/C++)

Code before:

inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {
#if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE)
    const int sve_register_length = svcntb() * 8;
    const int ggml_f16_epr = sve_register_length / 16;
    const int ggml_f16_step = 2 * ggml_f16_epr;

    GGML_F16x_VEC vx =  GGML_F16x_VEC_SET1(v);
    const int np = (n & ~(ggml_f16_step - 1));
    svfloat16_t ay1, ay2;

    for (int i = 0; i < np; i += ggml_f16_step) {
        ay1 = GGML_F16x_VEC_LOAD(y + i + 0*ggml_f16_epr, 0);
        ay1 = GGML_F16x_VEC_MUL(ay1, vx);
        GGML_F16x_VEC_STORE(y + i + 0*ggml_f16_epr, ay1, 0);

        ay2 = GGML_F16x_VEC_LOAD(y + i + 1*ggml_f16_epr, 1);
        ay2 = GGML_F16x_VEC_MUL(ay2, vx);
        GGML_F16x_VEC_STORE(y + i + 1*ggml_f16_epr, ay2, 1);
    }
    // leftovers
    // maximum number of leftover elements will be less that ggmlF_16x_epr. Apply predicated svmad on available elements only
    if (np < n) {
        svbool_t pg = svwhilelt_b16(np, n);
        svfloat16_t hy = svld1_f16(pg, (__fp16 *)(y + np));
        svfloat16_t out = svmul_f16_m(pg, hy, vx);
        svst1_f16(pg, (__fp16 *)(y + np), out);
    }
#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfh)
    const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);
    const _Float16 scale = *(const _Float16*)(&s);

    // calculate step size
    const int epr = __riscv_vsetvlmax_e16m4();
    const int step = epr * 2;
    const int np = (n & ~(step - 1));

    // unroll by 2
    for (int i = 0; i < np; i += step) {
        vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);
        ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, epr);
        __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);
        __asm__ __volatile__ ("" ::: "memory");

        vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);
        ay1 = __riscv_vfmul_vf_f16m4(ay1, scale, epr);
        __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);
        __asm__ __volatile__ ("" ::: "memory");
    }

    // leftovers

Review comment (suggestion):
I admire the commitment to remove the unnecessary float32 proxy and use float16 directly, but using RVV merely to emulate fixed-length SIMD seems like a missed opportunity for elegance. It would be delightful to see an implementation that actually leverages the hardware's native agility.

Code after:
    const int ggml_f16_epr = sve_register_length / 16;
    const int ggml_f16_step = 2 * ggml_f16_epr;

    GGML_F16x_VEC vx =  GGML_F16x_VEC_SET1(v);
    const int np = (n & ~(ggml_f16_step - 1));
    svfloat16_t ay1, ay2;

    for (int i = 0; i < np; i += ggml_f16_step) {
        ay1 = GGML_F16x_VEC_LOAD(y + i + 0*ggml_f16_epr, 0);
        ay1 = GGML_F16x_VEC_MUL(ay1, vx);
        GGML_F16x_VEC_STORE(y + i + 0*ggml_f16_epr, ay1, 0);

        ay2 = GGML_F16x_VEC_LOAD(y + i + 1*ggml_f16_epr, 1);
        ay2 = GGML_F16x_VEC_MUL(ay2, vx);
        GGML_F16x_VEC_STORE(y + i + 1*ggml_f16_epr, ay2, 1);
    }
    // leftovers
    // maximum number of leftover elements will be less that ggmlF_16x_epr. Apply predicated svmad on available elements only
    if (np < n) {
        svbool_t pg = svwhilelt_b16(np, n);
        svfloat16_t hy = svld1_f16(pg, (__fp16 *)(y + np));
        svfloat16_t out = svmul_f16_m(pg, hy, vx);
        svst1_f16(pg, (__fp16 *)(y + np), out);
    }
#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfh)
    const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);
    const _Float16 scale = *(const _Float16*)(&s);

    // calculate step size
    const int epr = __riscv_vsetvlmax_e16m4();
    const int step = epr * 2;
    const int np = (n & ~(step - 1));

    // unroll by 2
    for (int i = 0; i < np; i += step) {
        vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);
        ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, epr);
        __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);
        __asm__ __volatile__ ("" ::: "memory");

        vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);
        ay1 = __riscv_vfmul_vf_f16m4(ay1, scale, epr);
        __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);
        __asm__ __volatile__ ("" ::: "memory");
    }

    // leftovers
    int vl;
    for (int i = np; i < n; i += vl) {
        vl = __riscv_vsetvl_e16m4(n - i);
        vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, vl);