Repository: ggerganov/llama.cpp
File: ggml/src/ggml-hexagon/htp/matmul-ops.c (C)

Code before:
    uint8_t * restrict tmp_data = (uint8_t *) spad->data + (spad->size_per_thread * ith);

    const size_t src_row_size_padded = htp_round_up(src_row_size, QK_Q8_0x4x2 * sizeof(float));
    memset(tmp_data, 0, src_row_size_padded);  // zero-out temp row data for padding

    for (uint32_t i = ir_first; i < ir_last; ++i) {
        htp_l2fetch(src_data, 2, src_row_size, src_row_size);
        hvx_copy_fp32_aa(tmp_data, src_data, ne0);

        // FARF(HIGH, "quantize-q8x4-row: %u\n", i);
        quantize_row_fp32_q8x4x2((float *) tmp_data, dst_data, ne0, quantize_row_func);
        dst_data += dst_row_size;
        src_data += src_row_size;
    }

    uint64_t t2 = HAP_perf_get_qtimer_count();

    FARF(HIGH, "quantize-fp32-q8x4: %u/%u : n-rows %u (%u:%u) row-size %u -> %u usec %u\n", ith, nth, nrows, ir_first,
         ir_last, src_row_size, dst_row_size, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
}

static void htp_quantize_fp32_q8x4x2(unsigned int n, unsigned int i, void * data) {
    struct htp_ops_context * octx = data;
    // quantize_block_fp32_q8x4: use group size 128: tested on Qwen3:0.6B k proj layer 0 on 256 tokens, Relative L2 1.7%
    // quantize_block_fp32_q8x2: use group size 64 : Relative L2 1.3%
    // quantize_block_fp32_q8x1: use group size 32 : Relative L2 1.1%
    quantize_block_fp32_q8_func_t quantize_row_func = quantize_block_fp32_q8x1;
    quantize_fp32_q8x4x2(&octx->src1, octx->src1_spad.data, &octx->src0_spad, n, i, octx->src1_nrows_per_thread, quantize_row_func);
}

// ** matmul callbacks for worker_pool

static void htp_matvec_q4x4x2_q8x4x2(unsigned int n, unsigned int i, void * data) {
    struct htp_ops_context * octx = data;

    struct htp_matmul_type mt;
    mt.type        = "q4x4x2-q8x4x2";
    mt.vec_dot     = vec_dot_q4x4x2_q8x4x2;
    mt.vec_dot_rx2 = vec_dot_q4x4x2_q8x4x2_rx2;

    matvec(&mt, &octx->src0, &octx->src1, &octx->dst, &octx->src0_spad, &octx->src1_spad, &octx->dst_spad, n, i,
           octx->src0_nrows_per_thread, octx->ctx->dma[i]);
}

static void htp_matmul_q4x4x2_q8x4x2(unsigned int n, unsigned int i, void * data) {
    struct htp_ops_context * octx = data;

    struct htp_matmul_type mt;
    mt.type        = "q4x4x2-q8x4x2";
    mt.vec_dot     = vec_dot_q4x4x2_q8x4x2;
    mt.vec_dot_rx2 = vec_dot_q4x4x2_q8x4x2_rx2;

Review comment (suggestion):
Sorry to pop up with a maybe off-topic question: 
Is there a convenient way to test op level L2 divergence in the current codebase? Aware of `llama-perplexity`, but it seems to be a model-level metric rather than something you can use for a single op.

Code after:
    const uint32_t ne3 = src->ne[3];

    const uint32_t nrows = ne1 * ne2 * ne3;                             // total n_rows

    const uint32_t ir_first = nrows_per_thread * ith;                   // first row
    const uint32_t ir_last  = MIN(ir_first + nrows_per_thread, nrows);  // last row

    const size_t src_row_size = src->nb[1];
    const size_t dst_row_size = q8x4x2_row_size(ne0);

    uint8_t * restrict src_data = (uint8_t *) src->data + (src_row_size * ir_first);
    uint8_t * restrict dst_data = (uint8_t *) dst + (dst_row_size * ir_first);
    uint8_t * restrict tmp_data = (uint8_t *) spad->data + (spad->size_per_thread * ith);

    const size_t src_row_size_padded = htp_round_up(src_row_size, QK_Q8_0x4x2 * sizeof(float));
    memset(tmp_data, 0, src_row_size_padded);  // zero-out temp row data for padding

    for (uint32_t i = ir_first; i < ir_last; ++i) {
        htp_l2fetch(src_data, 2, src_row_size, src_row_size);
        hvx_copy_fp32_aa(tmp_data, src_data, ne0);

        // FARF(HIGH, "quantize-q8x4-row: %u\n", i);
        quantize_row_fp32_q8x4x2((float *) tmp_data, dst_data, ne0);
        dst_data += dst_row_size;
        src_data += src_row_size;
    }

    uint64_t t2 = HAP_perf_get_qtimer_count();

    FARF(HIGH, "quantize-fp32-q8x4: %u/%u : n-rows %u (%u:%u) row-size %u -> %u usec %u\n", ith, nth, nrows, ir_first,
         ir_last, src_row_size, dst_row_size, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
}

static void htp_quantize_fp32_q8x4x2(unsigned int n, unsigned int i, void * data) {
    struct htp_ops_context * octx = data;
    quantize_fp32_q8x4x2(&octx->src1, octx->src1_spad.data, &octx->src0_spad, n, i, octx->src1_nrows_per_thread);
}

// ** matmul callbacks for worker_pool

static void htp_matvec_q4x4x2_q8x4x2(unsigned int n, unsigned int i, void * data) {
    struct htp_ops_context * octx = data;

    struct htp_matmul_type mt;
    mt.type        = "q4x4x2-q8x4x2";
    mt.vec_dot     = vec_dot_q4x4x2_q8x4x2;
    mt.vec_dot_rx2 = vec_dot_q4x4x2_q8x4x2_rx2;

    matvec(&mt, &octx->src0, &octx->src1, &octx->dst, &octx->src0_spad, &octx->src1_spad, &octx->dst_spad, n, i,
           octx->src0_nrows_per_thread, octx->ctx->dma[i]);
}