Repository: ggerganov/llama.cpp
File: ggml/src/ggml-cpu/simd-gemm.h (C/C++)

Code before:
#pragma once

// Computes C[M x N] += A[M x K] * B[K x N]

#include "ggml-cpu-impl.h"
#include "vec.h"
#include "common.h"


// TODO: add support for sizeless vector types
#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)

// TODO: untested on avx512 and arm
// These are in units of GGML_F32_EPR
#if defined(__AVX512F__) || defined (__ARM_NEON__)
    static constexpr int GEMM_RM = 6;
    static constexpr int GEMM_RN = 4; // 24+4+1 = 29/32
#elif defined(__AVX2__) || defined(__AVX__)
    static constexpr int GEMM_RM = 6;
    static constexpr int GEMM_RN = 2; // 12+2+1 = 15/16
#else
    static constexpr int GEMM_RM = 2;
    static constexpr int GEMM_RN = 2;
#endif

template <int RM, int RN>
static inline void simd_gemm_ukernel(
    float       * GGML_RESTRICT C,
    const float * GGML_RESTRICT A,
    const float * GGML_RESTRICT B,
    int64_t K, int64_t N,
    int ii, int64_t jj)
{
    static constexpr int KN = GGML_F32_EPR;

    GGML_F32_VEC acc[RM][RN];
    for (int i = 0; i < RM; i++) {
        for (int r = 0; r < RN; r++) {
            acc[i][r] = GGML_F32_VEC_LOAD(C + (ii + i) * N + jj + r * KN);
        }
    }

    for (int64_t kk = 0; kk < K; kk++) {

Review comment (suggestion):
On M2 Ultra, I get better results using:

```bash
# GGML_F32_EPR = 4

GEMM_RM = 4
GEMM_RN = 4
```

Here is before and after:

- 6x4

| Model                 | Test        |   t/s master |   t/s pr/19422 |   Speedup |
|:----------------------|:------------|-------------:|---------------:|----------:|
| gpt-oss 20B MXFP4 MoE | pp512@d1024 |       166.40 |         177.12 |      1.06 |
| gpt-oss 20B MXFP4 MoE | pp512@d4096 |       134.16 |         155.12 |      1.16 |
| qwen2 1.5B F16        | pp512@d1024 |       298.68 |         307.97 |      1.03 |
| qwen2 1.5B F16        | pp512@d4096 |       226.77 |         247.86 |      1.09 |
| qwen2 3B F16          | pp512@d1024 |       143.64 |         151.70 |      1.06 |
| qwen2 3B F16          | pp512@d4096 |       112.81 |         131.39 |      1.16 |

- 4x4

| Model                 | Test        |   t/s master |   t/s pr/19422 |   Speedup |
|:----------------------|:------------|-------------:|---------------:|----------:|
| gpt-oss 20B MXFP4 MoE | pp512@d1024 |       167.38 |         178.00 |      1.06 |
| gpt-oss 20B MXFP4 MoE | pp512@d4096 |       134.28 |         160.09 |      1.19 |
| qwen2 1.5B F16        | pp512@d1024 |       299.04 |         313.51 |      1.05 |
| qwen2 1.5B F16        | pp512@d4096 |       226.24 |         259.23 |      1.15 |
| qwen2 3B F16          | pp512@d1024 |       144.24 |         152.88 |      1.06 |
| qwen2 3B F16          | pp512@d4096 |       113.59 |         135.72 |      1.19 |


Code after:
#pragma once

// Computes C[M x N] += A[M x K] * B[K x N]

#include "ggml-cpu-impl.h"
#include "vec.h"
#include "common.h"
#include "simd-mappings.h"

// TODO: add support for sizeless vector types
#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)

// TODO: untested on avx512
// These are in units of GGML_F32_EPR
#if defined(__AVX512F__) || defined (__ARM_NEON__)
    static constexpr int GEMM_RM = 4;
    static constexpr int GEMM_RN = 4; // 16+4+1 = 25/32
#elif defined(__AVX2__) || defined(__AVX__)
    static constexpr int GEMM_RM = 6;
    static constexpr int GEMM_RN = 2; // 12+2+1 = 15/16
#else
    static constexpr int GEMM_RM = 2;
    static constexpr int GEMM_RN = 2;
#endif

#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Waggressive-loop-optimizations"
#endif

template <int RM, int RN>
static inline void simd_gemm_ukernel(
    float       * GGML_RESTRICT C,
    const float * GGML_RESTRICT A,
    const float * GGML_RESTRICT B,
    int64_t K, int64_t N,
    int64_t ii, int64_t jj)
{
    static constexpr int KN = GGML_F32_EPR;

    GGML_F32_VEC acc[RM][RN];
    for (int i = 0; i < RM; i++) {
        for (int r = 0; r < RN; r++) {