Repository: ggml-org/llama.cpp
Pull request: HIP: Use mmq on MFMA devices for MUL_MAT_ID in cases where a lot of splits would be generated

On MFMA hardware, MMQ performs better for medium sized problems, while dequant+rocblas performs better for large problem sizes.

currently ggml_cuda_should_use_mmq choses based on batch size and data type. This is suboptimal for MUL_MAT_ID as, even if the involved tensors are large, we end up calling rocblas for a large number of small tensors if the number of experts is high, causing poor performance.
This pr addresses this by choosing MMQ when the number of experts is high.

branch marks on a MI100 @ 160W power limit.

| Model                   |   Microbatch size | Test   |   t/s master |   t/s mmidopt |   Speedup |
|:------------------------|------------------:|:-------|-------------:|--------------:|----------:|
| gpt-oss 20B MXFP4 MoE   |                32 | pp1024 |       737.25 |        745.02 |      1.01 |
| gpt-oss 20B MXFP4 MoE   |                64 | pp1024 |       962.68 |        974.75 |      1.01 |
| gpt-oss 20B MXFP4 MoE   |               128 | pp1024 |       955.28 |        967.76 |      1.01 |
| gpt-oss 20B MXFP4 MoE   |               256 | pp1024 |      1720.56 |       1725.10 |      1.00 |
| gpt-oss 20B MXFP4 MoE   |               512 | pp1024 |      2277.16 |       2291.13 |      1.01 |
| gpt-oss 20B MXFP4 MoE   |              1024 | pp1024 |      2665.15 |       2685.24 |      1.01 |
| qwen3moe 30B.A3B Q4_K_M |                32 | pp1024 |       436.42 |        434.94 |      1.00 |
| qwen3moe 30B.A3B Q4_K_M |                64 | pp1024 |       562.45 |        563.55 |      1.00 |
| qwen3moe 30B.A3B Q4_K_M |               128 | pp1024 |       716.47 |        721.23 |      1.01 |
| qwen3moe 30B.A3B Q4_K_M |               256 | pp1024 |      1032.03 |       1124.19 |      1.09 |
| qwen3moe 30B.A3B Q4_K_M |               512 | pp1024 |       782.11 |       1497.25 |      1.91 |
| qwen3moe 30B.A3B Q4_K_M |              1024 | pp1024 |      1058.36 |       1738.98 |      1.64 |



future note: possibly it would be better to select based on the size of the resulting splits.

Changed files:
filename: ggml/src/ggml-cuda/ggml-cuda.cu
patch: @@ -2211,15 +2211,15 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
 
             const int cc            = ggml_cuda_info().devices[id].cc;
             const int warp_size     = ggml_cuda_info().devices[id].warp_size;
-            use_mul_mat_q           = use_mul_mat_q             && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1]);
+            use_mul_mat_q           = use_mul_mat_q             && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0);
             use_mul_mat_f           = use_mul_mat_f             && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false);
             use_mul_mat_vec_f       = use_mul_mat_vec_f         && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]);
             any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16   || !fast_fp16_hardware_available(cc);
         }
     } else {
         const int cc            = ggml_cuda_info().devices[ctx.device].cc;
         const int warp_size     = ggml_cuda_info().devices[ctx.device].warp_size;
-        use_mul_mat_q           = use_mul_mat_q             && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1]);
+        use_mul_mat_q           = use_mul_mat_q             && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0);
         use_mul_mat_f           = use_mul_mat_f             && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false);
         use_mul_mat_vec_f       = use_mul_mat_vec_f         && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]);
         any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16   || !fast_fp16_hardware_available(cc);
@@ -2287,7 +2287,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
             return;
         }
 
-        if (ggml_cuda_should_use_mmq(src0->type, cc, ne12)) {
+        if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) {
             ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst);
             return;
         }

filename: ggml/src/ggml-cuda/mmq.cu
patch: @@ -259,7 +259,7 @@ void ggml_cuda_op_mul_mat_q(
     GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_padded_row_size);
 }
 
-bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) {
+bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) {
 #ifdef GGML_CUDA_FORCE_CUBLAS
     return false;
 #endif // GGML_CUDA_FORCE_CUBLAS
@@ -320,7 +320,10 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) {
         if (GGML_CUDA_CC_IS_CDNA3(cc)) {
             return true;
         }
-        if (ne11 <= 128 || type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) {
+        if (n_experts > 64 || ne11 <= 128) {
+            return true;
+        }
+        if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) {
             return true;
         }
         if (ne11 <= 256 && (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K)) {

filename: ggml/src/ggml-cuda/mmq.cuh
patch: @@ -4082,4 +4082,4 @@ void ggml_cuda_op_mul_mat_q(
     const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
     const int64_t src1_padded_row_size, cudaStream_t stream);
 
-bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11);
+bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);