Repository: ggml-org/llama.cpp
Pull request: CUDA: tune GLM 4.7 Flash FA kernel selection logic

Follow-up to https://github.com/ggml-org/llama.cpp/pull/19092 .

Adjusts the kernel selection logic as a function of context depth to squeeze out a few more % on Ampere/Blackwell.

| GPU      | Model               |   Microbatch size | Test          |   t/s master |   t/s 8a8b9a8bd |   Speedup |
|:---------|:--------------------|------------------:|:--------------|-------------:|----------------:|----------:|
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512         |       118.65 |          123.66 |      1.04 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d2048   |       101.58 |          110.02 |      1.08 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d4096   |        95.63 |          104.31 |      1.09 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d6144   |        92.47 |          100.51 |      1.09 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d8192   |        89.57 |           96.76 |      1.08 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d10240  |        87.83 |           93.34 |      1.06 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d12288  |        87.17 |           90.35 |      1.04 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d14336  |        84.97 |           88.56 |      1.04 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d16384  |        83.03 |           83.05 |      1.00 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512         |       113.89 |          116.80 |      1.03 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d2048   |       100.98 |          111.95 |      1.11 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d4096   |        99.43 |          109.44 |      1.10 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d6144   |        97.09 |          106.74 |      1.10 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d8192   |        94.91 |          104.08 |      1.10 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d10240  |        94.02 |          102.02 |      1.09 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d12288  |        93.81 |          100.86 |      1.08 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d14336  |        92.17 |           99.54 |      1.08 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d16384  |        90.79 |           90.45 |      1.00 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512         |       182.03 |          178.11 |      0.98 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d2048   |       172.29 |          168.43 |      0.98 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d4096   |       168.03 |          165.62 |      0.99 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d6144   |       163.17 |          159.79 |      0.98 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d8192   |       158.58 |          156.56 |      0.99 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d10240  |       155.02 |          152.07 |      0.98 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d12288  |       154.56 |          150.89 |      0.98 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d14336  |       148.98 |          147.98 |      0.99 |
| RTX 3090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d16384  |       147.72 |          146.79 |      0.99 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d65536  |        98.96 |           99.67 |      1.01 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d81920  |        89.42 |           92.71 |      1.04 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d98304  |        81.72 |           86.95 |      1.06 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d114688 |        75.28 |           81.75 |      1.09 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 1 | pp512@d131072 |        69.74 |           77.13 |      1.11 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d65536  |       112.80 |          109.57 |      0.97 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d81920  |       105.80 |          103.89 |      0.98 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d98304  |        99.90 |           99.11 |      0.99 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d114688 |        94.97 |           95.13 |      1.00 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 2 | pp512@d131072 |        90.27 |           90.97 |      1.01 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d65536  |       183.49 |          184.99 |      1.01 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d81920  |       169.88 |          175.53 |      1.03 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d98304  |       157.74 |          165.91 |      1.05 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d114688 |       148.10 |          158.23 |      1.07 |
| RTX 5090 | deepseek2 ?B Q2_K_M |                 4 | pp512@d131072 |       139.40 |          151.00 |      1.08 |


Changed files:
filename: ggml/src/ggml-cuda/fattn.cu
patch: @@ -148,6 +148,10 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
             const int gqa_ratio = Q->ne[2] / K->ne[2];
             if (gqa_ratio == 20) { // GLM 4.7 Flash
                 if (cc >= GGML_CUDA_CC_BLACKWELL) {
+                    if (Q->ne[1] <= 4 && K->ne[1] >= 65536) {
+                        ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
+                        break;
+                    }
                     ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);
                     break;
                 }
@@ -161,6 +165,10 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
                 }
                 if (cc >= GGML_CUDA_CC_TURING) {
                     if (Q->ne[1] <= 4) {
+                        if (K->ne[1] <= 16384) {
+                            ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
+                            break;
+                        }
                         ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 32>(ctx, dst);
                         break;
                     }