Repository: ggml-org/llama.cpp
Pull request: CUDA: limit number of FA stream-k CUDA blocks

On master the CUDA mma FA kernel can launch superfluous CUDA blocks that do not do any useful work but cause overhead. This can happen when running small models on GPUs with many streaming multiprocessors at low batch sizes. This PR fixes this by limiting the number of CUDA blocks to the number that can do useful work.

<details>
<summary>Performance changes</summary>

| GPU            | Model                   | Microbatch size | Test  | t/s 617db241a | t/s cc1232a41 | Speedup |
| :---------     | :---------------------- | --------------: | :---- | ------------: | ------------: | ------: |
| MI100          | gemma 2B Q4_0           |               1 | pp512 |        307.07 |        307.69 |    1.00 |
| MI100          | gemma 2B Q4_0           |               2 | pp512 |        513.62 |        512.21 |    1.00 |
| MI100          | gemma 2B Q4_0           |               4 | pp512 |        558.99 |        558.87 |    1.00 |
| MI100          | gemma 2B Q4_0           |               8 | pp512 |        757.43 |        760.76 |    1.00 |
| MI100          | gemma 2B Q4_0           |              16 | pp512 |       1787.58 |       1784.75 |    1.00 |
| MI100          | gemma 2B Q4_0           |              32 | pp512 |       3004.19 |       3002.58 |    1.00 |
| MI100          | gemma 2B Q4_0           |              64 | pp512 |       4399.17 |       4400.27 |    1.00 |
| MI100          | gemma 2B Q4_0           |             128 | pp512 |       4861.50 |       4840.62 |    1.00 |
| MI100          | gemma 2B Q4_0           |             256 | pp512 |       6766.90 |       6730.57 |    0.99 |
| MI100          | gemma 2B Q4_0           |             512 | pp512 |       7691.90 |       7687.11 |    1.00 |
| MI100          | gpt-oss 20B MXFP4 MoE   |               1 | pp512 |        164.89 |        166.20 |    1.01 |
| MI100          | gpt-oss 20B MXFP4 MoE   |               2 | pp512 |        199.15 |        198.04 |    0.99 |
| MI100          | gpt-oss 20B MXFP4 MoE   |               4 | pp512 |        211.46 |        210.79 |    1.00 |
| MI100          | gpt-oss 20B MXFP4 MoE   |               8 | pp512 |        283.54 |        288.24 |    1.02 |
| MI100          | gpt-oss 20B MXFP4 MoE   |              16 | pp512 |        507.92 |        509.63 |    1.00 |
| MI100          | gpt-oss 20B MXFP4 MoE   |              32 | pp512 |        744.14 |        746.61 |    1.00 |
| MI100          | gpt-oss 20B MXFP4 MoE   |              64 | pp512 |       1014.15 |       1014.56 |    1.00 |
| MI100          | gpt-oss 20B MXFP4 MoE   |             128 | pp512 |       1030.22 |       1030.16 |    1.00 |
| MI100          | gpt-oss 20B MXFP4 MoE   |             256 | pp512 |       1883.15 |       1874.98 |    1.00 |
| MI100          | gpt-oss 20B MXFP4 MoE   |             512 | pp512 |       2819.48 |       2808.94 |    1.00 |
| MI100          | llama 1B Q4_0           |               1 | pp512 |        479.22 |        473.80 |    0.99 |
| MI100          | llama 1B Q4_0           |               2 | pp512 |        744.77 |        737.25 |    0.99 |
| MI100          | llama 1B Q4_0           |               4 | pp512 |        914.88 |        927.97 |    1.01 |
| MI100          | llama 1B Q4_0           |               8 | pp512 |       1331.92 |       1343.15 |    1.01 |
| MI100          | llama 1B Q4_0           |              16 | pp512 |       2415.02 |       2513.70 |    1.04 |
| MI100          | llama 1B Q4_0           |              32 | pp512 |       3878.65 |       3846.42 |    0.99 |
| MI100          | llama 1B Q4_0           |              64 | pp512 |       6463.76 |       6445.40 |    1.00 |
| MI100          | llama 1B Q4_0           |             128 | pp512 |       7625.89 |       7645.00 |    1.00 |
| MI100          | llama 1B Q4_0           |             256 | pp512 |      11350.46 |      11466.49 |    1.01 |
| MI100          | llama 1B Q4_0           |             512 | pp512 |      13278.27 |      13282.67 |    1.00 |
| MI100          | llama 8B Q4_0           |               1 | pp512 |        130.68 |        126.33 |    0.97 |
| MI100          | llama 8B Q4_0           |               2 | pp512 |        206.14 |        205.80 |    1.00 |
| MI100          | llama 8B Q4_0           |               4 | pp512 |        228.55 |        228.66 |    1.00 |
| MI100          | llama 8B Q4_0           |               8 | pp512 |        327.48 |        327.08 |    1.00 |
| MI100          | llama 8B Q4_0           |              16 | pp512 |        752.53 |        749.26 |    1.00 |
| MI100          | llama 8B Q4_0           |              32 | pp512 |        975.10 |        958.94 |    0.98 |
| MI100          | llama 8B Q4_0           |              64 | pp512 |       1465.00 |       1457.39 |    0.99 |
| MI100          | llama 8B Q4_0           |             128 | pp512 |       1796.11 |       1795.85 |    1.00 |
| MI100          | llama 8B Q4_0           |             256 | pp512 |       2257.35 |       2256.04 |    1.00 |
| MI100          | llama 8B Q4_0           |             512 | pp512 |       2545.60 |       2539.18 |    1.00 |
| RTX 3090       | gemma 2B Q4_0           |               1 | pp512 |        422.55 |        464.68 |    1.10 |
| RTX 3090       | gemma 2B Q4_0           |               2 | pp512 |        694.10 |        833.14 |    1.20 |
| RTX 3090       | gemma 2B Q4_0           |               4 | pp512 |       1081.69 |       1297.35 |    1.20 |
| RTX 3090       | gemma 2B Q4_0           |               8 | pp512 |       1347.14 |       1617.25 |    1.20 |
| RTX 3090       | gemma 2B Q4_0           |              16 | pp512 |       2825.58 |       3316.05 |    1.17 |
| RTX 3090       | gemma 2B Q4_0           |              32 | pp512 |       4705.99 |       5205.30 |    1.11 |
| RTX 3090       | gemma 2B Q4_0           |              64 | pp512 |       7283.66 |       7449.79 |    1.02 |
| RTX 3090       | gemma 2B Q4_0           |             128 | pp512 |       9297.05 |       9437.46 |    1.02 |
| RTX 3090       | gemma 2B Q4_0           |             256 | pp512 |      12184.15 |      12291.84 |    1.01 |
| RTX 3090       | gemma 2B Q4_0           |             512 | pp512 |      15292.65 |      15226.77 |    1.00 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |               1 | pp512 |        238.89 |        237.49 |    0.99 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |               2 | pp512 |        283.62 |        286.24 |    1.01 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |               4 | pp512 |        339.24 |        337.89 |    1.00 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |               8 | pp512 |        507.06 |        514.70 |    1.02 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |              16 | pp512 |        808.75 |        815.90 |    1.01 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |              32 | pp512 |       1228.01 |       1227.84 |    1.00 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |              64 | pp512 |       1671.74 |       1663.09 |    0.99 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |             128 | pp512 |       2036.35 |       2018.82 |    0.99 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |             256 | pp512 |       3086.19 |       3047.10 |    0.99 |
| RTX 3090       | gpt-oss 20B MXFP4 MoE   |             512 | pp512 |       4739.20 |       4702.81 |    0.99 |
| RTX 3090       | llama 1B Q4_0           |               1 | pp512 |        743.97 |        760.83 |    1.02 |
| RTX 3090       | llama 1B Q4_0           |               2 | pp512 |       1307.89 |       1365.19 |    1.04 |
| RTX 3090       | llama 1B Q4_0           |               4 | pp512 |       2021.03 |       2168.20 |    1.07 |
| RTX 3090       | llama 1B Q4_0           |               8 | pp512 |       2715.88 |       2864.56 |    1.05 |
| RTX 3090       | llama 1B Q4_0           |              16 | pp512 |       4766.38 |       5162.49 |    1.08 |
| RTX 3090       | llama 1B Q4_0           |              32 | pp512 |       7697.95 |       8166.61 |    1.06 |
| RTX 3090       | llama 1B Q4_0           |              64 | pp512 |      11651.23 |      11981.28 |    1.03 |
| RTX 3090       | llama 1B Q4_0           |             128 | pp512 |      15378.65 |      14816.65 |    0.96 |
| RTX 3090       | llama 1B Q4_0           |             256 | pp512 |      21673.09 |      20635.84 |    0.95 |
| RTX 3090       | llama 1B Q4_0           |             512 | pp512 |      25056.60 |      24964.06 |    1.00 |
| RTX 3090       | llama 8B Q4_0           |               1 | pp512 |        164.04 |        166.27 |    1.01 |
| RTX 3090       | llama 8B Q4_0           |               2 | pp512 |        305.31 |        308.39 |    1.01 |
| RTX 3090       | llama 8B Q4_0           |               4 | pp512 |        457.66 |        472.09 |    1.03 |
| RTX 3090       | llama 8B Q4_0           |               8 | pp512 |        538.71 |        553.41 |    1.03 |
| RTX 3090       | llama 8B Q4_0           |              16 | pp512 |       1205.67 |       1261.46 |    1.05 |
| RTX 3090       | llama 8B Q4_0           |              32 | pp512 |       1916.97 |       1959.62 |    1.02 |
| RTX 3090       | llama 8B Q4_0           |              64 | pp512 |       2864.33 |       2916.22 |    1.02 |
| RTX 3090       | llama 8B Q4_0           |             128 | pp512 |       3647.53 |       3649.13 |    1.00 |
| RTX 3090       | llama 8B Q4_0           |             256 | pp512 |       4325.46 |       4355.88 |    1.01 |
| RTX 3090       | llama 8B Q4_0           |             512 | pp512 |       5410.30 |       5464.14 |    1.01 |
| RTX 4090       | gemma 2B Q4_0           |               1 | pp512 |        557.81 |        557.54 |    1.00 |
| RTX 4090       | gemma 2B Q4_0           |               2 | pp512 |        823.40 |       1003.94 |    1.22 |
| RTX 4090       | gemma 2B Q4_0           |               4 | pp512 |       1530.09 |       1951.30 |    1.28 |
| RTX 4090       | gemma 2B Q4_0           |               8 | pp512 |       2062.45 |       2611.77 |    1.27 |
| RTX 4090       | gemma 2B Q4_0           |              16 | pp512 |       4024.08 |       4791.91 |    1.19 |
| RTX 4090       | gemma 2B Q4_0           |              32 | pp512 |       7158.53 |       8123.96 |    1.13 |
| RTX 4090       | gemma 2B Q4_0           |              64 | pp512 |      12206.44 |      13099.76 |    1.07 |
| RTX 4090       | gemma 2B Q4_0           |             128 | pp512 |      17926.24 |      18306.34 |    1.02 |
| RTX 4090       | gemma 2B Q4_0           |             256 | pp512 |      27200.39 |      27160.55 |    1.00 |
| RTX 4090       | gemma 2B Q4_0           |             512 | pp512 |      30618.51 |      30572.96 |    1.00 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |               1 | pp512 |        309.82 |        309.67 |    1.00 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |               2 | pp512 |        416.98 |        432.13 |    1.04 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |               4 | pp512 |        603.79 |        622.67 |    1.03 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |               8 | pp512 |        809.64 |        838.75 |    1.04 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |              16 | pp512 |       1332.66 |       1365.21 |    1.02 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |              32 | pp512 |       2230.18 |       2259.18 |    1.01 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |              64 | pp512 |       3343.92 |       3354.38 |    1.00 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |             128 | pp512 |       4365.63 |       4363.30 |    1.00 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |             256 | pp512 |       6794.91 |       6798.89 |    1.00 |
| RTX 4090       | gpt-oss 20B MXFP4 MoE   |             512 | pp512 |       9560.12 |       9569.74 |    1.00 |
| RTX 4090       | llama 1B Q4_0           |               1 | pp512 |        911.95 |        909.54 |    1.00 |
| RTX 4090       | llama 1B Q4_0           |               2 | pp512 |       1587.68 |       1629.96 |    1.03 |
| RTX 4090       | llama 1B Q4_0           |               4 | pp512 |       2847.47 |       3088.60 |    1.08 |
| RTX 4090       | llama 1B Q4_0           |               8 | pp512 |       4145.63 |       4431.80 |    1.07 |
| RTX 4090       | llama 1B Q4_0           |              16 | pp512 |       6461.51 |       7055.87 |    1.09 |
| RTX 4090       | llama 1B Q4_0           |              32 | pp512 |      11226.27 |      11901.37 |    1.06 |
| RTX 4090       | llama 1B Q4_0           |              64 | pp512 |      18262.44 |      18554.64 |    1.02 |
| RTX 4090       | llama 1B Q4_0           |             128 | pp512 |      26331.86 |      26501.36 |    1.01 |
| RTX 4090       | llama 1B Q4_0           |             256 | pp512 |      43321.86 |      43137.85 |    1.00 |
| RTX 4090       | llama 1B Q4_0           |             512 | pp512 |      48314.49 |      49416.56 |    1.02 |
| RTX 4090       | llama 8B Q4_0           |               1 | pp512 |        193.09 |        193.09 |    1.00 |
| RTX 4090       | llama 8B Q4_0           |               2 | pp512 |        362.51 |        364.29 |    1.00 |
| RTX 4090       | llama 8B Q4_0           |               4 | pp512 |        690.20 |        707.50 |    1.03 |
| RTX 4090       | llama 8B Q4_0           |               8 | pp512 |       1111.10 |       1142.71 |    1.03 |
| RTX 4090       | llama 8B Q4_0           |              16 | pp512 |       1916.39 |       2018.92 |    1.05 |
| RTX 4090       | llama 8B Q4_0           |              32 | pp512 |       3344.64 |       3460.47 |    1.03 |
| RTX 4090       | llama 8B Q4_0           |              64 | pp512 |       5727.82 |       5776.43 |    1.01 |
| RTX 4090       | llama 8B Q4_0           |             128 | pp512 |       8172.87 |       8172.86 |    1.00 |
| RTX 4090       | llama 8B Q4_0           |             256 | pp512 |      11117.59 |      11120.71 |    1.00 |
| RTX 4090       | llama 8B Q4_0           |             512 | pp512 |      12925.19 |      12936.87 |    1.00 |
| RTX 5090       | gemma 2B Q4_0           |               1 | pp512 |        687.88 |        687.19 |    1.00 |
| RTX 5090       | gemma 2B Q4_0           |               2 | pp512 |        911.18 |       1236.06 |    1.36 |
| RTX 5090       | gemma 2B Q4_0           |               4 | pp512 |       1716.52 |       2343.91 |    1.37 |
| RTX 5090       | gemma 2B Q4_0           |               8 | pp512 |       2336.98 |       3084.92 |    1.32 |
| RTX 5090       | gemma 2B Q4_0           |              16 | pp512 |       4333.46 |       5174.20 |    1.19 |
| RTX 5090       | gemma 2B Q4_0           |              32 | pp512 |       7654.92 |       8585.02 |    1.12 |
| RTX 5090       | gemma 2B Q4_0           |              64 | pp512 |      12550.24 |      13449.50 |    1.07 |
| RTX 5090       | gemma 2B Q4_0           |             128 | pp512 |      17955.61 |      18560.71 |    1.03 |
| RTX 5090       | gemma 2B Q4_0           |             256 | pp512 |      27933.36 |      28123.87 |    1.01 |
| RTX 5090       | gemma 2B Q4_0           |             512 | pp512 |      37924.19 |      37733.84 |    0.99 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |               1 | pp512 |        394.96 |        394.80 |    1.00 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |               2 | pp512 |        501.73 |        528.24 |    1.05 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |               4 | pp512 |        741.15 |        768.15 |    1.04 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |               8 | pp512 |        987.88 |       1020.05 |    1.03 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |              16 | pp512 |       1606.30 |       1633.32 |    1.02 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |              32 | pp512 |       2775.19 |       2803.78 |    1.01 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |              64 | pp512 |       4588.16 |       4601.48 |    1.00 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |             128 | pp512 |       6485.87 |       6490.55 |    1.00 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |             256 | pp512 |      10128.73 |      10142.42 |    1.00 |
| RTX 5090       | gpt-oss 20B MXFP4 MoE   |             512 | pp512 |      13916.71 |      13866.32 |    1.00 |
| RTX 5090       | llama 1B Q4_0           |               1 | pp512 |       1057.85 |       1057.12 |    1.00 |
| RTX 5090       | llama 1B Q4_0           |               2 | pp512 |       1742.10 |       1860.73 |    1.07 |
| RTX 5090       | llama 1B Q4_0           |               4 | pp512 |       3103.04 |       3443.54 |    1.11 |
| RTX 5090       | llama 1B Q4_0           |               8 | pp512 |       4619.95 |       4983.78 |    1.08 |
| RTX 5090       | llama 1B Q4_0           |              16 | pp512 |       6849.35 |       7335.65 |    1.07 |
| RTX 5090       | llama 1B Q4_0           |              32 | pp512 |      11592.58 |      12132.77 |    1.05 |
| RTX 5090       | llama 1B Q4_0           |              64 | pp512 |      18429.29 |      18822.11 |    1.02 |
| RTX 5090       | llama 1B Q4_0           |             128 | pp512 |      25792.56 |      25912.05 |    1.00 |
| RTX 5090       | llama 1B Q4_0           |             256 | pp512 |      41661.89 |      41701.35 |    1.00 |
| RTX 5090       | llama 1B Q4_0           |             512 | pp512 |      57142.33 |      56959.08 |    1.00 |
| RTX 5090       | llama 8B Q4_0           |               1 | pp512 |        275.25 |        275.10 |    1.00 |
| RTX 5090       | llama 8B Q4_0           |               2 | pp512 |        503.77 |        512.34 |    1.02 |
| RTX 5090       | llama 8B Q4_0           |               4 | pp512 |        908.15 |        967.76 |    1.07 |
| RTX 5090       | llama 8B Q4_0           |               8 | pp512 |       1319.40 |       1382.43 |    1.05 |
| RTX 5090       | llama 8B Q4_0           |              16 | pp512 |       2362.79 |       2498.42 |    1.06 |
| RTX 5090       | llama 8B Q4_0           |              32 | pp512 |       4077.67 |       4236.84 |    1.04 |
| RTX 5090       | llama 8B Q4_0           |              64 | pp512 |       6414.21 |       6524.84 |    1.02 |
| RTX 5090       | llama 8B Q4_0           |             128 | pp512 |       8774.79 |       8808.16 |    1.00 |
| RTX 5090       | llama 8B Q4_0           |             256 | pp512 |      12471.68 |      12403.33 |    0.99 |
| RTX 5090       | llama 8B Q4_0           |             512 | pp512 |      16136.64 |      16129.48 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |               1 | pp512 |        182.50 |        182.13 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |               2 | pp512 |        237.47 |        237.41 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |               4 | pp512 |        478.23 |        478.71 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |               8 | pp512 |        811.14 |        806.99 |    0.99 |
| RX 9060 XT     | gemma 2B Q4_0           |              16 | pp512 |       1336.67 |       1323.07 |    0.99 |
| RX 9060 XT     | gemma 2B Q4_0           |              32 | pp512 |       2158.31 |       2157.41 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |              64 | pp512 |       3265.54 |       3266.48 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |             128 | pp512 |       4864.51 |       4870.09 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |             256 | pp512 |       7261.16 |       7240.06 |    1.00 |
| RX 9060 XT     | gemma 2B Q4_0           |             512 | pp512 |       7339.91 |       7261.32 |    0.99 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |               1 | pp512 |         99.73 |         99.57 |    1.00 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |               2 | pp512 |        124.53 |        120.78 |    0.97 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |               4 | pp512 |        191.55 |        192.02 |    1.00 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |               8 | pp512 |        345.46 |        345.36 |    1.00 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |              16 | pp512 |        514.15 |        514.15 |    1.00 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |              32 | pp512 |        741.42 |        740.95 |    1.00 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |              64 | pp512 |       1107.00 |       1103.84 |    1.00 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |             128 | pp512 |       1366.49 |       1359.26 |    0.99 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |             256 | pp512 |       2060.42 |       2053.39 |    1.00 |
| RX 9060 XT     | gpt-oss 20B MXFP4 MoE   |             512 | pp512 |       2609.33 |       2616.85 |    1.00 |
| RX 9060 XT     | llama 1B Q4_0           |               1 | pp512 |        290.88 |        291.12 |    1.00 |
| RX 9060 XT     | llama 1B Q4_0           |               2 | pp512 |        375.01 |        373.21 |    1.00 |
| RX 9060 XT     | llama 1B Q4_0           |               4 | pp512 |        548.88 |        534.50 |    0.97 |
| RX 9060 XT     | llama 1B Q4_0           |               8 | pp512 |       1105.95 |       1115.81 |    1.01 |
| RX 9060 XT     | llama 1B Q4_0           |              16 | pp512 |       2406.95 |       2389.08 |    0.99 |
| RX 9060 XT     | llama 1B Q4_0           |              32 | pp512 |       3577.62 |       3568.49 |    1.00 |
| RX 9060 XT     | llama 1B Q4_0           |              64 | pp512 |       4831.80 |       4832.51 |    1.00 |
| RX 9060 XT     | llama 1B Q4_0           |             128 | pp512 |       7606.72 |       7578.95 |    1.00 |
| RX 9060 XT     | llama 1B Q4_0           |             256 | pp512 |      11539.12 |      11427.88 |    0.99 |
| RX 9060 XT     | llama 1B Q4_0           |             512 | pp512 |      10711.84 |      10745.54 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |               1 | pp512 |         64.94 |         65.08 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |               2 | pp512 |         94.96 |         95.01 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |               4 | pp512 |        170.64 |        171.06 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |               8 | pp512 |        211.71 |        210.62 |    0.99 |
| RX 9060 XT     | llama 8B Q4_0           |              16 | pp512 |        617.47 |        616.50 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |              32 | pp512 |        863.41 |        863.60 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |              64 | pp512 |       1485.96 |       1487.08 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |             128 | pp512 |       2257.72 |       2247.18 |    1.00 |
| RX 9060 XT     | llama 8B Q4_0           |             256 | pp512 |       2469.44 |       2451.48 |    0.99 |
| RX 9060 XT     | llama 8B Q4_0           |             512 | pp512 |       2501.56 |       2479.28 |    0.99 |
| V100-PCIE-32GB | gemma 2B Q4_0           |               1 | pp512 |        369.91 |        368.88 |    1.00 |
| V100-PCIE-32GB | gemma 2B Q4_0           |               2 | pp512 |        646.23 |        643.04 |    1.00 |
| V100-PCIE-32GB | gemma 2B Q4_0           |               4 | pp512 |        737.05 |        800.70 |    1.09 |
| V100-PCIE-32GB | gemma 2B Q4_0           |               8 | pp512 |       1160.11 |       1238.05 |    1.07 |
| V100-PCIE-32GB | gemma 2B Q4_0           |              16 | pp512 |       1797.88 |       1886.12 |    1.05 |
| V100-PCIE-32GB | gemma 2B Q4_0           |              32 | pp512 |       2927.14 |       2981.89 |    1.02 |
| V100-PCIE-32GB | gemma 2B Q4_0           |              64 | pp512 |       2114.79 |       2116.72 |    1.00 |
| V100-PCIE-32GB | gemma 2B Q4_0           |             128 | pp512 |       3823.66 |       3822.37 |    1.00 |
| V100-PCIE-32GB | gemma 2B Q4_0           |             256 | pp512 |       6517.56 |       6521.83 |    1.00 |
| V100-PCIE-32GB | gemma 2B Q4_0           |             512 | pp512 |       9116.61 |       9128.69 |    1.00 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |               1 | pp512 |        181.04 |        181.04 |    1.00 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |               2 | pp512 |        244.60 |        244.25 |    1.00 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |               4 | pp512 |        293.49 |        297.35 |    1.01 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |               8 | pp512 |        282.79 |        285.44 |    1.01 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |              16 | pp512 |        419.85 |        421.70 |    1.00 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |              32 | pp512 |        573.44 |        573.66 |    1.00 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |              64 | pp512 |        934.23 |        930.83 |    1.00 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |             128 | pp512 |       1297.05 |       1290.63 |    1.00 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |             256 | pp512 |       1756.90 |       1744.24 |    0.99 |
| V100-PCIE-32GB | gpt-oss 20B MXFP4 MoE   |             512 | pp512 |       2382.27 |       2358.11 |    0.99 |
| V100-PCIE-32GB | llama 1B Q4_0           |               1 | pp512 |        579.07 |        572.24 |    0.99 |
| V100-PCIE-32GB | llama 1B Q4_0           |               2 | pp512 |       1004.24 |        992.88 |    0.99 |
| V100-PCIE-32GB | llama 1B Q4_0           |               4 | pp512 |       1454.77 |       1442.09 |    0.99 |
| V100-PCIE-32GB | llama 1B Q4_0           |               8 | pp512 |       2097.35 |       2160.19 |    1.03 |
| V100-PCIE-32GB | llama 1B Q4_0           |              16 | pp512 |       2939.44 |       3052.26 |    1.04 |
| V100-PCIE-32GB | llama 1B Q4_0           |              32 | pp512 |       4794.12 |       4858.80 |    1.01 |
| V100-PCIE-32GB | llama 1B Q4_0           |              64 | pp512 |       3986.32 |       3992.48 |    1.00 |
| V100-PCIE-32GB | llama 1B Q4_0           |             128 | pp512 |       7108.66 |       7102.53 |    1.00 |
| V100-PCIE-32GB | llama 1B Q4_0           |             256 | pp512 |      11616.36 |      11611.04 |    1.00 |
| V100-PCIE-32GB | llama 1B Q4_0           |             512 | pp512 |      15055.69 |      14903.88 |    0.99 |
| V100-PCIE-32GB | llama 8B Q4_0           |               1 | pp512 |        141.99 |        142.26 |    1.00 |
| V100-PCIE-32GB | llama 8B Q4_0           |               2 | pp512 |        260.11 |        260.53 |    1.00 |
| V100-PCIE-32GB | llama 8B Q4_0           |               4 | pp512 |        359.38 |        360.00 |    1.00 |
| V100-PCIE-32GB | llama 8B Q4_0           |               8 | pp512 |        509.31 |        519.99 |    1.02 |
| V100-PCIE-32GB | llama 8B Q4_0           |              16 | pp512 |        727.72 |        747.77 |    1.03 |
| V100-PCIE-32GB | llama 8B Q4_0           |              32 | pp512 |       1111.04 |       1125.32 |    1.01 |
| V100-PCIE-32GB | llama 8B Q4_0           |              64 | pp512 |        637.62 |        638.83 |    1.00 |
| V100-PCIE-32GB | llama 8B Q4_0           |             128 | pp512 |       1160.66 |       1161.38 |    1.00 |
| V100-PCIE-32GB | llama 8B Q4_0           |             256 | pp512 |       2048.29 |       2049.06 |    1.00 |
| V100-PCIE-32GB | llama 8B Q4_0           |             512 | pp512 |       3092.83 |       3095.46 |    1.00 |

</details>

Changed files:
filename: ggml/src/ggml-cuda/fattn-common.cuh
patch: @@ -892,7 +892,7 @@ void launch_fattn(
     const int ntiles_x     = ((Q->ne[1] + ncols1 - 1) / ncols1);
     const int gqa_ratio    = Q->ne[2] / K->ne[2];
     const int ntiles_z_gqa = ((gqa_ratio + ncols2 - 1) / ncols2);
-    const int ntiles_total = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3];
+    const int ntiles_dst   = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3];
 
     // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.
     // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
@@ -919,37 +919,37 @@ void launch_fattn(
     GGML_ASSERT(max_blocks_per_sm > 0);
     int parallel_blocks = max_blocks_per_sm;
 
+    const int ntiles_KV = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.
+
     dim3 blocks_num;
     if (stream_k) {
         // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup.
         const int max_blocks = max_blocks_per_sm*nsm;
-        const int tiles_nwaves = (ntiles_total + max_blocks - 1) / max_blocks;
-        const int tiles_efficiency_percent = 100 * ntiles_total / (max_blocks*tiles_nwaves);
+        const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
+        const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves);
 
-        const int nblocks_stream_k = max_blocks;
+        const int nblocks_stream_k = std::min(max_blocks, ntiles_KV*ntiles_dst);
 
         const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75;
 
-        blocks_num.x = use_stream_k ? nblocks_stream_k : ntiles_total;
+        blocks_num.x = use_stream_k ? nblocks_stream_k : ntiles_dst;
         blocks_num.y = 1;
         blocks_num.z = 1;
 
-        if (ntiles_total % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.
+        if (ntiles_dst % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.
             dst_tmp_meta.alloc((size_t(blocks_num.x) * ncols * (2 + DV/2)));
         }
     } else {
-        const int ntiles_KQ = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by tensor size.
-
         // parallel_blocks must not be larger than what the tensor size allows:
-        parallel_blocks = std::min(parallel_blocks, ntiles_KQ);
+        parallel_blocks = std::min(parallel_blocks, ntiles_KV);
 
         // If ntiles_total % blocks_per_wave != 0 then some efficiency is lost due to tail effects.
         // Test whether parallel_blocks can be set to a higher value for better efficiency.
         const int blocks_per_wave = nsm * max_blocks_per_sm;
         int nwaves_best = 0;
         int efficiency_percent_best = 0;
-        for (int parallel_blocks_test = parallel_blocks; parallel_blocks_test <= ntiles_KQ; ++parallel_blocks_test) {
-            const int nblocks_total = ntiles_total * parallel_blocks_test;
+        for (int parallel_blocks_test = parallel_blocks; parallel_blocks_test <= ntiles_KV; ++parallel_blocks_test) {
+            const int nblocks_total = ntiles_dst * parallel_blocks_test;
             const int nwaves = (nblocks_total + blocks_per_wave - 1) / blocks_per_wave;
             const int efficiency_percent = 100 * nblocks_total / (nwaves*blocks_per_wave);
 
@@ -1015,7 +1015,7 @@ void launch_fattn(
     CUDA_CHECK(cudaGetLastError());
 
     if (stream_k) {
-        if (ntiles_total % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.
+        if (ntiles_dst % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles.
             const dim3 block_dim_combine(DV, 1, 1);
             const dim3 blocks_num_combine = {blocks_num.x, ncols1, ncols2};