Repository: ggml-org/llama.cpp
Pull request: HIP: enable mul_mat_f for RDNA4

Enable mul_mat_f for RDNA4 and move the n >= 3 workload from mmvf to mmf based on the result of test-backend-ops.

Use a weird unreached branch to force rocm compiler to generate better performance code for RDNA4, a bug shall be submitted to rocm.

System:  Ubuntu 24.04.3 LTS
ROCm: 7.1.0
Driver: amdgpu version: 6.16.6   ROCm version: 7.1.0
GPU: 9070XT

<details>

<summary>MUL_MAT results</summary>

| Backend   | GGML op   | Op parameters                                                                            |   TFLOPS master |   TFLOPS 698c9f24 |   Speedup |
|:----------|:----------|:-----------------------------------------------------------------------------------------|----------------:|------------------:|----------:|
| ROCm0     | MUL_MAT   | type_a=bf16,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            0.61 |              0.61 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=bf16,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            1.20 |              1.20 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=bf16,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            1.65 |              1.87 |      1.13 |
| ROCm0     | MUL_MAT   | type_a=bf16,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            1.98 |              2.49 |      1.26 |
| ROCm0     | MUL_MAT   | type_a=bf16,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            2.50 |              3.11 |      1.25 |
| ROCm0     | MUL_MAT   | type_a=bf16,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           91.95 |             93.59 |      1.02 |
| ROCm0     | MUL_MAT   | type_a=bf16,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.92 |              4.95 |      1.26 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=128,n=1,k=16416,bs=[8,1],nr=[4,1],per=[0,1,2,3],k_v=32832,o=1    |            1.38 |              1.39 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=16416,n=1,k=128,bs=[8,1],nr=[4,1],per=[0,2,1,3],k_v=0,o=1        |            0.34 |              0.34 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            0.61 |              0.61 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            1.21 |              1.21 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            1.77 |              1.87 |      1.06 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            2.27 |              2.49 |      1.10 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            2.68 |              3.11 |      1.16 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |           96.01 |             96.28 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f16,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            4.04 |              4.95 |      1.23 |
| ROCm0     | MUL_MAT   | type_a=f32,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            0.31 |              0.31 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f32,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            0.63 |              0.63 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f32,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            0.94 |              0.94 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f32,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            1.25 |              1.25 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f32,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            1.55 |              1.54 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f32,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            3.48 |              3.49 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=f32,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1       |            2.30 |              2.31 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_m,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            3.63 |              3.67 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq1_m,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            5.95 |              5.99 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq1_m,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            7.03 |              7.08 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq1_m,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            7.86 |              7.89 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_m,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            8.21 |              8.32 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq1_m,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |           74.78 |             75.01 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_m,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            9.38 |              9.34 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_s,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            4.16 |              4.18 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_s,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            6.90 |              6.92 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_s,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            7.42 |              7.49 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq1_s,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            8.25 |              8.29 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_s,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            8.87 |              8.97 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq1_s,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |           75.22 |             75.50 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq1_s,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            9.05 |              8.92 |      0.99 |
| ROCm0     | MUL_MAT   | type_a=iq2_s,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            1.61 |              1.62 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq2_s,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            2.86 |              2.88 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq2_s,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            3.97 |              3.99 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_s,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            4.81 |              4.83 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_s,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            5.11 |              5.12 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_s,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |           74.32 |             74.48 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_s,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            6.53 |              6.56 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xs,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            2.20 |              2.22 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq2_xs,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            3.85 |              3.88 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq2_xs,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            5.07 |              5.10 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq2_xs,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            5.90 |              5.92 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xs,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            6.64 |              6.67 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xs,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1  |           74.33 |             74.51 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xs,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            7.80 |              7.83 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xxs,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            1.66 |              1.69 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq2_xxs,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            3.00 |              3.01 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xxs,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            4.17 |              4.19 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xxs,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            5.05 |              5.08 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xxs,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            5.84 |              5.86 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xxs,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1 |           74.68 |             74.76 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq2_xxs,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            6.93 |              6.96 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_s,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            1.56 |              1.57 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq3_s,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            2.88 |              2.90 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq3_s,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            4.05 |              4.06 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_s,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            5.03 |              5.04 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_s,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            5.86 |              5.88 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_s,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |           73.48 |             73.44 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_s,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            7.07 |              7.09 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_xxs,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            2.14 |              2.16 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq3_xxs,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            3.81 |              3.83 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_xxs,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            5.10 |              5.12 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq3_xxs,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            5.83 |              5.85 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_xxs,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            6.88 |              6.90 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_xxs,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1 |           73.48 |             73.65 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq3_xxs,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |            7.71 |              7.75 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq4_nl,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            3.81 |              3.83 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_nl,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            5.39 |              5.41 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_nl,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            7.30 |              7.35 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq4_nl,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            8.74 |              8.77 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_nl,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            8.56 |              8.58 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_nl,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1  |           72.57 |             72.68 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_nl,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            9.27 |              9.36 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq4_xs,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            3.78 |              3.77 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_xs,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            6.57 |              6.59 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_xs,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            8.99 |              9.02 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_xs,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |            9.64 |              9.70 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=iq4_xs,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           10.28 |             10.32 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_xs,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1  |           72.64 |             72.71 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=iq4_xs,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           10.15 |             10.20 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=mxfp4,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            3.58 |              3.61 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=mxfp4,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            5.08 |              5.13 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=mxfp4,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            7.19 |              7.25 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=mxfp4,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            8.55 |              8.65 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=mxfp4,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            8.72 |              8.72 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=mxfp4,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1   |           74.08 |             74.29 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=mxfp4,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1     |            9.15 |              9.24 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q2_K,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            2.87 |              2.88 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q2_K,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.71 |              3.73 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q2_K,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.03 |              4.07 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q2_K,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.22 |              4.25 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q2_K,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.33 |              4.35 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q2_K,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           72.45 |             72.60 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q2_K,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.37 |              4.40 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q3_K,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            1.72 |              1.74 |      1.02 |
| ROCm0     | MUL_MAT   | type_a=q3_K,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            2.83 |              2.85 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q3_K,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.42 |              3.46 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q3_K,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.84 |              3.88 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q3_K,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.12 |              4.13 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q3_K,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           68.80 |             68.89 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q3_K,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.42 |              4.44 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_0,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.05 |              4.08 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_0,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            5.50 |              5.53 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_0,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            7.40 |              7.45 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_0,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.77 |              8.82 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_0,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.64 |              8.63 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_0,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           72.69 |             72.98 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_0,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            9.48 |              9.51 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_1,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.99 |              4.00 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_1,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            6.92 |              6.96 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_1,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            7.65 |              7.70 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_1,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            9.05 |              9.12 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_1,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.75 |              8.76 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_1,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           72.43 |             72.55 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_1,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            9.74 |              9.86 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_K,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            2.71 |              2.75 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_K,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.70 |              3.73 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_K,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.06 |              4.08 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_K,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.25 |              4.28 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_K,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.38 |              4.40 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q4_K,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           72.11 |             72.19 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q4_K,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.58 |              4.61 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_0,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.12 |              3.14 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_0,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            5.24 |              5.26 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_0,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            6.44 |              6.47 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_0,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            7.83 |              7.88 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_0,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.18 |              8.18 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_0,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           63.07 |             63.16 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_0,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.89 |              8.97 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_1,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.37 |              3.39 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_1,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            5.62 |              5.60 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_1,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            7.07 |              7.14 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_1,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.08 |              8.10 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_1,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.82 |              8.84 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_1,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           63.58 |             63.86 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_1,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            9.56 |              9.69 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_K,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            2.55 |              2.57 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_K,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.55 |              3.57 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_K,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.92 |              3.95 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_K,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.13 |              4.16 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_K,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.31 |              4.33 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q5_K,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           69.48 |             69.57 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q5_K,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.50 |              4.53 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q6_K,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            1.84 |              1.86 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q6_K,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            2.91 |              2.94 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q6_K,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            3.67 |              3.71 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q6_K,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.16 |              4.20 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q6_K,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.52 |              4.54 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q6_K,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           69.97 |             70.02 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q6_K,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            5.16 |              5.20 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q8_0,type_b=f32,m=4096,n=1,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            2.57 |              2.56 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q8_0,type_b=f32,m=4096,n=2,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            4.66 |              4.67 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q8_0,type_b=f32,m=4096,n=3,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            6.47 |              6.54 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q8_0,type_b=f32,m=4096,n=4,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            7.42 |              7.50 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q8_0,type_b=f32,m=4096,n=5,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            7.81 |              7.88 |      1.01 |
| ROCm0     | MUL_MAT   | type_a=q8_0,type_b=f32,m=4096,n=512,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1    |           68.41 |             68.60 |      1.00 |
| ROCm0     | MUL_MAT   | type_a=q8_0,type_b=f32,m=4096,n=8,k=14336,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1      |            8.04 |              8.15 |      1.01 |

</details>

model: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B

<details>

<summary>bf16</summary>

| GPU        | Model           |   Microbatch size | Test   |   t/s master |   t/s 698c9f24 |   Speedup |
|:-----------|:----------------|------------------:|:-------|-------------:|---------------:|----------:|
| RX 9070 XT | qwen2 1.5B BF16 |                 1 | pp512  |       156.89 |         157.12 |      1.00 |
| RX 9070 XT | qwen2 1.5B BF16 |                 2 | pp512  |       250.88 |         250.35 |      1.00 |
| RX 9070 XT | qwen2 1.5B BF16 |                 4 | pp512  |       375.40 |         518.08 |      1.38 |
| RX 9070 XT | qwen2 1.5B BF16 |                 8 | pp512  |       744.81 |        1039.92 |      1.40 |
| RX 9070 XT | qwen2 1.5B BF16 |                16 | pp512  |      1466.76 |        2014.38 |      1.37 |
| RX 9070 XT | qwen2 1.5B BF16 |                32 | pp512  |      2803.02 |        2758.97 |      0.98 |
| RX 9070 XT | qwen2 1.5B BF16 |                64 | pp512  |      4754.45 |        4704.70 |      0.99 |
| RX 9070 XT | qwen2 1.5B BF16 |               128 | pp512  |      7872.05 |        7756.98 |      0.99 |
| RX 9070 XT | qwen2 1.5B BF16 |               256 | pp512  |     11419.86 |       11277.21 |      0.99 |
| RX 9070 XT | qwen2 1.5B BF16 |               512 | pp512  |     14612.64 |       14515.26 |      0.99 |

</details>

<details>

<summary>f16</summary>

| GPU        | Model          |   Microbatch size | Test   |   t/s master |   t/s 698c9f24 |   Speedup |
|:-----------|:---------------|------------------:|:-------|-------------:|---------------:|----------:|
| RX 9070 XT | qwen2 1.5B F16 |                 1 | pp512  |       156.74 |         156.50 |      1.00 |
| RX 9070 XT | qwen2 1.5B F16 |                 2 | pp512  |       245.37 |         245.48 |      1.00 |
| RX 9070 XT | qwen2 1.5B F16 |                 4 | pp512  |       398.99 |         518.60 |      1.30 |
| RX 9070 XT | qwen2 1.5B F16 |                 8 | pp512  |       796.58 |        1033.86 |      1.30 |
| RX 9070 XT | qwen2 1.5B F16 |                16 | pp512  |      1592.32 |        2017.26 |      1.27 |
| RX 9070 XT | qwen2 1.5B F16 |                32 | pp512  |      3022.76 |        2982.65 |      0.99 |
| RX 9070 XT | qwen2 1.5B F16 |                64 | pp512  |      5170.13 |        5083.67 |      0.98 |
| RX 9070 XT | qwen2 1.5B F16 |               128 | pp512  |      8504.90 |        8457.53 |      0.99 |
| RX 9070 XT | qwen2 1.5B F16 |               256 | pp512  |     12656.47 |       12706.78 |      1.00 |
| RX 9070 XT | qwen2 1.5B F16 |               512 | pp512  |     18188.82 |       18325.39 |      1.01 |

</details>

<details>

<summary>f32</summary>

| GPU        | Model              |   Microbatch size | Test   |   t/s master |   t/s 698c9f24 |   Speedup |
|:-----------|:-------------------|------------------:|:-------|-------------:|---------------:|----------:|
| RX 9070 XT | qwen2 1.5B all F32 |                 1 | pp512  |        99.78 |          99.41 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |                 2 | pp512  |       185.40 |         185.05 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |                 4 | pp512  |       343.46 |         341.67 |      0.99 |
| RX 9070 XT | qwen2 1.5B all F32 |                 8 | pp512  |       494.70 |         493.65 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |                16 | pp512  |       725.07 |         723.68 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |                32 | pp512  |      1144.63 |        1145.15 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |                64 | pp512  |      1409.09 |        1404.45 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |               128 | pp512  |       846.13 |         842.86 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |               256 | pp512  |      1263.61 |        1263.73 |      1.00 |
| RX 9070 XT | qwen2 1.5B all F32 |               512 | pp512  |      1370.07 |        1361.65 |      0.99 |

</details>

Best Regards
Hui


Changed files:
filename: ggml/src/ggml-cuda/mmf.cu
patch: @@ -151,7 +151,7 @@ bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const
             return false;
         }
     } else {
-        if (src1_ncols > 16 || GGML_CUDA_CC_IS_RDNA4(cc)) {
+        if (src1_ncols > 16) {
             return false;
         }
     }