Repository: ggml-org/llama.cpp
Pull request: Vulkan: Tune Vulkan mmq int dot shader for performance

Retune the DP4A matmul shaders I added in #12135 to extract more performance from them after a bugfix reduced their performance in #12722. Here are my test results:

RTX 3090:

| model                          |       size |     params | backend    | ngl |          test |             t/s fp16 |   t/s int dot master |       t/s int dot PR |         t/s coopmat1 |         t/s coopmat2 |             t/s CUDA |
| ------------------------------ | ---------: | ---------: | ---------- | --: | ------------: | -------------------: | -------------------: | -------------------: | -------------------: | -------------------: | -------------------: |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | Vulkan     |  99 |         pp512 |       1025.95 ± 1.11 |       1928.35 ± 8.34 |       1925.12 ± 6.65 |      3138.60 ± 28.29 |      4247.01 ± 60.18 |      5069.38 ± 18.59 |
| llama 8B Q8_0                  |   7.95 GiB |     8.03 B | Vulkan     |  99 |         pp512 |       1000.95 ± 2.89 |       1898.78 ± 4.69 |       1894.99 ± 7.38 |       2749.46 ± 3.65 |      4329.58 ± 16.31 |      4932.49 ± 14.11 |


AMD Radeon RX 6800 XT:

| model                          |       size |     params | backend    | ngl |          test |    t/s fp16 |   t/s int dot master |       t/s int dot PR |             t/s ROCm |
| ------------------------------ | ---------: | ---------: | ---------- | --: | ------------: | -------------------: | -------------------: | -------------------: | -------------------: |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | Vulkan     |  99 |         pp512 |        922.14 ± 1.04 |       1284.95 ± 2.76 |       1463.92 ± 1.74 |       1678.14 ± 2.28 |
| llama 8B Q8_0                  |   7.95 GiB |     8.03 B | Vulkan     |  99 |         pp512 |        902.82 ± 0.80 |       1060.93 ± 0.74 |       1250.73 ± 1.23 |       1618.84 ± 1.25 |


AMD Radeon Pro VII:

| model                          |       size |     params | backend    | ngl |          test |    t/s fp16 |   t/s int dot master |       t/s int dot PR |             t/s ROCm |
| ------------------------------ | ---------: | ---------: | ---------- | --: | ------------: | -------------------: | -------------------: | -------------------: | -------------------: |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | Vulkan     |  99 |         pp512 |        313.00 ± 0.60 |        390.98 ± 1.52 |        588.09 ± 0.42 |       1012.39 ± 0.46 |
| llama 8B Q8_0                  |   7.95 GiB |     8.03 B | Vulkan     |  99 |         pp512 |        309.01 ± 0.48 |        317.19 ± 0.91 |        501.33 ± 0.55 |        398.53 ± 0.06 |


Intel A770:

| model                          |       size |     params | backend    | ngl |          test |    t/s fp16 |   t/s int dot master |       t/s int dot PR |             t/s SYCL |
| ------------------------------ | ---------: | ---------: | ---------- | --: | ------------: | -------------------: | -------------------: | -------------------: | -------------------: |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | Vulkan     |  99 |         pp512 |        165.47 ± 0.14 |        508.05 ± 0.95 |        734.05 ± 1.60 |        917.26 ± 6.47 |
| llama 8B Q8_0                  |   7.95 GiB |     8.03 B | Vulkan     |  99 |         pp512 |        157.58 ± 0.17 |        493.80 ± 0.76 |        678.78 ± 0.75 |        893.41 ± 4.35 |

Changed files:
filename: ggml/src/ggml-vulkan/ggml-vulkan.cpp
patch: @@ -1736,19 +1736,9 @@ static void ggml_vk_load_shaders(vk_device& device) {
         m_warptile_mmq = { 128,  64,  64, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
         s_warptile_mmq = { subgroup_size_32, 32, 32, 32, 32, 32, 2, tm_s, tn_s, tk_s, subgroup_size_8 };
 
-        const uint32_t tm_int_l = device->coopmat_int_support ? device->coopmat_int_m : 4;
-        const uint32_t tm_int_m = device->coopmat_int_support ? device->coopmat_int_m : 4;
-        const uint32_t tm_int_s = device->coopmat_int_support ? device->coopmat_int_m : 2;
-        const uint32_t tn_int_l = device->coopmat_int_support ? device->coopmat_int_n : 4;
-        const uint32_t tn_int_m = device->coopmat_int_support ? device->coopmat_int_n : 2;
-        const uint32_t tn_int_s = device->coopmat_int_support ? device->coopmat_int_n : 2;
-        const uint32_t tk_int_l = device->coopmat_int_support ? device->coopmat_int_k : 1;
-        const uint32_t tk_int_m = device->coopmat_int_support ? device->coopmat_int_k : 1;
-        const uint32_t tk_int_s = device->coopmat_int_support ? device->coopmat_int_k : 1;
-
-        l_warptile_mmq_int = { 128, 128, 128, 32, subgroup_size_8 * 2, 64, 2, tm_int_l, tn_int_l, tk_int_l, subgroup_size_8 };
-        m_warptile_mmq_int = { 128,  64,  64, 32, subgroup_size_8, 32, 2,     tm_int_m, tn_int_m, tk_int_m, subgroup_size_8 };
-        s_warptile_mmq_int = { subgroup_size_32, 32, 32, 32, 32, 32, 2,       tm_int_s, tn_int_s, tk_int_s, subgroup_size_8 };
+        l_warptile_mmq_int = { 128, 128, 128, 32, subgroup_size_8 * 2, 64, 2, 4, 4, 1, subgroup_size_8 };
+        m_warptile_mmq_int = { 128,  64,  64, 32, subgroup_size_8,     32, 2, 2, 2, 1, subgroup_size_8 };
+        s_warptile_mmq_int = { subgroup_size_32, 32, 32, 32, 32,       32, 2, 2, 1, 1, subgroup_size_8 };
 
         l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 };
         m_mmq_wg_denoms = m_wg_denoms = { 64,  64, 1 };
@@ -7922,7 +7912,7 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) {
         128, 49, 49,
         4096, 49, 4096,
     };
-    const size_t num_it = 1;
+    const size_t num_it = 100;
 
     ggml_vk_test_dequant_matmul(ctx, 4096, 512, 4096, 2, num_it, 1, 0, GGML_TYPE_Q4_0);
     ggml_vk_test_dequant_matmul(ctx, 4096, 512, 4096, 2, num_it, 1, 1, GGML_TYPE_Q4_0);