Repository: ggml-org/llama.cpp
Pull request: Vulkan: Optimize Matmul parameters for AMD GPUs with Coopmat support

I tuned this on AMD Radeon 8060S, but a brief test also showed improvements on AMD RX 9060 XT.

ggml_vulkan: 0 = Radeon 8060S Graphics (RADV GFX1151) (radv) | uma: 1 | fp16: 1 | bf16: 0 | warp size: 64 | shared memory: 65536 | int dot: 1 | matrix cores: KHR_coopmat

| model                 | size     | params | ngl | test          | t/s (ROCm)     | t/s (before)   | t/s (after)    | diff   |
|-----------------------|----------|--------|-----|---------------|----------------|----------------|----------------|--------|
| llama 8B Q4_0         | 4.33 GiB | 8.03 B | 99  | pp512         | 815.22 ± 16.88 | 644.84 ± 4.49  | 875.24 ± 17.99 | +35.7% |
| llama 8B Q4_0         | 4.33 GiB | 8.03 B | 99  | pp512 @ d8192 | 321.66 ± 3.02  | 384.27 ± 0.29  | 463.72 ± 1.32  | +20.7% |
| llama 8B Q4_K - Small | 4.36 GiB | 8.03 B | 99  | pp512         | 995.81 ± 32.32 | 529.50 ± 13.07 | 793.51 ± 9.67  | +49.9% |
| llama 8B Q4_K - Small | 4.36 GiB | 8.03 B | 99  | pp512 @ d8192 | 352.08 ± 3.17  | 341.86 ± 1.96  | 435.55 ± 2.63  | +27.4% |
| llama 8B Q8_0         | 7.95 GiB | 8.03 B | 99  | pp512         | 755.86 ± 21.11 | 422.37 ± 21.61 | 742.46 ± 15.49 | +75.8% |
| llama 8B Q8_0         | 7.95 GiB | 8.03 B | 99  | pp512 @ d8192 | 317.44 ± 2.31  | 306.97 ± 0.36  | 419.24 ± 4.59  | +36.6% |

ggml_vulkan: 0 = AMD Radeon RX 9060 XT (RADV GFX1200) (radv) | uma: 0 | fp16: 1 | bf16: 1 | warp size: 64 | shared memory: 65536 | int dot: 1 | matrix cores: KHR_coopmat

| model                 | size     | params | ngl | test          | t/s (ROCm)     | t/s (before)   | t/s (after)    | diff   |
|-----------------------|----------|--------|-----|---------------|----------------|----------------|----------------|--------|
| llama 8B Q4_K - Small | 4.36 GiB | 8.03 B | 99  | pp512         | 648.41 ± 14.37 | 1437.13 ± 1.77 | 1902.23 ± 1.65 | +32.4% |
| llama 8B Q4_K - Small | 4.36 GiB | 8.03 B | 99  | pp512 @ d8192 | 410.01 ± 6.22  | 757.59 ± 2.71  | 841.76 ± 3.02  | +11.1% |

Changed files:
filename: ggml/src/ggml-vulkan/ggml-vulkan.cpp
patch: @@ -3002,6 +3002,11 @@ static void ggml_vk_load_shaders(vk_device& device) {
         if ((device->architecture == AMD_GCN) && (device->driver_id != vk::DriverId::eAmdProprietary)) {
             m_warptile_mmq = m_warptile_mmq_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 };
             m_warptile_mmqid = m_warptile_mmqid_int = { 256, 64, 64, 32, 16, 16, 2, 2, 2, 1, 16 };
+        } else if (device->vendor_id == VK_VENDOR_ID_AMD && device->coopmat_support) {
+            // This is intentionally using tx_m values, slight performance increase
+            l_warptile = { 256, 128, 128, 16, subgroup_size_8, 64, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
+            l_warptile_mmq = l_warptile_mmq_int = { 256, 128, 128, 32, subgroup_size_8, 64, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
+            l_warptile_mmq_int_k = { 256, 128, 128, 32, subgroup_size_16, 64, 1, 4, 2, 1, subgroup_size_16 };
         } else if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && device->architecture == INTEL_XE2) {
             // Xe2/Xe3 with coopmat enabled - warptile performance tuning
             l_warptile = { 512, 128, 128, 16, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
@@ -5078,7 +5083,7 @@ static vk_device ggml_vk_get_device(size_t idx) {
             switch (device->vendor_id) {
 #ifndef GGML_VULKAN_RUN_TESTS
             case VK_VENDOR_ID_AMD:
-                device->mul_mat_l[i]    = false;
+                device->mul_mat_l[i]    = device->coopmat_support;
                 device->mul_mat_m[i]    = true;
                 device->mul_mat_s[i]    = true;
                 device->mul_mat_id_l[i] = false;