Repository: ggml-org/llama.cpp
Pull request: sycl: disable reorder for sycl mulmat

The reorder optimisation introduced a prompt processing performance regression for Q4_0 models. This PR disables reorder for the sycl mulmat which is the culprit of this regression. 

Some performance numbers on Arc A770

- Before this PR with `GGML_SYCL_DISABLE_OPT=1`

| model                          |       size |     params | backend    | ngl |    sm |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | ----: | --------------: | -------------------: |
| qwen2 1.5B Q4_0                | 1013.62 MiB |     1.78 B | SYCL       |  99 |  none |           pp512 |       4438.24 ± 4.02 |
| qwen2 1.5B Q4_0                | 1013.62 MiB |     1.78 B | SYCL       |  99 |  none |           tg128 |         40.81 ± 0.34 |
| llama 7B Q4_0                  |   3.57 GiB |     6.74 B | SYCL       |  99 |  none |           pp512 |       1711.58 ± 2.02 |
| llama 7B Q4_0                  |   3.57 GiB |     6.74 B | SYCL       |  99 |  none |           tg128 |         29.92 ± 0.21 |
| llama 7B Q4_0                  |   3.56 GiB |     6.74 B | SYCL       |  99 |  none |           pp512 |       1709.65 ± 1.31 |
| llama 7B Q4_0                  |   3.56 GiB |     6.74 B | SYCL       |  99 |  none |           tg128 |         29.90 ± 0.20 |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | SYCL       |  99 |  none |           pp512 |       1663.50 ± 2.72 |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | SYCL       |  99 |  none |           tg128 |         27.32 ± 0.23 |
| phi3 3B Q4_0                   |   2.03 GiB |     3.82 B | SYCL       |  99 |  none |           pp512 |       2453.67 ± 1.69 |
| phi3 3B Q4_0                   |   2.03 GiB |     3.82 B | SYCL       |  99 |  none |           tg128 |         36.18 ± 0.38 |
| qwen3 4B Q4_0                  |   2.20 GiB |     4.02 B | SYCL       |  99 |  none |           pp512 |       2237.05 ± 1.96 |
| qwen3 4B Q4_0                  |   2.20 GiB |     4.02 B | SYCL       |  99 |  none |           tg128 |         27.97 ± 0.26 |
| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | SYCL       |  99 |  none |           pp512 |       2246.32 ± 1.79 |
| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | SYCL       |  99 |  none |           tg128 |         27.76 ± 0.21 |

build: 24e86cae (5377)

- Before this PR with `GGML_SYCL_DISABLE_OPT=0`

| model                          |       size |     params | backend    | ngl |    sm |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | ----: | --------------: | -------------------: |
| qwen2 1.5B Q4_0                | 1013.62 MiB |     1.78 B | SYCL       |  99 |  none |           pp512 |       4092.76 ± 7.60 |
| qwen2 1.5B Q4_0                | 1013.62 MiB |     1.78 B | SYCL       |  99 |  none |           tg128 |         45.07 ± 0.22 |
| llama 7B Q4_0                  |   3.57 GiB |     6.74 B | SYCL       |  99 |  none |           pp512 |       1468.83 ± 0.91 |
| llama 7B Q4_0                  |   3.57 GiB |     6.74 B | SYCL       |  99 |  none |           tg128 |         33.96 ± 0.19 |
| llama 7B Q4_0                  |   3.56 GiB |     6.74 B | SYCL       |  99 |  none |           pp512 |       1463.11 ± 0.75 |
| llama 7B Q4_0                  |   3.56 GiB |     6.74 B | SYCL       |  99 |  none |           tg128 |         34.26 ± 0.24 |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | SYCL       |  99 |  none |           pp512 |       1406.29 ± 2.22 |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | SYCL       |  99 |  none |           tg128 |         31.41 ± 0.30 |
| phi3 3B Q4_0                   |   2.03 GiB |     3.82 B | SYCL       |  99 |  none |           pp512 |       2165.61 ± 1.38 |
| phi3 3B Q4_0                   |   2.03 GiB |     3.82 B | SYCL       |  99 |  none |           tg128 |         39.70 ± 0.25 |
| qwen3 4B Q4_0                  |   2.20 GiB |     4.02 B | SYCL       |  99 |  none |           pp512 |       1989.89 ± 1.32 |
| qwen3 4B Q4_0                  |   2.20 GiB |     4.02 B | SYCL       |  99 |  none |           tg128 |         31.69 ± 0.29 |
| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | SYCL       |  99 |  none |           pp512 |       1990.26 ± 2.98 |
| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | SYCL       |  99 |  none |           tg128 |         30.98 ± 0.91 |

build: 24e86cae (5377)

- This PR with  `GGML_SYCL_DISABLE_OPT=0`

| model                          |       size |     params | backend    | ngl |    sm |            test |                  t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | ----: | --------------: | -------------------: |
| qwen2 1.5B Q4_0                | 1013.62 MiB |     1.78 B | SYCL       |  99 |  none |           pp512 |      4448.51 ± 10.81 |
| qwen2 1.5B Q4_0                | 1013.62 MiB |     1.78 B | SYCL       |  99 |  none |           tg128 |         45.39 ± 0.33 |
| llama 7B Q4_0                  |   3.57 GiB |     6.74 B | SYCL       |  99 |  none |           pp512 |       1715.47 ± 1.71 |
| llama 7B Q4_0                  |   3.57 GiB |     6.74 B | SYCL       |  99 |  none |           tg128 |         34.33 ± 0.03 |
| llama 7B Q4_0                  |   3.56 GiB |     6.74 B | SYCL       |  99 |  none |           pp512 |       1716.05 ± 1.81 |
| llama 7B Q4_0                  |   3.56 GiB |     6.74 B | SYCL       |  99 |  none |           tg128 |         34.25 ± 0.22 |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | SYCL       |  99 |  none |           pp512 |       1662.76 ± 0.68 |
| llama 8B Q4_0                  |   4.33 GiB |     8.03 B | SYCL       |  99 |  none |           tg128 |         31.59 ± 0.23 |
| phi3 3B Q4_0                   |   2.03 GiB |     3.82 B | SYCL       |  99 |  none |           pp512 |       2455.08 ± 2.67 |
| phi3 3B Q4_0                   |   2.03 GiB |     3.82 B | SYCL       |  99 |  none |           tg128 |         39.73 ± 0.30 |
| qwen3 4B Q4_0                  |   2.20 GiB |     4.02 B | SYCL       |  99 |  none |           pp512 |       2242.72 ± 1.88 |
| qwen3 4B Q4_0                  |   2.20 GiB |     4.02 B | SYCL       |  99 |  none |           tg128 |         31.87 ± 0.27 |
| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | SYCL       |  99 |  none |           pp512 |       2252.19 ± 1.53 |
| qwen3 4B Q4_0                  |   2.21 GiB |     4.02 B | SYCL       |  99 |  none |           tg128 |         31.79 ± 0.24 |

build: 24e86cae (5377)



Changed files:
filename: ggml/src/ggml-sycl/ggml-sycl.cpp
patch: @@ -3028,7 +3028,7 @@ static bool should_reorder_tensor(ggml_backend_sycl_context& ctx, const ggml_ten
     return !g_ggml_sycl_disable_optimize && //allow optimize, controlled by $GGML_SYCL_DISABLE_OPT
             ctx.opt_feature.reorder &&      //allow this device due to good perf, skip the devices with bad perf.
             dst->op == GGML_OP_MUL_MAT &&   //limit to some supported cases of Q4_0, to do for more cases.
-            dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1;
+            dst->src[1]->ne[1]==1 && dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1;
 }
 
 static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * src0, const ggml_tensor * /* src1 */,
@@ -3151,8 +3151,6 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor
         ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_q, convert_src1_to_q8_1);
     } else {
         constexpr bool convert_src1_to_q8_1 = false;
-        // MUL_MAT_SYCL supports reorder
-        opt_for_reorder(&ctx, src0, src1, dst, mul_mat_algo::MUL_MAT_SYCL);
         ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_sycl, convert_src1_to_q8_1);
     }
     GGML_SYCL_DEBUG("call %s done\n", __func__);