Repository: ggml-org/llama.cpp
Pull request: metal : extend mul_mv_ext to BF16, Q2_K, Q3_K

## Summary

- Enable `mul_mv_ext` small-batch kernels (batch sizes 2-8) for **BF16**, **Q2_K**, and **Q3_K** quantization types
- These types previously fell through to the slower single-row `mul_mv` path
- BF16 uses the `float4` dequantize path (like F16); Q2_K/Q3_K use the `float4x4` K-quant path (like Q4_K/Q5_K/Q6_K)

## Performance

### M1 Max (32GB)

Ministral-3B Q3_K_S:

| Test | Master | Patched | Delta |
|------|-------:|--------:|------:|
| pp4 | 110 t/s | **149 t/s** | **+35.5%** |
| pp8 | 114 t/s | **177 t/s** | **+56.2%** |

Qwen3.5-0.8B BF16:

| Test | Master | Patched | Delta |
|------|-------:|--------:|------:|
| pp4 | 142 t/s | **194 t/s** | **+36.9%** |
| pp8 | 212 t/s | **366 t/s** | **+72.8%** |

Qwen3.5-0.8B Q3_K_S:

| Test | Master | Patched | Delta |
|------|-------:|--------:|------:|
| pp4 | 170 t/s | **194 t/s** | **+13.9%** |
| pp8 | 279 t/s | **347 t/s** | **+24.4%** |

Qwen3.5-0.8B Q4_K_M (control, no code change):

| Test | Master | Patched | Delta |
|------|-------:|--------:|------:|
| pp4 | 198 t/s | 199 t/s | +0.5% |
| pp8 | 358 t/s | 359 t/s | +0.2% |

### M4 Pro

Ministral-3B Q3_K_S:

| Test | Master | Patched | Delta |
|------|-------:|--------:|------:|
| pp4 | 148 t/s | **212 t/s** | **+42.8%** |
| pp8 | 169 t/s | **223 t/s** | **+31.6%** |

Qwen3.5-0.8B BF16:

| Test | Master | Patched | Delta |
|------|-------:|--------:|------:|
| pp4 | 288 t/s | **319 t/s** | **+10.6%** |
| pp8 | 467 t/s | **565 t/s** | **+21.1%** |

Note: IQ4_XS was evaluated but excluded due to a consistent small regression (~3%), likely from the complex lookup-table dequantizer interacting poorly with the multi-row kernel's register pressure. Can be revisited in a follow-up.

## Changes

- `ggml-metal.metal`: +18 lines — template instantiations for BF16 (with `GGML_METAL_HAS_BF16` guard), Q2_K, Q3_K
- `ggml-metal-ops.cpp`: +3 lines — type checks in mul_mv_ext dispatch

## Test plan

- [x] `cmake -B build -DGGML_METAL=ON && cmake --build build -j` compiles cleanly
- [x] `test-backend-ops -o MUL_MAT` — all tests pass (M4 Pro: 1014/1014, M1 Max: 3/3 backends)
- [x] `llama-bench` — no regressions on existing types
- [x] Validated on M4 Pro (0.8B, 3B models)
- [x] Validated on M1 Max (0.8B, 3B models)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Changed files:
filename: ggml/src/ggml-metal/ggml-metal-ops.cpp
patch: @@ -1963,6 +1963,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
           (
            op->src[0]->type == GGML_TYPE_F32  || // TODO: helper function
            op->src[0]->type == GGML_TYPE_F16  ||
+           op->src[0]->type == GGML_TYPE_BF16 ||
            op->src[0]->type == GGML_TYPE_Q4_0 ||
            op->src[0]->type == GGML_TYPE_Q4_1 ||
            op->src[0]->type == GGML_TYPE_Q5_0 ||
@@ -1977,6 +1978,8 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
            op->src[0]->type == GGML_TYPE_Q4_K ||
            op->src[0]->type == GGML_TYPE_Q5_K ||
            op->src[0]->type == GGML_TYPE_Q6_K ||
+           op->src[0]->type == GGML_TYPE_Q2_K ||
+           op->src[0]->type == GGML_TYPE_Q3_K ||
            false) && (ne11 >= 4 && ne11 <= 8)
          )
         )

filename: ggml/src/ggml-metal/ggml-metal.metal
patch: @@ -3481,6 +3481,13 @@ template [[host_name("kernel_mul_mv_ext_f16_f32_r1_3")]]    kernel mul_mv_ext_q4
 template [[host_name("kernel_mul_mv_ext_f16_f32_r1_4")]]    kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, half4,        4,  dequantize_f16_t4>;
 template [[host_name("kernel_mul_mv_ext_f16_f32_r1_5")]]    kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, half4,        4,  dequantize_f16_t4>;
 
+#if defined(GGML_METAL_HAS_BF16)
+template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_2")]]   kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, bfloat4,      4,  dequantize_bf16_t4>;
+template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_3")]]   kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, bfloat4,      4,  dequantize_bf16_t4>;
+template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_4")]]   kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, bfloat4,      4,  dequantize_bf16_t4>;
+template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_5")]]   kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, bfloat4,      4,  dequantize_bf16_t4>;
+#endif
+
 template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_2")]]   kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_0,   32, dequantize_q4_0_t4>;
 template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_3")]]   kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_0,   32, dequantize_q4_0_t4>;
 template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_4")]]   kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_0,   32, dequantize_q4_0_t4>;
@@ -3531,6 +3538,16 @@ template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_3")]] kernel mul_mv_ext_q4x4
 template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q6_K, 256, dequantize_q6_K>;
 template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q6_K, 256, dequantize_q6_K>;
 
+template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q2_K, 256, dequantize_q2_K>;
+template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q2_K, 256, dequantize_q2_K>;
+template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q2_K, 256, dequantize_q2_K>;
+template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q2_K, 256, dequantize_q2_K>;
+
+template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q3_K, 256, dequantize_q3_K>;
+template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q3_K, 256, dequantize_q3_K>;
+template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q3_K, 256, dequantize_q3_K>;
+template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q3_K, 256, dequantize_q3_K>;
+
 template<typename T0, typename T1, short NR0, typename args_t>
 void kernel_mul_mv_t_t_impl(
         args_t args,