Repository: ggml-org/llama.cpp
Pull request: kv-cache : pad the cache size to 256 for performance

fix #17037 #17058
cont #16812

~The small SWA caches can be padded to 256 without concerns about memory usage.~ Pad the cache size to 256. This is friendly for the CUDA backend since the FA implementation benefits from round sizes of the K/V tensors. Can also help other backends.

This is essentially a partial revert of #16812.

Comparing with before the regression in #16812:

```bash
GGML_CUDA=ON CUDA_VISIBLE_DEVICES=0 ./scripts/compare-commits.sh a8ca18b4b d2c30c61a llama-bench -m ~/.cache/llama.cpp/ggml-org_gpt-oss-20b-GGUF_gpt-oss-20b-mxfp4.gguf -m /home/ggerganov/.cache/llama.cpp/ggml-org_Qwen2.5-Coder-3B-Q8_0-GGUF_qwen2.5-coder-3b-q8_0.gguf -ngl 99 -d 4096,8192,16384,32768 -ub 512,4096 -b 4096 -fa 1 -n 32 -mmp 0
```

ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 1 CUDA devices:
  Device 0: NVIDIA GeForce RTX 5090, compute capability 12.0, VMM: yes

| Model                 |   Microbatch size | Test         |   t/s a8ca18b4b |   t/s d2c30c61a |   Speedup |
|:----------------------|------------------:|:-------------|----------------:|----------------:|----------:|
| gpt-oss 20B MXFP4 MoE |               512 | pp512@d4096  |         9702.25 |         9724.85 |      1.00 |
| gpt-oss 20B MXFP4 MoE |               512 | pp512@d8192  |         8609.17 |         8643.88 |      1.00 |
| gpt-oss 20B MXFP4 MoE |               512 | pp512@d16384 |         7169.27 |         7224.84 |      1.01 |
| gpt-oss 20B MXFP4 MoE |               512 | pp512@d32768 |         5349.99 |         5380.43 |      1.01 |
| gpt-oss 20B MXFP4 MoE |               512 | tg32@d4096   |          306.13 |          338.25 |      1.10 |
| gpt-oss 20B MXFP4 MoE |               512 | tg32@d8192   |          292.57 |          317.68 |      1.09 |
| gpt-oss 20B MXFP4 MoE |               512 | tg32@d16384  |          265.66 |          304.90 |      1.15 |
| gpt-oss 20B MXFP4 MoE |               512 | tg32@d32768  |          236.92 |          262.29 |      1.11 |
| gpt-oss 20B MXFP4 MoE |              4096 | pp512@d4096  |         8720.64 |         8735.30 |      1.00 |
| gpt-oss 20B MXFP4 MoE |              4096 | pp512@d8192  |         7908.52 |         7799.28 |      0.99 |
| gpt-oss 20B MXFP4 MoE |              4096 | pp512@d16384 |         6656.47 |         6583.46 |      0.99 |
| gpt-oss 20B MXFP4 MoE |              4096 | pp512@d32768 |         5063.06 |         4967.41 |      0.98 |
| gpt-oss 20B MXFP4 MoE |              4096 | tg32@d4096   |          296.76 |          318.91 |      1.07 |
| gpt-oss 20B MXFP4 MoE |              4096 | tg32@d8192   |          279.30 |          322.56 |      1.15 |
| gpt-oss 20B MXFP4 MoE |              4096 | tg32@d16384  |          251.35 |          283.65 |      1.13 |
| gpt-oss 20B MXFP4 MoE |              4096 | tg32@d32768  |          227.88 |          253.40 |      1.11 |
| qwen2 3B Q8_0         |               512 | pp512@d4096  |        17229.54 |        17278.24 |      1.00 |
| qwen2 3B Q8_0         |               512 | pp512@d8192  |        14011.21 |        14133.12 |      1.01 |
| qwen2 3B Q8_0         |               512 | pp512@d16384 |        10304.85 |        10307.07 |      1.00 |
| qwen2 3B Q8_0         |               512 | pp512@d32768 |         6612.16 |         6567.63 |      0.99 |
| qwen2 3B Q8_0         |               512 | tg32@d4096   |          279.56 |          294.93 |      1.05 |
| qwen2 3B Q8_0         |               512 | tg32@d8192   |          210.92 |          213.51 |      1.01 |
| qwen2 3B Q8_0         |               512 | tg32@d16384  |          185.15 |          188.19 |      1.02 |
| qwen2 3B Q8_0         |               512 | tg32@d32768  |          147.62 |          149.84 |      1.02 |
| qwen2 3B Q8_0         |              4096 | pp512@d4096  |        16599.01 |        16781.20 |      1.01 |
| qwen2 3B Q8_0         |              4096 | pp512@d8192  |        13664.67 |        13715.42 |      1.00 |
| qwen2 3B Q8_0         |              4096 | pp512@d16384 |        10056.36 |        10027.96 |      1.00 |
| qwen2 3B Q8_0         |              4096 | pp512@d32768 |         6585.53 |         6579.99 |      1.00 |
| qwen2 3B Q8_0         |              4096 | tg32@d4096   |          274.54 |          279.28 |      1.02 |
| qwen2 3B Q8_0         |              4096 | tg32@d8192   |          219.26 |          224.87 |      1.03 |
| qwen2 3B Q8_0         |              4096 | tg32@d16384  |          188.45 |          192.18 |      1.02 |
| qwen2 3B Q8_0         |              4096 | tg32@d32768  |          148.30 |          150.20 |      1.01 |

Changed files:
filename: include/llama.h
patch: @@ -463,6 +463,7 @@ extern "C" {
 
     // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions
     //       In some cases the requested values via llama_context_params may differ from the actual values used by the context
+    //       ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
     LLAMA_API uint32_t llama_n_ctx      (const struct llama_context * ctx);
     LLAMA_API uint32_t llama_n_ctx_seq  (const struct llama_context * ctx);
     LLAMA_API uint32_t llama_n_batch    (const struct llama_context * ctx);

filename: src/llama-context.cpp
patch: @@ -114,10 +114,14 @@ llama_context::llama_context(
         }
     }
 
+    // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
+    cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);
+
     if (cparams.kv_unified) {
         cparams.n_ctx_seq = cparams.n_ctx;
     } else {
         cparams.n_ctx_seq = cparams.n_ctx / cparams.n_seq_max;
+        cparams.n_ctx_seq = GGML_PAD(cparams.n_ctx_seq, 256);
 
         if (cparams.n_ctx_seq == 0) {
             throw std::runtime_error("n_ctx_seq == 0");

filename: src/llama-kv-cache-iswa.cpp
patch: @@ -45,7 +45,9 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
 
     const uint32_t size_base = kv_size;
 
-    uint32_t size_swa = std::min(size_base, GGML_PAD(hparams.n_swa*(unified ? n_seq_max : 1) + n_ubatch, n_pad));
+    // note: the SWA cache is always padded to 256 for performance
+    //       https://github.com/ggml-org/llama.cpp/issues/17037
+    uint32_t size_swa = GGML_PAD(std::min(size_base, hparams.n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256);
 
     // when using full-size SWA cache, we set the SWA cache size to be equal to the base cache size
     if (swa_full) {

filename: tools/server/tests/unit/test_speculative.py
patch: @@ -77,10 +77,10 @@ def test_different_draft_min_draft_max():
 
 def test_slot_ctx_not_exceeded():
     global server
-    server.n_ctx = 64
+    server.n_ctx = 256
     server.start()
     res = server.make_request("POST", "/completion", data={
-        "prompt": "Hello " * 56,
+        "prompt": "Hello " * 248,
         "temperature": 0.0,
         "top_k": 1,
         "speculative.p_min": 0.0,
@@ -91,19 +91,19 @@ def test_slot_ctx_not_exceeded():
 
 def test_with_ctx_shift():
     global server
-    server.n_ctx = 64
+    server.n_ctx = 256
     server.enable_ctx_shift = True
     server.start()
     res = server.make_request("POST", "/completion", data={
-        "prompt": "Hello " * 56,
+        "prompt": "Hello " * 248,
         "temperature": 0.0,
         "top_k": 1,
-        "n_predict": 64,
+        "n_predict": 256,
         "speculative.p_min": 0.0,
     })
     assert res.status_code == 200
     assert len(res.body["content"]) > 0
-    assert res.body["tokens_predicted"] == 64
+    assert res.body["tokens_predicted"] == 256
     assert res.body["truncated"] == True