Repository: ggml-org/llama.cpp
Pull request: context : print graph stats for memory-less contexts

Print stats for compute buffers and graph nodes with memory-less contexts (such as for embedding models).

Example:

```bash
llama-embedding -hf ggml-org/bge-small-en-v1.5-Q8_0-GGUF -p "test" -c 512 -b 512 -ub 512
```

```
0.00.379.367 I llama_context: constructing llama_context
0.00.379.370 I llama_context: n_seq_max     = 1
0.00.379.370 I llama_context: n_ctx         = 512
0.00.379.370 I llama_context: n_ctx_per_seq = 512
0.00.379.370 I llama_context: n_batch       = 512
0.00.379.370 I llama_context: n_ubatch      = 512
0.00.379.371 I llama_context: causal_attn   = 0
0.00.379.371 I llama_context: flash_attn    = 0
0.00.379.371 I llama_context: kv_unified    = true
0.00.379.371 I llama_context: freq_base     = 10000.0
0.00.379.372 I llama_context: freq_scale    = 1
0.00.379.372 I ggml_metal_init: allocating
0.00.379.401 I ggml_metal_init: found device: Apple M2 Ultra
0.00.379.404 I ggml_metal_init: picking default device: Apple M2 Ultra
0.00.379.997 I ggml_metal_load_library: using embedded metal library
0.00.384.503 I ggml_metal_init: GPU name:   Apple M2 Ultra
0.00.384.507 I ggml_metal_init: GPU family: MTLGPUFamilyApple8  (1008)
0.00.384.508 I ggml_metal_init: GPU family: MTLGPUFamilyCommon3 (3003)
0.00.384.508 I ggml_metal_init: GPU family: MTLGPUFamilyMetal3  (5001)
0.00.384.508 I ggml_metal_init: simdgroup reduction   = true
0.00.384.509 I ggml_metal_init: simdgroup matrix mul. = true
0.00.384.509 I ggml_metal_init: has residency sets    = true
0.00.384.509 I ggml_metal_init: has bfloat            = true
0.00.384.509 I ggml_metal_init: use bfloat            = true
0.00.384.510 I ggml_metal_init: hasUnifiedMemory      = true
0.00.384.511 I ggml_metal_init: recommendedMaxWorkingSetSize  = 154618.82 MB
0.00.405.574 I llama_context:        CPU  output buffer size =     0.12 MiB
0.00.406.905 I llama_context:      Metal compute buffer size =    16.75 MiB
0.00.406.909 I llama_context:        CPU compute buffer size =     2.51 MiB
0.00.406.909 I llama_context: graph nodes  = 431
0.00.406.910 I llama_context: graph splits = 2
```

Changed files:
filename: src/llama-context.cpp
patch: @@ -280,7 +280,7 @@ llama_context::llama_context(
     }
 
     // reserve worst-case graph
-    if (!hparams.vocab_only && memory) {
+    if (!hparams.vocab_only) {
         const uint32_t n_seqs = cparams.kv_unified ? 1 : cparams.n_seq_max;
         const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
 
@@ -292,11 +292,13 @@ llama_context::llama_context(
         int n_splits_tg = -1;
         int n_nodes_tg  = -1;
 
-        // simulate full KV cache
-
-        const auto mctx = memory->init_full();
-        if (!mctx) {
-            throw std::runtime_error("failed to initialize KV cache");
+        llama_memory_context_ptr mctx;
+        if (memory) {
+            LLAMA_LOG_DEBUG("%s: reserving full memory module\n", __func__);
+            mctx = memory->init_full();
+            if (!mctx) {
+                throw std::runtime_error("failed to initialize memory module");
+            }
         }
 
         cross.v_embd.clear();
@@ -1056,7 +1058,7 @@ int llama_context::decode(const llama_batch & batch_inp) {
         const auto * res = process_ubatch(ubatch, LLM_GRAPH_TYPE_DECODER, mctx.get(), status);
 
         if (!res) {
-            // the last ubatch failed or was aborted -> remove all positions of that ubatch from the KV cache
+            // the last ubatch failed or was aborted -> remove all positions of that ubatch from the memory module
             llama_pos pos_min[LLAMA_MAX_SEQ];
             for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {
                 pos_min[s] = std::numeric_limits<llama_pos>::max();
@@ -1073,7 +1075,7 @@ int llama_context::decode(const llama_batch & batch_inp) {
                     continue;
                 }
 
-                LLAMA_LOG_WARN("%s: removing KV cache entries for seq_id = %d, pos = [%d, +inf)\n", __func__, s, pos_min[s]);
+                LLAMA_LOG_WARN("%s: removing memory module entries for seq_id = %d, pos = [%d, +inf)\n", __func__, s, pos_min[s]);
 
                 memory->seq_rm(s, pos_min[s], -1);
             }
@@ -1857,7 +1859,7 @@ size_t llama_context::state_write_data(llama_io_write_i & io) {
     }
 
     if (memory != nullptr) {
-        LLAMA_LOG_DEBUG("%s: - writing KV self\n", __func__);
+        LLAMA_LOG_DEBUG("%s: - writing memory module\n", __func__);
         memory->state_write(io);
     }
 
@@ -1943,7 +1945,7 @@ size_t llama_context::state_read_data(llama_io_read_i & io) {
     }
 
     if (memory) {
-        LLAMA_LOG_DEBUG("%s: - reading KV self\n", __func__);
+        LLAMA_LOG_DEBUG("%s: - reading memory module\n", __func__);
 
         memory->state_read(io);
     }