Repository: ollama/ollama
Pull request: Tighten up memory prediction logging

Prior to this change, we logged the memory prediction multiple times as the scheduler iterates to find a suitable configuration, which can be confusing since only the last log before the server starts is actually valid. This now logs once just before starting the server on the final configuration. It also reports what library instead of always saying "offloading to gpu" when using CPU.

A few examples (non-debug regular logging level):
```
time=2024-06-17T19:07:15.507-07:00 level=INFO source=types.go:98 msg="inference compute" id=0 library=metal compute="" driver=0.0 name="" total="96.0 GiB" available="96.0 GiB"
[GIN] 2024/06/17 - 19:07:33 | 200 |     313.875µs |       127.0.0.1 | HEAD     "/"
[GIN] 2024/06/17 - 19:07:33 | 200 |    2.473333ms |       127.0.0.1 | POST     "/api/show"
time=2024-06-17T19:07:33.438-07:00 level=INFO source=memory.go:303 msg="offload to metal" layers.requested=-1 layers.model=27 layers.offload=27 layers.split="" memory.available="[96.0 GiB]" memory.required.full="3.2 GiB" memory.required.partial="3.2 GiB" memory.required.kv="650.0 MiB" memory.required.allocations="[3.2 GiB]" memory.weights.total="2.3 GiB" memory.weights.repeating="2.2 GiB" memory.weights.nonrepeating="103.8 MiB" memory.graph.full="157.0 MiB" memory.graph.partial="157.0 MiB"
time=2024-06-17T19:07:33.439-07:00 level=INFO source=server.go:359 msg="starting llama server" cmd="/var/folders/hs/0tcx8spd1vv390h0j6jq5vq80000gn/T/ollama3083603568/runners/metal/ollama_llama_server --model /Users/daniel/.ollama/models/blobs/sha256-66002b78c70a22ab25e16cc9a1736c6cc6335398c7312e3eb33db202350afe66 --ctx-size 2048 --batch-size 512 --embedding --log-disable --n-gpu-layers 27 --parallel 1 --port 63538"
```

```
time=2024-06-18T02:09:26.404Z level=WARN source=gpu.go:225 msg="CPU does not have minimum vector extensions, GPU inference disabled" required=avx detected="no vector extensions"
time=2024-06-18T02:09:26.405Z level=INFO source=types.go:98 msg="inference compute" id=0 library=cpu compute="" driver=0.0 name="" total="31.3 GiB" available="30.4 GiB"
[GIN] 2024/06/18 - 02:09:36 | 200 |     662.875µs |       127.0.0.1 | HEAD     "/"
[GIN] 2024/06/18 - 02:09:36 | 200 |    3.819958ms |       127.0.0.1 | POST     "/api/show"
time=2024-06-18T02:09:36.898Z level=INFO source=memory.go:303 msg="offload to cpu" layers.requested=-1 layers.model=27 layers.offload=0 layers.split="" memory.available="[30.4 GiB]" memory.required.full="2.7 GiB" memory.required.partial="0 B" memory.required.kv="650.0 MiB" memory.required.allocations="[2.7 GiB]" memory.weights.total="2.3 GiB" memory.weights.repeating="2.2 GiB" memory.weights.nonrepeating="103.8 MiB" memory.graph.full="157.0 MiB" memory.graph.partial="177.2 MiB"
time=2024-06-18T02:09:36.905Z level=INFO source=server.go:359 msg="starting llama server" cmd="/tmp/ollama284292296/runners/cpu/ollama_llama_server --model /root/.ollama/models/blobs/sha256-66002b78c70a22ab25e16cc9a1736c6cc6335398c7312e3eb33db202350afe66 --ctx-size 2048 --batch-size 512 --embedding --log-disable --parallel 1 --port 43251"
```

```
time=2024-06-17T19:02:51.398-07:00 level=INFO source=types.go:98 msg="inference compute" id=GPU-1c750365-54dc-7082-7c6b-9dd953a68ab6 library=cuda compute=6.1 driver=12.3 name="NVIDIA GeForce GTX 1060 6GB" total="5.9 GiB" available="5.7 GiB"
[GIN] 2024/06/17 - 19:02:57 | 200 |      28.835µs |       127.0.0.1 | HEAD     "/"
[GIN] 2024/06/17 - 19:02:57 | 200 |     453.399µs |       127.0.0.1 | POST     "/api/show"
time=2024-06-17T19:02:58.233-07:00 level=INFO source=memory.go:303 msg="offload to cuda" layers.requested=-1 layers.model=27 layers.offload=27 layers.split="" memory.available="[5.7 GiB]" memory.required.full="3.1 GiB" memory.required.partial="3.1 GiB" memory.required.kv="650.0 MiB" memory.required.allocations="[3.1 GiB]" memory.weights.total="2.3 GiB" memory.weights.repeating="2.2 GiB" memory.weights.nonrepeating="103.8 MiB" memory.graph.full="157.0 MiB" memory.graph.partial="177.2 MiB"
time=2024-06-17T19:02:58.233-07:00 level=INFO source=server.go:359 msg="starting llama server" cmd="/tmp/ollama3201791839/runners/cuda_v11/ollama_llama_server --model /home/daniel/.ollama/models/blobs/sha256-66002b78c70a22ab25e16cc9a1736c6cc6335398c7312e3eb33db202350afe66 --ctx-size 2048 --batch-size 512 --embedding --log-disable --n-gpu-layers 27 --parallel 1 --port 43155"
```

Changed files:
filename: llm/memory.go
patch: @@ -49,6 +49,18 @@ type MemoryEstimate struct {
 
 	// For multi-GPU scenarios, this is the size in bytes per GPU
 	GPUSizes []uint64
+
+	// internal fields for logging purposes
+	inferenceLibrary    string
+	layersRequested     int
+	layersModel         int
+	availableList       []string
+	kv                  uint64
+	allocationsList     []string
+	memoryWeights       uint64
+	memoryLayerOutput   uint64
+	graphFullOffload    uint64
+	graphPartialOffload uint64
 }
 
 // Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size
@@ -252,78 +264,86 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
 		allocationsList = append(allocationsList, format.HumanBytes2(a))
 	}
 
+	estimate := MemoryEstimate{
+		TotalSize: memoryRequiredTotal,
+		Layers:    0,
+		Graph:     0,
+		VRAMSize:  0,
+		GPUSizes:  []uint64{},
+
+		inferenceLibrary:    gpus[0].Library,
+		layersRequested:     opts.NumGPU,
+		layersModel:         int(ggml.KV().BlockCount()) + 1,
+		availableList:       availableList,
+		kv:                  kv,
+		allocationsList:     allocationsList,
+		memoryWeights:       memoryWeights,
+		memoryLayerOutput:   memoryLayerOutput,
+		graphFullOffload:    graphFullOffload,
+		graphPartialOffload: graphPartialOffload,
+	}
+
+	if gpus[0].Library == "cpu" {
+		return estimate
+	}
+	if layerCount == 0 {
+		slog.Debug("insufficient VRAM to load any model layers")
+		return estimate
+	}
+	estimate.Layers = layerCount
+	estimate.Graph = graphOffload
+	estimate.VRAMSize = memoryRequiredPartial
+	estimate.TotalSize = memoryRequiredTotal
+	estimate.TensorSplit = tensorSplit
+	estimate.GPUSizes = gpuAllocations
+	return estimate
+}
+
+func (m MemoryEstimate) log() {
 	slog.Info(
-		"offload to gpu",
+		"offload to "+m.inferenceLibrary,
 		slog.Group(
 			"layers",
 			// requested number of layers to offload
-			"requested", opts.NumGPU,
+			"requested", m.layersRequested,
 			// The number of layers the model has (including output)
-			"model", int(ggml.KV().BlockCount())+1,
+			"model", m.layersModel,
 			// estimated number of layers that can be offloaded
-			"offload", layerCount,
-			// multi-gpu split for tesnors
-			"split", tensorSplit,
+			"offload", m.Layers,
+			// multi-gpu split for tensors
+			"split", m.TensorSplit,
 		),
 		slog.Group(
 			"memory",
 			// memory available by GPU for offloading
-			"available", availableList,
+			"available", m.availableList,
 			slog.Group(
 				"required",
 				// memory required for full offloading
-				"full", format.HumanBytes2(memoryRequiredTotal),
+				"full", format.HumanBytes2(m.TotalSize),
 				// memory required to offload layers.estimate layers
-				"partial", format.HumanBytes2(memoryRequiredPartial),
+				"partial", format.HumanBytes2(m.VRAMSize),
 				// memory of KV cache
-				"kv", format.HumanBytes2(kv),
+				"kv", format.HumanBytes2(m.kv),
 				// Allocations across the GPUs
-				"allocations", allocationsList,
+				"allocations", m.allocationsList,
 			),
 			slog.Group(
 				"weights",
 				// memory of the weights
-				"total", format.HumanBytes2(memoryWeights),
+				"total", format.HumanBytes2(m.memoryWeights),
 				// memory of repeating layers
-				"repeating", format.HumanBytes2(memoryWeights-memoryLayerOutput),
+				"repeating", format.HumanBytes2(m.memoryWeights-m.memoryLayerOutput),
 				// memory of non-repeating layers
-				"nonrepeating", format.HumanBytes2(memoryLayerOutput),
+				"nonrepeating", format.HumanBytes2(m.memoryLayerOutput),
 			),
 			slog.Group(
 				"graph",
 				// memory of graph when fully offloaded
-				"full", format.HumanBytes2(graphFullOffload),
+				"full", format.HumanBytes2(m.graphFullOffload),
 				// memory of graph when not fully offloaded
-				"partial", format.HumanBytes2(graphPartialOffload),
+				"partial", format.HumanBytes2(m.graphPartialOffload),
 			),
 		),
 	)
-	if gpus[0].Library == "cpu" {
-		return MemoryEstimate{
-			Layers:    0,
-			Graph:     0,
-			VRAMSize:  0,
-			TotalSize: memoryRequiredTotal,
-			GPUSizes:  []uint64{},
-		}
-	}
-	if layerCount == 0 {
-		slog.Debug("insufficient VRAM to load any model layers")
-		return MemoryEstimate{
-			Layers:    0,
-			Graph:     0,
-			VRAMSize:  0,
-			TotalSize: memoryRequiredTotal,
-			GPUSizes:  []uint64{},
-		}
-	}
-
-	return MemoryEstimate{
-		Layers:      layerCount,
-		Graph:       graphOffload,
-		VRAMSize:    memoryRequiredPartial,
-		TotalSize:   memoryRequiredTotal,
-		TensorSplit: tensorSplit,
-		GPUSizes:    gpuAllocations,
-	}
 }

filename: llm/server.go
patch: @@ -116,6 +116,8 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 		}
 	}
 
+	estimate.log()
+
 	// Loop through potential servers
 	finalErr := errors.New("no suitable llama servers found")