Repository: ollama/ollama
File: llm/llm.go (Go)

Code before:
		for opts.NumGPU = 0; opts.NumGPU < int(ggml.KV().BlockCount()) && vram > mem; opts.NumGPU++ {
			layer := ggml.LayerSize(fmt.Sprintf("blk.%d.", opts.NumGPU))
			layerKV := kv / int64(ggml.KV().BlockCount())
			if vram <= mem+layer+layerKV {
				break
			}

			mem += layer + layerKV
		}

		// only offload output tensors if the repeating tensors have been offloaded
		if layer := ggml.LayerSize("output."); opts.NumGPU >= int(ggml.KV().BlockCount()) && vram > mem+layer {
			opts.NumGPU++
			mem += layer
		}

		opts.VRAMUsed = mem
		opts.VRAMTotal = vram
	}

	if opts.NumGPU == 0 || slices.Contains(cpuOnlyFamilies, ggml.KV().Architecture()) {
		info.Library = "cpu"
	}

	slog.Info("llm", "model", model, "adapters", adapters, "projectors", projectors, "opts", opts, "info", info)
	slog.Info("mem", "mem", mem, "vram", vram, "kv", kv, "graph", graph, "layers", opts.NumGPU)
	return newLlmServer(info, model, adapters, projectors, opts)
}

func projectorMemoryRequirements(filename string) int64 {
	file, err := os.Open(filename)
	if err != nil {
		return 0
	}
	defer file.Close()

	ggml, err := DecodeGGML(file)
	if err != nil {
		return 0
	}

	prefixes := make(map[string]struct{})
	for _, layer := range ggml.Tensors() {
		parts := strings.Split(layer.Name, ".")
		prefixes[strings.Join(parts[:2], ".")] = struct{}{}
	}

	var ask int64
	for prefix := range prefixes {
		ask += ggml.LayerSize(prefix)
	}

Review comment (bug):
These sort of feel more like debug statements, but I totally see why setting these at info in the short-term makes sense in case we have any problems, so we can quickly spot where things went sideways on the calculations.  If that was your intent, maybe a comment breadcrumb to remind us to quiet this down to debug level after a few releases?  If your intent was to keep them around longer, maybe expand the message so it's more actionable/understandable for users.

Code after:
		info.Library = "cpu"
	}

	requiredMemory := usedMemory

	var layers int
	for i := 0; i < int(ggml.KV().BlockCount()); i++ {
		layerMemory := ggml.LayerSize(fmt.Sprintf("blk.%d.", i)) + kv/int64(ggml.KV().BlockCount())
		requiredMemory += layerMemory

		if availableMemory > usedMemory+layerMemory && (opts.NumGPU < 0 || layers < opts.NumGPU) {
			usedMemory += layerMemory
			layers++
		}
	}

	memOutputLayer := ggml.LayerSize("output.")
	requiredMemory += memOutputLayer

	// only offload output layer if all repeating layers are offloaded
	if layers >= int(ggml.KV().BlockCount()) && availableMemory > usedMemory+memOutputLayer {
		usedMemory += memOutputLayer
		layers++
	}

	slog.Info(
		"offload to gpu",
		"layers", layers,
		"required", format.HumanBytes2(requiredMemory),
		"used", format.HumanBytes2(usedMemory),
		"available", format.HumanBytes2(availableMemory),
		"kv", format.HumanBytes2(kv),
		"graph", format.HumanBytes2(graph),
	)

	if opts.NumGPU < 0 && info.Library != "cpu" {
		opts.NumGPU = layers
	}

	return newLlmServer(info, model, adapters, projectors, opts)
}

func projectorMemoryRequirements(filename string) int64 {
	file, err := os.Open(filename)
	if err != nil {
		return 0
	}
	defer file.Close()

	ggml, _, err := DecodeGGML(file)
	if err != nil {