Repository: ollama/ollama
File: llm/llm.go (Go)

Code before:

	// this amount is the overhead + tensors in memory
	// TODO: get this from the llama.cpp's graph calcluations instead of
	// guessing it's ~1/8th of the kv cache times gqa
	// this is slightly (~25%) higher than what it is in reality
	// TODO: this needs to account for batch size != 512
	requiredAlloc := int64(ggml.NumGQA()) * requiredKv / 8

	requiredTotal := requiredModel + requiredKv + requiredAlloc

	fmt.Println("system memory ", available)
	fmt.Println("required model", requiredModel)
	fmt.Println("required kv", requiredKv)
	fmt.Println("required alloc", requiredAlloc)
	fmt.Println("required total", requiredTotal)

	if opts.NumGPU == -1 {
		opts.NumGPU = int(ggml.NumLayers()) + 1
	}

	// decide how many layers to put on the GPU
	if opts.NumGPU > 0 {
		switch runtime.GOOS {
		case "darwin":
			if requiredTotal > available {
				fmt.Println("not enough vram available, falling back to CPU only")
				opts.NumGPU = 0
			}
		default:
			info := gpu.GetGPUInfo()
			if info.Library == "cpu" || info.Library == "default" {
				opts.NumGPU = 0
				break
			}

			if requiredTotal <= available {
				break
			}

			// overhead + tensors are always loaded into
			// scratch memory and so don't offload if there isn't even
			// enough room for that
			if requiredAlloc > available {
				fmt.Println("not enough vram available, falling back to CPU only")
				opts.NumGPU = 0
				break
			}

			available -= requiredAlloc

			// fill remaining vram with layers

Review comment (suggestion):
```suggestion
				log.Printf("not enough vram available, %sMB required, falling back to CPU only", requiredTotal)
```
This is probably logged elsewhere, in that case ignore this comment

Code after:

	// this amount is the overhead + tensors in memory
	// TODO: get this from the llama.cpp's graph calcluations instead of
	// guessing it's ~1/7th of the kv cache times gqa
	requiredAlloc := int64(ggml.NumGQA()) * requiredKv / 7

	requiredTotal := requiredModel + requiredKv + requiredAlloc

	log.Println("system memory bytes:", available)
	log.Println("required model bytes:", requiredModel)
	log.Println("required kv bytes:", requiredKv)
	log.Println("required alloc bytes:", requiredAlloc)
	log.Println("required total bytes:", requiredTotal)

	info := gpu.GetGPUInfo()
	library := info.Library

	if opts.NumGPU == -1 {
		// default to offloading all layers
		opts.NumGPU = int(ggml.NumLayers()) + 1
	}

	// decide how many layers to put on the GPU
	if opts.NumGPU > 0 {
		switch runtime.GOOS {
		case "darwin":
			if requiredTotal > available {
				log.Println("not enough vram available, falling back to CPU only")
				opts.NumGPU = 0
			}
		default:
			if library == "cpu" || library == "default" {
				opts.NumGPU = 0
				break
			}

			// no offloading required
			if requiredTotal <= available {
				break
			}

			// This handles two cases:
			// 1. overhead + tensors are always loaded into scratch memory even with num_gpu 0
			// 2. it seems llama.cpp always tries to allocate the entire kv cache (even if later split into layers) into vram or crashes
			if requiredAlloc > available || requiredKv > available {
				log.Printf("not enough vram available, falling back to CPU only")
				library = "cpu"
				opts.NumGPU = 0
				break
			}