Repository: ollama/ollama
Pull request: Prevent loading models larger than total memory

Users may not realize the shiny new model they're trying to load fits on their disk, but can't load into system+GPU memory.  Today we crash, but with this fix, we'll give them a better error message before even trying to load it.

Fixes #3837 #4955 


Verified by using `stress-ng` to saturate system memory, and loaded a secondary model on another ollama instance to use up GPU memory, then tried to load a model
```
% ollama run gemma:7b
Error: requested model (5.5 GiB) is too large for this system (4.5 GiB)
```

Debug server logs from the manual test
```
time=2024-07-03T15:12:11.901-07:00 level=DEBUG source=gpu.go:336 msg="updating system memory data" before.total="31.3 GiB" before.free="4.0 GiB" now.total="31.3 GiB" now.free="3.9 GiB"
CUDA driver version: 11.4
time=2024-07-03T15:12:12.000-07:00 level=DEBUG source=gpu.go:377 msg="updating cuda memory data" gpu=GPU-1c750365-54dc-7082-7c6b-9dd953a68ab6 name="NVIDIA GeForce GTX 1060 6GB" before.total="5.9 GiB" before.free="548.9 MiB" now.total="5.9 GiB" now.free="548.9 MiB" now.used="5.4 GiB"
releasing cuda driver library
time=2024-07-03T15:12:12.000-07:00 level=DEBUG source=sched.go:186 msg="updating default concurrency" OLLAMA_MAX_LOADED_MODELS=3 gpu_count=1
time=2024-07-03T15:12:12.033-07:00 level=DEBUG source=memory.go:101 msg=evaluating library=cuda gpu_count=1 available="[548.9 MiB]"
time=2024-07-03T15:12:12.034-07:00 level=DEBUG source=memory.go:168 msg="gpu has too little memory to allocate any layers" gpu="{memInfo:{TotalMemory:6372196352 FreeMemory:575537152} Library:cuda Variant:no vector extensions MinimumMemory:479199232 DependencyPath: EnvWorkarounds:[] UnreliableFreeMemory:false ID:GPU-1c750365-54dc-7082-7c6b-9dd953a68ab6 Name:NVIDIA GeForce GTX 1060 6GB Compute:6.1 DriverMajor:11 DriverMinor:4}"
time=2024-07-03T15:12:12.034-07:00 level=DEBUG source=memory.go:296 msg="insufficient VRAM to load any model layers"
time=2024-07-03T15:12:12.034-07:00 level=WARN source=sched.go:216 msg="model request too large for system" requested="5.5 GiB" system="4.5 GiB"
[GIN] 2024/07/03 - 15:12:12 | 500 |  200.768652ms |      10.16.0.83 | POST     "/api/chat"
```

The system under test:
```
% free -h; nvidia-smi
               total        used        free      shared  buff/cache   available
Mem:            31Gi        26Gi       394Mi       197Mi       4.2Gi       3.9Gi
Swap:             0B          0B          0B
Wed Jul  3 15:15:17 2024
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 470.256.02   Driver Version: 470.256.02   CUDA Version: 11.4     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA GeForce ...  Off  | 00000000:01:00.0 Off |                  N/A |
| 33%   33C    P8    13W / 120W |   5467MiB /  6077MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A   1582218      C   ...a_v11/ollama_llama_server     5465MiB |
+-----------------------------------------------------------------------------+
```

Changed files:
filename: server/sched.go
patch: @@ -139,6 +139,11 @@ func (s *Scheduler) processPending(ctx context.Context) {
 			}
 
 			for {
+				cpus := s.getCpuFn()
+				var systemMem gpu.GpuInfo
+				if len(cpus) > 0 {
+					systemMem = cpus[0]
+				}
 				var runnerToExpire *runnerRef
 				s.loadedMu.Lock()
 				runner := s.loaded[pending.model.ModelPath]
@@ -192,6 +197,27 @@ func (s *Scheduler) processPending(ctx context.Context) {
 						break
 					}
 
+					// Block attempting to load a model larger than system memory + GPU memory
+					estimate := llm.EstimateGPULayers(gpus, ggml, pending.model.ProjectorPaths, pending.opts)
+					maxSize := systemMem.FreeMemory
+					for _, gpu := range gpus {
+						if gpu.Library == "cpu" {
+							continue
+						}
+						if loadedCount == 0 {
+							// If no other models are loaded, set the limit based on what's available
+							maxSize += gpu.FreeMemory
+						} else {
+							// Other models could be unloaded, favor total memory for limit
+							maxSize += gpu.TotalMemory
+						}
+					}
+					if estimate.TotalSize > maxSize {
+						slog.Warn("model request too large for system", "requested", format.HumanBytes2(estimate.TotalSize), "system", format.HumanBytes2(maxSize))
+						pending.errCh <- fmt.Errorf("requested model (%s) is too large for this system (%s)", format.HumanBytes2(estimate.TotalSize), format.HumanBytes2(maxSize))
+						break
+					}
+
 					// Evaluate if the model will fit in the available system memory, or if we should unload a model first
 					if len(gpus) == 1 && gpus[0].Library == "cpu" {
 						// simplifying assumption of defaultParallel when in CPU mode

filename: server/sched_test.go
patch: @@ -199,6 +199,8 @@ func TestRequests(t *testing.T) {
 		require.Equal(t, resp.llama, scenario1a.srv)
 		require.Empty(t, s.pendingReqCh)
 		require.Empty(t, scenario1a.req.errCh)
+	case err := <-scenario1a.req.errCh:
+		t.Fatal(err.Error())
 	case <-ctx.Done():
 		t.Fatal("timeout")
 	}
@@ -212,6 +214,8 @@ func TestRequests(t *testing.T) {
 		require.Equal(t, resp.llama, scenario1a.srv)
 		require.Empty(t, s.pendingReqCh)
 		require.Empty(t, scenario1b.req.errCh)
+	case err := <-scenario1b.req.errCh:
+		t.Fatal(err.Error())
 	case <-ctx.Done():
 		t.Fatal("timeout")
 	}
@@ -230,6 +234,8 @@ func TestRequests(t *testing.T) {
 		require.Equal(t, resp.llama, scenario2a.srv)
 		require.Empty(t, s.pendingReqCh)
 		require.Empty(t, scenario2a.req.errCh)
+	case err := <-scenario2a.req.errCh:
+		t.Fatal(err.Error())
 	case <-ctx.Done():
 		t.Fatal("timeout")
 	}
@@ -246,6 +252,8 @@ func TestRequests(t *testing.T) {
 		require.Equal(t, resp.llama, scenario3a.srv)
 		require.Empty(t, s.pendingReqCh)
 		require.Empty(t, scenario3a.req.errCh)
+	case err := <-scenario3a.req.errCh:
+		t.Fatal(err.Error())
 	case <-ctx.Done():
 		t.Fatal("timeout")
 	}
@@ -262,6 +270,8 @@ func TestRequests(t *testing.T) {
 		require.Equal(t, resp.llama, scenario3b.srv)
 		require.Empty(t, s.pendingReqCh)
 		require.Empty(t, scenario3b.req.errCh)
+	case err := <-scenario3b.req.errCh:
+		t.Fatal(err.Error())
 	case <-ctx.Done():
 		t.Fatal("timeout")
 	}
@@ -278,6 +288,8 @@ func TestRequests(t *testing.T) {
 		require.Equal(t, resp.llama, scenario3c.srv)
 		require.Empty(t, s.pendingReqCh)
 		require.Empty(t, scenario3c.req.errCh)
+	case err := <-scenario3c.req.errCh:
+		t.Fatal(err.Error())
 	case <-ctx.Done():
 		t.Fatal("timeout")
 	}