Repository: ollama/ollama
Pull request: Fix rare nil pointer dereference when model unloads

While testing concurrency I noticed a segfault happen occasionally when loading, canceling, and loading the same model repeatedly over and over again with a script like this:

```
#!/bin/bash

# Command to run
COMMAND="ollama run llama3 hello"

# Number of times to run the command concurrently
N=100

# Running the command N times concurrently
for i in $(seq 1 $N); do
    $COMMAND &
done
```

The error looked like this:


```
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x2 addr=0x0 pc=0x101298ae0]

goroutine 51 [running]:
github.com/ollama/ollama/server.(*runnerRef).needsReload(0x14000ffc5a0, {0x1018b17a0, 0x1400017d8b0}, 0x140003e20f0)
	/Users/jmorgan/git/ollama/server/sched.go:472 +0x150
github.com/ollama/ollama/server.(*Scheduler).processPending(0x1400017d900, {0x1018b17a0, 0x1400017d8b0})
	/Users/jmorgan/git/ollama/server/sched.go:143 +0x3d0
github.com/ollama/ollama/server.(*Scheduler).Run.func1()
	/Users/jmorgan/git/ollama/server/sched.go:120 +0x28
created by github.com/ollama/ollama/server.(*Scheduler).Run in goroutine 1
	/Users/jmorgan/git/ollama/server/sched.go:119 +0xc4
```



Changed files:
filename: server/sched.go
patch: @@ -116,7 +116,7 @@ func (s *Scheduler) processPending(ctx context.Context) {
 					}
 				} else if envconfig.MaxRunners > 0 && loadedCount >= envconfig.MaxRunners {
 					slog.Debug("max runners achieved, unloading one to make room", "runner_count", loadedCount)
-					runnerToExpire = s.findRunnerToUnload(pending)
+					runnerToExpire = s.findRunnerToUnload()
 				} else {
 					// Either no models are loaded or below envconfig.MaxRunners
 					// Get a refreshed GPU list
@@ -157,7 +157,7 @@ func (s *Scheduler) processPending(ctx context.Context) {
 						s.loadFn(pending, ggml, gpus)
 						break
 					}
-					runnerToExpire = s.findRunnerToUnload(pending)
+					runnerToExpire = s.findRunnerToUnload()
 				}
 
 				if runnerToExpire == nil {
@@ -257,9 +257,9 @@ func (s *Scheduler) processCompleted(ctx context.Context) {
 				continue
 			}
 
+			s.loadedMu.Lock()
 			slog.Debug("got lock to unload", "model", runner.model)
 			runner.unload()
-			s.loadedMu.Lock()
 			delete(s.loaded, runner.model)
 			s.loadedMu.Unlock()
 			slog.Debug("runner released", "model", runner.model)
@@ -504,7 +504,7 @@ func pickBestFitGPUs(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList) gpu.
 }
 
 // findRunnerToUnload finds a runner to unload to make room for a new model
-func (s *Scheduler) findRunnerToUnload(req *LlmRequest) *runnerRef {
+func (s *Scheduler) findRunnerToUnload() *runnerRef {
 	s.loadedMu.Lock()
 	runnerList := make([]*runnerRef, 0, len(s.loaded))
 	for _, r := range s.loaded {

filename: server/sched_test.go
patch: @@ -473,10 +473,7 @@ func TestUpdateFreeSpace(t *testing.T) {
 func TestFindRunnerToUnload(t *testing.T) {
 	ctx, done := context.WithTimeout(context.Background(), 100*time.Millisecond)
 	defer done()
-	req := &LlmRequest{
-		ctx:  ctx,
-		opts: api.DefaultOptions(),
-	}
+
 	r1 := &runnerRef{refCount: 1, sessionDuration: 1}
 	r2 := &runnerRef{sessionDuration: 2}
 
@@ -486,10 +483,10 @@ func TestFindRunnerToUnload(t *testing.T) {
 	s.loaded["b"] = r2
 	s.loadedMu.Unlock()
 
-	resp := s.findRunnerToUnload(req)
+	resp := s.findRunnerToUnload()
 	require.Equal(t, r2, resp)
 	r2.refCount = 1
-	resp = s.findRunnerToUnload(req)
+	resp = s.findRunnerToUnload()
 	require.Equal(t, r1, resp)
 
 }