Repository: ollama/ollama
File: server/sched.go (Go)

Code before:
		numParallelToTry = append(numParallelToTry, defaultParallel, 1)
	} else {
		numParallelToTry = []int{*numParallel}
	}

	for _, gl := range gpus.ByLibrary() {
		sgl := append(make(discover.GpuInfoList, 0, len(gl)), gl...)

		// TODO - potentially sort by performance capability, existing models loaded, etc.
		// TODO - Eliminate any GPUs that already have envconfig.MaxRunners loaded on them
		// Note: at present, this will favor most current available VRAM descending and ignoring faster GPU speed in mixed setups
		sort.Sort(sort.Reverse(discover.ByFreeMemory(sgl)))

		if !envconfig.SchedSpread() {
			for _, p := range numParallelToTry {
				req.opts.NumCtx = req.origNumCtx * p
				// Try to pack into as few GPUs as possible, starting from 1 GPU
				for numGPUs := 1; numGPUs <= len(sgl); numGPUs++ {
					gpuSubset := sgl[:numGPUs]
					ok, _ := llm.PredictServerFit(gpuSubset, f, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts, p)

					if ok {
						var totalFreeMemory uint64
						for _, g := range gpuSubset {
							totalFreeMemory += g.FreeMemory
						}
						*numParallel = p
						return gpuSubset
					}
				}
			}
		}

		// TODO future refinements
		// - if multiple Libraries, see if any single GPU in any Library will fit
		// - try subsets of GPUs instead of just falling back to 1 or all in a family

		// Now try all the GPUS (OLLAMA_SCHED_SPREAD is set)
		for _, p := range numParallelToTry {
			req.opts.NumCtx = req.origNumCtx * p
			if envconfig.SchedSpread() {
				if ok, _ := llm.PredictServerFit(sgl, f, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts, p); ok {
					*numParallel = p
					return sgl
				}
			}
		}
	}
	return nil
}


Review comment (suggestion):
I think this calculation of `totalFreeMemory` is unused.

It would be good to keep the existing log message about loading the model though - you can just print something similar to the current multi-GPU case, where it doesn't display the free memory. i.e.
`slog.Info("new model will fit in available VRAM, loading", "model", req.model.ModelPath, "library", sgl[0].Library, "parallel", p, "required", format.HumanBytes2(estimatedVRAM))`

Code after:
		numParallelToTry = append(numParallelToTry, defaultParallel, 1)
	} else {
		numParallelToTry = []int{*numParallel}
	}

	for _, gl := range gpus.ByLibrary() {
		sgl := append(make(discover.GpuInfoList, 0, len(gl)), gl...)

		// TODO - potentially sort by performance capability, existing models loaded, etc.
		// TODO - Eliminate any GPUs that already have envconfig.MaxRunners loaded on them
		// Note: at present, this will favor most current available VRAM descending and ignoring faster GPU speed in mixed setups
		sort.Sort(sort.Reverse(discover.ByFreeMemory(sgl)))

		if !envconfig.SchedSpread() {
			for _, p := range numParallelToTry {
				req.opts.NumCtx = req.origNumCtx * p
				// Try to pack into as few GPUs as possible, starting from 1 GPU
				for numGPUs := 1; numGPUs <= len(sgl); numGPUs++ {
					gpuSubset := sgl[:numGPUs]
					ok, estimatedVRAM := llm.PredictServerFit(gpuSubset, f, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts, p)

					if ok {
						slog.Info("new model will fit in available VRAM across minimum required GPUs, loading",
							"model", req.model.ModelPath,
							"library", sgl[0].Library,
							"parallel", p,
							"required", format.HumanBytes2(estimatedVRAM),
							"gpus", numGPUs)
						*numParallel = p
						return gpuSubset
					}
				}
			}
		} else {
			// TODO future refinements
			// - if multiple Libraries, see if any single GPU in any Library will fit
			// - try subsets of GPUs instead of just falling back to 1 or all in a family

			// Now try all the GPUS (OLLAMA_SCHED_SPREAD is set)
			for _, p := range numParallelToTry {
				req.opts.NumCtx = req.origNumCtx * p
				if ok, estimatedVRAM := llm.PredictServerFit(sgl, f, req.model.AdapterPaths, req.model.ProjectorPaths, req.opts, p); ok {
					slog.Info("new model will fit in available VRAM, loading",
						"model", req.model.ModelPath,
						"library", sgl[0].Library,
						"parallel", p,
						"required", format.HumanBytes2(estimatedVRAM),
						"gpus", len(sgl))
					*numParallel = p
					return sgl
				}