Repository: ollama/ollama
File: llm/server.go (Go)

Code before:
// LoadModel will load a model from disk. The model must be in the GGML format.
//
// It collects array values for arrays with a size less than or equal to
// maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If
// the maxArraySize is negative, all arrays are collected.
func LoadModel(model string, maxArraySize int) (*GGML, error) {
	if _, err := os.Stat(model); err != nil {
		return nil, err
	}

	f, err := os.Open(model)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ggml, _, err := DecodeGGML(f, maxArraySize)
	return ggml, err
}

// setCacheTypeParams sets the K/V cache type parameters if specified
func setCacheTypeParams(params *[]string, flashAttnEnabled bool, opts api.Options) {
	// if the cache types are both empty and opts.F16KV is not set, don't set any cache types
	if envconfig.CacheTypeK() == "" && envconfig.CacheTypeV() == "" && !opts.F16KV {
		return
	}

	// K/V cache quantization types supported by llama.cpp server
	validKVCacheTypes := map[string]bool{
		"f16": true, "f32": true, "q8_0": true, "q4_0": true,
	}

	setCacheTypeParam := func(paramName, cacheType string) {
		if !validKVCacheTypes[cacheType] {
			if cacheType != "" {
				slog.Warn("invalid cache type", "param", paramName, "type", cacheType)
			}
			return
		}

		if cacheType == "f16" || cacheType == "f32" {
			*params = append(*params, paramName, cacheType)
			slog.Debug("Setting cache type", "param", paramName, "type", cacheType)
		} else if flashAttnEnabled {
			*params = append(*params, paramName, cacheType)
			slog.Debug("Setting cache type", "param", paramName, "type", cacheType)
		} else {
			slog.Warn("requested cache type requires flash attention to be enabled, ignoring",
				"param", paramName, "type", cacheType, "flash_attention", flashAttnEnabled)
			// Fallback to default f16
			*params = append(*params, paramName, "f16")

Review comment (performance):
From some comments in the PR and our discussion, it seems we may need to disable KV cache quantization for embedding models.

We do this today by looking for the pooling layer (i.e. a `<architecture>.pooling_type` KV in the GGUF file) in `images.go`: https://github.com/ollama/ollama/blob/main/server/images.go#L99

Code after:
	gpus         discover.GpuInfoList // Recorded just before the model loaded, free space will be incorrect
	loadDuration time.Duration        // Record how long it took the model to load
	loadProgress float32

	sem *semaphore.Weighted
}

// LoadModel will load a model from disk. The model must be in the GGML format.
//
// It collects array values for arrays with a size less than or equal to
// maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If
// the maxArraySize is negative, all arrays are collected.
func LoadModel(model string, maxArraySize int) (*GGML, error) {
	if _, err := os.Stat(model); err != nil {
		return nil, err
	}

	f, err := os.Open(model)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ggml, _, err := DecodeGGML(f, maxArraySize)
	return ggml, err
}

// NewLlamaServer will run a server for the given GPUs
// The gpu list must be a single family.
func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {
	var err error
	var cpuRunner string
	var estimate MemoryEstimate
	var systemTotalMemory uint64
	var systemFreeMemory uint64
	var systemSwapFreeMemory uint64

	systemInfo := discover.GetSystemInfo()
	systemTotalMemory = systemInfo.System.TotalMemory
	systemFreeMemory = systemInfo.System.FreeMemory
	systemSwapFreeMemory = systemInfo.System.FreeSwap
	slog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))

	// If the user wants zero GPU layers, reset the gpu list to be CPU/system ram info
	if opts.NumGPU == 0 {
		gpus = discover.GetCPUInfo()
	}
	if len(gpus) == 1 && gpus[0].Library == "cpu" {
		cpuRunner = runners.ServerForCpu()
		estimate = EstimateGPULayers(gpus, ggml, projectors, opts)
	} else {