ronantakizawa/codereview-bench huggingface dataset:
Repository: ollama/ollama
File: llama/llama.go (Go)

Code before:
		return nil, errors.New("llama: tokenize embedding")
	}

	retval := C.llama_eval(llm.ctx, unsafe.SliceData(tokens), C.int(len(tokens)), C.llama_get_kv_cache_token_count(llm.ctx), C.int(llm.NumThread))
	if retval != 0 {
		return nil, errors.New("llama: eval")
	}

	n := int(C.llama_n_embd(llm.ctx))
	if n <= 0 {
		return nil, errors.New("llama: no embeddings generated")
	}

	embedPtr := C.llama_get_embeddings(llm.ctx)
	if embedPtr == nil {
		return nil, errors.New("llama: embedding retrieval failed")
	}

	header := reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(embedPtr)),
		Len:  n,
		Cap:  n,
	}
	embedSlice := *(*[]float64)(unsafe.Pointer(&header))

	return embedSlice, nil
}


Review comment (suggestion):
Since go1.20, you can try `unsafe.Slice`

```suggestion
	n := C.llama_n_embd(llm.ctx)
	if n <= 0 {
		return nil, errors.New("llama: no embeddings generated")
	}

	embeddings := unsafe.Slice(C.llama_get_embeddings(llm.ctx), n)
```

Code after:
	tokens := llm.tokenize(input)
	if tokens == nil {
		return nil, errors.New("llama: tokenize embedding")
	}

	retval := C.llama_eval(llm.ctx, unsafe.SliceData(tokens), C.int(len(tokens)), C.llama_get_kv_cache_token_count(llm.ctx), C.int(llm.NumThread))
	if retval != 0 {
		return nil, errors.New("llama: eval")
	}

	n := int(C.llama_n_embd(llm.ctx))
	if n <= 0 {
		return nil, errors.New("llama: no embeddings generated")
	}

	embedPtr := C.llama_get_embeddings(llm.ctx)
	if embedPtr == nil {
		return nil, errors.New("llama: embedding retrieval failed")
	}

	header := reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(embedPtr)),
		Len:  n,
		Cap:  n,
	}
	embedSlice := *(*[]float64)(unsafe.Pointer(&header))

	return embedSlice, nil
}
Repository: ollama/ollama
File: server/images.go (Go)

Code before:
		return "", err
	}

	var vars struct {
		First  bool
		System string
		Prompt string
		Embed  string

		// deprecated: versions <= 0.0.7 used this to omit the system prompt
		Context []int
	}

	vars.First = len(request.Context) == 0
	vars.System = m.System
	vars.Prompt = request.Prompt
	vars.Context = request.Context

	if len(m.Embeddings) > 0 {
		promptEmbed, err := loaded.llm.Embedding(request.Prompt)
		if err != nil {
			return "", fmt.Errorf("failed to get embedding for prompt: %v", err)
		}
		// TODO: set embed_top from specified parameters in modelfile
		embed_top := 3
		embed := vector.TopK(embed_top, mat.NewVecDense(len(promptEmbed), promptEmbed), loaded.Embeddings)
		toEmbed := ""
		for _, e := range embed {
			toEmbed = fmt.Sprintf("%s %s", toEmbed, e.Embedding.Data)
		}
		vars.Embed = toEmbed
	}

	var sb strings.Builder
	if err := tmpl.Execute(&sb, vars); err != nil {
		return "", err
	}

	return sb.String(), nil
}

type ManifestV2 struct {
	SchemaVersion int      `json:"schemaVersion"`
	MediaType     string   `json:"mediaType"`
	Config        Layer    `json:"config"`
	Layers        []*Layer `json:"layers"`
}

type Layer struct {
	MediaType string `json:"mediaType"`
	Digest    string `json:"digest"`

Review comment (suggestion):
I think the bulk of the code here should be moved to the caller. There's no guarantee `loaded.llm` is non-nil. It'll be better if the caller calculates `toEmbed` and passes it in to `Prompt` as a parameter.

Code after:
	}

	tmpl, err := template.New("").Parse(t)
	if err != nil {
		return "", err
	}

	var vars struct {
		First  bool
		System string
		Prompt string
		Embed  string

		// deprecated: versions <= 0.0.7 used this to omit the system prompt
		Context []int
	}

	vars.First = len(request.Context) == 0
	vars.System = m.System
	vars.Prompt = request.Prompt
	vars.Context = request.Context
	vars.Embed = embedding

	if request.System != "" {
		vars.System = request.System
	}

	var sb strings.Builder
	if err := tmpl.Execute(&sb, vars); err != nil {
		return "", err
	}

	return sb.String(), nil
}

type ManifestV2 struct {
	SchemaVersion int      `json:"schemaVersion"`
	MediaType     string   `json:"mediaType"`
	Config        Layer    `json:"config"`
	Layers        []*Layer `json:"layers"`
}

type Layer struct {
	MediaType string `json:"mediaType"`
	Digest    string `json:"digest"`
	Size      int    `json:"size"`
}

type LayerReader struct {
	Layer
	io.Reader
Repository: ollama/ollama
File: server/images.go (Go)

Code before:
		},
		Layers: layers,
	}

	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		return err
	}

	fp, err := mp.GetManifestPath(true)
	if err != nil {
		return err
	}
	return os.WriteFile(fp, manifestJSON, 0o644)
}

func GetLayerWithBufferFromLayer(layer *Layer) (*LayerReader, error) {
	fp, err := GetBlobsPath(layer.Digest)
	if err != nil {
		return nil, err
	}

	file, err := os.Open(fp)
	if err != nil {
		return nil, fmt.Errorf("could not open blob: %w", err)
	}
	defer file.Close()

	newLayer, err := CreateLayer(file)
	if err != nil {
		return nil, err
	}
	newLayer.MediaType = layer.MediaType
	return newLayer, nil
}

// paramsToReader converts specified parameter options to their correct types, and returns a reader for the json
func paramsToReader(params map[string][]string) (io.ReadSeeker, error) {
	opts := api.Options{}
	valueOpts := reflect.ValueOf(&opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts)           // types of the fields in the options struct

	// build map of json struct tags to their types
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}


Review comment (suggestion):
`llm` should be non-nil so it should be fine to `defer llm.Close()` directly unless I'm missing something

Code after:
func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
	layers := []*LayerReader{}
	if len(e.files) > 0 {
		if _, err := os.Stat(e.model); err != nil {
			if os.IsNotExist(err) {
				// this is a model name rather than the file
				model, err := GetModel(e.model)
				if err != nil {
					return nil, fmt.Errorf("failed to get model to generate embeddings: %v", err)
				}
				e.model = model.ModelPath
			} else {
				return nil, fmt.Errorf("failed to get model file to generate embeddings: %v", err)
			}
		}

		e.opts.EmbeddingOnly = true
		llm, err := llama.New(e.model, e.opts)
		if err != nil {
			return nil, fmt.Errorf("load model to generate embeddings: %v", err)
		}
		defer func() {
			if llm != nil {
				llm.Close()
			}
		}()

		addedFiles := make(map[string]bool) // keep track of files that have already been added
		for _, filePattern := range e.files {
			matchingFiles, err := filepath.Glob(filePattern)
			if err != nil {
				return nil, fmt.Errorf("could not find files with pattern %s: %w", filePattern, err)
			}

			for _, filePath := range matchingFiles {
				if addedFiles[filePath] {
					continue
				}
				addedFiles[filePath] = true
				// TODO: check file type
				f, err := os.Open(filePath)
				if err != nil {
					return nil, fmt.Errorf("could not open embed file: %w", err)
				}
				scanner := bufio.NewScanner(f)
				scanner.Split(bufio.ScanLines)

				data := []string{}
				for scanner.Scan() {
					data = append(data, scanner.Text())
				}
Repository: ollama/ollama
File: server/auth.go (Go)

Code before:
	rawKey, err := ioutil.ReadFile(keyPath)
	if err != nil {
		log.Printf("Failed to load private key: %v", err)
		return "", err
	}

	s := SignatureData{
		Method: "GET",
		Path:   url,
		Data:   nil,
	}

	if !strings.HasPrefix(s.Path, "http") {
		if regOpts.Insecure {
			s.Path = "http://" + url
		} else {
			s.Path = "https://" + url
		}
	}

	sig, err := s.Sign(rawKey)
	if err != nil {
		return "", err
	}

	log.Printf("sig = %s", sig)

	headers := map[string]string{
		"Authorization": sig,
	}

	resp, err := makeRequest("GET", url, headers, nil, regOpts)
	if err != nil {
		log.Printf("couldn't get token: %q", err)
	}
	defer resp.Body.Close()

	// Check for success: For a successful upload, the Docker registry will respond with a 201 Created
	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		return "", fmt.Errorf("on pull registry responded with code %d: %s", resp.StatusCode, body)
	}

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var tok api.TokenResponse
	if err := json.Unmarshal(respBody, &tok); err != nil {
		return "", err

Review comment (suggestion):
Looks like data here is never used in this PR? I think we should hold off on merging this for now, or at least add the timestamp in that case. Makes it pretty open to replays where only the URL creates the signature. 

Code after:
	return fmt.Sprintf("%s?service=%s&scope=%s&ts=%d&nonce=%s", r.Realm, r.Service, r.Scope, time.Now().Unix(), nonce), nil
}

func getAuthToken(redirData AuthRedirect, regOpts *RegistryOptions) (string, error) {
	url, err := redirData.URL()
	if err != nil {
		return "", err
	}

	home, err := os.UserHomeDir()
	if err != nil {
		return "", err
	}

	keyPath := path.Join(home, ".ollama/id_ed25519")

	rawKey, err := ioutil.ReadFile(keyPath)
	if err != nil {
		log.Printf("Failed to load private key: %v", err)
		return "", err
	}

	s := SignatureData{
		Method: "GET",
		Path:   url,
		Data:   nil,
	}

	if !strings.HasPrefix(s.Path, "http") {
		if regOpts.Insecure {
			s.Path = "http://" + url
		} else {
			s.Path = "https://" + url
		}
	}

	sig, err := s.Sign(rawKey)
	if err != nil {
		return "", err
	}

	headers := map[string]string{
		"Authorization": sig,
	}

	resp, err := makeRequest("GET", url, headers, nil, regOpts)
	if err != nil {
		log.Printf("couldn't get token: %q", err)
	}
	defer resp.Body.Close()
Repository: ollama/ollama
File: server/auth.go (Go)

Code before:
	log.Printf("data = %v hash = %v hashHex = %v contentHash = %v", s.Data, hash, string(hashHex), contentHash)

	log.Printf("string = '%s'", strings.Join([]string{s.Method, s.Path, contentHash}, ","))
	// bytesToSign e.g.: "GET,http://localhost,OTdkZjM1O...
	bytesToSign := []byte(strings.Join([]string{s.Method, s.Path, contentHash}, ","))

	return bytesToSign
}

// SignData takes a SignatureData object and signs it with a raw private key
func (s SignatureData) Sign(rawKey []byte) (string, error) {
	privateKey, err := ssh.ParseRawPrivateKey(rawKey)
	if err != nil {
		return "", err
	}

	signer, err := ssh.NewSignerFromKey(privateKey)
	if err != nil {
		return "", err
	}

	// get the pubkey, but remove the type
	pubKey := ssh.MarshalAuthorizedKey(signer.PublicKey())
	parts := bytes.Split(pubKey, []byte(" "))
	if len(parts) < 2 {
		return "", fmt.Errorf("malformed private key")
	}

	signedData, err := signer.Sign(nil, s.Bytes())
	if err != nil {
		return "", err
	}

	// signature is <pubkey>:<signature>
	sig := fmt.Sprintf("%s:%s", bytes.TrimSpace(parts[1]), base64.StdEncoding.EncodeToString(signedData.Blob))
	return sig, nil
}


Review comment (suggestion):
```suggestion
		return "", fmt.Errorf("malformed public key")
```

Code after:
	if err := json.Unmarshal(respBody, &tok); err != nil {
		return "", err
	}

	return tok.Token, nil
}

// Bytes returns a byte slice of the data to sign for the request
func (s SignatureData) Bytes() []byte {
	// We first derive the content hash of the request body using:
	//     base64(hex(sha256(request body)))

	hash := sha256.Sum256(s.Data)
	hashHex := make([]byte, hex.EncodedLen(len(hash)))
	hex.Encode(hashHex, hash[:])
	contentHash := base64.StdEncoding.EncodeToString(hashHex)

	// We then put the entire request together in a serialize string using:
	//       "<method>,<uri>,<content hash>"
	// e.g.  "GET,http://localhost,OTdkZjM1O..."

	return []byte(strings.Join([]string{s.Method, s.Path, contentHash}, ","))
}

// SignData takes a SignatureData object and signs it with a raw private key
func (s SignatureData) Sign(rawKey []byte) (string, error) {
	privateKey, err := ssh.ParseRawPrivateKey(rawKey)
	if err != nil {
		return "", err
	}

	signer, err := ssh.NewSignerFromKey(privateKey)
	if err != nil {
		return "", err
	}

	// get the pubkey, but remove the type
	pubKey := ssh.MarshalAuthorizedKey(signer.PublicKey())
	parts := bytes.Split(pubKey, []byte(" "))
	if len(parts) < 2 {
		return "", fmt.Errorf("malformed public key")
	}

	signedData, err := signer.Sign(nil, s.Bytes())
	if err != nil {
		return "", err
	}

	// signature is <pubkey>:<signature>
	sig := fmt.Sprintf("%s:%s", bytes.TrimSpace(parts[1]), base64.StdEncoding.EncodeToString(signedData.Blob))
	return sig, nil
Repository: ollama/ollama
File: server/auth.go (Go)

Code before:
	if err != nil {
		log.Printf("couldn't get token: %q", err)
	}
	defer resp.Body.Close()

	// Check for success: For a successful upload, the Docker registry will respond with a 201 Created
	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		return "", fmt.Errorf("on pull registry responded with code %d: %s", resp.StatusCode, body)
	}

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var tok api.TokenResponse
	if err := json.Unmarshal(respBody, &tok); err != nil {
		return "", err
	}

	return tok.Token, nil
}

func (s SignatureData) Bytes() []byte {
	// contentHash = base64(hex(sha256(s.Data)))
	hash := sha256.Sum256(s.Data)
	hashHex := make([]byte, hex.EncodedLen(len(hash)))
	hex.Encode(hashHex, hash[:])
	contentHash := base64.StdEncoding.EncodeToString(hashHex)
	log.Printf("data = %v hash = %v hashHex = %v contentHash = %v", s.Data, hash, string(hashHex), contentHash)

	log.Printf("string = '%s'", strings.Join([]string{s.Method, s.Path, contentHash}, ","))
	// bytesToSign e.g.: "GET,http://localhost,OTdkZjM1O...
	bytesToSign := []byte(strings.Join([]string{s.Method, s.Path, contentHash}, ","))

	return bytesToSign
}

// SignData takes a SignatureData object and signs it with a raw private key
func (s SignatureData) Sign(rawKey []byte) (string, error) {
	privateKey, err := ssh.ParseRawPrivateKey(rawKey)
	if err != nil {
		return "", err
	}

	signer, err := ssh.NewSignerFromKey(privateKey)
	if err != nil {
		return "", err
	}


Review comment (suggestion):
```suggestion
	// contentHash = base64(hex(sha256(s.Data)))
```

Code after:
			s.Path = "https://" + url
		}
	}

	sig, err := s.Sign(rawKey)
	if err != nil {
		return "", err
	}

	headers := map[string]string{
		"Authorization": sig,
	}

	resp, err := makeRequest("GET", url, headers, nil, regOpts)
	if err != nil {
		log.Printf("couldn't get token: %q", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		return "", fmt.Errorf("on pull registry responded with code %d: %s", resp.StatusCode, body)
	}

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var tok api.TokenResponse
	if err := json.Unmarshal(respBody, &tok); err != nil {
		return "", err
	}

	return tok.Token, nil
}

// Bytes returns a byte slice of the data to sign for the request
func (s SignatureData) Bytes() []byte {
	// We first derive the content hash of the request body using:
	//     base64(hex(sha256(request body)))

	hash := sha256.Sum256(s.Data)
	hashHex := make([]byte, hex.EncodedLen(len(hash)))
	hex.Encode(hashHex, hash[:])
	contentHash := base64.StdEncoding.EncodeToString(hashHex)

	// We then put the entire request together in a serialize string using:
	//       "<method>,<uri>,<content hash>"
	// e.g.  "GET,http://localhost,OTdkZjM1O..."
Repository: ollama/ollama
File: server/auth.go (Go)

Code before:
		return "", fmt.Errorf("on pull registry responded with code %d: %s", resp.StatusCode, body)
	}

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var tok api.TokenResponse
	if err := json.Unmarshal(respBody, &tok); err != nil {
		return "", err
	}

	return tok.Token, nil
}

func (s SignatureData) Bytes() []byte {
	// contentHash = base64(hex(sha256(s.Data)))
	hash := sha256.Sum256(s.Data)
	hashHex := make([]byte, hex.EncodedLen(len(hash)))
	hex.Encode(hashHex, hash[:])
	contentHash := base64.StdEncoding.EncodeToString(hashHex)
	log.Printf("data = %v hash = %v hashHex = %v contentHash = %v", s.Data, hash, string(hashHex), contentHash)

	log.Printf("string = '%s'", strings.Join([]string{s.Method, s.Path, contentHash}, ","))
	// bytesToSign e.g.: "GET,http://localhost,OTdkZjM1O...
	bytesToSign := []byte(strings.Join([]string{s.Method, s.Path, contentHash}, ","))

	return bytesToSign
}

// SignData takes a SignatureData object and signs it with a raw private key
func (s SignatureData) Sign(rawKey []byte) (string, error) {
	privateKey, err := ssh.ParseRawPrivateKey(rawKey)
	if err != nil {
		return "", err
	}

	signer, err := ssh.NewSignerFromKey(privateKey)
	if err != nil {
		return "", err
	}

	// get the pubkey, but remove the type
	pubKey := ssh.MarshalAuthorizedKey(signer.PublicKey())
	parts := bytes.Split(pubKey, []byte(" "))
	if len(parts) < 2 {
		return "", fmt.Errorf("malformed private key")
	}

	signedData, err := signer.Sign(nil, s.Bytes())

Review comment (suggestion):
```suggestion
	// bytesToSign e.g.: "GET,http://localhost,OTdkZjM1O...
```

Code after:

	headers := map[string]string{
		"Authorization": sig,
	}

	resp, err := makeRequest("GET", url, headers, nil, regOpts)
	if err != nil {
		log.Printf("couldn't get token: %q", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		body, _ := io.ReadAll(resp.Body)
		return "", fmt.Errorf("on pull registry responded with code %d: %s", resp.StatusCode, body)
	}

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var tok api.TokenResponse
	if err := json.Unmarshal(respBody, &tok); err != nil {
		return "", err
	}

	return tok.Token, nil
}

// Bytes returns a byte slice of the data to sign for the request
func (s SignatureData) Bytes() []byte {
	// We first derive the content hash of the request body using:
	//     base64(hex(sha256(request body)))

	hash := sha256.Sum256(s.Data)
	hashHex := make([]byte, hex.EncodedLen(len(hash)))
	hex.Encode(hashHex, hash[:])
	contentHash := base64.StdEncoding.EncodeToString(hashHex)

	// We then put the entire request together in a serialize string using:
	//       "<method>,<uri>,<content hash>"
	// e.g.  "GET,http://localhost,OTdkZjM1O..."

	return []byte(strings.Join([]string{s.Method, s.Path, contentHash}, ","))
}

// SignData takes a SignatureData object and signs it with a raw private key
func (s SignatureData) Sign(rawKey []byte) (string, error) {
	privateKey, err := ssh.ParseRawPrivateKey(rawKey)
	if err != nil {
		return "", err
Repository: ollama/ollama
File: server/modelpath.go (Go)

Code before:
}

const (
	DefaultRegistry       = "registry.ollama.ai"
	DefaultNamespace      = "library"
	DefaultTag            = "latest"
	DefaultProtocolScheme = "https"
)

var protocolPattern = regexp.MustCompile(`^.*://`)

func ParseModelPath(name string) ModelPath {
	name = protocolPattern.ReplaceAllString(name, "")

	slashParts := strings.Split(name, "/")

	registry := DefaultRegistry
	namespace := DefaultNamespace
	repository := ""
	tag := DefaultTag

	switch len(slashParts) {
	case 3:
		registry = slashParts[0]
		namespace = slashParts[1]
		repository = slashParts[2]
	case 2:
		namespace = slashParts[0]
		repository = slashParts[1]
	case 1:
		repository = slashParts[0]
	default:
		fmt.Println("Invalid image format.")
		return ModelPath{}
	}

	repoParts := strings.Split(repository, ":")
	if len(repoParts) == 2 {
		repository = repoParts[0]
		tag = repoParts[1]
	}

	return ModelPath{
		ProtocolScheme: DefaultProtocolScheme,
		Registry:       registry,
		Namespace:      namespace,
		Repository:     repository,
		Tag:            tag,
	}
}


Review comment (suggestion):
Hello! Thanks for opening a PR 😊 

This is close, but I think you'll want to apply this to `registry` vs `repository`. I added an example here: https://github.com/jmorganca/ollama/issues/371


Code after:
}

const (
	DefaultRegistry       = "registry.ollama.ai"
	DefaultNamespace      = "library"
	DefaultTag            = "latest"
	DefaultProtocolScheme = "https"
)

var (
	ErrInvalidImageFormat = errors.New("invalid image format")
	ErrInvalidProtocol    = errors.New("invalid protocol scheme")
	ErrInsecureProtocol   = errors.New("insecure protocol http")
)

func ParseModelPath(name string, allowInsecure bool) (ModelPath, error) {
	mp := ModelPath{
		ProtocolScheme: DefaultProtocolScheme,
		Registry:       DefaultRegistry,
		Namespace:      DefaultNamespace,
		Repository:     "",
		Tag:            DefaultTag,
	}

	protocol, rest, didSplit := strings.Cut(name, "://")
	if didSplit {
		if protocol == "https" || protocol == "http" && allowInsecure {
			mp.ProtocolScheme = protocol
			name = rest
		} else if protocol == "http" && !allowInsecure {
			return ModelPath{}, ErrInsecureProtocol
		} else {
			return ModelPath{}, ErrInvalidProtocol
		}
	}

	slashParts := strings.Split(name, "/")
	switch len(slashParts) {
	case 3:
		mp.Registry = slashParts[0]
		mp.Namespace = slashParts[1]
		mp.Repository = slashParts[2]
	case 2:
		mp.Namespace = slashParts[0]
		mp.Repository = slashParts[1]
	case 1:
		mp.Repository = slashParts[0]
	default:
		return ModelPath{}, ErrInvalidImageFormat
	}
Repository: ollama/ollama
File: llm/llama.go (Go)

Code before:
		return "Q6_K"
	default:
		return "Unknown"
	}
}

type Running struct {
	Port   int
	Cmd    *exec.Cmd
	Cancel context.CancelFunc
}

type ModelRunner struct {
	Path string // path to the model runner executable
}

type llama struct {
	api.Options
	Running
}

var errNoGPU = errors.New("nvidia-smi command failed")

// CheckVRAM returns the available VRAM in MiB on Linux machines with NVIDIA GPUs
func CheckVRAM() (int, error) {
	cmd := exec.Command("nvidia-smi", "--query-gpu=memory.free", "--format=csv,noheader,nounits")
	var stdout bytes.Buffer
	cmd.Stdout = &stdout
	err := cmd.Run()
	if err != nil {
		return 0, errNoGPU
	}

	var total int
	scanner := bufio.NewScanner(&stdout)
	for scanner.Scan() {
		line := scanner.Text()
		vram, err := strconv.Atoi(line)
		if err != nil {
			return 0, fmt.Errorf("failed to parse available VRAM: %v", err)
		}

		total += vram
	}

	return total, nil
}

func NumGPU(opts api.Options) int {
	if opts.NumGPU != -1 {
		return opts.NumGPU

Review comment (performance):
Would it be better to use `memory.total`? I believe worst case nvidia GPUs will begin to [evict](https://developer.nvidia.com/blog/improving-gpu-memory-oversubscription-performance/) existing memory pages to system memory (RAM) which, in theory, should be fast. And it also means the model will get as much memory as possible when loaded here (vs only what's "available").

Code after:
		return "Q6_K"
	default:
		return "Unknown"
	}
}

type Running struct {
	Port   int
	Cmd    *exec.Cmd
	Cancel context.CancelFunc
}

type ModelRunner struct {
	Path string // path to the model runner executable
}

type llama struct {
	api.Options
	Running
}

var errNoGPU = errors.New("nvidia-smi command failed")

// CheckVRAM returns the available VRAM in MiB on Linux machines with NVIDIA GPUs
func CheckVRAM() (int, error) {
	cmd := exec.Command("nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits")
	var stdout bytes.Buffer
	cmd.Stdout = &stdout
	err := cmd.Run()
	if err != nil {
		return 0, errNoGPU
	}

	var total int
	scanner := bufio.NewScanner(&stdout)
	for scanner.Scan() {
		line := scanner.Text()
		vram, err := strconv.Atoi(line)
		if err != nil {
			return 0, fmt.Errorf("failed to parse available VRAM: %v", err)
		}

		total += vram
	}

	return total, nil
}

func NumGPU(opts api.Options) int {
	if opts.NumGPU != -1 {
		return opts.NumGPU