ronantakizawa/codereview-bench huggingface dataset:
Repository: ollama/ollama
File: cmd/cmd.go (Go)

Code before:
		}
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"NAME", "ID", "SIZE", "PROCESSOR", "CONTEXT", "UNTIL"})
	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeaderLine(false)
	table.SetBorder(false)
	table.SetNoWhiteSpace(true)
	table.SetTablePadding("    ")
	table.AppendBulk(data)
	table.Render()

	return nil
}

func DeleteHandler(cmd *cobra.Command, args []string) error {
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return err
	}

	for _, arg := range args {
		name := model.ParseName(arg)
		if name.Namespace != "library" || !strings.HasSuffix(name.Model, "-cloud") {
			// Unload the model if it's running before deletion
			if err := loadOrUnloadModel(cmd, &runOptions{
				Model:     args[0],
				KeepAlive: &api.Duration{Duration: 0},
			}); err != nil {
				if !strings.Contains(strings.ToLower(err.Error()), "not found") {
					fmt.Fprintf(os.Stderr, "Warning: unable to stop model '%s'\n", args[0])
				}
			}
		}

		req := api.DeleteRequest{Name: name.String()}
		if err := client.Delete(cmd.Context(), &req); err != nil {
			return err
		}
		fmt.Printf("deleted '%s'\n", arg)
	}
	return nil
}

func ShowHandler(cmd *cobra.Command, args []string) error {
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return err
	}

Review comment (suggestion):
this heuristic isn't ideal but short of calling show and looking at `RemoteHost`/`RemoteModel`, this is the best we can do

Code after:
			if delta > 0 {
				until = "Stopping..."
			} else {
				until = format.HumanTime(m.ExpiresAt, "Never")
			}
			ctxStr := strconv.Itoa(m.ContextLength)
			data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), procStr, ctxStr, until})
		}
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"NAME", "ID", "SIZE", "PROCESSOR", "CONTEXT", "UNTIL"})
	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeaderLine(false)
	table.SetBorder(false)
	table.SetNoWhiteSpace(true)
	table.SetTablePadding("    ")
	table.AppendBulk(data)
	table.Render()

	return nil
}

func DeleteHandler(cmd *cobra.Command, args []string) error {
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return err
	}

	for _, arg := range args {
		// Unload the model if it's running before deletion
		if err := loadOrUnloadModel(cmd, &runOptions{
			Model:     args[0],
			KeepAlive: &api.Duration{Duration: 0},
		}); err != nil {
			if !strings.Contains(strings.ToLower(err.Error()), "not found") {
				fmt.Fprintf(os.Stderr, "Warning: unable to stop model '%s'\n", args[0])
			}
		}

		if err := client.Delete(cmd.Context(), &api.DeleteRequest{Name: arg}); err != nil {
			return err
		}
		fmt.Printf("deleted '%s'\n", arg)
	}
	return nil
}

func ShowHandler(cmd *cobra.Command, args []string) error {
	client, err := api.ClientFromEnvironment()
Repository: ollama/ollama
File: integration/embed_test.go (Go)

Code before:
}

func embedTestHelper(ctx context.Context, client *api.Client, t *testing.T, req api.EmbedRequest) (*api.EmbedResponse, error) {
	t.Helper()

	if err := PullIfMissing(ctx, client, req.Model); err != nil {
		t.Fatal(err)
	}

	return client.Embed(ctx, &req)
}

func TestEmbedTruncation(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
	defer cancel()
	client, _, cleanup := InitServerConnection(ctx, t)
	defer cleanup()

	t.Run("single input token count", func(t *testing.T) {
		req := api.EmbedRequest{
			Model: "all-minilm",
			Input: "why is the sky blue?",
		}

		res, err := embedTestHelper(ctx, client, t, req)
		if err != nil {
			t.Fatal(err)
		}

		if res.PromptEvalCount <= 0 {
			t.Fatalf("expected positive token count, got %d", res.PromptEvalCount)
		}
	})

	t.Run("batch parallel token counting", func(t *testing.T) {
		req := api.EmbedRequest{
			Model: "all-minilm",
			Input: []string{"cat", "dog and mouse", "bird"},
		}

		res, err := embedTestHelper(ctx, client, t, req)
		if err != nil {
			t.Fatal(err)
		}

		if len(res.Embeddings) != 3 {
			t.Fatalf("expected 3 embeddings, got %d", len(res.Embeddings))
		}

		if res.PromptEvalCount <= 0 {
			t.Fatalf("expected positive token count, got %d", res.PromptEvalCount)

Review comment (suggestion):
You can wrap this with
```
        for _, model := range libraryEmbedModels {
```

and change `all-minilm` below this to pick up each of the embed models.

Code after:
	}

	return client.Embeddings(ctx, &req)
}

func embedTestHelper(ctx context.Context, client *api.Client, t *testing.T, req api.EmbedRequest) (*api.EmbedResponse, error) {
	t.Helper()

	if err := PullIfMissing(ctx, client, req.Model); err != nil {
		t.Fatal(err)
	}

	return client.Embed(ctx, &req)
}

func TestEmbedTruncation(t *testing.T) {
	// Use test deadline if set, otherwise default to 2 minutes
	timeout := 2 * time.Minute
	if deadline, ok := t.Deadline(); ok {
		timeout = time.Until(deadline) - 10*time.Second // Reserve 10s buffer
	}
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	client, _, cleanup := InitServerConnection(ctx, t)
	defer cleanup()

	for _, model := range libraryEmbedModels {
		model := model
		t.Run(model, func(t *testing.T) {
			// Check if we're running out of time (reserve 20s for current model)
			if deadline, ok := t.Deadline(); ok && time.Until(deadline) < 20*time.Second {
				t.Skip("skipping remaining tests to avoid timeout")
			}

			// Give each model its own budget to account for first-time pulls/loads
			mctx, mcancel := context.WithTimeout(ctx, 3*time.Minute)
			defer mcancel()

			t.Run("truncation batch", func(t *testing.T) {
				truncTrue := true
				req := api.EmbedRequest{
					Model:    model,
					Input:    []string{"short", strings.Repeat("long ", 100), "medium text"},
					Truncate: &truncTrue,
					Options:  map[string]any{"num_ctx": 30},
				}

				res, err := embedTestHelper(mctx, client, t, req)
				if err != nil {
					t.Fatal(err)
				}
Repository: ollama/ollama
File: runner/llamarunner/runner.go (Go)

Code before:
		return nil, fmt.Errorf("failed to process inputs: %w", err)
	} else if len(inputs) == 0 {
		return nil, errors.New("no input provided")
	}

	if params.numKeep < 0 {
		params.numKeep = len(inputs)
	}

	if s.model.AddBOSToken() {
		params.numKeep += 1
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if len(inputs) > s.cache.numCtx {
		discard := len(inputs) - s.cache.numCtx
		if !params.truncate {
			return nil, errorInputTooLong
		}

		// If the original prompt ended with an EOG token, add it back after truncation
		lastIsEOG := false

		eogToken := 0
		if len(inputs) > 0 && s.model.TokenIsEog(inputs[len(inputs)-1].token) {
			lastIsEOG = true
			eogToken = inputs[len(inputs)-1].token
		}

		var newInputs []input

		// For embeddings, truncate only at the front and keep first N tokens
		if params.embedding {
			newLimit := s.cache.numCtx
			if s.model.AddBOSToken() {
				newLimit--
			}

			if lastIsEOG {
				newLimit--
			}

			if newLimit <= 0 {
				return nil, fmt.Errorf("input after truncation exceeds maximum context length")
			}

			newInputs = inputs[:newLimit]
		} else {
			// Otherwise, truncate in the middle

Review comment (bug):
I guess before we were simply truncating out the EOG token? If so, this is a good catch. However handling this here makes the code a bit hard to follow.

Did we do this previously? If this is a separate bug from truncation, I'd recommended we revisit the EOG handling in a follow up PR. I get the feeling we may want to better handle these truncation edge cases than putting a bunch of branches here in the code

Code after:
		return nil, fmt.Errorf("failed to process inputs: %w", err)
	} else if len(inputs) == 0 {
		return nil, errors.New("no input provided")
	}

	if params.numKeep < 0 {
		params.numKeep = len(inputs)
	}

	if s.model.AddBOSToken() {
		params.numKeep += 1
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if len(inputs) > s.cache.numCtx {
		discard := len(inputs) - s.cache.numCtx
		if !params.truncate {
			return nil, errorInputTooLong
		}

		newInputs := inputs[:params.numKeep]
		newInputs = append(newInputs, inputs[params.numKeep+discard:]...)

		slog.Warn("truncating input prompt", "limit", s.cache.numCtx, "prompt", len(inputs), "keep", params.numKeep, "new", len(newInputs))
		inputs = newInputs
	}

	var sc *llama.SamplingContext
	if params.samplingParams != nil {
		sc, err = llama.NewSamplingContext(s.model, *params.samplingParams)
		if err != nil {
			return nil, err
		}
		for _, input := range inputs {
			if input.embed == nil {
				sc.Accept(input.token, false)
			}
		}
	}

	return &Sequence{
		inputs:           inputs,
		numPromptInputs:  len(inputs),
		numPredict:       params.numPredict,
		pendingResponses: make([]string, 0),
		responses:        make(chan string, 100),
		quit:             make(chan bool, 1),
		embedding:        make(chan []float32, 1),
		samplingCtx:      sc,
Repository: ollama/ollama
File: runner/ollamarunner/runner.go (Go)

Code before:
					}

					sameBatch = inp.SameBatch
				}
			}

			if promptStart >= int32(len(inputs)) {
				return nil, errors.New("entire prompt removed by truncation")
			}

			newInputs = inputs[:params.numKeep]
			newInputs = append(newInputs, inputs[promptStart:]...)
		}

		slog.Warn("truncating input prompt", "limit", s.cache.numCtx, "prompt", len(inputs), "keep", params.numKeep, "new", len(newInputs))
		inputs = newInputs
	}

	// TODO(jessegross): Ingest cached history for grammar
	promptCount := len(inputs)

	return &Sequence{
		ctxs:             ctxs,
		mmStore:          mmStore,
		inputs:           inputs,
		numPromptInputs:  promptCount,
		numPredict:       params.numPredict,
		pendingResponses: make([]string, 0),
		responses:        make(chan string, 100),
		quit:             make(chan bool, 1),
		embedding:        make(chan []float32, 1),
		sampler:          params.sampler,
		embeddingOnly:    params.embedding,
		stop:             params.stop,
		numKeep:          params.numKeep,
		shift:            params.shift,
	}, nil
}

// inputs processes the prompt and images into a list of inputs
// by splitting the prompt on [img-<n>] tags, tokenizing text and
// decoding images
func (s *Server) inputs(prompt string, images []llm.ImageData) ([]*input.Input, []ml.Context, multimodalStore, error) {
	var inputs []*input.Input
	var ctxs []ml.Context
	var mmStore multimodalStore

	var parts []string
	var matches [][]string

	multimodalProcessor, visionModel := s.model.(model.MultimodalProcessor)

Review comment (suggestion):
let's remove `promptCount` and keep this `len(inputs)` as before to minimize the diff and remove the unnecessary variable

Code after:
	}, nil
}

// inputs processes the prompt and images into a list of inputs
// by splitting the prompt on [img-<n>] tags, tokenizing text and
// decoding images
func (s *Server) inputs(prompt string, images []llm.ImageData) ([]*input.Input, []ml.Context, multimodalStore, error) {
	var inputs []*input.Input
	var ctxs []ml.Context
	var mmStore multimodalStore

	var parts []string
	var matches [][]string

	multimodalProcessor, visionModel := s.model.(model.MultimodalProcessor)

	if visionModel {
		re := regexp.MustCompile(`\[img-(\d+)\]`)
		parts = re.Split(prompt, -1)
		matches = re.FindAllStringSubmatch(prompt, -1)
		mmStore = newMultimodalStore()
	} else {
		parts = []string{prompt}
	}

	postTokenize := false
	for i, part := range parts {
		// text - tokenize
		tokens, err := s.model.(model.TextProcessor).Encode(part, i == 0)
		if err != nil {
			return nil, nil, nil, err
		}

		for _, t := range tokens {
			inputs = append(inputs, &input.Input{Token: t})
		}

		// image - decode and store
		if i < len(matches) {
			n, _ := strconv.Atoi(matches[i][1])

			imageIndex := -1
			for j := range images {
				if images[j].ID == n {
					imageIndex = j
					break
				}
			}

			if imageIndex < 0 {
				return nil, nil, nil, fmt.Errorf("invalid image index: %d", n)
Repository: ollama/ollama
File: runner/ollamarunner/runner.go (Go)

Code before:
	if err != nil {
		return nil, fmt.Errorf("failed to process inputs: %w", err)
	} else if len(inputs) == 0 {
		return nil, errors.New("no input provided")
	}

	if params.numKeep < 0 {
		params.numKeep = int32(len(inputs))
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if int32(len(inputs)) > s.cache.numCtx {
		if !params.truncate {
			return nil, errorInputTooLong
		}

		discard := int32(len(inputs)) - s.cache.numCtx

		var newInputs []*input.Input

		if params.embedding {
			var lastIsEOG bool

			eogToken := int32(0)
			if len(inputs) > 0 {
				if textProcessor, ok := s.model.(model.TextProcessor); ok {
					lastToken := inputs[len(inputs)-1]
					if textProcessor.Is(lastToken.Token, model.SpecialEOS) {
						eogToken = lastToken.Token
					}
				}
			}

			// If embedding only, truncate to the maximum context length - 1 (to account for BOS token)
			newLimit := s.cache.numCtx - 1

			if s.model.(model.TextProcessor).Is(inputs[len(inputs)-1].Token, model.SpecialEOS) && !s.model.(model.TextProcessor).Is(inputs[newLimit-1].Token, model.SpecialEOS) {
				lastIsEOG = true
				newLimit--
			}

			if newLimit <= 0 {
				return nil, fmt.Errorf("input after truncation exceeds maximum context length")
			}

			newInputs = inputs[:newLimit]

			if lastIsEOG {
				newInputs = append(newInputs, &input.Input{Token: eogToken})

Review comment (suggestion):
Unless I'm mistaken, if we're truncating, I think you can always assume the last token should be EOS and replace it with that anyways. In other words, you don't have to check to see if it's an EOS token. Simply truncate to n values, and swap the last token for EOS

Code after:
	if err != nil {
		return nil, fmt.Errorf("failed to process inputs: %w", err)
	} else if len(inputs) == 0 {
		return nil, errors.New("no input provided")
	}

	if params.numKeep < 0 {
		params.numKeep = int32(len(inputs))
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if int32(len(inputs)) > s.cache.numCtx {
		if !params.truncate {
			return nil, errorInputTooLong
		}

		discard := int32(len(inputs)) - s.cache.numCtx

		promptStart := params.numKeep + discard

		// If we need to truncate in the middle of a unbreakable batch, remove the entire batch
		sameBatch := 0
		for i, inp := range inputs {
			if sameBatch > 0 {
				sameBatch--

				if promptStart == int32(i) {
					promptStart++
				}
			} else if promptStart == int32(i) {
				break
			}

			if inp.SameBatch != 0 {
				if int32(i) < params.numKeep {
					return nil, fmt.Errorf("SameBatch may not be specified within numKeep (index: %v numKeep: %v SameBatch: %v)", i, params.numKeep, inp.SameBatch)
				}

				sameBatch = inp.SameBatch
			}
		}

		if promptStart >= int32(len(inputs)) {
			return nil, errors.New("entire prompt removed by truncation")
		}

		newInputs := inputs[:params.numKeep]
		newInputs = append(newInputs, inputs[promptStart:]...)
Repository: ollama/ollama
File: server/routes.go (Go)

Code before:
		return
	}

	streamResponse(c, ch)
}

func (s *Server) EmbedHandler(c *gin.Context) {
	checkpointStart := time.Now()
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	truncate := true
	if req.Truncate != nil && !*req.Truncate {
		truncate = false
	}

	// Basic server-side guard: if truncation is enabled and num_ctx <= 1, return an error
	if truncate && req.Options != nil {
		if v, ok := req.Options["num_ctx"]; ok {
			var numCtx int
			switch t := v.(type) {
			case float64:
				numCtx = int(t)
			case int:
				numCtx = t
			case int64:
				numCtx = int(t)
			case json.Number:
				if i, err := t.Int64(); err == nil {
					numCtx = int(i)
				}
			}
			if numCtx <= 1 {
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "input after truncation exceeds maximum context length"})
				return
			}
		}
	}

	var input []string

	switch i := req.Input.(type) {
	case string:

Review comment (bug):
This check is a good addition. However I think we may want to do this in a follow up PR somewhere here https://github.com/ollama/ollama/blob/d2158ca6f4ef64968e639b4474e156ff7c92883a/server/routes.go#L143 to check boundary conditions once the type of `"num_ctx"` has been resolved. Is there any major downside to taking it out of this PR? (e.g. a _new_ crash?)

Code after:
		return
	}

	streamResponse(c, ch)
}

func (s *Server) EmbedHandler(c *gin.Context) {
	checkpointStart := time.Now()
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	var input []string

	switch i := req.Input.(type) {
	case string:
		if len(i) > 0 {
			input = append(input, i)
		}
	case []any:
		for _, v := range i {
			if _, ok := v.(string); !ok {
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
				return
			}
			input = append(input, v.(string))
		}
	default:
		if req.Input != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
			return
		}
	}

	name, err := getExistingName(model.ParseName(req.Model))
	if err != nil {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		return
	}

	r, m, opts, err := s.scheduleRunner(c.Request.Context(), name.String(), []model.Capability{}, req.Options, req.KeepAlive)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
Repository: ollama/ollama
File: runner/ollamarunner/runner.go (Go)

Code before:
	}

	if params.numKeep < 0 {
		params.numKeep = int32(len(inputs))
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if int32(len(inputs)) > s.cache.numCtx {
		if !params.truncate {
			return nil, errorInputTooLong
		}

		var newInputs []*input.Input

		if params.embedding {
			// If embedding only, truncate to the maximum context length - 1 (to account for BOS token)
			newLimit := s.cache.numCtx - 1

			if newLimit <= 0 {
				return nil, fmt.Errorf("input after truncation exceeds maximum context length")
			}

			// Get the EOS token from the original input
			eogToken := int32(0)
			if textProcessor, ok := s.model.(model.TextProcessor); ok {
				lastToken := inputs[len(inputs)-1]
				if textProcessor.Is(lastToken.Token, model.SpecialEOS) {
					eogToken = lastToken.Token
				}
			}

			// Truncate and always end with EOS
			newInputs = inputs[:newLimit-1]
			newInputs = append(newInputs, &input.Input{Token: eogToken})
		} else {
			// Otherwise, truncate in the middle
			discard := int32(len(inputs)) - s.cache.numCtx
			promptStart := params.numKeep + discard

			// If we need to truncate in the middle of a unbreakable batch, remove the entire batch
			sameBatch := 0
			for i, inp := range inputs {
				if sameBatch > 0 {
					sameBatch--

					if promptStart == int32(i) {
						promptStart++
					}
				} else if promptStart == int32(i) {

Review comment (suggestion):
I'd default to `-1`. The issue is that `0` is usually valid token (although it's rarely the eog token, there may be a change it _is_ in a given vocabulary).



Code after:
	}

	if params.numKeep < 0 {
		params.numKeep = int32(len(inputs))
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if int32(len(inputs)) > s.cache.numCtx {
		if !params.truncate {
			return nil, errorInputTooLong
		}

		discard := int32(len(inputs)) - s.cache.numCtx

		promptStart := params.numKeep + discard

		// If we need to truncate in the middle of a unbreakable batch, remove the entire batch
		sameBatch := 0
		for i, inp := range inputs {
			if sameBatch > 0 {
				sameBatch--

				if promptStart == int32(i) {
					promptStart++
				}
			} else if promptStart == int32(i) {
				break
			}

			if inp.SameBatch != 0 {
				if int32(i) < params.numKeep {
					return nil, fmt.Errorf("SameBatch may not be specified within numKeep (index: %v numKeep: %v SameBatch: %v)", i, params.numKeep, inp.SameBatch)
				}

				sameBatch = inp.SameBatch
			}
		}

		if promptStart >= int32(len(inputs)) {
			return nil, errors.New("entire prompt removed by truncation")
		}

		newInputs := inputs[:params.numKeep]
		newInputs = append(newInputs, inputs[promptStart:]...)

		slog.Warn("truncating input prompt", "limit", s.cache.numCtx, "prompt", len(inputs), "keep", params.numKeep, "new", len(newInputs))
		inputs = newInputs
	}
Repository: ollama/ollama
File: llm/server.go (Go)

Code before:
				return nil
			}
		}
	}

	if err := scanner.Err(); err != nil {
		if strings.Contains(err.Error(), "unexpected EOF") || strings.Contains(err.Error(), "forcibly closed") {
			s.Close()
			var msg string
			if s.status != nil && s.status.LastErrMsg != "" {
				msg = s.status.LastErrMsg
			} else {
				msg = err.Error()
			}
			return fmt.Errorf("an error was encountered while running the model: %s", msg)
		}

		return fmt.Errorf("error reading llm response: %v", err)
	}

	return nil
}

type EmbeddingRequest struct {
	Content  string `json:"content"`
	Truncate bool   `json:"truncate"`
}

type EmbeddingResponse struct {
	Embedding       []float32 `json:"embedding"`
	PromptEvalCount int       `json:"prompt_eval_count"`
}

func (s *llmServer) Embedding(ctx context.Context, input string, truncate bool) ([]float32, int, error) {
	logutil.Trace("embedding request", "input", input)

	if err := s.sem.Acquire(ctx, 1); err != nil {
		if errors.Is(err, context.Canceled) {
			slog.Info("aborting embedding request due to client closing the connection")
		} else {
			slog.Error("Failed to acquire semaphore", "error", err)
		}
		return nil, 0, err
	}
	defer s.sem.Release(1)

	// Make sure the server is ready
	status, err := s.getServerStatusRetry(ctx)
	if err != nil {
		return nil, 0, err
	} else if status != ServerStatusReady {

Review comment (suggestion):
Similar to the comment below, I'm not sure we need this anymore as we will always error on truncate in the runner 😊 

Code after:
				return nil
			}
		}
	}

	if err := scanner.Err(); err != nil {
		if strings.Contains(err.Error(), "unexpected EOF") || strings.Contains(err.Error(), "forcibly closed") {
			s.Close()
			var msg string
			if s.status != nil && s.status.LastErrMsg != "" {
				msg = s.status.LastErrMsg
			} else {
				msg = err.Error()
			}
			return fmt.Errorf("an error was encountered while running the model: %s", msg)
		}

		return fmt.Errorf("error reading llm response: %v", err)
	}

	return nil
}

type EmbeddingRequest struct {
	Content string `json:"content"`
}

type EmbeddingResponse struct {
	Embedding       []float32 `json:"embedding"`
	PromptEvalCount int       `json:"prompt_eval_count"`
}

func (s *llmServer) Embedding(ctx context.Context, input string) ([]float32, int, error) {
	logutil.Trace("embedding request", "input", input)

	if err := s.sem.Acquire(ctx, 1); err != nil {
		if errors.Is(err, context.Canceled) {
			slog.Info("aborting embedding request due to client closing the connection")
		} else {
			slog.Error("Failed to acquire semaphore", "error", err)
		}
		return nil, 0, err
	}
	defer s.sem.Release(1)

	// Make sure the server is ready
	status, err := s.getServerStatusRetry(ctx)
	if err != nil {
		return nil, 0, err
	} else if status != ServerStatusReady {
		return nil, 0, fmt.Errorf("unexpected server status: %s", status)