Repository: ollama/ollama
File: runner/llamarunner/runner.go (Go)

Code before:

	if s.model.AddBOSToken() {
		params.numKeep += 1
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if len(inputs) > s.cache.numCtx {
		discard := len(inputs) - s.cache.numCtx

		// If truncation is not enabled, return an error
		if !params.truncate {
			return nil, errorInputTooLong
		}

		var newInputs []input

		// TODO: EOG token handling should be moved to tokenizer
		if params.embedding {
			// If the original prompt ended with an EOG token, add it back after truncation
			lastIsEOG := false

			eogToken := 0

			if len(inputs) > 0 && s.model.TokenIsEog(inputs[len(inputs)-1].token) {
				eogToken = inputs[len(inputs)-1].token
			}

			// If embedding only, truncate to the maximum context length - 1 (to account for BOS token)
			newLimit := s.cache.numCtx - 1

			if s.model.TokenIsEog(inputs[len(inputs)-1].token) && !s.model.TokenIsEog(inputs[newLimit-1].token) {
				lastIsEOG = true
				newLimit--
			}

			if newLimit <= 0 {
				return nil, fmt.Errorf("input after truncation exceeds maximum context length")
			}

			newInputs = inputs[:newLimit]

			if lastIsEOG {
				newInputs = append(newInputs, input{token: eogToken})
			}
		} else {
			// Otherwise, truncate in the middle
			newInputs = inputs[:params.numKeep]
			newInputs = append(newInputs, inputs[params.numKeep+discard:]...)
		}

Review comment (performance):
I think we can clean this up a little:

```
// make sure to end the truncated sequence with the eog token if one was originally provided
eogToken := -1
if len(inputs) > 0 && s.model.TokenIsEog(inputs[len(inputs)-1].token) {
  eogToken = inputs[len(inputs)-1].token
}

newInputs = inputs[:s.cache.numCtx]
if eogToken >= 0 {
  newInputs[len(newInputs) - 1] = input{token: eogToken}
}
 ```


Code after:

	if s.model.AddBOSToken() {
		params.numKeep += 1
	}

	// Ensure that at least 1 input can be discarded during shift
	params.numKeep = min(params.numKeep, s.cache.numCtx-1)

	if len(inputs) > s.cache.numCtx {
		discard := len(inputs) - s.cache.numCtx
		if !params.truncate {
			return nil, errorInputTooLong
		}

		newInputs := inputs[:params.numKeep]
		newInputs = append(newInputs, inputs[params.numKeep+discard:]...)

		slog.Warn("truncating input prompt", "limit", s.cache.numCtx, "prompt", len(inputs), "keep", params.numKeep, "new", len(newInputs))
		inputs = newInputs
	}

	var sc *llama.SamplingContext
	if params.samplingParams != nil {
		sc, err = llama.NewSamplingContext(s.model, *params.samplingParams)
		if err != nil {
			return nil, err
		}
		for _, input := range inputs {
			if input.embed == nil {
				sc.Accept(input.token, false)
			}
		}
	}

	return &Sequence{
		inputs:           inputs,
		numPromptInputs:  len(inputs),
		numPredict:       params.numPredict,
		pendingResponses: make([]string, 0),
		responses:        make(chan string, 100),
		quit:             make(chan bool, 1),
		embedding:        make(chan []float32, 1),
		samplingCtx:      sc,
		embeddingOnly:    params.embedding,
		stop:             params.stop,
		numKeep:          params.numKeep,
		shift:            params.shift,
	}, nil
}

// inputs processes the prompt and images into a list of inputs