Repository: ollama/ollama
Pull request: token repeat limit for prediction requests

- abort prediction (generate/chat) requests when a token repeat limit is hit
- this prevent json format infinite loops
- this prevents a stuck request from starving the other queued requests
- move completion cancellation to its own function

Tested with this code from #1910 
```python
import requests 
import json
country = "france"
schema = {
	"city": {
		"type": "string",
		"description": "Name of the city"
	},
	"lat":{
		"type": "float",
		"description": "Decimal Latitude of the city"
	},
	"lon":{
		"type": "float",
		"description": "Decimal Longitude of the city"
	}
}
payload = {
	"model": "mistral-no-repeat",
	"messages": [
		{"role": "system", "content": f"You are a helpful AI assistant. The user will enter a country name and the assistant will return the decimal latitude and decimal longitude of the capital of the country. Output in JSON using the schema defined here: {schema}."},
		{"role": "user", "content": "japan"},
		{"role": "assistant", "content": "{\"city\": \"Tokyo\", \"lat\": 35.6748, \"lon\": 139.7624}"},
		{"role": "user", "content": country},
		],
		"format": "json",
		"stream": False
		
}
response = requests.post ("http://localhost:11434/api/chat", json=payload)
response.raise_for_status()
chat = response.json()
try:
    message_content_json = json.loads(chat['message']['content'])
    print(message_content_json)
except json.JSONDecodeError:
    print("JSONDecodeError: The content is not in proper JSON format.")
```

output is more reliable, with occasional JSON format failures which can be handled with a retry:
```bash
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
{'city': 'Paris', ' lat ': 48.8566, ' lon': 2.3522}
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
{'city': 'Paris', 'lat': 48.8566, ' lon ': 2.3522}
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
{'city': 'Paris', 'lat': 48.8566, ' lon ': 2.3522}
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
{'city': 'Paris', ' lat': 48.8566, ' lon': 2.3522}
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
{'city': 'Paris', 'lat': 48.8566, ' lon ': 2.3522}
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
{'city': 'Paris', ' lat ': 48.8566, ' lon': 2.3522}
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
JSONDecodeError: The content is not in proper JSON format.
bruce@Bruces-MBP triage % poetry run python3 make_json_request.py
{'city': 'Paris', ' lat ': 48.8534, ' lon': 2.3522}
```

resolves #1910 

Changed files:
filename: llm/dyn_ext_server.go
patch: @@ -228,17 +228,14 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu
 		}
 
 		retryNeeded := false
+		// keep track of the last token generated, this is used to abort if the model starts looping
+		var lastToken string
+		var tokenRepeat int
 	out:
 		for {
 			select {
 			case <-ctx.Done():
-				// This handles the request cancellation
-				C.dyn_llama_server_completion_cancel(llm.s, resp.id, &resp)
-				if resp.id < 0 {
-					return extServerResponseToErr(resp)
-				} else {
-					return nil
-				}
+				return cancelCompletion(llm, resp)
 			default:
 				var result C.ext_server_task_result_t
 				C.dyn_llama_server_completion_next_result(llm.s, resp.id, &result)
@@ -261,6 +258,20 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu
 					break out
 				}
 
+				switch {
+				case strings.TrimSpace(p.Content) == lastToken:
+					tokenRepeat++
+				default:
+					lastToken = strings.TrimSpace(p.Content)
+					tokenRepeat = 0
+				}
+
+				// 30 picked as an arbitrary max token repeat limit, modify as needed
+				if tokenRepeat > 30 {
+					slog.Debug("prediction aborted, token repeat limit reached")
+					return cancelCompletion(llm, resp)
+				}
+
 				if p.Content != "" {
 					fn(PredictResult{
 						Content: p.Content,
@@ -288,6 +299,15 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu
 	return fmt.Errorf("max retries exceeded")
 }
 
+func cancelCompletion(llm *dynExtServer, resp C.ext_server_resp_t) error {
+	C.dyn_llama_server_completion_cancel(llm.s, resp.id, &resp)
+	if resp.id < 0 {
+		return extServerResponseToErr(resp)
+	} else {
+		return nil
+	}
+}
+
 func (llm *dynExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
 	data, err := json.Marshal(TokenizeRequest{Content: prompt})
 	if err != nil {