Repository: ggml-org/llama.cpp
Pull request: Thinking model disabled assistant prefill

Fixes: #15401 

cc @matteoserva from [the original PR](https://github.com/ggml-org/llama.cpp/pull/13196/files#r2134727153)

## Changes

* Auto-detect whether `enable_thinking` should be true based on whether the rendered template changes when enabled
* Always set `input.enable_thinking` from `chat_template_kwargs.enable_thinking` if set explicitly there
* Fix the logical condition to reject agent-prefill when `enable_thinking` is true only when set by default for thinking models or explicitly via kwarg

## Testing

All model types tested using the following set of calls

```sh
# Test without setting enable_thinking
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}]}'

# Test with enable_thinking: true
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": true}}'

# Test with enable_thinking: false
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": false}}'
```

### Thinking model w/ thinking enabled

```sh
./bin/llama-server -hf ibm-granite/granite-3.2-8b-instruct-GGUF --jinja
```

```sh
# Reasoning enabled implicitly => error
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}]}'
# {"error":{"code":500,"message":"Assistant response prefill is incompatible with enable_thinking.","type":"server_error"}}

# Reasoning enabled explicitly => error
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": true}}'
# {"error":{"code":500,"message":"Assistant response prefill is incompatible with enable_thinking.","type":"server_error"}}

# Reasoning disabled explicitly => ok
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": true}}' 
# {"prompt":"
system
Knowledge Cutoff Date: April 2024.\nToday's Date: August 18, 2025.\nYou are Granite, developed by IBM. You are a helpful AI assistant.
\n
user
hello world
\n
assistant
hi hi"}
```

### Thinking model w/ thinking disabled

```sh
./bin/llama-server -hf ibm-granite/granite-3.2-8b-instruct-GGUF --jinja --reasoning-budget 0
```

```sh
# Reasoning disabled implicitly => ok
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}]}'
# {"prompt":"
system
Knowledge Cutoff Date: April 2024.\nToday's Date: August 18, 2025.\nYou are Granite, developed by IBM. You are a helpful AI assistant.
\n
user
hello world
\n
assistant
hi hi"}

# Reasoning enabled explicitly => error
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": true}}'
# {"error":{"code":500,"message":"Assistant response prefill is incompatible with enable_thinking.","type":"server_error"}}

# Reasoning disabled explicitly => ok
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": true}}' 
# {"prompt":"
system
Knowledge Cutoff Date: April 2024.\nToday's Date: August 18, 2025.\nYou are Granite, developed by IBM. You are a helpful AI assistant.
\n
user
hello world
\n
assistant
hi hi"}
```

### Non-Thinking model

```sh
./bin/llama-server -m ~/models/granite-3.0-2b-instruct/granite-3.0-2B-instruct-F16.gguf --jinja
```

```sh
# Reasoning disabled implicitly => ok
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}]}'
# {"prompt":"
user
hello world
\n
assistant
hi hi"}

# Reasoning enabled explicitly => error
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": true}}'
# {"error":{"code":500,"message":"Assistant response prefill is incompatible with enable_thinking.","type":"server_error"}}

# Reasoning disabled explicitly => ok
curl http://localhost:8080/apply-template -d '{"messages": [{"role": "user", "content": "hello world"}, {"role": "assistant", "content": "hi hi"}], "chat_template_kwargs": {"enable_thinking": true}}' 
# {"prompt":"
user
hello world
\n
assistant
hi hi"}
```

Changed files:
filename: common/chat.cpp
patch: @@ -163,6 +163,19 @@ common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::strin
     throw std::runtime_error("Invalid tool_choice: " + tool_choice);
 }
 
+bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates) {
+    common_chat_templates_inputs dummy_inputs;
+    common_chat_msg msg;
+    msg.role = "user";
+    msg.content = "test";
+    dummy_inputs.messages = {msg};
+    dummy_inputs.enable_thinking = false;
+    const auto rendered_no_thinking = common_chat_templates_apply(chat_templates, dummy_inputs);
+    dummy_inputs.enable_thinking = true;
+    const auto rendered_with_thinking = common_chat_templates_apply(chat_templates, dummy_inputs);
+    return rendered_no_thinking.prompt != rendered_with_thinking.prompt;
+}
+
 template <>
 std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messages) {
     std::vector<common_chat_msg> msgs;

filename: common/chat.h
patch: @@ -199,6 +199,8 @@ common_chat_msg           common_chat_parse(const std::string & input, bool is_p
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice);
 
+bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);
+
 // Parses a JSON array of messages in OpenAI's chat completion API format.
 // T can be std::string containing JSON or nlohmann::ordered_json
 template <class T> std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const T & messages);

filename: tools/server/server.cpp
patch: @@ -2267,6 +2267,12 @@ struct server_context {
 
         metrics.init();
 
+        // thinking is enabled if:
+        // 1. It's not explicitly disabled (reasoning_budget == 0)
+        // 2. The chat template supports it
+        const bool enable_thinking = params_base.reasoning_budget != 0 && common_chat_templates_support_enable_thinking(chat_templates.get());
+        SRV_INF("Enable thinking? %d\n", enable_thinking);
+
         oai_parser_opt = {
             /* use_jinja             */ params_base.use_jinja,
             /* prefill_assistant     */ params_base.prefill_assistant,
@@ -2275,7 +2281,7 @@ struct server_context {
             /* common_chat_templates */ chat_templates.get(),
             /* allow_image           */ mctx ? mtmd_support_vision(mctx) : false,
             /* allow_audio           */ mctx ? mtmd_support_audio (mctx) : false,
-            /* enable_thinking       */ params_base.reasoning_budget != 0,
+            /* enable_thinking       */ enable_thinking,
         };
     }
 

filename: tools/server/utils.hpp
patch: @@ -54,8 +54,8 @@ static T json_value(const json & body, const std::string & key, const T & defaul
     if (body.contains(key) && !body.at(key).is_null()) {
         try {
             return body.at(key);
-        } catch (NLOHMANN_JSON_NAMESPACE::detail::type_error const &) {
-            LOG_WRN("Wrong type supplied for parameter '%s'. Expected '%s', using default value\n", key.c_str(), json(default_value).type_name());
+        } catch (NLOHMANN_JSON_NAMESPACE::detail::type_error const & err) {
+            LOG_WRN("Wrong type supplied for parameter '%s'. Expected '%s', using default value: %s\n", key.c_str(), json(default_value).type_name(), err.what());
             return default_value;
         }
     } else {
@@ -708,6 +708,16 @@ static json oaicompat_chat_params_parse(
         inputs.chat_template_kwargs[item.key()] = item.value().dump();
     }
 
+    // parse the "enable_thinking" kwarg to override the default value
+    auto enable_thinking_kwarg = json_value(inputs.chat_template_kwargs, "enable_thinking", std::string(""));
+    if (enable_thinking_kwarg == "true") {
+        inputs.enable_thinking = true;
+    } else if (enable_thinking_kwarg == "false") {
+        inputs.enable_thinking = false;
+    } else if (!enable_thinking_kwarg.empty() && enable_thinking_kwarg[0] == '"') {
+        throw std::runtime_error("invalid type for \"enable_thinking\" (expected boolean, got string)");
+    }
+
     // if the assistant message appears at the end of list, we do not add end-of-turn token
     // for ex. this can be useful to modify the reasoning process in reasoning models
     bool prefill_assistant_message = !inputs.messages.empty() && inputs.messages.back().role == "assistant" && opt.prefill_assistant;
@@ -724,7 +734,7 @@ static json oaicompat_chat_params_parse(
         /* TODO: test this properly */
         inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE;
 
-        if ( (!inputs.enable_thinking) || inputs.chat_template_kwargs.find("enable_thinking") != inputs.chat_template_kwargs.end()) {
+        if ( inputs.enable_thinking ) {
             throw std::runtime_error("Assistant response prefill is incompatible with enable_thinking.");
         }