Repository: ggml-org/llama.cpp
Pull request: `server`: fix format of streamed tool call deltas (diff name, fix id location)

Fix format of tool call diffs, now much more conformant to the OAI format:

- Define `id` on the tool_call itself, not inside `tool_call.function.id`
- Return `tool_call.function.name` as a delta, not repeated over and over.
- Define `tool_call.type` (`"function"`)
- Return empty `tool_call.arguments` on the first tool call delta

Fixes https://github.com/ggml-org/llama.cpp/issues/13774 (also reported [here](https://github.com/ggml-org/llama.cpp/pull/12379#issuecomment-2908620307) cc/ @bgs4free @Rane2021 @TheTerrasque @making @PrideIsLife @tctien342 )

Note this still sends the tool name in one go, as both OpenAI & Anthropic APIs seem to do (which is probably what confused me initially)

Follow up to https://github.com/ggml-org/llama.cpp/pull/12379

<details>
<summary>Example of outputs</summary>

```bash
clear && curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gpt-4o",
    "tools": [{
          "type": "function",
          "function": {
              "name": "get_current_weather",
              "description": "Get the current weather",
              "parameters": {
                  "type": "object",
                  "properties": {
                      "location": {
                          "type": "string",
                          "description": "The city and state, e.g. San Francisco, CA"
                      },
                      "format": {
                          "type": "string",
                          "enum": ["celsius", "fahrenheit"],
                          "description": "The temperature unit to use. Infer this from the users location."
                      }
                  },
                  "required": ["location", "format"]
              }
          }
      }, {
          "type": "function",
          "function": {
              "name": "get_n_day_weather_forecast",
              "description": "Get an N-day weather forecast",
              "parameters": {
                  "type": "object",
                  "properties": {
                      "location": {
                          "type": "string",
                          "description": "The city and state, e.g. San Francisco, CA"
                      },
                      "format": {
                          "type": "string",
                          "enum": ["celsius", "fahrenheit"],
                          "description": "The temperature unit to use. Infer this from the users location."
                      },
                      "num_days": {
                          "type": "integer",
                          "description": "The number of days to forecast"
                      }
                  },
                  "required": ["location", "format", "num_days"]
              }
          }
      }],
    "messages": [
      {"role": "system", "content": "You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Do not make assumptions about what values to plug into functions. Here are the available tools:"},
      {"role": "user", "content": "what is the weather going to be like in San Francisco and Glasgow over the next 4 days"}
    ],
    "stream": true
  }'
```

```
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1748263544,"id":"chatcmpl-xxesoUemboS3eSaTCzm7PRweLJGrESU2","model":"gpt-4o","system_fingerprint":"b5496-a7e6f3941","object":"chat.completion.chunk"}

data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"id":"iL1vo9XXQNwkXzcKwyYF4wpGwvRF5HAP","type":"function","function":{"name":"get_current_weather","arguments":""}}]}}],"created":1748263544,"id":"chatcmpl-xxesoUemboS3eSaTCzm7PRweLJGrESU2","model":"gpt-4o","system_fingerprint":"b5496-a7e6f3941","object":"chat.completion.chunk"}

data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{"}}]}}],"created":1748263544,"id":"chatcmpl-xxesoUemboS3eSaTCzm7PRweLJGrESU2","model":"gpt-4o","system_fingerprint":"b5496-a7e6f3941","object":"chat.completion.chunk"}

data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]}}],"created":1748263544,"id":"chatcmpl-xxesoUemboS3eSaTCzm7PRweLJGrESU2","model":"gpt-4o","system_fingerprint":"b5496-a7e6f3941","object":"chat.completion.chunk"}

data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"location"}}]}}],"created":1748263544,"id":"chatcmpl-xxesoUemboS3eSaTCzm7PRweLJGrESU2","model":"gpt-4o","system_fingerprint":"b5496-a7e6f3941","object":"chat.completion.chunk"}

data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]}}],"created":1748263544,"id":"chatcmpl-xxesoUemboS3eSaTCzm7PRweLJGrESU2","model":"gpt-4o","system_fingerprint":"b5496-a7e6f3941","object":"chat.completion.chunk"}

data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]}}],"created":1748263544,"id":"chatcmpl-xxesoUemboS3eSaTCzm7PRweLJGrESU2","model":"gpt-4o","system_fingerprint":"b5496-a7e6f3941","object":"chat.completion.chunk"}

...
```

</details>

Changed files:
filename: common/chat.cpp
patch: @@ -106,9 +106,9 @@ std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const comm
         if (!args_diff.empty() || pref.id != newf.id) {
             auto & diff = diffs.emplace_back();
             diff.tool_call_index = idx;
-            diff.tool_call_delta.name = newf.name;
             if (pref.id != newf.id) {
                 diff.tool_call_delta.id = newf.id;
+                diff.tool_call_delta.name = newf.name;
             }
             diff.tool_call_delta.arguments = args_diff;
         }
@@ -392,22 +392,19 @@ template <> json common_chat_msg_diff_to_json_oaicompat(const common_chat_msg_di
         delta["content"] = diff.content_delta;
     }
     if (diff.tool_call_index != std::string::npos) {
+        json tool_call;
+        tool_call["index"] = diff.tool_call_index;
+        if (!diff.tool_call_delta.id.empty()) {
+            tool_call["id"] = diff.tool_call_delta.id;
+            tool_call["type"] = "function";
+        }
         json function = json::object();
         if (!diff.tool_call_delta.name.empty()) {
             function["name"] = diff.tool_call_delta.name;
         }
-        if (!diff.tool_call_delta.id.empty()) {
-            function["id"] = diff.tool_call_delta.id;
-        }
-        if (!diff.tool_call_delta.arguments.empty()) {
-            function["arguments"] = diff.tool_call_delta.arguments;
-        }
-        delta["tool_calls"] = json::array({
-            json {
-                {"index", diff.tool_call_index},
-                {"function", function}
-            }
-        });
+        function["arguments"] = diff.tool_call_delta.arguments;
+        tool_call["function"] = function;
+        delta["tool_calls"] = json::array({tool_call});
     }
     return delta;
 }

filename: tests/test-chat.cpp
patch: @@ -1356,8 +1356,7 @@ static void test_msg_diffs_compute() {
 
         common_chat_msg_diff diff12;
         diff12.tool_call_index = 0;
-        diff12.tool_call_delta.name = "special_function";
-        // Note: id doesnt change here.
+        // Note: neither id nor name change here.
         diff12.tool_call_delta.arguments = "g1\": 1}";
 
         assert_equals(

filename: tools/server/tests/utils.py
patch: @@ -328,6 +328,10 @@ def make_any_request(
                     if 'function' not in tc:
                         raise ValueError(f"Expected function type, got {tc['type']}")
                     if tc['index'] >= len(tool_calls):
+                        assert 'id' in tc
+                        assert tc.get('type') == 'function'
+                        assert 'function' in tc and 'name' in tc['function'] and len(tc['function']['name']) > 0, \
+                            f"Expected function call with name, got {tc.get('function')}"
                         tool_calls.append(dict(
                             id="",
                             type="function",
@@ -340,10 +344,10 @@ def make_any_request(
                     if tc.get('id') is not None:
                         tool_call['id'] = tc['id']
                     fct = tc['function']
+                    assert 'id' not in fct, f"Function call should not have id: {fct}"
                     if fct.get('name') is not None:
-                        tool_call['function']['name'] = fct['name']
+                        tool_call['function']['name'] = tool_call['function'].get('name', '') + fct['name']
                     if fct.get('arguments') is not None:
-                        assert len(fct['arguments']) > 0, f'Expected non empty arguments delta!'
                         tool_call['function']['arguments'] += fct['arguments']
 
             print(f'Streamed response had {content_parts} content parts, {tool_call_parts} tool call parts incl. {arguments_parts} arguments parts')