Repository: ggml-org/llama.cpp
Pull request: Add Mistral-Small-3.2-24B-Instruct-2506.jinja

After referencing:

https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407
https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503

and how Qwen does tool calling, plus some help from my good friend Claude, was able to figure out proper tool calling support and what seems to be a proper chat template that follows the best practices from mistral themselves

Hosted with this command:

```
./llama-server -m /models/Mistral-Small-3.2-24B-Instruct-2506-GGUF/mistralai_Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -ngl 100 --host 0.0.0.0 -fa --slots --jinja --chat-template-file /models/Mistral-Small-3.2-24B-Instruct-2506.jinja
```

Used the following script to validate:

<details>
<summary>test_mistral.py script for checking the tool use works</summary>

```
#!/usr/bin/env python3
"""
Tool calling test script for llama.cpp OpenAI-compatible endpoint
Tests the full flow: initial request -> tool call -> mock result -> final response
"""

import json
import requests
from typing import Dict, Any, List

# Configuration
API_URL = "http://localhost:8080/v1/chat/completions"
MODEL = "mistralai_Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf"

# Tool definition
WEATHER_TOOL = {
    "type": "function",
    "function": {
        "name": "get_current_weather",
        "description": "Get the current weather in a given location",
        "parameters": {
            "type": "object",
            "properties": {
                "location": {
                    "type": "string",
                    "description": "The city and country/state, e.g. `San Francisco, CA`, or `Paris, France`"
                }
            },
            "required": ["location"]
        }
    }
}

def make_request(messages: List[Dict[str, Any]], tools: List[Dict[str, Any]] = None) -> Dict[str, Any]:
    """Make a request to the API"""
    payload = {
        "model": MODEL,
        "messages": messages
    }
    
    if tools:
        payload["tools"] = tools
    
    response = requests.post(API_URL, json=payload)
    response.raise_for_status()
    return response.json()

def get_mock_weather(location: str) -> Dict[str, Any]:
    """Generate mock weather data for a location"""
    # Mock weather data
    mock_data = {
        "Istanbul": {
            "temperature": 18,
            "condition": "partly cloudy",
            "humidity": 65,
            "wind_speed": 12
        },
        "default": {
            "temperature": 22,
            "condition": "sunny",
            "humidity": 50,
            "wind_speed": 8
        }
    }
    
    weather = mock_data.get(location.split(",")[0].strip(), mock_data["default"])
    
    return {
        "location": location,
        "temperature_celsius": weather["temperature"],
        "condition": weather["condition"],
        "humidity_percent": weather["humidity"],
        "wind_speed_kmh": weather["wind_speed"]
    }

def main():
    print("🚀 Testing tool calling with llama.cpp endpoint\n")
    
    # Initial conversation
    messages = [
        {"role": "system", "content": "You are a chatbot that uses tools/functions. Don't overthink things."},
        {"role": "user", "content": "What is the weather in Istanbul?"}
    ]
    
    print("📤 Sending initial request with tool definition...")
    print(f"User: {messages[-1]['content']}\n")
    
    # Step 1: Initial request with tool
    try:
        response = make_request(messages, [WEATHER_TOOL])
        print("📥 Received response:")
        print(json.dumps(response, indent=2))
        print()
        
        # Extract the assistant's response
        assistant_message = response["choices"][0]["message"]
        messages.append(assistant_message)
        
        # Check if the model made a tool call
        if "tool_calls" in assistant_message and assistant_message["tool_calls"]:
            tool_call = assistant_message["tool_calls"][0]
            function_name = tool_call["function"]["name"]
            function_args = json.loads(tool_call["function"]["arguments"])
            
            print(f"🔧 Model called tool: {function_name}")
            print(f"   Arguments: {function_args}\n")
            
            # Step 2: Execute the tool (mock)
            if function_name == "get_current_weather":
                location = function_args.get("location", "Unknown")
                weather_data = get_mock_weather(location)
                
                print(f"🌤️  Mock weather data for {location}:")
                print(json.dumps(weather_data, indent=2))
                print()
                
                # Step 3: Send tool result back to model
                tool_result_message = {
                    "role": "tool",
                    "tool_call_id": tool_call["id"],
                    "content": json.dumps(weather_data)
                }
                messages.append(tool_result_message)
                
                print("📤 Sending tool result back to model...")
                final_response = make_request(messages)
                
                print("📥 Final response from model:")
                print(json.dumps(final_response, indent=2))
                print()
                
                # Extract and display the final answer
                final_content = final_response["choices"][0]["message"]["content"]
                print("💬 Assistant's final answer:")
                print(final_content)
                
                # Add the assistant's response to messages
                messages.append({
                    "role": "assistant",
                    "content": final_content
                })
                
                # Continue with more turns
                print("\n" + "="*60 + "\n")
                test_additional_turns(messages)
            else:
                print(f"❓ Unknown tool called: {function_name}")
        else:
            print("⚠️  Model did not make a tool call")
            if "content" in assistant_message:
                print(f"Model response: {assistant_message['content']}")
                
    except requests.exceptions.RequestException as e:
        print(f"❌ Request error: {e}")
    except json.JSONDecodeError as e:
        print(f"❌ JSON parsing error: {e}")
    except KeyError as e:
        print(f"❌ Unexpected response format, missing key: {e}")
    except Exception as e:
        print(f"❌ Unexpected error: {e}")

def test_additional_turns(messages: List[Dict[str, Any]]):
    """Test additional conversation turns after tool usage"""
    print("🔄 Testing additional conversation turns...\n")
    
    # Turn 1: Follow-up question about the weather (no tool needed)
    print("📍 Turn 1: Follow-up question")
    messages.append({
        "role": "user",
        "content": "Is that good weather for sightseeing?"
    })
    print(f"User: {messages[-1]['content']}\n")
    
    try:
        response = make_request(messages)
        assistant_response = response["choices"][0]["message"]["content"]
        print(f"💬 Assistant: {assistant_response}\n")
        messages.append({
            "role": "assistant",
            "content": assistant_response
        })
    except Exception as e:
        print(f"❌ Error in turn 1: {e}\n")
        return
    
    print("-" * 40 + "\n")
    
    # Turn 2: Different topic (no tool needed)
    print("📍 Turn 2: Different topic")
    messages.append({
        "role": "user",
        "content": "By the way, what's the capital of Turkey?"
    })
    print(f"User: {messages[-1]['content']}\n")
    
    try:
        response = make_request(messages)
        assistant_response = response["choices"][0]["message"]["content"]
        print(f"💬 Assistant: {assistant_response}\n")
        messages.append({
            "role": "assistant",
            "content": assistant_response
        })
    except Exception as e:
        print(f"❌ Error in turn 2: {e}\n")
        return
    
    print("-" * 40 + "\n")
    
    # Turn 3: Another weather request (should trigger tool use again)
    print("📍 Turn 3: Another weather request with tool")
    messages.append({
        "role": "user",
        "content": "What about the weather in Paris, France?"
    })
    print(f"User: {messages[-1]['content']}\n")
    
    try:
        response = make_request(messages, [WEATHER_TOOL])
        assistant_message = response["choices"][0]["message"]
        
        if "tool_calls" in assistant_message and assistant_message["tool_calls"]:
            tool_call = assistant_message["tool_calls"][0]
            function_args = json.loads(tool_call["function"]["arguments"])
            
            print(f"🔧 Model called tool again: {tool_call['function']['name']}")
            print(f"   Arguments: {function_args}\n")
            
            # Get mock weather for Paris
            location = function_args.get("location", "Unknown")
            weather_data = get_mock_weather(location)
            
            print(f"🌤️  Mock weather data for {location}:")
            print(json.dumps(weather_data, indent=2))
            print()
            
            # Send tool result
            messages.append(assistant_message)
            messages.append({
                "role": "tool",
                "tool_call_id": tool_call["id"],
                "content": json.dumps(weather_data)
            })
            
            final_response = make_request(messages)
            final_content = final_response["choices"][0]["message"]["content"]
            print(f"💬 Assistant: {final_content}\n")
            messages.append({
                "role": "assistant",
                "content": final_content
            })
        else:
            print("⚠️  Model did not make a tool call for Paris weather")
            if "content" in assistant_message:
                print(f"Model response: {assistant_message['content']}")
    except Exception as e:
        print(f"❌ Error in turn 3: {e}\n")
        return
    
    print("-" * 40 + "\n")
    
    # Turn 4: Comparison question (no tool needed)
    print("📍 Turn 4: Comparison question")
    messages.append({
        "role": "user",
        "content": "Which city has better weather for visiting, Istanbul or Paris?"
    })
    print(f"User: {messages[-1]['content']}\n")
    
    try:
        response = make_request(messages)
        assistant_response = response["choices"][0]["message"]["content"]
        print(f"💬 Assistant: {assistant_response}\n")
    except Exception as e:
        print(f"❌ Error in turn 4: {e}\n")
    
    print("✅ Multi-turn conversation test complete!")

if __name__ == "__main__":
    main()
```
</details>

Which gave this proper output:

<details>
<summary>test_mistral.py script results</summary>

```
 python3 test_mistral.py 
🚀 Testing tool calling with llama.cpp endpoint

📤 Sending initial request with tool definition...
User: What is the weather in Istanbul?

📥 Received response:
{
  "choices": [
    {
      "finish_reason": "tool_calls",
      "index": 0,
      "message": {
        "role": "assistant",
        "content": null,
        "tool_calls": [
          {
            "type": "function",
            "function": {
              "name": "get_current_weather",
              "arguments": "{\"location\":\"Istanbul, Turkey.\"}"
            },
            "id": "a0b2a9fb4"
          }
        ]
      }
    }
  ],
  "created": 1750711340,
  "model": "mistralai_Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf",
  "system_fingerprint": "b5697-ef035803",
  "object": "chat.completion",
  "usage": {
    "completion_tokens": 39,
    "prompt_tokens": 120,
    "total_tokens": 159
  },
  "id": "chatcmpl-rdyqGaribJmnaex1JO4tMR46XfvLGQlL",
  "timings": {
    "prompt_n": 100,
    "prompt_ms": 100.287,
    "prompt_per_token_ms": 1.0028700000000002,
    "prompt_per_second": 997.1382133277492,
    "predicted_n": 39,
    "predicted_ms": 904.706,
    "predicted_per_token_ms": 23.197589743589745,
    "predicted_per_second": 43.10792677400172
  }
}

🔧 Model called tool: get_current_weather
   Arguments: {'location': 'Istanbul, Turkey.'}

🌤️  Mock weather data for Istanbul, Turkey.:
{
  "location": "Istanbul, Turkey.",
  "temperature_celsius": 18,
  "condition": "partly cloudy",
  "humidity_percent": 65,
  "wind_speed_kmh": 12
}

📤 Sending tool result back to model...
📥 Final response from model:
{
  "choices": [
    {
      "finish_reason": "stop",
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "The current weather in Istanbul, Turkey is partly cloudy with a temperature of 18\u00b0C, humidity of 65%, and wind speed of 12 km/h."
      }
    }
  ],
  "created": 1750711341,
  "model": "mistralai_Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf",
  "system_fingerprint": "b5697-ef035803",
  "object": "chat.completion",
  "usage": {
    "completion_tokens": 37,
    "prompt_tokens": 120,
    "total_tokens": 157
  },
  "id": "chatcmpl-3sAqhVS4CS8P3TOwm7ewGK5KVP315kq1",
  "timings": {
    "prompt_n": 100,
    "prompt_ms": 79.474,
    "prompt_per_token_ms": 0.79474,
    "prompt_per_second": 1258.2731459345193,
    "predicted_n": 37,
    "predicted_ms": 790.148,
    "predicted_per_token_ms": 21.355351351351352,
    "predicted_per_second": 46.82667044654925
  }
}

💬 Assistant's final answer:
The current weather in Istanbul, Turkey is partly cloudy with a temperature of 18°C, humidity of 65%, and wind speed of 12 km/h.

============================================================

🔄 Testing additional conversation turns...

📍 Turn 1: Follow-up question
User: Is that good weather for sightseeing?

💬 Assistant: Based on the current weather conditions in Istanbul, it seems to be quite pleasant for sightseeing. A temperature of 18°C (64°F) is comfortable for outdoor activities, and the partly cloudy skies suggest that you won't have to deal with intense sun or heavy rain. Just make sure to dress in layers as the weather can change throughout the day, and perhaps carry a light jacket in case it gets chilly. Enjoy your exploration of Istanbul!

----------------------------------------

📍 Turn 2: Different topic
User: By the way, what's the capital of Turkey?

💬 Assistant: The capital of Turkey is Ankara.

----------------------------------------

📍 Turn 3: Another weather request with tool
User: What about the weather in Paris, France?

🔧 Model called tool again: get_current_weather
   Arguments: {'location': 'Paris, France.'}

🌤️  Mock weather data for Paris, France.:
{
  "location": "Paris, France.",
  "temperature_celsius": 22,
  "condition": "sunny",
  "humidity_percent": 50,
  "wind_speed_kmh": 8
}

💬 Assistant: The current weather in Paris, France is sunny with a temperature of 22°C, humidity of 50%, and wind speed of 8 km/h.

----------------------------------------

📍 Turn 4: Comparison question
User: Which city has better weather for visiting, Istanbul or Paris?

💬 Assistant: Both cities have pleasant weather, but the better option depends on your preferences:

* If you enjoy slightly cooler temperatures and lower humidity, Istanbul might be a better choice with its current temperature of 18°C (64°F) and partly cloudy skies.
* If you prefer warmer, sunnier weather with lower humidity, Paris might be more appealing with its current temperature of 22°C (72°F) and sunny skies.

Since the weather in both cities is generally good for sightseeing, you may want to consider other factors, such as the specific attractions you're interested in, when deciding which city to visit.

✅ Multi-turn conversation test complete!
```
</details>

Only thing left is that the `yesterday` functionality doesn't work, but I think it's best to try to get a change in to minja to support that


Changed files:
filename: models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja
patch: @@ -0,0 +1,124 @@
+{%- set today = strftime_now("%Y-%m-%d") %}
+{%- set default_system_message = "You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\nYour knowledge base was last updated on 2023-10-01. The current date is " + today + ".\n\nWhen you're not sure about some information or when the user's request requires up-to-date or specific data, you must use the available tools to fetch the information. Do not hesitate to use tools whenever they can provide a more accurate or complete response. If no relevant tools are available, then clearly state that you don't have the information and avoid making up anything.
+
+If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\").
+You are always very attentive to dates, and when asked about information at specific dates, you discard information that is at another date.
+You follow these instructions in all languages, and always respond to the user in the language they use or request.
+Next sections describe the capabilities that you have.
+
+# WEB BROWSING INSTRUCTIONS
+
+You cannot perform any web search or access internet to open URLs, links etc. If it seems like the user is expecting you to do so, you clarify the situation and ask the user to copy paste the text directly in the chat.
+
+# MULTI-MODAL INSTRUCTIONS
+
+You have the ability to read images, but you cannot generate images. You also cannot transcribe audio files or videos.
+You cannot read nor transcribe audio files or videos.
+
+# TOOL CALLING INSTRUCTIONS
+
+You may have access to tools that you can use to fetch information or perform actions. You must use these tools in the following situations:
+
+1. When the request requires up-to-date information.
+2. When the request requires specific data that you do not have in your knowledge base.
+3. When the request involves actions that you cannot perform without tools.
+
+Always prioritize using tools to provide the most accurate and helpful response. If tools are not available, inform the user that you cannot perform the requested action at the moment." %}
+
+{{- bos_token }}
+
+{%- set system_prompt = default_system_message %}
+{%- set loop_messages = messages %}
+
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{%- if messages|length > 0 and messages[0]['role'] == 'system' %}
+    {%- if messages[0]['content'] is string %}
+        {%- set system_prompt = messages[0]['content'] %}
+    {%- else %}
+        {%- set system_prompt = messages[0]['content'][0]['text'] %}
+    {%- endif %}
+    {%- set loop_messages = messages[1:] %}
+{%- endif %}
+
+{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
+
+{%- set ns = namespace(index=0) %}
+{%- for message in loop_messages %}
+    {%- if not (message.role == "tool" or (message.get('tool_calls'))) %}
+        {%- if (message["role"] == "user") != (ns.index % 2 == 0) %}
+            {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
+        {%- endif %}
+        {%- set ns.index = ns.index + 1 %}
+    {%- endif %}
+{%- endfor %}
+
+{{- '[SYSTEM_PROMPT]' + system_prompt + '[/SYSTEM_PROMPT]' }}
+
+{%- for message in loop_messages %}
+    {%- if message['role'] == 'system' %}
+        {%- if message['content'] is string %}
+            {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}
+        {%- else %}
+            {{- '[SYSTEM_PROMPT]' + message['content'][0]['text'] + '[/SYSTEM_PROMPT]' }}
+        {%- endif %}
+    {%- elif message['role'] == 'user' %}
+        {%- if tools is not none and (message == user_messages[-1]) %}
+            {{- '[AVAILABLE_TOOLS]' + tools|tojson + '[/AVAILABLE_TOOLS]' }}
+        {%- endif %}
+        {{- '
' }}
+        {%- if message['content'] is string %}
+            {{- message['content'] }}
+        {%- else %}
+            {%- for block in message['content'] %}
+                {%- if block['type'] == 'text' %}
+                    {{- block['text'] }}
+                {%- elif block['type'] in ['image', 'image_url'] %}
+                    {{- '[IMG]' }}
+                {%- else %}
+                    {{- raise_exception('Only text and image blocks are supported in message content!') }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        {{- '
' }}
+    {%- elif message['role'] == 'assistant' %}
+        {%- if message.get('tool_calls') %}
+            {%- for tool_call in message.tool_calls %}
+                {{- '[TOOL_CALLS]' + tool_call.function.name }}
+                {%- if not tool_call.id is defined or tool_call.id is not string or tool_call.id|length != 9 %}
+                    {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+                {%- endif %}
+                {{- '[CALL_ID]' + tool_call.id }}
+                {{- '[ARGS]' + tool_call['function']['arguments']|tojson }}
+            {%- endfor %}
+            {{- eos_token }}
+        {%- elif message['content'] is string %}
+            {{- message['content'] + eos_token }}
+        {%- else %}
+            {%- for block in message['content'] %}
+                {%- if block['type'] == 'text' %}
+                    {{- block['text'] }}
+                {%- elif block['type'] in ['image', 'image_url'] %}
+                    {{- '[IMG]' }}
+                {%- else %}
+                    {{- raise_exception('Only text and image blocks are supported in assistant content!') }}
+                {%- endif %}
+            {%- endfor %}
+            {{- eos_token }}
+        {%- endif %}
+    {%- elif message['role'] == 'tool_results' or message['role'] == 'tool' %}
+        {%- if message.content is defined and message.content.content is defined %}
+            {%- set content = message.content.content %}
+        {%- else %}
+            {%- set content = message.content %}
+        {%- endif %}
+        {%- if not message.tool_call_id is defined or message.tool_call_id is not string or message['tool_call_id']|length != 9 %}
+            {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+        {%- endif %}
+        {{- '[TOOL_RESULTS]' + message.tool_call_id + '[TOOL_CONTENT]' + content|string + '[/TOOL_RESULTS]' }}
+    {%- else %}
+        {{- raise_exception('Only system, user, assistant, and tool roles are supported!') }}
+    {%- endif %}
+{%- endfor %}