Repository: promptfoo/promptfoo
Issue: Responses API issues with tool calling
Root cause: Algorithmic Error

A fix was applied in #5172 to support the OpenAI Responses API when using tool callbacks. The immediate issue I reported was resolved, but the final response is still not correct.

Here's a simple example that works correctly with the `openai:gpt-4o` provider but fails with `openai:responses:gpt-4`:

```
module.exports = /** @type {import('promptfoo').TestSuiteConfig} */ ({
  prompts: [
    'Please add the following numbers together: {{a}} and {{b}}',
  ],
  providers: [
    {
      id: 'openai:responses:gpt-4o',
      config: {
        model: 'gpt-4o',
        temperature: 0,
        tools: [
          {
            type: 'function',
            name: 'addNumbers',
            function: {
              name: 'addNumbers',
              description: 'Add two numbers together',
              parameters: {
                type: 'object',
                properties: {
                  a: { type: 'number' },
                  b: { type: 'number' },
                },
                required: ['a', 'b'],
              },
            },
          },
        ],
        tool_choice: 'auto',
        functionToolCallbacks: {
          addNumbers: (parametersJsonString) => {
            const { a, b } = JSON.parse(parametersJsonString);
            return JSON.stringify(a + b);
          },
        },
      },
    },
  ],
  tests: [
    {
      vars: { a: 5, b: 6 },
      assert: [
        {
          type: 'equals',
          value: '11',
        },
      ],
    },
  ],
});
```

The final output is an empty string instead of the tool result `"11"`, so the test fails. Here's my full log:

```
[main.js:88] Verbose mode enabled via --verbose flag
[migrate.js:58] Running database migrations...
[migrate.js:60] Database migrations completed
[index.js:156] Reading prompts from ["Please add the following numbers together: {{a}} and {{b}}"]
[filterTests.js:45] Starting filterTests with options: {}
[filterTests.js:46] Initial test count: 1
[eval.js:171] Inserting prompt 600fc45c7ae71955cf7be351bc650a4c6084fe37810cf2aa31600089a4d6e391
[eval.js:194] Inserting dataset 9e0ae52a96963a70c5d60e3b550ac7b08398544cab373133bf5d20edaf50e3ae
[evaluatorTracing.js:83] [EvaluatorTracing] Checking tracing config: undefined
[evaluatorTracing.js:84] [EvaluatorTracing] testSuite keys: description,tags,prompts,providers,providerPromptMap,tests,scenarios,defaultTest,derivedMetrics,nunjucksFilters,extensions,tracing
[evaluatorTracing.js:85] [EvaluatorTracing] Full testSuite.tracing: undefined
[evaluatorTracing.js:111] [EvaluatorTracing] Tracing not enabled or OTLP HTTP receiver not configured
[evaluatorTracing.js:112] [EvaluatorTracing] tracing.enabled: undefined
[evaluatorTracing.js:113] [EvaluatorTracing] tracing.otlp.http.enabled: undefined
[evaluator.js:592] Starting evaluation eval-Ezs-2025-08-26T22:54:01
[evaluator.js:1126] Progress bar settings: showProgressBar=false, isWebUI=false
[evaluator.js:1161] Running 1 test cases (up to 4 at a time)...
[evaluator.js:288] Provider type: openai:responses:gpt-4o
[responses.js:167] Calling OpenAI Responses API: {"model":"gpt-4o","input":"Please add the following numbers together: 5 and 6","max_output_tokens":1024,"tools":[{"type":"function","name":"addNumbers","function":{"name":"addNumbers","description":"Add two numbers together","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"]}}}],"tool_choice":"auto","text":{"format":{"type":"text"}}}
[cache.js:121] Returning cached response for https://api.openai.com/v1/responses: {"data":{"id":"resp_68ae2c0797c08191ab22276ded1571d9061d9d321cbfcb2e","object":"response","created_at":1756244999,"status":"completed","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":1024,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[{"id":"fc_68ae2c084c9081918aec2fd382c3d6d3061d9d321cbfcb2e","type":"function_call","status":"completed","arguments":"{}","call_id":"call_UA11D4k1VhcCDmIE9cKxE1o6","name":"addNumbers"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":null,"name":"addNumbers","parameters":{"additionalProperties":false,"type":"object","properties":{},"required":[]},"strict":true}],"top_logprobs":0,"top_p":1,"truncation":"disabled","usage":{"input_tokens":37,"input_tokens_details":{"cached_tokens":0},"output_tokens":11,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":48},"user":null,"metadata":{}},"status":200,"statusText":"OK","headers":{"alt-svc":"h3=\":443\"; ma=86400","cf-cache-status":"DYNAMIC","cf-ray":"97568acd8b17e5ad-DFW","connection":"keep-alive","content-encoding":"br","content-type":"application/json","date":"Tue, 26 Aug 2025 21:50:00 GMT","openai-organization":"move37-3iz7md","openai-processing-ms":"793","openai-project":"proj_ktlBsIbDeO8G2F8irVE4tZy2","openai-version":"2020-10-01","server":"cloudflare","set-cookie":"_cfuvid=0rZNABm5sJrVmxD34pugOwTlOv_5x..DEx.5nY2qfb4-1756245000415-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None","strict-transport-security":"max-age=31536000; includeSubDomains; preload","transfer-encoding":"chunked","x-content-type-options":"nosniff","x-envoy-upstream-service-time":"795","x-ratelimit-limit-requests":"10000","x-ratelimit-limit-tokens":"30000000","x-ratelimit-remaining-requests":"9999","x-ratelimit-remaining-tokens":"29999746","x-ratelimit-reset-requests":"6ms","x-ratelimit-reset-tokens":"0s","x-request-id":"req_83e1a82c6cf5465c1a081cd710277e1d"}}
[responses.js:208] 	OpenAI Responses API response: {"id":"resp_68ae2c0797c08191ab22276ded1571d9061d9d321cbfcb2e","object":"response","created_at":1756244999,"status":"completed","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":1024,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","output":[{"id":"fc_68ae2c084c9081918aec2fd382c3d6d3061d9d321cbfcb2e","type":"function_call","status":"completed","arguments":"{}","call_id":"call_UA11D4k1VhcCDmIE9cKxE1o6","name":"addNumbers"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":null,"name":"addNumbers","parameters":{"additionalProperties":false,"type":"object","properties":{},"required":[]},"strict":true}],"top_logprobs":0,"top_p":1,"truncation":"disabled","usage":{"input_tokens":37,"input_tokens_details":{"cached_tokens":0},"output_tokens":11,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":48},"user":null,"metadata":{}}
[evaluator.js:310] Provider response properties: output, tokenUsage, cached, cost, raw
[evaluator.js:311] Provider response cached property explicitly: true
[evaluator.js:330] Evaluator response = {"output":"","tokenUsage":{"cached":48,"total":48},"cached":true,"cost":0.00020250000000000002,"raw"...
[evaluator.js:331] Evaluator checking cached flag: response.cached = true, provider.delay = 0
[evaluator.js:337] Skipping delay because response is cached
[tokenUsage.js:39] Tracked token usage for openai:responses:gpt-4o (OpenAiResponsesProvider): total=48, cached=48
[signal.js:20] Writing to signal file /Users/collin/.promptfoo/evalLastWritten
[signal.js:22] Successfully wrote to signal file
[evaluator.js:1142] Eval #1 complete (1 of 1)
[signal.js:20] Writing to signal file /Users/collin/.promptfoo/evalLastWritten
[signal.js:22] Successfully wrote to signal file
[eval.js:308] At least one evaluation failure occurred. This might be caused by the underlying call to the provider, or a test failure. Context: 
[{"raw":"Please add the following numbers together: {{a}} and {{b}}","label":"Please add the following numbers together: {{a}} and {{b}}","id":"600fc45c7ae71955cf7be351bc650a4c6084fe37810cf2aa31600089a4d6e391","provider":"openai:responses:gpt-4o","metrics":{"score":0,"testPassCount":0,"testFailCount":1,"testErrorCount":0,"assertPassCount":0,"assertFailCount":1,"totalLatencyMs":5,"tokenUsage":{"prompt":0,"completion":0,"cached":48,"total":48,"numRequests":1,"completionDetails":{"reasoning":0,"acceptedPrediction":0,"rejectedPrediction":0},"assertions":{"total":0,"prompt":0,"completion":0,"cached":0,"numRequests":0,"completionDetails":{"reasoning":0,"acceptedPrediction":0,"rejectedPrediction":0}}},"namedScores":{},"namedScoresCount":{},"cost":0.00020250000000000002}}]
[index.js:312] ============================================================================================================================
[eval.js:329] ✔ Evaluation complete. ID: eval-Ezs-2025-08-26T22:54:01

[eval.js:330] » Run promptfoo view to use the local web viewer
[eval.js:335] » Do you want to share this with your team? Sign up for free at https://promptfoo.app
[eval.js:337] » This project needs your feedback. What's one thing we can improve? https://promptfoo.dev/feedback
[index.js:312] ============================================================================================================================
[eval.js:362] Token Usage Summary:
[eval.js:367] 
  Evaluation:
[eval.js:368]     Total: 48
[eval.js:369]     Prompt: 0
[eval.js:370]     Completion: 0
[eval.js:372]     Cached: 48
[eval.js:434] 
  Grand Total: 48 tokens
[index.js:312] ============================================================================================================================
[eval.js:437] Duration: 0s (concurrency: 4)
[eval.js:438] Successes: 0
[eval.js:439] Failures: 1
[eval.js:441] Errors: 0
[eval.js:444] Pass Rate: 0.00%
[index.js:312] ============================================================================================================================
```