Repository: promptfoo/promptfoo
Pull request #3421: feat: multimodal live function callbacks
Author: login: abrayne
Created: 2025-03-18T18:02:23Z | Merged: 2025-03-19T17:33:36Z

Add ability to use function callbacks in google multimodal live tests as in vertex provider.


Diff:
diff --git a/examples/google-multimodal-live/promptfooconfig.callback.js b/examples/google-multimodal-live/promptfooconfig.callback.js
new file mode 100644
index 000000000000..25a1fc6840cc
--- /dev/null
+++ b/examples/google-multimodal-live/promptfooconfig.callback.js
@@ -0,0 +1,61 @@
+module.exports = /** @type {import('promptfoo').TestSuiteConfig} */ ({
+  description: 'Function calling and callback execution demonstration',
+  prompts: [
+    'Please add the following numbers together: {{a}} and {{b}}',
+    'What is the sum of {{a}} and {{b}}?',
+  ],
+  providers: [
+    {
+      id: 'google:live:gemini-2.0-flash-exp',
+      config: {
+        generationConfig: { response_modalities: ['text'] },
+        timeoutMs: 10000,
+        tools: [
+          {
+            functionDeclarations: [
+              {
+                name: 'addNumbers',
+                description: 'Add two numbers together',
+                parameters: {
+                  type: 'object',
+                  properties: {
+                    a: { type: 'number' },
+                    b: { type: 'number' },
+                  },
+                  required: ['a', 'b'],
+                },
+              },
+            ],
+          },
+        ],
+        functionToolCallbacks: {
+          addNumbers: (parametersJsonString) => {
+            const { a, b } = JSON.parse(parametersJsonString);
+            return { sum: a + b };
+          },
+        },
+      },
+    },
+  ],
+  tests: [
+    {
+      vars: { a: 5, b: 6 },
+      assert: [
+        {
+          type: 'contains',
+          value: '11',
+          transform: 'JSON.parse(output).text',
+        },
+      ],
+    },
+    {
+      vars: { a: 10, b: 20 },
+      assert: [
+        {
+          type: 'javascript',
+          value: "JSON.parse(output).text.includes('30')",
+        },
+      ],
+    },
+  ],
+});
diff --git a/src/providers/googleMultimodalLive.ts b/src/providers/googleMultimodalLive.ts
index b566e20a2356..0b73ad5d8a99 100644
--- a/src/providers/googleMultimodalLive.ts
+++ b/src/providers/googleMultimodalLive.ts
@@ -115,6 +115,12 @@ interface CompletionOptions {
 
   tools?: Tool[];
 
+  /**
+   * If set, automatically call these functions when the assistant activates
+   * these function tools.
+   */
+  functionToolCallbacks?: Record<string, (arg: string) => Promise<any>>;
+
   systemInstruction?: Content;
 }
 
@@ -231,24 +237,46 @@ export class GoogleMMLiveProvider implements ApiProvider {
             response_text_total =
               response_text_total + response.serverContent.modelTurn.parts[0].text;
           } else if (response.toolCall?.functionCalls) {
-            const functionResponses: any[] = [];
             for (const functionCall of response.toolCall.functionCalls) {
               function_calls_total.push(functionCall);
               if (functionCall && functionCall.id && functionCall.name) {
-                functionResponses.push({
-                  id: functionCall.id,
-                  name: functionCall.name,
-                  // TODO: add mocking of function response here
-                  response: {},
-                });
+                let callbackResponse = {};
+
+                // Handle function tool callbacks
+                const functionName = functionCall.name;
+                if (
+                  this.config.functionToolCallbacks &&
+                  this.config.functionToolCallbacks[functionName]
+                ) {
+                  try {
+                    callbackResponse = await this.config.functionToolCallbacks[functionName](
+                      JSON.stringify(
+                        typeof functionCall.args === 'string'
+                          ? JSON.parse(functionCall.args)
+                          : functionCall.args,
+                      ),
+                    );
+                  } catch (error) {
+                    callbackResponse = {
+                      error: `Error executing function ${functionName}: ${error}`,
+                    };
+                    logger.error(`Error executing function ${functionName}: ${error}`);
+                  }
+                }
+
+                const toolMessage = {
+                  tool_response: {
+                    function_responses: {
+                      id: functionCall.id,
+                      name: functionName,
+                      response: callbackResponse,
+                    },
+                  },
+                };
+                logger.debug(`WebSocket sent: ${JSON.stringify(toolMessage)}`);
+                ws.send(JSON.stringify(toolMessage));
               }
             }
-            const toolMessage = {
-              tool_response: {
-                function_responses: functionResponses,
-              },
-            };
-            ws.send(JSON.stringify(toolMessage));
           } else if (response.serverContent?.turnComplete) {
             if (contentIndex < contents.length) {
               const contentMessage = formatContentMessage(contents, contentIndex);
diff --git a/test/providers/googleMultimodalLive.test.ts b/test/providers/googleMultimodalLive.test.ts
index 3f0ace2c65eb..b4c87cf26c2b 100644
--- a/test/providers/googleMultimodalLive.test.ts
+++ b/test/providers/googleMultimodalLive.test.ts
@@ -331,4 +331,146 @@ describe('GoogleMMLiveProvider', () => {
       process.env.GOOGLE_API_KEY = originalApiKey;
     }
   });
+
+  it('should handle function tool callbacks correctly', async () => {
+    jest.mocked(WebSocket).mockImplementation(() => {
+      setTimeout(() => {
+        mockWs.onopen?.({ type: 'open', target: mockWs } as WebSocket.Event);
+        simulateSetupMessage(mockWs);
+        simulatePartsMessage(mockWs, [
+          {
+            executableCode: {
+              language: 'PYTHON',
+              code: 'print(default_api.addNumbers(a=5, b=6))\n',
+            },
+          },
+        ]);
+        simulateFunctionCallMessage(mockWs, [
+          { name: 'addNumbers', args: { a: 5, b: 6 }, id: 'function-call-13767088400406609799' },
+        ]);
+        simulatePartsMessage(mockWs, [
+          { codeExecutionResult: { outcome: 'OUTCOME_OK', output: '{"sum": 11}\n' } },
+        ]);
+        simulateTextMessage(mockWs, 'The sum of 5 and 6 is 11.\n');
+        simulateCompletionMessage(mockWs);
+      }, 60);
+      return mockWs;
+    });
+
+    const mockAddNumbers = jest.fn().mockResolvedValue({ sum: 5 + 6 });
+
+    provider = new GoogleMMLiveProvider('gemini-2.0-flash-exp', {
+      config: {
+        generationConfig: {
+          response_modalities: ['text'],
+        },
+        timeoutMs: 500,
+        apiKey: 'test-api-key',
+        tools: [
+          {
+            functionDeclarations: [
+              {
+                name: 'addNumbers',
+                description: 'Add two numbers together',
+                parameters: {
+                  type: 'object',
+                  properties: {
+                    a: { type: 'number' },
+                    b: { type: 'number' },
+                  },
+                  required: ['a', 'b'],
+                },
+              },
+            ],
+          },
+        ],
+        functionToolCallbacks: {
+          addNumbers: mockAddNumbers,
+        },
+      },
+    });
+
+    const response = await provider.callApi('What is the sum of 5 and 6?');
+    expect(response).toEqual({
+      output: JSON.stringify({
+        text: 'The sum of 5 and 6 is 11.\n',
+        toolCall: {
+          functionCalls: [
+            { name: 'addNumbers', args: { a: 5, b: 6 }, id: 'function-call-13767088400406609799' },
+          ],
+        },
+      }),
+    });
+    expect(mockAddNumbers).toHaveBeenCalledTimes(1);
+    expect(mockAddNumbers).toHaveBeenCalledWith('{"a":5,"b":6}');
+  });
+
+  it('should handle errors in function tool callbacks', async () => {
+    jest.mocked(WebSocket).mockImplementation(() => {
+      setTimeout(() => {
+        mockWs.onopen?.({ type: 'open', target: mockWs } as WebSocket.Event);
+        simulateSetupMessage(mockWs);
+        simulatePartsMessage(mockWs, [
+          { executableCode: { language: 'PYTHON', code: 'print(default_api.errorFunction())\n' } },
+        ]);
+        simulateFunctionCallMessage(mockWs, [
+          { name: 'errorFunction', args: {}, id: 'function-call-7580472343952164416' },
+        ]);
+        simulatePartsMessage(mockWs, [
+          {
+            codeExecutionResult: {
+              outcome: 'OUTCOME_OK',
+              output: "{'error': 'Error executing function errorFunction: Error: Test error'}\n",
+            },
+          },
+        ]);
+        simulateTextMessage(
+          mockWs,
+          'The function `errorFunction` has been called and it returned an error as expected.',
+        );
+        simulateCompletionMessage(mockWs);
+      }, 60);
+      return mockWs;
+    });
+    provider = new GoogleMMLiveProvider('gemini-2.0-flash-exp', {
+      config: {
+        generationConfig: {
+          response_modalities: ['text'],
+        },
+        timeoutMs: 500,
+        apiKey: 'test-api-key',
+        tools: [
+          {
+            functionDeclarations: [
+              {
+                name: 'errorFunction',
+                description: 'A function that always throws an error',
+                parameters: {
+                  type: 'OBJECT',
+                  properties: {},
+                },
+              },
+            ],
+          },
+        ],
+        functionToolCallbacks: {
+          errorFunction: () => {
+            throw new Error('Test error');
+          },
+        },
+      },
+    });
+
+    const response = await provider.callApi('Call the error function');
+    expect(response).toEqual({
+      output: JSON.stringify({
+        text: 'The function `errorFunction` has been called and it returned an error as expected.',
+        toolCall: {
+          functionCalls: [
+            { name: 'errorFunction', args: {}, id: 'function-call-7580472343952164416' },
+          ],
+        },
+      }),
+    });
+  });
 });


Reviews:
user: login: ellipsis-dev[bot]
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T18:06:31Z
user: login: abrayne
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T18:09:38Z
user: login: abrayne
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T18:10:01Z
user: login: mldangelo
body: This is great! Thanks so much @abrayne! 
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-19T17:33:14Z

Review comments:
diff_hunk: @@ -115,6 +115,12 @@ interface CompletionOptions {
 
   tools?: Tool[];
 
+  /**
+   * If set, automatically call these functions when the assistant activates
+   * these function tools.
+   */
+  functionToolCallbacks?: Record<string, (arg: string) => Promise<string>>;
path: src/providers/googleMultimodalLive.ts
user: login: ellipsis-dev[bot]
body: The type for `functionToolCallbacks` is declared as returning a `Promise<string>`, but the callbacks (e.g., in the demo config and tests) return objects. Consider updating the type (e.g., to `Promise<any>` or `Promise<unknown>`) so that it accurately reflects the return value.
```suggestion
  functionToolCallbacks?: Record<string, (arg: string) => Promise<any>>;
```
created_at: 2025-03-18T18:06:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -331,4 +331,141 @@ describe('GoogleMMLiveProvider', () => {
       process.env.GOOGLE_API_KEY = originalApiKey;
     }
   });
+
+  it('should handle function tool callbacks correctly', async () => {
+    jest.mocked(WebSocket).mockImplementation(() => {
+      setTimeout(() => {
+        mockWs.onopen?.({ type: 'open', target: mockWs } as WebSocket.Event);
+        simulateSetupMessage(mockWs);
+        simulatePartsMessage(mockWs, [
+          {
+            executableCode: {
+              language: 'PYTHON',
+              code: 'print(default_api.addNumbers(a=5, b=6))\n',
+            },
+          },
+        ]);
+        simulateFunctionCallMessage(mockWs, [
+          { name: 'addNumbers', args: { a: 5, b: 6 }, id: 'function-call-13767088400406609799' },
+        ]);
+        simulatePartsMessage(mockWs, [
+          { codeExecutionResult: { outcome: 'OUTCOME_OK', output: '{"sum": 11}\n' } },
+        ]);
+        simulateTextMessage(mockWs, 'The sum of 5 and 6 is 11.\n');
+        simulateCompletionMessage(mockWs);
+      }, 60);
+      return mockWs;
+    });
+
+    const mockAddNumbers = jest.fn().mockResolvedValue({ sum: 5 + 6 });
+
+    provider = new GoogleMMLiveProvider('gemini-2.0-flash-exp', {
+      config: {
+        generationConfig: {
+          response_modalities: ['text'],
+        },
+        timeoutMs: 500,
+        apiKey: 'test-api-key',
+        tools: [
+          {
+            functionDeclarations: [
+              {
+                name: 'addNumbers',
+                description: 'Add two numbers together',
+                parameters: {
+                  type: 'object',
+                  properties: {
+                    a: { type: 'number' },
+                    b: { type: 'number' },
+                  },
+                  required: ['a', 'b'],
+                },
+              },
+            ],
+          },
+        ],
+        functionToolCallbacks: {
+          addNumbers: mockAddNumbers,
+        },
+      },
+    });
+
+    const response = await provider.callApi('What is the sum of 5 and 6?');
+    expect(response).toEqual({
+      output: JSON.stringify({
+        text: 'The sum of 5 and 6 is 11.\n',
+        toolCall: {
+          functionCalls: [
+            { name: 'addNumbers', args: { a: 5, b: 6 }, id: 'function-call-13767088400406609799' },
+          ],
+        },
+      }),
+    });
+    expect(mockAddNumbers).toHaveBeenCalledTimes(1);
+    expect(mockAddNumbers).toHaveBeenCalledWith('{"a":5,"b":6}');
+  });
+
+  it('should handle errors in function tool callbacks', async () => {
+    jest.mocked(WebSocket).mockImplementation(() => {
+      setTimeout(() => {
+        mockWs.onopen?.({ type: 'open', target: mockWs } as WebSocket.Event);
+        simulateSetupMessage(mockWs);
+        simulatePartsMessage(mockWs, [
+          { executableCode: { language: 'PYTHON', code: 'print(default_api.errorFunction())\n' } },
+        ]);
+        simulateFunctionCallMessage(mockWs, [
+          { name: 'errorFunction', args: {}, id: 'function-call-7580472343952164416' },
+        ]);
+        simulatePartsMessage(mockWs, [
+          {
+            codeExecutionResult: {
+              outcome: 'OUTCOME_OK',
+              output: "{'error': 'Error executing function errorFunction: Error: Test error'}\n",
+            },
+          },
+        ]);
+        simulateTextMessage(
+          mockWs,
+          'The function `errorFunction` has been called and it returned an error as expected.',
+        );
+        simulateCompletionMessage(mockWs);
+      }, 60);
+      return mockWs;
+    });
+    provider = new GoogleMMLiveProvider('gemini-2.0-flash-exp', {
+      config: {
+        tools: [
+          {
+            functionDeclarations: [
+              {
+                name: 'errorFunction',
+                description: 'A function that always throws an error',
+                parameters: {
+                  type: 'OBJECT',
path: test/providers/googleMultimodalLive.test.ts
user: login: ellipsis-dev[bot]
body: Typo: In the function declaration for `errorFunction`, the parameter type is set to `OBJECT` (uppercase). To maintain consistency with other function declarations (e.g., `addNumbers` uses `object` in lowercase), please change `OBJECT` to `object`.
```suggestion
                  type: 'object',
```
created_at: 2025-03-18T18:06:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -331,4 +331,141 @@ describe('GoogleMMLiveProvider', () => {
       process.env.GOOGLE_API_KEY = originalApiKey;
     }
   });
+
+  it('should handle function tool callbacks correctly', async () => {
+    jest.mocked(WebSocket).mockImplementation(() => {
+      setTimeout(() => {
+        mockWs.onopen?.({ type: 'open', target: mockWs } as WebSocket.Event);
+        simulateSetupMessage(mockWs);
+        simulatePartsMessage(mockWs, [
+          {
+            executableCode: {
+              language: 'PYTHON',
+              code: 'print(default_api.addNumbers(a=5, b=6))\n',
+            },
+          },
+        ]);
+        simulateFunctionCallMessage(mockWs, [
+          { name: 'addNumbers', args: { a: 5, b: 6 }, id: 'function-call-13767088400406609799' },
+        ]);
+        simulatePartsMessage(mockWs, [
+          { codeExecutionResult: { outcome: 'OUTCOME_OK', output: '{"sum": 11}\n' } },
+        ]);
+        simulateTextMessage(mockWs, 'The sum of 5 and 6 is 11.\n');
+        simulateCompletionMessage(mockWs);
+      }, 60);
+      return mockWs;
+    });
+
+    const mockAddNumbers = jest.fn().mockResolvedValue({ sum: 5 + 6 });
+
+    provider = new GoogleMMLiveProvider('gemini-2.0-flash-exp', {
+      config: {
+        generationConfig: {
+          response_modalities: ['text'],
+        },
+        timeoutMs: 500,
+        apiKey: 'test-api-key',
+        tools: [
+          {
+            functionDeclarations: [
+              {
+                name: 'addNumbers',
+                description: 'Add two numbers together',
+                parameters: {
+                  type: 'object',
+                  properties: {
+                    a: { type: 'number' },
+                    b: { type: 'number' },
+                  },
+                  required: ['a', 'b'],
+                },
+              },
+            ],
+          },
+        ],
+        functionToolCallbacks: {
+          addNumbers: mockAddNumbers,
+        },
+      },
+    });
+
+    const response = await provider.callApi('What is the sum of 5 and 6?');
+    expect(response).toEqual({
+      output: JSON.stringify({
+        text: 'The sum of 5 and 6 is 11.\n',
+        toolCall: {
+          functionCalls: [
+            { name: 'addNumbers', args: { a: 5, b: 6 }, id: 'function-call-13767088400406609799' },
+          ],
+        },
+      }),
+    });
+    expect(mockAddNumbers).toHaveBeenCalledTimes(1);
+    expect(mockAddNumbers).toHaveBeenCalledWith('{"a":5,"b":6}');
+  });
+
+  it('should handle errors in function tool callbacks', async () => {
+    jest.mocked(WebSocket).mockImplementation(() => {
+      setTimeout(() => {
+        mockWs.onopen?.({ type: 'open', target: mockWs } as WebSocket.Event);
+        simulateSetupMessage(mockWs);
+        simulatePartsMessage(mockWs, [
+          { executableCode: { language: 'PYTHON', code: 'print(default_api.errorFunction())\n' } },
+        ]);
+        simulateFunctionCallMessage(mockWs, [
+          { name: 'errorFunction', args: {}, id: 'function-call-7580472343952164416' },
+        ]);
+        simulatePartsMessage(mockWs, [
+          {
+            codeExecutionResult: {
+              outcome: 'OUTCOME_OK',
+              output: "{'error': 'Error executing function errorFunction: Error: Test error'}\n",
+            },
+          },
+        ]);
+        simulateTextMessage(
+          mockWs,
+          'The function `errorFunction` has been called and it returned an error as expected.',
+        );
+        simulateCompletionMessage(mockWs);
+      }, 60);
+      return mockWs;
+    });
+    provider = new GoogleMMLiveProvider('gemini-2.0-flash-exp', {
+      config: {
+        tools: [
+          {
+            functionDeclarations: [
+              {
+                name: 'errorFunction',
+                description: 'A function that always throws an error',
+                parameters: {
+                  type: 'OBJECT',
path: test/providers/googleMultimodalLive.test.ts
user: login: abrayne
body: The schema copied from vertex provider uses upper case, so I will leave it as that.
created_at: 2025-03-18T18:09:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -115,6 +115,12 @@ interface CompletionOptions {
 
   tools?: Tool[];
 
+  /**
+   * If set, automatically call these functions when the assistant activates
+   * these function tools.
+   */
+  functionToolCallbacks?: Record<string, (arg: string) => Promise<string>>;
path: src/providers/googleMultimodalLive.ts
user: login: abrayne
body: Done!
created_at: 2025-03-18T18:10:01Z
author_association: CONTRIBUTOR

PR comments:
user: login: gru-agent[bot]
created_at: 2025-03-18T18:03:28Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/43a82877-31b5-44a8-b081-babadf9357c5) | 811c05fddd128239f38ca37370625f9fd68c4f48 | 🚫 Skipped | No files need to be tested {"examples/google-multimodal-live/promptfooconfig.callback.js":"target file(examples/google-multimodal-live/promptfooconfig.callback.js) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","src/providers/googleMultimodalLive.ts":"target file(src/providers/googleMultimodalLive.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js","test/providers/googleMultimodalLive.test.ts":"target file(test/providers/googleMultimodalLive.test.ts) not in work scope \n include: src/app/src/**/*.ts \n exclude: node_modules,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.d.ts,**/*.test.js,**/*.spec.js"} |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR
user: login: gru-agent[bot]
created_at: 2025-03-18T18:03:34Z
body: ## TestGru Assignment

### Summary

| Link | CommitId | Status | Reason |
| ---- | -------- | ------ | ------ |
| [Detail](https://gru.ai/:test/promptfoo@github/promptfoo/e35338e3-246f-4c26-82ff-dd364b9314cd) | 811c05fddd128239f38ca37370625f9fd68c4f48 | 🚫 Skipped |  |

### Files

| File | Pull Request |
| ---- | ------------ |
| src/providers/googleMultimodalLive.ts | 🚫 Skipped (There's no need to update the test code) |

> [!TIP]
> You can `@gru-agent` and leave your feedback. TestGru will make adjustments based on your input

author_association: CONTRIBUTOR
user: login: abrayne
created_at: 2025-03-19T11:39:18Z
body: @mldangelo this should be good to go when you are happy with it.
author_association: CONTRIBUTOR