Repository: lzell/AIProxySwift
Pull request #148: Add support for OpenAI Realtime API tool calls
Author: login: timmywheels
Created: 2025-04-24T12:13:09Z | Merged: 2025-04-25T03:17:13Z

This change adds support for OpenAI Realtime tool calls.

I've never authored any Swift code before, so would love some feedback @lzell !

**Note**
I noticed in the OpenAI [docs](https://platform.openai.com/docs/api-reference/realtime-server-events/response/function_call_arguments/done), the `name` (i.e. the tool name) property of the `response.function_call_arguments.done` event is not present, however in the OpenAI playground the `name` is part of the payload, see screenshot below.

![CleanShot 2025-04-24 at 08 14 34@2x](https://github.com/user-attachments/assets/85a753e2-21f9-4a87-a898-b7ca8625eba8)



Diff:
diff --git a/Sources/AIProxy/OpenAI/OpenAIRealtimeConversationItemCreate.swift b/Sources/AIProxy/OpenAI/OpenAIRealtimeConversationItemCreate.swift
index 368bcfd4..1d070c85 100644
--- a/Sources/AIProxy/OpenAI/OpenAIRealtimeConversationItemCreate.swift
+++ b/Sources/AIProxy/OpenAI/OpenAIRealtimeConversationItemCreate.swift
@@ -7,24 +7,38 @@
 
 import Foundation
 
+/// https://platform.openai.com/docs/api-reference/realtime-client-events/conversation/item/create
 public struct OpenAIRealtimeConversationItemCreate: Encodable {
     public let type = "conversation.item.create"
     public let item: Item
+
+    public init(item: Item) {
+        self.item = item
+    }
 }
 
 // MARK: -
 public extension OpenAIRealtimeConversationItemCreate {
     struct Item: Encodable {
-        let type = "message"
-        let role: String
-        let content: [Content]
+        public let type = "message"
+        public let role: String
+        public let content: [Content]
+
+        public init(role: String, text: String) {
+            self.role = role
+            self.content = [.init(text: text)]
+        }
     }
 }
 
 // MARK: -
 public extension OpenAIRealtimeConversationItemCreate.Item {
     struct Content: Encodable {
-        let type = "input_text"
-        let text: String
+        public let type = "input_text"
+        public let text: String
+
+        public init(text: String) {
+            self.text = text
+        }
     }
 }
diff --git a/Sources/AIProxy/OpenAI/OpenAIRealtimeMessage.swift b/Sources/AIProxy/OpenAI/OpenAIRealtimeMessage.swift
index 7c119d9c..a2c15425 100644
--- a/Sources/AIProxy/OpenAI/OpenAIRealtimeMessage.swift
+++ b/Sources/AIProxy/OpenAI/OpenAIRealtimeMessage.swift
@@ -12,4 +12,5 @@ public enum OpenAIRealtimeMessage {
     case responseCreated // "response.created"
     case responseAudioDelta(String) // "response.audio.delta"
     case inputAudioBufferSpeechStarted // "input_audio_buffer.speech_started"
+    case responseFunctionCallArgumentsDone(String, String) // "response.function_call_arguments.done"
 }
diff --git a/Sources/AIProxy/OpenAI/OpenAIRealtimeResponseCreate.swift b/Sources/AIProxy/OpenAI/OpenAIRealtimeResponseCreate.swift
index d8c32497..84779de8 100644
--- a/Sources/AIProxy/OpenAI/OpenAIRealtimeResponseCreate.swift
+++ b/Sources/AIProxy/OpenAI/OpenAIRealtimeResponseCreate.swift
@@ -7,6 +7,7 @@
 
 import Foundation
 
+/// https://platform.openai.com/docs/api-reference/realtime-client-events/response
 public struct OpenAIRealtimeResponseCreate: Encodable {
     public let type = "response.create"
     public let response: Response?
@@ -18,17 +19,36 @@ public struct OpenAIRealtimeResponseCreate: Encodable {
 
 // MARK: -
 extension OpenAIRealtimeResponseCreate {
-
     public struct Response: Encodable {
         public let instructions: String?
         public let modalities: [String]?
+        public let tools: [Tool]?
 
         public init(
             instructions: String? = nil,
-            modalities: [String]? = nil
+            modalities: [String]? = nil,
+            tools: [Tool]? = nil
         ) {
-            self.modalities = modalities
             self.instructions = instructions
+            self.modalities = modalities
+            self.tools = tools
+        }
+    }
+}
+
+// MARK: -
+extension OpenAIRealtimeResponseCreate.Response {
+    public struct Tool: Encodable {
+        public let name: String
+        public let description: String
+        public let parameters: [String: AIProxyJSONValue]
+        public let type = "function"
+
+        public init(name: String, description: String, parameters: [String: AIProxyJSONValue]) {
+            self.name = name
+            self.description = description
+            self.parameters = parameters
+            
         }
     }
 }
diff --git a/Sources/AIProxy/OpenAI/OpenAIRealtimeResponseFunctionCallArgumentsDone.swift b/Sources/AIProxy/OpenAI/OpenAIRealtimeResponseFunctionCallArgumentsDone.swift
new file mode 100644
index 00000000..ba74be53
--- /dev/null
+++ b/Sources/AIProxy/OpenAI/OpenAIRealtimeResponseFunctionCallArgumentsDone.swift
@@ -0,0 +1,22 @@
+//
+//  OpenAIRealtimeResponseDone.swift
+//  AIProxy
+//
+//  Created by Tim Wheeler on 4/22/25.
+//
+
+import Foundation
+
+/// Returned when the model-generated function call arguments are done streaming.
+/// Also emitted when a Response is interrupted, incomplete, or cancelled.
+/// https://platform.openai.com/docs/api-reference/realtime-server-events/response/function_call_arguments/done
+public struct OpenAIRealtimeResponseFunctionCallArgumentsDone: Encodable {
+    public let type = "response.function_call_arguments.done"
+    public let name: String?
+    public let arguments: String?
+
+    public init(name: String, arguments: String) {
+        self.name = name
+        self.arguments = arguments
+    }
+}
diff --git a/Sources/AIProxy/OpenAI/OpenAIRealtimeSession.swift b/Sources/AIProxy/OpenAI/OpenAIRealtimeSession.swift
index 22e80a2d..fff2a433 100644
--- a/Sources/AIProxy/OpenAI/OpenAIRealtimeSession.swift
+++ b/Sources/AIProxy/OpenAI/OpenAIRealtimeSession.swift
@@ -141,6 +141,11 @@ open class OpenAIRealtimeSession {
             self.continuation?.yield(.responseCreated)
         case "input_audio_buffer.speech_started":
             self.continuation?.yield(.inputAudioBufferSpeechStarted)
+        case "response.function_call_arguments.done":
+            if let name = json["name"] as? String,
+               let arguments = json["arguments"] as? String {
+                self.continuation?.yield(.responseFunctionCallArgumentsDone(name, arguments))
+            }
         default:
             break
         }
diff --git a/Sources/AIProxy/OpenAI/OpenAIRealtimeSessionConfiguration.swift b/Sources/AIProxy/OpenAI/OpenAIRealtimeSessionConfiguration.swift
index acefd3c5..4eda25b5 100644
--- a/Sources/AIProxy/OpenAI/OpenAIRealtimeSessionConfiguration.swift
+++ b/Sources/AIProxy/OpenAI/OpenAIRealtimeSessionConfiguration.swift
@@ -9,6 +9,54 @@
 /// https://platform.openai.com/docs/api-reference/realtime-client-events/session/update#realtime-client-events/session/update-session
 public struct OpenAIRealtimeSessionConfiguration: Encodable {
 
+    public enum ToolChoice: Encodable {
+
+        /// The model will not call any tool and instead generates a message.
+        /// This is the default when no tools are present in the request body
+        case none
+
+        /// The model can pick between generating a message or calling one or more tools.
+        /// This is the default when tools are present in the request body
+        case auto
+
+        /// The model must call one or more tools
+        case required
+
+        /// Forces the model to call a specific tool
+        case specific(functionName: String)
+
+        private enum RootKey: CodingKey {
+            case type
+            case function
+        }
+
+        private enum FunctionKey: CodingKey {
+            case name
+        }
+
+        public func encode(to encoder: any Encoder) throws {
+            switch self {
+            case .none:
+                var container = encoder.singleValueContainer()
+                try container.encode("none")
+            case .auto:
+                var container = encoder.singleValueContainer()
+                try container.encode("auto")
+            case .required:
+                var container = encoder.singleValueContainer()
+                try container.encode("required")
+            case .specific(let functionName):
+                var container = encoder.container(keyedBy: RootKey.self)
+                try container.encode("function", forKey: .type)
+                var functionContainer = container.nestedContainer(
+                    keyedBy: FunctionKey.self,
+                    forKey: .function
+                )
+                try functionContainer.encode(functionName, forKey: .name)
+            }
+        }
+    }
+    
     /// The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`.
     public let inputAudioFormat: AudioFormat?
 
@@ -44,13 +92,11 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable {
     /// Sampling temperature for the model.
     public let temperature: Double?
 
-    /// Tools are not yet implemented.
     /// Tools (functions) available to the model.
-    /// public let tools: [Tool]?
+    public let tools: [Tool]?
 
-    /// Tools are not yet implemented.
     /// How the model chooses tools. Options are "auto", "none", "required", or specify a function.
-    /// public let toolChoice: ToolChoice?
+    public let toolChoice: ToolChoice?
 
     /// Configuration for turn detection. Set to nil to turn off.
     public let turnDetection: TurnDetection?
@@ -67,8 +113,8 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable {
         case modalities
         case outputAudioFormat = "output_audio_format"
         case temperature
-        // case tools
-        // case toolChoice = "tool_choice"
+        case tools
+        case toolChoice = "tool_choice"
         case turnDetection = "turn_detection"
         case voice
     }
@@ -81,6 +127,8 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable {
         modalities: [OpenAIRealtimeSessionConfiguration.Modality]? = nil,
         outputAudioFormat: OpenAIRealtimeSessionConfiguration.AudioFormat? = nil,
         temperature: Double? = nil,
+        tools: [OpenAIRealtimeSessionConfiguration.Tool]? = nil,
+        toolChoice: OpenAIRealtimeSessionConfiguration.ToolChoice? = nil,
         turnDetection: OpenAIRealtimeSessionConfiguration.TurnDetection? = nil,
         voice: String? = nil
     ) {
@@ -91,6 +139,8 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable {
         self.modalities = modalities
         self.outputAudioFormat = outputAudioFormat
         self.temperature = temperature
+        self.tools = tools
+        self.toolChoice = toolChoice
         self.turnDetection = turnDetection
         self.voice = voice
     }
@@ -129,15 +179,22 @@ extension OpenAIRealtimeSessionConfiguration {
 extension OpenAIRealtimeSessionConfiguration {
     public struct Tool: Encodable {
         /// The description of the function
-        let description: String
+        public let description: String
 
         /// The name of the function
-        let name: String
+        public let name: String
 
-        let parameters: [String: AIProxyJSONValue]
+        /// The function parameters
+        public let parameters: [String: AIProxyJSONValue]
 
         /// The type of the tool, e.g., "function".
-        let type: String
+        public let type = "function"
+        
+        public init(name: String, description: String, parameters: [String: AIProxyJSONValue]) {
+            self.name = name
+            self.description = description
+            self.parameters = parameters
+        }
     }
 }
 


Reviews:
user: login: timmywheels
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T12:20:11Z
user: login: lzell
body: Thank you @timmywheels! I have a few comments, but I bet we can get this merged in relatively easily
state: COMMENTED
author_association: OWNER
submitted_at: 2025-04-24T16:36:34Z
user: login: timmywheels
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T02:45:37Z
user: login: lzell
state: APPROVED
author_association: OWNER
submitted_at: 2025-04-25T03:17:00Z

Review comments:
diff_hunk: @@ -0,0 +1,34 @@
+//
+//  OpenAIRealtimeResponseDone.swift
+//  AIProxy
+//
+//  Created by Tim Wheeler on 4/22/25.
+//
+
+import Foundation
+
+public struct OpenAIRealtimeResponseFunctionCallArgumentsDone: Encodable {
+    public let type = "response.function_call_arguments.done"
+    public let response: Response?
+
+    public init(response: Response? = nil) {
+        self.response = response
+    }
+}
+
+// MARK: -
+extension OpenAIRealtimeResponseFunctionCallArgumentsDone {
+    
+    public struct Response: Encodable {
+        public let name: String?
+        public let arguments: String?
path: Sources/AIProxy/OpenAI/OpenAIRealtimeResponseFunctionCallArgumentsDone.swift
user: login: timmywheels
body: I didn't include all property payloads such as `eventId` etc., which follows existing patterns I've seen in this codebase.
created_at: 2025-04-24T12:20:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -18,17 +18,33 @@ public struct OpenAIRealtimeResponseCreate: Encodable {
 
 // MARK: -
 extension OpenAIRealtimeResponseCreate {
+    
+    public struct Tool: Encodable {
path: Sources/AIProxy/OpenAI/OpenAIRealtimeResponseCreate.swift
user: login: lzell
body: A few things here:
1. All the properties should be public
2. Don't we need a public initializer on this?
3. Stylistically, the way I would structure this file is like this:

```
public struct OpenAIRealtimeResponseCreate: Encodable {
    public let response: Response?
    <snip>
}

extension OpenAIRealtimeResponseCreate {
   public struct Response: Encodable {
      public let tools: [Tool]
      <snip>
   }
}

extension OpenAIRealtimeResponseCreate.Response {
   public struct Tool {
      <snip>
   }
}
```      

Finally, can you fix my omission and add the following (it helps when trying to quickly see what our properties should look like): 

```
/// https://platform.openai.com/docs/api-reference/realtime-client-events/response
public struct OpenAIRealtimeResponseCreate: Encodable
```
created_at: 2025-04-24T16:21:17Z
author_association: OWNER
diff_hunk: @@ -0,0 +1,34 @@
+//
+//  OpenAIRealtimeResponseDone.swift
+//  AIProxy
+//
+//  Created by Tim Wheeler on 4/22/25.
+//
+
+import Foundation
+
path: Sources/AIProxy/OpenAI/OpenAIRealtimeResponseFunctionCallArgumentsDone.swift
user: login: lzell
body: Can you include this docstring above the struct:
```
/// Returned when the model-generated function call arguments are done streaming.
/// Also emitted when a Response is interrupted, incomplete, or cancelled.
/// https://platform.openai.com/docs/api-reference/realtime-server-events/response/function_call_arguments/done
```
created_at: 2025-04-24T16:23:34Z
author_association: OWNER
diff_hunk: @@ -0,0 +1,34 @@
+//
+//  OpenAIRealtimeResponseDone.swift
+//  AIProxy
+//
+//  Created by Tim Wheeler on 4/22/25.
+//
+
+import Foundation
+
+public struct OpenAIRealtimeResponseFunctionCallArgumentsDone: Encodable {
+    public let type = "response.function_call_arguments.done"
+    public let response: Response?
path: Sources/AIProxy/OpenAI/OpenAIRealtimeResponseFunctionCallArgumentsDone.swift
user: login: lzell
body: This looks like the wrong internal structure to me. Seems like you want to have `arguments` here instead of one level of indirection away (currently you have it in the nested `Response` struct)
![Screenshot 2025-04-24 at 09 27 02](https://github.com/user-attachments/assets/d791d9e1-814c-412c-8212-757def99e564)

created_at: 2025-04-24T16:29:08Z
author_association: OWNER
diff_hunk: @@ -141,6 +141,11 @@ open class OpenAIRealtimeSession {
             self.continuation?.yield(.responseCreated)
         case "input_audio_buffer.speech_started":
             self.continuation?.yield(.inputAudioBufferSpeechStarted)
+        case "response.function_call_arguments.done":
+            if let name = json["name"] as? String,
path: Sources/AIProxy/OpenAI/OpenAIRealtimeSession.swift
user: login: lzell
body: Perhaps there is an error in OpenAI's documentation. Are you seeing this branch be entered? I don't see a `name` field on `response.function_call_arguments.done`: 
https://platform.openai.com/docs/api-reference/realtime-server-events/response/function_call_arguments/done
created_at: 2025-04-24T16:30:38Z
author_association: OWNER
diff_hunk: @@ -141,6 +191,15 @@ extension OpenAIRealtimeSessionConfiguration {
     }
 }
 
+extension OpenAIRealtimeSessionConfiguration.Tool {
+    public init(name: String, description: String, parameters: [String: AIProxyJSONValue]) {
path: Sources/AIProxy/OpenAI/OpenAIRealtimeSessionConfiguration.swift
user: login: lzell
body: You can pull this initializer into the definition of `public struct Tool`, line 190
created_at: 2025-04-24T16:34:05Z
author_association: OWNER
diff_hunk: @@ -141,6 +191,15 @@ extension OpenAIRealtimeSessionConfiguration {
     }
 }
 
+extension OpenAIRealtimeSessionConfiguration.Tool {
+    public init(name: String, description: String, parameters: [String: AIProxyJSONValue]) {
path: Sources/AIProxy/OpenAI/OpenAIRealtimeSessionConfiguration.swift
user: login: lzell
body: And while you are in there, can you make the properties on `Tool` all public. I'm trying to remember why I didn't make any of these properties public, but right now that seems like a bug!

created_at: 2025-04-24T16:35:03Z
author_association: OWNER
diff_hunk: @@ -141,6 +141,11 @@ open class OpenAIRealtimeSession {
             self.continuation?.yield(.responseCreated)
         case "input_audio_buffer.speech_started":
             self.continuation?.yield(.inputAudioBufferSpeechStarted)
+        case "response.function_call_arguments.done":
+            if let name = json["name"] as? String,
path: Sources/AIProxy/OpenAI/OpenAIRealtimeSession.swift
user: login: timmywheels
body: Yeah I noticed their docs are missing the `name` field. In the OpenAI Playground I was able to confirm it's part of the payload. That's what I'm using in my application to key on which function I should to invoke.

![CleanShot 2025-04-24 at 22 42 48@2x](https://github.com/user-attachments/assets/d032e123-6a2e-4130-a877-7982e1811e0a)

created_at: 2025-04-25T02:45:37Z
author_association: CONTRIBUTOR

PR comments:
user: login: timmywheels
created_at: 2025-04-24T19:44:26Z
body: Thanks @lzell will respond + update PR later today 🙏 
author_association: CONTRIBUTOR
user: login: lzell
created_at: 2025-04-25T03:33:20Z
body: Thank you Tim! This is now available in 0.96.0: https://github.com/lzell/AIProxySwift/releases/tag/0.96.0

@timmywheels 
author_association: OWNER