Repository: sashabaranov/go-openai
Pull request #598 (2023-11-18 23:31:29)

Add completion-with-tool example
**Describe the change**
Add a completion-with-tool example showing how to correctly utilize the function calling capability of GPT4

**Describe your solution**
This is based on the existing completion example and uses the same get_current_weather() python function that the OpenAI documentation includes.

**Tests**
Manually run the new example.

**Additional context**
As a user of this go SDK it was not immediately intuitive to me how to correctly utilize the function calling capability of GPT4 (https://platform.openai.com/docs/guides/function-calling). While the aformentioned link provides a helpful example written in python, I initially tripped over how to correctly translate the specification of function arguments when using this go SDK.


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/sashabaranov/go-openai/pull/598?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Comparison is base [(`a130cfe`)](https://app.codecov.io/gh/sashabaranov/go-openai/commit/a130cfee26427b99ae0bf957be74e32ca8a7f567?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) 98.38% compared to head [(`8ea0d17`)](https://app.codecov.io/gh/sashabaranov/go-openai/pull/598?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) 98.38%.


<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##           master     #598   +/-   ##
=======================================
  Coverage   98.38%   98.38%           
=======================================
  Files          24       24           
  Lines        1300     1300           
=======================================
  Hits         1279     1279           
  Misses         15       15           
  Partials        6        6           
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/sashabaranov/go-openai/pull/598?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None).


Patch:
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 44bf697..222c065 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -8,11 +8,14 @@ Thanks for submitting a pull request! Please provide enough information so that
 **Describe the change**
 Please provide a clear and concise description of the changes you're proposing. Explain what problem it solves or what feature it adds.
 
+**Provide OpenAI documentation link**
+Provide a relevant API doc from https://platform.openai.com/docs/api-reference
+
 **Describe your solution**
 Describe how your changes address the problem or how they add the feature. This should include a brief description of your approach and any new libraries or dependencies you're using.
 
 **Tests**
-Briefly describe how you have tested these changes. 
+Briefly describe how you have tested these changes. If possible — please add integration tests.
 
 **Additional context**
 Add any other context or screenshots or logs about your pull request here. If the pull request relates to an open issue, please link to it.
diff --git a/chat.go b/chat.go
index ebdc0e2..5b87b6b 100644
--- a/chat.go
+++ b/chat.go
@@ -2,6 +2,7 @@ package openai
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"net/http"
 )
@@ -20,6 +21,7 @@ const chatCompletionsSuffix = "/chat/completions"
 var (
 	ErrChatCompletionInvalidModel       = errors.New("this model is not supported with this method, please use CreateCompletion client method instead") //nolint:lll
 	ErrChatCompletionStreamNotSupported = errors.New("streaming is not supported with this method, please use CreateChatCompletionStream")              //nolint:lll
+	ErrContentFieldsMisused             = errors.New("can't use both Content and MultiContent properties simultaneously")
 )
 
 type Hate struct {
@@ -51,9 +53,36 @@ type PromptAnnotation struct {
 	ContentFilterResults ContentFilterResults `json:"content_filter_results,omitempty"`
 }
 
+type ImageURLDetail string
+
+const (
+	ImageURLDetailHigh ImageURLDetail = "high"
+	ImageURLDetailLow  ImageURLDetail = "low"
+	ImageURLDetailAuto ImageURLDetail = "auto"
+)
+
+type ChatMessageImageURL struct {
+	URL    string         `json:"url,omitempty"`
+	Detail ImageURLDetail `json:"detail,omitempty"`
+}
+
+type ChatMessagePartType string
+
+const (
+	ChatMessagePartTypeText     ChatMessagePartType = "text"
+	ChatMessagePartTypeImageURL ChatMessagePartType = "image_url"
+)
+
+type ChatMessagePart struct {
+	Type     ChatMessagePartType  `json:"type,omitempty"`
+	Text     string               `json:"text,omitempty"`
+	ImageURL *ChatMessageImageURL `json:"image_url,omitempty"`
+}
+
 type ChatCompletionMessage struct {
-	Role    string `json:"role"`
-	Content string `json:"content"`
+	Role         string `json:"role"`
+	Content      string `json:"content"`
+	MultiContent []ChatMessagePart
 
 	// This property isn't in the official documentation, but it's in
 	// the documentation for the official library for python:
@@ -70,6 +99,64 @@ type ChatCompletionMessage struct {
 	ToolCallID string `json:"tool_call_id,omitempty"`
 }
 
+func (m ChatCompletionMessage) MarshalJSON() ([]byte, error) {
+	if m.Content != "" && m.MultiContent != nil {
+		return nil, ErrContentFieldsMisused
+	}
+	if len(m.MultiContent) > 0 {
+		msg := struct {
+			Role         string            `json:"role"`
+			Content      string            `json:"-"`
+			MultiContent []ChatMessagePart `json:"content,omitempty"`
+			Name         string            `json:"name,omitempty"`
+			FunctionCall *FunctionCall     `json:"function_call,omitempty"`
+			ToolCalls    []ToolCall        `json:"tool_calls,omitempty"`
+			ToolCallID   string            `json:"tool_call_id,omitempty"`
+		}(m)
+		return json.Marshal(msg)
+	}
+	msg := struct {
+		Role         string            `json:"role"`
+		Content      string            `json:"content"`
+		MultiContent []ChatMessagePart `json:"-"`
+		Name         string            `json:"name,omitempty"`
+		FunctionCall *FunctionCall     `json:"function_call,omitempty"`
+		ToolCalls    []ToolCall        `json:"tool_calls,omitempty"`
+		ToolCallID   string            `json:"tool_call_id,omitempty"`
+	}(m)
+	return json.Marshal(msg)
+}
+
+func (m *ChatCompletionMessage) UnmarshalJSON(bs []byte) error {
+	msg := struct {
+		Role         string `json:"role"`
+		Content      string `json:"content"`
+		MultiContent []ChatMessagePart
+		Name         string        `json:"name,omitempty"`
+		FunctionCall *FunctionCall `json:"function_call,omitempty"`
+		ToolCalls    []ToolCall    `json:"tool_calls,omitempty"`
+		ToolCallID   string        `json:"tool_call_id,omitempty"`
+	}{}
+	if err := json.Unmarshal(bs, &msg); err == nil {
+		*m = ChatCompletionMessage(msg)
+		return nil
+	}
+	multiMsg := struct {
+		Role         string `json:"role"`
+		Content      string
+		MultiContent []ChatMessagePart `json:"content"`
+		Name         string            `json:"name,omitempty"`
+		FunctionCall *FunctionCall     `json:"function_call,omitempty"`
+		ToolCalls    []ToolCall        `json:"tool_calls,omitempty"`
+		ToolCallID   string            `json:"tool_call_id,omitempty"`
+	}{}
+	if err := json.Unmarshal(bs, &multiMsg); err != nil {
+		return err
+	}
+	*m = ChatCompletionMessage(multiMsg)
+	return nil
+}
+
 type ToolCall struct {
 	// Index is not nil only in chat completion chunk object
 	Index    *int         `json:"index,omitempty"`
diff --git a/examples/completion-with-tool/main.go b/examples/completion-with-tool/main.go
new file mode 100644
index 0000000..2c7fedc
--- /dev/null
+++ b/examples/completion-with-tool/main.go
@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/sashabaranov/go-openai"
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+func main() {
+	ctx := context.Background()
+	client := openai.NewClient(os.Getenv("OPENAI_API_KEY"))
+
+	// describe the function & its inputs
+	params := jsonschema.Definition{
+		Type: jsonschema.Object,
+		Properties: map[string]jsonschema.Definition{
+			"location": {
+				Type:        jsonschema.String,
+				Description: "The city and state, e.g. San Francisco, CA",
+			},
+			"unit": {
+				Type: jsonschema.String,
+				Enum: []string{"celsius", "fahrenheit"},
+			},
+		},
+		Required: []string{"location"},
+	}
+	f := openai.FunctionDefinition{
+		Name:        "get_current_weather",
+		Description: "Get the current weather in a given location",
+		Parameters:  params,
+	}
+	t := openai.Tool{
+		Type:     openai.ToolTypeFunction,
+		Function: f,
+	}
+
+	// simulate user asking a question that requires the function
+	dialogue := []openai.ChatCompletionMessage{
+		{Role: openai.ChatMessageRoleUser, Content: "What is the weather in Boston today?"},
+	}
+	fmt.Printf("Asking OpenAI '%v' and providing it a '%v()' function...\n",
+		dialogue[0].Content, f.Name)
+	resp, err := client.CreateChatCompletion(ctx,
+		openai.ChatCompletionRequest{
+			Model:    openai.GPT4TurboPreview,
+			Messages: dialogue,
+			Tools:    []openai.Tool{t},
+		},
+	)
+	if err != nil || len(resp.Choices) != 1 {
+		fmt.Printf("Completion error: err:%v len(choices):%v\n", err,
+			len(resp.Choices))
+		return
+	}
+	msg := resp.Choices[0].Message
+	if len(msg.ToolCalls) != 1 {
+		fmt.Printf("Completion error: len(toolcalls): %v\n", len(msg.ToolCalls))
+		return
+	}
+
+	// simulate calling the function & responding to OpenAI
+	dialogue = append(dialogue, msg)
+	fmt.Printf("OpenAI called us back wanting to invoke our function '%v' with params '%v'\n",
+		msg.ToolCalls[0].Function.Name, msg.ToolCalls[0].Function.Arguments)
+	dialogue = append(dialogue, openai.ChatCompletionMessage{
+		Role:       openai.ChatMessageRoleTool,
+		Content:    "Sunny and 80 degrees.",
+		Name:       msg.ToolCalls[0].Function.Name,
+		ToolCallID: msg.ToolCalls[0].ID,
+	})
+	fmt.Printf("Sending OpenAI our '%v()' function's response and requesting the reply to the original question...\n",
+		f.Name)
+	resp, err = client.CreateChatCompletion(ctx,
+		openai.ChatCompletionRequest{
+			Model:    openai.GPT4TurboPreview,
+			Messages: dialogue,
+			Tools:    []openai.Tool{t},
+		},
+	)
+	if err != nil || len(resp.Choices) != 1 {
+		fmt.Printf("2nd completion error: err:%v len(choices):%v\n", err,
+			len(resp.Choices))
+		return
+	}
+
+	// display OpenAI's response to the original question utilizing our function
+	msg = resp.Choices[0].Message
+	fmt.Printf("OpenAI answered the original request with: %v\n",
+		msg.Content)
+}
diff --git a/speech.go b/speech.go
index a3d5f5d..f2442b9 100644
--- a/speech.go
+++ b/speech.go
@@ -10,8 +10,9 @@ import (
 type SpeechModel string
 
 const (
-	TTSModel1   SpeechModel = "tts-1"
-	TTsModel1HD SpeechModel = "tts-1-hd"
+	TTSModel1      SpeechModel = "tts-1"
+	TTSModel1HD    SpeechModel = "tts-1-hd"
+	TTSModelCanary SpeechModel = "canary-tts"
 )
 
 type SpeechVoice string
@@ -57,7 +58,7 @@ func contains[T comparable](s []T, e T) bool {
 }
 
 func isValidSpeechModel(model SpeechModel) bool {
-	return contains([]SpeechModel{TTSModel1, TTsModel1HD}, model)
+	return contains([]SpeechModel{TTSModel1, TTSModel1HD, TTSModelCanary}, model)
 }
 
 func isValidVoice(voice SpeechVoice) bool {


Test patch:
diff --git a/chat_test.go b/chat_test.go
index 8377809..520bf5c 100644
--- a/chat_test.go
+++ b/chat_test.go
@@ -3,6 +3,7 @@ package openai_test
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -296,6 +297,108 @@ func TestAzureChatCompletions(t *testing.T) {
 	checks.NoError(t, err, "CreateAzureChatCompletion error")
 }
 
+func TestMultipartChatCompletions(t *testing.T) {
+	client, server, teardown := setupAzureTestServer()
+	defer teardown()
+	server.RegisterHandler("/openai/deployments/*", handleChatCompletionEndpoint)
+
+	_, err := client.CreateChatCompletion(context.Background(), openai.ChatCompletionRequest{
+		MaxTokens: 5,
+		Model:     openai.GPT3Dot5Turbo,
+		Messages: []openai.ChatCompletionMessage{
+			{
+				Role: openai.ChatMessageRoleUser,
+				MultiContent: []openai.ChatMessagePart{
+					{
+						Type: openai.ChatMessagePartTypeText,
+						Text: "Hello!",
+					},
+					{
+						Type: openai.ChatMessagePartTypeImageURL,
+						ImageURL: &openai.ChatMessageImageURL{
+							URL:    "URL",
+							Detail: openai.ImageURLDetailLow,
+						},
+					},
+				},
+			},
+		},
+	})
+	checks.NoError(t, err, "CreateAzureChatCompletion error")
+}
+
+func TestMultipartChatMessageSerialization(t *testing.T) {
+	jsonText := `[{"role":"system","content":"system-message"},` +
+		`{"role":"user","content":[{"type":"text","text":"nice-text"},` +
+		`{"type":"image_url","image_url":{"url":"URL","detail":"high"}}]}]`
+
+	var msgs []openai.ChatCompletionMessage
+	err := json.Unmarshal([]byte(jsonText), &msgs)
+	if err != nil {
+		t.Fatalf("Expected no error: %s", err)
+	}
+	if len(msgs) != 2 {
+		t.Errorf("unexpected number of messages")
+	}
+	if msgs[0].Role != "system" || msgs[0].Content != "system-message" || msgs[0].MultiContent != nil {
+		t.Errorf("invalid user message: %v", msgs[0])
+	}
+	if msgs[1].Role != "user" || msgs[1].Content != "" || len(msgs[1].MultiContent) != 2 {
+		t.Errorf("invalid user message")
+	}
+	parts := msgs[1].MultiContent
+	if parts[0].Type != "text" || parts[0].Text != "nice-text" {
+		t.Errorf("invalid text part: %v", parts[0])
+	}
+	if parts[1].Type != "image_url" || parts[1].ImageURL.URL != "URL" || parts[1].ImageURL.Detail != "high" {
+		t.Errorf("invalid image_url part")
+	}
+
+	s, err := json.Marshal(msgs)
+	if err != nil {
+		t.Fatalf("Expected no error: %s", err)
+	}
+	res := strings.ReplaceAll(string(s), " ", "")
+	if res != jsonText {
+		t.Fatalf("invalid message: %s", string(s))
+	}
+
+	invalidMsg := []openai.ChatCompletionMessage{
+		{
+			Role:    "user",
+			Content: "some-text",
+			MultiContent: []openai.ChatMessagePart{
+				{
+					Type: "text",
+					Text: "nice-text",
+				},
+			},
+		},
+	}
+	_, err = json.Marshal(invalidMsg)
+	if !errors.Is(err, openai.ErrContentFieldsMisused) {
+		t.Fatalf("Expected error: %s", err)
+	}
+
+	err = json.Unmarshal([]byte(`["not-a-message"]`), &msgs)
+	if err == nil {
+		t.Fatalf("Expected error")
+	}
+
+	emptyMultiContentMsg := openai.ChatCompletionMessage{
+		Role:         "user",
+		MultiContent: []openai.ChatMessagePart{},
+	}
+	s, err = json.Marshal(emptyMultiContentMsg)
+	if err != nil {
+		t.Fatalf("Unexpected error")
+	}
+	res = strings.ReplaceAll(string(s), " ", "")
+	if res != `{"role":"user","content":""}` {
+		t.Fatalf("invalid message: %s", string(s))
+	}
+}
+
 // handleChatCompletionEndpoint Handles the ChatGPT completion endpoint by the test server.
 func handleChatCompletionEndpoint(w http.ResponseWriter, r *http.Request) {
 	var err error