Repository: sashabaranov/go-openai
Pull request #267 (2023-04-19 04:14:49)

Adds support for audio captioning with Whisper
Closes https://github.com/sashabaranov/go-openai/issues/143

Allows users to add a `Format: ` argument to the request to get captions from the audio apis.

Demonstration (using the same code added for README)

https://user-images.githubusercontent.com/4222666/232965446-78a47bc5-d7eb-4439-a161-3ff13f2600ff.mov

Broke `decodeResponse` out of client, because cyclomatic complexity  got above 20 with the string response changes.

Added tests for `decodeResponse` to not decrease code coverage

codecov[bot]: ## [Codecov](https://codecov.io/gh/sashabaranov/go-openai/pull/267?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) Report
> Merging [#267](https://codecov.io/gh/sashabaranov/go-openai/pull/267?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (cc78107) into [master](https://codecov.io/gh/sashabaranov/go-openai/commit/061c97ef7ee75e3ad782797cb46925949e548746?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (061c97e) will **decrease** coverage by `0.11%`.
> The diff coverage is `87.09%`.

```diff
@@            Coverage Diff             @@
##           master     #267      +/-   ##
==========================================
- Coverage   91.45%   91.34%   -0.11%     
==========================================
  Files          22       22              
  Lines         620      647      +27     
==========================================
+ Hits          567      591      +24     
- Misses         39       41       +2     
- Partials       14       15       +1     
```


| [Impacted Files](https://codecov.io/gh/sashabaranov/go-openai/pull/267?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) | Coverage Δ | |
|---|---|---|
| [client.go](https://codecov.io/gh/sashabaranov/go-openai/pull/267?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-Y2xpZW50Lmdv) | `80.89% <63.63%> (-1.82%)` | :arrow_down: |
| [audio.go](https://codecov.io/gh/sashabaranov/go-openai/pull/267?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-YXVkaW8uZ28=) | `85.93% <100.00%> (+2.91%)` | :arrow_up: |
| [image.go](https://codecov.io/gh/sashabaranov/go-openai/pull/267?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-aW1hZ2UuZ28=) | `92.77% <100.00%> (+0.77%)` | :arrow_up: |

Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None). Have a feature suggestion? [Share it here.](https://app.codecov.io/gh/feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None)


Patch:
diff --git a/README.md b/README.md
index 898465c..7526ea3 100644
--- a/README.md
+++ b/README.md
@@ -223,6 +223,47 @@ func main() {
 ```
 </details>
 
+<details>
+<summary>Audio Captions</summary>
+
+```go
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	openai "github.com/sashabaranov/go-openai"
+)
+
+func main() {
+	c := openai.NewClient(os.Getenv("OPENAI_KEY"))
+
+	req := openai.AudioRequest{
+		Model:    openai.Whisper1,
+		FilePath: os.Args[1],
+		Format:   openai.AudioResponseFormatSRT,
+	}
+	resp, err := c.CreateTranscription(context.Background(), req)
+	if err != nil {
+		fmt.Printf("Transcription error: %v\n", err)
+		return
+	}
+	f, err := os.Create(os.Args[1] + ".srt")
+	if err != nil {
+		fmt.Printf("Could not open file: %v\n", err)
+		return
+	}
+	defer f.Close()
+	if _, err := f.WriteString(resp.Text); err != nil {
+		fmt.Printf("Error writing to file: %v\n", err)
+		return
+	}
+}
+```
+</details>
+
 <details>
 <summary>DALL-E 2 image generation</summary>
 
@@ -420,4 +461,4 @@ func main() {
 	fmt.Println(resp.Choices[0].Message.Content)
 }
 ```
-</details>
\ No newline at end of file
+</details>
diff --git a/audio.go b/audio.go
index 9db9298..46c3711 100644
--- a/audio.go
+++ b/audio.go
@@ -13,6 +13,15 @@ const (
 	Whisper1 = "whisper-1"
 )
 
+// Response formats; Whisper uses AudioResponseFormatJSON by default.
+type AudioResponseFormat string
+
+const (
+	AudioResponseFormatJSON AudioResponseFormat = "json"
+	AudioResponseFormatSRT  AudioResponseFormat = "srt"
+	AudioResponseFormatVTT  AudioResponseFormat = "vtt"
+)
+
 // AudioRequest represents a request structure for audio API.
 // ResponseFormat is not supported for now. We only return JSON text, which may be sufficient.
 type AudioRequest struct {
@@ -21,6 +30,7 @@ type AudioRequest struct {
 	Prompt      string // For translation, it should be in English
 	Temperature float32
 	Language    string // For translation, just do not use it. It seems "en" works, not confirmed...
+	Format      AudioResponseFormat
 }
 
 // AudioResponse represents a response structure for audio API.
@@ -66,10 +76,19 @@ func (c *Client) callAudioAPI(
 	}
 	req.Header.Add("Content-Type", builder.formDataContentType())
 
-	err = c.sendRequest(req, &response)
+	if request.HasJSONResponse() {
+		err = c.sendRequest(req, &response)
+	} else {
+		err = c.sendRequest(req, &response.Text)
+	}
 	return
 }
 
+// HasJSONResponse returns true if the response format is JSON.
+func (r AudioRequest) HasJSONResponse() bool {
+	return r.Format == "" || r.Format == AudioResponseFormatJSON
+}
+
 // audioMultipartForm creates a form with audio file contents and the name of the model to use for
 // audio processing.
 func audioMultipartForm(request AudioRequest, b formBuilder) error {
@@ -97,6 +116,14 @@ func audioMultipartForm(request AudioRequest, b formBuilder) error {
 		}
 	}
 
+	// Create a form field for the format (if provided)
+	if request.Format != "" {
+		err = b.writeField("response_format", string(request.Format))
+		if err != nil {
+			return fmt.Errorf("writing format: %w", err)
+		}
+	}
+
 	// Create a form field for the temperature (if provided)
 	if request.Temperature != 0 {
 		err = b.writeField("temperature", fmt.Sprintf("%.2f", request.Temperature))
diff --git a/chat_stream.go b/chat_stream.go
index 8211292..b5257cc 100644
--- a/chat_stream.go
+++ b/chat_stream.go
@@ -3,6 +3,7 @@ package openai
 import (
 	"bufio"
 	"context"
+	"net/http"
 )
 
 type ChatCompletionStreamChoiceDelta struct {
@@ -53,6 +54,9 @@ func (c *Client) CreateChatCompletionStream(
 	if err != nil {
 		return
 	}
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
+		return nil, c.handleErrorResp(resp)
+	}
 
 	stream = &ChatCompletionStream{
 		streamReader: &streamReader[ChatCompletionStreamResponse]{
diff --git a/client.go b/client.go
index c1f76d7..e17ded2 100644
--- a/client.go
+++ b/client.go
@@ -43,7 +43,7 @@ func NewOrgClient(authToken, org string) *Client {
 	return NewClientWithConfig(config)
 }
 
-func (c *Client) sendRequest(req *http.Request, v interface{}) error {
+func (c *Client) sendRequest(req *http.Request, v any) error {
 	req.Header.Set("Accept", "application/json; charset=utf-8")
 	// Azure API Key authentication
 	if c.config.APIType == APITypeAzure {
@@ -72,25 +72,29 @@ func (c *Client) sendRequest(req *http.Request, v interface{}) error {
 	defer res.Body.Close()
 
 	if res.StatusCode < http.StatusOK || res.StatusCode >= http.StatusBadRequest {
-		var errRes ErrorResponse
-		err = json.NewDecoder(res.Body).Decode(&errRes)
-		if err != nil || errRes.Error == nil {
-			reqErr := RequestError{
-				StatusCode: res.StatusCode,
-				Err:        err,
-			}
-			return fmt.Errorf("error, %w", &reqErr)
-		}
-		errRes.Error.StatusCode = res.StatusCode
-		return fmt.Errorf("error, status code: %d, message: %w", res.StatusCode, errRes.Error)
+		return c.handleErrorResp(res)
 	}
 
-	if v != nil {
-		if err = json.NewDecoder(res.Body).Decode(v); err != nil {
-			return err
-		}
+	return decodeResponse(res.Body, v)
+}
+
+func decodeResponse(body io.Reader, v any) error {
+	if v == nil {
+		return nil
 	}
 
+	if result, ok := v.(*string); ok {
+		return decodeString(body, result)
+	}
+	return json.NewDecoder(body).Decode(v)
+}
+
+func decodeString(body io.Reader, output *string) error {
+	b, err := io.ReadAll(body)
+	if err != nil {
+		return err
+	}
+	*output = string(b)
 	return nil
 }
 
@@ -132,3 +136,17 @@ func (c *Client) newStreamRequest(
 	}
 	return req, nil
 }
+
+func (c *Client) handleErrorResp(resp *http.Response) error {
+	var errRes ErrorResponse
+	err := json.NewDecoder(resp.Body).Decode(&errRes)
+	if err != nil || errRes.Error == nil {
+		reqErr := RequestError{
+			HTTPStatusCode: resp.StatusCode,
+			Err:            err,
+		}
+		return fmt.Errorf("error, %w", &reqErr)
+	}
+	errRes.Error.HTTPStatusCode = resp.StatusCode
+	return fmt.Errorf("error, status code: %d, message: %w", resp.StatusCode, errRes.Error)
+}
diff --git a/error.go b/error.go
index 32ffa6c..8aee670 100644
--- a/error.go
+++ b/error.go
@@ -7,17 +7,17 @@ import (
 
 // APIError provides error information returned by the OpenAI API.
 type APIError struct {
-	Code       any     `json:"code,omitempty"`
-	Message    string  `json:"message"`
-	Param      *string `json:"param,omitempty"`
-	Type       string  `json:"type"`
-	StatusCode int     `json:"-"`
+	Code           any     `json:"code,omitempty"`
+	Message        string  `json:"message"`
+	Param          *string `json:"param,omitempty"`
+	Type           string  `json:"type"`
+	HTTPStatusCode int     `json:"-"`
 }
 
 // RequestError provides informations about generic request errors.
 type RequestError struct {
-	StatusCode int
-	Err        error
+	HTTPStatusCode int
+	Err            error
 }
 
 type ErrorResponse struct {
@@ -73,7 +73,7 @@ func (e *RequestError) Error() string {
 	if e.Err != nil {
 		return e.Err.Error()
 	}
-	return fmt.Sprintf("status code %d", e.StatusCode)
+	return fmt.Sprintf("status code %d", e.HTTPStatusCode)
 }
 
 func (e *RequestError) Unwrap() error {
diff --git a/stream.go b/stream.go
index 64688cd..95662db 100644
--- a/stream.go
+++ b/stream.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"errors"
+	"net/http"
 )
 
 var (
@@ -43,6 +44,9 @@ func (c *Client) CreateCompletionStream(
 	if err != nil {
 		return
 	}
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
+		return nil, c.handleErrorResp(resp)
+	}
 
 	stream = &CompletionStream{
 		streamReader: &streamReader[CompletionResponse]{


Test patch:
diff --git a/api_test.go b/api_test.go
index d6ad789..ecba256 100644
--- a/api_test.go
+++ b/api_test.go
@@ -1,16 +1,15 @@
 package openai_test
 
 import (
-	"encoding/json"
-
-	. "github.com/sashabaranov/go-openai"
-	"github.com/sashabaranov/go-openai/internal/test/checks"
-
 	"context"
+	"encoding/json"
 	"errors"
 	"io"
 	"os"
 	"testing"
+
+	. "github.com/sashabaranov/go-openai"
+	"github.com/sashabaranov/go-openai/internal/test/checks"
 )
 
 func TestAPI(t *testing.T) {
@@ -119,8 +118,8 @@ func TestAPIError(t *testing.T) {
 		t.Fatalf("Error is not an APIError: %+v", err)
 	}
 
-	if apiErr.StatusCode != 401 {
-		t.Fatalf("Unexpected API error status code: %d", apiErr.StatusCode)
+	if apiErr.HTTPStatusCode != 401 {
+		t.Fatalf("Unexpected API error status code: %d", apiErr.HTTPStatusCode)
 	}
 
 	switch v := apiErr.Code.(type) {
@@ -239,8 +238,8 @@ func TestRequestError(t *testing.T) {
 		t.Fatalf("Error is not a RequestError: %+v", err)
 	}
 
-	if reqErr.StatusCode != 418 {
-		t.Fatalf("Unexpected request error status code: %d", reqErr.StatusCode)
+	if reqErr.HTTPStatusCode != 418 {
+		t.Fatalf("Unexpected request error status code: %d", reqErr.HTTPStatusCode)
 	}
 
 	if reqErr.Unwrap() == nil {
diff --git a/audio_test.go b/audio_test.go
index 9d2abfc..daf51f2 100644
--- a/audio_test.go
+++ b/audio_test.go
@@ -112,6 +112,7 @@ func TestAudioWithOptionalArgs(t *testing.T) {
 				Prompt:      "用简体中文",
 				Temperature: 0.5,
 				Language:    "zh",
+				Format:      AudioResponseFormatSRT,
 			}
 			_, err = tc.createFn(ctx, req)
 			checks.NoError(t, err, "audio API error")
@@ -179,6 +180,7 @@ func TestAudioWithFailingFormBuilder(t *testing.T) {
 		Prompt:      "test",
 		Temperature: 0.5,
 		Language:    "en",
+		Format:      AudioResponseFormatSRT,
 	}
 
 	mockFailedErr := fmt.Errorf("mock form builder fail")
@@ -202,7 +204,7 @@ func TestAudioWithFailingFormBuilder(t *testing.T) {
 		return nil
 	}
 
-	failOn := []string{"model", "prompt", "temperature", "language"}
+	failOn := []string{"model", "prompt", "temperature", "language", "response_format"}
 	for _, failingField := range failOn {
 		failForField = failingField
 		mockFailedErr = fmt.Errorf("mock form builder fail on field %s", failingField)
diff --git a/chat_stream_test.go b/chat_stream_test.go
index 24046db..afcb86d 100644
--- a/chat_stream_test.go
+++ b/chat_stream_test.go
@@ -204,6 +204,57 @@ func TestCreateChatCompletionStreamError(t *testing.T) {
 	t.Logf("%+v\n", apiErr)
 }
 
+func TestCreateChatCompletionStreamRateLimitError(t *testing.T) {
+	server := test.NewTestServer()
+	server.RegisterHandler("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(429)
+
+		// Send test responses
+		dataBytes := []byte(`{"error":{` +
+			`"message": "You are sending requests too quickly.",` +
+			`"type":"rate_limit_reached",` +
+			`"param":null,` +
+			`"code":"rate_limit_reached"}}`)
+
+		_, err := w.Write(dataBytes)
+		checks.NoError(t, err, "Write error")
+	})
+	ts := server.OpenAITestServer()
+	ts.Start()
+	defer ts.Close()
+
+	// Client portion of the test
+	config := DefaultConfig(test.GetTestToken())
+	config.BaseURL = ts.URL + "/v1"
+	config.HTTPClient.Transport = &tokenRoundTripper{
+		test.GetTestToken(),
+		http.DefaultTransport,
+	}
+
+	client := NewClientWithConfig(config)
+	ctx := context.Background()
+
+	request := ChatCompletionRequest{
+		MaxTokens: 5,
+		Model:     GPT3Dot5Turbo,
+		Messages: []ChatCompletionMessage{
+			{
+				Role:    ChatMessageRoleUser,
+				Content: "Hello!",
+			},
+		},
+		Stream: true,
+	}
+
+	var apiErr *APIError
+	_, err := client.CreateChatCompletionStream(ctx, request)
+	if !errors.As(err, &apiErr) {
+		t.Errorf("TestCreateChatCompletionStreamRateLimitError did not return APIError")
+	}
+	t.Logf("%+v\n", apiErr)
+}
+
 // Helper funcs.
 func compareChatResponses(r1, r2 ChatCompletionStreamResponse) bool {
 	if r1.ID != r2.ID || r1.Object != r2.Object || r1.Created != r2.Created || r1.Model != r2.Model {
diff --git a/client_test.go b/client_test.go
index 1c15985..7bea6dd 100644
--- a/client_test.go
+++ b/client_test.go
@@ -1,6 +1,8 @@
 package openai //nolint:testpackage // testing private field
 
 import (
+	"bytes"
+	"io"
 	"testing"
 )
 
@@ -20,3 +22,38 @@ func TestClient(t *testing.T) {
 		t.Errorf("Client does not contain proper orgID")
 	}
 }
+
+func TestDecodeResponse(t *testing.T) {
+	stringInput := ""
+
+	testCases := []struct {
+		name  string
+		value interface{}
+		body  io.Reader
+	}{
+		{
+			name:  "nil input",
+			value: nil,
+			body:  bytes.NewReader([]byte("")),
+		},
+		{
+			name:  "string input",
+			value: &stringInput,
+			body:  bytes.NewReader([]byte("test")),
+		},
+		{
+			name:  "map input",
+			value: &map[string]interface{}{},
+			body:  bytes.NewReader([]byte(`{"test": "test"}`)),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := decodeResponse(tc.body, tc.value)
+			if err != nil {
+				t.Errorf("Unexpected error: %v", err)
+			}
+		})
+	}
+}
diff --git a/error_accumulator_test.go b/error_accumulator_test.go
index 637bf36..ecf954d 100644
--- a/error_accumulator_test.go
+++ b/error_accumulator_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"net/http"
 	"testing"
 
 	"github.com/sashabaranov/go-openai/internal/test"
@@ -71,7 +72,11 @@ func TestErrorByteWriteErrors(t *testing.T) {
 
 func TestErrorAccumulatorWriteErrors(t *testing.T) {
 	var err error
-	ts := test.NewTestServer().OpenAITestServer()
+	server := test.NewTestServer()
+	server.RegisterHandler("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "error", 200)
+	})
+	ts := server.OpenAITestServer()
 	ts.Start()
 	defer ts.Close()
 
diff --git a/stream_test.go b/stream_test.go
index a80504d..a5c591f 100644
--- a/stream_test.go
+++ b/stream_test.go
@@ -1,16 +1,16 @@
 package openai_test
 
 import (
-	. "github.com/sashabaranov/go-openai"
-	"github.com/sashabaranov/go-openai/internal/test"
-	"github.com/sashabaranov/go-openai/internal/test/checks"
-
 	"context"
 	"errors"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	. "github.com/sashabaranov/go-openai"
+	"github.com/sashabaranov/go-openai/internal/test"
+	"github.com/sashabaranov/go-openai/internal/test/checks"
 )
 
 func TestCompletionsStreamWrongModel(t *testing.T) {
@@ -171,6 +171,52 @@ func TestCreateCompletionStreamError(t *testing.T) {
 	t.Logf("%+v\n", apiErr)
 }
 
+func TestCreateCompletionStreamRateLimitError(t *testing.T) {
+	server := test.NewTestServer()
+	server.RegisterHandler("/v1/completions", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(429)
+
+		// Send test responses
+		dataBytes := []byte(`{"error":{` +
+			`"message": "You are sending requests too quickly.",` +
+			`"type":"rate_limit_reached",` +
+			`"param":null,` +
+			`"code":"rate_limit_reached"}}`)
+
+		_, err := w.Write(dataBytes)
+		checks.NoError(t, err, "Write error")
+	})
+	ts := server.OpenAITestServer()
+	ts.Start()
+	defer ts.Close()
+
+	// Client portion of the test
+	config := DefaultConfig(test.GetTestToken())
+	config.BaseURL = ts.URL + "/v1"
+	config.HTTPClient.Transport = &tokenRoundTripper{
+		test.GetTestToken(),
+		http.DefaultTransport,
+	}
+
+	client := NewClientWithConfig(config)
+	ctx := context.Background()
+
+	request := CompletionRequest{
+		MaxTokens: 5,
+		Model:     GPT3Ada,
+		Prompt:    "Hello!",
+		Stream:    true,
+	}
+
+	var apiErr *APIError
+	_, err := client.CreateCompletionStream(ctx, request)
+	if !errors.As(err, &apiErr) {
+		t.Errorf("TestCreateCompletionStreamRateLimitError did not return APIError")
+	}
+	t.Logf("%+v\n", apiErr)
+}
+
 // A "tokenRoundTripper" is a struct that implements the RoundTripper
 // interface, specifically to handle the authentication token by adding a token
 // to the request header. We need this because the API requires that each