Repository: aallam/openai-kotlin (kotlin)

Issue:
TranscriptionRequest responseFormat always text
Hi, as I understand we can request  a responseFormat from TranscriptionRequest . But whether I use json, or verbose_json transcription.text is always int text format. I can see json respone in the body when I enable logging

Question is how to get json formatted version of the response?

Also, when I use  srt or vtt I get below error. I can see actual response in the log output but looks like ktor does not know how to handle it.

com.aallam.openai.api.exception.OpenAIHttpException
	at com.aallam.openai.client.internal.http.HttpTransport.perform(HttpTransport.kt:23)
	at com.aallam.openai.client.internal.http.HttpTransport$perform$1.invokeSuspend(HttpTransport.kt:0)
	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
	at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:106)
	at kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.common.kt:284)
	at kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:85)
	at kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:59)
	at kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source:1)
	at kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking$default(Builders.kt:38)
	at kotlinx.coroutines.BuildersKt.runBlocking$default(Unknown Source:1)
	at com.test.client.openai.conector.OpenAiWhisperConnector.upload(OpenAiWhisperConnector.kt:43)
	at com.test.client.openai.OpenAiWhisperClient.upload(OpenAiWhisperClient.kt:21)

Pull request description:
| Q                 | A
| ----------------- | ----------
| Bug fix?          | no
| New feature?      | yes    <!-- please update the /CHANGELOG.md file -->
| BC breaks?        | no
| Related Issue     | Fix #124 

## Describe your change

Add support of other audio formats (e.g. verbose_json, text)

Patch:
diff --git a/openai-client/src/commonMain/kotlin/com.aallam.openai.client/Audio.kt b/openai-client/src/commonMain/kotlin/com.aallam.openai.client/Audio.kt
index 75fddea2..1ad830bb 100644
--- a/openai-client/src/commonMain/kotlin/com.aallam.openai.client/Audio.kt
+++ b/openai-client/src/commonMain/kotlin/com.aallam.openai.client/Audio.kt
@@ -1,10 +1,7 @@
 package com.aallam.openai.client
 
 import com.aallam.openai.api.BetaOpenAI
-import com.aallam.openai.api.audio.Transcription
-import com.aallam.openai.api.audio.TranscriptionRequest
-import com.aallam.openai.api.audio.Translation
-import com.aallam.openai.api.audio.TranslationRequest
+import com.aallam.openai.api.audio.*
 
 /**
  * Learn how to turn audio into text.
diff --git a/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/api/AudioApi.kt b/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/api/AudioApi.kt
index faac6521..7d6e2da8 100644
--- a/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/api/AudioApi.kt
+++ b/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/api/AudioApi.kt
@@ -17,30 +17,58 @@ import io.ktor.client.request.forms.*
 internal class AudioApi(val requester: HttpRequester) : Audio {
     @BetaOpenAI
     override suspend fun transcription(request: TranscriptionRequest): Transcription {
+        return when (request.responseFormat) {
+            "json", "verbose_json", null -> transcriptionAsJson(request)
+            else -> transcriptionAsString(request)
+        }
+    }
+
+    private suspend fun transcriptionAsJson(request: TranscriptionRequest): Transcription {
         return requester.perform {
             it.submitFormWithBinaryData(url = TranscriptionPathV1, formData = formDataOf(request))
         }
     }
 
+    private suspend fun transcriptionAsString(request: TranscriptionRequest): Transcription {
+        val text = requester.perform<String> {
+            it.submitFormWithBinaryData(url = TranscriptionPathV1, formData = formDataOf(request))
+        }
+        return Transcription(text)
+    }
+
     /**
      * Build transcription request as form-data.
      */
-    private fun formDataOf(request: TranscriptionRequest) = formData {
+    private fun formDataOf(request: TranscriptionRequest, format: String? = null) = formData {
         appendFileSource("file", request.audio)
         append(key = "model", value = request.model.id)
         request.prompt?.let { prompt -> append(key = "prompt", value = prompt) }
-        request.responseFormat?.let { append(key = "response_format", value = it) }
+        val responseFormat = format ?: request.responseFormat
+        responseFormat?.let { append(key = "response_format", value = it) }
         request.temperature?.let { append(key = "temperature", value = it) }
         request.language?.let { append(key = "language", value = it) }
     }
 
     @BetaOpenAI
     override suspend fun translation(request: TranslationRequest): Translation {
+        return when (request.responseFormat) {
+            "json", "verbose_json", null -> translationAsJson(request)
+            else -> translationAsString(request)
+        }
+    }
+
+    private suspend fun translationAsJson(request: TranslationRequest): Translation {
         return requester.perform {
             it.submitFormWithBinaryData(url = TranslationPathV1, formData = formDataOf(request))
         }
     }
 
+    private suspend fun translationAsString(request: TranslationRequest): Translation {
+        val text = requester.perform<String> {
+            it.submitFormWithBinaryData(url = TranslationPathV1, formData = formDataOf(request))
+        }
+        return Translation(text)
+    }
 
     /**
      * Build transcription request as form-data.
diff --git a/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/extension/Stream.kt b/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/extension/Stream.kt
index fffafaa6..4efc1f3e 100644
--- a/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/extension/Stream.kt
+++ b/openai-client/src/commonMain/kotlin/com.aallam.openai.client/internal/extension/Stream.kt
@@ -20,10 +20,7 @@ internal suspend inline fun <reified T> FlowCollector<T>.streamEventsFrom(respon
         val line = channel.readUTF8Line() ?: continue
         val value: T = when {
             line.startsWith(STREAM_END_TOKEN) -> break
-            line.startsWith(STREAM_PREFIX) -> JsonLenient.decodeFromString(
-                line.removePrefix(STREAM_PREFIX)
-            )
-
+            line.startsWith(STREAM_PREFIX) -> JsonLenient.decodeFromString(line.removePrefix(STREAM_PREFIX))
             else -> continue
         }
         emit(value)
diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Segment.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Segment.kt
new file mode 100644
index 00000000..fd406808
--- /dev/null
+++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Segment.kt
@@ -0,0 +1,21 @@
+package com.aallam.openai.api.audio
+
+import com.aallam.openai.api.BetaOpenAI
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@BetaOpenAI
+@Serializable
+public data class Segment(
+    @SerialName("id") val id: Int,
+    @SerialName("seek") val seek: Int,
+    @SerialName("start") val start: Double,
+    @SerialName("end") val end: Double,
+    @SerialName("text") val text: String,
+    @SerialName("tokens") val tokens: List<Int>,
+    @SerialName("temperature") val temperature: Double,
+    @SerialName("avg_logprob") val avgLogprob: Double,
+    @SerialName("compression_ratio") val compressionRatio: Double,
+    @SerialName("no_speech_prob") val noSpeechProb: Double,
+    @SerialName("transient") val transient: Boolean,
+)
diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Transcription.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Transcription.kt
index 12086c03..1c3989cd 100644
--- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Transcription.kt
+++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Transcription.kt
@@ -7,13 +7,18 @@ import kotlinx.serialization.Serializable
 /**
  * Create transcription response.
  *
- * Note: This does not currently support the non-default response format types.
+ * [text] format depends on [TranscriptionRequest]'s `responseFormat`.
+ * Remaining field are provided only in case of response format `verbose_json`.
  */
 @BetaOpenAI
 @Serializable
 public data class Transcription(
     /**
-     * Transcription text.
+     * Transcription [text].
+     * The format depends on [TranscriptionRequest]'s `responseFormat`.
      */
-    @SerialName("text") val text: String
+    @SerialName("text") val text: String,
+    @SerialName("language") val language: String? = null,
+    @SerialName("duration") val duration: Double? = null,
+    @SerialName("segments") val segments: List<Segment>? = null,
 )
diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Translation.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Translation.kt
index e98bb4b8..73b59119 100644
--- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Translation.kt
+++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/audio/Translation.kt
@@ -7,13 +7,19 @@ import kotlinx.serialization.Serializable
 /**
  * Create translation response.
  *
- * Note: This does not currently support the non-default response format types.
+ * [text] format depends on [TranslationRequest]'s `responseFormat`.
+ * Remaining field are provided only in case of response format `verbose_json`.
  */
 @BetaOpenAI
 @Serializable
 public data class Translation(
     /**
      * Translation text.
+     *
+     * The format depends on [TranslationRequest]'s `responseFormat`.
      */
-    @SerialName("text") val text: String
+    @SerialName("text") val text: String,
+    @SerialName("language") val language: String? = null,
+    @SerialName("duration") val duration: Double? = null,
+    @SerialName("segments") val segments: List<Segment>? = null,
 )


Test patch:
diff --git a/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestAudio.kt b/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestAudio.kt
index 92327ca3..842b4fc3 100644
--- a/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestAudio.kt
+++ b/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestAudio.kt
@@ -10,6 +10,7 @@ import io.ktor.client.call.*
 import io.ktor.client.request.*
 import kotlinx.coroutines.test.runTest
 import kotlin.test.Test
+import kotlin.test.assertEquals
 import kotlin.test.assertTrue
 
 class TestAudio : TestOpenAI() {
@@ -24,6 +25,7 @@ class TestAudio : TestOpenAI() {
             audio = FileSource(name = "micro-machines.wav", source = audioBytes.asSource())
             model = ModelId("whisper-1")
         }
+
         val transcription = openAI.transcription(request)
         assertTrue { transcription.text.isNotEmpty() }
         assertTrue {
@@ -34,6 +36,35 @@ class TestAudio : TestOpenAI() {
         }
     }
 
+    @Test
+    fun transcriptionText() = runTest {
+        val speedTalkingUrl = "https://github.com/aallam/sample-data/raw/main/openai/audio/micro-machines.wav"
+        val audioBytes: ByteArray = httpClient.get(speedTalkingUrl).body()
+        val request = transcriptionRequest {
+            audio = FileSource(name = "micro-machines.wav", source = audioBytes.asSource())
+            model = ModelId("whisper-1")
+            responseFormat = "text"
+        }
+        val transcription = openAI.transcription(request)
+        assertTrue { transcription.text.isNotEmpty() }
+    }
+
+    @Test
+    fun transcriptionJsonVerbose() = runTest {
+        val speedTalkingUrl = "https://github.com/aallam/sample-data/raw/main/openai/audio/micro-machines.wav"
+        val audioBytes: ByteArray = httpClient.get(speedTalkingUrl).body()
+        val request = transcriptionRequest {
+            audio = FileSource(name = "micro-machines.wav", source = audioBytes.asSource())
+            model = ModelId("whisper-1")
+            responseFormat = "verbose_json"
+        }
+        val transcription = openAI.transcription(request)
+        assertTrue { transcription.text.isNotEmpty() }
+        assertEquals(transcription.language, "english")
+        assertEquals(transcription.duration, 29.88)
+        assertTrue { transcription.segments?.isNotEmpty() ?: false }
+    }
+
     @Test
     fun translation() = runTest {
         val multilingualUrl = "https://github.com/aallam/sample-data/raw/main/openai/audio/multilingual.wav"
@@ -51,4 +82,33 @@ class TestAudio : TestOpenAI() {
             )
         }
     }
+
+    @Test
+    fun translationText() = runTest {
+        val multilingualUrl = "https://github.com/aallam/sample-data/raw/main/openai/audio/multilingual.wav"
+        val audioBytes: ByteArray = httpClient.get(multilingualUrl).body()
+        val request = translationRequest {
+            audio = FileSource(name = "multilingual.wav", source = audioBytes.asSource())
+            model = ModelId("whisper-1")
+            responseFormat = "text"
+        }
+        val translation = openAI.translation(request)
+        assertTrue { translation.text.isNotEmpty() }
+    }
+
+    @Test
+    fun translationJsonVerbose() = runTest {
+        val multilingualUrl = "https://github.com/aallam/sample-data/raw/main/openai/audio/multilingual.wav"
+        val audioBytes: ByteArray = httpClient.get(multilingualUrl).body()
+        val request = translationRequest {
+            audio = FileSource(name = "multilingual.wav", source = audioBytes.asSource())
+            model = ModelId("whisper-1")
+            responseFormat = "verbose_json"
+        }
+        val translation = openAI.translation(request)
+        assertTrue { translation.text.isNotEmpty() }
+        assertEquals(translation.language, "english")
+        assertEquals(translation.duration, 42.06)
+        assertTrue { translation.segments?.isNotEmpty() ?: false }
+    }
 }