Repository: livekit/agents
Pull request #1643: resample audio for realtime model and audio sink
Author: login: longcw
Created: 2025-03-13T04:45:26Z | Merged: 2025-03-14T01:22:28Z



Diff:
diff --git a/livekit-agents/livekit/agents/voice/generation.py b/livekit-agents/livekit/agents/voice/generation.py
index 7b5646e171..50f02dc04b 100644
--- a/livekit-agents/livekit/agents/voice/generation.py
+++ b/livekit-agents/livekit/agents/voice/generation.py
@@ -205,15 +205,38 @@ async def _audio_forwarding_task(
     out: _AudioOutput,
 ) -> None:
     try:
+        resampler: rtc.AudioResampler | None = None
         async for frame in tts_output:
             out.audio.append(frame)
-            await audio_output.capture_frame(frame)
+
+            if (
+                not out.first_frame_fut.done()
+                and audio_output.sample_rate is not None
+                and frame.sample_rate != audio_output.sample_rate
+                and resampler is None
+            ):
+                resampler = rtc.AudioResampler(
+                    input_rate=frame.sample_rate,
+                    output_rate=audio_output.sample_rate,
+                    num_channels=frame.num_channels,
+                )
+
+            if resampler:
+                for f in resampler.push(frame):
+                    await audio_output.capture_frame(f)
+            else:
+                await audio_output.capture_frame(frame)
+
             if not out.first_frame_fut.done():
                 out.first_frame_fut.set_result(None)
     finally:
         if isinstance(tts_output, _ACloseable):
             await tts_output.aclose()
 
+        if resampler:
+            for frame in resampler.flush():
+                await audio_output.capture_frame(frame)
+
         audio_output.flush()
 
 
diff --git a/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py b/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
index 62886e8d46..ca5a3fbe81 100644
--- a/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
+++ b/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
@@ -3,7 +3,7 @@
 import asyncio
 import base64
 from dataclasses import dataclass
-from typing import Literal, Optional
+from typing import Iterator, Literal, Optional
 
 from livekit import rtc
 from livekit.agents import llm, utils
@@ -128,6 +128,7 @@ def __init__(self, realtime_model: RealtimeModel) -> None:
         self._realtime_model = realtime_model
         self._fnc_ctx = llm.FunctionContext.empty()
         self._msg_ch = utils.aio.Chan[RealtimeClientEvent]()
+        self._input_resampler: rtc.AudioResampler | None = None
 
         self._conn: AsyncRealtimeConnection | None = None
         self._main_atask = asyncio.create_task(self._main_task(), name="RealtimeSession._main_task")
@@ -323,12 +324,13 @@ async def update_instructions(self, instructions: str) -> None:
         )
 
     def push_audio(self, frame: rtc.AudioFrame) -> None:
-        self._msg_ch.send_nowait(
-            InputAudioBufferAppendEvent(
-                type="input_audio_buffer.append",
-                audio=base64.b64encode(frame.data).decode("utf-8"),
+        for f in self._resample_audio(frame):
+            self._msg_ch.send_nowait(
+                InputAudioBufferAppendEvent(
+                    type="input_audio_buffer.append",
+                    audio=base64.b64encode(f.data).decode("utf-8"),
+                )
             )
-        )
 
     def generate_reply(
         self, *, instructions: NotGivenOr[str] = NOT_GIVEN
@@ -372,6 +374,27 @@ async def aclose(self) -> None:
         if self._conn is not None:
             await self._conn.close()
 
+    def _resample_audio(self, frame: rtc.AudioFrame) -> Iterator[rtc.AudioFrame]:
+        if self._input_resampler:
+            if frame.sample_rate != self._input_resampler._input_rate:
+                # input audio changed to a different sample rate
+                self._input_resampler = None
+
+        if self._input_resampler is None and (
+            frame.sample_rate != SAMPLE_RATE or frame.num_channels != NUM_CHANNELS
+        ):
+            self._input_resampler = rtc.AudioResampler(
+                input_rate=frame.sample_rate,
+                output_rate=SAMPLE_RATE,
+                num_channels=NUM_CHANNELS,
+            )
+
+        if self._input_resampler:
+            # TODO(long): flush the resampler when the input source is changed
+            yield from self._input_resampler.push(frame)
+        else:
+            yield frame
+
     def _handle_input_audio_buffer_speech_started(
         self, _: InputAudioBufferSpeechStartedEvent
     ) -> None:


Reviews:
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:56:47Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:57:03Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:57:44Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:59:09Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:59:39Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T13:06:04Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T13:06:37Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T13:10:13Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:15:49Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T14:32:45Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T14:33:07Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:41:18Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:41:51Z
user: login: theomonnom
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-13T14:51:01Z

Review comments:
diff_hunk: @@ -50,13 +50,15 @@ class PlaybackFinishedEvent:
 
 
 class AudioSink(ABC, rtc.EventEmitter[Literal["playback_finished"]]):
-    def __init__(self, *, sample_rate: int | None = None) -> None:
+    def __init__(self, *, sample_rate: int | None = None, num_channels: int | None = None) -> None:
path: livekit-agents/livekit/agents/voice/io.py
user: login: theomonnom
body: We can't add num_channels yet, we don't have audio remixing
created_at: 2025-03-13T12:56:47Z
author_association: MEMBER
diff_hunk: @@ -50,13 +50,15 @@ class PlaybackFinishedEvent:
 
 
 class AudioSink(ABC, rtc.EventEmitter[Literal["playback_finished"]]):
-    def __init__(self, *, sample_rate: int | None = None) -> None:
+    def __init__(self, *, sample_rate: int | None = None, num_channels: int | None = None) -> None:
path: livekit-agents/livekit/agents/voice/io.py
user: login: theomonnom
body: Feel free to ignore the channels for now 
created_at: 2025-03-13T12:57:02Z
author_association: MEMBER
diff_hunk: @@ -323,12 +324,13 @@ async def update_instructions(self, instructions: str) -> None:
         )
 
     def push_audio(self, frame: rtc.AudioFrame) -> None:
-        self._msg_ch.send_nowait(
-            InputAudioBufferAppendEvent(
-                type="input_audio_buffer.append",
-                audio=base64.b64encode(frame.data).decode("utf-8"),
+        for f in self._resample_audio(frame):
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body:  do we have to use a generator?
 For reference this is how it's done in STT, seems simple enough https://github.com/livekit/agents/blob/648c850af6df1c60016751cd5eca596b6c985624/livekit-agents/livekit/agents/stt/stt.py#L278
created_at: 2025-03-13T12:57:44Z
author_association: MEMBER
diff_hunk: @@ -204,16 +204,52 @@ async def _audio_forwarding_task(
     tts_output: AsyncIterable[rtc.AudioFrame],
     out: _AudioOutput,
 ) -> None:
+    def _needs_resampling(frame: rtc.AudioFrame) -> bool:
+        if audio_output.sample_rate is not None and frame.sample_rate != audio_output.sample_rate:
+            return True
+        if (
+            audio_output.num_channels is not None
+            and frame.num_channels != audio_output.num_channels
+        ):
+            return True
+        return False
+
     try:
+        resampler: rtc.AudioResampler | None = None
         async for frame in tts_output:
             out.audio.append(frame)
-            await audio_output.capture_frame(frame)
+
+            if not out.first_frame_fut.done() and _needs_resampling(frame) and resampler is None:
path: livekit-agents/livekit/agents/voice/generation.py
user: login: theomonnom
body: Do we need to check first_frame_fut, seems like it isn't needed? We can't just check if a resampler is there 
created_at: 2025-03-13T12:59:09Z
author_association: MEMBER
diff_hunk: @@ -204,16 +204,52 @@ async def _audio_forwarding_task(
     tts_output: AsyncIterable[rtc.AudioFrame],
     out: _AudioOutput,
 ) -> None:
+    def _needs_resampling(frame: rtc.AudioFrame) -> bool:
+        if audio_output.sample_rate is not None and frame.sample_rate != audio_output.sample_rate:
+            return True
+        if (
+            audio_output.num_channels is not None
+            and frame.num_channels != audio_output.num_channels
+        ):
+            return True
+        return False
+
     try:
+        resampler: rtc.AudioResampler | None = None
         async for frame in tts_output:
             out.audio.append(frame)
-            await audio_output.capture_frame(frame)
+
+            if not out.first_frame_fut.done() and _needs_resampling(frame) and resampler is None:
path: livekit-agents/livekit/agents/voice/generation.py
user: login: theomonnom
body: We don't have to check every frame,  it is OK to assume that the first frame will determine the sample_rate and the sample_rate will not change 
created_at: 2025-03-13T12:59:39Z
author_association: MEMBER
diff_hunk: @@ -323,12 +324,13 @@ async def update_instructions(self, instructions: str) -> None:
         )
 
     def push_audio(self, frame: rtc.AudioFrame) -> None:
-        self._msg_ch.send_nowait(
-            InputAudioBufferAppendEvent(
-                type="input_audio_buffer.append",
-                audio=base64.b64encode(frame.data).decode("utf-8"),
+        for f in self._resample_audio(frame):
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: imaging that the input audio may change the sample rate bc of the change of `agent.input`, we maye need to check the sample rate even there is already a `resampler` so moved the logic to a method to be clear.
created_at: 2025-03-13T13:06:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -204,16 +204,52 @@ async def _audio_forwarding_task(
     tts_output: AsyncIterable[rtc.AudioFrame],
     out: _AudioOutput,
 ) -> None:
+    def _needs_resampling(frame: rtc.AudioFrame) -> bool:
+        if audio_output.sample_rate is not None and frame.sample_rate != audio_output.sample_rate:
+            return True
+        if (
+            audio_output.num_channels is not None
+            and frame.num_channels != audio_output.num_channels
+        ):
+            return True
+        return False
+
     try:
+        resampler: rtc.AudioResampler | None = None
         async for frame in tts_output:
             out.audio.append(frame)
-            await audio_output.capture_frame(frame)
+
+            if not out.first_frame_fut.done() and _needs_resampling(frame) and resampler is None:
path: livekit-agents/livekit/agents/voice/generation.py
user: login: longcw
body: yes
created_at: 2025-03-13T13:06:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -50,13 +50,15 @@ class PlaybackFinishedEvent:
 
 
 class AudioSink(ABC, rtc.EventEmitter[Literal["playback_finished"]]):
-    def __init__(self, *, sample_rate: int | None = None) -> None:
+    def __init__(self, *, sample_rate: int | None = None, num_channels: int | None = None) -> None:
path: livekit-agents/livekit/agents/voice/io.py
user: login: longcw
body: Do all TTS always output with one channel? otherwise if user misconfigured the num_channel of TTS and RoomIO.AudioSource it will raise the error.

Maybe that's too much, just let it fail seems ok?
created_at: 2025-03-13T13:10:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -50,13 +50,15 @@ class PlaybackFinishedEvent:
 
 
 class AudioSink(ABC, rtc.EventEmitter[Literal["playback_finished"]]):
-    def __init__(self, *, sample_rate: int | None = None) -> None:
+    def __init__(self, *, sample_rate: int | None = None, num_channels: int | None = None) -> None:
path: livekit-agents/livekit/agents/voice/io.py
user: login: longcw
body: updated
created_at: 2025-03-13T14:15:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -323,12 +324,13 @@ async def update_instructions(self, instructions: str) -> None:
         )
 
     def push_audio(self, frame: rtc.AudioFrame) -> None:
-        self._msg_ch.send_nowait(
-            InputAudioBufferAppendEvent(
-                type="input_audio_buffer.append",
-                audio=base64.b64encode(frame.data).decode("utf-8"),
+        for f in self._resample_audio(frame):
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: ah I see, I think it is also a OK assumption to say the input must always use the same samplerate
created_at: 2025-03-13T14:32:45Z
author_association: MEMBER
diff_hunk: @@ -50,13 +50,15 @@ class PlaybackFinishedEvent:
 
 
 class AudioSink(ABC, rtc.EventEmitter[Literal["playback_finished"]]):
-    def __init__(self, *, sample_rate: int | None = None) -> None:
+    def __init__(self, *, sample_rate: int | None = None, num_channels: int | None = None) -> None:
path: livekit-agents/livekit/agents/voice/io.py
user: login: theomonnom
body: Generally, all TTS outputs one channel (at least by default).
Let's fail for now it's easier 
created_at: 2025-03-13T14:33:07Z
author_association: MEMBER
diff_hunk: @@ -323,12 +324,13 @@ async def update_instructions(self, instructions: str) -> None:
         )
 
     def push_audio(self, frame: rtc.AudioFrame) -> None:
-        self._msg_ch.send_nowait(
-            InputAudioBufferAppendEvent(
-                type="input_audio_buffer.append",
-                audio=base64.b64encode(frame.data).decode("utf-8"),
+        for f in self._resample_audio(frame):
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: I think that's true for the single input. But if we allow user to change the input, and will auto resample for the first input (so user don't care what is the sample rate of the first input they given), I think we should do that for the second, third inputs as well to make the behavior consistent (user doesn't need to take care of sample rate for any new inputs). 
created_at: 2025-03-13T14:41:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -50,13 +50,15 @@ class PlaybackFinishedEvent:
 
 
 class AudioSink(ABC, rtc.EventEmitter[Literal["playback_finished"]]):
-    def __init__(self, *, sample_rate: int | None = None) -> None:
+    def __init__(self, *, sample_rate: int | None = None, num_channels: int | None = None) -> None:
path: livekit-agents/livekit/agents/voice/io.py
user: login: longcw
body: Yeah, removed for now. The `AudioSource` will fail if the number of channel mismatch.
created_at: 2025-03-13T14:41:50Z
author_association: CONTRIBUTOR

PR comments:
user: login: changeset-bot[bot]
created_at: 2025-03-13T04:45:30Z
body: ###  ⚠️  No Changeset found

Latest commit: 3c04278e4fa6224a604e8fd47bd33325750d0a96

Merging this PR will not cause a version bump for any packages. If these changes should not result in a new version, you're good to go. **If these changes should result in a version bump, you need to add a changeset.**

<details><summary>This PR includes no changesets</summary>

  When changesets are added to this PR, you'll see the packages that this PR includes changesets for and the associated semver types

</details>

[Click here to learn what changesets are, and how to add one](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md).

[Click here if you're a maintainer who wants to add a changeset to this PR](https://github.com/livekit/agents/new/longc/resample-input-audio?filename=.changeset/forty-cats-run.md&value=---%0A%22livekit-agents%22%3A%20patch%0A%22livekit-plugins-openai%22%3A%20patch%0A---%0A%0Aresample%20audio%20for%20realtime%20model%20and%20audio%20sink%0A)


author_association: NONE
user: login: theomonnom
created_at: 2025-03-14T01:22:44Z
body: merging to avoid future conflicts since a lot is happening due to the refactor
author_association: MEMBER