Repository: livekit/agents
Pull request #1910: add realtime audio node
Author: login: longcw
Created: 2025-04-07T05:49:34Z | Merged: 2025-04-07T13:31:35Z



Diff:
diff --git a/.github/next-release/changeset-bdfd50f8.md b/.github/next-release/changeset-bdfd50f8.md
new file mode 100644
index 0000000000..db505491eb
--- /dev/null
+++ b/.github/next-release/changeset-bdfd50f8.md
@@ -0,0 +1,5 @@
+---
+"livekit-agents": patch
+---
+
+add realtime audio node (#1910)
diff --git a/examples/voice_agents/speedup_output_audio.py b/examples/voice_agents/speedup_output_audio.py
new file mode 100644
index 0000000000..e01a5e6592
--- /dev/null
+++ b/examples/voice_agents/speedup_output_audio.py
@@ -0,0 +1,126 @@
+import logging
+from collections.abc import AsyncIterable
+
+import numpy as np
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import (
+    Agent,
+    AgentSession,
+    JobContext,
+    JobProcess,
+    ModelSettings,
+    WorkerOptions,
+    cli,
+    utils,
+)
+from livekit.plugins import deepgram, openai, silero
+
+try:
+    import librosa
+except ImportError:
+    raise ImportError(
+        "librosa is required to run this example, install it with `pip install librosa`"
+    ) from None
+
+
+logger = logging.getLogger("speedup-output-audio")
+logging.getLogger("numba").setLevel(logging.WARNING)
+
+load_dotenv()
+
+## This example demonstrates how to add post-processing to the output audio of the agent.
+
+
+class MyAgent(Agent):
+    def __init__(self, *, speed_factor: float = 1.2) -> None:
+        super().__init__(
+            instructions="Your name is Jenna. You would interact with users via voice."
+            "with that in mind keep your responses concise and to the point."
+            "You are curious and friendly, and have a sense of humor.",
+        )
+        self.speed_factor = speed_factor
+
+    async def tts_node(
+        self, text: AsyncIterable[str], model_settings: ModelSettings
+    ) -> AsyncIterable[rtc.AudioFrame]:
+        # process for tts output
+        async for frame in self._process_audio_stream(super().tts_node(text, model_settings)):
+            yield frame
+
+    async def realtime_audio_output_node(
+        self, audio: AsyncIterable[rtc.AudioFrame], model_settings: ModelSettings
+    ) -> AsyncIterable[rtc.AudioFrame]:
+        # process for realtime audio output
+        async for frame in self._process_audio_stream(
+            super().realtime_audio_output_node(audio, model_settings)
+        ):
+            yield frame
+
+    async def _process_audio_stream(
+        self, audio: AsyncIterable[rtc.AudioFrame]
+    ) -> AsyncIterable[rtc.AudioFrame]:
+        stream: utils.audio.AudioByteStream | None = None
+        async for frame in audio:
+            if stream is None:
+                stream = utils.audio.AudioByteStream(
+                    sample_rate=frame.sample_rate,
+                    num_channels=frame.num_channels,
+                    samples_per_channel=frame.sample_rate // 10,  # 100ms
+                )
+            # TODO: find a streamed way to process the audio
+            for f in stream.push(frame.data):
+                yield self._process_audio(f)
+
+        for f in stream.flush():
+            yield self._process_audio(f)
+
+    def _process_audio(self, frame: rtc.AudioFrame) -> rtc.AudioFrame:
+        # time-stretch without pitch change
+        audio_data = np.frombuffer(frame.data, dtype=np.int16)
+
+        stretched = librosa.effects.time_stretch(
+            audio_data.astype(np.float32) / np.iinfo(np.int16).max,
+            rate=self.speed_factor,
+        )
+        stretched = (stretched * np.iinfo(np.int16).max).astype(np.int16)
+        return rtc.AudioFrame(
+            data=stretched.tobytes(),
+            sample_rate=frame.sample_rate,
+            num_channels=frame.num_channels,
+            samples_per_channel=stretched.shape[-1],
+        )
+
+
+def prewarm(proc: JobProcess):
+    proc.userdata["vad"] = silero.VAD.load()
+
+    # warmup the librosa
+    librosa.effects.time_stretch(
+        np.random.randn(16000).astype(np.float32),
+        rate=1.2,
+    )
+
+
+async def entrypoint(ctx: JobContext):
+    # each log entry will include these fields
+    ctx.log_context_fields = {
+        "room": ctx.room.name,
+        "user_id": "your user_id",
+    }
+    await ctx.connect()
+
+    session = AgentSession(
+        vad=ctx.proc.userdata["vad"],
+        llm=openai.LLM(model="gpt-4o-mini"),
+        stt=deepgram.STT(model="nova-3"),
+        tts=openai.TTS(voice="ash"),
+        # llm=openai.realtime.RealtimeModel(voice="alloy"),
+    )
+    await session.start(agent=MyAgent(), room=ctx.room)
+    # session.say("Hello, how can I help you today?")
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint, prewarm_fnc=prewarm))
diff --git a/livekit-agents/livekit/agents/voice/agent.py b/livekit-agents/livekit/agents/voice/agent.py
index 82e8deb253..d1ed28570b 100644
--- a/livekit-agents/livekit/agents/voice/agent.py
+++ b/livekit-agents/livekit/agents/voice/agent.py
@@ -420,6 +420,15 @@ async def _forward_input():
             finally:
                 await utils.aio.cancel_and_wait(forward_task)
 
+    async def realtime_audio_output_node(
+        self, audio: AsyncIterable[rtc.AudioFrame], model_settings: ModelSettings
+    ) -> AsyncIterable[rtc.AudioFrame]:
+        """A node processing the audio from the realtime LLM session before it is played out."""
+        self.__get_activity_or_raise()
+        async for frame in audio:
+            yield frame
+        # flush the buffer if any
+
     def __get_activity_or_raise(self) -> AgentActivity:
         """Get the current activity context for this task (internal)"""
         if self._activity is None:
diff --git a/livekit-agents/livekit/agents/voice/agent_activity.py b/livekit-agents/livekit/agents/voice/agent_activity.py
index 904e5592b4..72f054861e 100644
--- a/livekit-agents/livekit/agents/voice/agent_activity.py
+++ b/livekit-agents/livekit/agents/voice/agent_activity.py
@@ -1192,7 +1192,10 @@ async def _read_messages(
                     audio_out = None
                     if audio_output is not None:
                         forward_task, audio_out = perform_audio_forwarding(
-                            audio_output=audio_output, tts_output=msg.audio_stream
+                            audio_output=audio_output,
+                            tts_output=self._agent.realtime_audio_output_node(
+                                msg.audio_stream, model_settings
+                            ),
                         )
                         forward_tasks.append(forward_task)
                         audio_out.first_frame_fut.add_done_callback(_on_first_frame)


Reviews:
user: login: davidzhao
body: nice!
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-07T06:36:55Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T06:52:03Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T07:27:44Z
user: login: theomonnom
body: lgtm!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-07T12:58:39Z

Review comments:
diff_hunk: @@ -1192,7 +1194,8 @@ async def _read_messages(
                     audio_out = None
                     if audio_output is not None:
                         forward_task, audio_out = perform_audio_forwarding(
path: livekit-agents/livekit/agents/voice/agent_activity.py
user: login: davidzhao
body: wouldn't be cleaner to apply this within `perform_audio_forwarding` ?
created_at: 2025-04-07T06:36:10Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,119 @@
+import logging
+from collections.abc import AsyncIterable
+
+import numpy as np
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import (
+    Agent,
+    AgentSession,
+    JobContext,
+    JobProcess,
+    RoomInputOptions,
+    RoomOutputOptions,
+    WorkerOptions,
+    cli,
+    utils,
+)
+from livekit.plugins import deepgram, openai, silero
+
+try:
+    import librosa
+except ImportError:
+    raise ImportError(
+        "librosa is required to run this example, install it with `pip install librosa`"
+    ) from None
+
+# uncomment to enable Krisp background voice/noise cancellation
path: examples/voice_agents/process_output_audio.py
user: login: davidzhao
body: I think we can remove krisp related stuff from here, and replace with a paragraph about what this example is demonstrating.
created_at: 2025-04-07T06:36:45Z
author_association: MEMBER
diff_hunk: @@ -1192,7 +1194,8 @@ async def _read_messages(
                     audio_out = None
                     if audio_output is not None:
                         forward_task, audio_out = perform_audio_forwarding(
path: livekit-agents/livekit/agents/voice/agent_activity.py
user: login: longcw
body: I think this makes `perform_audio_forwarding` cleaner, it take a audio stream and forward it to the AudioOutput but don't care where the audio stream comes from. 
created_at: 2025-04-07T06:52:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,119 @@
+import logging
+from collections.abc import AsyncIterable
+
+import numpy as np
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import (
+    Agent,
+    AgentSession,
+    JobContext,
+    JobProcess,
+    RoomInputOptions,
+    RoomOutputOptions,
+    WorkerOptions,
+    cli,
+    utils,
+)
+from livekit.plugins import deepgram, openai, silero
+
+try:
+    import librosa
+except ImportError:
+    raise ImportError(
+        "librosa is required to run this example, install it with `pip install librosa`"
+    ) from None
+
+# uncomment to enable Krisp background voice/noise cancellation
path: examples/voice_agents/process_output_audio.py
user: login: longcw
body: updated
created_at: 2025-04-07T07:27:44Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-07T05:49:47Z
body: <!-- changeset-checker -->
### ✅ Changeset File Detected

The following changeset entries were found:

- `patch` - `livekit-agents`

**Change description:**
add realtime audio node (#1910)
author_association: CONTRIBUTOR
user: login: theomonnom
created_at: 2025-04-07T12:59:08Z
body: Can you change the example name to smtgh like "speed up" audio
author_association: MEMBER