Repository: livekit/agents
Pull request #1695: add turn detection options for agent 1.0
Author: login: longcw
Created: 2025-03-21T08:25:12Z | Merged: 2025-03-24T07:21:28Z

+ turn detection options
+ realtime API with local VAD example
+ push to talk example
+ support use custom STT with realtime model for user transcription
+ support EOU model with realtime model


Diff:
diff --git a/examples/push_to_talk.py b/examples/push_to_talk.py
new file mode 100644
index 0000000000..6f3a209b02
--- /dev/null
+++ b/examples/push_to_talk.py
@@ -0,0 +1,61 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import Agent, AgentSession, JobContext, RoomIO, WorkerOptions, cli
+from livekit.agents.llm import ChatContext, ChatMessage
+from livekit.plugins import cartesia, deepgram, openai
+
+logger = logging.getLogger("push-to-talk-example")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+class MyAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are a helpful assistant.",
+            stt=deepgram.STT(),
+            llm=openai.LLM(model="gpt-4o-mini"),
+            tts=cartesia.TTS(),
+        )
+
+    async def on_end_of_turn(
+        self, chat_ctx: ChatContext, new_message: ChatMessage, generating_reply: bool
+    ) -> None:
+        # callback when user input is transcribed
+        chat_ctx = chat_ctx.copy()
+        chat_ctx.items.append(new_message)
+        await self.update_chat_ctx(chat_ctx)
+        logger.info("add user message to chat context", extra={"content": new_message.content})
+
+
+async def entrypoint(ctx: JobContext):
+    await ctx.connect()
+
+    session = AgentSession(turn_detection="manual")
+    room_io = RoomIO(session, room=ctx.room)
+    await room_io.start()
+    await session.start(agent=MyAgent())
+
+    # disable input audio at the start
+    session.input.set_audio_enabled(False)
+
+    @ctx.room.local_participant.register_rpc_method("start_turn")
+    async def start_turn(data: rtc.RpcInvocationData):
+        session.interrupt()
+
+        # listen to the caller if multi-user
+        room_io.set_participant(data.caller_identity)
+        session.input.set_audio_enabled(True)
+
+    @ctx.room.local_participant.register_rpc_method("end_turn")
+    async def end_turn(data: rtc.RpcInvocationData):
+        session.input.set_audio_enabled(False)
+        session.generate_reply()
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))
diff --git a/examples/vad_realtime_example.py b/examples/vad_realtime_example.py
new file mode 100644
index 0000000000..4f9f7c8144
--- /dev/null
+++ b/examples/vad_realtime_example.py
@@ -0,0 +1,55 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit.agents import Agent, AgentSession, JobContext, JobProcess, WorkerOptions, cli, llm
+from livekit.plugins import deepgram, openai, silero
+from livekit.plugins.turn_detector import EOUModel
+
+logger = logging.getLogger("vad-realtime-example")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+class MyAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Alloy.",
+            llm=openai.realtime.RealtimeModel(
+                voice="alloy", turn_detection=None, input_audio_transcription=None
+            ),
+        )
+
+    async def on_enter(self):
+        self.session.generate_reply()
+
+    @llm.function_tool
+    async def fetch_weather_today(self) -> str:
+        """Called when the user asks for the weather today"""
+        return "The weather today is sunny and 70 degrees."
+
+    @llm.function_tool
+    async def fetch_weather_tomorrow(self) -> str:
+        """Called when the user asks for the weather tomorrow"""
+        return "The weather tomorrow is rainy and 60 degrees."
+
+
+async def entrypoint(ctx: JobContext):
+    await ctx.connect()
+
+    session = AgentSession(
+        turn_detection=EOUModel(),
+        vad=ctx.proc.userdata["vad"],
+        stt=deepgram.STT(),
+        allow_interruptions=True,
+    )
+    await session.start(agent=MyAgent(), room=ctx.room)
+
+
+def prewarm(proc: JobProcess):
+    proc.userdata["vad"] = silero.VAD.load()
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint, prewarm_fnc=prewarm))
diff --git a/livekit-agents/livekit/agents/llm/chat_context.py b/livekit-agents/livekit/agents/llm/chat_context.py
index ced190ae84..90edebcddd 100644
--- a/livekit-agents/livekit/agents/llm/chat_context.py
+++ b/livekit-agents/livekit/agents/llm/chat_context.py
@@ -24,6 +24,7 @@
 from livekit.agents.utils.misc import is_given
 
 from .. import utils
+from ..log import logger
 
 
 class ImageContent(BaseModel):
@@ -194,17 +195,22 @@ def from_dict(cls, data: dict) -> ChatContext:
         items = item_adapter.validate_python(data["items"])
         return cls(items)
 
+    @property
+    def readonly(self) -> bool:
+        return False
+
 
 class _ReadOnlyChatContext(ChatContext):
     """A read-only wrapper for ChatContext that prevents modifications."""
 
     error_msg = (
-        "This is a read-only reference to the chat context. "
-        "Please use .copy() method to make a mutable copy."
+        "trying to modify a read-only chat context, "
+        "please use .copy() and agent.update_chat_ctx() to modify the chat context"
     )
 
     class _ImmutableList(list):
         def _raise_error(self, *args, **kwargs):
+            logger.error(_ReadOnlyChatContext.error_msg)
             raise RuntimeError(_ReadOnlyChatContext.error_msg)
 
         # override all mutating methods to raise errors
@@ -216,3 +222,7 @@ def copy(self):
 
     def __init__(self, items: list[ChatItem]):
         self._items = self._ImmutableList(items)
+
+    @property
+    def readonly(self) -> bool:
+        return True
diff --git a/livekit-agents/livekit/agents/llm/realtime.py b/livekit-agents/livekit/agents/llm/realtime.py
index 0d2c0fa88a..230a11043f 100644
--- a/livekit-agents/livekit/agents/llm/realtime.py
+++ b/livekit-agents/livekit/agents/llm/realtime.py
@@ -48,6 +48,7 @@ class ErrorEvent:
 class RealtimeCapabilities:
     message_truncation: bool
     turn_detection: bool
+    user_transcription: bool
 
 
 class RealtimeError(Exception):
@@ -130,6 +131,10 @@ def generate_reply(
         instructions: NotGivenOr[str] = NOT_GIVEN,
     ) -> asyncio.Future[GenerationCreatedEvent]: ...  # can raise RealtimeError on Timeout
 
+    # commit the input audio buffer to the server
+    @abstractmethod
+    def commit_input_audio(self) -> None: ...
+
     # cancel the current generation (do nothing if no generation is in progress)
     @abstractmethod
     def interrupt(self) -> None: ...
diff --git a/livekit-agents/livekit/agents/voice/agent.py b/livekit-agents/livekit/agents/voice/agent.py
index 61d5360866..b1943ee85a 100644
--- a/livekit-agents/livekit/agents/voice/agent.py
+++ b/livekit-agents/livekit/agents/voice/agent.py
@@ -253,7 +253,9 @@ async def on_exit(self) -> None:
         """Called when the task is exited"""
         pass
 
-    async def on_end_of_turn(self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage) -> None:
+    async def on_end_of_turn(
+        self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage, generating_reply: bool
+    ) -> None:
         """Called when the user has finished speaking, and the LLM is about to respond
 
         This is a good opportunity to update the chat context or edit the new message before it is
diff --git a/livekit-agents/livekit/agents/voice/agent_activity.py b/livekit-agents/livekit/agents/voice/agent_activity.py
index 2c9f08ee59..a303c90283 100644
--- a/livekit-agents/livekit/agents/voice/agent_activity.py
+++ b/livekit-agents/livekit/agents/voice/agent_activity.py
@@ -14,6 +14,7 @@
 from ..metrics import AgentMetrics
 from ..types import NOT_GIVEN, AgentState, NotGivenOr
 from ..utils.misc import is_given
+from .agent import Agent, ModelSettings
 from .audio_recognition import AudioRecognition, RecognitionHooks, _TurnDetector
 from .events import (
     MetricsCollectedEvent,
@@ -41,8 +42,6 @@ def log_event(event: str, **kwargs) -> None:
     debug.Tracing.log_event(event, kwargs)
 
 
-from .agent import Agent, ModelSettings  # noqa: E402
-
 if TYPE_CHECKING:
     from .agent_session import AgentSession
 
@@ -74,17 +73,79 @@ def __init__(self, agent: Agent, sess: AgentSession) -> None:
 
         from .. import llm as large_language_model
 
-        if (
-            isinstance(self.llm, large_language_model.RealtimeModel)
-            and self.llm.capabilities.turn_detection
-            and not self.allow_interruptions
+        self._turn_detection_mode = self._session.turn_detection_mode
+
+        if self._turn_detection_mode == "vad" and not self.vad:
+            logger.warning("turn_detection is set to 'vad', but no VAD model is provided")
+            self._turn_detection_mode = None
+
+        if self._turn_detection_mode == "stt" and (
+            not self.stt or not self.stt.capabilities.streaming
         ):
-            raise ValueError(
-                "the RealtimeModel uses a server-side turn detection, allow_interruptions cannot be False, "  # noqa: E501
-                "disable turn_detection in the RealtimeModel and use VAD on the AgentTask/VoiceAgent instead"  # noqa: E501
+            logger.warning(
+                "turn_detection is set to 'stt', but no STT model is provided or the STT model "
+                "does not support streaming, ignoring the turn_detection setting"
             )
+            self._turn_detection_mode = None
+
+        if isinstance(self.llm, large_language_model.RealtimeModel):
+            if self.llm.capabilities.turn_detection and not self.allow_interruptions:
+                raise ValueError(
+                    "the RealtimeModel uses a server-side turn detection, "
+                    "allow_interruptions cannot be False, disable turn_detection in "
+                    "the RealtimeModel and use VAD on the AgentSession instead"
+                )
 
-        if not self.vad and self.stt and isinstance(self.llm, llm.LLM) and self.allow_interruptions:
+            if (
+                self._turn_detection_mode == "realtime_llm"
+                and not self.llm.capabilities.turn_detection
+            ):
+                logger.warning(
+                    "turn_detection is set to 'realtime_llm', but the LLM is not a RealtimeModel "
+                    "or the server-side turn detection is not supported/enabled, "
+                    "ignoring the turn_detection setting"
+                )
+                self._turn_detection_mode = None
+
+            if self._turn_detection_mode == "stt":
+                logger.warning(
+                    "turn_detection is set to 'stt', but the LLM is a RealtimeModel, "
+                    "ignoring the turn_detection setting"
+                )
+                self._turn_detection_mode = None
+
+            elif (
+                self._turn_detection_mode
+                and self._turn_detection_mode != "realtime_llm"
+                and self.llm.capabilities.turn_detection
+            ):
+                logger.warning(
+                    f"turn_detection is set to '{self._turn_detection_mode}', but the LLM "
+                    "is a RealtimeModel and server-side turn detection enabled, "
+                    "ignoring the turn_detection setting"
+                )
+                self._turn_detection_mode = None
+
+            # fallback to VAD if server side turn detection is disabled and VAD is available
+            if (
+                not self.llm.capabilities.turn_detection
+                and self.vad
+                and self._turn_detection_mode is None
+            ):
+                self._turn_detection_mode = "vad"
+        elif self._turn_detection_mode == "realtime_llm":
+            logger.warning(
+                "turn_detection is set to 'realtime_llm', but the LLM is not a RealtimeModel"
+            )
+            self._turn_detection_mode = None
+
+        if (
+            not self.vad
+            and self.stt
+            and isinstance(self.llm, llm.LLM)
+            and self.allow_interruptions
+            and self._turn_detection_mode is None
+        ):
             logger.warning(
                 "VAD is not set. Enabling VAD is recommended when using LLM and STT "
                 "for more responsive interruption handling."
@@ -242,14 +303,15 @@ async def start(self) -> None:
             if isinstance(self.tts, tts.TTS):
                 self.tts.on("metrics_collected", self._on_metrics_collected)
 
-            if isinstance(self.vad, vad.VAD):
-                self.vad.on("metrics_collected", self._on_metrics_collected)
+            agent_vad = self.vad if self._turn_detection_mode in ("vad", None) else None
+            if isinstance(agent_vad, vad.VAD):
+                agent_vad.on("metrics_collected", self._on_metrics_collected)
 
             self._main_atask = asyncio.create_task(self._main_task(), name="_main_task")
             self._audio_recognition = AudioRecognition(
                 hooks=self,
                 stt=self._agent.stt_node if self.stt else None,
-                vad=self.vad,
+                vad=agent_vad,
                 turn_detector=self.turn_detector,
                 min_endpointing_delay=self._session.options.min_endpointing_delay,
             )
@@ -295,6 +357,7 @@ def push_audio(self, frame: rtc.AudioFrame) -> None:
             return
 
         if self._current_speech and not self._current_speech.allow_interruptions:
+            # TODO(long): make this optional if user want to save the transcript for later response?
             # drop the audio if the current speech is not interruptable
             return
 
@@ -529,6 +592,9 @@ def on_end_of_speech(self, ev: vad.VADEvent) -> None:
         self._session.emit("user_stopped_speaking", UserStoppedSpeakingEvent())
 
     def on_vad_inference_done(self, ev: vad.VADEvent) -> None:
+        if self._turn_detection_mode == "manual":
+            return
+
         if ev.speech_duration > self._session.options.min_interruption_duration:
             if (
                 self._current_speech is not None
@@ -542,12 +608,20 @@ def on_vad_inference_done(self, ev: vad.VADEvent) -> None:
                 self._current_speech.interrupt()
 
     def on_interim_transcript(self, ev: stt.SpeechEvent) -> None:
+        if isinstance(self.llm, llm.RealtimeModel) and self.llm.capabilities.user_transcription:
+            # skip stt transcription if user_transcription is enabled on the realtime model
+            return
+
         self._session.emit(
             "user_input_transcribed",
             UserInputTranscribedEvent(transcript=ev.alternatives[0].text, is_final=False),
         )
 
     def on_final_transcript(self, ev: stt.SpeechEvent) -> None:
+        if isinstance(self.llm, llm.RealtimeModel) and self.llm.capabilities.user_transcription:
+            # skip stt transcription if user_transcription is enabled on the realtime model
+            return
+
         self._session.emit(
             "user_input_transcribed",
             UserInputTranscribedEvent(transcript=ev.alternatives[0].text, is_final=True),
@@ -555,17 +629,37 @@ def on_final_transcript(self, ev: stt.SpeechEvent) -> None:
 
     async def on_end_of_turn(self, new_transcript: str) -> None:
         # When the audio recognition detects the end of a user turn:
+        #  - check if the turn detection mode is set to "manual"
+        #  - check if realtime model server-side turn detection is enabled
         #  - check if there is no current generation happening
         #  - cancel the current generation if it allows interruptions (otherwise skip this current
         #  turn)
         #  - generate a reply to the user input
 
+        user_message = llm.ChatMessage(role="user", content=[new_transcript])
+        if self._turn_detection_mode == "manual":
+            await self._agent.on_end_of_turn(
+                self._agent.chat_ctx, user_message, generating_reply=False
+            )
+            return
+
+        if isinstance(self.llm, llm.RealtimeModel) and self._rt_session is not None:
+            if self.llm.capabilities.turn_detection:
+                return
+            # ignore stt transcription for realtime model and commit the audio buffer
+            new_transcript = ""
+            self._rt_session.commit_input_audio()
+
         if self._current_speech is not None:
             if not self._current_speech.allow_interruptions:
                 logger.warning(
-                    "skipping user input, current speech generation cannot be interrupted",
+                    "skipping reply to user input, current speech generation cannot be interrupted",
                     extra={"user_input": new_transcript},
                 )
+                # in agent.on_end_of_turn user can ignore the message or save it for later
+                await self._agent.on_end_of_turn(
+                    self._agent.chat_ctx, user_message, generating_reply=False
+                )
                 return
 
             log_event(
@@ -573,6 +667,8 @@ async def on_end_of_turn(self, new_transcript: str) -> None:
                 speech_id=self._current_speech.id,
             )
             self._current_speech.interrupt()
+            if self._rt_session is not None:
+                self._rt_session.interrupt()
 
         if self.draining:
             logger.warning(
@@ -587,9 +683,8 @@ async def on_end_of_turn(self, new_transcript: str) -> None:
 
         await self._agent.on_end_of_turn(
             self._agent.chat_ctx,
-            llm.ChatMessage(
-                role="user", content=[new_transcript]
-            ),  # TODO(theomonnom): This doesn't allow edits yet
+            new_message=user_message,  # TODO(theomonnom): This doesn't allow edits yet
+            generating_reply=True,
         )
 
         if self.draining:
diff --git a/livekit-agents/livekit/agents/voice/agent_session.py b/livekit-agents/livekit/agents/voice/agent_session.py
index 3ff0fbca58..0619d0c72d 100644
--- a/livekit-agents/livekit/agents/voice/agent_session.py
+++ b/livekit-agents/livekit/agents/voice/agent_session.py
@@ -4,7 +4,7 @@
 import copy
 from collections.abc import AsyncIterable
 from dataclasses import dataclass
-from typing import Generic, TypeVar
+from typing import Generic, Literal, TypeVar
 
 from livekit import rtc
 
@@ -32,12 +32,27 @@ class VoiceOptions:
 
 Userdata_T = TypeVar("Userdata_T")
 
+TurnDetectionMode = Literal["stt", "vad", "realtime_llm", "manual"]
+"""
+The mode of turn detection to use.
+
+- "stt": use speech-to-text result to detect the end of the user's turn
+- "vad": use VAD to detect the start and end of the user's turn
+- "realtime_llm": use server-side turn detection provided by the realtime LLM
+- "manual": manually manage the turn detection
+- (default) None or not provided: automatically choose the best mode
+    based on available models (realtime_llm -> vad -> stt -> manual)
+
+If the model (VAD, STT, or RealtimeModel) is not provided, fallback to the default mode.
+If a _TurnDetector is provided, use the default mode with the provided turn detector.
+"""
+
 
 class AgentSession(rtc.EventEmitter[EventTypes], Generic[Userdata_T]):
     def __init__(
         self,
         *,
-        turn_detector: NotGivenOr[_TurnDetector] = NOT_GIVEN,
+        turn_detection: NotGivenOr[TurnDetectionMode | _TurnDetector | None] = NOT_GIVEN,
         stt: NotGivenOr[stt.STT] = NOT_GIVEN,
         vad: NotGivenOr[vad.VAD] = NOT_GIVEN,
         llm: NotGivenOr[llm.LLM | llm.RealtimeModel] = NOT_GIVEN,
@@ -61,7 +76,13 @@ def __init__(
             max_tool_steps=max_tool_steps,
         )
         self._started = False
-        self._turn_detector = turn_detector or None
+        self._turn_detection_mode: TurnDetectionMode | None = None
+        self._turn_detector: _TurnDetector | None = None
+        if isinstance(turn_detection, str):
+            self._turn_detection_mode = turn_detection
+        else:
+            self._turn_detector = turn_detection or None
+
         self._stt = stt or None
         self._vad = vad or None
         self._llm = llm or None
@@ -101,6 +122,10 @@ def userdata(self) -> Userdata_T:
     def userdata(self, value: Userdata_T) -> None:
         self._userdata = value
 
+    @property
+    def turn_detection_mode(self) -> TurnDetectionMode | None:
+        return self._turn_detection_mode
+
     @property
     def turn_detector(self) -> _TurnDetector | None:
         return self._turn_detector
diff --git a/livekit-agents/livekit/agents/voice/audio_recognition.py b/livekit-agents/livekit/agents/voice/audio_recognition.py
index bfcf444fcd..83b5f59966 100644
--- a/livekit-agents/livekit/agents/voice/audio_recognition.py
+++ b/livekit-agents/livekit/agents/voice/audio_recognition.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import time
 from collections.abc import AsyncIterable
 from typing import Protocol
 
@@ -56,6 +57,7 @@ def __init__(
         self._vad = vad
 
         self._speaking = False
+        self._last_speaking_time: float = 0
         self._audio_transcript = ""
         self._last_language: str | None = None
         self._vad_graph = tracing.Tracing.add_graph(
@@ -123,6 +125,7 @@ async def _on_stt_event(self, ev: stt.SpeechEvent) -> None:
         if ev.type == stt.SpeechEventType.FINAL_TRANSCRIPT:
             self._hooks.on_final_transcript(ev)
             transcript = ev.alternatives[0].text
+            self._last_language = ev.alternatives[0].language
             if not transcript:
                 return
 
@@ -143,6 +146,10 @@ async def _on_stt_event(self, ev: stt.SpeechEvent) -> None:
             self._audio_transcript = self._audio_transcript.lstrip()
 
             if not self._speaking:
+                if not self._vad:
+                    # vad disabled, use stt timestamp
+                    self._last_speaking_time = time.time()
+
                 chat_ctx = self._hooks.retrieve_chat_ctx().copy()
                 self._run_eou_detection(chat_ctx)
         elif ev.type == stt.SpeechEventType.INTERIM_TRANSCRIPT:
@@ -163,23 +170,24 @@ async def _on_vad_event(self, ev: vad.VADEvent) -> None:
         elif ev.type == vad.VADEventType.END_OF_SPEECH:
             self._hooks.on_end_of_speech(ev)
             self._speaking = False
+            self._last_speaking_time = time.time()
 
-            if not self._speaking:
-                chat_ctx = self._hooks.retrieve_chat_ctx().copy()
-                self._run_eou_detection(chat_ctx)
+            chat_ctx = self._hooks.retrieve_chat_ctx().copy()
+            self._run_eou_detection(chat_ctx)
 
     def _run_eou_detection(self, chat_ctx: llm.ChatContext) -> None:
-        if not self._audio_transcript:
+        if self._stt and not self._audio_transcript:
+            # stt enabled but no transcript yet
             return
 
-        # TODO
-        # chat_ctx = self._agent._chat_ctx.copy()
-        # chat_ctx.append(role="user", text=self._audio_transcript)
-        turn_detector = self._turn_detector
+        chat_ctx = chat_ctx.copy()
+        chat_ctx.add_message(role="user", content=self._audio_transcript)
+        turn_detector = self._turn_detector if self._audio_transcript else None
 
         @utils.log_exceptions(logger=logger)
         async def _bounce_eou_task() -> None:
-            await asyncio.sleep(self._min_endpointing_delay)
+            delay = max(self._last_speaking_time + self._min_endpointing_delay - time.time(), 0)
+            await asyncio.sleep(delay)
 
             if turn_detector is not None and turn_detector.supports_language(self._last_language):
                 end_of_turn_probability = await turn_detector.predict_end_of_turn(chat_ctx)
@@ -188,8 +196,14 @@ async def _bounce_eou_task() -> None:
                     {"probability": end_of_turn_probability},
                 )
                 unlikely_threshold = turn_detector.unlikely_threshold()
-                if end_of_turn_probability > unlikely_threshold:
-                    await asyncio.sleep(self.UNLIKELY_END_OF_TURN_EXTRA_DELAY)
+                if end_of_turn_probability < unlikely_threshold:
+                    delay = max(
+                        self._last_speaking_time
+                        + self.UNLIKELY_END_OF_TURN_EXTRA_DELAY
+                        - time.time(),
+                        0,
+                    )
+                    await asyncio.sleep(delay)
 
             tracing.Tracing.log_event("end of user turn", {"transcript": self._audio_transcript})
             await self._hooks.on_end_of_turn(self._audio_transcript)
diff --git a/livekit-agents/livekit/agents/voice/room_io/_output.py b/livekit-agents/livekit/agents/voice/room_io/_output.py
index c3b3189a82..fd2828d09a 100644
--- a/livekit-agents/livekit/agents/voice/room_io/_output.py
+++ b/livekit-agents/livekit/agents/voice/room_io/_output.py
@@ -31,8 +31,10 @@ def __init__(
         self._audio_source = rtc.AudioSource(sample_rate, num_channels, queue_size_ms)
         self._publish_options = track_publish_options
         self._publication: rtc.LocalTrackPublication | None = None
+
         self._republish_task: asyncio.Task | None = None  # used to republish track on reconnection
         self._flush_task: asyncio.Task | None = None
+        self._interrupted_event = asyncio.Event()
 
         self._pushed_duration: float = 0.0
         self._interrupted: bool = False
@@ -76,25 +78,35 @@ def flush(self) -> None:
             logger.error("flush called while playback is in progress")
             self._flush_task.cancel()
 
-        def _playback_finished(task: asyncio.Task[None]) -> None:
-            pushed_duration, interrupted = self._pushed_duration, self._interrupted
-            self._pushed_duration = 0
-            self._interrupted = False
-            self.on_playback_finished(playback_position=pushed_duration, interrupted=interrupted)
-
-        self._flush_task = asyncio.create_task(self._audio_source.wait_for_playout())
-        self._flush_task.add_done_callback(_playback_finished)
+        self._flush_task = asyncio.create_task(self._wait_for_playout())
 
     def clear_buffer(self) -> None:
         super().clear_buffer()
-
         if not self._pushed_duration:
             return
+        self._interrupted_event.set()
+
+    async def _wait_for_playout(self) -> None:
+        wait_for_interruption = asyncio.create_task(self._interrupted_event.wait())
+        wait_for_playout = asyncio.create_task(self._audio_source.wait_for_playout())
+        await asyncio.wait(
+            [wait_for_playout, wait_for_interruption],
+            return_when=asyncio.FIRST_COMPLETED,
+        )
+
+        interrupted = wait_for_interruption.done()
+        pushed_duration = self._pushed_duration
+
+        if interrupted:
+            pushed_duration = max(pushed_duration - self._audio_source.queued_duration, 0)
+            self._audio_source.clear_queue()
+            wait_for_playout.cancel()
+        else:
+            wait_for_interruption.cancel()
 
-        queued_duration = self._audio_source.queued_duration
-        self._pushed_duration = max(self._pushed_duration - queued_duration, 0)
-        self._interrupted = True
-        self._audio_source.clear_queue()
+        self._pushed_duration = 0
+        self._interrupted_event.clear()
+        self.on_playback_finished(playback_position=pushed_duration, interrupted=interrupted)
 
 
 class _ParticipantLegacyTranscriptionOutput(io.TextOutput):
diff --git a/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py b/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
index 7689779dd9..0962876f3a 100644
--- a/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
+++ b/livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
@@ -7,7 +7,7 @@
 import os
 import weakref
 from collections.abc import Iterator
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from typing import Literal, Union
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
@@ -29,6 +29,7 @@
     ConversationItemTruncateEvent,
     ErrorEvent,
     InputAudioBufferAppendEvent,
+    InputAudioBufferCommitEvent,
     InputAudioBufferSpeechStartedEvent,
     InputAudioBufferSpeechStoppedEvent,
     RealtimeClientEvent,
@@ -74,7 +75,18 @@ class _InputAudioTranscription:
     model: Literal["whisper-1"] = "whisper-1"
 
 
+@dataclass
+class _TurnDetection:
+    type: Literal["server_vad"] = "server_vad"
+    create_response: bool | None = True
+    interrupt_response: bool | None = True
+    prefix_padding_ms: int | None = 300
+    silence_duration_ms: int | None = 500
+    threshold: float | None = 0.65
+
+
 DEFAULT_INPUT_AUDIO_TRANSCRIPTION = _InputAudioTranscription()
+DEFAULT_TURN_DETECTION = _TurnDetection()
 
 
 @dataclass
@@ -82,6 +94,7 @@ class _RealtimeOptions:
     model: str
     voice: str
     input_audio_transcription: _InputAudioTranscription | None
+    turn_detection: _TurnDetection | None
     api_key: str
     base_url: str
 
@@ -110,12 +123,16 @@ def __init__(
         base_url: NotGivenOr[str] = NOT_GIVEN,
         input_audio_transcription: _InputAudioTranscription
         | None = DEFAULT_INPUT_AUDIO_TRANSCRIPTION,
+        turn_detection: _TurnDetection | None = DEFAULT_TURN_DETECTION,
         api_key: str | None = None,
         http_session: aiohttp.ClientSession | None = None,
     ) -> None:
         super().__init__(
-            # TODO(theomonnom): add a way to disable turn detection (And use VAD on the VoiceAgent)
-            capabilities=llm.RealtimeCapabilities(message_truncation=True, turn_detection=True)
+            capabilities=llm.RealtimeCapabilities(
+                message_truncation=True,
+                turn_detection=turn_detection is not None,
+                user_transcription=input_audio_transcription is not None,
+            )
         )
 
         api_key = api_key or os.environ.get("OPENAI_API_KEY")
@@ -130,6 +147,7 @@ def __init__(
             model=model,
             voice=voice,
             input_audio_transcription=input_audio_transcription,
+            turn_detection=turn_detection,
             api_key=api_key,
             base_url=base_url,
         )
@@ -215,6 +233,7 @@ def __init__(self, realtime_model: RealtimeModel) -> None:
             NUM_CHANNELS,
             samples_per_channel=SAMPLE_RATE // 10,  # 100ms
         )
+        self._pushed_duration_s = 0  # duration of audio pushed to the OpenAI Realtime API
 
     def send_event(self, event: RealtimeClientEvent | dict) -> None:
         with contextlib.suppress(utils.aio.channel.ChanClosed):
@@ -248,7 +267,7 @@ async def _send_task() -> None:
                 try:
                     if isinstance(msg, BaseModel):
                         msg = msg.model_dump(
-                            by_alias=True, exclude_unset=True, exclude_defaults=True
+                            by_alias=True, exclude_unset=True, exclude_defaults=False
                         )
 
                     self.emit("openai_client_event_queued", msg)
@@ -351,6 +370,12 @@ async def _recv_task() -> None:
                 model=self._realtime_model._opts.input_audio_transcription.model,
             )
 
+        turn_detection: session_update_event.SessionTurnDetection | None = None
+        if self._realtime_model._opts.turn_detection:
+            turn_detection = session_update_event.SessionTurnDetection.model_validate(
+                asdict(self._realtime_model._opts.turn_detection)
+            )
+
         # initial session update
         self.send_event(
             SessionUpdateEvent(
@@ -363,14 +388,7 @@ async def _recv_task() -> None:
                     input_audio_format="pcm16",
                     output_audio_format="pcm16",
                     modalities=["text", "audio"],
-                    turn_detection=session_update_event.SessionTurnDetection(
-                        type="server_vad",
-                        create_response=True,
-                        interrupt_response=True,
-                        prefix_padding_ms=300,
-                        silence_duration_ms=500,
-                        threshold=0.65,
-                    ),
+                    turn_detection=turn_detection,
                     input_audio_transcription=input_audio_transcription,
                 ),
                 event_id=utils.shortuuid("session_update_"),
@@ -518,6 +536,7 @@ def push_audio(self, frame: rtc.AudioFrame) -> None:
                         audio=base64.b64encode(f.data).decode("utf-8"),
                     )
                 )
+                self._pushed_duration_s += f.duration
 
     def generate_reply(
         self, *, instructions: NotGivenOr[str] = NOT_GIVEN
@@ -557,6 +576,11 @@ def truncate(self, *, message_id: str, audio_end_ms: int) -> None:
             )
         )
 
+    def commit_input_audio(self) -> None:
+        if self._pushed_duration_s > 0.1:  # OpenAI requires at least 100ms of audio
+            self.send_event(InputAudioBufferCommitEvent(type="input_audio_buffer.commit"))
+            self._pushed_duration_s = 0
+
     async def aclose(self) -> None:
         self._msg_ch.close()
         await self._main_atask
diff --git a/livekit-plugins/livekit-plugins-turn-detector/livekit/plugins/turn_detector/eou.py b/livekit-plugins/livekit-plugins-turn-detector/livekit/plugins/turn_detector/eou.py
index b6c8b43728..5cf9e607b7 100644
--- a/livekit-plugins/livekit-plugins-turn-detector/livekit/plugins/turn_detector/eou.py
+++ b/livekit-plugins/livekit-plugins-turn-detector/livekit/plugins/turn_detector/eou.py
@@ -140,7 +140,7 @@ async def predict_end_of_turn(
     ) -> float:
         messages = []
 
-        for msg in chat_ctx.messages:
+        for msg in chat_ctx.items:
             if msg.role not in ("user", "assistant"):
                 continue
 


Reviews:
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T08:27:05Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T15:28:10Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T15:28:34Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T15:29:17Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T15:37:48Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T15:47:10Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T15:49:14Z
user: login: lukasIO
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T15:54:31Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T21:55:51Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T21:58:59Z
user: login: lukasIO
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T01:55:45Z
user: login: lukasIO
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T01:59:34Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T03:20:36Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T05:35:11Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-22T16:54:28Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-22T16:55:09Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-22T16:59:30Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T17:16:13Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-22T17:23:19Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T17:54:02Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-22T19:40:38Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-22T19:41:09Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-23T02:05:51Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-23T02:08:46Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-23T02:46:16Z
user: login: davidzhao
body: lgtm, only a few nits
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-24T04:49:51Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T04:54:06Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T05:07:43Z
user: login: davidzhao
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T05:10:35Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T11:59:09Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:01:52Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:02:06Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:02:20Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T12:03:21Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:05:54Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:10:44Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:14:05Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T12:16:12Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:19:51Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T12:20:31Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T12:29:54Z
user: login: theomonnom
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T13:30:27Z
user: login: longcw
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T14:40:44Z

Review comments:
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: longcw
body: @theomonnom wdyt about make `agent.chat_ctx` mutable for regular LLM?
created_at: 2025-03-21T08:27:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -191,6 +191,10 @@ def from_dict(cls, data: dict) -> ChatContext:
         items = item_adapter.validate_python(data["items"])
         return cls(items)
 
+    @property
+    def mutable(self) -> bool:
path: livekit-agents/livekit/agents/llm/chat_context.py
user: login: theomonnom
body: let's rename to `readonly`, it's more common IMO
created_at: 2025-03-21T15:28:10Z
author_association: MEMBER
diff_hunk: @@ -63,6 +63,10 @@ def __init__(self, *, capabilities: RealtimeCapabilities) -> None:
     def capabilities(self) -> RealtimeCapabilities:
         return self._capabilities
 
+    @property
+    @abstractmethod
+    def server_side_turn_detection(self) -> bool: ...
path: livekit-agents/livekit/agents/llm/realtime.py
user: login: theomonnom
body: No need for this function, we already expose it via the capabilities, if turn_detection is True, it means it's being used
created_at: 2025-03-21T15:28:34Z
author_association: MEMBER
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: theomonnom
body: I understand why we want to do it, but I would not create major differences between both today. This may be very confusing when switching an agent using RealtimeLLM to a stateless LLM. The code will break
created_at: 2025-03-21T15:29:17Z
author_association: MEMBER
diff_hunk: @@ -63,6 +63,10 @@ def __init__(self, *, capabilities: RealtimeCapabilities) -> None:
     def capabilities(self) -> RealtimeCapabilities:
         return self._capabilities
 
+    @property
+    @abstractmethod
+    def server_side_turn_detection(self) -> bool: ...
path: livekit-agents/livekit/agents/llm/realtime.py
user: login: longcw
body: So should we change the `capabilities` based on the args of Realtime model? I though it was fixed for a type of model.
created_at: 2025-03-21T15:37:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: longcw
body: I found it's a little bit annoying and inefficient if we make `chat_ctx` of regular LLM readonly, like in this example https://github.com/livekit/agents/pull/1695/files#diff-88ec3d07b4e65630ae21331f8576f5f677e9d5ad4a52e0e8d15c33b573b81846R35-R36

user should know what LLM is using in an agent for most of the case, or they can use `chat_ctx.readonly` flag to use different methods if it's for both RealtimeModel and LLM.

```python
        chat_ctx = chat_ctx.copy()
        chat_ctx.items.append(new_message)
        self.update_chat_ctx(chat_ctx)
        # we copied the chat_ctx 3 times here
 
        # or 
        assert chat_ctx.mutable  # asset not chat_ctx.readonly
        chat_ctx.items.append(new_message)
```

But I don't have a strong opinion here, maybe we wait for the future. But I think there will be a lot questions about that, @lukasIO just met this issue that modifying the readonly chat_ctx failed silently because it was in a async method... So maybe at least it's better to show the error message in a log then raise the error. 
created_at: 2025-03-21T15:47:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: longcw
body: but anyway, raise the error is better than allow it silently, which is much hard to find lol
created_at: 2025-03-21T15:49:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: lukasIO
body: how are users expected to use `conversation_item_added` event effectively when they need to reconstruct the whole chat ctx each time they update it 🤔 
created_at: 2025-03-21T15:54:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -63,6 +63,10 @@ def __init__(self, *, capabilities: RealtimeCapabilities) -> None:
     def capabilities(self) -> RealtimeCapabilities:
         return self._capabilities
 
+    @property
+    @abstractmethod
+    def server_side_turn_detection(self) -> bool: ...
path: livekit-agents/livekit/agents/llm/realtime.py
user: login: theomonnom
body: Since you can't access the capabilities statically, I took the decision to make them dynamic. So it is indeed stateful
created_at: 2025-03-21T21:55:51Z
author_association: MEMBER
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: theomonnom
body: Where was the async method that was failing silently? We should absolutely log all errors from tasks we create. 

I think I would say to see that later, my opinion is that a code you see on our docs/github should directly be compatible with what you have locally, no matter what model you're using.

I think as a first step, we should maybe allow update_chat_ctx to not be necessarily `awaited`. But I think it's fine for v1.0? 

I wouldn't worry about the efficiency here, we only do shallow copies, it's very fast



We could also improve the pattern like:

```
self.update_chat_ctx(self.chat_ctx.copy().append("Hello world"))
```
created_at: 2025-03-21T21:58:59Z
author_association: MEMBER
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: lukasIO
body: I think a "blessed" way to hook into the chat ctx for a persistence layer should not be an afterthought but rather an integral part of 1.0. 

If it's not the `conversation_item_added` event, it's fine for it to be a custom text sink, but if that's the case, then we need all necessary bits exposed publicly
created_at: 2025-03-22T01:55:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: lukasIO
body: As I understand the main concern is a difference in behaviour between realtime model and regular llm. 

I guess we could just abstract that away for realtime models? where for realtime models a call to `add_message` would update the chat context with the copy? 
That seems more obvious than requiring to copy + update the chat ctx for all models in user land
created_at: 2025-03-22T01:59:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -70,15 +70,17 @@ def tools(self) -> list[llm.FunctionTool]:
     @property
     def chat_ctx(self) -> llm.ChatContext:
         """
-        Provides a read-only view of the agent's current chat context.
-
         Returns:
-            llm.ChatContext: A read-only version of the agent's conversation history.
+            llm.ChatContext: A read-only version of the agent's conversation history if the agent's
+            LLM is a RealtimeModel. Otherwise, it returns the internal mutable chat context.
 
         See Also:
             update_chat_ctx: Method to update the internal chat context.
         """
-        return _ReadOnlyChatContext(self._chat_ctx.items)
+        if isinstance(self.llm, llm.RealtimeModel):
+            return _ReadOnlyChatContext(self._chat_ctx.items)
+        else:
+            return self._chat_ctx
path: livekit-agents/livekit/agents/voice/agent.py
user: login: longcw
body: It may failed in user defined async function without log. But anyway I'll revert it for now to unblock this pr.
created_at: 2025-03-22T03:20:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -72,25 +75,35 @@ def flush(self) -> None:
             logger.error("flush called while playback is in progress")
             self._flush_task.cancel()
 
-        def _playback_finished(task: asyncio.Task[None]) -> None:
-            pushed_duration, interrupted = self._pushed_duration, self._interrupted
-            self._pushed_duration = 0
-            self._interrupted = False
-            self.on_playback_finished(playback_position=pushed_duration, interrupted=interrupted)
-
-        self._flush_task = asyncio.create_task(self._audio_source.wait_for_playout())
-        self._flush_task.add_done_callback(_playback_finished)
+        self._flush_task = asyncio.create_task(self._wait_for_playout())
 
     def clear_buffer(self) -> None:
         super().clear_buffer()
-
         if not self._pushed_duration:
             return
+        self._interrupted_event.set()
+
+    async def _wait_for_playout(self) -> None:
path: livekit-agents/livekit/agents/voice/room_io/_output.py
user: login: longcw
body: fixed the clear_buffer sometimes doesn't work
created_at: 2025-03-22T05:35:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -72,25 +75,35 @@ def flush(self) -> None:
             logger.error("flush called while playback is in progress")
             self._flush_task.cancel()
 
-        def _playback_finished(task: asyncio.Task[None]) -> None:
-            pushed_duration, interrupted = self._pushed_duration, self._interrupted
-            self._pushed_duration = 0
-            self._interrupted = False
-            self.on_playback_finished(playback_position=pushed_duration, interrupted=interrupted)
-
-        self._flush_task = asyncio.create_task(self._audio_source.wait_for_playout())
-        self._flush_task.add_done_callback(_playback_finished)
+        self._flush_task = asyncio.create_task(self._wait_for_playout())
 
     def clear_buffer(self) -> None:
         super().clear_buffer()
-
         if not self._pushed_duration:
             return
+        self._interrupted_event.set()
+
+    async def _wait_for_playout(self) -> None:
path: livekit-agents/livekit/agents/voice/room_io/_output.py
user: login: theomonnom
body: Is it related to the issue you pinged me on Linear? https://linear.app/livekit/issue/AGT-1043/audio-source-clear-queue-only-clears-the-first-second-of-audio#comment-9705354d
created_at: 2025-03-22T16:54:28Z
author_association: MEMBER
diff_hunk: @@ -48,6 +48,7 @@ class ErrorEvent:
 class RealtimeCapabilities:
     message_truncation: bool
     turn_detection: bool
+    speech_to_text: bool
path: livekit-agents/livekit/agents/llm/realtime.py
user: login: theomonnom
body: `user_transcription`?
created_at: 2025-03-22T16:55:08Z
author_association: MEMBER
diff_hunk: @@ -519,10 +536,15 @@ def push_audio(self, frame: rtc.AudioFrame) -> None:
                         audio=base64.b64encode(f.data).decode("utf-8"),
                     )
                 )
+                self._pushed_duration_s += f.duration
 
     def generate_reply(
         self, *, instructions: NotGivenOr[str] = NOT_GIVEN
     ) -> asyncio.Future[llm.GenerationCreatedEvent]:
+        if not self._realtime_model.capabilities.turn_detection and self._pushed_duration_s > 0:
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: Could this break our tool responses, where we add a new FunctionToolOutput item and call `generate_reply`.  Since push_audio is continuous, commit_audio will always be called, are there any side effects?
created_at: 2025-03-22T16:59:30Z
author_association: MEMBER
diff_hunk: @@ -72,25 +75,35 @@ def flush(self) -> None:
             logger.error("flush called while playback is in progress")
             self._flush_task.cancel()
 
-        def _playback_finished(task: asyncio.Task[None]) -> None:
-            pushed_duration, interrupted = self._pushed_duration, self._interrupted
-            self._pushed_duration = 0
-            self._interrupted = False
-            self.on_playback_finished(playback_position=pushed_duration, interrupted=interrupted)
-
-        self._flush_task = asyncio.create_task(self._audio_source.wait_for_playout())
-        self._flush_task.add_done_callback(_playback_finished)
+        self._flush_task = asyncio.create_task(self._wait_for_playout())
 
     def clear_buffer(self) -> None:
         super().clear_buffer()
-
         if not self._pushed_duration:
             return
+        self._interrupted_event.set()
+
+    async def _wait_for_playout(self) -> None:
path: livekit-agents/livekit/agents/voice/room_io/_output.py
user: login: longcw
body: not sure if it's related, but I do encountered an issue when testing VAD with realtime model that after `clear_queue` the audio was still playing for tens of seconds. I though it was some race conditions that after `clear_queue` there is new audio frames comes in so `wait_for_playout` didn't return.

But that's also weird, since before `clear_queue` we already interrupted the realtime API so there is no new audio came in, and the buffer of AudioSource is bigger enough we shouldn't have a lot of audio frames buffered somewhere before the `capture_frame` when calling the `clear_queue`.

The current solution fixed it and should be better than before.
created_at: 2025-03-22T17:16:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -249,7 +267,7 @@ async def _send_task() -> None:
                 try:
                     if isinstance(msg, BaseModel):
                         msg = msg.model_dump(
-                            by_alias=True, exclude_unset=True, exclude_defaults=True
+                            by_alias=True, exclude_unset=True, exclude_defaults=False
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: The exclude_defaults=True is actually from their SDK
created_at: 2025-03-22T17:23:19Z
author_association: MEMBER
diff_hunk: @@ -249,7 +267,7 @@ async def _send_task() -> None:
                 try:
                     if isinstance(msg, BaseModel):
                         msg = msg.model_dump(
-                            by_alias=True, exclude_unset=True, exclude_defaults=True
+                            by_alias=True, exclude_unset=True, exclude_defaults=False
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: I guess it's a bug, with that the disable turn detection won't be added to update as the default value in sdk is None while they create the session with turn detection enabled lol
created_at: 2025-03-22T17:54:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -249,7 +267,7 @@ async def _send_task() -> None:
                 try:
                     if isinstance(msg, BaseModel):
                         msg = msg.model_dump(
-                            by_alias=True, exclude_unset=True, exclude_defaults=True
+                            by_alias=True, exclude_unset=True, exclude_defaults=False
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: lol I see
created_at: 2025-03-22T19:40:37Z
author_association: MEMBER
diff_hunk: @@ -249,7 +267,7 @@ async def _send_task() -> None:
                 try:
                     if isinstance(msg, BaseModel):
                         msg = msg.model_dump(
-                            by_alias=True, exclude_unset=True, exclude_defaults=True
+                            by_alias=True, exclude_unset=True, exclude_defaults=False
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: seems like this is https://github.com/openai/openai-python/issues/2199
created_at: 2025-03-22T19:41:09Z
author_association: MEMBER
diff_hunk: @@ -519,10 +536,15 @@ def push_audio(self, frame: rtc.AudioFrame) -> None:
                         audio=base64.b64encode(f.data).decode("utf-8"),
                     )
                 )
+                self._pushed_duration_s += f.duration
 
     def generate_reply(
         self, *, instructions: NotGivenOr[str] = NOT_GIVEN
     ) -> asyncio.Future[llm.GenerationCreatedEvent]:
+        if not self._realtime_model.capabilities.turn_detection and self._pushed_duration_s > 0:
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: It's a good point, I'll do some tests
created_at: 2025-03-23T02:05:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -249,7 +267,7 @@ async def _send_task() -> None:
                 try:
                     if isinstance(msg, BaseModel):
                         msg = msg.model_dump(
-                            by_alias=True, exclude_unset=True, exclude_defaults=True
+                            by_alias=True, exclude_unset=True, exclude_defaults=False
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: actually I think it should never use `exclude_defaults=True`, otherwise you won't be able to update the model back to default config if you updated configs to some other values before. I guess they introduce `exclude_defaults=True` only for init so you only need to send a few configs at the begginning.
created_at: 2025-03-23T02:08:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -519,10 +536,15 @@ def push_audio(self, frame: rtc.AudioFrame) -> None:
                         audio=base64.b64encode(f.data).decode("utf-8"),
                     )
                 )
+                self._pushed_duration_s += f.duration
 
     def generate_reply(
         self, *, instructions: NotGivenOr[str] = NOT_GIVEN
     ) -> asyncio.Future[llm.GenerationCreatedEvent]:
+        if not self._realtime_model.capabilities.turn_detection and self._pushed_duration_s > 0:
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: added a `rt_session.commit_input_audio()` so we call the commit input audio buffer in `on_end_of_turn`
created_at: 2025-03-23T02:46:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,77 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import Agent, AgentSession, JobContext, RunContext, WorkerOptions, cli
+from livekit.agents.llm import ChatContext, ChatMessage, function_tool
+from livekit.plugins import cartesia, deepgram, openai
+
+logger = logging.getLogger("push-to-talk-example")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+class EchoAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Echo.",
+            stt=deepgram.STT(),
+            llm=openai.LLM(model="gpt-4o-mini"),
+            tts=cartesia.TTS(),
+        )
+
+    @function_tool
+    async def talk_to_alloy(self, context: RunContext):
+        """Called when want to talk to Alloy."""
+        return AlloyAgent(), "Transferring you to Alloy."
+
+    async def on_end_of_turn(
+        self, chat_ctx: ChatContext, new_message: ChatMessage, generating_reply: bool
+    ) -> None:
+        chat_ctx = chat_ctx.copy()
+        chat_ctx.items.append(new_message)
+        await self.update_chat_ctx(chat_ctx)
+        logger.info("add user message to chat context", extra={"content": new_message.content})
+
+
+class AlloyAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Alloy.",
+            llm=openai.realtime.RealtimeModel(voice="alloy", turn_detection=None),
+        )
+
+    @function_tool
+    async def talk_to_echo(self, context: RunContext):
+        """Called when want to talk to Echo."""
+        return EchoAgent(), "Transferring you to Echo."
+
+
+async def entrypoint(ctx: JobContext):
+    await ctx.connect()
+
+    session = AgentSession(turn_detection="manual")
+
+    await session.start(
+        agent=EchoAgent(),
+        room=ctx.room,
+    )
+
+    # disable input audio if in PPT mode
+    session.input.set_audio_enabled(False)
+
+    @ctx.room.local_participant.register_rpc_method("push")
+    async def on_pushed(data: rtc.RpcInvocationData):
path: examples/push_to_talk.py
user: login: davidzhao
body: nit: just making method names more clear:

`on_pushed` -> `start_turn`
`on_talk` -> `end_turn`
created_at: 2025-03-24T04:30:38Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,77 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import Agent, AgentSession, JobContext, RunContext, WorkerOptions, cli
+from livekit.agents.llm import ChatContext, ChatMessage, function_tool
+from livekit.plugins import cartesia, deepgram, openai
+
+logger = logging.getLogger("push-to-talk-example")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+class EchoAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Echo.",
+            stt=deepgram.STT(),
+            llm=openai.LLM(model="gpt-4o-mini"),
+            tts=cartesia.TTS(),
+        )
+
+    @function_tool
+    async def talk_to_alloy(self, context: RunContext):
+        """Called when want to talk to Alloy."""
+        return AlloyAgent(), "Transferring you to Alloy."
+
+    async def on_end_of_turn(
+        self, chat_ctx: ChatContext, new_message: ChatMessage, generating_reply: bool
+    ) -> None:
+        chat_ctx = chat_ctx.copy()
+        chat_ctx.items.append(new_message)
+        await self.update_chat_ctx(chat_ctx)
+        logger.info("add user message to chat context", extra={"content": new_message.content})
+
+
+class AlloyAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Alloy.",
+            llm=openai.realtime.RealtimeModel(voice="alloy", turn_detection=None),
+        )
+
+    @function_tool
+    async def talk_to_echo(self, context: RunContext):
+        """Called when want to talk to Echo."""
+        return EchoAgent(), "Transferring you to Echo."
+
+
+async def entrypoint(ctx: JobContext):
+    await ctx.connect()
+
+    session = AgentSession(turn_detection="manual")
+
+    await session.start(
+        agent=EchoAgent(),
+        room=ctx.room,
+    )
+
+    # disable input audio if in PPT mode
+    session.input.set_audio_enabled(False)
+
+    @ctx.room.local_participant.register_rpc_method("push")
+    async def on_pushed(data: rtc.RpcInvocationData):
path: examples/push_to_talk.py
user: login: davidzhao
body: IMO it'd also be good to link to the participant who has pushed the button, to demonstrate multi-user
created_at: 2025-03-24T04:31:26Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,77 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import Agent, AgentSession, JobContext, RunContext, WorkerOptions, cli
+from livekit.agents.llm import ChatContext, ChatMessage, function_tool
+from livekit.plugins import cartesia, deepgram, openai
+
+logger = logging.getLogger("push-to-talk-example")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+class EchoAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Echo.",
+            stt=deepgram.STT(),
+            llm=openai.LLM(model="gpt-4o-mini"),
+            tts=cartesia.TTS(),
+        )
+
+    @function_tool
+    async def talk_to_alloy(self, context: RunContext):
+        """Called when want to talk to Alloy."""
+        return AlloyAgent(), "Transferring you to Alloy."
path: examples/push_to_talk.py
user: login: davidzhao
body: since the goal is to show push to talk, can we remove the mention of alloy/echo.. and just having a single agent?
created_at: 2025-03-24T04:31:57Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,55 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit.agents import Agent, AgentSession, JobContext, JobProcess, WorkerOptions, cli, llm
+from livekit.plugins import deepgram, openai, silero
+from livekit.plugins.turn_detector import EOUModel
+
+logger = logging.getLogger("vad-realtime-example")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+class AlloyAgent(Agent):
path: examples/vad_realtime_example.py
user: login: davidzhao
body: same here. just making things a bit more generic

```suggestion
class AlloyAgent(Agent):
```
created_at: 2025-03-24T04:32:44Z
author_association: MEMBER
diff_hunk: @@ -48,6 +48,7 @@ class ErrorEvent:
 class RealtimeCapabilities:
     message_truncation: bool
     turn_detection: bool
+    user_transcription: bool
path: livekit-agents/livekit/agents/llm/realtime.py
user: login: davidzhao
body: With Gemini realtime, currently we are using a separate session to generate transcriptions. should we not do that anymore and instead set `user_transcription` to false?

CC: @jayeshp19 
created_at: 2025-03-24T04:34:09Z
author_association: MEMBER
diff_hunk: @@ -188,7 +189,7 @@ async def _bounce_eou_task() -> None:
                     {"probability": end_of_turn_probability},
                 )
                 unlikely_threshold = turn_detector.unlikely_threshold()
-                if end_of_turn_probability > unlikely_threshold:
+                if end_of_turn_probability < unlikely_threshold:
                     await asyncio.sleep(self.UNLIKELY_END_OF_TURN_EXTRA_DELAY)
path: livekit-agents/livekit/agents/voice/audio_recognition.py
user: login: davidzhao
body: not part of the PR but just noting that this isn't correct.. (both here and `await asyncio.sleep(self._min_endpointing_delay)`)

the time that we wait need to subtract the time already spent waiting for final transcripts.. i.e. `max(_min_endpointing_delay-(final_transcript_time - vad_end_speech_time), 0)`
created_at: 2025-03-24T04:47:50Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,77 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit import rtc
+from livekit.agents import Agent, AgentSession, JobContext, RunContext, WorkerOptions, cli
+from livekit.agents.llm import ChatContext, ChatMessage, function_tool
+from livekit.plugins import cartesia, deepgram, openai
+
+logger = logging.getLogger("push-to-talk-example")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+class EchoAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Echo.",
+            stt=deepgram.STT(),
+            llm=openai.LLM(model="gpt-4o-mini"),
+            tts=cartesia.TTS(),
+        )
+
+    @function_tool
+    async def talk_to_alloy(self, context: RunContext):
+        """Called when want to talk to Alloy."""
+        return AlloyAgent(), "Transferring you to Alloy."
path: examples/push_to_talk.py
user: login: longcw
body: yeah, it was to test if the manual turn detection works for both pipeline and realtime agents.
created_at: 2025-03-24T04:54:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -188,7 +189,7 @@ async def _bounce_eou_task() -> None:
                     {"probability": end_of_turn_probability},
                 )
                 unlikely_threshold = turn_detector.unlikely_threshold()
-                if end_of_turn_probability > unlikely_threshold:
+                if end_of_turn_probability < unlikely_threshold:
                     await asyncio.sleep(self.UNLIKELY_END_OF_TURN_EXTRA_DELAY)
path: livekit-agents/livekit/agents/voice/audio_recognition.py
user: login: longcw
body: probably `sleep( (user_stop_speaking_time + delay) - current_time)`?

`user_stop_speaking` time could be either `vad_end_speech_time` or `final_transcript_time` (if vad disabled), `delay` is `_min_endpointing_delay` or `UNLIKELY_END_OF_TURN_EXTRA_DELAY`.
created_at: 2025-03-24T05:07:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -188,7 +189,7 @@ async def _bounce_eou_task() -> None:
                     {"probability": end_of_turn_probability},
                 )
                 unlikely_threshold = turn_detector.unlikely_threshold()
-                if end_of_turn_probability > unlikely_threshold:
+                if end_of_turn_probability < unlikely_threshold:
                     await asyncio.sleep(self.UNLIKELY_END_OF_TURN_EXTRA_DELAY)
path: livekit-agents/livekit/agents/voice/audio_recognition.py
user: login: davidzhao
body: yeah that makes sense. otherwise we'd be longer than we have to
created_at: 2025-03-24T05:10:35Z
author_association: MEMBER
diff_hunk: @@ -130,6 +131,10 @@ def generate_reply(
         instructions: NotGivenOr[str] = NOT_GIVEN,
     ) -> asyncio.Future[GenerationCreatedEvent]: ...  # can raise RealtimeError on Timeout
 
+    # commit the input audio buffer to the server
+    @abstractmethod
+    def commit_input_audio(self) -> None: ...
path: livekit-agents/livekit/agents/llm/realtime.py
user: login: theomonnom
body: Do you think we should rename to just `commit_audio`? since we already have `push_audio` that doesn't imply the "input"
created_at: 2025-03-24T11:59:09Z
author_association: MEMBER
diff_hunk: @@ -32,12 +32,27 @@ class VoiceOptions:
 
 Userdata_T = TypeVar("Userdata_T")
 
+TurnDetectionMode = Literal["stt", "vad", "realtime_llm", "manual"]
path: livekit-agents/livekit/agents/voice/agent_session.py
user: login: theomonnom
body: Can we also include the _TurnDetector here, it's hard to see it is possible to use it IMO. Let's put everything inside the same type?
created_at: 2025-03-24T12:01:52Z
author_association: MEMBER
diff_hunk: @@ -101,6 +122,10 @@ def userdata(self) -> Userdata_T:
     def userdata(self, value: Userdata_T) -> None:
         self._userdata = value
 
+    @property
+    def turn_detection_mode(self) -> TurnDetectionMode | None:
path: livekit-agents/livekit/agents/voice/agent_session.py
user: login: theomonnom
body: Can we remove this in favor of turn_detection?
created_at: 2025-03-24T12:02:05Z
author_association: MEMBER
diff_hunk: @@ -101,6 +122,10 @@ def userdata(self) -> Userdata_T:
     def userdata(self, value: Userdata_T) -> None:
         self._userdata = value
 
+    @property
+    def turn_detection_mode(self) -> TurnDetectionMode | None:
path: livekit-agents/livekit/agents/voice/agent_session.py
user: login: theomonnom
body: returning _TurnDetector as well if needed
created_at: 2025-03-24T12:02:20Z
author_association: MEMBER
diff_hunk: @@ -130,6 +131,10 @@ def generate_reply(
         instructions: NotGivenOr[str] = NOT_GIVEN,
     ) -> asyncio.Future[GenerationCreatedEvent]: ...  # can raise RealtimeError on Timeout
 
+    # commit the input audio buffer to the server
+    @abstractmethod
+    def commit_input_audio(self) -> None: ...
path: livekit-agents/livekit/agents/llm/realtime.py
user: login: longcw
body: Sounds good to me.
created_at: 2025-03-24T12:03:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -74,14 +75,26 @@ class _InputAudioTranscription:
     model: Literal["whisper-1"] = "whisper-1"
 
 
+@dataclass
+class _TurnDetection:
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: Should we expose directly the OAI class?
created_at: 2025-03-24T12:05:54Z
author_association: MEMBER
diff_hunk: @@ -74,14 +75,26 @@ class _InputAudioTranscription:
     model: Literal["whisper-1"] = "whisper-1"
 
 
+@dataclass
+class _TurnDetection:
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: https://github.com/openai/openai-python/blob/f66d2e6fdc51c4528c99bb25a8fbca6f9b9b872d/src/openai/types/beta/realtime/session_update_event_param.py#L69

It is also a TypeDict, so maybe easier for users?

_TurnDetection is also private here
created_at: 2025-03-24T12:10:43Z
author_association: MEMBER
diff_hunk: @@ -74,14 +75,26 @@ class _InputAudioTranscription:
     model: Literal["whisper-1"] = "whisper-1"
 
 
+@dataclass
+class _TurnDetection:
+    type: Literal["server_vad"] = "server_vad"
+    create_response: bool | None = True
+    interrupt_response: bool | None = True
+    prefix_padding_ms: int | None = 300
+    silence_duration_ms: int | None = 500
+    threshold: float | None = 0.65
+
+
 DEFAULT_INPUT_AUDIO_TRANSCRIPTION = _InputAudioTranscription()
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: theomonnom
body: Maybe we can do the same for 
https://github.com/openai/openai-python/blob/f66d2e6fdc51c4528c99bb25a8fbca6f9b9b872d/src/openai/types/beta/realtime/session_update_event_param.py#L27
created_at: 2025-03-24T12:14:05Z
author_association: MEMBER
diff_hunk: @@ -74,14 +75,26 @@ class _InputAudioTranscription:
     model: Literal["whisper-1"] = "whisper-1"
 
 
+@dataclass
+class _TurnDetection:
+    type: Literal["server_vad"] = "server_vad"
+    create_response: bool | None = True
+    interrupt_response: bool | None = True
+    prefix_padding_ms: int | None = 300
+    silence_duration_ms: int | None = 500
+    threshold: float | None = 0.65
+
+
 DEFAULT_INPUT_AUDIO_TRANSCRIPTION = _InputAudioTranscription()
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: sounds good, will update it.
created_at: 2025-03-24T12:16:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -542,37 +608,67 @@ def on_vad_inference_done(self, ev: vad.VADEvent) -> None:
                 self._current_speech.interrupt()
 
     def on_interim_transcript(self, ev: stt.SpeechEvent) -> None:
+        if isinstance(self.llm, llm.RealtimeModel) and self.llm.capabilities.user_transcription:
+            # skip stt transcription if user_transcription is enabled on the realtime model
+            return
+
         self._session.emit(
             "user_input_transcribed",
             UserInputTranscribedEvent(transcript=ev.alternatives[0].text, is_final=False),
         )
 
     def on_final_transcript(self, ev: stt.SpeechEvent) -> None:
+        if isinstance(self.llm, llm.RealtimeModel) and self.llm.capabilities.user_transcription:
+            # skip stt transcription if user_transcription is enabled on the realtime model
+            return
+
         self._session.emit(
             "user_input_transcribed",
             UserInputTranscribedEvent(transcript=ev.alternatives[0].text, is_final=True),
         )
 
     async def on_end_of_turn(self, new_transcript: str) -> None:
         # When the audio recognition detects the end of a user turn:
+        #  - check if the turn detection mode is set to "manual"
+        #  - check if realtime model server-side turn detection is enabled
         #  - check if there is no current generation happening
         #  - cancel the current generation if it allows interruptions (otherwise skip this current
         #  turn)
         #  - generate a reply to the user input
 
+        user_message = llm.ChatMessage(role="user", content=[new_transcript])
+        if self._turn_detection_mode == "manual":
+            await self._agent.on_end_of_turn(
+                self._agent.chat_ctx, user_message, generating_reply=False
+            )
+            return
+
+        if isinstance(self.llm, llm.RealtimeModel) and self._rt_session is not None:
+            if self.llm.capabilities.turn_detection:
+                return
+            # ignore stt transcription for realtime model and commit the audio buffer
+            new_transcript = ""
+            self._rt_session.commit_input_audio()
+
         if self._current_speech is not None:
             if not self._current_speech.allow_interruptions:
                 logger.warning(
-                    "skipping user input, current speech generation cannot be interrupted",
+                    "skipping reply to user input, current speech generation cannot be interrupted",
                     extra={"user_input": new_transcript},
                 )
+                # in agent.on_end_of_turn user can ignore the message or save it for later
+                await self._agent.on_end_of_turn(
path: livekit-agents/livekit/agents/voice/agent_activity.py
user: login: theomonnom
body: I don't think we should emit the event in this case?
created_at: 2025-03-24T12:19:51Z
author_association: MEMBER
diff_hunk: @@ -253,7 +253,9 @@ async def on_exit(self) -> None:
         """Called when the task is exited"""
         pass
 
-    async def on_end_of_turn(self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage) -> None:
+    async def on_end_of_turn(
+        self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage, generating_reply: bool
path: livekit-agents/livekit/agents/voice/agent.py
user: login: theomonnom
body: I'm not a fan of having the generating_reply field here, is it needed? 
If turn_detection is set to manual, you already expect that no generation will be triggered in the current Agent
created_at: 2025-03-24T12:20:31Z
author_association: MEMBER
diff_hunk: @@ -253,7 +253,9 @@ async def on_exit(self) -> None:
         """Called when the task is exited"""
         pass
 
-    async def on_end_of_turn(self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage) -> None:
+    async def on_end_of_turn(
+        self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage, generating_reply: bool
path: livekit-agents/livekit/agents/voice/agent.py
user: login: longcw
body: It's actually for https://github.com/livekit/agents/blob/dev-1.0/livekit-agents/livekit/agents/voice/agent_activity.py#L659-L662
```python
            if not self._current_speech.allow_interruptions:
                logger.warning(
                    "skipping reply to user input, current speech generation cannot be interrupted",
                    extra={"user_input": new_transcript},
                )
                # in agent.on_end_of_turn user can ignore the message or save it for later
                await self._agent.on_end_of_turn(
                    self._agent.chat_ctx, user_message, generating_reply=False
                )
                return
```

Some users may want to keep the user text even not allow interruptions https://github.com/livekit/agents/issues/1661
created_at: 2025-03-24T12:29:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -253,7 +253,9 @@ async def on_exit(self) -> None:
         """Called when the task is exited"""
         pass
 
-    async def on_end_of_turn(self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage) -> None:
+    async def on_end_of_turn(
+        self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage, generating_reply: bool
path: livekit-agents/livekit/agents/voice/agent.py
user: login: theomonnom
body: I see, but I think it should be handled separately? I don't think it should be part of this event? 
But more like a flag saying "Continue to push audio even if allow_interruptions is False" ? Something inside the AgentSession parameters. To set what is the expected behavior of someone speaking when interruptions is disallowed

Because there is no "end_of_turn here"
see https://github.com/livekit/agents/pull/1695#discussion_r2010071459
created_at: 2025-03-24T13:30:27Z
author_association: MEMBER
diff_hunk: @@ -74,14 +75,26 @@ class _InputAudioTranscription:
     model: Literal["whisper-1"] = "whisper-1"
 
 
+@dataclass
+class _TurnDetection:
path: livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py
user: login: longcw
body: a problem here, those TypeDict doesn't have default values, so user has to pass the full config? or it's okay to have missing value in the type dict?
created_at: 2025-03-24T14:40:44Z
author_association: CONTRIBUTOR

PR comments:
user: login: changeset-bot[bot]
created_at: 2025-03-21T08:25:16Z
body: ###  ⚠️  No Changeset found

Latest commit: 76119d3b43577c234ca26b82b7b55b79fe76ce2f

Merging this PR will not cause a version bump for any packages. If these changes should not result in a new version, you're good to go. **If these changes should result in a version bump, you need to add a changeset.**



[Click here to learn what changesets are, and how to add one](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md).

[Click here if you're a maintainer who wants to add a changeset to this PR](https://github.com/livekit/agents/new/longc/turn-detection-options?filename=.changeset/cyan-pandas-call.md&value=---%0A%22%40fake-scope%2Ffake-pkg%22%3A%20patch%0A---%0A%0Aadd%20turn%20detection%20options%20for%20agent%201.0%0A)


author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-23T02:26:23Z
body: <!-- changeset-checker -->
### :warning: Changeset Required

We detected changes in the following package(s) but **no changeset file was found**. Please add one for proper versioning:

- `livekit-agents`
- `livekit-plugins-openai`
- `livekit-plugins-turn-detector`

👉 Create a changeset file by [clicking here](https://github.com/livekit/agents/new/longc/turn-detection-options?filename=.github%2Fnext-release%2Fchangeset-ede32221.md&value=---%0A%22livekit-agents%22%3A+patch%0A%22livekit-plugins-openai%22%3A+patch%0A%22livekit-plugins-turn-detector%22%3A+patch%0A---%0A%0Aadd+turn+detection+options+for+agent+1.0+%28%231695%29).
author_association: CONTRIBUTOR