Repository: pipecat-ai/pipecat
Pull request #1592: Smart turn
Author: login: filipi87
Created: 2025-04-15T13:22:29Z | Merged: 2025-04-17T21:21:48Z

Smart turn


Diff:
diff --git a/dot-env.template b/dot-env.template
index f0b5bdc0f2..18ddbee0af 100644
--- a/dot-env.template
+++ b/dot-env.template
@@ -92,4 +92,8 @@ ASSEMBLYAI_API_KEY=...
 OPENROUTER_API_KEY=...
 
 # Piper
-PIPER_BASE_URL=...
\ No newline at end of file
+PIPER_BASE_URL=...
+
+# Smart turn
+LOCAL_SMART_TURN_MODEL_PATH=
+REMOTE_SMART_TURN_URL=
\ No newline at end of file
diff --git a/examples/foundational/38-smart-turn.py b/examples/foundational/38-smart-turn.py
new file mode 100644
index 0000000000..03d530b903
--- /dev/null
+++ b/examples/foundational/38-smart-turn.py
@@ -0,0 +1,111 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.smart_turn import SmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+            vad_audio_passthrough=True,
+            end_of_turn_analyzer=SmartTurnAnalyzer(url=remote_smart_turn_url),
+        ),
+    )
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+        },
+    ]
+
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    pipeline = Pipeline(
+        [
+            transport.input(),  # Transport user input
+            stt,
+            context_aggregator.user(),  # User responses
+            llm,  # LLM
+            tts,  # TTS
+            transport.output(),  # Transport bot output
+            context_aggregator.assistant(),  # Assistant spoken responses
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            allow_interruptions=True,
+            enable_metrics=True,
+            enable_usage_metrics=True,
+            report_only_initial_ttfb=True,
+        ),
+    )
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Client connected")
+        # Kick off the conversation.
+        messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Client disconnected")
+
+    @transport.event_handler("on_client_closed")
+    async def on_client_closed(transport, client):
+        logger.info(f"Client closed connection")
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=False)
+
+    await runner.run(task)
+
+
+if __name__ == "__main__":
+    from run import main
+
+    main()
diff --git a/examples/foundational/38a-local-smart-turn.py b/examples/foundational/38a-local-smart-turn.py
new file mode 100644
index 0000000000..7baedf10e1
--- /dev/null
+++ b/examples/foundational/38a-local-smart-turn.py
@@ -0,0 +1,129 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.local_smart_turn import LocalCoreMLSmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH
+    # to the path where the smart-turn repo is cloned.
+    #
+    # Example setup:
+    #
+    #   # Git LFS (Large File Storage)
+    #   brew install git-lfs
+    #   # Hugging Face uses LFS to store large model files, including .mlpackage
+    #   git lfs install
+    #   # Clone the repo with the smart_turn_classifier.mlpackage
+    #   git clone https://huggingface.co/pipecat-ai/smart-turn
+    #
+    # Then set the env variable:
+    #   export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
+    # or add it to your .env file
+    smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+            vad_audio_passthrough=True,
+            end_of_turn_analyzer=LocalCoreMLSmartTurnAnalyzer(
+                smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams()
+            ),
+        ),
+    )
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+        },
+    ]
+
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    pipeline = Pipeline(
+        [
+            transport.input(),  # Transport user input
+            stt,
+            context_aggregator.user(),  # User responses
+            llm,  # LLM
+            tts,  # TTS
+            transport.output(),  # Transport bot output
+            context_aggregator.assistant(),  # Assistant spoken responses
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            allow_interruptions=True,
+            enable_metrics=True,
+            enable_usage_metrics=True,
+            report_only_initial_ttfb=True,
+        ),
+    )
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Client connected")
+        # Kick off the conversation.
+        messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Client disconnected")
+
+    @transport.event_handler("on_client_closed")
+    async def on_client_closed(transport, client):
+        logger.info(f"Client closed connection")
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=False)
+
+    await runner.run(task)
+
+
+if __name__ == "__main__":
+    from run import main
+
+    main()
diff --git a/pyproject.toml b/pyproject.toml
index cb0cd35207..e10e6daf9d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,6 +79,8 @@ qwen = []
 rime = [ "websockets~=13.1" ]
 riva = [ "nvidia-riva-client~=2.19.0" ]
 sentry = [ "sentry-sdk~=2.23.1" ]
+local-smart-turn = [ "coremltools>=8.0", "transformers", "torch==2.5.0", "torchaudio==2.5.0" ]
+remote-smart-turn = []
 silero = [ "onnxruntime~=1.20.1" ]
 simli = [ "simli-ai~=0.1.10"]
 soundfile = [ "soundfile~=0.13.0" ]
diff --git a/src/pipecat/audio/turn/__init__.py b/src/pipecat/audio/turn/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/src/pipecat/audio/turn/base_smart_turn.py b/src/pipecat/audio/turn/base_smart_turn.py
new file mode 100644
index 0000000000..0716d4e7c4
--- /dev/null
+++ b/src/pipecat/audio/turn/base_smart_turn.py
@@ -0,0 +1,182 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 3
+PRE_SPEECH_MS = 0
+MAX_DURATION_SECONDS = 8  # Max allowed segment duration
+USE_ONLY_LAST_VAD_SEGMENT = True
+
+
+class SmartTurnParams(BaseModel):
+    stop_secs: float = STOP_SECS
+    pre_speech_ms: float = PRE_SPEECH_MS
+    max_duration_secs: float = MAX_DURATION_SECONDS
+    # not exposing this for now yet until the model can handle it.
+    # use_only_last_vad_segment: bool = USE_ONLY_LAST_VAD_SEGMENT
+
+
+class BaseSmartTurn(ABC):
+    def __init__(
+        self, *, sample_rate: Optional[int] = None, params: SmartTurnParams = SmartTurnParams()
+    ):
+        self._init_sample_rate = sample_rate
+        self._params = params
+        # Configuration
+        self._sample_rate = 0
+        self._stop_ms = self._params.stop_secs * 1000  # silence threshold in ms
+        # Inference state
+        self._audio_buffer = []
+        self._speech_triggered = False
+        self._silence_ms = 0
+        self._speech_start_time = None
+
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
+    def set_sample_rate(self, sample_rate: int):
+        self._sample_rate = sample_rate
+
+    @property
+    def speech_triggered(self) -> bool:
+        return self._speech_triggered
+
+    def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
+        # Convert raw audio to float32 format and append to the buffer
+        audio_int16 = np.frombuffer(buffer, dtype=np.int16)
+        audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
+        self._audio_buffer.append((time.time(), audio_float32))
+
+        state = EndOfTurnState.INCOMPLETE
+
+        if is_speech:
+            # Reset silence tracking on speech
+            self._silence_ms = 0
+            self._speech_triggered = True
+            if self._speech_start_time is None:
+                self._speech_start_time = time.time()
+                logger.debug(f"Speech started at {self._speech_start_time}")
+        else:
+            if self._speech_triggered:
+                chunk_duration_ms = len(audio_int16) / (self._sample_rate / 1000)
+                self._silence_ms += chunk_duration_ms
+                # If silence exceeds threshold, mark end of turn
+                if self._silence_ms >= self._stop_ms:
+                    logger.debug(
+                        f"End of Turn complete due to stop_secs. Silence in ms: {self._silence_ms}"
+                    )
+                    state = EndOfTurnState.COMPLETE
+                    self._clear(state)
+            else:
+                # Trim buffer to prevent unbounded growth before speech
+                max_buffer_time = (
+                    (self._params.pre_speech_ms / 1000)
+                    + self._params.stop_secs
+                    + self._params.max_duration_secs
+                )
+                while (
+                    self._audio_buffer and self._audio_buffer[0][0] < time.time() - max_buffer_time
+                ):
+                    self._audio_buffer.pop(0)
+
+        return state
+
+    def analyze_end_of_turn(self) -> EndOfTurnState:
+        logger.debug("Analyzing End of Turn...")
+        state = self._process_speech_segment(self._audio_buffer)
+        if state == EndOfTurnState.COMPLETE or USE_ONLY_LAST_VAD_SEGMENT:
+            self._clear(state)
+        logger.debug(f"End of Turn result: {state}")
+        return state
+
+    def _clear(self, turn_state: EndOfTurnState):
+        # Reset internal state for next turn
+        logger.debug("Clearing audio buffer...")
+        # If the state is still incomplete, keep the _speech_triggered as True
+        self._speech_triggered = turn_state == EndOfTurnState.INCOMPLETE
+        self._audio_buffer = []
+        self._speech_start_time = None
+        self._silence_ms = 0
+
+    def _process_speech_segment(self, audio_buffer) -> EndOfTurnState:
+        state = EndOfTurnState.INCOMPLETE
+
+        if not audio_buffer:
+            return state
+
+        # Extract recent audio segment for prediction
+        start_time = self._speech_start_time - (self._params.pre_speech_ms / 1000)
+        start_index = 0
+        for i, (t, _) in enumerate(audio_buffer):
+            if t >= start_time:
+                start_index = i
+                break
+
+        end_index = len(audio_buffer) - 1
+
+        # Extract the audio segment
+        segment_audio_chunks = [chunk for _, chunk in audio_buffer[start_index : end_index + 1]]
+        segment_audio = np.concatenate(segment_audio_chunks)
+
+        logger.debug(f"Segment audio chunks after start index: {len(segment_audio)}")
+
+        # Limit maximum duration
+        max_samples = int(self._params.max_duration_secs * self.sample_rate)
+        if len(segment_audio) > max_samples:
+            # slices the array to keep the last max_samples samples, discarding the earlier part.
+            segment_audio = segment_audio[-max_samples:]
+
+        logger.debug(f"Segment audio chunks after limiting duration: {len(segment_audio)}")
+
+        if len(segment_audio) > 0:
+            start_time = time.perf_counter()
+            result = self._predict_endpoint(segment_audio)
+            state = (
+                EndOfTurnState.COMPLETE if result["prediction"] == 1 else EndOfTurnState.INCOMPLETE
+            )
+            end_time = time.perf_counter()
+
+            logger.debug("--------")
+            logger.debug(f"Prediction: {'Complete' if result['prediction'] == 1 else 'Incomplete'}")
+            logger.debug(f"Probability of complete: {result['probability']:.4f}")
+            logger.debug(f"Prediction took {(end_time - start_time) * 1000:.2f}ms seconds")
+        else:
+            logger.debug(f"params: {self._params}, stop_ms: {self._stop_ms}")
+            logger.debug("Captured empty audio segment, skipping prediction.")
+
+        return state
+
+    @abstractmethod
+    def _predict_endpoint(self, buffer: np.ndarray) -> Dict[str, any]:
+        """
+        Abstract method to predict if a turn has ended based on audio.
+
+        Args:
+            buffer: Float32 numpy array of audio samples at 16kHz.
+
+        Returns:
+            Dictionary with:
+              - prediction: 1 if turn is complete, else 0
+              - probability: Confidence of the prediction
+        """
+        pass
diff --git a/src/pipecat/audio/turn/local_smart_turn.py b/src/pipecat/audio/turn/local_smart_turn.py
new file mode 100644
index 0000000000..665e4b64da
--- /dev/null
+++ b/src/pipecat/audio/turn/local_smart_turn.py
@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import os
+from typing import Dict
+
+import numpy as np
+import torch
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+try:
+    import coremltools as ct
+    from transformers import AutoFeatureExtractor
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, smart_turn_model_path: str, **kwargs):
+        super().__init__(**kwargs)
+
+        if not smart_turn_model_path:
+            logger.error("smart_turn_model_path is not set.")
+            raise Exception("smart_turn_model_path must be provided.")
+
+        core_ml_model_path = f"{smart_turn_model_path}/coreml/smart_turn_classifier.mlpackage"
+
+        logger.debug("Loading Local Smart Turn model...")
+        # Only load the processor, not the torch model
+        self._turn_processor = AutoFeatureExtractor.from_pretrained(smart_turn_model_path)
+        self._turn_model = ct.models.MLModel(core_ml_model_path)
+        logger.debug("Loaded Local Smart Turn")
+
+    def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, any]:
+        inputs = self._turn_processor(
+            audio_array,
+            sampling_rate=16000,
+            padding="max_length",
+            truncation=True,
+            max_length=800,  # Maximum length as specified in training
+            return_attention_mask=True,
+            return_tensors="pt",
+        )
+
+        output = self._turn_model.predict(dict(inputs))
+        logits = output["logits"]  # Core ML returns numpy array
+        logits_tensor = torch.tensor(logits)
+        probabilities = torch.nn.functional.softmax(logits_tensor, dim=1)
+        completion_prob = probabilities[0, 1].item()  # Probability of class 1 (Complete)
+        prediction = 1 if completion_prob > 0.5 else 0
+
+        return {
+            "prediction": prediction,
+            "probability": completion_prob,
+        }
diff --git a/src/pipecat/audio/turn/smart_turn.py b/src/pipecat/audio/turn/smart_turn.py
new file mode 100644
index 0000000000..5378e4b5b7
--- /dev/null
+++ b/src/pipecat/audio/turn/smart_turn.py
@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, url: str, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = url
+
+        if not self.remote_smart_turn_url:
+            logger.error("remote_smart_turn_url is not set.")
+            raise Exception("remote_smart_turn_url must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.trace("Serializing NumPy array to bytes...")
+        buffer = io.BytesIO()
+        np.save(buffer, audio_array)
+        serialized_bytes = buffer.getvalue()
+        logger.trace(f"Serialized size: {len(serialized_bytes)} bytes")
+        return serialized_bytes
+
+    def _send_raw_request(self, data_bytes: bytes):
+        headers = {"Content-Type": "application/octet-stream"}
+        logger.trace(
+            f"Sending {len(data_bytes)} bytes as raw body to {self.remote_smart_turn_url}..."
+        )
+        try:
+            response = self.session.post(
+                self.remote_smart_turn_url,
+                data=data_bytes,
+                headers=headers,
+                timeout=60,
+            )
+
+            logger.trace("\n--- Response ---")
+            logger.trace(f"Status Code: {response.status_code}")
+
+            if response.ok:
+                try:
+                    logger.trace("Response JSON:")
+                    logger.trace(response.json())
+                    return response.json()
+                except requests.exceptions.JSONDecodeError:
+                    logger.trace("Response Content (non-JSON):")
+                    logger.trace(response.text)
+            else:
+                logger.trace("Response Content (Error):")
+                logger.trace(response.text)
+                response.raise_for_status()
+
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to send raw request to Daily Smart Turn: {e}")
+            raise Exception("Failed to send raw request to Daily Smart Turn.")
+
+    def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, any]:
+        serialized_array = self._serialize_array(audio_array)
+        return self._send_raw_request(serialized_array)
diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py
index 26f3865761..3c7cf868de 100644
--- a/src/pipecat/transports/base_input.py
+++ b/src/pipecat/transports/base_input.py
@@ -10,6 +10,7 @@
 
 from loguru import logger
 
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn, EndOfTurnState
 from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
 from pipecat.frames.frames import (
     BotInterruptionFrame,
@@ -64,12 +65,19 @@ def sample_rate(self) -> int:
     def vad_analyzer(self) -> Optional[VADAnalyzer]:
         return self._params.vad_analyzer
 
+    @property
+    def end_of_turn_analyzer(self) -> Optional[BaseSmartTurn]:
+        return self._params.end_of_turn_analyzer
+
     async def start(self, frame: StartFrame):
         self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
 
         # Configure VAD analyzer.
         if self._params.vad_enabled and self._params.vad_analyzer:
             self._params.vad_analyzer.set_sample_rate(self._sample_rate)
+        # Configure End of turn analyzer.
+        if self._params.end_of_turn_analyzer:
+            self._params.end_of_turn_analyzer.set_sample_rate(self._sample_rate)
         # Start audio filter.
         if self._params.audio_in_filter:
             await self._params.audio_in_filter.start(self._sample_rate)
@@ -187,10 +195,18 @@ async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState
             and new_vad_state != VADState.STOPPING
         ):
             frame = None
-            if new_vad_state == VADState.SPEAKING:
-                frame = UserStartedSpeakingFrame()
-            elif new_vad_state == VADState.QUIET:
-                frame = UserStoppedSpeakingFrame()
+            # If the turn analyser is enabled, this will prevent:
+            # - Creating the UserStoppedSpeakingFrame
+            # - Creating the UserStartedSpeakingFrame multiple times
+            can_create_user_frames = (
+                self._params.end_of_turn_analyzer is None
+                or not self._params.end_of_turn_analyzer.speech_triggered
+            )
+            if can_create_user_frames:
+                if new_vad_state == VADState.SPEAKING:
+                    frame = UserStartedSpeakingFrame()
+                elif new_vad_state == VADState.QUIET:
+                    frame = UserStoppedSpeakingFrame()
 
             if frame:
                 await self._handle_user_interruption(frame)
@@ -198,6 +214,29 @@ async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState
             vad_state = new_vad_state
         return vad_state
 
+    async def _handle_end_of_turn(self):
+        if self.end_of_turn_analyzer:
+            state = await self.get_event_loop().run_in_executor(
+                self._executor, self.end_of_turn_analyzer.analyze_end_of_turn
+            )
+            await self._handle_end_of_turn_complete(state)
+
+    async def _handle_end_of_turn_complete(self, state: EndOfTurnState):
+        if state == EndOfTurnState.COMPLETE:
+            await self._handle_user_interruption(UserStoppedSpeakingFrame())
+
+    async def _run_turn_analyzer(
+        self, frame: InputAudioRawFrame, vad_state: VADState, previous_vad_state: VADState
+    ):
+        is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
+        # If silence exceeds threshold, we are going to receive EndOfTurnState.COMPLETE
+        end_of_turn_state = self._params.end_of_turn_analyzer.append_audio(frame.audio, is_speech)
+        if end_of_turn_state == EndOfTurnState.COMPLETE:
+            await self._handle_end_of_turn_complete(end_of_turn_state)
+        # Otherwise we are going to trigger to check if the turn is completed based on the VAD
+        elif vad_state == VADState.QUIET and vad_state != previous_vad_state:
+            await self._handle_end_of_turn()
+
     async def _audio_task_handler(self):
         vad_state: VADState = VADState.QUIET
         while True:
@@ -211,10 +250,14 @@ async def _audio_task_handler(self):
 
             # Check VAD and push event if necessary. We just care about
             # changes from QUIET to SPEAKING and vice versa.
+            previous_vad_state = vad_state
             if self._params.vad_enabled:
                 vad_state = await self._handle_vad(frame, vad_state)
                 audio_passthrough = self._params.vad_audio_passthrough
 
+            if self._params.end_of_turn_analyzer:
+                await self._run_turn_analyzer(frame, vad_state, previous_vad_state)
+
             # Push audio downstream if passthrough.
             if audio_passthrough:
                 await self.push_frame(frame)
diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py
index 411456071f..79c876fc49 100644
--- a/src/pipecat/transports/base_transport.py
+++ b/src/pipecat/transports/base_transport.py
@@ -11,6 +11,7 @@
 
 from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
 from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
 from pipecat.audio.vad.vad_analyzer import VADAnalyzer
 from pipecat.processors.frame_processor import FrameProcessor
 from pipecat.utils.base_object import BaseObject
@@ -41,6 +42,7 @@ class TransportParams(BaseModel):
     vad_enabled: bool = False
     vad_audio_passthrough: bool = False
     vad_analyzer: Optional[VADAnalyzer] = None
+    end_of_turn_analyzer: Optional[BaseSmartTurn] = None
 
 
 class BaseTransport(BaseObject):


Reviews:
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T12:02:47Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T12:03:47Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T01:36:26Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T12:27:44Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T14:10:27Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T14:41:03Z
user: login: markbackman
body: This is looking good! I think this is everything we discussed. Once this is in, I'll do one more testing pass.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T16:55:26Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:32:12Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:32:22Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:32:33Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:32:44Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:34:50Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:36:33Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:36:51Z
user: login: aconchillo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:41:25Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:07:18Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:07:27Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:07:33Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:14:24Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:14:28Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:14:32Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:14:36Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:29:05Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:29:11Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T19:46:37Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:22:45Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:41:37Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:41:53Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:42:28Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:44:35Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:44:43Z
user: login: markbackman
body: So good! Approving only with a few minor suggestions.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:44:58Z
user: login: filipi87
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:49:20Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:50:51Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:52:58Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:54:26Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T20:56:41Z
user: login: markbackman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T21:20:26Z

Review comments:
diff_hunk: @@ -571,6 +571,15 @@ class EmulateUserStoppedSpeakingFrame(SystemFrame):
     pass
 
 
+@dataclass
+class UserEndOfTurnFrame(SystemFrame):
path: src/pipecat/frames/frames.py
user: login: filipi87
body: We need to decide this:
- In the same user turn, should we emit multiple `UserStartedSpeaking` and `UserStoppedSpeaking` events, and also include a new event to mark the end of the turn, in this case, `UserEndOfTurnFrame`?
  - If we take this approach, we will need change the LLM services to react to the `UserEndOfTurnFrame` instead of to `UserStoppedSpeaking`.
- Or should we only emit a single `UserStartedSpeaking` and `UserStoppedSpeaking` event ? In that case, do we still need a `UserEndOfTurnFrame` ?

created_at: 2025-04-16T12:02:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -189,7 +201,9 @@ async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState
             frame = None
             if new_vad_state == VADState.SPEAKING:
                 frame = UserStartedSpeakingFrame()
-            elif new_vad_state == VADState.QUIET:
+            # TODO: need to double check if this is the expected behavior
+            # Not triggering the UserStoppedSpeakingFrame if the turn analyser is enabled
+            elif new_vad_state == VADState.QUIET and not self.end_of_turn_analyzer:
path: src/pipecat/transports/base_input.py
user: login: filipi87
body: Same here.
And if we choose this approach, I will need to refactor to only trigger the `UserStartedSpeakingFrame` once. 
created_at: 2025-04-16T12:03:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -571,6 +571,15 @@ class EmulateUserStoppedSpeakingFrame(SystemFrame):
     pass
 
 
+@dataclass
+class UserEndOfTurnFrame(SystemFrame):
path: src/pipecat/frames/frames.py
user: login: markbackman
body: I'm interested in @aconchillo's POV.

To date `UserStoppedSpeakingFrame` signifies that the VAD detects end of speech and that the user's turn had ended. Since the two were one and the same, a single event was sufficient.

With smart-turn, end of speech and end of turn can now occur at different times. Given this, we should consider adding a new event for `UserEndOfTurnFrame`.

If we do this, we'll probably want to add a `UserEndOfTurnFrame` to the non-smart-turn case as well, so our logic doesn't vary too much between smart-turn and non-smart-turn cases.

@aconchillo do you have any insights?
created_at: 2025-04-17T01:36:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -571,6 +571,15 @@ class EmulateUserStoppedSpeakingFrame(SystemFrame):
     pass
 
 
+@dataclass
+class UserEndOfTurnFrame(SystemFrame):
path: src/pipecat/frames/frames.py
user: login: filipi87
body: > With smart-turn, end of speech and end of turn can now occur at different times. Given this, we should consider adding a new event for UserEndOfTurnFrame.

Exactly, if we take this approach, we will need change the LLM services to react to the `UserEndOfTurnFrame` instead of to `UserStoppedSpeaking`. And probably in a couple of other places in our code.

> If we do this, we'll probably want to add a UserEndOfTurnFrame to the non-smart-turn case as well, so our logic doesn't vary too much between smart-turn and non-smart-turn cases.

Yep. That would help to smooth things.
created_at: 2025-04-17T12:27:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,172 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
+PRE_SPEECH_MS = 0
+MAX_DURATION_SECONDS = 16  # Max allowed segment duration
+
+
+class SmartTurnParams(BaseModel):
+    stop_secs: float = STOP_SECS
+    pre_speech_ms: float = PRE_SPEECH_MS
+    max_duration_secs: float = MAX_DURATION_SECONDS
+
+
+class BaseSmartTurn(ABC):
+    def __init__(
+        self, *, sample_rate: Optional[int] = None, params: SmartTurnParams = SmartTurnParams()
+    ):
+        self._init_sample_rate = sample_rate
+        self._params = params
+        # Configuration
+        self._sample_rate = 0
+        self._stop_ms = self._params.stop_secs * 1000  # silence threshold in ms
+        # Inference state
+        self._audio_buffer = []
+        self._speech_triggered = False
+        self._silence_ms = 0
+        self._speech_start_time = None
+
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
+    def set_sample_rate(self, sample_rate: int):
+        self._sample_rate = sample_rate
+
+    def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
+        # Convert raw audio to float32 format and append to the buffer
+        audio_int16 = np.frombuffer(buffer, dtype=np.int16)
+        audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
+        self._audio_buffer.append((time.time(), audio_float32))
+
+        state = EndOfTurnState.INCOMPLETE
+
+        if is_speech:
+            # Reset silence tracking on speech
+            self._silence_ms = 0
+            self._speech_triggered = True
+            if self._speech_start_time is None:
+                self._speech_start_time = time.time()
+                logger.debug(f"Speech started at {self._speech_start_time}")
+        else:
+            if self._speech_triggered:
+                chunk_duration_ms = len(audio_int16) / (self._sample_rate / 1000)
+                self._silence_ms += chunk_duration_ms
+                # If silence exceeds threshold, mark end of turn
+                if self._silence_ms >= self._stop_ms:
+                    logger.debug(
+                        f"End of Turn complete due to stop_secs. Silence in ms: {self._silence_ms}"
+                    )
+                    state = EndOfTurnState.COMPLETE
+                    self._clear()
+            else:
+                # Trim buffer to prevent unbounded growth before speech
+                max_buffer_time = (
+                    (self._params.pre_speech_ms / 1000)
+                    + self._params.stop_secs
+                    + self._params.max_duration_secs
+                )
+                while (
+                    self._audio_buffer and self._audio_buffer[0][0] < time.time() - max_buffer_time
+                ):
+                    self._audio_buffer.pop(0)
+
+        return state
+
+    def analyze_end_of_turn(self) -> EndOfTurnState:
+        logger.debug("Analyzing End of Turn...")
+        state = self._process_speech_segment(self._audio_buffer)
+        if state == EndOfTurnState.COMPLETE:
+            self._clear()
+        logger.debug(f"End of Turn result: {state}")
+        return state
+
+    def _clear(self):
+        # Reset internal state for next turn
+        logger.debug("Clearing audio buffer...")
+        self._speech_triggered = False
+        self._audio_buffer = []
+        self._speech_start_time = None
+        self._silence_ms = 0
+
+    def _process_speech_segment(self, audio_buffer) -> EndOfTurnState:
+        state = EndOfTurnState.INCOMPLETE
+
+        if not audio_buffer:
+            return state
+
+        # Extract recent audio segment for prediction
+        start_time = self._speech_start_time - (self._params.pre_speech_ms / 1000)
+        start_index = 0
+        for i, (t, _) in enumerate(audio_buffer):
+            if t >= start_time:
+                start_index = i
+                break
+
+        end_index = len(audio_buffer) - 1
+
+        # Extract the audio segment
+        segment_audio_chunks = [chunk for _, chunk in audio_buffer[start_index : end_index + 1]]
+        segment_audio = np.concatenate(segment_audio_chunks)
+
+        logger.debug(f"Segment audio chunks after start index: {len(segment_audio)}")
+
+        # Limit maximum duration
+        if len(segment_audio) / self.sample_rate > self._params.max_duration_secs:
+            segment_audio = segment_audio[: int(self._params.max_duration_secs * self.sample_rate)]
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: markbackman
body: Does this keep only the first `max_duration_secs` of audio? In the event the user's input is long, we always want to send the last `max_duration_secs` of audio to the model.
created_at: 2025-04-17T14:10:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,172 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
+PRE_SPEECH_MS = 0
+MAX_DURATION_SECONDS = 16  # Max allowed segment duration
+
+
+class SmartTurnParams(BaseModel):
+    stop_secs: float = STOP_SECS
+    pre_speech_ms: float = PRE_SPEECH_MS
+    max_duration_secs: float = MAX_DURATION_SECONDS
+
+
+class BaseSmartTurn(ABC):
+    def __init__(
+        self, *, sample_rate: Optional[int] = None, params: SmartTurnParams = SmartTurnParams()
+    ):
+        self._init_sample_rate = sample_rate
+        self._params = params
+        # Configuration
+        self._sample_rate = 0
+        self._stop_ms = self._params.stop_secs * 1000  # silence threshold in ms
+        # Inference state
+        self._audio_buffer = []
+        self._speech_triggered = False
+        self._silence_ms = 0
+        self._speech_start_time = None
+
+    @property
+    def sample_rate(self) -> int:
+        return self._sample_rate
+
+    def set_sample_rate(self, sample_rate: int):
+        self._sample_rate = sample_rate
+
+    def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
+        # Convert raw audio to float32 format and append to the buffer
+        audio_int16 = np.frombuffer(buffer, dtype=np.int16)
+        audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
+        self._audio_buffer.append((time.time(), audio_float32))
+
+        state = EndOfTurnState.INCOMPLETE
+
+        if is_speech:
+            # Reset silence tracking on speech
+            self._silence_ms = 0
+            self._speech_triggered = True
+            if self._speech_start_time is None:
+                self._speech_start_time = time.time()
+                logger.debug(f"Speech started at {self._speech_start_time}")
+        else:
+            if self._speech_triggered:
+                chunk_duration_ms = len(audio_int16) / (self._sample_rate / 1000)
+                self._silence_ms += chunk_duration_ms
+                # If silence exceeds threshold, mark end of turn
+                if self._silence_ms >= self._stop_ms:
+                    logger.debug(
+                        f"End of Turn complete due to stop_secs. Silence in ms: {self._silence_ms}"
+                    )
+                    state = EndOfTurnState.COMPLETE
+                    self._clear()
+            else:
+                # Trim buffer to prevent unbounded growth before speech
+                max_buffer_time = (
+                    (self._params.pre_speech_ms / 1000)
+                    + self._params.stop_secs
+                    + self._params.max_duration_secs
+                )
+                while (
+                    self._audio_buffer and self._audio_buffer[0][0] < time.time() - max_buffer_time
+                ):
+                    self._audio_buffer.pop(0)
+
+        return state
+
+    def analyze_end_of_turn(self) -> EndOfTurnState:
+        logger.debug("Analyzing End of Turn...")
+        state = self._process_speech_segment(self._audio_buffer)
+        if state == EndOfTurnState.COMPLETE:
+            self._clear()
+        logger.debug(f"End of Turn result: {state}")
+        return state
+
+    def _clear(self):
+        # Reset internal state for next turn
+        logger.debug("Clearing audio buffer...")
+        self._speech_triggered = False
+        self._audio_buffer = []
+        self._speech_start_time = None
+        self._silence_ms = 0
+
+    def _process_speech_segment(self, audio_buffer) -> EndOfTurnState:
+        state = EndOfTurnState.INCOMPLETE
+
+        if not audio_buffer:
+            return state
+
+        # Extract recent audio segment for prediction
+        start_time = self._speech_start_time - (self._params.pre_speech_ms / 1000)
+        start_index = 0
+        for i, (t, _) in enumerate(audio_buffer):
+            if t >= start_time:
+                start_index = i
+                break
+
+        end_index = len(audio_buffer) - 1
+
+        # Extract the audio segment
+        segment_audio_chunks = [chunk for _, chunk in audio_buffer[start_index : end_index + 1]]
+        segment_audio = np.concatenate(segment_audio_chunks)
+
+        logger.debug(f"Segment audio chunks after start index: {len(segment_audio)}")
+
+        # Limit maximum duration
+        if len(segment_audio) / self.sample_rate > self._params.max_duration_secs:
+            segment_audio = segment_audio[: int(self._params.max_duration_secs * self.sample_rate)]
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: filipi87
body: Yes, you were right. I have pushed a fix for that. 
Nice catch.
created_at: 2025-04-17T14:41:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,176 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: markbackman
body: Based on our conversation, let's set the default to 3 seconds.
created_at: 2025-04-17T16:53:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,176 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
+PRE_SPEECH_MS = 0
+MAX_DURATION_SECONDS = 8  # Max allowed segment duration
+USE_ONLY_LAST_VAD_SEGMENT = False
+
+
+class SmartTurnParams(BaseModel):
+    stop_secs: float = STOP_SECS
+    pre_speech_ms: float = PRE_SPEECH_MS
+    max_duration_secs: float = MAX_DURATION_SECONDS
+    use_only_last_vad_segment: bool = USE_ONLY_LAST_VAD_SEGMENT
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: markbackman
body: For now, let's not expose this. We probably want to maintain this logic, but not to expose the input param yet until the model can handle it.
created_at: 2025-04-17T16:54:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,176 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
+PRE_SPEECH_MS = 0
+MAX_DURATION_SECONDS = 8  # Max allowed segment duration
+USE_ONLY_LAST_VAD_SEGMENT = False
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: markbackman
body: Based on performance, let's set the default to True.
created_at: 2025-04-17T16:54:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
+        buffer = io.BytesIO()
+        np.save(buffer, audio_array)
+        serialized_bytes = buffer.getvalue()
+        logger.debug(f"Serialized size: {len(serialized_bytes)} bytes")
+        return serialized_bytes
+
+    def _send_raw_request(self, data_bytes: bytes):
+        headers = {"Content-Type": "application/octet-stream"}
+        logger.debug(
+            f"Sending {len(data_bytes)} bytes as raw body to {self.remote_smart_turn_url}..."
+        )
+        try:
+            response = self.session.post(
+                self.remote_smart_turn_url,
+                data=data_bytes,
+                headers=headers,
+                timeout=60,
+            )
+
+            logger.debug("\n--- Response ---")
+            logger.debug(f"Status Code: {response.status_code}")
+
+            if response.ok:
+                try:
+                    logger.debug("Response JSON:")
+                    logger.debug(response.json())
+                    return response.json()
+                except requests.exceptions.JSONDecodeError:
+                    logger.debug("Response Content (non-JSON):")
+                    logger.debug(response.text)
+            else:
+                logger.debug("Response Content (Error):")
+                logger.debug(response.text)
path: src/pipecat/audio/turn/smart_turn.py
user: login: aconchillo
body: We probably want all this to be `logger.trace`
created_at: 2025-04-17T17:32:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
+        buffer = io.BytesIO()
+        np.save(buffer, audio_array)
+        serialized_bytes = buffer.getvalue()
+        logger.debug(f"Serialized size: {len(serialized_bytes)} bytes")
+        return serialized_bytes
+
+    def _send_raw_request(self, data_bytes: bytes):
+        headers = {"Content-Type": "application/octet-stream"}
+        logger.debug(
+            f"Sending {len(data_bytes)} bytes as raw body to {self.remote_smart_turn_url}..."
+        )
path: src/pipecat/audio/turn/smart_turn.py
user: login: aconchillo
body: Logger trace
created_at: 2025-04-17T17:32:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
+        buffer = io.BytesIO()
+        np.save(buffer, audio_array)
+        serialized_bytes = buffer.getvalue()
+        logger.debug(f"Serialized size: {len(serialized_bytes)} bytes")
path: src/pipecat/audio/turn/smart_turn.py
user: login: aconchillo
body: Logger trace.
created_at: 2025-04-17T17:32:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
path: src/pipecat/audio/turn/smart_turn.py
user: login: aconchillo
body: Logger trace
created_at: 2025-04-17T17:32:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,81 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import os
+from typing import Dict
+
+import numpy as np
+import torch
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+try:
+    import coremltools as ct
+    from transformers import AutoFeatureExtractor
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH
+        # to the path where the smart-turn repo is cloned.
+        #
+        # Example setup:
+        #
+        #   # Git LFS (Large File Storage)
+        #   brew install git-lfs
+        #   # Hugging Face uses LFS to store large model files, including .mlpackage
+        #   git lfs install
+        #   # Clone the repo with the smart_turn_classifier.mlpackage
+        #   git clone https://huggingface.co/pipecat-ai/smart-turn
+        #
+        # Then set the env variable:
+        #   export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
+        # or add it to your .env file
+        smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH")
path: src/pipecat/audio/turn/local_smart_turn.py
user: login: aconchillo
body: I'd pass a `model_path` argument instead of defining an environment variable. You can use the environment variable on the test files.
created_at: 2025-04-17T17:34:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
path: src/pipecat/audio/turn/smart_turn.py
user: login: aconchillo
body: We should pass this as a `base_url` instead of an environment variable. Use the variable in the test file.
created_at: 2025-04-17T17:36:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
path: src/pipecat/audio/turn/smart_turn.py
user: login: aconchillo
body: or simply `url`
created_at: 2025-04-17T17:36:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -211,10 +234,23 @@ async def _audio_task_handler(self):
 
             # Check VAD and push event if necessary. We just care about
             # changes from QUIET to SPEAKING and vice versa.
+            previous_vad_state = vad_state
             if self._params.vad_enabled:
                 vad_state = await self._handle_vad(frame, vad_state)
                 audio_passthrough = self._params.vad_audio_passthrough
 
+            if self._params.end_of_turn_analyzer:
+                is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
+                # If silence exceeds threshold, we are going to receive EndOfTurnState.COMPLETE
+                end_of_turn_state = self._params.end_of_turn_analyzer.append_audio(
+                    frame.audio, is_speech
+                )
+                if end_of_turn_state == EndOfTurnState.COMPLETE:
+                    await self._handle_end_of_turn_complete(end_of_turn_state)
+                # Otherwise we are going to trigger to check if the turn is completed based on the VAD
+                elif vad_state == VADState.QUIET and vad_state != previous_vad_state:
+                    await self._handle_end_of_turn()
+
path: src/pipecat/transports/base_input.py
user: login: aconchillo
body: Can we move this to a function to keep this function smaller:

```
if self._params.end_of_turn_analyzer:
    await self._run_turn_analyzer(frame, vad_state, previous_vad_state)
```
created_at: 2025-04-17T17:41:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,176 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: filipi87
body: Done!
created_at: 2025-04-17T19:07:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,176 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
+PRE_SPEECH_MS = 0
+MAX_DURATION_SECONDS = 8  # Max allowed segment duration
+USE_ONLY_LAST_VAD_SEGMENT = False
+
+
+class SmartTurnParams(BaseModel):
+    stop_secs: float = STOP_SECS
+    pre_speech_ms: float = PRE_SPEECH_MS
+    max_duration_secs: float = MAX_DURATION_SECONDS
+    use_only_last_vad_segment: bool = USE_ONLY_LAST_VAD_SEGMENT
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: filipi87
body: Done.
created_at: 2025-04-17T19:07:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,176 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, Optional
+
+import numpy as np
+from loguru import logger
+from pydantic import BaseModel
+
+
+# Enum for end-of-turn detection states
+class EndOfTurnState(Enum):
+    COMPLETE = 1
+    INCOMPLETE = 2
+
+
+# Default timing parameters
+STOP_SECS = 1
+PRE_SPEECH_MS = 0
+MAX_DURATION_SECONDS = 8  # Max allowed segment duration
+USE_ONLY_LAST_VAD_SEGMENT = False
path: src/pipecat/audio/turn/base_smart_turn.py
user: login: filipi87
body: Done.
created_at: 2025-04-17T19:07:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
+        buffer = io.BytesIO()
+        np.save(buffer, audio_array)
+        serialized_bytes = buffer.getvalue()
+        logger.debug(f"Serialized size: {len(serialized_bytes)} bytes")
+        return serialized_bytes
+
+    def _send_raw_request(self, data_bytes: bytes):
+        headers = {"Content-Type": "application/octet-stream"}
+        logger.debug(
+            f"Sending {len(data_bytes)} bytes as raw body to {self.remote_smart_turn_url}..."
+        )
+        try:
+            response = self.session.post(
+                self.remote_smart_turn_url,
+                data=data_bytes,
+                headers=headers,
+                timeout=60,
+            )
+
+            logger.debug("\n--- Response ---")
+            logger.debug(f"Status Code: {response.status_code}")
+
+            if response.ok:
+                try:
+                    logger.debug("Response JSON:")
+                    logger.debug(response.json())
+                    return response.json()
+                except requests.exceptions.JSONDecodeError:
+                    logger.debug("Response Content (non-JSON):")
+                    logger.debug(response.text)
+            else:
+                logger.debug("Response Content (Error):")
+                logger.debug(response.text)
path: src/pipecat/audio/turn/smart_turn.py
user: login: filipi87
body: Done.
created_at: 2025-04-17T19:14:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
+        buffer = io.BytesIO()
+        np.save(buffer, audio_array)
+        serialized_bytes = buffer.getvalue()
+        logger.debug(f"Serialized size: {len(serialized_bytes)} bytes")
+        return serialized_bytes
+
+    def _send_raw_request(self, data_bytes: bytes):
+        headers = {"Content-Type": "application/octet-stream"}
+        logger.debug(
+            f"Sending {len(data_bytes)} bytes as raw body to {self.remote_smart_turn_url}..."
+        )
path: src/pipecat/audio/turn/smart_turn.py
user: login: filipi87
body: Done.
created_at: 2025-04-17T19:14:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
+        buffer = io.BytesIO()
+        np.save(buffer, audio_array)
+        serialized_bytes = buffer.getvalue()
+        logger.debug(f"Serialized size: {len(serialized_bytes)} bytes")
path: src/pipecat/audio/turn/smart_turn.py
user: login: filipi87
body: Done.
created_at: 2025-04-17T19:14:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+        if not self.remote_smart_turn_url:
+            logger.error("REMOTE_SMART_TURN_URL is not set.")
+            raise Exception("REMOTE_SMART_TURN_URL environment variable must be provided.")
+
+        # Use a session to reuse connections (keep-alive)
+        self.session = requests.Session()
+        self.session.headers.update({"Connection": "keep-alive"})
+
+    def _serialize_array(self, audio_array: np.ndarray) -> bytes:
+        logger.debug("Serializing NumPy array to bytes...")
path: src/pipecat/audio/turn/smart_turn.py
user: login: filipi87
body: Done.
created_at: 2025-04-17T19:14:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,81 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import os
+from typing import Dict
+
+import numpy as np
+import torch
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+try:
+    import coremltools as ct
+    from transformers import AutoFeatureExtractor
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
+
+class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH
+        # to the path where the smart-turn repo is cloned.
+        #
+        # Example setup:
+        #
+        #   # Git LFS (Large File Storage)
+        #   brew install git-lfs
+        #   # Hugging Face uses LFS to store large model files, including .mlpackage
+        #   git lfs install
+        #   # Clone the repo with the smart_turn_classifier.mlpackage
+        #   git clone https://huggingface.co/pipecat-ai/smart-turn
+        #
+        # Then set the env variable:
+        #   export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
+        # or add it to your .env file
+        smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH")
path: src/pipecat/audio/turn/local_smart_turn.py
user: login: filipi87
body: done.
created_at: 2025-04-17T19:29:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,75 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import io
+import os
+from typing import Dict
+
+import numpy as np
+import requests
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import BaseSmartTurn
+
+
+class SmartTurnAnalyzer(BaseSmartTurn):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
path: src/pipecat/audio/turn/smart_turn.py
user: login: filipi87
body: done.
created_at: 2025-04-17T19:29:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -211,10 +234,23 @@ async def _audio_task_handler(self):
 
             # Check VAD and push event if necessary. We just care about
             # changes from QUIET to SPEAKING and vice versa.
+            previous_vad_state = vad_state
             if self._params.vad_enabled:
                 vad_state = await self._handle_vad(frame, vad_state)
                 audio_passthrough = self._params.vad_audio_passthrough
 
+            if self._params.end_of_turn_analyzer:
+                is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
+                # If silence exceeds threshold, we are going to receive EndOfTurnState.COMPLETE
+                end_of_turn_state = self._params.end_of_turn_analyzer.append_audio(
+                    frame.audio, is_speech
+                )
+                if end_of_turn_state == EndOfTurnState.COMPLETE:
+                    await self._handle_end_of_turn_complete(end_of_turn_state)
+                # Otherwise we are going to trigger to check if the turn is completed based on the VAD
+                elif vad_state == VADState.QUIET and vad_state != previous_vad_state:
+                    await self._handle_end_of_turn()
+
path: src/pipecat/transports/base_input.py
user: login: filipi87
body: Done.
created_at: 2025-04-17T19:46:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -187,17 +196,49 @@ async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState
             and new_vad_state != VADState.STOPPING
         ):
             frame = None
-            if new_vad_state == VADState.SPEAKING:
-                frame = UserStartedSpeakingFrame()
-            elif new_vad_state == VADState.QUIET:
-                frame = UserStoppedSpeakingFrame()
+            # If the turn analyser is enabled, this will prevent:
+            # - Creating the UserStoppedSpeakingFrame
+            # - Creating the UserStartedSpeakingFrame multiple times
+            can_create_user_frames = (
+                self._params.end_of_turn_analyzer is None
+                or not self._params.end_of_turn_analyzer.speech_triggered
+            )
+            if can_create_user_frames:
+                if new_vad_state == VADState.SPEAKING:
+                    frame = UserStartedSpeakingFrame()
+                elif new_vad_state == VADState.QUIET:
+                    frame = UserStoppedSpeakingFrame()
path: src/pipecat/transports/base_input.py
user: login: filipi87
body: @markbackman , @aconchillo , this is the logic preventing emitting these events if the turn analyser is enabled.
created_at: 2025-04-17T20:22:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,111 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.smart_turn import SmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+            vad_audio_passthrough=True,
+            end_of_turn_analyzer=SmartTurnAnalyzer(url=remote_smart_turn_url),
+        ),
+    )
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
path: examples/foundational/38-smart-turn.py
user: login: markbackman
body: Let's remove the model so we don't have to update it later.

```suggestion
    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
```
created_at: 2025-04-17T20:41:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,129 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.local_smart_turn import LocalCoreMLSmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH
+    # to the path where the smart-turn repo is cloned.
+    #
+    # Example setup:
+    #
+    #   # Git LFS (Large File Storage)
+    #   brew install git-lfs
+    #   # Hugging Face uses LFS to store large model files, including .mlpackage
+    #   git lfs install
+    #   # Clone the repo with the smart_turn_classifier.mlpackage
+    #   git clone https://huggingface.co/pipecat-ai/smart-turn
+    #
+    # Then set the env variable:
+    #   export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
+    # or add it to your .env file
+    smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+            vad_audio_passthrough=True,
+            end_of_turn_analyzer=LocalCoreMLSmartTurnAnalyzer(
+                smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams(stop_secs=5)
+            ),
+        ),
+    )
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
path: examples/foundational/38a-local-smart-turn.py
user: login: markbackman
body: Same here.

```suggestion
    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
```
created_at: 2025-04-17T20:41:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,129 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.local_smart_turn import LocalCoreMLSmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH
+    # to the path where the smart-turn repo is cloned.
+    #
+    # Example setup:
+    #
+    #   # Git LFS (Large File Storage)
+    #   brew install git-lfs
+    #   # Hugging Face uses LFS to store large model files, including .mlpackage
+    #   git lfs install
+    #   # Clone the repo with the smart_turn_classifier.mlpackage
+    #   git clone https://huggingface.co/pipecat-ai/smart-turn
+    #
+    # Then set the env variable:
+    #   export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
+    # or add it to your .env file
+    smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+            vad_audio_passthrough=True,
+            end_of_turn_analyzer=LocalCoreMLSmartTurnAnalyzer(
+                smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams(stop_secs=5)
path: examples/foundational/38a-local-smart-turn.py
user: login: markbackman
body: Should we just leave the default?

```suggestion
                smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams()
```
created_at: 2025-04-17T20:42:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,129 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.local_smart_turn import LocalCoreMLSmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH
+    # to the path where the smart-turn repo is cloned.
+    #
+    # Example setup:
+    #
+    #   # Git LFS (Large File Storage)
+    #   brew install git-lfs
+    #   # Hugging Face uses LFS to store large model files, including .mlpackage
+    #   git lfs install
+    #   # Clone the repo with the smart_turn_classifier.mlpackage
+    #   git clone https://huggingface.co/pipecat-ai/smart-turn
+    #
+    # Then set the env variable:
+    #   export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
+    # or add it to your .env file
+    smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+            vad_audio_passthrough=True,
+            end_of_turn_analyzer=LocalCoreMLSmartTurnAnalyzer(
+                smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams(stop_secs=5)
path: examples/foundational/38a-local-smart-turn.py
user: login: filipi87
body: Yep, I have changed for testing, but we can restore to the default values.
created_at: 2025-04-17T20:44:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,111 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.smart_turn import SmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
path: examples/foundational/38-smart-turn.py
user: login: markbackman
body: One tricky part about using this will be that the VADParams value will need to be adjusted to a shorter time, like 0.2 second. I think we need to leave this part to developers and just write good documentation on also adjusting the VAD's stop_secs.

We could alternatively be tricky and provide a different VADParams(stop_secs) when the SmartTurnAnalyzer is used. I'm not sure if that's a good idea though. WDYT?
created_at: 2025-04-17T20:44:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,111 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.smart_turn import SmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
path: examples/foundational/38-smart-turn.py
user: login: filipi87
body: I think for this first version, we should leave it as is and recommend changing the stop_secs inside VADParams to 0.2.
Once we're really comfortable with it, we can reconsider changing its default value based on whether the SmartTurnAnalyzer is enabled.
If we go this route, maybe the user shouldn't even need to define the vad_analyzer. It could be automatically enabled when SmartTurnAnalyzer is enabled.
In any case, I think that should be implemented in a follow up PR.
created_at: 2025-04-17T20:49:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -187,17 +196,49 @@ async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState
             and new_vad_state != VADState.STOPPING
         ):
             frame = None
-            if new_vad_state == VADState.SPEAKING:
-                frame = UserStartedSpeakingFrame()
-            elif new_vad_state == VADState.QUIET:
-                frame = UserStoppedSpeakingFrame()
+            # If the turn analyser is enabled, this will prevent:
+            # - Creating the UserStoppedSpeakingFrame
+            # - Creating the UserStartedSpeakingFrame multiple times
+            can_create_user_frames = (
+                self._params.end_of_turn_analyzer is None
+                or not self._params.end_of_turn_analyzer.speech_triggered
+            )
+            if can_create_user_frames:
+                if new_vad_state == VADState.SPEAKING:
+                    frame = UserStartedSpeakingFrame()
+                elif new_vad_state == VADState.QUIET:
+                    frame = UserStoppedSpeakingFrame()
path: src/pipecat/transports/base_input.py
user: login: markbackman
body: I added an Observer for my testing and can confirm that the `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` work as expected.
created_at: 2025-04-17T20:50:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -571,6 +571,15 @@ class EmulateUserStoppedSpeakingFrame(SystemFrame):
     pass
 
 
+@dataclass
+class UserEndOfTurnFrame(SystemFrame):
path: src/pipecat/frames/frames.py
user: login: markbackman
body: Oh! Just catching this. We should delete this frame as it's unused.
created_at: 2025-04-17T20:52:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -187,17 +196,49 @@ async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState
             and new_vad_state != VADState.STOPPING
         ):
             frame = None
-            if new_vad_state == VADState.SPEAKING:
-                frame = UserStartedSpeakingFrame()
-            elif new_vad_state == VADState.QUIET:
-                frame = UserStoppedSpeakingFrame()
+            # If the turn analyser is enabled, this will prevent:
+            # - Creating the UserStoppedSpeakingFrame
+            # - Creating the UserStartedSpeakingFrame multiple times
+            can_create_user_frames = (
+                self._params.end_of_turn_analyzer is None
+                or not self._params.end_of_turn_analyzer.speech_triggered
+            )
+            if can_create_user_frames:
+                if new_vad_state == VADState.SPEAKING:
+                    frame = UserStartedSpeakingFrame()
+                elif new_vad_state == VADState.QUIET:
+                    frame = UserStoppedSpeakingFrame()
 
             if frame:
                 await self._handle_user_interruption(frame)
 
             vad_state = new_vad_state
         return vad_state
 
+    async def _handle_end_of_turn(self):
+        if self.end_of_turn_analyzer:
+            state = await self.get_event_loop().run_in_executor(
+                self._executor, self.end_of_turn_analyzer.analyze_end_of_turn
+            )
+            await self._handle_end_of_turn_complete(state)
+
+    async def _handle_end_of_turn_complete(self, state: EndOfTurnState):
+        if state == EndOfTurnState.COMPLETE:
+            await self.push_frame(UserEndOfTurnFrame())
path: src/pipecat/transports/base_input.py
user: login: markbackman
body: I guess we should just remove this, too.
created_at: 2025-04-17T20:54:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,111 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.smart_turn import SmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection):
+    logger.info(f"Starting bot")
+
+    remote_smart_turn_url = os.getenv("REMOTE_SMART_TURN_URL")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
path: examples/foundational/38-smart-turn.py
user: login: markbackman
body: Yup! I agree with all of this. No change right now. I'll make sure to make this clear in the docs, which I'll write right now.
created_at: 2025-04-17T20:56:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -79,6 +79,8 @@ qwen = []
 rime = [ "websockets~=13.1" ]
 riva = [ "nvidia-riva-client~=2.19.0" ]
 sentry = [ "sentry-sdk~=2.23.1" ]
+local-smart-turn = [ "coremltools>=8.0", "transformers", "torch==2.5.0", "torchaudio==2.5.0" ]
+remote-smart-turn = []
path: pyproject.toml
user: login: markbackman
body: @filipi87 maybe this should just be `smart-turn` to match the package name?
created_at: 2025-04-17T21:20:26Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-04-15T13:24:11Z
body: ## [Codecov](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai) Report
Attention: Patch coverage is `18.13953%` with `176 lines` in your changes missing coverage. Please review.

| [Files with missing lines](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai) | Patch % | Lines |
|---|---|---|
| [src/pipecat/audio/turn/base\_smart\_turn.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Faudio%2Fturn%2Fbase_smart_turn.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvYXVkaW8vdHVybi9iYXNlX3NtYXJ0X3R1cm4ucHk=) | 29.80% | [73 Missing :warning: ](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai) |
| [src/pipecat/audio/turn/smart\_turn.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Faudio%2Fturn%2Fsmart_turn.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvYXVkaW8vdHVybi9zbWFydF90dXJuLnB5) | 0.00% | [47 Missing :warning: ](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai) |
| [src/pipecat/audio/turn/local\_smart\_turn.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Faudio%2Fturn%2Flocal_smart_turn.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvYXVkaW8vdHVybi9sb2NhbF9zbWFydF90dXJuLnB5) | 0.00% | [33 Missing :warning: ](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai) |
| [src/pipecat/transports/base\_input.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Ftransports%2Fbase_input.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvdHJhbnNwb3J0cy9iYXNlX2lucHV0LnB5) | 20.68% | [23 Missing :warning: ](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai) |

| [Files with missing lines](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai) | Coverage Δ | |
|---|---|---|
| [src/pipecat/transports/base\_transport.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Ftransports%2Fbase_transport.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvdHJhbnNwb3J0cy9iYXNlX3RyYW5zcG9ydC5weQ==) | `88.88% <100.00%> (+0.51%)` | :arrow_up: |
| [src/pipecat/transports/base\_input.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Ftransports%2Fbase_input.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvdHJhbnNwb3J0cy9iYXNlX2lucHV0LnB5) | `21.62% <20.68%> (+0.48%)` | :arrow_up: |
| [src/pipecat/audio/turn/local\_smart\_turn.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Faudio%2Fturn%2Flocal_smart_turn.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvYXVkaW8vdHVybi9sb2NhbF9zbWFydF90dXJuLnB5) | `0.00% <0.00%> (ø)` | |
| [src/pipecat/audio/turn/smart\_turn.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Faudio%2Fturn%2Fsmart_turn.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvYXVkaW8vdHVybi9zbWFydF90dXJuLnB5) | `0.00% <0.00%> (ø)` | |
| [src/pipecat/audio/turn/base\_smart\_turn.py](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592?src=pr&el=tree&filepath=src%2Fpipecat%2Faudio%2Fturn%2Fbase_smart_turn.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai#diff-c3JjL3BpcGVjYXQvYXVkaW8vdHVybi9iYXNlX3NtYXJ0X3R1cm4ucHk=) | `29.80% <29.80%> (ø)` | |

... and [7 files with indirect coverage changes](https://app.codecov.io/gh/pipecat-ai/pipecat/pull/1592/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pipecat-ai)

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: markbackman
created_at: 2025-04-17T01:46:18Z
body: @filipi87 this is very cool!

I tried testing by setting the `SmartTurnParams` to `stop_secs` of 5.0 seconds. In doing this, I noticed two important things:
1. The frame counting mechanism results in the wrong time. I'm seeing the waiting time amount to half of the `stop_secs` time (e.g. 2.5 seconds of waiting instead of 5.0 seconds). I think this is due to differences in the chunk size. Can you take a look to ensure that the timing matches the setting? You might be able to calcuate the actual duration of the chunk instead of just a fixed frame (e.g. `actual_chunk_duration_ms = len(audio_int16) / (self._sample_rate / 1000)`).
2. Using the slower stop_secs time, it's easier to test how well the end of turn logic works. I've noticed that if I break a sentence across two phrases, I see an unexpected result. Here's what I see:
  a) Say "What's the uh...", receive INCOMPLETE
  b) Within the next 5 seconds, say "the capital of Spain?"
  
  I would expect to get a COMPLETE after b), but, I get an INCOMPLETE. I think what's happening (haven't checked) is that we're sending only the most audio chunk. That is, the model sees only "the capital of Spain" and deems that to be incomplete. Instead, after a) is returned as incomplete, I think we want to keep the chunk from a) and continue buffering audio until the next time the user stops speaking. This way, the smart-turn model gets the full input (e.g. "What's the uh... <pause> the capital of Spain?"). I have a hunch that this will result in a COMPLETE response after b) as the full audio input is closer to the training data and is an actual complete thought.
author_association: CONTRIBUTOR
user: login: filipi87
created_at: 2025-04-17T13:33:27Z
body: Hi @markbackman,

> I tried testing by setting the SmartTurnParams to stop_secs of 5.0 seconds. In doing this, I noticed two important things:

Yes, you were right. Somehow, when I looked at the property `audio_out_10ms_chunks`, I misread it as `audio_10ms_chunk` and thought the audio inputs would always be that size. It's already fixed now. Thank you for catching this.

> Regarding your second question:
 Using the slower stop_secs time, it’s easier to test how well the end of turn logic works. I’ve noticed that if I break a sentence across two phrases, I see an unexpected result. Here’s what I see:
a) Say “What’s the uh...“, receive INCOMPLETE
b) Within the next 5 seconds, say “the capital of Spain?”

This one it is just related with the `max_duration_secs`, the audio that we send to the smart turn, it is capped based on this maximum duration. Right now we are using the default value of 8 seconds.

author_association: CONTRIBUTOR