Repository: livekit/agents
File: livekit-agents/livekit/agents/voice/recorder_io/recorder_io.py (Python)

Code before:
        return self.__started_time

    def take_buf(self) -> list[rtc.AudioFrame]:
        frames = self.__acc_frames
        self.__acc_frames = []
        return frames

    def __aiter__(self) -> AsyncIterator[rtc.AudioFrame]:
        return self

    async def __anext__(self) -> rtc.AudioFrame:
        frame = await self.__audio_input.__anext__()

        if self.__recording_io.recording:
            if self.__started_time is None:
                self.__started_time = time.time()

            self.__acc_frames.append(frame)

        return frame


class RecorderAudioOutput(io.AudioOutput):
    def __init__(
        self,
        *,
        recording_io: RecorderIO,
        audio_output: io.AudioOutput | None = None,
        write_fnc: Callable[[list[rtc.AudioFrame]], Any],
    ) -> None:
        super().__init__(
            label="RecorderIO",
            next_in_chain=audio_output,
            sample_rate=audio_output.sample_rate if audio_output else None,
            # TODO: support pause
            capabilities=io.AudioOutputCapabilities(pause=True),  # depends on the next_in_chain
        )
        self.__recording_io = recording_io
        self.__write = write_fnc
        self.__acc_frames: list[rtc.AudioFrame] = []
        self.__started_time: None | float = None

        # pause tracking
        self.__current_pause_start: float | None = None
        self.__pause_wall_times: list[tuple[float, float]] = []

    @property
    def started_wall_time(self) -> float | None:
        return self.__started_time

    @property

Review comment (suggestion):
if I understand correctly, the issue is the silent duration is not counted when both input and output buffers are empty. so it's not only the duration after the first response to the time user enables the mic, but also before the first response generated (LLM + TTS time).

can we record the time the `RecordIO.start` called as `last_take_time`, then modify `RecorderAudioInput.take_buf` to return a silence frame with duration `current_time - last_take_time` before there is any frame added to the input buffer?

then the total padding duration should be `started_wall_time - RecordIO.start_wall_time`


Code after:
        self.__started_time: None | float = None
        self.__padded: bool = False

    @property
    def started_wall_time(self) -> float | None:
        return self.__started_time

    def take_buf(self, pad_since: float | None = None) -> list[rtc.AudioFrame]:
        frames = self.__acc_frames
        self.__acc_frames = []
        if (
            pad_since
            and self.__started_time
            and (padding := self.__started_time - pad_since) > 0
            and not self.__padded
            and len(frames) > 0
        ):
            logger.warning(
                "input speech started after last agent speech ended",
                extra={
                    "last_agent_speech_time": pad_since,
                    "input_started_time": self.__started_time,
                },
            )
            self.__padded = True
            frames = [
                _create_silence_frame(padding, frames[0].sample_rate, frames[0].num_channels),
                *frames,
            ]
        # we could pad with silence here with some fixed SR and channels,
        # but it's better for the user to know that this is happening
        elif pad_since and self.__started_time is None and not self.__padded and not frames:
            logger.warning(
                "input speech hasn't started yet, skipping silence padding, "
                "recording may be inaccurate until the speech starts"
            )

        return frames

    def __aiter__(self) -> AsyncIterator[rtc.AudioFrame]:
        return self

    async def __anext__(self) -> rtc.AudioFrame:
        frame = await self.__audio_input.__anext__()

        if self.__recording_io.recording:
            if self.__started_time is None:
                self.__started_time = time.time()

            self.__acc_frames.append(frame)