Repository: livekit/agents
File: livekit-agents/livekit/agents/llm/fallback_adapter.py (Python)

Code before:
                            "llm_availability_changed",
                            AvailabilityChangedEvent(llm=llm, available=False),
                        )

                    if text_sent or tool_calls_sent:
                        extra = {"text_sent": text_sent, "tool_calls_sent": tool_calls_sent}
                        if not self._fallback_adapter._retry_on_chunk_sent:
                            logger.error(
                                f"{llm.label} failed after sending chunk, skip retrying. "
                                "Set `retry_on_chunk_sent` to `True` to enable retrying after chunks are sent.",
                                extra=extra,
                            )
                            raise

                        logger.warning(
                            f"{llm.label} failed after sending chunk, retrying..",
                            extra=extra,
                        )

            self._try_recovery(llm)

        raise APIConnectionError(
            f"all LLMs failed ({[llm.label for llm in self._fallback_adapter._llm_instances]}) after {time.time() - start_time} seconds"  # noqa: E501
        )

    # Overriding the parent class method to skip nested duplicate traces
    async def _main_task(self) -> None:
        for i in range(self._conn_options.max_retry + 1):
            try:
                try:
                    return await self._run()
                except Exception:
                    raise
            except APIError as e:
                retry_interval = self._conn_options._interval_for_retry(i)

                if self._conn_options.max_retry == 0 or not e.retryable:
                    self._emit_error(e, recoverable=False)
                    raise
                elif i == self._conn_options.max_retry:
                    self._emit_error(e, recoverable=False)
                    raise APIConnectionError(
                        f"failed to generate LLM completion after {self._conn_options.max_retry + 1} attempts",  # noqa: E501
                    ) from e

                else:
                    self._emit_error(e, recoverable=True)
                    logger.warning(
                        f"failed to generate LLM completion, retrying in {retry_interval}s",  # noqa: E501
                        exc_info=e,
                        extra={

Review comment (bug):
I was thinking the nested spans from fallback and stream adapters are expected, maybe it makes the trace more complex but it reflects the actual behavior when these adapters are used. one benefit is when a LLM failed, you will see the failed one and the success one nested under the same fallback adapter span.

btw, if we are going to avoid this, we should remove it for both fallback and stream adapter, and also TTS and STT.

Code after:
                        llm_status.available = False
                        self._fallback_adapter.emit(
                            "llm_availability_changed",
                            AvailabilityChangedEvent(llm=llm, available=False),
                        )

                    if text_sent or tool_calls_sent:
                        extra = {"text_sent": text_sent, "tool_calls_sent": tool_calls_sent}
                        if not self._fallback_adapter._retry_on_chunk_sent:
                            logger.error(
                                f"{llm.label} failed after sending chunk, skip retrying. "
                                "Set `retry_on_chunk_sent` to `True` to enable retrying after chunks are sent.",
                                extra=extra,
                            )
                            raise

                        logger.warning(
                            f"{llm.label} failed after sending chunk, retrying..",
                            extra=extra,
                        )

            self._try_recovery(llm)

        raise APIConnectionError(
            f"all LLMs failed ({[llm.label for llm in self._fallback_adapter._llm_instances]}) after {time.time() - start_time} seconds"  # noqa: E501
        )

    async def _metrics_monitor_task(self, event_aiter: AsyncIterable[ChatChunk]) -> None:
        return