""" This module contains component intended to use in combination with `InferencePipeline` to ensure observability. Please consider them internal details of implementation. """ from abc import ABC, abstractmethod from collections import deque from datetime import datetime from typing import Any, Deque, Iterable, List, Optional, TypeVar import supervision as sv from inference.core.interfaces.camera.entities import StatusUpdate, UpdateSeverity from inference.core.interfaces.camera.video_source import VideoSource from inference.core.interfaces.stream.entities import ( LatencyMonitorReport, ModelActivityEvent, PipelineStateReport, ) T = TypeVar("T") MAX_LATENCY_CONTEXT = 64 MAX_UPDATES_CONTEXT = 512 class PipelineWatchDog(ABC): def __init__(self): pass @abstractmethod def register_video_source(self, video_source: VideoSource) -> None: pass @abstractmethod def on_status_update(self, status_update: StatusUpdate) -> None: pass @abstractmethod def on_model_preprocessing_started( self, frame_timestamp: datetime, frame_id: int ) -> None: pass @abstractmethod def on_model_inference_started( self, frame_timestamp: datetime, frame_id: int ) -> None: pass @abstractmethod def on_model_postprocessing_started( self, frame_timestamp: datetime, frame_id: int ) -> None: pass @abstractmethod def on_model_prediction_ready( self, frame_timestamp: datetime, frame_id: int ) -> None: pass @abstractmethod def get_report(self) -> Optional[PipelineStateReport]: pass class NullPipelineWatchdog(PipelineWatchDog): def register_video_source(self, video_source: VideoSource) -> None: pass def on_status_update(self, status_update: StatusUpdate) -> None: pass def on_model_preprocessing_started( self, frame_timestamp: datetime, frame_id: int ) -> None: pass def on_model_inference_started( self, frame_timestamp: datetime, frame_id: int ) -> None: pass def on_model_postprocessing_started( self, frame_timestamp: datetime, frame_id: int ) -> None: pass def on_model_prediction_ready( self, frame_timestamp: datetime, frame_id: int ) -> None: pass def get_report(self) -> Optional[PipelineStateReport]: return None class LatencyMonitor: def __init__(self): self._preprocessing_start_event: Optional[ModelActivityEvent] = None self._inference_start_event: Optional[ModelActivityEvent] = None self._postprocessing_start_event: Optional[ModelActivityEvent] = None self._prediction_ready_event: Optional[ModelActivityEvent] = None self._reports: Deque[LatencyMonitorReport] = deque(maxlen=MAX_LATENCY_CONTEXT) def register_preprocessing_start( self, frame_timestamp: datetime, frame_id: int ) -> None: self._preprocessing_start_event = ModelActivityEvent( event_timestamp=datetime.now(), frame_id=frame_id, frame_decoding_timestamp=frame_timestamp, ) def register_inference_start( self, frame_timestamp: datetime, frame_id: int ) -> None: self._inference_start_event = ModelActivityEvent( event_timestamp=datetime.now(), frame_id=frame_id, frame_decoding_timestamp=frame_timestamp, ) def register_postprocessing_start( self, frame_timestamp: datetime, frame_id: int ) -> None: self._postprocessing_start_event = ModelActivityEvent( event_timestamp=datetime.now(), frame_id=frame_id, frame_decoding_timestamp=frame_timestamp, ) def register_prediction_ready( self, frame_timestamp: datetime, frame_id: int ) -> None: self._prediction_ready_event = ModelActivityEvent( event_timestamp=datetime.now(), frame_id=frame_id, frame_decoding_timestamp=frame_timestamp, ) self._generate_report() def summarise_reports(self) -> LatencyMonitorReport: avg_frame_decoding_latency = average_property_values( examined_objects=self._reports, property_name="frame_decoding_latency" ) avg_pre_processing_latency = average_property_values( examined_objects=self._reports, property_name="pre_processing_latency" ) avg_inference_latency = average_property_values( examined_objects=self._reports, property_name="inference_latency" ) avg_pos_processing_latency = average_property_values( examined_objects=self._reports, property_name="post_processing_latency" ) avg_model_latency = average_property_values( examined_objects=self._reports, property_name="model_latency" ) avg_e2e_latency = average_property_values( examined_objects=self._reports, property_name="e2e_latency" ) return LatencyMonitorReport( frame_decoding_latency=avg_frame_decoding_latency, pre_processing_latency=avg_pre_processing_latency, inference_latency=avg_inference_latency, post_processing_latency=avg_pos_processing_latency, model_latency=avg_model_latency, e2e_latency=avg_e2e_latency, ) def _generate_report(self) -> None: frame_decoding_latency = None if self._preprocessing_start_event is not None: frame_decoding_latency = ( self._preprocessing_start_event.event_timestamp - self._preprocessing_start_event.frame_decoding_timestamp ).total_seconds() event_pairs = [ (self._preprocessing_start_event, self._inference_start_event), (self._inference_start_event, self._postprocessing_start_event), (self._postprocessing_start_event, self._prediction_ready_event), (self._preprocessing_start_event, self._prediction_ready_event), ] event_pairs_results = [] for earlier_event, later_event in event_pairs: latency = compute_events_latency( earlier_event=earlier_event, later_event=later_event, ) event_pairs_results.append(latency) ( pre_processing_latency, inference_latency, post_processing_latency, model_latency, ) = event_pairs_results e2e_latency = None if self._prediction_ready_event is not None: e2e_latency = ( self._prediction_ready_event.event_timestamp - self._prediction_ready_event.frame_decoding_timestamp ).total_seconds() self._reports.append( LatencyMonitorReport( frame_decoding_latency=frame_decoding_latency, pre_processing_latency=pre_processing_latency, inference_latency=inference_latency, post_processing_latency=post_processing_latency, model_latency=model_latency, e2e_latency=e2e_latency, ) ) def average_property_values( examined_objects: Iterable, property_name: str ) -> Optional[float]: values = get_not_empty_properties( examined_objects=examined_objects, property_name=property_name ) return safe_average(values=values) def get_not_empty_properties( examined_objects: Iterable, property_name: str ) -> List[Any]: results = [ getattr(examined_object, property_name, None) for examined_object in examined_objects ] return [e for e in results if e is not None] def safe_average(values: List[float]) -> Optional[float]: if len(values) == 0: return None return sum(values) / len(values) def compute_events_latency( earlier_event: Optional[ModelActivityEvent], later_event: Optional[ModelActivityEvent], ) -> Optional[float]: if not are_events_compatible(events=[earlier_event, later_event]): return None return (later_event.event_timestamp - earlier_event.event_timestamp).total_seconds() def are_events_compatible(events: List[Optional[ModelActivityEvent]]) -> bool: if any(e is None for e in events): return False if len(events) == 0: return False frame_ids = [e.frame_id for e in events] return all(e == frame_ids[0] for e in frame_ids) class BasePipelineWatchDog(PipelineWatchDog): """ Implementation to be used from single inference thread, as it keeps state assumed to represent status of consecutive stage of prediction process in latency monitor. """ def __init__(self): super().__init__() self._video_source: Optional[VideoSource] = None self._inference_throughput_monitor = sv.FPSMonitor() self._latency_monitor = LatencyMonitor() self._stream_updates = deque(maxlen=MAX_UPDATES_CONTEXT) def register_video_source(self, video_source: VideoSource) -> None: self._video_source = video_source def on_status_update(self, status_update: StatusUpdate) -> None: if status_update.severity.value <= UpdateSeverity.DEBUG.value: return None self._stream_updates.append(status_update) def on_model_preprocessing_started( self, frame_timestamp: datetime, frame_id: int ) -> None: self._latency_monitor.register_preprocessing_start( frame_timestamp=frame_timestamp, frame_id=frame_id ) def on_model_inference_started( self, frame_timestamp: datetime, frame_id: int ) -> None: self._latency_monitor.register_inference_start( frame_timestamp=frame_timestamp, frame_id=frame_id ) def on_model_postprocessing_started( self, frame_timestamp: datetime, frame_id: int ) -> None: self._latency_monitor.register_postprocessing_start( frame_timestamp=frame_timestamp, frame_id=frame_id ) def on_model_prediction_ready( self, frame_timestamp: datetime, frame_id: int ) -> None: self._latency_monitor.register_prediction_ready( frame_timestamp=frame_timestamp, frame_id=frame_id ) self._inference_throughput_monitor.tick() def get_report(self) -> PipelineStateReport: source_metadata = None if self._video_source is not None: source_metadata = self._video_source.describe_source() return PipelineStateReport( video_source_status_updates=list(self._stream_updates), latency_report=self._latency_monitor.summarise_reports(), inference_throughput=self._inference_throughput_monitor(), source_metadata=source_metadata, )