Repository: apify/crawlee-python
File: src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py (Python)

Code before:

    async def __aenter__(self) -> RenderingTypePredictor:
        """Initialize the predictor upon entering the context manager."""
        await self.initialize()
        return self

    async def __aexit__(
        self,
        exc_type: type[BaseException] | None,
        exc_value: BaseException | None,
        exc_traceback: TracebackType | None,
    ) -> None:
        """Clear the predictor upon exiting the context manager."""
        await self.clear()


@docs_group('Other')
class DefaultRenderingTypePredictor(RenderingTypePredictor):
    """Stores rendering type for previously crawled URLs and predicts the rendering type for unvisited urls.

    `RenderingTypePredictor` implementation based on logistic regression:
    https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
    """

    def __init__(
        self, detection_ratio: float = 0.1, *, persistence_enabled: bool = False, persist_state_key: str | None = None
    ) -> None:
        """Initialize a new instance.

        Args:
            detection_ratio: A number between 0 and 1 that determines the desired ratio of rendering type detections.
            persist_state_key: Key in the key-value storage where the trained model parameters will be saved.
            If None, defaults to 'rendering-type-predictor-state'.
            persistence_enabled: Whether to enable persistence of the trained model parameters for reuse.

        """
        super().__init__()

        self._rendering_type_detection_results: dict[RenderingType, dict[str, list[UrlComponents]]] = {
            'static': defaultdict(list),
            'client only': defaultdict(list),
        }
        self._detection_ratio = max(0, min(1, detection_ratio))

        # Used to increase detection probability recommendation for initial recommendations of each label.
        # Reaches 1 (no additional increase) after n samples of specific label is already present in
        # `self._rendering_type_detection_results`.
        n = 3

        self._state = RecoverableState(
            default_state=RenderingTypePredictorState(

Review comment (suggestion):
Maybe directly set default, or is there a reason for `None`?
persist_state_key: str  = "rendering-type-predictor-state"

Code after:

    async def __aenter__(self) -> RenderingTypePredictor:
        """Initialize the predictor upon entering the context manager."""
        await self.initialize()
        return self

    async def __aexit__(
        self,
        exc_type: type[BaseException] | None,
        exc_value: BaseException | None,
        exc_traceback: TracebackType | None,
    ) -> None:
        """Clear the predictor upon exiting the context manager."""
        await self.clear()


@docs_group('Other')
class DefaultRenderingTypePredictor(RenderingTypePredictor):
    """Stores rendering type for previously crawled URLs and predicts the rendering type for unvisited urls.

    `RenderingTypePredictor` implementation based on logistic regression:
    https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html
    """

    def __init__(
        self,
        detection_ratio: float = 0.1,
        *,
        persistence_enabled: bool = False,
        persist_state_key: str = 'rendering-type-predictor-state',
    ) -> None:
        """Initialize a new instance.

        Args:
            detection_ratio: A number between 0 and 1 that determines the desired ratio of rendering type detections.
            persist_state_key: Key in the key-value storage where the trained model parameters will be saved.
            If None, defaults to 'rendering-type-predictor-state'.
            persistence_enabled: Whether to enable persistence of the trained model parameters for reuse.

        """
        super().__init__()

        self._rendering_type_detection_results: dict[RenderingType, dict[str, list[UrlComponents]]] = {
            'static': defaultdict(list),
            'client only': defaultdict(list),
        }
        self._detection_ratio = max(0, min(1, detection_ratio))

        # Used to increase detection probability recommendation for initial recommendations of each label.
        # Reaches 1 (no additional increase) after n samples of specific label is already present in
        # `self._rendering_type_detection_results`.