Repository: apify/crawlee-python
File: tests/unit/crawlers/_adaptive_playwright/test_predictor.py (Python)

Code before:
    assert persisted_data['model']['is_fitted'] is True


async def test_persistent_prediction_recovery() -> None:
    """Test that the model and resources is recovered from KeyValueStore."""
    persist_key = 'test-persistent-state-recovery'

    async with DefaultRenderingTypePredictor(
        detection_ratio=0.01, persistence_enabled=True, persist_state_key=persist_key
    ) as predictor:
        # Learn some data
        predictor.store_result(
            Request.from_url(url='http://www.aaa.com/some/stuff', label='some label'), rendering_type='static'
        )
        before_recover_prediction = predictor.predict(
            Request.from_url(url='http://www.aaa.com/some/stuff', label='some label')
        )

    # Recover predictor
    async with DefaultRenderingTypePredictor(
        detection_ratio=0.01, persistence_enabled=True, persist_state_key=persist_key
    ) as recover_predictor:
        after_recover_prediction = recover_predictor.predict(
            Request.from_url(url='http://www.aaa.com/some/stuff', label='some label')
        )

    assert (
        before_recover_prediction.detection_probability_recommendation
        == after_recover_prediction.detection_probability_recommendation
    )


@pytest.mark.parametrize(
    ('url_1', 'url_2', 'expected_rounded_similarity'),
    [
        (
            'https://docs.python.org/3/library/itertools.html#itertools.zip_longest',
            'https://docs.python.org/3.7/library/itertools.html#itertools.zip_longest',
            0.67,
        ),
        ('https://differente.com/same', 'https://differenta.com/same', 0),
        ('https://same.com/almost_the_same', 'https://same.com/almost_the_sama', 1),
        ('https://same.com/same/extra', 'https://same.com/same', 0.5),
    ],
)
def test_url_similarity(url_1: str, url_2: str, expected_rounded_similarity: float) -> None:
    assert (
        round(calculate_url_similarity(url_1=get_url_components(url_1), url_2=get_url_components(url_2)), 2)
        == expected_rounded_similarity
    )


Review comment (suggestion):
Could we also add to this test that another instance of `DefaultRenderingTypePredictor`  created without persistence gives a different prediction? (To avoid a situation where we would assert some default behavior of the predictor regardless of the persistence)

Code after:
    assert persisted_data['model']['is_fitted'] is True


@pytest.mark.parametrize(
    ('persistence_enabled', 'same_result'),
    [
        pytest.param(True, True, id='with persistence'),
        pytest.param(False, False, id='without persistence'),
    ],
)
async def test_persistent_prediction_recovery(*, persistence_enabled: bool, same_result: bool) -> None:
    """Test that the model and resources is recovered from KeyValueStore."""
    persist_key = 'test-persistent-state-recovery'

    async with DefaultRenderingTypePredictor(
        detection_ratio=0.01, persistence_enabled=persistence_enabled, persist_state_key=persist_key
    ) as predictor:
        # Learn some data
        predictor.store_result(
            Request.from_url(url='http://www.aaa.com/some/stuff', label='some label'), rendering_type='static'
        )
        before_recover_prediction = predictor.predict(
            Request.from_url(url='http://www.aaa.com/some/stuff', label='some label')
        )

    # Recover predictor
    async with DefaultRenderingTypePredictor(
        detection_ratio=0.01, persistence_enabled=True, persist_state_key=persist_key
    ) as recover_predictor:
        after_recover_prediction = recover_predictor.predict(
            Request.from_url(url='http://www.aaa.com/some/stuff', label='some label')
        )

    # If persistence is enabled, the predicted results must be the same.
    if same_result:
        assert (
            before_recover_prediction.detection_probability_recommendation
            == after_recover_prediction.detection_probability_recommendation
        )
    else:
        assert (
            before_recover_prediction.detection_probability_recommendation
            != after_recover_prediction.detection_probability_recommendation
        )


@pytest.mark.parametrize(
    ('url_1', 'url_2', 'expected_rounded_similarity'),
    [
        (
            'https://docs.python.org/3/library/itertools.html#itertools.zip_longest',