Repository: apify/crawlee-python
File: tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py (Python)

Code before:

async def test_adaptive_context_query_non_existing_element(test_urls: list[str]) -> None:
    """Test that querying non-existing selector returns `None`"""
    browser_only_predictor_no_detection = _SimpleRenderingTypePredictor(
        rendering_types=cycle(['client only']), detection_probability_recommendation=cycle([0])
    )

    crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
        max_request_retries=1,
        rendering_type_predictor=browser_only_predictor_no_detection,
    )

    mocked_h3_handler = Mock()

    @crawler.router.default_handler
    async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
        mocked_h3_handler(await context.query_selector_one('non sense selector', timeout=timedelta(milliseconds=1)))

    await crawler.run(test_urls[:1])

    mocked_h3_handler.assert_called_once_with(None)


async def test_change_context_state_after_handling(server_url: URL) -> None:
    """Test that context state is saved after handling the request."""
    static_only_predictor_no_detection = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0]))

    test_mapper = {}
    request_queue = await RequestQueue.open(name='state-test')
    async with SessionPool() as session_pool:
        crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
            rendering_type_predictor=static_only_predictor_no_detection,
            session_pool=session_pool,
            request_manager=request_queue,
        )

        @crawler.router.default_handler
        async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
            if context.session is not None:
                test_mapper['session_id'] = context.session.id
                context.session.user_data['session_state'] = True
            context.request.user_data['request_state'] = True

        request = Request.from_url(str(server_url))

        await crawler.run([request])

        assert test_mapper['session_id'] is not None

        session = await session_pool.get_session_by_id(test_mapper['session_id'])
        check_request = await request_queue.get_request(request.unique_key)

Review comment (suggestion):
I think that the test should cover all three possible cases - static crawl, browser crawl and rendering type detection (both crawl types).

Also, please make sure that the modifications to user data are not done twice in the case of a detection. Adding something to a list-typed user data item should do nicely.

Code after:


async def test_adaptive_context_query_non_existing_element(test_urls: list[str]) -> None:
    """Test that querying non-existing selector returns `None`"""
    browser_only_predictor_no_detection = _SimpleRenderingTypePredictor(
        rendering_types=cycle(['client only']), detection_probability_recommendation=cycle([0])
    )

    crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
        max_request_retries=1,
        rendering_type_predictor=browser_only_predictor_no_detection,
    )

    mocked_h3_handler = Mock()

    @crawler.router.default_handler
    async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
        mocked_h3_handler(await context.query_selector_one('non sense selector', timeout=timedelta(milliseconds=1)))

    await crawler.run(test_urls[:1])

    mocked_h3_handler.assert_called_once_with(None)


@pytest.mark.parametrize(
    'test_input',
    [
        pytest.param(
            TestInput(
                expected_pw_count=0,
                expected_static_count=2,
                rendering_types=cycle(['static']),
                detection_probability_recommendation=cycle([0]),
            ),
            id='Static only',
        ),
        pytest.param(
            TestInput(
                expected_pw_count=2,
                expected_static_count=0,
                rendering_types=cycle(['client only']),
                detection_probability_recommendation=cycle([0]),
            ),
            id='Client only',
        ),
        pytest.param(
            TestInput(
                expected_pw_count=2,
                expected_static_count=2,
                rendering_types=cycle(['static', 'client only']),
                detection_probability_recommendation=cycle([1]),