Repository: apify/crawlee-python
File: tests/unit/http_crawler/test_http_crawler.py (Python)

Code before:
                f'{httpbin}/cookies/set?b=2',
                f'{httpbin}/cookies/set?c=3',
            ]
        ),
        session_pool=session_pool,
    )

    @crawler.router.default_handler
    async def handler(context: HttpCrawlingContext) -> None:
        visit(context.request.url)
        track_session_usage(context.session.id if context.session else None)

    await crawler.run()

    visited = {call[0][0] for call in visit.call_args_list}
    assert len(visited) == 3

    session_ids = {call[0][0] for call in track_session_usage.call_args_list}
    assert len(session_ids) == 1

    session = await session_pool.get_session_by_id(session_ids.pop())
    assert session is not None
    assert session.cookies == {'a': '1', 'b': '2', 'c': '3'}


async def test_do_not_retry_on_client_erros(crawler: HttpCrawler, server: respx.MockRouter) -> None:
    await crawler.add_requests(['https://test.io/bad_request'])
    stats = await crawler.run()

    # by default, client errors are not retried
    assert stats.requests_failed == 1
    assert stats.retry_histogram == [1]
    assert stats.requests_total == 1

    assert len(server['bad_request_endpoint'].calls) == 1


async def test_http_status_statistics(crawler: HttpCrawler, server: respx.MockRouter) -> None:
    await crawler.add_requests([f'https://test.io/500?id={i}' for i in range(10)])
    await crawler.add_requests([f'https://test.io/404?id={i}' for i in range(10)])
    await crawler.add_requests([f'https://test.io/html?id={i}' for i in range(10)])

    await crawler.run()

    assert crawler.statistics.state.requests_with_status_code == {
        '200': 10,
        '404': 10,  # client errors are not retried by default
        '500': 30,  # server errors are retried by default
    }

    assert len(server['html_endpoint'].calls) == 10

Review comment (suggestion):
```suggestion
async def test_do_not_retry_on_client_errors(crawler: HttpCrawler, server: respx.MockRouter) -> None:
```

Code after:
                f'{httpbin}/cookies/set?b=2',
                f'{httpbin}/cookies/set?c=3',
            ]
        ),
        session_pool=session_pool,
    )

    @crawler.router.default_handler
    async def handler(context: HttpCrawlingContext) -> None:
        visit(context.request.url)
        track_session_usage(context.session.id if context.session else None)

    await crawler.run()

    visited = {call[0][0] for call in visit.call_args_list}
    assert len(visited) == 3

    session_ids = {call[0][0] for call in track_session_usage.call_args_list}
    assert len(session_ids) == 1

    session = await session_pool.get_session_by_id(session_ids.pop())
    assert session is not None
    assert session.cookies == {'a': '1', 'b': '2', 'c': '3'}


async def test_do_not_retry_on_client_errors(crawler: HttpCrawler, server: respx.MockRouter) -> None:
    await crawler.add_requests(['https://test.io/bad_request'])
    stats = await crawler.run()

    # by default, client errors are not retried
    assert stats.requests_failed == 1
    assert stats.retry_histogram == [1]
    assert stats.requests_total == 1

    assert len(server['bad_request_endpoint'].calls) == 1


async def test_http_status_statistics(crawler: HttpCrawler, server: respx.MockRouter) -> None:
    await crawler.add_requests([f'https://test.io/500?id={i}' for i in range(10)])
    await crawler.add_requests([f'https://test.io/404?id={i}' for i in range(10)])
    await crawler.add_requests([f'https://test.io/html?id={i}' for i in range(10)])

    await crawler.run()

    assert crawler.statistics.state.requests_with_status_code == {
        '200': 10,
        '404': 10,  # client errors are not retried by default
        '500': 30,  # server errors are retried by default
    }

    assert len(server['html_endpoint'].calls) == 10