Repository: apify/crawlee-python
File: tests/unit/crawlers/_http/test_http_crawler.py (Python)

Code before:
    [
        pytest.param(CurlImpersonateHttpClient, id='curl'),
        pytest.param(HttpxHttpClient, id='httpx'),
    ],
)
async def test_isolation_cookies(http_client_class: type[BaseHttpClient], httpbin: URL) -> None:
    http_client = http_client_class()
    sessions_ids: list[str] = []
    sessions_cookies: dict[str, dict[str, str]] = {}
    response_cookies: dict[str, dict[str, str]] = {}

    crawler = HttpCrawler(
        session_pool=SessionPool(max_pool_size=1),
        http_client=http_client,
        concurrency_settings=ConcurrencySettings(max_concurrency=1),
    )

    @crawler.router.default_handler
    async def handler(context: HttpCrawlingContext) -> None:
        if context.session:
            sessions_ids.append(context.session.id)
            if context.request.unique_key in {'1', '2'}:
                sessions_cookies[context.session.id] = context.session.cookies
                response_data = json.loads(context.http_response.read())
                response_cookies[context.session.id] = response_data.get('cookies')
                if context.request.unique_key == '1':
                    context.session.retire()

    await crawler.run(
        [
            # The first request sets the cookie in the session
            str(httpbin.with_path('/cookies/set').extend_query(a=1)),
            # With the second request, we check the cookies in the session and set retire
            Request.from_url(str(httpbin.with_path('/cookies')), unique_key='1'),
            # The third request is made with a new session to make sure it does not use another session's cookies
            Request.from_url(str(httpbin.with_path('/cookies')), unique_key='2'),
        ]
    )

    assert len(sessions_cookies) == 2
    assert len(response_cookies) == 2

    assert sessions_ids[0] == sessions_ids[1]

    cookie_session_id = sessions_ids[0]
    clean_session_id = sessions_ids[2]

    assert cookie_session_id != clean_session_id

    # The initiated cookies must match in both the response and the session store
    assert sessions_cookies[cookie_session_id] == response_cookies[cookie_session_id] == {'a': '1'}

Review comment (suggestion):
Instead of "encoding" this into the unique key, you could use a flag in `user_data`, e.g., `if context.request.user_data.get('retire_session', False)`

Code after:
    [
        pytest.param(CurlImpersonateHttpClient, id='curl'),
        pytest.param(HttpxHttpClient, id='httpx'),
    ],
)
async def test_isolation_cookies(http_client_class: type[BaseHttpClient], httpbin: URL) -> None:
    http_client = http_client_class()
    sessions_ids: list[str] = []
    sessions_cookies: dict[str, dict[str, str]] = {}
    response_cookies: dict[str, dict[str, str]] = {}

    crawler = HttpCrawler(
        session_pool=SessionPool(max_pool_size=1),
        http_client=http_client,
        concurrency_settings=ConcurrencySettings(max_concurrency=1),
    )

    @crawler.router.default_handler
    async def handler(context: HttpCrawlingContext) -> None:
        if not context.session:
            return

        sessions_ids.append(context.session.id)

        if context.request.unique_key not in {'1', '2'}:
            return

        sessions_cookies[context.session.id] = context.session.cookies
        response_data = json.loads(context.http_response.read())
        response_cookies[context.session.id] = response_data.get('cookies')

        if context.request.user_data.get('retire_session'):
            context.session.retire()

    await crawler.run(
        [
            # The first request sets the cookie in the session
            str(httpbin.with_path('/cookies/set').extend_query(a=1)),
            # With the second request, we check the cookies in the session and set retire
            Request.from_url(str(httpbin.with_path('/cookies')), unique_key='1', user_data={'retire_session': True}),
            # The third request is made with a new session to make sure it does not use another session's cookies
            Request.from_url(str(httpbin.with_path('/cookies')), unique_key='2'),
        ]
    )

    assert len(sessions_cookies) == 2
    assert len(response_cookies) == 2

    assert sessions_ids[0] == sessions_ids[1]

    cookie_session_id = sessions_ids[0]