Repository: apify/crawlee-python
File: tests/unit/crawlers/_playwright/test_playwright_crawler.py (Python)

Code before:
    error_handler = mock.AsyncMock(return_value=None)
    crawler.error_handler(error_handler)

    failed_handler = mock.AsyncMock(return_value=None)
    crawler.failed_request_handler(failed_handler)

    @crawler.router.default_handler
    async def request_handler(_context: PlaywrightCrawlingContext) -> None:
        pass

    await crawler.run(['https://this-does-not-exist-22343434.com'])
    assert error_handler.call_count == 3
    assert failed_handler.call_count == 1


async def test_redirect_handling(httpbin: URL) -> None:
    crawler = PlaywrightCrawler()
    handled_urls = set[str]()

    @crawler.router.default_handler
    async def request_handler(context: PlaywrightCrawlingContext) -> None:
        handled_urls.add(context.request.loaded_url or '')

    request = Request.from_url(
        url=str((httpbin / 'redirect-to').update_query(url='https://apify.com')),
    )

    request.crawlee_data.enqueue_strategy = EnqueueStrategy.SAME_ORIGIN

    await crawler.run([request])
    assert handled_urls == set()


async def test_chromium_headless_headers(httpbin: URL) -> None:
    crawler = PlaywrightCrawler(headless=True, browser_type='chromium')
    headers = dict[str, str]()

    @crawler.router.default_handler
    async def request_handler(context: PlaywrightCrawlingContext) -> None:
        response = await context.response.text()
        response_headers = dict(json.loads(response)).get('headers', {})

        for key, val in response_headers.items():
            headers[key] = val

    await crawler.run([str(httpbin / 'get')])

    assert 'User-Agent' in headers
    assert 'Sec-Ch-Ua' in headers
    assert 'Sec-Ch-Ua-Mobile' in headers
    assert 'Sec-Ch-Ua-Platform' in headers

Review comment (suggestion):
I would suggest to add a comment that describes this very specific behavior of httpbin site, that creates redirect based on this specific query parameter.
Maybe add link. To me most readable was https://httpbin.org/legacy  (but maybe there is better)

Code after:
    error_handler = mock.AsyncMock(return_value=None)
    crawler.error_handler(error_handler)

    failed_handler = mock.AsyncMock(return_value=None)
    crawler.failed_request_handler(failed_handler)

    @crawler.router.default_handler
    async def request_handler(_context: PlaywrightCrawlingContext) -> None:
        pass

    await crawler.run(['https://this-does-not-exist-22343434.com'])
    assert error_handler.call_count == 3
    assert failed_handler.call_count == 1


async def test_redirect_handling(httpbin: URL) -> None:
    # Set up a dummy crawler that tracks visited URLs
    crawler = PlaywrightCrawler()
    handled_urls = set[str]()

    @crawler.router.default_handler
    async def request_handler(context: PlaywrightCrawlingContext) -> None:
        handled_urls.add(context.request.loaded_url or '')

    # Craft a request that points to httpbin initially, but redirects to apify.com
    request = Request.from_url(
        url=str((httpbin / 'redirect-to').update_query(url='https://apify.com')),
    )

    # Ensure that the request uses the SAME_ORIGIN strategy - apify.com will be considered out of scope
    request.crawlee_data.enqueue_strategy = EnqueueStrategy.SAME_ORIGIN

    # No URLs should be visited in the run
    await crawler.run([request])
    assert handled_urls == set()


async def test_chromium_headless_headers(httpbin: URL) -> None:
    crawler = PlaywrightCrawler(headless=True, browser_type='chromium')
    headers = dict[str, str]()

    @crawler.router.default_handler
    async def request_handler(context: PlaywrightCrawlingContext) -> None:
        response = await context.response.text()
        response_headers = dict(json.loads(response)).get('headers', {})

        for key, val in response_headers.items():
            headers[key] = val

    await crawler.run([str(httpbin / 'get')])