Repository: apify/crawlee-python
File: src/crawlee/crawlers/_playwright/_playwright_crawler.py (Python)

Code before:
        if self._browser_pool is None:
            raise ValueError('Browser pool is not initialized.')

        # Create a new browser page
        crawlee_page = await self._browser_pool.new_page(proxy_info=context.proxy_info)

        pre_navigation_context = PlaywrightPreNavCrawlingContext(
            request=context.request,
            session=context.session,
            add_requests=context.add_requests,
            send_request=context.send_request,
            push_data=context.push_data,
            use_state=context.use_state,
            proxy_info=context.proxy_info,
            get_key_value_store=context.get_key_value_store,
            log=context.log,
            page=crawlee_page.page,
            block_requests=partial(block_requests, page=crawlee_page.page),
        )

        async with browser_page_context(crawlee_page.page):
            for hook in self._pre_navigation_hooks:
                await hook(pre_navigation_context)
        yield pre_navigation_context

    def _prepare_request_handler(
        self,
        method: HttpMethod = 'GET',
        headers: HttpHeaders | dict[str, str] | None = None,
        payload: HttpPayload | None = None,
    ) -> Callable:
        """Create a route handler for Playwright to support non-GET methods with custom parameters.

        Args:
            method: HTTP method to use for the request.
            headers: Custom HTTP headers to send with the request.
            payload: Request body data for POST/PUT requests.
        """

        async def route_handler(route: Route, _: PlaywrightRequest) -> None:
            await route.continue_(method=method, headers=dict(headers) if headers else None, post_data=payload)

        return route_handler

    async def _navigate(
        self,
        context: PlaywrightPreNavCrawlingContext,
    ) -> AsyncGenerator[PlaywrightCrawlingContext, Exception | None]:
        """Execute an HTTP request utilizing the `BrowserPool` and the `Playwright` library.

        Args:

Review comment (suggestion):
`request_handler` is already taken, let's come up with a better name. How does `_prepare_non_get_request_interceptor` sound? Not great, huh... but it's a start, I guess.

Code after:
        if self._browser_pool is None:
            raise ValueError('Browser pool is not initialized.')

        # Create a new browser page
        crawlee_page = await self._browser_pool.new_page(proxy_info=context.proxy_info)

        pre_navigation_context = PlaywrightPreNavCrawlingContext(
            request=context.request,
            session=context.session,
            add_requests=context.add_requests,
            send_request=context.send_request,
            push_data=context.push_data,
            use_state=context.use_state,
            proxy_info=context.proxy_info,
            get_key_value_store=context.get_key_value_store,
            log=context.log,
            page=crawlee_page.page,
            block_requests=partial(block_requests, page=crawlee_page.page),
        )

        async with browser_page_context(crawlee_page.page):
            for hook in self._pre_navigation_hooks:
                await hook(pre_navigation_context)
        yield pre_navigation_context

    def _prepare_request_interceptor(
        self,
        method: HttpMethod = 'GET',
        headers: HttpHeaders | dict[str, str] | None = None,
        payload: HttpPayload | None = None,
    ) -> Callable:
        """Create a request interceptor for Playwright to support non-GET methods with custom parameters.

        The interceptor modifies requests by adding custom headers and payload before they are sent.

        Args:
            method: HTTP method to use for the request.
            headers: Custom HTTP headers to send with the request.
            payload: Request body data for POST/PUT requests.
        """

        async def route_handler(route: Route, _: PlaywrightRequest) -> None:
            await route.continue_(method=method, headers=dict(headers) if headers else None, post_data=payload)

        return route_handler

    async def _navigate(
        self,
        context: PlaywrightPreNavCrawlingContext,
    ) -> AsyncGenerator[PlaywrightCrawlingContext, Exception | None]:
        """Execute an HTTP request utilizing the `BrowserPool` and the `Playwright` library.