Repository: apify/crawlee-python
File: src/crawlee/http_crawler/_http_crawler.py (Python)

Code before:
from __future__ import annotations

import logging
from typing import TYPE_CHECKING, AsyncGenerator, Iterable

from typing_extensions import Unpack

from crawlee.basic_crawler import BasicCrawler, BasicCrawlerOptions, ContextPipeline
from crawlee.errors import SessionError
from crawlee.http_clients import HttpxHttpClient
from crawlee.http_crawler._http_crawling_context import HttpCrawlingContext

if TYPE_CHECKING:
    from crawlee._types import BasicCrawlingContext


class HttpCrawler(BasicCrawler[HttpCrawlingContext]):
    """A web crawler for performing HTTP requests.

    The `HttpCrawler` builds on top of the `BasicCrawler`, which means it inherits all of its features. On top
    of that it implements the HTTP communication using the HTTP clients. The class allows integration with
    any HTTP client that implements the `BaseHttpClient` interface. The HTTP client is provided to the crawler
    as an input parameter to the constructor.

    The HTTP client-based crawlers are ideal for websites that do not require JavaScript execution. However,
    if you need to execute client-side JavaScript, consider using browser-based crawler like the `PlaywrightCrawler`.

    ### Usage

    ```python
    from crawlee.http_crawler import HttpCrawler, HttpCrawlingContext

    crawler = HttpCrawler()

    # Define the default request handler, which will be called for every request.
    @crawler.router.default_handler
    async def request_handler(context: HttpCrawlingContext) -> None:
        context.log.info(f'Processing {context.request.url} ...')

        # Extract data from the page.
        data = {
            'url': context.request.url,
            'response': context.http_response.read().decode()[:100],
        }

        # Push the extracted data to the default dataset.
        await context.push_data(data)

    await crawler.run(['https://crawlee.dev/'])
    ```
    """

Review comment (suggestion):
```suggestion
    if you need to execute client-side JavaScript, consider using a browser-based crawler like the `PlaywrightCrawler`.
```

Code after:
from __future__ import annotations

import logging
from typing import TYPE_CHECKING, AsyncGenerator, Iterable

from typing_extensions import Unpack

from crawlee.basic_crawler import BasicCrawler, BasicCrawlerOptions, ContextPipeline
from crawlee.errors import SessionError
from crawlee.http_clients import HttpxHttpClient
from crawlee.http_crawler._http_crawling_context import HttpCrawlingContext

if TYPE_CHECKING:
    from crawlee._types import BasicCrawlingContext


class HttpCrawler(BasicCrawler[HttpCrawlingContext]):
    """A web crawler for performing HTTP requests.

    The `HttpCrawler` builds on top of the `BasicCrawler`, which means it inherits all of its features. On top
    of that it implements the HTTP communication using the HTTP clients. The class allows integration with
    any HTTP client that implements the `BaseHttpClient` interface. The HTTP client is provided to the crawler
    as an input parameter to the constructor.

    The HTTP client-based crawlers are ideal for websites that do not require JavaScript execution. However,
    if you need to execute client-side JavaScript, consider using a browser-based crawler like the `PlaywrightCrawler`.

    ### Usage

    ```python
    from crawlee.http_crawler import HttpCrawler, HttpCrawlingContext

    crawler = HttpCrawler()

    # Define the default request handler, which will be called for every request.
    @crawler.router.default_handler
    async def request_handler(context: HttpCrawlingContext) -> None:
        context.log.info(f'Processing {context.request.url} ...')

        # Extract data from the page.
        data = {
            'url': context.request.url,
            'response': context.http_response.read().decode()[:100],
        }

        # Push the extracted data to the default dataset.
        await context.push_data(data)

    await crawler.run(['https://crawlee.dev/'])
    ```
    """