Repository: apify/crawlee-python
File: src/crawlee/_utils/robots.py (Python)

Code before:
from __future__ import annotations

from typing import TYPE_CHECKING

from protego import Protego  # type: ignore[import-untyped]
from yarl import URL

from crawlee._utils.web import is_status_code_client_error

if TYPE_CHECKING:
    from typing_extensions import Self

    from crawlee.http_clients import HttpClient
    from crawlee.proxy_configuration import ProxyInfo


class RobotsTxtFile:
    def __init__(self, url: str, robots: Protego) -> None:
        self._robots = robots
        self._original_url = URL(url).origin()

    @classmethod
    async def from_content(cls, url: str, content: str) -> Self:
        """Create a RobotsTxtFile instance from the given content.

        Args:
            url: the URL of the robots.txt file
            content: the content of the robots.txt file
        """
        robots = Protego.parse(content)
        return cls(url, robots)

    @classmethod
    async def find(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None) -> Self:
        """Determine the location of a robots.txt file for a URL and fetch it.

        Args:
            url: the URL to fetch robots.txt for
            proxy_info: a `ProxyInfo` to be used for fetching the robots.txt file
            http_client: the HTTP client to use for fetching the robots.txt file
        """
        robots_url = URL(url).with_path('/robots.txt')
        return await cls.load(str(robots_url), http_client, proxy_info)

    @classmethod
    async def load(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None) -> Self:
        """Load the robots.txt file for a given URL.

        Args:

Review comment (suggestion):
```suggestion
        """Create a `RobotsTxtFile` instance from the given content.
```

Code after:
from __future__ import annotations

from typing import TYPE_CHECKING

from protego import Protego
from yarl import URL

from crawlee._utils.web import is_status_code_client_error

if TYPE_CHECKING:
    from typing_extensions import Self

    from crawlee.http_clients import HttpClient
    from crawlee.proxy_configuration import ProxyInfo


class RobotsTxtFile:
    def __init__(self, url: str, robots: Protego) -> None:
        self._robots = robots
        self._original_url = URL(url).origin()

    @classmethod
    async def from_content(cls, url: str, content: str) -> Self:
        """Create a `RobotsTxtFile` instance from the given content.

        Args:
            url: The URL associated with the robots.txt file.
            content: The raw string content of the robots.txt file to be parsed.
        """
        robots = Protego.parse(content)
        return cls(url, robots)

    @classmethod
    async def find(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None) -> Self:
        """Determine the location of a robots.txt file for a URL and fetch it.

        Args:
            url: The URL whose domain will be used to find the corresponding robots.txt file.
            http_client: Optional `ProxyInfo` to be used when fetching the robots.txt file. If None, no proxy is used.
            proxy_info: The `HttpClient` instance used to perform the network request for fetching the robots.txt file.
        """
        robots_url = URL(url).with_path('/robots.txt')
        return await cls.load(str(robots_url), http_client, proxy_info)

    @classmethod
    async def load(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | None = None) -> Self:
        """Load the robots.txt file for a given URL.

        Args: