Repository: apify/crawlee-python
File: src/crawlee/proxy_configuration.py (Python)

Code before:
        request: Request | None = None,
    ) -> str | None | Awaitable[str | None]: ...


class ProxyConfiguration:
    """Configures connection to a proxy server with the provided options.

    Proxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or
    blacklists. Setting proxy configuration in your crawlers automatically configures them to use the selected proxies
    for all connections. You can get information about the currently used proxy by inspecting the {@apilink ProxyInfo}
    property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.

    If you want to use your own proxies, use the {@apilink ProxyConfigurationOptions.proxyUrls} option. Your list of
    proxy URLs will be rotated by the configuration if this option is provided.
    """

    def __init__(
        self,
        *,
        proxy_urls: list[str] | None = None,
        new_url_function: NewUrlFunction | None = None,
        tiered_proxy_urls: list[list[str]] | None = None,
    ) -> None:
        """Initialize a proxy configuration object.

        Exactly one of `proxy_urls`, `tiered_proxy_urls` and `new_url_function` must be specified.

        Args:
            proxy_urls: A list of URLs of proxies that will be rotated in a round-robin fashion
            tiered_proxy_urls: A list of URL tiers (where a tier is a list of proxy URLs). Crawlers will automatically
                try to use the left-most (lowest index) tier where blocking does not happen. The proxy URLs in
                the selected tier will be rotated in a round-robin fashion.
            new_url_function: A function that returns a proxy URL for a given Request. This provides full control over
                the proxy selection mechanism.
        """
        self._next_custom_url_index = 0
        self._used_proxy_urls = dict[str, URL]()
        self._url_validator = TypeAdapter(AnyHttpUrl)

        # Validation
        if sum(map(bool, (proxy_urls, new_url_function, list(flatten(tiered_proxy_urls or []))))) != 1:
            raise ValueError(
                'Exactly one of `proxy_urls`, `tiered_proxy_urls` and `new_url_function` '
                'must be specified (and non-empty).'
            )

        self._proxy_urls = (
            [URL(url) for url in proxy_urls if self._url_validator.validate_python(url)] if proxy_urls else []
        )
        self._proxy_tier_tracker = (
            ProxyTierTracker(

Review comment (suggestion):
```suggestion
        Exactly one of `proxy_urls`, `tiered_proxy_urls` or `new_url_function` must be specified.
```

Code after:
        request: Request | None = None,
    ) -> str | None | Awaitable[str | None]: ...


class ProxyConfiguration:
    """Configures connection to a proxy server with the provided options.

    Proxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or
    blacklists. Setting proxy configuration in your crawlers automatically configures them to use the selected proxies
    for all connections. You can get information about the currently used proxy by inspecting the {@apilink ProxyInfo}
    property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.

    If you want to use your own proxies, use the {@apilink ProxyConfigurationOptions.proxyUrls} option. Your list of
    proxy URLs will be rotated by the configuration if this option is provided.
    """

    def __init__(
        self,
        *,
        proxy_urls: list[str] | None = None,
        new_url_function: NewUrlFunction | None = None,
        tiered_proxy_urls: list[list[str]] | None = None,
    ) -> None:
        """Initialize a proxy configuration object.

        Exactly one of `proxy_urls`, `tiered_proxy_urls` or `new_url_function` must be specified.

        Args:
            proxy_urls: A list of URLs of proxies that will be rotated in a round-robin fashion
            tiered_proxy_urls: A list of URL tiers (where a tier is a list of proxy URLs). Crawlers will automatically
                try to use the lowest tier (smallest index) where blocking does not happen. The proxy URLs in
                the selected tier will be rotated in a round-robin fashion.
            new_url_function: A function that returns a proxy URL for a given Request. This provides full control over
                the proxy selection mechanism.
        """
        self._next_custom_url_index = 0
        self._used_proxy_urls = dict[str, URL]()
        self._url_validator = TypeAdapter(AnyHttpUrl)

        # Validation
        if sum(map(bool, (proxy_urls, new_url_function, list(flatten(tiered_proxy_urls or []))))) != 1:
            raise ValueError(
                'Exactly one of `proxy_urls`, `tiered_proxy_urls` and `new_url_function` '
                'must be specified (and non-empty).'
            )

        self._proxy_urls = (
            [URL(url) for url in proxy_urls if self._url_validator.validate_python(url)] if proxy_urls else []
        )
        self._proxy_tier_tracker = (
            ProxyTierTracker(