Repository: apify/crawlee-python
File: src/crawlee/_request.py (Python)

Code before:
    """Data needed to create a new crawling request."""

    model_config = ConfigDict(populate_by_name=True)

    url: Annotated[str, BeforeValidator(validate_http_url), Field()]
    """URL of the web page to crawl"""

    unique_key: Annotated[str, Field(alias='uniqueKey')]
    """A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing
    to the same URL.

    If `unique_key` is not provided, then it is automatically generated by normalizing the URL.
    For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`
    of `http://www.example.com/something`.

    Pass an arbitrary non-empty text value to the `unique_key` property
    to override the default behavior and specify which URLs shall be considered equal.
    """

    method: HttpMethod = 'GET'
    """HTTP request method."""

    headers: Annotated[HttpHeaders, Field(default_factory=HttpHeaders)] = HttpHeaders()
    """HTTP request headers."""

    query_params: Annotated[HttpQueryParams, Field(alias='queryParams', default_factory=dict)] = {}
    """URL query parameters."""

    payload: Annotated[
        HttpPayload | None,
        PlainValidator(lambda value: None if value is None else str(value).encode('utf-8')),
    ] = None
    """HTTP request payload."""

    user_data: Annotated[
        dict[str, JsonSerializable],  # Internally, the model contains `UserData`, this is just for convenience
        Field(alias='userData', default_factory=lambda: UserData()),
        PlainValidator(user_data_adapter.validate_python),
        PlainSerializer(
            lambda instance: user_data_adapter.dump_python(
                instance,
                by_alias=True,
                exclude_none=True,
                exclude_unset=True,
                exclude_defaults=True,
            )
        ),
    ] = {}
    """Custom user data assigned to the request. Use this to save any request related data to the
    request's scope, keeping them accessible on retries, failures etc.
    """

Review comment (suggestion):
We should probably get rid of this one, or make it a property that changes the `url` field. Unfortunately, Apify API does not accept `queryParams`.

Code after:
    """Data needed to create a new crawling request."""

    model_config = ConfigDict(populate_by_name=True)

    url: Annotated[str, BeforeValidator(validate_http_url), Field()]
    """URL of the web page to crawl"""

    unique_key: Annotated[str, Field(alias='uniqueKey')]
    """A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing
    to the same URL.

    If `unique_key` is not provided, then it is automatically generated by normalizing the URL.
    For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`
    of `http://www.example.com/something`.

    Pass an arbitrary non-empty text value to the `unique_key` property
    to override the default behavior and specify which URLs shall be considered equal.
    """

    method: HttpMethod = 'GET'
    """HTTP request method."""

    headers: Annotated[HttpHeaders, Field(default_factory=HttpHeaders)] = HttpHeaders()
    """HTTP request headers."""

    query_params: Annotated[HttpQueryParams, Field(alias='queryParams', default_factory=dict)] = {}
    """URL query parameters."""

    payload: HttpPayload | None = None
    """HTTP request payload."""

    user_data: Annotated[
        dict[str, JsonSerializable],  # Internally, the model contains `UserData`, this is just for convenience
        Field(alias='userData', default_factory=lambda: UserData()),
        PlainValidator(user_data_adapter.validate_python),
        PlainSerializer(
            lambda instance: user_data_adapter.dump_python(
                instance,
                by_alias=True,
                exclude_none=True,
                exclude_unset=True,
                exclude_defaults=True,
            )
        ),
    ] = {}
    """Custom user data assigned to the request. Use this to save any request related data to the
    request's scope, keeping them accessible on retries, failures etc.
    """

    retry_count: Annotated[int, Field(alias='retryCount')] = 0