Repository: apify/crawlee-python
File: src/crawlee/_request.py (Python)

Code before:
    state: RequestState | None = None
    """Describes the request's current lifecycle state."""

    session_rotation_count: Annotated[int | None, Field(alias='sessionRotationCount')] = None

    skip_navigation: Annotated[bool, Field(alias='skipNavigation')] = False

    last_proxy_tier: Annotated[int | None, Field(alias='lastProxyTier')] = None

    forefront: Annotated[bool, Field()] = False


class UserData(BaseModel, MutableMapping[str, JsonValue]):
    """Represents the `user_data` part of a Request.

    Apart from the well-known attributes (`label` and `__crawlee`), it can also contain arbitrary JSON-compatible
    values.
    """

    model_config = ConfigDict(extra='allow')
    __pydantic_extra__: dict[str, JsonValue] = Field(init=False)  # pyright: ignore

    crawlee_data: Annotated[CrawleeRequestData | None, Field(alias='__crawlee')] = None
    label: Annotated[str | None, Field()] = None

    def __getitem__(self, key: str) -> JsonValue:
        return self.__pydantic_extra__[key]

    def __setitem__(self, key: str, value: JsonValue) -> None:
        if key == 'label':
            if value is not None and not isinstance(value, str):
                raise ValueError('`label` must be str or None')

            self.label = value
        self.__pydantic_extra__[key] = value

    def __delitem__(self, key: str) -> None:
        del self.__pydantic_extra__[key]

    def __iter__(self) -> Iterator[str]:  # type: ignore
        yield from self.__pydantic_extra__

    def __len__(self) -> int:
        return len(self.__pydantic_extra__)


user_data_adapter = TypeAdapter(UserData)


class BaseRequestData(BaseModel):
    """Data needed to create a new crawling request."""

Review comment (suggestion):
Suspicious default factory, do we really need both that and a regular default?

Also, doesn't this cause `Request` to serialize to `{..., "headers": {"headers": {...}}, ...}` now? Wouldn't a `RootModel` be more appropriate? :slightly_smiling_face: 

Code after:


class BaseRequestData(BaseModel):
    """Data needed to create a new crawling request."""

    model_config = ConfigDict(populate_by_name=True)

    url: Annotated[str, BeforeValidator(validate_http_url), Field()]
    """URL of the web page to crawl"""

    unique_key: Annotated[str, Field(alias='uniqueKey')]
    """A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing
    to the same URL.

    If `unique_key` is not provided, then it is automatically generated by normalizing the URL.
    For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`
    of `http://www.example.com/something`.

    Pass an arbitrary non-empty text value to the `unique_key` property
    to override the default behavior and specify which URLs shall be considered equal.
    """

    method: HttpMethod = 'GET'
    """HTTP request method."""

    headers: Annotated[HttpHeaders, Field(default_factory=HttpHeaders())] = HttpHeaders()
    """HTTP request headers."""

    query_params: Annotated[HttpQueryParams, Field(alias='queryParams', default_factory=dict)] = {}
    """URL query parameters."""

    payload: HttpPayload | None = None

    data: Annotated[dict[str, Any], Field(default_factory=dict)] = {}

    user_data: Annotated[
        dict[str, JsonValue],  # Internally, the model contains `UserData`, this is just for convenience
        Field(alias='userData', default_factory=lambda: UserData()),
        PlainValidator(user_data_adapter.validate_python),
        PlainSerializer(
            lambda instance: user_data_adapter.dump_python(
                instance,
                by_alias=True,
                exclude_none=True,
                exclude_unset=True,
                exclude_defaults=True,
            )
        ),
    ] = {}
    """Custom user data assigned to the request. Use this to save any request related data to the
    request's scope, keeping them accessible on retries, failures etc.