Repository: apify/crawlee-python
File: src/crawlee/models.py (Python)

Code before:
# ruff: noqa: TCH001, TCH002, TCH003

from __future__ import annotations

from datetime import datetime
from decimal import Decimal
from enum import Enum
from typing import Annotated, Any, Generic

from pydantic import BaseModel, ConfigDict, Field, HttpUrl
from typing_extensions import Self, TypeVar

from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
from crawlee.enqueue_strategy import EnqueueStrategy


class BaseRequestData(BaseModel):
    """Data needed to create a new crawling request."""

    model_config = ConfigDict(populate_by_name=True)

    url: Annotated[HttpUrl, Field()]
    """URL of the web page to crawl"""

    unique_key: Annotated[str, Field(alias='uniqueKey')]
    """A unique key identifying the request. Two requests with the same `uniqueKey` are considered as pointing to the
    same URL.

    If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
    For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
    of `http://www.example.com/something`.

    Pass an arbitrary non-empty text value to the `uniqueKey` property
    to override the default behavior and specify which URLs shall be considered equal.
    """

    method: str = 'get'

    payload: str | None = None

    headers: Annotated[dict[str, str] | None, Field(default_factory=dict)] = None

    user_data: Annotated[dict[str, Any], Field(alias='userData', default_factory=dict)]
    """Custom user data assigned to the request. Use this to save any request related data to the
    request's scope, keeping them accessible on retries, failures etc.
    """

    retry_count: Annotated[int, Field(alias='retryCount')] = 0

Review comment (suggestion):
I'd wager the `Field()` annotation is redundant here. But I'm not sure myself if I'd prefer to keep it or not.

Code after:
# ruff: noqa: TCH001, TCH002, TCH003

from __future__ import annotations

from datetime import datetime
from decimal import Decimal
from enum import Enum
from typing import Annotated, Any, Generic
from urllib.parse import parse_qs

from pydantic import BaseModel, ConfigDict, Field, HttpUrl
from typing_extensions import Self, TypeVar

from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
from crawlee.enqueue_strategy import EnqueueStrategy


class BaseRequestData(BaseModel):
    """Data needed to create a new crawling request."""

    model_config = ConfigDict(populate_by_name=True)

    url: Annotated[HttpUrl, Field()]
    """URL of the web page to crawl"""

    unique_key: Annotated[str, Field(alias='uniqueKey')]
    """A unique key identifying the request. Two requests with the same `uniqueKey` are considered as pointing to the
    same URL.

    If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
    For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
    of `http://www.example.com/something`.

    Pass an arbitrary non-empty text value to the `uniqueKey` property
    to override the default behavior and specify which URLs shall be considered equal.
    """

    method: str = 'get'

    payload: str | None = None

    headers: Annotated[dict[str, str] | None, Field(default_factory=dict)] = None

    user_data: Annotated[dict[str, Any], Field(alias='userData', default_factory=dict)]
    """Custom user data assigned to the request. Use this to save any request related data to the
    request's scope, keeping them accessible on retries, failures etc.
    """