Repository: apify/crawlee-python
File: tests/unit/_utils/test_requests.py (Python)

Code before:
import pytest

from crawlee._utils.requests import compute_unique_key, normalize_headers, normalize_url, unique_key_to_request_id

if TYPE_CHECKING:
    from crawlee._types import HttpMethod, HttpPayload


# Tests for normalize_headers
def test_normalize_headers_whitelist() -> None:
    headers = {
        'Content-Type': 'application/json',
        'Authorization': 'Bearer token',
        'Custom-Header': 'should be ignored',
        'Accept': 'text/html',
    }
    normalized = normalize_headers(headers)
    expected = {
        'content-type': 'application/json',
        'authorization': 'Bearer token',
        'accept': 'text/html',
    }
    assert normalized == expected, 'Only whitelisted headers should be included, and keys should be lowercased.'


def test_normalize_headers_strip_values() -> None:
    headers = {
        'Content-Type': ' application/json ',
        'Authorization': '  Bearer token  ',
        'Accept': 'text/html ',
    }
    normalized = normalize_headers(headers)
    expected = {
        'content-type': 'application/json',
        'authorization': 'Bearer token',
        'accept': 'text/html',
    }
    assert normalized == expected, 'Values should be stripped of leading and trailing whitespace.'


def test_normalize_headers_empty_input() -> None:
    headers: dict[str, str] = {}
    normalized = normalize_headers(headers)
    assert normalized == {}, 'Empty header input should return an empty dictionary.'


def test_unique_key_to_request_id_length() -> None:
    unique_key = 'exampleKey123'
    request_id = unique_key_to_request_id(unique_key, request_id_length=15)
    assert len(request_id) == 15, 'Request ID should have the correct length.'


Review comment (suggestion):
```suggestion
def test_normalize_headers_strips_whitespaces() -> None:
```

Code after:
import pytest

from crawlee._types import HttpHeaders
from crawlee._utils.requests import compute_unique_key, normalize_url, unique_key_to_request_id

if TYPE_CHECKING:
    from crawlee._types import HttpMethod, HttpPayload


def test_unique_key_to_request_id_length() -> None:
    unique_key = 'exampleKey123'
    request_id = unique_key_to_request_id(unique_key, request_id_length=15)
    assert len(request_id) == 15, 'Request ID should have the correct length.'


def test_unique_key_to_request_id_consistency() -> None:
    unique_key = 'consistentKey'
    request_id_1 = unique_key_to_request_id(unique_key)
    request_id_2 = unique_key_to_request_id(unique_key)
    assert request_id_1 == request_id_2, 'The same unique key should generate consistent request IDs.'


@pytest.mark.parametrize(
    ('unique_key', 'expected_request_id'),
    [
        ('abc', 'ungWv48BzpBQUDe'),
        ('uniqueKey', 'xiWPs083cree7mH'),
        ('', '47DEQpj8HBSaTIm'),
        ('测试中文', 'lKPdJkdvw8MXEUp'),
        ('test+/=', 'XZRQjhoG0yjfnYD'),
    ],
    ids=[
        'basic_abc',
        'keyword_uniqueKey',
        'empty_string',
        'non_ascii_characters',
        'url_unsafe_characters',
    ],
)
def test_unique_key_to_request_id_known_values(unique_key: str, expected_request_id: str) -> None:
    request_id = unique_key_to_request_id(unique_key)
    assert request_id == expected_request_id, f'Unique key "{unique_key}" should produce the expected request ID.'


@pytest.mark.parametrize(
    ('url', 'expected_output', 'keep_url_fragment'),
    [
        ('https://example.com/?utm_source=test&utm_medium=test&key=value', 'https://example.com?key=value', False),
        (
            'http://example.com/?key=value&another_key=another_value',
            'http://example.com?another_key=another_value&key=value',