Repository: yt-dlp/yt-dlp
File: yt_dlp/utils/networking.py (Python)

Code before:
    The constructor can take multiple dicts, in which keys in the latter are prioritised.
    """

    def __init__(self, *args, **kwargs):
        super().__init__()
        for dct in args:
            if dct is not None:
                self.update(dct)
        self.update(kwargs)

    def __setitem__(self, key, value):
        if isinstance(value, bytes):
            value = value.decode('latin-1')
        super().__setitem__(key.title(), str(value).strip())

    def __getitem__(self, key):
        return super().__getitem__(key.title())

    def __delitem__(self, key):
        super().__delitem__(key.title())

    def __contains__(self, key):
        return super().__contains__(key.title() if isinstance(key, str) else key)


class CaseSensitiveDict(HTTPHeaderDict):
    def __setitem__(self, key, value):
        if isinstance(value, bytes):
            value = value.decode('latin-1')
        self.data[key] = str(value).strip()

    def __getitem__(self, key):
        return self.data[key]

    def __delitem__(self, key):
        del self.data[key]

    def __contains__(self, key):
        return key in self.data


std_headers = HTTPHeaderDict({
    'User-Agent': random_user_agent(),
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-us,en;q=0.5',
    'Sec-Fetch-Mode': 'navigate',
})


def clean_proxies(proxies: dict, headers: HTTPHeaderDict):
    req_proxy = headers.pop('Ytdl-Request-Proxy', None)

Review comment (security):
I think we should put this into the extractor file, since its only a temporary workaround until we make `HTTPHeaderDict` store sensitive keys and the handler extension will select pre-normalization of headers

Code after:
    )
    return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)


class HTTPHeaderDict(dict):
    """
    Store and access keys case-insensitively.
    The constructor can take multiple dicts, in which keys in the latter are prioritised.

    Retains a case sensitive mapping of the headers, which can be accessed via `.sensitive()`.
    """
    def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> typing.Self:
        obj = dict.__new__(cls, *args, **kwargs)
        obj.__sensitive_map = {}
        return obj

    def __init__(self, /, *args, **kwargs):
        super().__init__()
        self.__sensitive_map = {}

        for dct in filter(None, args):
            self.update(dct)
        if kwargs:
            self.update(kwargs)

    def sensitive(self, /) -> dict[str, str]:
        return {
            self.__sensitive_map[key]: value
            for key, value in self.items()
        }

    def __contains__(self, key: str, /) -> bool:
        return super().__contains__(key.title() if isinstance(key, str) else key)

    def __delitem__(self, key: str, /) -> None:
        key = key.title()
        del self.__sensitive_map[key]
        super().__delitem__(key)

    def __getitem__(self, key, /) -> str:
        return super().__getitem__(key.title())

    def __ior__(self, other, /):
        if isinstance(other, type(self)):
            other = other.sensitive()
        if isinstance(other, dict):
            self.update(other)
            return
        return NotImplemented

    def __or__(self, other, /) -> typing.Self: