CVE-2023-46121: Inconsistent Interpretation of HTTP Requests ('HTTP Request/Response Smuggling') (CWE-444)
value: yt-dlp is a youtube-dl fork with additional features and fixes. The Generic Extractor in yt-dlp is vulnerable to an attacker setting an arbitrary proxy for a request to an arbitrary url, allowing the attacker to MITM the request made from yt-dlp's HTTP session. This could lead to cookie exfiltration in some cases. Version 2023.11.14 removed the ability to smuggle `http_headers` to the Generic extractor, as well as other extractors that use the same pattern. Users are advised to upgrade. Users unable to upgrade should disable the Ggneric extractor (or only pass trusted sites with trusted content) and ake caution when using `--no-check-certificate`.
CWE description: The product acts as an intermediary HTTP agent
         (such as a proxy or firewall) in the data flow between two
         entities such as a client and server, but it does not
         interpret malformed HTTP requests or responses in ways that
         are consistent with how the messages will be processed by
         those entities that are at the ultimate destination.
CVSS v3 base score: 3.7
Published: 2023-11-15T00:15Z
Repository: https://github.com/yt-dlp/yt-dlp
Language: Python
Files changed: test/test_networking.py, yt_dlp/extractor/cybrary.py, yt_dlp/extractor/duboku.py, yt_dlp/extractor/embedly.py, yt_dlp/extractor/generic.py, yt_dlp/extractor/slideslive.py, yt_dlp/extractor/storyfire.py, yt_dlp/extractor/vimeo.py, yt_dlp/utils/networking.py
Commit date: 2023-11-14 22:04:25 +0100
Commit message: [ie] Do not smuggle `http_headers`

See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x

Authored by: coletdjnz

Diff: diff --git a/test/test_networking.py b/test/test_networking.py
index 689161fb22..4466fc0485 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -1291,10 +1291,14 @@ def test_clean_header(self):
         with FakeYDL({'http_headers': {'Youtubedl-no-compression': True}}) as ydl:
             rh = self.build_handler(ydl)
             assert 'Youtubedl-no-compression' not in rh.headers
             assert rh.headers.get('Accept-Encoding') == 'identity'
 
+        with FakeYDL({'http_headers': {'Ytdl-socks-proxy': 'socks://localhost:1080'}}) as ydl:
+            rh = self.build_handler(ydl)
+            assert 'Ytdl-socks-proxy' not in rh.headers
+
     def test_build_handler_params(self):
         with FakeYDL({
             'http_headers': {'test': 'testtest'},
             'socket_timeout': 2,
             'proxy': 'http://127.0.0.1:8080',
diff --git a/yt_dlp/extractor/cybrary.py b/yt_dlp/extractor/cybrary.py
index aeffe93b41..c4c78ee1b3 100644
--- a/yt_dlp/extractor/cybrary.py
+++ b/yt_dlp/extractor/cybrary.py
@@ -103,11 +103,11 @@ def _real_extract(self, url):
             'series_id': str_or_none(traverse_obj(course, ('content_description', 'id'))),
             'id': vimeo_id,
             'chapter': module.get('title'),
             'chapter_id': str_or_none(module.get('id')),
             'title': activity.get('title'),
-            'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'http_headers': {'Referer': 'https://api.cybrary.it'}})
+            'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'referer': 'https://api.cybrary.it'})
         }
 
 
 class CybraryCourseIE(CybraryBaseIE):
     _VALID_URL = r'https://app\.cybrary\.it/browse/course/(?P<id>[\w-]+)/?(?:$|[#?])'
diff --git a/yt_dlp/extractor/duboku.py b/yt_dlp/extractor/duboku.py
index fb0546caeb..fc9564cef7 100644
--- a/yt_dlp/extractor/duboku.py
+++ b/yt_dlp/extractor/duboku.py
@@ -136,11 +136,11 @@ def _real_extract(self, url):
         if data_from == 'iframe':
             # use _type url_transparent to retain the meaningful details
             # of the video.
             return {
                 '_type': 'url_transparent',
-                'url': smuggle_url(data_url, {'http_headers': headers}),
+                'url': smuggle_url(data_url, {'referer': webpage_url}),
                 'id': video_id,
                 'title': title,
                 'series': series_title,
                 'season_number': int_or_none(season_id),
                 'season_id': season_id,
diff --git a/yt_dlp/extractor/embedly.py b/yt_dlp/extractor/embedly.py
index 458aaa0a08..a424b49df7 100644
--- a/yt_dlp/extractor/embedly.py
+++ b/yt_dlp/extractor/embedly.py
@@ -104,6 +104,6 @@ def _real_extract(self, url):
         src = urllib.parse.unquote(traverse_obj(qs, ('url', 0)) or '')
         if src and YoutubeTabIE.suitable(src):
             return self.url_result(src, YoutubeTabIE)
         return self.url_result(smuggle_url(
             urllib.parse.unquote(traverse_obj(qs, ('src', 0), ('url', 0))),
-            {'http_headers': {'Referer': url}}))
+            {'referer': url}))
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index ac7cc673f3..1503e5146e 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -15,10 +15,11 @@
     UnsupportedError,
     determine_ext,
     determine_protocol,
     dict_get,
     extract_basic_auth,
+    filter_dict,
     format_field,
     int_or_none,
     is_html,
     js_to_json,
     merge_dicts,
@@ -2433,14 +2434,14 @@ def _real_extract(self, url):
         # that will always result in downloading the whole file that is not desirable.
         # Therefore for extraction pass we have to override Accept-Encoding to any in order
         # to accept raw bytes and being able to download only a chunk.
         # It may probably better to solve this by checking Content-Type for application/octet-stream
         # after a HEAD request, but not sure if we can rely on this.
-        full_response = self._request_webpage(url, video_id, headers={
+        full_response = self._request_webpage(url, video_id, headers=filter_dict({
             'Accept-Encoding': 'identity',
-            **smuggled_data.get('http_headers', {})
-        })
+            'Referer': smuggled_data.get('referer'),
+        }))
         new_url = full_response.url
         url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl()
         if new_url != extract_basic_auth(url)[0]:
             self.report_following_redirect(new_url)
             if force_videoid:
@@ -2456,11 +2457,11 @@ def _real_extract(self, url):
         # Check for direct link to a video
         content_type = full_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             self.report_detected('direct video link')
-            headers = smuggled_data.get('http_headers', {})
+            headers = filter_dict({'Referer': smuggled_data.get('referer')})
             format_id = str(m.group('format_id'))
             ext = determine_ext(url, default_ext=None) or urlhandle_detect_ext(full_response)
             subtitles = {}
             if format_id.endswith('mpegurl') or ext == 'm3u8':
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers)
@@ -2708,11 +2709,11 @@ def _extract_embeds(self, url, webpage, *, urlh=None, info_dict={}):
             return [merge_dicts({
                 '_type': 'video' if is_direct else 'url_transparent',
                 'url': smuggle_url(json_ld['url'], {
                     'force_videoid': video_id,
                     'to_generic': True,
-                    'http_headers': {'Referer': url},
+                    'referer': url,
                 }),
             }, json_ld)]
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py
index 13f3109d74..df2af3b35d 100644
--- a/yt_dlp/extractor/slideslive.py
+++ b/yt_dlp/extractor/slideslive.py
@@ -528,11 +528,11 @@ def _real_extract(self, url):
                 'display_id': video_id,
             })
             if service_name == 'vimeo':
                 info['url'] = smuggle_url(
                     f'https://player.vimeo.com/video/{service_id}',
-                    {'http_headers': {'Referer': url}})
+                    {'referer': url})
 
         video_slides = traverse_obj(slides, ('slides', ..., 'video', 'id'))
         if not video_slides:
             return info
 
diff --git a/yt_dlp/extractor/storyfire.py b/yt_dlp/extractor/storyfire.py
index 035747c31c..566f777827 100644
--- a/yt_dlp/extractor/storyfire.py
+++ b/yt_dlp/extractor/storyfire.py
@@ -30,13 +30,11 @@ def _parse_video(self, video):
             'id': vimeo_id,
             'title': title,
             'description': video.get('description'),
             'url': smuggle_url(
                 'https://player.vimeo.com/video/' + vimeo_id, {
-                    'http_headers': {
-                        'Referer': 'https://storyfire.com/',
-                    }
+                    'referer': 'https://storyfire.com/',
                 }),
             'thumbnail': video.get('storyImage'),
             'view_count': int_or_none(video.get('views')),
             'like_count': int_or_none(video.get('likesCount')),
             'comment_count': int_or_none(video.get('commentsCount')),
diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index e72fa50fa8..e5e8144bb1 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -35,18 +35,18 @@ class VimeoBaseInfoExtractor(InfoExtractor):
     _LOGIN_REQUIRED = False
     _LOGIN_URL = 'https://vimeo.com/log_in'
 
     @staticmethod
     def _smuggle_referrer(url, referrer_url):
-        return smuggle_url(url, {'http_headers': {'Referer': referrer_url}})
+        return smuggle_url(url, {'referer': referrer_url})
 
     def _unsmuggle_headers(self, url):
         """@returns (url, smuggled_data, headers)"""
         url, data = unsmuggle_url(url, {})
         headers = self.get_param('http_headers').copy()
-        if 'http_headers' in data:
-            headers.update(data['http_headers'])
+        if 'referer' in data:
+            headers['Referer'] = data['referer']
         return url, data, headers
 
     def _perform_login(self, username, password):
         webpage = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
diff --git a/yt_dlp/utils/networking.py b/yt_dlp/utils/networking.py
index ba0493cc2b..ed02500110 100644
--- a/yt_dlp/utils/networking.py
+++ b/yt_dlp/utils/networking.py
@@ -121,10 +121,11 @@ def clean_proxies(proxies: dict, headers: HTTPHeaderDict):
 
 def clean_headers(headers: HTTPHeaderDict):
     if 'Youtubedl-No-Compression' in headers:  # compat
         del headers['Youtubedl-No-Compression']
         headers['Accept-Encoding'] = 'identity'
+    headers.pop('Ytdl-socks-proxy', None)
 
 
 def remove_dot_segments(path):
     # Implements RFC3986 5.2.4 remote_dot_segments
     # Pseudo-code: https://tools.ietf.org/html/rfc3986#section-5.2.4

Vulnerable code:             'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'http_headers': {'Referer': 'https://api.cybrary.it'}})
                'url': smuggle_url(data_url, {'http_headers': headers}),
            {'http_headers': {'Referer': url}}))
        full_response = self._request_webpage(url, video_id, headers={
            **smuggled_data.get('http_headers', {})
        })
            headers = smuggled_data.get('http_headers', {})
                    'http_headers': {'Referer': url},
                    {'http_headers': {'Referer': url}})
                    'http_headers': {
                        'Referer': 'https://storyfire.com/',
                    }
        return smuggle_url(url, {'http_headers': {'Referer': referrer_url}})
        if 'http_headers' in data:
            headers.update(data['http_headers'])
Fixed code:         with FakeYDL({'http_headers': {'Ytdl-socks-proxy': 'socks://localhost:1080'}}) as ydl:
            rh = self.build_handler(ydl)
            assert 'Ytdl-socks-proxy' not in rh.headers

            'url': smuggle_url(f'https://player.vimeo.com/video/{vimeo_id}', {'referer': 'https://api.cybrary.it'})
                'url': smuggle_url(data_url, {'referer': webpage_url}),
            {'referer': url}))
    filter_dict,
        full_response = self._request_webpage(url, video_id, headers=filter_dict({
            'Referer': smuggled_data.get('referer'),
        }))
            headers = filter_dict({'Referer': smuggled_data.get('referer')})
                    'referer': url,
                    {'referer': url})
                    'referer': 'https://storyfire.com/',
        return smuggle_url(url, {'referer': referrer_url})
        if 'referer' in data:
            headers['Referer'] = data['referer']
    headers.pop('Ytdl-socks-proxy', None)