Repository: yt-dlp/yt-dlp
Pull request #11763 (2024-12-08 11:48:16)

[cleanup] Make more playlist extractions lazy
**IMPORTANT**: PRs without the template will be CLOSED

### Description of your *pull request* and other information

<!--

Explanation of your *pull request* in arbitrary form goes here. Please **make sure the description explains the purpose and effect** of your *pull request* and is worded well enough to be understood. Provide as much **context and examples** as possible

-->

Make playlist extraction in multiple extractors lazy by using generator expressions instead of list comprehension.


<details open><summary>Template</summary> <!-- OPEN is intentional -->

<!--

# PLEASE FOLLOW THE GUIDE BELOW

- You will be asked some questions, please read them **carefully** and answer honestly
- Put an `x` into all the boxes `[ ]` relevant to your *pull request* (like [x])
- Use *Preview* tab to see how your *pull request* will actually look like

-->

### Before submitting a *pull request* make sure you have:
- [x] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions)
- [x] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests

### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check all of the following options that apply:
- [x] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)

### What is the purpose of your *pull request*?
- [x] Fix or improvement to an extractor (Make sure to add/update tests)
- [ ] New extractor ([Piracy websites will not be accepted](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy))
- [ ] Core bug fix/improvement
- [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))

</details>


grqz: <details><summary>there are more</summary>

```code-search
# Query: \\.playlist_result[\\s\\n]*\\([\\s\\n]*\\[
# Flags: RegExp
# ContextLines: 3

14 results - 14 files

yt_dlp/extractor/abc.py:
  138              urls_info = [urls_info]
  139  
  140          if youtube:
  141:             return self.playlist_result([
  142                  self.url_result(url_info['url']) for url_info in urls_info])
  143  
  144          formats = []

yt_dlp/extractor/banbye.py:
  26  
  27      def _extract_playlist(self, playlist_id):
  28          data = self._download_json(f'{self._API_BASE}/playlists/{playlist_id}', playlist_id)
  29:         return self.playlist_result([
  30              self.url_result(f'{self._VIDEO_BASE}/{video_id}', BanByeIE)
  31              for video_id in data['videoIds']], playlist_id, data.get('name'))
  32  

yt_dlp/extractor/bbc.py:
  1444          entries.extend(re.findall(
  1445              rf'setPlaylist\("({EMBED_URL})"\)', webpage))
  1446          if entries:
  1447:             return self.playlist_result(
  1448:                 [self.url_result(entry_, 'BBCCoUk') for entry_ in entries],
  1449                  playlist_id, playlist_title, playlist_description)
  1450  
  1451          # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)

yt_dlp/extractor/cineverse.py:
  135  
  136          season = traverse_obj(pageprops, ('seasonEpisodes', ..., 'episodes', lambda _, v: v['item_id'] and v['title']))
  137          if season:
  138:             return self.playlist_result([item_result(ep) for ep in season], playlist_id=series_id,
  139                                          playlist_title=traverse_obj(pageprops, ('itemDetailsData', 'title')))
  140          return item_result(pageprops['itemDetailsData'])

yt_dlp/extractor/duboku.py:
  239                  f'Cannot find {fragment}' if fragment else 'Cannot extract playlist')
  240  
  241          # return url results
  242:         return self.playlist_result([
  243              self.url_result(
  244                  urllib.parse.urljoin('https://w.duboku.io', x['href']),
  245                  ie=DubokuIE.ie_key(), video_title=x.get('title'))

yt_dlp/extractor/francetv.py:
  437  
  438          dailymotion_urls = tuple(DailymotionIE._extract_embed_urls(url, webpage))
  439          if dailymotion_urls:
  440:             return self.playlist_result([
  441                  self.url_result(dailymotion_url, DailymotionIE.ie_key())
  442                  for dailymotion_url in dailymotion_urls])
  443  

yt_dlp/extractor/gamejolt.py:
  410          game_id = self._match_id(url)
  411          game_overview = self._call_api(
  412              f'web/discover/games/overview/{game_id}', game_id, note='Downloading soundtrack info', errnote='Unable to download soundtrack info')
  413:         return self.playlist_result([{
  414              'id': str_or_none(song.get('id')),
  415              'title': str_or_none(song.get('title')),
  416              'url': str_or_none(song.get('url')),

yt_dlp/extractor/rtvcplay.py:
  170                  if not podcast_episodes:
  171                      raise ExtractorError('Could not find asset_id nor program playlist nor podcast episodes')
  172  
  173:                 return self.playlist_result([
  174                      self.url_result(episode['file'], url_transparent=True, **traverse_obj(episode, {
  175                          'title': 'title',
  176                          'description': ('description', {clean_html}),

yt_dlp/extractor/s4c.py:
   97                  'show_prog_in_series': 'Y',
   98              }, note='Downloading series details JSON')
   99  
  100:         return self.playlist_result(
  101:             [self.url_result(f'https://www.s4c.cymru/clic/programme/{episode_id}', S4CIE, episode_id)
  102               for episode_id in traverse_obj(series_details, ('other_progs_in_series', ..., 'id'))],
  103              series_id, traverse_obj(series_details, ('full_prog_details', 0, 'series_title', {str})))

yt_dlp/extractor/sbscokr.py:
  164                  'currentCount': '10000',
  165              })
  166  
  167:         return self.playlist_result(
  168:             [self.url_result(f'https://allvod.sbs.co.kr/allvod/vodEndPage.do?mdaId={video_id}', SBSCoKrIE)
  169               for video_id in traverse_obj(details, ('list', ..., 'mdaId'))], program_id)
  170  
  171  

yt_dlp/extractor/seznamzpravy.py:
  149          title = info.get('title') or self._og_search_title(webpage, fatal=False)
  150          description = info.get('description') or self._og_search_description(webpage)
  151  
  152:         return self.playlist_result([
  153              self.url_result(entry_url, ie=SeznamZpravyIE.ie_key())
  154              for entry_url in SeznamZpravyIE._extract_embed_urls(url, webpage)],
  155              article_id, title, description)

yt_dlp/extractor/testurl.py:
  37  
  38          testcases = tuple(extractor.get_testcases(True))
  39          if num == 'all':
  40:             return self.playlist_result(
  41:                 [self.url_result(tc['url'], extractor) for tc in testcases],
  42                  url, f'{extractor.IE_NAME} tests')
  43          try:
  44              tc = testcases[int(num or 0)]

yt_dlp/extractor/youtube.py:
  6873          return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
  6874  
  6875      def _empty_playlist(self, item_id, data):
  6876:         return self.playlist_result([], item_id, **self._extract_metadata_from_tabs(item_id, data))
  6877  
  6878      @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
  6879      def _real_extract(self, url, smuggled_data):

yt_dlp/extractor/zaiko.py:
  137          webpage = self._download_real_webpage(url, ticket_id)
  138          eticket = self._parse_vue_element_attr('eticket', webpage, ticket_id)
  139  
  140:         return self.playlist_result(
  141:             [self.url_result(stream, ZaikoIE) for stream in traverse_obj(eticket, ('streams', ..., 'url'))],
  142              ticket_id, **traverse_obj(eticket, ('ticket-details', {
  143                  'title': 'event_name',
  144                  'thumbnail': 'event_img_url',
```


</details>


seproDev: I didn't include those since no network requests are being made

Patch:
diff --git a/yt_dlp/extractor/brightcove.py b/yt_dlp/extractor/brightcove.py
index 2526f25da..3ada1fd5d 100644
--- a/yt_dlp/extractor/brightcove.py
+++ b/yt_dlp/extractor/brightcove.py
@@ -31,6 +31,7 @@
     update_url_query,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class BrightcoveLegacyIE(InfoExtractor):
@@ -935,8 +936,8 @@ def extract_policy_key():
 
         if content_type == 'playlist':
             return self.playlist_result(
-                [self._parse_brightcove_metadata(vid, vid.get('id'), headers)
-                 for vid in json_data.get('videos', []) if vid.get('id')],
+                (self._parse_brightcove_metadata(vid, vid['id'], headers)
+                 for vid in traverse_obj(json_data, ('videos', lambda _, v: v['id']))),
                 json_data.get('id'), json_data.get('name'),
                 json_data.get('description'))
 
diff --git a/yt_dlp/extractor/dvtv.py b/yt_dlp/extractor/dvtv.py
index 3e442b339..52d67d2bd 100644
--- a/yt_dlp/extractor/dvtv.py
+++ b/yt_dlp/extractor/dvtv.py
@@ -162,7 +162,7 @@ def _real_extract(self, url):
         items = re.findall(r'(?s)playlist\.push\(({.+?})\);', webpage)
         if items:
             return self.playlist_result(
-                [self._parse_video_metadata(i, video_id, timestamp) for i in items],
+                (self._parse_video_metadata(i, video_id, timestamp) for i in items),
                 video_id, self._html_search_meta('twitter:title', webpage))
 
         item = self._search_regex(
diff --git a/yt_dlp/extractor/nytimes.py b/yt_dlp/extractor/nytimes.py
index 9ef57410a..a97add71a 100644
--- a/yt_dlp/extractor/nytimes.py
+++ b/yt_dlp/extractor/nytimes.py
@@ -343,7 +343,7 @@ def _real_extract(self, url):
         if media_ids:
             media_ids.append(lead_video_id)
             return self.playlist_result(
-                [self._extract_video(media_id) for media_id in media_ids], page_id, title, description)
+                map(self._extract_video, media_ids), page_id, title, description)
 
         return {
             **self._extract_video(lead_video_id),
diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py
index 6bdeaf157..a0e831a5c 100644
--- a/yt_dlp/extractor/patreon.py
+++ b/yt_dlp/extractor/patreon.py
@@ -457,7 +457,7 @@ class PatreonCampaignIE(PatreonBaseIE):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?patreon\.com/(?:
             (?:m|api/campaigns)/(?P<campaign_id>\d+)|
-            (?P<vanity>(?!creation[?/]|posts/|rss[?/])[\w-]+)
+            (?:c/)?(?P<vanity>(?!creation[?/]|posts/|rss[?/])[\w-]+)
         )(?:/posts)?/?(?:$|[?#])'''
     _TESTS = [{
         'url': 'https://www.patreon.com/dissonancepod/',
@@ -509,6 +509,26 @@ class PatreonCampaignIE(PatreonBaseIE):
             'thumbnail': r're:^https?://.*$',
         },
         'playlist_mincount': 201,
+    }, {
+        'url': 'https://www.patreon.com/c/OgSog',
+        'info_dict': {
+            'id': '8504388',
+            'title': 'OGSoG',
+            'description': r're:(?s)Hello and welcome to our Patreon page. We are Mari, Lasercorn, .+',
+            'channel': 'OGSoG',
+            'channel_id': '8504388',
+            'channel_url': 'https://www.patreon.com/OgSog',
+            'uploader_url': 'https://www.patreon.com/OgSog',
+            'uploader_id': '72323575',
+            'uploader': 'David Moss',
+            'thumbnail': r're:https?://.+/.+',
+            'channel_follower_count': int,
+            'age_limit': 0,
+        },
+        'playlist_mincount': 331,
+    }, {
+        'url': 'https://www.patreon.com/c/OgSog/posts',
+        'only_matching': True,
     }, {
         'url': 'https://www.patreon.com/dissonancepod/posts',
         'only_matching': True,
diff --git a/yt_dlp/extractor/soundcloud.py b/yt_dlp/extractor/soundcloud.py
index e0dda8ff8..66bc5f9c5 100644
--- a/yt_dlp/extractor/soundcloud.py
+++ b/yt_dlp/extractor/soundcloud.py
@@ -259,6 +259,8 @@ def invalid_url(url):
             preset_base = preset.partition('_')[0]
 
             protocol = traverse_obj(t, ('format', 'protocol', {str})) or 'http'
+            if protocol.startswith(('ctr-', 'cbc-')):
+                continue
             if protocol == 'progressive':
                 protocol = 'http'
             if protocol != 'hls' and '/hls' in format_url:
diff --git a/yt_dlp/extractor/vidyard.py b/yt_dlp/extractor/vidyard.py
index 2f6d1f4c5..89a89b13f 100644
--- a/yt_dlp/extractor/vidyard.py
+++ b/yt_dlp/extractor/vidyard.py
@@ -421,5 +421,5 @@ def _real_extract(self, url):
             return self._process_video_json(video_json['chapters'][0], video_id)
 
         return self.playlist_result(
-            [self._process_video_json(chapter, video_id) for chapter in video_json['chapters']],
+            (self._process_video_json(chapter, video_id) for chapter in video_json['chapters']),
             str(video_json['playerUuid']), video_json.get('name'))
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index e69373ba2..fd9c7107c 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -3127,9 +3127,9 @@ def _parse_sig_js(self, jscode):
         # ;N&&(N=sig(decodeURIComponent(N)),J.set(R,encodeURIComponent(N)));return J};
         # {var H=u,k=f.sp,v=sig(decodeURIComponent(f.s));H.set(k,encodeURIComponent(v))}
         funcname = self._search_regex(
-            (r'\b(?P<var>[a-zA-Z0-9$]+)&&\((?P=var)=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\((?P=var)\)\)',
-             r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*(?P<arg>[a-zA-Z0-9$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
+            (r'\b(?P<var>[a-zA-Z0-9_$]+)&&\((?P=var)=(?P<sig>[a-zA-Z0-9_$]{2,})\(decodeURIComponent\((?P=var)\)\)',
+             r'(?P<sig>[a-zA-Z0-9_$]+)\s*=\s*function\(\s*(?P<arg>[a-zA-Z0-9_$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)',
+             r'(?:\b|[^a-zA-Z0-9_$])(?P<sig>[a-zA-Z0-9_$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9_$]{2}\.[a-zA-Z0-9_$]{2}\(a,\d+\))?',
              # Old patterns
              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
@@ -4067,10 +4067,12 @@ def build_fragments(f):
                 if height:
                     res_qualities[height] = quality
 
+            display_name = audio_track.get('displayName') or ''
+            is_original = 'original' in display_name.lower()
+            is_descriptive = 'descriptive' in display_name.lower()
             is_default = audio_track.get('audioIsDefault')
-            is_descriptive = 'descriptive' in (audio_track.get('displayName') or '').lower()
             language_code = audio_track.get('id', '').split('.')[0]
-            if language_code and is_default:
+            if language_code and (is_original or (is_default and not original_language)):
                 original_language = language_code
 
             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
@@ -4151,7 +4153,7 @@ def build_fragments(f):
                 'filesize': int_or_none(fmt.get('contentLength')),
                 'format_id': f'{itag}{"-drc" if fmt.get("isDrc") else ""}',
                 'format_note': join_nonempty(
-                    join_nonempty(audio_track.get('displayName'), is_default and ' (default)', delim=''),
+                    join_nonempty(display_name, is_default and ' (default)', delim=''),
                     name, fmt.get('isDrc') and 'DRC',
                     try_get(fmt, lambda x: x['projectionType'].replace('RECTANGULAR', '').lower()),
                     try_get(fmt, lambda x: x['spatialAudioType'].replace('SPATIAL_AUDIO_TYPE_', '').lower()),
@@ -4170,7 +4172,7 @@ def build_fragments(f):
                 'url': fmt_url,
                 'width': int_or_none(fmt.get('width')),
                 'language': join_nonempty(language_code, 'desc' if is_descriptive else '') or None,
-                'language_preference': PREFERRED_LANG_VALUE if is_default else -10 if is_descriptive else -1,
+                'language_preference': PREFERRED_LANG_VALUE if is_original else 5 if is_default else -10 if is_descriptive else -1,
                 # Strictly de-prioritize broken, damaged and 3gp formats
                 'preference': -20 if is_broken else -10 if is_damaged else -2 if itag == '17' else None,
             }
@@ -4689,7 +4691,7 @@ def process_language(container, base_url, lang_code, sub_name, query):
                     (?=(?P<artist>[^\n]+))(?P=artist)\n+
                     (?=(?P<album>[^\n]+))(?P=album)\n
                     (?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?
-                    (?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
+                    (?:.+?Released\ on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?
                     (.+?\nArtist\s*:\s*
                         (?=(?P<clean_artist>[^\n]+))(?P=clean_artist)\n
                     )?.+\nAuto-generated\ by\ YouTube\.\s*$
@@ -5282,6 +5284,7 @@ def _extract_entries(self, parent_renderer, continuation_list):
                     'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
                     'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)],
                     'richGridRenderer': lambda x: self._extract_entries(x, continuation_list),
+                    'lockupViewModel': lambda x: [self._extract_lockup_view_model(x)],
                 }
                 for key, renderer in isr_content.items():
                     if key not in known_renderers:


Test patch:
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 2a99436a6..13436f088 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -73,6 +73,11 @@
         '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
         'MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
     ),
+    (
+        'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q',
+    ),
 ]
 
 _NSIG_TESTS = [
@@ -192,6 +197,10 @@
         'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js',
         'gK15nzVyaXE9RsMP3z', 'ZFFWFLPWx9DEgQ',
     ),
+    (
+        'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js',
+        'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP',
+    ),
 ]