Repository: ytdl-org/youtube-dl

Issue:
Error with Top Tracks playlist
Similar to #1839 

Eg. From Youtube homepage, click the Music channel on the left, then Play next to Top Tracks in the main body area.

$ youtube-dl -xkt --verbose --audio-format=mp3 --playlist-end=20 'http://www.youtube.com/watch?v=E0CazRHB0so&list=TLvYGWqBK1SGSiVcFok6SzMDjrayc8VmSo'
[debug] System config: []
[debug] User config: []
[debug] Command-line args: ['-xkt', '--verbose', '--audio-format=mp3', '--playlist-end=20', 'http://www.youtube.com/watch?v=E0CazRHB0so&list=TLvYGWqBK1SGSiVcFok6SzMDjrayc8VmSo']
WARNING: Assuming --restrict-filenames since file system encoding cannot encode all charactes. Set the LC_ALL environment variable to fix this.
[debug] youtube-dl version 2013.11.29
[debug] Git HEAD: b138de7
[debug] Python version 3.3.2 - Linux-3.12.0-1-ARCH-i686-with-arch
[debug] Proxy map: {}
[youtube:playlist] Downloading playlist PLTLvYGWqBK1SGSiVcFok6SzMDjrayc8VmSo - add --no-playlist to just download video E0CazRHB0so
[youtube:playlist] TLvYGWqBK1SGSiVcFok6SzMDjrayc8VmSo: Downloading page #1
ERROR: Unable to extract OpenGraph title; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.
Traceback (most recent call last):
  File "/home/user/src/youtube-dl/youtube_dl/YoutubeDL.py", line 428, in extract_info
    ie_result = ie.extract(url)
  File "/home/user/src/youtube-dl/youtube_dl/extractor/common.py", line 131, in extract
    return self._real_extract(url)
  File "/home/user/src/youtube-dl/youtube_dl/extractor/youtube.py", line 1595, in _real_extract
    playlist_title = self._og_search_title(page)
  File "/home/user/src/youtube-dl/youtube_dl/extractor/common.py", line 356, in _og_search_title
    return self._og_search_property('title', html, *_kargs)
  File "/home/user/src/youtube-dl/youtube_dl/extractor/common.py", line 344, in _og_search_property
    escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, *_kargs)
  File "/home/user/src/youtube-dl/youtube_dl/extractor/common.py", line 284, in _search_regex
    raise RegexNotFoundError(u'Unable to extract %s' % _name)
youtube_dl.utils.RegexNotFoundError: Unable to extract OpenGraph title; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.



Discussion:

Resolving patch:
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -194,6 +194,7 @@
     YoutubeWatchLaterIE,
     YoutubeFavouritesIE,
     YoutubeHistoryIE,
+    YoutubeTopListIE,
 )
 from .zdf import ZDFIE
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1576,6 +1576,9 @@ def _real_extract(self, url):
         if len(playlist_id) == 13:  # 'RD' + 11 characters for the video id
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
+        if playlist_id.startswith('TL'):
+            raise ExtractorError(u'For downloading YouTube.com top lists, use '
+                u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
 
         # Extract the video ids from the playlist pages
         ids = []
@@ -1598,6 +1601,38 @@ def _real_extract(self, url):
         return self.playlist_result(url_results, playlist_id, playlist_title)
 
 
+class YoutubeTopListIE(YoutubePlaylistIE):
+    IE_NAME = u'youtube:toplist'
+    IE_DESC = (u'YouTube.com top lists, "yttoplist:{channel}:{list title}"'
+        u' (Example: "yttoplist:music:Top Tracks")')
+    _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        channel = mobj.group('chann')
+        title = mobj.group('title')
+        query = compat_urllib_parse.urlencode({'title': title})
+        playlist_re = 'href="([^"]+?%s[^"]+?)"' % re.escape(query)
+        channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title)
+        link = self._html_search_regex(playlist_re, channel_page, u'list')
+        url = compat_urlparse.urljoin('https://www.youtube.com/', link)
+        
+        video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
+        ids = []
+        # sometimes the webpage doesn't contain the videos
+        # retry until we get them
+        for i in itertools.count(0):
+            msg = u'Downloading Youtube mix'
+            if i > 0:
+                msg += ', retry #%d' % i
+            webpage = self._download_webpage(url, title, msg)
+            ids = orderedSet(re.findall(video_re, webpage))
+            if ids:
+                break
+        url_results = self._ids_to_results(ids)
+        return self.playlist_result(url_results, playlist_title=title)
+
+
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = u'YouTube.com channels'
     _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"


Test patch:
diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -15,6 +15,7 @@
     YoutubeIE,
     YoutubeChannelIE,
     YoutubeShowIE,
+    YoutubeTopListIE,
 )
 
 
@@ -116,5 +117,12 @@ def test_youtube_mix(self):
         original_video = entries[0]
         self.assertEqual(original_video['id'], 'rjFaenf1T-Y')
 
+    def test_youtube_toplist(self):
+        dl = FakeYDL()
+        ie = YoutubeTopListIE(dl)
+        result = ie.extract('yttoplist:music:Top Tracks')
+        entries = result['entries']
+        self.assertTrue(len(entries) >= 9)
+
 if __name__ == '__main__':
     unittest.main()