Repository: yt-dlp/yt-dlp
Pull request #1581 (2021-11-06 23:51:47)

[IMDB] Thumbnail lookup fix
## Please follow the guide below

- You will be asked some questions, please read them **carefully** and answer honestly
- Put an `x` into all the boxes [ ] relevant to your *pull request* (like that [x])
- Use *Preview* tab to see how your *pull request* will actually look like

---

### Before submitting a *pull request* make sure you have:
- [x] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions)
- [x] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)

### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
- [x] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)

### What is the purpose of your *pull request*?
- [ ] Bug fix
- [x] Improvement
- [ ] New extractor
- [ ] New feature

---

### Description of your *pull request* and other information

Explanation of your *pull request* in arbitrary form goes here. Please make sure the description explains the purpose and effect of your *pull request* and is worded well enough to be understood. Provide as much context and examples as possible.

---

The IMDB extractor wasn't returning any thumbnails so I took a look and can see that we are `try_get`ing off the wrong hash. Should be `info` (like all the attribute lookups) and not `video_metadata`.

Simple.



Patch:
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 0fbb3baa7..92e430fda 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2985,7 +2985,7 @@ def sanitize_info(info_dict, remove_private_keys=False):
             return info_dict
         info_dict.setdefault('epoch', int(time.time()))
         remove_keys = {'__original_infodict'}  # Always remove this since this may contain a copy of the entire dict
-        keep_keys = ['_type'],  # Always keep this to facilitate load-info-json
+        keep_keys = ['_type']  # Always keep this to facilitate load-info-json
         if remove_private_keys:
             remove_keys |= {
                 'requested_formats', 'requested_subtitles', 'requested_entries',
diff --git a/yt_dlp/extractor/imdb.py b/yt_dlp/extractor/imdb.py
index a31301985..24f1fde64 100644
--- a/yt_dlp/extractor/imdb.py
+++ b/yt_dlp/extractor/imdb.py
@@ -111,7 +111,7 @@ def _real_extract(self, url):
             'formats': formats,
             'description': info.get('videoDescription'),
             'thumbnail': url_or_none(try_get(
-                video_metadata, lambda x: x['videoSlate']['source'])),
+                info, lambda x: x['videoSlate']['source'])),
             'duration': parse_duration(info.get('videoRuntime')),
         }
 
diff --git a/yt_dlp/extractor/la7.py b/yt_dlp/extractor/la7.py
index 363fbd6a5..de985e450 100644
--- a/yt_dlp/extractor/la7.py
+++ b/yt_dlp/extractor/la7.py
@@ -7,8 +7,9 @@
 from ..utils import (
     determine_ext,
     float_or_none,
+    HEADRequest,
+    int_or_none,
     parse_duration,
-    smuggle_url,
     unified_strdate,
 )
 
@@ -25,19 +26,38 @@ class LA7IE(InfoExtractor):
         'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
         'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
         'info_dict': {
-            'id': '0_42j6wd36',
+            'id': 'inccool8-02-10-2015-163722',
             'ext': 'mp4',
             'title': 'Inc.Cool8',
             'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
             'thumbnail': 're:^https?://.*',
-            'uploader_id': 'kdla7pillole@iltrovatore.it',
-            'timestamp': 1443814869,
             'upload_date': '20151002',
         },
     }, {
         'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
         'only_matching': True,
     }]
+    _HOST = 'https://awsvodpkg.iltrovatore.it'
+
+    def _generate_mp4_url(self, quality, m3u8_formats):
+        for f in m3u8_formats:
+            if f['vcodec'] != 'none' and quality in f['url']:
+                http_url = '%s%s.mp4' % (self._HOST, quality)
+
+                urlh = self._request_webpage(
+                    HEADRequest(http_url), quality,
+                    note='Check filesize', fatal=False)
+                if urlh:
+                    http_f = f.copy()
+                    del http_f['manifest_url']
+                    http_f.update({
+                        'format_id': http_f['format_id'].replace('hls-', 'https-'),
+                        'url': http_url,
+                        'protocol': 'https',
+                        'filesize_approx': int_or_none(urlh.headers.get('Content-Length', None)),
+                    })
+                    return http_f
+                return None
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -46,22 +66,30 @@ def _real_extract(self, url):
             url = '%s//%s' % (self.http_scheme(), url)
 
         webpage = self._download_webpage(url, video_id)
+        video_path = self._search_regex(r'(/content/.*?).mp4', webpage, 'video_path')
 
-        player_data = self._search_regex(
-            [r'(?s)videoParams\s*=\s*({.+?});', r'videoLa7\(({[^;]+})\);'],
-            webpage, 'player data')
-        vid = self._search_regex(r'vid\s*:\s*"(.+?)",', player_data, 'vid')
+        formats = self._extract_mpd_formats(
+            f'{self._HOST}/local/dash/,{video_path}.mp4.urlset/manifest.mpd',
+            video_id, mpd_id='dash', fatal=False)
+        m3u8_formats = self._extract_m3u8_formats(
+            f'{self._HOST}/local/hls/,{video_path}.mp4.urlset/master.m3u8',
+            video_id, 'mp4', m3u8_id='hls', fatal=False)
+        formats.extend(m3u8_formats)
+
+        for q in filter(None, video_path.split(',')):
+            http_f = self._generate_mp4_url(q, m3u8_formats)
+            if http_f:
+                formats.append(http_f)
+
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url_transparent',
-            'url': smuggle_url('kaltura:103:%s' % vid, {
-                'service_url': 'http://nkdam.iltrovatore.it',
-            }),
             'id': video_id,
             'title': self._og_search_title(webpage, default=None),
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
-            'ie_key': 'Kaltura',
+            'formats': formats,
+            'upload_date': unified_strdate(self._search_regex(r'datetime="(.+?)"', webpage, 'upload_date', fatal=False))
         }
 
 
diff --git a/yt_dlp/extractor/yandexvideo.py b/yt_dlp/extractor/yandexvideo.py
index 9974d65d6..67095f2fd 100644
--- a/yt_dlp/extractor/yandexvideo.py
+++ b/yt_dlp/extractor/yandexvideo.py
@@ -7,6 +7,7 @@
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    extract_attributes,
     int_or_none,
     try_get,
     url_or_none,
@@ -148,7 +149,7 @@ def _real_extract(self, url):
 
 
 class ZenYandexIE(InfoExtractor):
-    _VALID_URL = r'https?://zen\.yandex\.ru/media/(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-(?P<id>[a-z0-9-]+)'
+    _VALID_URL = r'https?://zen\.yandex\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
     _TESTS = [{
         'url': 'https://zen.yandex.ru/media/popmech/izverjenie-vulkana-iz-spichek-zreliscnyi-opyt-6002240ff8b1af50bb2da5e3',
         'info_dict': {
@@ -156,19 +157,38 @@ class ZenYandexIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Извержение вулкана из спичек: зрелищный опыт',
             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
-            'thumbnail': 'https://avatars.mds.yandex.net/get-zen-pub-og/3558619/pub_6002240ff8b1af50bb2da5e3_600bad814d953e4132a30b5e/orig',
+            'thumbnail': 're:^https://avatars.mds.yandex.net/',
             'uploader': 'Популярная механика',
         },
+        'params': {
+            'skip_download': 'm3u8',
+        },
     }, {
         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
         'info_dict': {
             'id': '60c7c443da18892ebfe85ed7',
             'ext': 'mp4',
             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
-            'description': 'md5:8684912f6086f298f8078d4af0e8a600',
-            'thumbnail': 'https://avatars.mds.yandex.net/get-zen-pub-og/4410519/pub_60c7c443da18892ebfe85ed7_60c7c48e060a163121f42cc3/orig',
+            'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
+            'thumbnail': 're:^https://avatars.mds.yandex.net/',
             'uploader': 'AcademeG DailyStream'
         },
+        'params': {
+            'skip_download': 'm3u8',
+            'format': 'bestvideo',
+        },
+    }, {
+        'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
+        'info_dict': {
+            'id': '6002240ff8b1af50bb2da5e3',
+            'ext': 'mp4',
+            'title': 'Извержение вулкана из спичек: зрелищный опыт',
+            'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
+            'uploader': 'Популярная механика',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
     }, {
         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
         'only_matching': True,
@@ -177,23 +197,37 @@ class ZenYandexIE(InfoExtractor):
     def _real_extract(self, url):
         id = self._match_id(url)
         webpage = self._download_webpage(url, id)
-        data_json = self._parse_json(self._search_regex(r'w\._data\s?=\s?({.+?});', webpage, 'metadata'), id)
-        stream_json = try_get(data_json, lambda x: x['publication']['content']['gifContent'], dict)
-        stream_url = stream_json.get('stream') or try_get(stream_json, lambda x: x['streams']['url'])
-        formats = self._extract_m3u8_formats(stream_url, id)
+        data_json = self._parse_json(
+            self._search_regex(r'data\s*=\s*({["\']_*serverState_*video.+?});', webpage, 'metadata'), id)
+        serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
+                                         webpage, 'server state').replace('State', 'Settings')
+        uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
+                                      webpage, 'uploader', default='<a>')
+        uploader_name = extract_attributes(uploader).get('aria-label')
+        video_json = try_get(data_json, lambda x: x[serverstate]['exportData']['video'], dict)
+        stream_urls = try_get(video_json, lambda x: x['video']['streams'])
+        formats = []
+        for s_url in stream_urls:
+            ext = determine_ext(s_url)
+            if ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(s_url, id, mpd_id='dash'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4'))
         self._sort_formats(formats)
         return {
             'id': id,
-            'title': try_get(data_json, (lambda x: x['og']['title'], lambda x: x['publication']['content']['preview']['title'])),
-            'uploader': data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
-            'description': try_get(data_json, lambda x: x['og']['description']),
-            'thumbnail': try_get(data_json, lambda x: x['og']['imageUrl']),
+            'title': video_json.get('title') or self._og_search_title(webpage),
             'formats': formats,
+            'duration': int_or_none(video_json.get('duration')),
+            'view_count': int_or_none(video_json.get('views')),
+            'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
+            'description': self._og_search_description(webpage) or try_get(data_json, lambda x: x['og']['description']),
+            'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
         }
 
 
 class ZenYandexChannelIE(InfoExtractor):
-    _VALID_URL = r'https?://zen\.yandex\.ru/(?!media)(?:id/)?(?P<id>[a-z0-9-_]+)'
+    _VALID_URL = r'https?://zen\.yandex\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
     _TESTS = [{
         'url': 'https://zen.yandex.ru/tok_media',
         'info_dict': {
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 75b4ed61b..875ab5e72 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -6442,10 +6442,10 @@ def traverse_obj(
 
     def _traverse_obj(obj, path, _current_depth=0):
         nonlocal depth
-        if obj is None:
-            return None
         path = tuple(variadic(path))
         for i, key in enumerate(path):
+            if obj is None:
+                return None
             if isinstance(key, (list, tuple)):
                 obj = [_traverse_obj(obj, sub_key, _current_depth) for sub_key in key]
                 key = ...


Test patch:
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bd2d752e2..40c4169c8 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -737,6 +737,7 @@ def expect_same_infodict(out):
         test(NA_TEST_OUTTMPL, 'NA-NA-def-1234.mp4')
         test(NA_TEST_OUTTMPL, 'none-none-def-1234.mp4', outtmpl_na_placeholder='none')
         test(NA_TEST_OUTTMPL, '--def-1234.mp4', outtmpl_na_placeholder='')
+        test('%(non_existent.0)s', 'NA')
 
         # String formatting
         FMT_TEST_OUTTMPL = '%%(height)%s.%%(ext)s'