Repository: yt-dlp/yt-dlp
Pull request #5170 (2022-10-07 17:30:01)

[utils] `traverse_obj`: return `[]` if last path branches without default
**IMPORTANT**: PRs without the template will be CLOSED

### Description of your *pull request* and other information

</details>

It is common to see a pattern like `traverse_obj(obj, ...) or []` since the default is currently `None`.
This can cause developers to fall into the trap of doing `for item in traverse_obj(obj, ...)`.
This PR addresses this issue by defaulting to `[]` if the last path did not match and no default was provided.

<details open><summary>Template</summary> <!-- OPEN is intentional -->

### Before submitting a *pull request* make sure you have:
- [x] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions)
- [x] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
- [x] Checked the code with [flake8](https://pypi.python.org/pypi/flake8) and [ran relevant tests](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions)

### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
- [x] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)

### What is the purpose of your *pull request*?
- [ ] Fix or improvement to an extractor (Make sure to add/update tests)
- [ ] New extractor ([Piracy websites will not be accepted](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy))
- [x] Core bug fix/improvement
- [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))

Closes #5162


Patch:
diff --git a/yt_dlp/extractor/tubitv.py b/yt_dlp/extractor/tubitv.py
index d91a46500..f5ed950be 100644
--- a/yt_dlp/extractor/tubitv.py
+++ b/yt_dlp/extractor/tubitv.py
@@ -22,6 +22,19 @@ class TubiTvIE(InfoExtractor):
     _NETRC_MACHINE = 'tubitv'
     _GEO_COUNTRIES = ['US']
     _TESTS = [{
+        'url': 'https://tubitv.com/movies/383676/tracker',
+        'md5': '566fa0f76870302d11af0de89511d3f0',
+        'info_dict': {
+            'id': '383676',
+            'ext': 'mp4',
+            'title': 'Tracker',
+            'description': 'md5:ff320baf43d0ad2655e538c1d5cd9706',
+            'uploader_id': 'f866e2677ea2f0dff719788e4f7f9195',
+            'release_year': 2010,
+            'thumbnail': r're:^https?://.+\.(jpe?g|png)$',
+            'duration': 6122,
+        },
+    }, {
         'url': 'http://tubitv.com/video/283829/the_comedian_at_the_friday',
         'md5': '43ac06be9326f41912dc64ccf7a80320',
         'info_dict': {
@@ -31,12 +44,10 @@ class TubiTvIE(InfoExtractor):
             'description': 'A stand up comedian is forced to look at the decisions in his life while on a one week trip to the west coast.',
             'uploader_id': 'bc168bee0d18dd1cb3b86c68706ab434',
         },
+        'skip': 'Content Unavailable'
     }, {
         'url': 'http://tubitv.com/tv-shows/321886/s01_e01_on_nom_stories',
         'only_matching': True,
-    }, {
-        'url': 'http://tubitv.com/movies/383676/tracker',
-        'only_matching': True,
     }, {
         'url': 'https://tubitv.com/movies/560057/penitentiary?start=true',
         'info_dict': {
@@ -47,11 +58,13 @@ class TubiTvIE(InfoExtractor):
             'uploader_id': 'd8fed30d4f24fcb22ec294421b9defc2',
             'release_year': 1979,
         },
-        'params': {
-            'skip_download': True,
-        },
+        'skip': 'Content Unavailable'
     }]
 
+    # DRM formats are included only to raise appropriate error
+    _UNPLAYABLE_FORMATS = ('hlsv6_widevine', 'hlsv6_widevine_nonclearlead', 'hlsv6_playready_psshv0',
+                           'hlsv6_fairplay', 'dash_widevine', 'dash_widevine_nonclearlead')
+
     def _perform_login(self, username, password):
         self.report_login()
         form_data = {
@@ -69,17 +82,26 @@ def _perform_login(self, username, password):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(
-            'https://tubitv.com/oz/videos/%s/content?video_resources=dash&video_resources=hlsv3&video_resources=hlsv6' % video_id, video_id)
+        video_data = self._download_json(f'https://tubitv.com/oz/videos/{video_id}/content', video_id, query={
+            'video_resources': ['dash', 'hlsv3', 'hlsv6', *self._UNPLAYABLE_FORMATS],
+        })
         title = video_data['title']
 
         formats = []
+        drm_formats = False
 
         for resource in video_data['video_resources']:
             if resource['type'] in ('dash', ):
                 formats += self._extract_mpd_formats(resource['manifest']['url'], video_id, mpd_id=resource['type'], fatal=False)
             elif resource['type'] in ('hlsv3', 'hlsv6'):
                 formats += self._extract_m3u8_formats(resource['manifest']['url'], video_id, 'mp4', m3u8_id=resource['type'], fatal=False)
+            elif resource['type'] in self._UNPLAYABLE_FORMATS:
+                drm_formats = True
+
+        if not formats and drm_formats:
+            self.report_drm(video_id)
+        elif not formats and not video_data.get('policy_match'):  # policy_match is False if content was removed
+            raise ExtractorError('This content is currently unavailable', expected=True)
 
         self._sort_formats(formats)
 
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index d0be7f19e..7d8e97162 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5294,7 +5294,7 @@ def load_plugins(name, suffix, namespace):
 
 
 def traverse_obj(
-        obj, *paths, default=None, expected_type=None, get_all=True,
+        obj, *paths, default=NO_DEFAULT, expected_type=None, get_all=True,
         casesense=True, is_user_input=False, traverse_string=False):
     """
     Safely traverse nested `dict`s and `Sequence`s
@@ -5304,6 +5304,7 @@ def traverse_obj(
     "value"
 
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
+    The next path will also be tested if the path branched but no results could be found.
     A value of None is treated as the absence of a value.
 
     The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
@@ -5342,6 +5343,7 @@ def traverse_obj(
     @returns                The result of the object traversal.
                             If successful, `get_all=True`, and the path branches at least once,
                             then a list of results is returned instead.
+                            A list is always returned if the last path branches and no `default` is given.
     """
     is_sequence = lambda x: isinstance(x, collections.abc.Sequence) and not isinstance(x, (str, bytes))
     casefold = lambda k: k.casefold() if isinstance(k, str) else k
@@ -5385,7 +5387,7 @@ def apply_key(key, obj):
         elif isinstance(key, dict):
             iter_obj = ((k, _traverse_obj(obj, v)) for k, v in key.items())
             yield {k: v if v is not None else default for k, v in iter_obj
-                   if v is not None or default is not None}
+                   if v is not None or default is not NO_DEFAULT}
 
         elif isinstance(obj, dict):
             yield (obj.get(key) if casesense or (key in obj)
@@ -5426,18 +5428,22 @@ def apply_path(start_obj, path):
 
         return has_branched, objs
 
-    def _traverse_obj(obj, path):
+    def _traverse_obj(obj, path, use_list=True):
         has_branched, results = apply_path(obj, path)
         results = LazyList(x for x in map(type_test, results) if x is not None)
-        if results:
-            return results.exhaust() if get_all and has_branched else results[0]
 
-    for path in paths:
-        result = _traverse_obj(obj, path)
+        if get_all and has_branched:
+            return results.exhaust() if results or use_list else None
+
+        return results[0] if results else None
+
+    for index, path in enumerate(paths, 1):
+        use_list = default is NO_DEFAULT and index == len(paths)
+        result = _traverse_obj(obj, path, use_list)
         if result is not None:
             return result
 
-    return default
+    return None if default is NO_DEFAULT else default
 
 
 def traverse_dict(dictn, keys, casesense=True):


Test patch:
diff --git a/test/test_utils.py b/test/test_utils.py
index 69313564a..6f3f6cb91 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1890,6 +1890,7 @@ def test_traverse_obj(self):
                 {'index': 2},
                 {'index': 3},
             ),
+            'dict': {},
         }
 
         # Test base functionality
@@ -1926,11 +1927,15 @@ def test_traverse_obj(self):
 
         # Test alternative paths
         self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
-                         msg='multiple `path_list` should be treated as alternative paths')
+                         msg='multiple `paths` should be treated as alternative paths')
         self.assertEqual(traverse_obj(_TEST_DATA, 'str', 100), 'str',
                          msg='alternatives should exit early')
         self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'fail'), None,
                          msg='alternatives should return `default` if exhausted')
+        self.assertEqual(traverse_obj(_TEST_DATA, (..., 'fail'), 100), 100,
+                         msg='alternatives should track their own branching return')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', ...), ('data', ...)), list(_TEST_DATA['data']),
+                         msg='alternatives on empty objects should search further')
 
         # Test branch and path nesting
         self.assertEqual(traverse_obj(_TEST_DATA, ('urls', (3, 0), 'url')), ['https://www.example.com/0'],
@@ -1963,8 +1968,16 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, ('fail', 'url')), (0, 'url')))}),
                          {0: ['https://www.example.com/1', 'https://www.example.com/0']},
                          msg='tripple nesting in dict path should be treated as branches')
-        self.assertEqual(traverse_obj({}, {0: 1}, default=...), {0: ...},
-                         msg='do not remove `None` values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
+                         msg='remove `None` values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=...), {0: ...},
+                         msg='do not remove `None` values if `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {0: {}},
+                         msg='do not remove empty values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=...), {0: {}},
+                         msg='do not remove empty values when dict key and a default')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', ...)}), {0: []},
+                         msg='if branch in dict key not successful, return `[]`')
 
         # Testing default parameter behavior
         _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
@@ -1981,7 +1994,13 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', 10)), None,
                          msg='`IndexError` should result in `default`')
         self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail'), default=1), 1,
-                         msg='if branched but not successfull return `default`, not `[]`')
+                         msg='if branched but not successful return `default` if defined, not `[]`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail'), default=None), None,
+                         msg='if branched but not successful return `default` even if `default` is `None`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail')), [],
+                         msg='if branched but not successful return `[]`, not `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', ...)), [],
+                         msg='if branched but object is empty return `[]`, not `default`')
 
         # Testing expected_type behavior
         _EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0}