Bugfixes (#1058)

* Add catch for suggested search results; accounts for edge case of no views on result.

* Added exception handling for incorrect cached js files.

* Now allows you to *actually* set filenames, instead of doing partial overrides to create safe filenames.

* Innertube improvements, and skeleton code for future innertube work

pytube/__main__.py

@@ -220,7 +220,17 @@ class YouTube:
                 extract.apply_descrambler(self.vid_info, fmt)
             extract.apply_descrambler(self.player_config_args, fmt)
 
-            extract.apply_signature(self.player_config_args, fmt, self.js)
+            # If the cached js doesn't work, try fetching a new js file
+            # https://github.com/pytube/pytube/issues/1054
+            try:
+                extract.apply_signature(self.player_config_args, fmt, self.js)
+            except exceptions.ExtractError:
+                # To force an update to the js file, we clear the cache and retry
+                self._js = None
+                self._js_url = None
+                pytube.__js__ = None
+                pytube.__js_url__ = None
+                extract.apply_signature(self.player_config_args, fmt, self.js)
 
             # build instances of :class:`Stream <Stream>`
             # Initialize stream objects

pytube/cipher.py

@@ -17,7 +17,7 @@ import re
 from itertools import chain
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
-from pytube.exceptions import RegexMatchError
+from pytube.exceptions import ExtractError, RegexMatchError
 from pytube.helpers import cache, regex_search
 from pytube.parser import find_object_from_startpoint, throttling_array_split
 
@@ -60,7 +60,7 @@ class Cipher:
             if not callable(curr_func):
                 logger.debug(f'{curr_func} is not callable.')
                 logger.debug(f'Throttling array:\n{self.throttling_array}\n')
-                raise TypeError(f'{curr_func} is not callable.')
+                raise ExtractError(f'{curr_func} is not callable.')
 
             first_arg = self.throttling_array[int(step[1])]
 

pytube/contrib/search.py

@@ -137,6 +137,14 @@ class Search:
                 if 'channelRenderer' in video_details:
                     continue
 
+                # Skip 'people also searched for' results
+                if 'horizontalCardListRenderer' in video_details:
+                    continue
+
+                # Can't seem to reproduce, probably related to typo fix suggestions
+                if 'didYouMeanRenderer' in video_details:
+                    continue
+
                 if 'videoRenderer' not in video_details:
                     logger.warn('Unexpected renderer encountered.')
                     logger.warn(f'Renderer name: {video_details.keys()}')
@@ -166,7 +174,11 @@ class Search:
                     else:
                         vid_view_count_text = vid_renderer['viewCountText']['simpleText']
                     # Strip ' views' text, then remove commas
-                    vid_view_count = int(vid_view_count_text.split()[0].replace(',',''))
+                    stripped_text = vid_view_count_text.split()[0].replace(',','')
+                    if stripped_text == 'No':
+                        vid_view_count = 0
+                    else:
+                        vid_view_count = int(stripped_text)
                 else:
                     vid_view_count = 0
                 if 'lengthText' in vid_renderer:

pytube/innertube.py

@@ -5,6 +5,7 @@ interfaces returns raw results. These should instead be parsed to extract
 the useful information for the end user.
 """
 # Native python imports
+from datetime import datetime
 import json
 from urllib import parse
 
@@ -12,8 +13,53 @@ from urllib import parse
 from pytube import request
 
 
+_default_clients = {
+    'WEB': {
+        'context': {
+            'client': {
+                'clientName': 'WEB',
+                'clientVersion': '2.20200720.00.02'
+            }
+        },
+        'api_key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+    },
+    'ANDROID': {
+        'context': {
+            'client': {
+                'clientName': 'ANDROID',
+                'clientVersion': '16.20'
+            }
+        },
+        'api_key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+    }
+}
+_token_timeout = 1800
+
+
 class InnerTube:
     """Object for interacting with the innertube API."""
+    def __init__(self, client='WEB', bearer_token=None):
+        self.context = _default_clients[client]['context']
+        self.api_key = _default_clients[client]['api_key']
+        self.bearer_token = bearer_token
+        self.last_refresh = None
+        self.refresh_bearer_token()
+
+    def refresh_bearer_token(self, force=False):
+        """Refreshes the OAuth token.
+
+        This is skeleton code for potential future functionality, so it is incomplete.
+        """
+        # Skip refresh if it's been less than 30 minutes
+        if self.last_refresh and not force:
+            # Use a 30-minute timer.
+            if (datetime.now() - self.last_refresh).total_seconds() < _token_timeout:
+                return
+
+        # TODO: Refresh the token
+
+        self.last_refresh = datetime.now()
+
     @property
     def base_url(self):
         """Return the base url endpoint for the innertube API."""
@@ -23,30 +69,31 @@ class InnerTube:
     def base_data(self):
         """Return the base json data to transmit to the innertube API."""
         return {
-            'context': {
-                'client': {
-                    'clientName': 'WEB',
-                    'clientVersion': '2.20200720.00.02'
-                }
-            }
+            'context': self.context
         }
 
     @property
     def base_params(self):
         """Return the base query parameters to transmit to the innertube API."""
         return {
-            'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+            'key': self.api_key
         }
 
     def _call_api(self, endpoint, query, data):
         """Make a request to a given endpoint with the provided query parameters and data."""
         endpoint_url = f'{endpoint}?{parse.urlencode(query)}'
+        headers = {
+            'Content-Type': 'application/json',
+        }
+        # Add the bearer token if applicable
+        if self.bearer_token:
+            self.refresh_bearer_token()
+            headers['authorization'] = f'Bearer {self.bearer_token}'
+
         response = request._execute_request(
             endpoint_url,
             'POST',
-            headers={
-                'Content-Type': 'application/json',
-            },
+            headers=headers,
             data=data
         )
         return json.loads(response.read())
@@ -122,3 +169,42 @@ class InnerTube:
             data['continuation'] = continuation
         data.update(self.base_data)
         return self._call_api(endpoint, query, data)
+
+    def verify_age(self, video_id):
+        """Make a request to the age_verify endpoint.
+
+        Notable examples of the types of video this verification step is for:
+        * https://www.youtube.com/watch?v=QLdAhwSBZ3w
+        * https://www.youtube.com/watch?v=hc0ZDaAZQT0
+
+        :param str video_id:
+            The video id to get player info for.
+        :rtype: dict
+        :returns:
+            Returns information that includes a URL for bypassing certain restrictions.
+        """
+        endpoint = f'{self.base_url}/verify_age'
+        data = {
+            'nextEndpoint': {
+                'urlEndpoint': {
+                    'url': f'/watch?v={video_id}'
+                }
+            },
+            'setControvercy': True
+        }
+        data.update(self.base_data)
+        result = self._call_api(endpoint, self.base_params, data)
+        return result
+
+    def get_transcript(self, video_id):
+        """Make a request to the get_transcript endpoint.
+
+        This is likely related to captioning for videos, but is currently untested.
+        """
+        endpoint = f'{self.base_url}/get_transcript'
+        query = {
+            'videoId': video_id,
+        }
+        query.update(self.base_params)
+        result = self._call_api(endpoint, query, self.base_data)
+        return result

pytube/streams.py

@@ -282,16 +282,14 @@ class Stream:
 
     def get_file_path(
         self,
-        filename: Optional[str],
-        output_path: Optional[str],
+        filename: Optional[str] = None,
+        output_path: Optional[str] = None,
         filename_prefix: Optional[str] = None,
     ) -> str:
-        if filename:
-            filename = f"{safe_filename(filename)}.{self.subtype}"
-        else:
+        if not filename:
             filename = self.default_filename
         if filename_prefix:
-            filename = f"{safe_filename(filename_prefix)}{filename}"
+            filename = f"{filename_prefix}{filename}"
         return os.path.join(target_directory(output_path), filename)
 
     def exists_at_path(self, file_path: str) -> bool:

tests/test_streams.py

@@ -111,7 +111,7 @@ def test_views(cipher_signature):
 
 
 @mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "6796391"})
+    "pytube.request.head", MagicMock(return_value={"content-length": "6796391"})
 )
 @mock.patch(
     "pytube.request.stream",
@@ -124,7 +124,7 @@ def test_download(cipher_signature):
 
 
 @mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "16384"})
+    "pytube.request.head", MagicMock(return_value={"content-length": "16384"})
 )
 @mock.patch(
     "pytube.request.stream",
@@ -142,7 +142,7 @@ def test_download_with_prefix(cipher_signature):
 
 
 @mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "16384"})
+    "pytube.request.head", MagicMock(return_value={"content-length": "16384"})
 )
 @mock.patch(
     "pytube.request.stream",
@@ -155,12 +155,12 @@ def test_download_with_filename(cipher_signature):
         file_path = stream.download(filename="cool name bro")
         assert file_path == os.path.join(
             "/target",
-            "cool name bro.mp4"
+            "cool name bro"
         )
 
 
 @mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "16384"})
+    "pytube.request.head", MagicMock(return_value={"content-length": "16384"})
 )
 @mock.patch(
     "pytube.request.stream",
@@ -181,7 +181,7 @@ def test_download_with_existing(cipher_signature):
 
 
 @mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "16384"})
+    "pytube.request.head", MagicMock(return_value={"content-length": "16384"})
 )
 @mock.patch(
     "pytube.request.stream",
@@ -212,7 +212,7 @@ def test_progressive_streams_return_includes_video_track(cipher_signature):
 
 
 @mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "16384"})
+    "pytube.request.head", MagicMock(return_value={"content-length": "16384"})
 )
 @mock.patch(
     "pytube.request.stream",
@@ -233,7 +233,7 @@ def test_on_progress_hook(cipher_signature):
 
 
 @mock.patch(
-    "pytube.streams.request.head", MagicMock(return_value={"content-length": "16384"})
+    "pytube.request.head", MagicMock(return_value={"content-length": "16384"})
 )
 @mock.patch(
     "pytube.request.stream",