Repository: pytube/pytube

Issue:
[BUG]
**Before creating an issue**

Please confirm that you are on the latest version of pytube by installing from the source.
You can do this by running `python -m pip install git+https://github.com/pytube/pytube`.
Sometimes, the pypi library repository is not up to date, and your issue may have been fixed already!

**Describe the bug**
urllib.error.HTTPError: HTTP Error 404: Not Found

**To Reproduce**
>>> import pytube
>>> p = pytube.Playlist('https://www.youtube.com/playlist?list=PLg0OBzfAXdsrNhPhH85_LSs7fDjYW_7sX')
>>> p.videos # <generator object Playlist.videos at 0x7feffe799c80>
>>> p  <- error Occured

**Expected behavior**
>>> p 
>>> ['video1_url', 'video2_url',...] 

**Output**
Traceback (most recent call last):
  File "<input>", line 1, in <module>
  File "/Users/dooseop/PycharmProjects/snji_bot/venv/lib/python3.8/site-packages/pytube/contrib/playlist.py", line 230, in __repr__
    return f"{self.video_urls}"
  File "/Users/dooseop/PycharmProjects/snji_bot/venv/lib/python3.8/site-packages/pytube/contrib/playlist.py", line 211, in video_urls
    for page in list(self._paginate())
  File "/Users/dooseop/PycharmProjects/snji_bot/venv/lib/python3.8/site-packages/pytube/contrib/playlist.py", line 78, in _paginate
    req = request.get(load_more_url, extra_headers=headers)
  File "/Users/dooseop/PycharmProjects/snji_bot/venv/lib/python3.8/site-packages/pytube/request.py", line 42, in get
    return _execute_request(url, headers=extra_headers).read().decode("utf-8")
  File "/Users/dooseop/PycharmProjects/snji_bot/venv/lib/python3.8/site-packages/pytube/request.py", line 26, in _execute_request
    return urlopen(request)  # nosec
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/urllib/request.py", line 222, in urlopen
    return opener.open(url, data, timeout)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/urllib/request.py", line 531, in open
    response = meth(req, response)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/urllib/request.py", line 640, in http_response
    response = self.parent.error(
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/urllib/request.py", line 569, in error
    return self._call_chain(*args)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/urllib/request.py", line 502, in _call_chain
    result = func(*args)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/urllib/request.py", line 649, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found

**System information**
Please provide the following information:
- Python 3.8 
- MacOS Bigsur
- pytube version 10.5.1

Yesterday, It worked without any problem.
However, when I ran the program today, the following error occurred. I don't know why. 
Please check.

Thanks a lot.

Discussion:
bigbluedeer: i dont know what is going on with the CI testing. while manually testing its working as intended
tfdahlin: hey @bigbluedeer I'll try to review this in the evening, and I'll fix the tests as necessary if it all looks good to me. Looks like the reason that the tests are failing is because of the way that the mocks are set up conflicting with the new `post` method you've implemented
tfdahlin: Got my days mixed up, thought yesterday was Friday. Will get to this today
msemple1111: I had a playlist that was broken, tested and can confirm this works on the two i tried
tfdahlin: Alright, I like this fix, but I think we do need to dynamically extract the API key from it. Working on a solution for that now, then I'll fix the tests for you, then I'll merge it
PatrickPoncet: Not super pro with pip module & git but I did "python -m pip install git+https://github.com/pytube/pytube" and still getting version pytube-10.5.1 typing-extensions-3.7.4.3

Am I missing something in order to get the latest version to get this fix?
tfdahlin: @PatrickPoncet this fix has not yet been merged. I just finished making sure the tests pass, and that flake8 issues are fixed, and will be merging + updating pypi right now. Try again in like 5 minutes and it should work!

Patch:
diff --git a/pytube/contrib/playlist.py b/pytube/contrib/playlist.py
index d781187..83036ea 100644
--- a/pytube/contrib/playlist.py
+++ b/pytube/contrib/playlist.py
@@ -30,12 +30,33 @@ class Playlist(Sequence):
         if proxies:
             install_proxy(proxies)
 
+        # These need to be initialized as None for the properties.
+        self._html = None
+        self._ytcfg = None
+
         self.playlist_id = extract.playlist_id(url)
 
         self.playlist_url = (
             f"https://www.youtube.com/playlist?list={self.playlist_id}"
         )
-        self.html = request.get(self.playlist_url)
+
+    @property
+    def html(self):
+        if self._html:
+            return self._html
+        self._html = request.get(self.playlist_url)
+        return self._html
+
+    @property
+    def ytcfg(self):
+        if self._ytcfg:
+            return self._ytcfg
+        self._ytcfg = extract.get_ytcfg(self.html)
+        return self._ytcfg
+
+    @property
+    def yt_api_key(self):
+        return self.ytcfg['INNERTUBE_API_KEY']
 
     def _paginate(
         self, until_watch_id: Optional[str] = None
@@ -49,7 +70,6 @@ class Playlist(Sequence):
         :rtype: Iterable[List[str]]
         :returns: Iterable of lists of YouTube watch ids
         """
-        req = self.html
         videos_urls, continuation = self._extract_videos(
             json.dumps(extract.initial_data(self.html))
         )
@@ -67,15 +87,15 @@ class Playlist(Sequence):
         # than 100 songs inside a playlist, so we need to add further requests
         # to gather all of them
         if continuation:
-            load_more_url, headers = self._build_continuation_url(continuation)
+            load_more_url, headers, data = self._build_continuation_url(continuation)
         else:
-            load_more_url, headers = None, None
+            load_more_url, headers, data = None, None, None
 
-        while load_more_url and headers:  # there is an url found
+        while load_more_url and headers and data:  # there is an url found
             logger.debug("load more url: %s", load_more_url)
             # requesting the next page of videos with the url generated from the
-            # previous page
-            req = request.get(load_more_url, extra_headers=headers)
+            # previous page, needs to be a post
+            req = request.post(load_more_url, extra_headers=headers, data=data)
             # extract up to 100 songs from the page loaded
             # returns another continuation if more videos are available
             videos_urls, continuation = self._extract_videos(req)
@@ -89,32 +109,43 @@ class Playlist(Sequence):
             yield videos_urls
 
             if continuation:
-                load_more_url, headers = self._build_continuation_url(
+                load_more_url, headers, data = self._build_continuation_url(
                     continuation
                 )
             else:
-                load_more_url, headers = None, None
+                load_more_url, headers, data = None, None, None
 
-    @staticmethod
-    def _build_continuation_url(continuation: str) -> Tuple[str, dict]:
+    def _build_continuation_url(self, continuation: str) -> Tuple[str, dict, dict]:
         """Helper method to build the url and headers required to request
         the next page of videos
 
         :param str continuation: Continuation extracted from the json response
             of the last page
-        :rtype: Tuple[str, dict]
+        :rtype: Tuple[str, dict, dict]
         :returns: Tuple of an url and required headers for the next http
             request
         """
         return (
             (
-                f"https://www.youtube.com/browse_ajax?ctoken="
-                f"{continuation}&continuation={continuation}"
+                # was changed to this format (and post requests)
+                # between 2021.03.02 and 2021.03.03
+                "https://www.youtube.com/youtubei/v1/browse?key="
+                f"{self.yt_api_key}"
             ),
             {
                 "X-YouTube-Client-Name": "1",
                 "X-YouTube-Client-Version": "2.20200720.00.02",
             },
+            # extra data required for post request
+            {
+                "continuation": continuation,
+                "context": {
+                    "client": {
+                        "clientName": "WEB",
+                        "clientVersion": "2.20200720.00.02"
+                    }
+                }
+            }
         )
 
     @staticmethod
@@ -150,9 +181,9 @@ class Playlist(Sequence):
             try:
                 # this is the json tree structure, if the json was directly sent
                 # by the server in a continuation response
-                important_content = initial_data[1]['response']['onResponseReceivedActions'][
-                    0
-                ]['appendContinuationItemsAction']['continuationItems']
+                # no longer a list and no longer has the "response" key
+                important_content = initial_data['onResponseReceivedActions'][0][
+                    'appendContinuationItemsAction']['continuationItems']
                 videos = important_content
             except (KeyError, IndexError, TypeError) as p:
                 print(p)
diff --git a/pytube/extract.py b/pytube/extract.py
index 7a87fa2..adce3bb 100644
--- a/pytube/extract.py
+++ b/pytube/extract.py
@@ -24,6 +24,7 @@ from pytube.exceptions import RegexMatchError
 from pytube.helpers import regex_search
 from pytube.metadata import YouTubeMetadata
 from pytube.parser import parse_for_object
+from pytube.parser import parse_for_all_objects
 
 logger = logging.getLogger(__name__)
 
@@ -354,6 +355,40 @@ def get_ytplayer_config(html: str) -> Any:
     )
 
 
+def get_ytcfg(html: str) -> str:
+    """Get the entirety of the ytcfg object.
+
+    This is built over multiple pieces, so we have to find all matches and
+    combine the dicts together.
+
+    :param str html:
+        The html contents of the watch page.
+    :rtype: str
+    :returns:
+        Substring of the html containing the encoded manifest data.
+    """
+    ytcfg = {}
+    ytcfg_patterns = [
+        r"ytcfg\s=\s",
+        r"ytcfg\.set\("
+    ]
+    for pattern in ytcfg_patterns:
+        # Try each pattern consecutively and try to build a cohesive object
+        try:
+            found_objects = parse_for_all_objects(html, pattern)
+            for obj in found_objects:
+                ytcfg.update(obj)
+        except HTMLParseError:
+            continue
+
+    if len(ytcfg) > 0:
+        return ytcfg
+
+    raise RegexMatchError(
+        caller="get_ytcfg", pattern="ytcfg_pattenrs"
+    )
+
+
 def apply_signature(config_args: Dict, fmt: str, js: str) -> None:
     """Apply the decrypted signature to the stream manifest.
 
diff --git a/pytube/parser.py b/pytube/parser.py
index 2ddefdc..8a71b0c 100644
--- a/pytube/parser.py
+++ b/pytube/parser.py
@@ -4,6 +4,39 @@ import re
 from pytube.exceptions import HTMLParseError
 
 
+def parse_for_all_objects(html, preceding_regex):
+    """Parses input html to find all matches for the input starting point.
+
+    :param str html:
+        HTML to be parsed for an object.
+    :param str preceding_regex:
+        Regex to find the string preceding the object.
+    :rtype list:
+    :returns:
+        A list of dicts created from parsing the objects.
+    """
+    result = []
+    regex = re.compile(preceding_regex)
+    match_iter = regex.finditer(html)
+    for match in match_iter:
+        if match:
+            start_index = match.end()
+            try:
+                obj = parse_for_object_from_startpoint(html, start_index)
+            except HTMLParseError:
+                # Some of the instances might fail because set is technically
+                # a method of the ytcfg object. We'll skip these since they
+                # don't seem relevant at the moment.
+                continue
+            else:
+                result.append(obj)
+
+    if len(result) == 0:
+        raise HTMLParseError(f'No matches for regex {preceding_regex}')
+
+    return result
+
+
 def parse_for_object(html, preceding_regex):
     """Parses input html to find the end of a JavaScript object.
 
@@ -20,11 +53,11 @@ def parse_for_object(html, preceding_regex):
     if not result:
         raise HTMLParseError(f'No matches for regex {preceding_regex}')
 
-    start_index = result.span()[1]
+    start_index = result.end()
     return parse_for_object_from_startpoint(html, start_index)
 
 
-def parse_for_object_from_startpoint(html, start_point):
+def find_object_from_startpoint(html, start_point):
     """Parses input html to find the end of a JavaScript object.
 
     :param str html:
@@ -77,10 +110,25 @@ def parse_for_object_from_startpoint(html, start_point):
         i += 1
 
     full_obj = html[:i]
+    return full_obj  # noqa: R504
+
+
+def parse_for_object_from_startpoint(html, start_point):
+    """JSONifies an object parsed from HTML.
+
+    :param str html:
+        HTML to be parsed for an object.
+    :param int start_point:
+        Index of where the object starts.
+    :rtype dict:
+    :returns:
+        A dict created from parsing the object.
+    """
+    full_obj = find_object_from_startpoint(html, start_point)
     try:
         return json.loads(full_obj)
     except json.decoder.JSONDecodeError:
         try:
             return ast.literal_eval(full_obj)
-        except ValueError:
+        except (ValueError, SyntaxError):
             raise HTMLParseError('Could not parse object.')
diff --git a/pytube/request.py b/pytube/request.py
index 182561b..274cf0f 100644
--- a/pytube/request.py
+++ b/pytube/request.py
@@ -3,6 +3,7 @@
 import logging
 from functools import lru_cache
 import re
+import json
 from urllib import parse
 from urllib.request import Request
 from urllib.request import urlopen
@@ -15,12 +16,16 @@ default_chunk_size = 4096  # 4kb
 default_range_size = 9437184  # 9MB
 
 
-def _execute_request(url, method=None, headers=None):
+def _execute_request(url, method=None, headers=None, data=None):
     base_headers = {"User-Agent": "Mozilla/5.0", "accept-language": "en-US,en"}
     if headers:
         base_headers.update(headers)
+    if data:
+        # encode data for request
+        if not isinstance(data, bytes):
+            data = bytes(json.dumps(data), encoding="utf-8")
     if url.lower().startswith("http"):
-        request = Request(url, headers=base_headers, method=method)
+        request = Request(url, headers=base_headers, method=method, data=data)
     else:
         raise ValueError("Invalid URL")
     return urlopen(request)  # nosec
@@ -42,6 +47,31 @@ def get(url, extra_headers=None):
     return _execute_request(url, headers=extra_headers).read().decode("utf-8")
 
 
+def post(url, extra_headers=None, data=None):
+    """Send an http POST request.
+
+    :param str url:
+        The URL to perform the POST request for.
+    :param dict extra_headers:
+        Extra headers to add to the request
+    :param dict data:
+        The data to send on the POST request
+    :rtype: str
+    :returns:
+        UTF-8 encoded string of response
+    """
+    # could technically be implemented in get,
+    # but to avoid confusion implemented like this
+    if extra_headers is None:
+        extra_headers = {}
+    if data is None:
+        data = {}
+    # required because the youtube servers are strict on content type
+    # raises HTTPError [400]: Bad Request otherwise
+    extra_headers.update({"Content-Type": "application/json"})
+    return _execute_request(url, headers=extra_headers, data=data).read().decode("utf-8")
+
+
 def seq_stream(url, chunk_size=default_chunk_size, range_size=default_range_size):
     """Read the response in sequence.
     :param str url: The URL to perform the GET request for.


Test patch:
diff --git a/tests/contrib/test_playlist.py b/tests/contrib/test_playlist.py
index 2ae42c3..b2d60b9 100644
--- a/tests/contrib/test_playlist.py
+++ b/tests/contrib/test_playlist.py
@@ -1,12 +1,11 @@
 # -*- coding: utf-8 -*-
 import datetime
 from unittest import mock
-from unittest.mock import MagicMock
 
 from pytube import Playlist
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_title(request_get):
     request_get.return_value = (
         "<title>(149) Python Tutorial for Beginners "
@@ -24,7 +23,7 @@ def test_title(request_get):
     )
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_init_with_playlist_url(request_get):
     request_get.return_value = ""
     url = (
@@ -35,7 +34,7 @@ def test_init_with_playlist_url(request_get):
     assert playlist.playlist_url == url
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_init_with_watch_url(request_get):
     request_get.return_value = ""
     url = (
@@ -49,7 +48,7 @@ def test_init_with_watch_url(request_get):
     )
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_last_updated(request_get, playlist_html):
     expected = datetime.date(2020, 3, 11)
     request_get.return_value = playlist_html
@@ -60,13 +59,11 @@ def test_last_updated(request_get, playlist_html):
     assert playlist.last_updated == expected
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_video_urls(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
-    request_get.assert_called()
     assert playlist.video_urls == [
         "https://www.youtube.com/watch?v=ujTCoH21GlA",
         "https://www.youtube.com/watch?v=45ryDIPHdGg",
@@ -81,15 +78,23 @@ def test_video_urls(request_get, playlist_html):
         "https://www.youtube.com/watch?v=g1Zbuk1gAfk",
         "https://www.youtube.com/watch?v=zixd-si9Q-o",
     ]
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
-def test_repr(request_get, playlist_html):
+@mock.patch("pytube.request.get")
+def test_html(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
+    assert playlist.html == playlist_html
     request_get.assert_called()
+
+
+@mock.patch("pytube.request.get")
+def test_repr(request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = playlist_html
+    playlist = Playlist(url)
     assert (
         repr(playlist) == "["
         "'https://www.youtube.com/watch?v=ujTCoH21GlA', "
@@ -106,30 +111,29 @@ def test_repr(request_get, playlist_html):
         "'https://www.youtube.com/watch?v=zixd-si9Q-o'"
         "]"
     )
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_sequence(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
     assert playlist[0] == "https://www.youtube.com/watch?v=ujTCoH21GlA"
     assert len(playlist) == 12
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 @mock.patch("pytube.cli.YouTube.__init__", return_value=None)
 def test_videos(youtube, request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
-    request_get.assert_called()
     assert len(list(playlist.videos)) == 12
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 @mock.patch("pytube.cli.YouTube.__init__", return_value=None)
 def test_load_more(youtube, request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
@@ -138,12 +142,11 @@ def test_load_more(youtube, request_get, playlist_html):
         '{"content_html":"", "load_more_widget_html":""}',
     ]
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(side_effect=["dummy", None])
-    request_get.assert_called()
     assert len(list(playlist.videos)) == 12
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 @mock.patch("pytube.contrib.playlist.install_proxy", return_value=None)
 def test_proxy(install_proxy, request_get):
     url = "https://www.fakeurl.com/playlist?list=whatever"
@@ -152,31 +155,34 @@ def test_proxy(install_proxy, request_get):
     install_proxy.assert_called_with({"http": "things"})
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_trimmed(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
-    assert request_get.call_count == 1
     trimmed = list(playlist.trimmed("1BYu65vLKdA"))
     assert trimmed == [
         "https://www.youtube.com/watch?v=ujTCoH21GlA",
         "https://www.youtube.com/watch?v=45ryDIPHdGg",
     ]
+    assert request_get.call_count == 1
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
-def test_playlist_failed_pagination(request_get, playlist_long_html):
+@mock.patch("pytube.request.get")
+@mock.patch("pytube.request.post")
+def test_playlist_failed_pagination(request_post, request_get, playlist_long_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.side_effect = [
         playlist_long_html,
-        "{}",
+    ]
+    request_post.side_effect = [
+        "{}"
     ]
     playlist = Playlist(url)
     video_urls = playlist.video_urls
     assert len(video_urls) == 100
-    assert request_get.call_count == 2
+    assert request_get.call_count == 1
+    assert request_post.call_count == 1
     # TODO: Cannot get this test to work probably
     # request_get.assert_called_with(
     #    "https://www.youtube.com/browse_ajax?ctoken" # noqa
@@ -189,11 +195,14 @@ def test_playlist_failed_pagination(request_get, playlist_long_html):
     # ) # noqa
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
-def test_playlist_pagination(request_get, playlist_html, playlist_long_html):
+@mock.patch("pytube.request.get")
+@mock.patch("pytube.request.post")
+def test_playlist_pagination(request_post, request_get, playlist_html, playlist_long_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.side_effect = [
-        playlist_long_html,
+        playlist_long_html
+    ]
+    request_post.side_effect = [
         '{"content_html":"<a '
         'href=\\"/watch?v=BcWz41-4cDk&amp;feature=plpp_video&amp;ved'
         '=CCYQxjQYACITCO33n5-pn-cCFUG3xAodLogN2yj6LA\\">}", '
@@ -202,10 +211,11 @@ def test_playlist_pagination(request_get, playlist_html, playlist_long_html):
     ]
     playlist = Playlist(url)
     assert len(playlist.video_urls) == 100
-    assert request_get.call_count == 2
+    assert request_get.call_count == 1
+    assert request_post.call_count == 1
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_trimmed_pagination(request_get, playlist_html, playlist_long_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.side_effect = [
@@ -222,7 +232,7 @@ def test_trimmed_pagination(request_get, playlist_html, playlist_long_html):
 
 
 # TODO: Test case not clear to me
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_trimmed_pagination_not_found(
     request_get, playlist_html, playlist_long_html
 ):
@@ -241,7 +251,7 @@ def test_trimmed_pagination_not_found(
 
 
 # test case for playlist with submenus
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_playlist_submenu(
         request_get, playlist_submenu_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"