Repository: streamlink/streamlink
Pull request #1874 (2018-06-28 18:29:43)

plugins.vk: _url_re update, allow embedded content, plugin cleanup
- **_url_re** update
- allow **embedded** content
- plugin **cleanup** / use **API**

~Added some` itertags tests`~

---

close https://github.com/streamlink/streamlink/issues/1584
close https://github.com/streamlink/streamlink/pull/1603



Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 74a24292..801c74f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,27 @@
 # Changelog
 
+## streamlink 0.14.1 (2018-06-28)
+
+Only a few fixes in this small maintenance release.
+
+- Fix the tv3cat plugin (https://github.com/streamlink/streamlink/pull/1851)
+- VOD supported added to atresplayer plugin (https://github.com/streamlink/streamlink/pull/1852, https://github.com/streamlink/streamlink/pull/1853)
+- Fixed Twitch OAuth request flow (https://github.com/streamlink/streamlink/pull/1856)
+- Removed tv8cat plugin (https://github.com/streamlink/streamlink/pull/1860)
+
+```text
+NyanKiyoshi <NyanKiyoshi@users.noreply.github.com> (1):
+      Fix 404 error on oauth authorize url
+
+beardypig <beardypig@protonmail.com> (5):
+      plugins.t3cat: update validation rule, refactor plugin a little bit
+      plugins.atresplayer: update to support VOD streams
+      stream.dash: support for SegmentList streams with ranged segments
+      release: fix release notes manual install url
+      plugins.tv8cat: plugin removed - the live broadcast is no longer available
+```
+
+
 ## streamlink 0.14.0 (2018-06-26)
 
 Here are the changes to this months release!
diff --git a/docs/plugin_matrix.rst b/docs/plugin_matrix.rst
index 3b8e8970..edf47b74 100644
--- a/docs/plugin_matrix.rst
+++ b/docs/plugin_matrix.rst
@@ -123,11 +123,11 @@ mediaklikk              mediaklikk.hu        Yes   No    Streams may be geo-rest
 mips                    mips.tv              Yes   --    Requires rtmpdump with K-S-V patches.
 mitele                  mitele.es            Yes   No    Streams may be geo-restricted to Spain.
 mixer                   mixer.com            Yes   Yes
+mjunoon                 mjunoon.tv           Yes   Yes
 mlgtv                   mlg.tv               Yes   --
 nbc                     nbc.com              No    Yes   Streams are geo-restricted to USA. Authentication is not supported.
 nbcsports               nbcsports.com        No    Yes   Streams maybe be geo-restricted to USA. Authentication is not supported.
 nhkworld                nhk.or.jp/nhkworld   Yes   No
-nineanime               9anime.to            --    Yes
 nos                     nos.nl               Yes   Yes   Streams may be geo-restricted to Netherlands.
 npo                     - npo.nl             Yes   Yes   Streams may be geo-restricted to Netherlands.
                         - zapp.nl
diff --git a/script/release.sh b/script/release.sh
index e8437fbb..e6ed1382 100755
--- a/script/release.sh
+++ b/script/release.sh
@@ -62,7 +62,7 @@ success() {
 
 changelog() {
   temp_changes=$(mktemp) && trap "rm -rf ${temp_changes}" EXIT || exit 255
-  date=$(date +"%Y-%m-%d")
+  date=$(date -u +"%Y-%m-%d")
   shortlog=$(git shortlog --email --no-merges --pretty=%s ${1}..)
 
   echo -e "\n## streamlink $2 ($date)\n\n!! WRITE RELEASE NOTES HERE !!\n\n\`\`\`text\n${shortlog}\n\`\`\`\n" > "${temp_changes}"
diff --git a/src/streamlink/plugin/plugin.py b/src/streamlink/plugin/plugin.py
index d6613d50..3aed7c14 100644
--- a/src/streamlink/plugin/plugin.py
+++ b/src/streamlink/plugin/plugin.py
@@ -56,7 +56,7 @@ def stream_weight(stream):
         if stream in weights:
             return weights[stream], group
 
-    match = re.match(r"^(\d+)(k|p)?(\d+)?(\+)?(?:_(\d+)k)?(?:_(alt)(\d)?)?$", stream)
+    match = re.match(r"^(\d+)(k|p)?(\d+)?(\+)?(?:[a_](\d+)k)?(?:_(alt)(\d)?)?$", stream)
 
     if match:
         weight = 0
diff --git a/src/streamlink/plugins/mjunoon.py b/src/streamlink/plugins/mjunoon.py
new file mode 100644
index 00000000..ebf874fc
--- /dev/null
+++ b/src/streamlink/plugins/mjunoon.py
@@ -0,0 +1,34 @@
+import logging
+import re
+
+from streamlink.compat import urlparse, parse_qsl
+from streamlink.plugin import Plugin
+from streamlink.plugin.api.utils import itertags
+from streamlink.stream import HLSStream
+
+log = logging.getLogger(__name__)
+
+
+class Mjunoon(Plugin):
+    url_re = re.compile(r"""https?://(?:www.)?mjunoon.tv/""", re.VERBOSE)
+
+    @classmethod
+    def can_handle_url(cls, url):
+        return cls.url_re.match(url) is not None
+
+    def _get_streams(self):
+        res = self.session.http.get(self.url)
+        for script in itertags(res.text, 'script'):
+            if script.attributes.get("id") == "playerScript":
+                log.debug("Found the playerScript script tag")
+                urlparts = urlparse(script.attributes.get("src"))
+                i = 0
+
+                for key, url in parse_qsl(urlparts.query):
+                    if key == "streamUrl":
+                        i += 1
+                        for s in HLSStream.parse_variant_playlist(self.session, url, params=dict(id=i), verify=False).items():
+                            yield s
+
+
+__plugin__ = Mjunoon
diff --git a/src/streamlink/plugins/nineanime.py b/src/streamlink/plugins/nineanime.py
index 5e5fa038..0bac36f1 100644
--- a/src/streamlink/plugins/nineanime.py
+++ b/src/streamlink/plugins/nineanime.py
@@ -1,69 +1,1 @@
-import re
-from streamlink.plugin import Plugin
-from streamlink.plugin.api import http
-from streamlink.plugin.api import useragents
-from streamlink.plugin.api import validate
-from streamlink.stream import HTTPStream
-from streamlink.compat import urlparse
-
-
-class NineAnime(Plugin):
-    _episode_info_url = "//9anime.to/ajax/episode/info"
-
-    _info_schema = validate.Schema({
-        "grabber": validate.url(),
-        "params": {
-            "id": validate.text,
-            "token": validate.text,
-            "options": validate.text,
-        }
-    })
-
-    _streams_schema = validate.Schema({
-        "token": validate.text,
-        "error": None,
-        "data": [{
-            "label": validate.text,
-            "file": validate.url(),
-            "type": "mp4"
-        }]
-    })
-
-    _url_re = re.compile(r"https?://9anime.to/watch/(?:[^.]+?\.)(\w+)/(\w+)")
-
-    @classmethod
-    def can_handle_url(cls, url):
-        return cls._url_re.match(url) is not None
-
-    def add_scheme(self, url):
-        # update the scheme for the grabber url if required
-        if url.startswith("//"):
-            url = "{0}:{1}".format(urlparse(self.url).scheme, url)
-        return url
-
-    @Plugin.broken(1110)
-    def _get_streams(self):
-        match = self._url_re.match(self.url)
-        film_id, episode_id = match.groups()
-
-        headers = {
-            "Referer": self.url,
-            "User-Agent": useragents.FIREFOX
-        }
-
-        # Get the info about the Episode, including the Grabber API URL
-        info_res = http.get(self.add_scheme(self._episode_info_url),
-                            params=dict(update=0, film=film_id, id=episode_id),
-                            headers=headers)
-        info = http.json(info_res, schema=self._info_schema)
-
-        # Get the data about the streams from the Grabber API
-        grabber_url = self.add_scheme(info["grabber"])
-        stream_list_res = http.get(grabber_url, params=info["params"], headers=headers)
-        stream_data = http.json(stream_list_res, schema=self._streams_schema)
-
-        for stream in stream_data["data"]:
-            yield stream["label"], HTTPStream(self.session, stream["file"])
-
-
-__plugin__ = NineAnime
+# Plugin removed - https://github.com/streamlink/streamlink/issues/1862
diff --git a/src/streamlink/plugins/vk.py b/src/streamlink/plugins/vk.py
index d21983e3..085e2e2d 100644
--- a/src/streamlink/plugins/vk.py
+++ b/src/streamlink/plugins/vk.py
@@ -1,20 +1,25 @@
-import json
+# -*- coding: utf-8 -*-
+import logging
 import re
 
-from streamlink.plugin import Plugin
-from streamlink.plugin.api import http
-from streamlink.plugin.api import validate
 from streamlink.compat import urlparse, unquote
+from streamlink.plugin import Plugin
+from streamlink.plugin.api import http, useragents
+from streamlink.plugin.api.utils import itertags
 from streamlink.stream import HTTPStream, HLSStream
-from streamlink.utils import parse_json, update_scheme
-from streamlink.utils.crypto import unpad_pkcs5
+from streamlink.utils import update_scheme
+
+log = logging.getLogger(__name__)
 
 
 class VK(Plugin):
-    _url_re = re.compile(r"http(?:s)?://(\w+\.)?vk.com/video-[0-9]*_[0-9]*")
-    _url_catalog_re = re.compile(r"http(?:s)?://(\w+\.)?vk.com/videos-[0-9]*")
-    _livestream_sources_re = re.compile(r"<source src=\\\"(.*?)\\\" type=\\\"application\\\/vnd\.apple\.mpegurl\\\">")
-    _vod_sources_re = re.compile(r"<source src=\\\"(.*?)\\\" type=\\\"video\\\/mp4\\\">")
+
+    API_URL = 'https://vk.com/al_video.php'
+
+    _url_re = re.compile(r'''https?://(?:\w+\.)?vk\.com/video
+        (?:\?z=video)?(?P<video_id>-?[0-9]*_[0-9]*)
+        ''', re.VERBOSE)
+    _url_catalog_re = re.compile(r"https?://(\w+\.)?vk\.com/videos-?[0-9]*")
     _vod_quality_re = re.compile(r"\.([0-9]*?)\.mp4")
 
     @classmethod
@@ -27,15 +32,17 @@ class VK(Plugin):
 
     @classmethod
     def follow_vk_redirect(cls, url):
-        # If this is a 'videos' catalog URL with an video ID in the GET request, get that instead
+        # If this is a 'videos' catalog URL
+        # with an video ID in the GET request, get that instead
         parsed_url = urlparse(url)
-        if parsed_url.path.startswith('/videos-'):
+        if parsed_url.path.startswith('/videos'):
             query = dict((v[0], v[1]) for v in [q.split('=') for q in parsed_url.query.split('&')] if v[0] == 'z')
             try:
                 true_path = unquote(query['z']).split('/')[0]
                 return parsed_url.scheme + '://' + parsed_url.netloc + '/' + true_path
             except KeyError:
-                # No redirect found in query string, so return the catalog url and fail later
+                # No redirect found in query string,
+                # so return the catalog url and fail later
                 return url
         else:
             return url
@@ -45,55 +52,54 @@ class VK(Plugin):
         Find the streams for vk.com
         :return:
         """
-        # If this is a 'videos' catalog URL with an video ID in the GET request, get that instead
+        http.headers.update({'User-Agent': useragents.IPHONE_6})
+
+        # If this is a 'videos' catalog URL
+        # with an video ID in the GET request, get that instead
         url = self.follow_vk_redirect(self.url)
-        if url != self.url:
-            self.logger.debug('Grabbing data from real video page at {}', url)
-
-        headers = {}
-        res = http.get(url, headers=headers)
-        headers["Referer"] = url
-
-        # Try and find an HLS source (livestream)
-        stream_urls = self._livestream_sources_re.findall(res.text)
-        if len(stream_urls):
-            stream_url = stream_urls[0].replace('\/', '/')
-            self.logger.debug("Found live stream at {}", stream_url)
-            try:
-                # try to parse the stream as a variant playlist
-                variant = HLSStream.parse_variant_playlist(self.session, stream_url, headers=headers)
-                if variant:
-                    for q, s in variant.items():
-                        yield q, s
-                else:
-                    # and if that fails, try it as a plain HLS stream
-                    yield 'live', HLSStream(self.session, stream_url, headers=headers)
-            except IOError:
-                self.logger.warning("Could not open the stream, perhaps the channel is offline")
+
+        m = self._url_re.match(url)
+        if not m:
+            log.error('URL is not compatible: {0}'.format(url))
             return
 
-        # Try and find a set of MP4 sources (VOD)
-        vod_urls = self._vod_sources_re.findall(res.text)
-        if len(vod_urls):
-            vod_urls = [v.replace('\/', '/') for v in vod_urls]
-            # Try to get quality from URL
-            qualities = {}
-            for s in vod_urls:
-                q = self._vod_quality_re.findall(s)
-                if not len(q):
-                    break
-                qualities[s] = q[0]
+        video_id = m.group('video_id')
+        log.debug('video ID: {0}'.format(video_id))
 
-            try:
-                if len(qualities) == len(vod_urls):
-                    for s in vod_urls:
-                        yield qualities[s] + 'p', HTTPStream(self.session, s)
+        params = {
+            'act': 'show_inline',
+            'al': '1',
+            'video': video_id,
+        }
+        res = http.post(self.API_URL, params=params)
+
+        for _i in itertags(res.text, 'iframe'):
+            if _i.attributes.get('src'):
+                iframe_url = update_scheme(self.url, _i.attributes['src'])
+                log.debug('Found iframe: {0}'.format(iframe_url))
+                for s in self.session.streams(iframe_url).items():
+                    yield s
+
+        for _i in itertags(res.text, 'source'):
+            if _i.attributes.get('type') == 'application/vnd.apple.mpegurl':
+                video_url = _i.attributes['src']
+                # Remove invalid URL
+                if video_url.startswith('https://vk.com/'):
+                    continue
+                streams = HLSStream.parse_variant_playlist(self.session,
+                                                           video_url)
+                if not streams:
+                    yield 'live', HLSStream(self.session, video_url)
                 else:
-                    # Fall back to numerical ranking
-                    for q, s in enumerate(vod_urls):
-                        yield str(q), HTTPStream(self.session, s)
-            except IOError:
-                self.logger.warning("Could not open the stream, perhaps the channel is offline")
+                    for s in streams.items():
+                        yield s
+            elif _i.attributes.get('type') == 'video/mp4':
+                q = 'vod'
+                video_url = _i.attributes['src']
+                m = self._vod_quality_re.search(video_url)
+                if m:
+                    q = '{0}p'.format(m.group(1))
+                yield q, HTTPStream(self.session, video_url)
 
 
 __plugin__ = VK


Test patch:
diff --git a/tests/test_plugin_mjunoon.py b/tests/test_plugin_mjunoon.py
new file mode 100644
index 00000000..1aa08b1b
--- /dev/null
+++ b/tests/test_plugin_mjunoon.py
@@ -0,0 +1,36 @@
+import unittest
+import requests_mock
+
+from streamlink import Streamlink
+from tests.mock import patch, Mock, ANY, call
+from streamlink.plugins.mjunoon import Mjunoon
+
+
+class TestPluginMixer(unittest.TestCase):
+    def test_can_handle_url(self):
+        # should match
+        self.assertTrue(Mjunoon.can_handle_url('https://mjunoon.tv/news-live'))
+        self.assertTrue(Mjunoon.can_handle_url('http://mjunoon.tv/watch/some-long-vod-name23456'))
+        self.assertTrue(Mjunoon.can_handle_url('https://www.mjunoon.tv/other-live'))
+        self.assertTrue(Mjunoon.can_handle_url('https://www.mjunoon.tv/watch/something-else-2321'))
+
+    def test_can_handle_url_negative(self):
+        # shouldn't match
+        self.assertFalse(Mjunoon.can_handle_url('https://mjunoon.com'))
+
+    @patch('streamlink.plugins.mjunoon.HLSStream.parse_variant_playlist')
+    def test_get_streams(self, parse_variant_playlist):
+        session = Streamlink()
+        Mjunoon.bind(session, "test")
+        script_text = """
+        <script id="playerScript"  src="playerAssets/js/player.js?v=2.2&streamUrl=https://vod.mjunoon.tv:8181/live/41/41.m3u8&streamUrl=https://vod.mjunoon.tv:8181/live/17/17.m3u8"></script>
+        """
+        parse_variant_playlist.items.return_value = [("test", Mock())]
+        with requests_mock.Mocker() as rmock:
+            rmock.get("https://mjunoon.tv/news-live", text=script_text)
+            plugin = Mjunoon("https://mjunoon.tv/news-live")
+            _ = list(plugin.streams())
+            self.assertSequenceEqual(
+                [call(ANY, "https://vod.mjunoon.tv:8181/live/41/41.m3u8", params=dict(id=1), verify=False),
+                 call(ANY, "https://vod.mjunoon.tv:8181/live/17/17.m3u8", params=dict(id=2), verify=False)],
+                [parse_variant_playlist.mock_calls[0], parse_variant_playlist.mock_calls[3]])
diff --git a/tests/test_plugin_stream.py b/tests/test_plugin_stream.py
index 917b893d..091bf0e1 100644
--- a/tests/test_plugin_stream.py
+++ b/tests/test_plugin_stream.py
@@ -137,30 +137,44 @@ class TestPluginStream(unittest.TestCase):
             parse_params(""""conn=['B:1', 'S:authMe', 'O:1', 'NN:code:1.23', 'NS:flag:ok', 'O:0']""")
         )
 
+    def test_stream_weight_value(self):
+        self.assertEqual((720, "pixels"),
+                         stream_weight("720p"))
+
+        self.assertEqual((721, "pixels"),
+                         stream_weight("720p+"))
+
+        self.assertEqual((780, "pixels"),
+                         stream_weight("720p60"))
+
     def test_stream_weight(self):
-        self.assertEqual(
-            (720, "pixels"),
-            stream_weight("720p"))
-        self.assertEqual(
-            (721, "pixels"),
-            stream_weight("720p+"))
-        self.assertEqual(
-            (780, "pixels"),
-            stream_weight("720p60"))
-
-        self.assertTrue(
-            stream_weight("720p+") > stream_weight("720p"))
-        self.assertTrue(
-            stream_weight("720p") == stream_weight("720p"))
-        self.assertTrue(
-            stream_weight("720p_3000k") > stream_weight("720p_2500k"))
-        self.assertTrue(
-            stream_weight("720p60_3000k") > stream_weight("720p_3000k"))
-        self.assertTrue(
-            stream_weight("720p_3000k") < stream_weight("720p+_3000k"))
-
-        self.assertTrue(
-            stream_weight("3000k") > stream_weight("2500k"))
+        self.assertGreater(stream_weight("720p+"),
+                           stream_weight("720p"))
+
+        self.assertGreater(stream_weight("720p_3000k"),
+                           stream_weight("720p_2500k"))
+
+        self.assertGreater(stream_weight("720p60_3000k"),
+                           stream_weight("720p_3000k"))
+
+        self.assertGreater(stream_weight("3000k"),
+                           stream_weight("2500k"))
+
+        self.assertEqual(stream_weight("720p"),
+                         stream_weight("720p"))
+
+        self.assertLess(stream_weight("720p_3000k"),
+                        stream_weight("720p+_3000k"))
+
+    def test_stream_weight_and_audio(self):
+        self.assertGreater(stream_weight("720p+a256k"),
+                           stream_weight("720p+a128k"))
+
+        self.assertGreater(stream_weight("720p+a256k"),
+                           stream_weight("720p+a128k"))
+
+        self.assertGreater(stream_weight("720p+a128k"),
+                           stream_weight("360p+a256k"))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_plugin_vk.py b/tests/test_plugin_vk.py
index e0d3e06f..e1192639 100755
--- a/tests/test_plugin_vk.py
+++ b/tests/test_plugin_vk.py
@@ -14,6 +14,10 @@ class TestPluginVK(unittest.TestCase):
             "https://vk.com/videos-24136539?z=video-24136539_456241181%2Fpl_-24136539_-2"),
             "https://vk.com/video-24136539_456241181"
         )
+        self.assertEqual(VK.follow_vk_redirect(
+            "https://vk.com/videos132886594?z=video132886594_167211693"),
+            "https://vk.com/video132886594_167211693"
+        )
 
         # shouldn't redirect
         self.assertEqual(VK.follow_vk_redirect("http://vk.com/"), "http://vk.com/")
@@ -26,6 +30,11 @@ class TestPluginVK(unittest.TestCase):
         self.assertTrue(VK.can_handle_url("http://vk.com/video-24136539_456239830"))
         self.assertTrue(VK.can_handle_url("https://www.vk.com/video-34453259_456240574"))
         self.assertTrue(VK.can_handle_url("https://vk.com/videos-24136539?z=video-24136539_456241155%2Fpl_-24136539_-2"))
+        self.assertTrue(VK.can_handle_url("https://vk.com/video?z=video-15755094_456245149%2Fpl_cat_lives"))
+        self.assertTrue(VK.can_handle_url("https://vk.com/video?z=video132886594_167211693%2Fpl_cat_8"))
+        self.assertTrue(VK.can_handle_url("https://vk.com/video132886594_167211693"))
+        self.assertTrue(VK.can_handle_url("https://vk.com/videos132886594?z=video132886594_167211693"))
+        self.assertTrue(VK.can_handle_url("https://vk.com/video-73154028_456239128"))
 
         # shouldn't match
         self.assertFalse(VK.can_handle_url("https://vk.com/"))