Repository: streamlink/streamlink
Pull request #2351 (2019-03-09 14:54:20)

plugins.ntv: new Plugin for ntv.ru (#2346)
NTV is russian tv channel (#2346): 

https://en.wikipedia.org/wiki/NTV_(Russia)


Patch:
diff --git a/.travis.yml b/.travis.yml
index 45f67c54..0a83c747 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -66,7 +66,6 @@ deploy:
   - provider: releases
     api_key: "${RELEASES_API_KEY}"
     file: "${STREAMLINK_INSTALLER_DIST_DIR}/streamlink-${TRAVIS_TAG}.exe"
-    file_glob: true
     skip_cleanup: true
     on:
       tags: true
@@ -97,7 +96,7 @@ deploy:
       repo: streamlink/streamlink
   - provider: releases
     api_key: "${RELEASES_API_KEY}"
-    file: "${STREAMLINK_DIST_DIR}/streamlink-${TRAVIS_TAG}*"
+    file: "${STREAMLINK_DIST_DIR}/streamlink-${TRAVIS_TAG}.tar.gz{,.asc}"
     file_glob: true
     skip_cleanup: true
     on:
diff --git a/docs/plugin_matrix.rst b/docs/plugin_matrix.rst
index 6dd3bdde..e96d93f7 100644
--- a/docs/plugin_matrix.rst
+++ b/docs/plugin_matrix.rst
@@ -40,6 +40,7 @@ cdnbg                   - tv.bnt.bg          Yes   No    Streams may be geo-rest
                         - bgonair.bg
                         - bitelevision.com
                         - bloombergtv.bg
+                        - bstv.bg
                         - inlife.bg
                         - kanal3.bg
                         - mmtvmusic.com
@@ -81,7 +82,7 @@ eltrecetv               eltrecetv.com.ar     Yes   Yes   Streams may be geo-rest
 eurocom                 eurocom.bg           Yes   No
 euronews                euronews.com         Yes   No
 europaplus              europaplus.ru        Yes   --
-facebook                facebook.com         Yes   No    Only 360p HLS streams.
+facebook                facebook.com         Yes   Yes
 filmon                  filmon.com           Yes   Yes   Only SD quality streams.
 foxtr                   fox.com.tr           Yes   No
 funimationnow           - funimation.com     --    Yes   :ref:`Requires session cookies <cli-funimationnow>`
@@ -126,6 +127,7 @@ npo                     - npo.nl             Yes   Yes   Streams may be geo-rest
                         - zappelin.nl
 nrk                     - tv.nrk.no          Yes   Yes   Streams may be geo-restricted to Norway.
                         - radio.nrk.no
+ntv                     ntv.ru               Yes   No
 ok_live                 ok.ru                Yes   Yes
 olympicchannel          olympicchannel.com   Yes   Yes   Only non-premium content is available.
 onetv                   - 1tv.ru             Yes   Yes   Streams may be geo-restricted to Russia. VOD only for 1tv.ru
diff --git a/src/streamlink/compat.py b/src/streamlink/compat.py
index 0eedef5d..cb19ab77 100644
--- a/src/streamlink/compat.py
+++ b/src/streamlink/compat.py
@@ -34,12 +34,12 @@ elif is_py3:
 
 try:
     from urllib.parse import (
-        urlparse, urlunparse, urljoin, quote, unquote, parse_qsl, urlencode, urlsplit, urlunsplit
+        urlparse, urlunparse, urljoin, quote, unquote, unquote_plus, parse_qsl, urlencode, urlsplit, urlunsplit
     )
     import queue
 except ImportError:
     from urlparse import urlparse, urlunparse, urljoin, parse_qsl, urlsplit, urlunsplit
-    from urllib import quote, unquote, urlencode
+    from urllib import quote, unquote, unquote_plus, urlencode
     import Queue as queue
 
 try:
@@ -60,5 +60,5 @@ getargspec = getattr(inspect, "getfullargspec", inspect.getargspec)
 
 __all__ = ["is_py2", "is_py3", "is_py33", "is_win32", "str", "bytes",
            "urlparse", "urlunparse", "urljoin", "parse_qsl", "quote",
-           "unquote", "queue", "range", "urlencode", "devnull", "which",
+           "unquote", "unquote_plus", "queue", "range", "urlencode", "devnull", "which",
            "izip", "urlsplit", "urlunsplit", "getargspec", "html_unescape"]
diff --git a/src/streamlink/plugins/cdnbg.py b/src/streamlink/plugins/cdnbg.py
index dfb5591d..79b0172b 100644
--- a/src/streamlink/plugins/cdnbg.py
+++ b/src/streamlink/plugins/cdnbg.py
@@ -22,6 +22,7 @@ class CDNBG(Plugin):
             inlife\.bg|
             mmtvmusic\.com/live|
             mu-vi\.tv/LiveStreams/pages/Live\.aspx|
+            videochanel\.bstv\.bg|
             bloombergtv.bg/video
         )/?
     """, re.VERBOSE)
diff --git a/src/streamlink/plugins/facebook.py b/src/streamlink/plugins/facebook.py
index 537b4a5a..426649aa 100644
--- a/src/streamlink/plugins/facebook.py
+++ b/src/streamlink/plugins/facebook.py
@@ -1,53 +1,110 @@
+import logging
 import re
 
+from streamlink.compat import bytes, is_py3, unquote_plus, urlencode
 from streamlink.plugin import Plugin
 from streamlink.plugin.api import useragents
 from streamlink.stream import DASHStream, HTTPStream
 from streamlink.utils import parse_json
 
+log = logging.getLogger(__name__)
+
 
 class Facebook(Plugin):
-    _url_re = re.compile(r"https?://(?:www\.)?facebook\.com/[^/]+/(posts|videos)")
+    _url_re = re.compile(r'''(?x)https?://(?:www\.)?facebook(?:\.com|corewwwi.onion)
+        /[^/]+/(?:posts|videos)/(?P<video_id>[0-9]+)''')
     _src_re = re.compile(r'''(sd|hd)_src["']?\s*:\s*(?P<quote>["'])(?P<url>.+?)(?P=quote)''')
+    _dash_manifest_re = re.compile(r'''dash_manifest["']?\s*:\s*["'](?P<manifest>.+?)["'],''')
     _playlist_re = re.compile(r'''video:\[({url:".+?}\])''')
     _plurl_re = re.compile(r'''url:"(.*?)"''')
+    _pc_re = re.compile(r'''pkg_cohort["']\s*:\s*["'](.+?)["']''')
+    _rev_re = re.compile(r'''client_revision["']\s*:\s*(\d+),''')
+    _dtsg_re = re.compile(r'''DTSGInitialData["'],\s*\[\],\s*{\s*["']token["']\s*:\s*["'](.+?)["']''')
+    _DEFAULT_PC = "PHASED:DEFAULT"
+    _DEFAULT_REV = 4681796
+    _TAHOE_URL = "https://www.facebook.com/video/tahoe/async/{0}/?chain=true&isvideo=true&payloadtype=primary"
 
     @classmethod
     def can_handle_url(cls, url):
-        return cls._url_re.match(url)
-
-    def _get_streams(self):
-        res = self.session.http.get(self.url, headers={"User-Agent": useragents.CHROME})
-
-        streams = {}
-        vod_urls = set([])
+        return cls._url_re.match(url) is not None
 
+    def _parse_streams(self, res):
         for match in self._src_re.finditer(res.text):
             stream_url = match.group("url")
             if "\\/" in stream_url:
                 # if the URL is json encoded, decode it
                 stream_url = parse_json("\"{}\"".format(stream_url))
             if ".mpd" in stream_url:
-                streams.update(DASHStream.parse_manifest(self.session, stream_url))
+                for s in DASHStream.parse_manifest(self.session, stream_url).items():
+                    yield s
             elif ".mp4" in stream_url:
-                streams[match.group(1)] = HTTPStream(self.session, stream_url)
-                vod_urls.add(stream_url)
+                yield match.group(1), HTTPStream(self.session, stream_url)
             else:
-                self.logger.debug("Non-dash/mp4 stream: {0}".format(stream_url))
+                log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
 
-        if streams:
-            return streams
+        match = self._dash_manifest_re.search(res.text)
+        if match:
+            # facebook replaces "<" characters with the substring "\\x3C"
+            manifest = match.group("manifest").replace("\\/", "/")
+            if is_py3:
+                manifest = bytes(unquote_plus(manifest), "utf-8").decode("unicode_escape")
+            else:
+                manifest = unquote_plus(manifest).decode("string_escape")
+            # Ignore unsupported manifests until DASH SegmentBase support is implemented
+            if "SegmentBase" in manifest:
+                log.error("Skipped DASH manifest with SegmentBase streams")
+            else:
+                for s in DASHStream.parse_manifest(self.session, manifest).items():
+                    yield s
+
+    def _get_streams(self):
+        self.session.http.headers.update({'User-Agent': useragents.CHROME})
+        done = False
+        res = self.session.http.get(self.url)
+        for s in self._parse_streams(res):
+            done = True
+            yield s
+        if done:
+            return
 
         # fallback on to playlist
-        self.logger.debug("Falling back to playlist regex")
+        log.debug("Falling back to playlist regex")
         match = self._playlist_re.search(res.text)
         playlist = match and match.group(1)
         if playlist:
-            for url in dict(url.group(1) for url in self._plurl_re.finditer(playlist)):
-                if url not in vod_urls:
-                    streams["sd"] = HTTPStream(self.session, url)
+            match = self._plurl_re.search(playlist)
+            if match:
+                url = match.group(1)
+                yield "sd", HTTPStream(self.session, url)
+                return
+
+        # fallback to tahoe player url
+        log.debug("Falling back to tahoe player")
+        video_id = self._url_re.match(self.url).group("video_id")
+        url = self._TAHOE_URL.format(video_id)
+        data = {
+            "__a": 1,
+            "__pc": self._DEFAULT_PC,
+            "__rev": self._DEFAULT_REV,
+            "fb_dtsg": "",
+        }
+        match = self._pc_re.search(res.text)
+        if match:
+            data["__pc"] = match.group(1)
+        match = self._rev_re.search(res.text)
+        if match:
+            data["__rev"] = match.group(1)
+        match = self._dtsg_re.search(res.text)
+        if match:
+            data["fb_dtsg"] = match.group(1)
+        res = self.session.http.post(
+            url,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+            data=urlencode(data).encode("ascii")
+        )
 
-        return streams
+        for s in self._parse_streams(res):
+            yield s
 
 
 __plugin__ = Facebook
diff --git a/src/streamlink/plugins/ntv.py b/src/streamlink/plugins/ntv.py
new file mode 100644
index 00000000..4e276d98
--- /dev/null
+++ b/src/streamlink/plugins/ntv.py
@@ -0,0 +1,26 @@
+import re
+from streamlink.plugin import Plugin
+from streamlink.stream import HLSStream
+
+
+class NTV(Plugin):
+    url_re = re.compile(r'https?://www.ntv.ru/air/.*')
+
+    @classmethod
+    def can_handle_url(cls, url):
+        return cls.url_re.match(url) is not None
+
+    def _get_streams(self):
+        body = self.session.http.get(self.url).text
+        mrl = None
+        match = re.search(r'var camHlsURL = \'(.*)\'', body)
+        if match:
+            mrl = 'http:' + match.group(1)
+        else:
+            match = re.search(r'var hlsURL = \'(.*)\'', body)
+            if match:
+                mrl = match.group(1)
+        if mrl:
+            return HLSStream.parse_variant_playlist(self.session, mrl)
+
+__plugin__ = NTV
diff --git a/src/streamlink/plugins/pluzz.py b/src/streamlink/plugins/pluzz.py
index 259f6029..63edf383 100644
--- a/src/streamlink/plugins/pluzz.py
+++ b/src/streamlink/plugins/pluzz.py
@@ -23,7 +23,7 @@ class Pluzz(Plugin):
             www\.(ludo|zouzous)\.fr/heros/[\w-]+ |
             (.+\.)?francetvinfo\.fr)
     ''', re.VERBOSE)
-    _pluzz_video_id_re = re.compile(r'''videoId:\s*["'](?P<video_id>[^"']+)["']''')
+    _pluzz_video_id_re = re.compile(r'''(?P<q>["']*)videoId(?P=q):\s*["'](?P<video_id>[^"']+)["']''')
     _jeunesse_video_id_re = re.compile(r'playlist: \[{.*?,"identity":"(?P<video_id>.+?)@(?P<catalogue>Ludo|Zouzous)"')
     _sport_video_id_re = re.compile(r'data-video="(?P<video_id>.+?)"')
     _embed_video_id_re = re.compile(r'href="http://videos\.francetv\.fr/video/(?P<video_id>.+?)(?:@.+?)?"')
diff --git a/src/streamlink/plugins/tf1.py b/src/streamlink/plugins/tf1.py
index 1e4abb57..b5900bb7 100644
--- a/src/streamlink/plugins/tf1.py
+++ b/src/streamlink/plugins/tf1.py
@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import unicode_literals
 
 import logging
 import re
@@ -18,7 +18,7 @@ class TF1(Plugin):
     token = "07e45841-a17a-47cf-af64-a42311bdcc3d"
 
     def api_call(self, channel, useragent=useragents.CHROME):
-        url = urljoin(self.api_url_base, "L_"+channel.upper())
+        url = urljoin(self.api_url_base, "L_" + channel.upper())
         req = self.session.http.get(url,
                                     params=dict(token=self.token),
                                     headers={"User-Agent": useragent})
@@ -29,7 +29,7 @@ class TF1(Plugin):
             data = self.api_call(channel, useragent)
 
             if data.get("error"):
-                log.error("Failed to get stream for {0}: {error} ({code})".format(channel, **data))
+                log.error("Failed to get {format} stream for {0}: {error} ({code})".format(channel, **data))
             else:
                 log.debug("Got {format} stream {url}".format(**data))
                 yield data['format'], data['url']
@@ -42,7 +42,7 @@ class TF1(Plugin):
         m = self.url_re.match(self.url)
         if m:
             channel = m.group(1) or m.group(2)
-            self.logger.debug("Found channel {0}", channel)
+            log.debug("Found channel {0}".format(channel))
             for sformat, url in self.get_stream_urls(channel):
                 try:
                     if sformat == "dash":
@@ -55,4 +55,5 @@ class TF1(Plugin):
                     log.error("Could not open {0} stream".format(sformat))
                     log.debug("Failed with error: {0}".format(e))
 
+
 __plugin__ = TF1
diff --git a/src/streamlink/plugins/twitch.py b/src/streamlink/plugins/twitch.py
index 410a22b6..17ce4e0b 100644
--- a/src/streamlink/plugins/twitch.py
+++ b/src/streamlink/plugins/twitch.py
@@ -222,7 +222,7 @@ class TwitchAPI(object):
     # Private API calls
 
     def access_token(self, endpoint, asset, **params):
-        return self.call("/api/{0}/{1}/access_token".format(endpoint, asset), **params)
+        return self.call("/api/{0}/{1}/access_token".format(endpoint, asset), **dict(platform="_", **params))
 
     def token(self, **params):
         return self.call("/api/viewer/token", **params)
diff --git a/src/streamlink_cli/main.py b/src/streamlink_cli/main.py
index 9acb1094..b1a83a40 100644
--- a/src/streamlink_cli/main.py
+++ b/src/streamlink_cli/main.py
@@ -630,8 +630,11 @@ def authenticate_twitch_oauth():
     client_id = TWITCH_CLIENT_ID
     redirect_uri = "https://streamlink.github.io/twitch_oauth.html"
     url = ("https://api.twitch.tv/kraken/oauth2/authorize"
-           "?response_type=token&client_id={0}&redirect_uri="
-           "{1}&scope=user_read+user_subscriptions").format(client_id, redirect_uri)
+           "?response_type=token"
+           "&client_id={0}"
+           "&redirect_uri={1}"
+           "&scope=user_read+user_subscriptions"
+           "&force_verify=true").format(client_id, redirect_uri)
 
     console.msg("Attempting to open a browser to let you authenticate "
                 "Streamlink with Twitch")


Test patch:
diff --git a/tests/plugins/test_cdnbg.py b/tests/plugins/test_cdnbg.py
index 7b6026a6..b98b16ee 100644
--- a/tests/plugins/test_cdnbg.py
+++ b/tests/plugins/test_cdnbg.py
@@ -22,6 +22,7 @@ class TestPluginCDNBG(unittest.TestCase):
         self.assertTrue(CDNBG.can_handle_url("http://inlife.bg/"))
         self.assertTrue(CDNBG.can_handle_url("https://mmtvmusic.com/live/"))
         self.assertTrue(CDNBG.can_handle_url("http://mu-vi.tv/LiveStreams/pages/Live.aspx"))
+        self.assertTrue(CDNBG.can_handle_url("http://videochanel.bstv.bg/"))
         self.assertTrue(CDNBG.can_handle_url("https://www.bloombergtv.bg/video"))
 
         # shouldn't match
diff --git a/tests/plugins/test_facebook.py b/tests/plugins/test_facebook.py
index 76e72dff..33e2bcf6 100644
--- a/tests/plugins/test_facebook.py
+++ b/tests/plugins/test_facebook.py
@@ -10,6 +10,7 @@ class TestPluginFacebook(unittest.TestCase):
         self.assertTrue(Facebook.can_handle_url("https://www.facebook.com/nytfood/videos/1485091228202006/"))
         self.assertTrue(Facebook.can_handle_url("https://www.facebook.com/SporTurkTR/videos/798553173631138/"))
         self.assertTrue(Facebook.can_handle_url("https://www.facebook.com/119555411802156/posts/500665313691162/"))
+        self.assertTrue(Facebook.can_handle_url("https://www.facebookcorewwwi.onion/SporTurkTR/videos/798553173631138/"))
 
         # shouldn't match
         self.assertFalse(Facebook.can_handle_url("https://www.facebook.com"))
diff --git a/tests/plugins/test_ntv.py b/tests/plugins/test_ntv.py
new file mode 100644
index 00000000..752fca1e
--- /dev/null
+++ b/tests/plugins/test_ntv.py
@@ -0,0 +1,21 @@
+import unittest
+
+from streamlink.plugins.ntv import NTV
+
+
+class TestPluginNTV(unittest.TestCase):
+    def test_can_handle_url(self):
+        should_match = [
+            'https://www.ntv.ru/air/',
+            'http://www.ntv.ru/air/'
+        ]
+        for url in should_match:
+            self.assertTrue(NTV.can_handle_url(url))
+
+    def test_can_handle_url_negative(self):
+        should_not_match = [
+            'https://www.ntv.ru/',
+            'http://www.ntv.ru/'
+        ]
+        for url in should_not_match:
+            self.assertFalse(NTV.can_handle_url(url))