Repository: streamlink/streamlink
Pull request #2233 (2019-01-04 22:18:08)

fix zdf mediathek plugin
* json schema changed slightly
* new playId constant was introduced
* added required "Accept"-header

Closes streamlink/streamlink#2231


Patch:
diff --git a/.travis.yml b/.travis.yml
index 188d6eaa..a664f15f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,7 @@ matrix:
 
 before_install:
   - pip install --disable-pip-version-check --upgrade pip setuptools
-  - pip install -r dev-requirements.txt
+  - pip install --upgrade -r dev-requirements.txt
   - pip install pycountry
   - if [[ $BUILD_DOCS == 'yes' ]]; then
       pip install -r docs-requirements.txt;
diff --git a/docs/plugin_matrix.rst b/docs/plugin_matrix.rst
index 6bd754a1..c8318fcb 100644
--- a/docs/plugin_matrix.rst
+++ b/docs/plugin_matrix.rst
@@ -211,9 +211,12 @@ trtspor                 trtspor.com          Yes   No    Some streams are geo-re
 turkuvaz                - atv.com.tr         Yes   No
                         - a2tv.com.tr
                         - ahaber.com.tr
+                        - anews.com.tr
                         - aspor.com.tr
-                        - minikago.com.tr
+                        - atvavrupa.tv
                         - minikacocuk.com.tr
+                        - minikago.com.tr
+                        - sabah.com.tr
 tv1channel              tv1channel.org       Yes   Yes
 tv3cat                  tv3.cat              Yes   Yes   Streams may be geo-restricted to Spain.
 tv4play                 - tv4play.se         Yes   Yes   Streams may be geo-restricted to Sweden.
diff --git a/src/streamlink/plugins/btsports.py b/src/streamlink/plugins/btsports.py
index 30cd594f..29308c53 100644
--- a/src/streamlink/plugins/btsports.py
+++ b/src/streamlink/plugins/btsports.py
@@ -1,12 +1,15 @@
-import re
 import time
+
+import logging
+import re
 from uuid import uuid4
 
 from streamlink.compat import quote
 from streamlink.plugin import Plugin, PluginArguments, PluginArgument
 from streamlink.plugin.api import useragents
 from streamlink.stream import HLSStream
-from streamlink.utils import url_equal
+
+log = logging.getLogger(__name__)
 
 
 class BTSports(Plugin):
@@ -47,25 +50,23 @@ class BTSports(Plugin):
         return cls.url_re.match(url) is not None
 
     def login(self, username, password):
-        self.logger.debug("Logging in as {0}".format(username))
+        log.debug("Logging in as {0}".format(username))
 
-        redirect_to = "https://home.bt.com/ss/Satellite/secure/loginforward?redirectURL={0}".format(quote(self.url))
+        redirect_to = "https://home.bt.com/ss/Satellite/secure/loginforward?view=btsport&redirectURL={0}".format(quote(self.url))
         data = {
             "cookieExpp": "30",
             "Switch": "yes",
             "SMPostLoginUrl": "/appsyouraccount/secure/postlogin",
-            "loginforward": "https://home.bt.com/ss/Satellite/secure/loginforward",
+            "loginforward": "https://home.bt.com/ss/Satellite/secure/loginforward?view=btsport",
             "smauthreason": "0",
             "TARGET": redirect_to,
             "USER": username,
             "PASSWORD": password}
-
         res = self.session.http.post(self.login_url, data=data)
 
-        self.logger.debug("Redirected to: {0}".format(res.url))
-
+        log.debug("Redirected to: {0}".format(res.url))
 
-        if url_equal(res.url, self.url, ignore_scheme=True):
+        if "loginerror" not in res.text:
             self.logger.debug("Login successful, getting SAML token")
             res = self.session.http.get("https://samlfed.bt.com/sportgetfedwebhls?bt.cid={0}".format(self.acid()))
             d = self.saml_re.search(res.text)
@@ -81,7 +82,8 @@ class BTSports(Plugin):
                     self.logger.error("Failed to login: {0} - {1}".format(fed_json['errorDescription'],
                                                                           fed_json['message']))
                 return success
-        return False
+        else:
+            return False
 
     def device_id(self):
         device_id = self.cache.get("device_id") or str(uuid4())
@@ -108,19 +110,23 @@ class BTSports(Plugin):
     def _get_streams(self):
         if self.options.get("email") and self.options.get("password"):
             if self.login(self.options.get("email"), self.options.get("password")):
-                self.logger.debug("Logged in and authenticated with BT Sports.")
+                log.debug("Logged in and authenticated with BT Sports.")
 
                 res = self.session.http.get(self.url)
                 m = self.content_re.findall(res.text)
                 if m:
                     info = dict(m)
                     data = self._get_cdn(info.get("ID"), info.get("TYPE"))
+                    log.debug("CDN respsonse: {0}".format(data))
                     if data['resultCode'] == 'OK':
                         return HLSStream.parse_variant_playlist(self.session, data['resultObj']['src'])
                     else:
-                        self.logger.error("Failed to get stream with error: {0} - {1}".format(data['errorDescription'],
+                        log.error("Failed to get stream with error: {0} - {1}".format(data['errorDescription'],
                                                                                               data['message']))
+            else:
+                log.error("Login failed.")
         else:
-            self.logger.error("A username and password is required to use BT Sports")
+            log.error("A username and password is required to use BT Sports")
+
 
 __plugin__ = BTSports
diff --git a/src/streamlink/plugins/trt.py b/src/streamlink/plugins/trt.py
index bca9b992..6b274a04 100644
--- a/src/streamlink/plugins/trt.py
+++ b/src/streamlink/plugins/trt.py
@@ -1,19 +1,20 @@
-from __future__ import print_function
+import logging
 import re
+
 from base64 import b64decode
 
+from streamlink.compat import parse_qsl, urlparse
 from streamlink.plugin import Plugin
-from streamlink.plugin.api import validate
-from streamlink.compat import urlparse, parse_qsl
-from streamlink.stream import HDSStream
-from streamlink.stream import HLSStream
+from streamlink.stream import HDSStream, HLSStream
+
+log = logging.getLogger(__name__)
 
 
 class TRT(Plugin):
     """
     Support for the live TV streams on http://www.trt.net.tr/, some streams may be geo-locked
     """
-    url_re = re.compile(r"http://www.trt.net.tr/anasayfa/canli.aspx.*", re.I)
+    url_re = re.compile(r"https?://www\.trt\.net\.tr/anasayfa/canli\.aspx.*", re.I)
     stream_data_re = re.compile(r'<script>eval\(dcm1\("(.*?)"\)\);')
     f4mm_re = re.compile(r'''(?P<q>["'])(?P<url>http[^"']+?.f4m)(?P=q);''')
     m3u8_re = re.compile(r'''(?P<q>["'])(?P<url>http[^"']+?.m3u8)(?P=q);''')
@@ -27,7 +28,7 @@ class TRT(Plugin):
     def _get_streams(self):
         args = dict(parse_qsl(urlparse(self.url).query))
         if "k" in args:
-            self.logger.debug("Loading channel: {k}", **args)
+            log.debug("Loading channel: {0}".format(args["k"]))
             res = self.session.http.get(self.url)
             stream_data_m = self.stream_data_re.search(res.text)
             if stream_data_m:
diff --git a/src/streamlink/plugins/turkuvaz.py b/src/streamlink/plugins/turkuvaz.py
index 59c24fed..947f5c79 100644
--- a/src/streamlink/plugins/turkuvaz.py
+++ b/src/streamlink/plugins/turkuvaz.py
@@ -1,5 +1,5 @@
-import random
 import re
+
 from streamlink.plugin import Plugin
 from streamlink.plugin.api import useragents
 from streamlink.plugin.api import validate
@@ -11,13 +11,18 @@ class Turkuvaz(Plugin):
     Plugin to support ATV/A2TV Live streams from www.atv.com.tr and www.a2tv.com.tr
     """
 
-    _url_re = re.compile(r"""https?://(?:www.)?
-                             (?:(atvavrupa).tv|
-                                (atv|a2tv|ahaber|aspor|minikago|minikacocuk|anews).com.tr)
-                                /webtv/(live-broadcast|canli-yayin)""",
-                         re.VERBOSE)
-    _hls_url = "http://trkvz-live.ercdn.net/{channel}/{channel}.m3u8"
-    _token_url = "http://videotoken.tmgrup.com.tr/webtv/secure"
+    _url_re = re.compile(r"""(?x)https?://(?:www\.)?
+    (?:
+        (?:
+            (atvavrupa)\.tv
+            |
+            (atv|a2tv|ahaber|aspor|minikago|minikacocuk|anews)\.com\.tr
+        )/webtv/(?:live-broadcast|canli-yayin)
+    |
+        sabah\.com\.tr/(apara)/canli-yayin
+    )""")
+    _hls_url = "https://trkvz-live.ercdn.net/{channel}/{channel}.m3u8"
+    _token_url = "https://securevideotoken.tmgrup.com.tr/webtv/secure"
     _token_schema = validate.Schema(validate.all(
         {
             "Success": True,
@@ -32,19 +37,20 @@ class Turkuvaz(Plugin):
 
     def _get_streams(self):
         url_m = self._url_re.match(self.url)
-        domain = url_m.group(1) or url_m.group(2)
+        domain = url_m.group(1) or url_m.group(2) or url_m.group(3)
         # remap the domain to channel
         channel = {"atv": "atvhd",
                    "ahaber": "ahaberhd",
+                   "apara": "aparahd",
                    "aspor": "asporhd",
                    "anews": "anewshd",
                    "minikacocuk": "minikagococuk"}.get(domain, domain)
         hls_url = self._hls_url.format(channel=channel)
         # get the secure HLS URL
         res = self.session.http.get(self._token_url,
-                       params="url={0}".format(hls_url),
-                       headers={"Referer": self.url,
-                                "User-Agent": useragents.CHROME})
+                                    params="url={0}".format(hls_url),
+                                    headers={"Referer": self.url,
+                                             "User-Agent": useragents.CHROME})
 
         secure_hls_url = self.session.http.json(res, schema=self._token_schema)
 
diff --git a/src/streamlink/plugins/tvrby.py b/src/streamlink/plugins/tvrby.py
index a6bbc463..ace887f4 100644
--- a/src/streamlink/plugins/tvrby.py
+++ b/src/streamlink/plugins/tvrby.py
@@ -1,21 +1,23 @@
-from __future__ import print_function
 import re
 
 from streamlink.plugin import Plugin
 from streamlink.plugin.api import validate
 from streamlink.stream import HLSStream
-from streamlink.stream import RTMPStream
 
 
 class TVRBy(Plugin):
     url_re = re.compile(r"""https?://(?:www\.)?tvr.by/televidenie/belarus""")
-    file_re = re.compile(r"""(?P<q1>["']?)file(?P=q1)\s*:\s*(?P<q2>["'])(?P<url>(?:http.+?m3u8.*?|rtmp://.*?))(?P=q2)""")
+    file_re = re.compile(r"""(?P<url>https://stream\.hoster\.by[^"',]+\.m3u8[^"',]*)""")
+    player_re = re.compile(r"""["'](?P<url>[^"']+tvr\.by/plugines/online-tv-main\.php[^"']+)["']""")
 
     stream_schema = validate.Schema(
         validate.all(
             validate.transform(file_re.finditer),
             validate.transform(list),
-            [validate.get("url")]
+            [validate.get("url")],
+            # remove duplicates
+            validate.transform(set),
+            validate.transform(list),
         ),
     )
 
@@ -29,26 +31,21 @@ class TVRBy(Plugin):
     def can_handle_url(cls, url):
         return cls.url_re.match(url) is not None
 
-    @Plugin.broken()
     def _get_streams(self):
         res = self.session.http.get(self.url)
+        m = self.player_re.search(res.text)
+        if not m:
+            return
+
+        player_url = m.group("url")
+        res = self.session.http.get(player_url)
         stream_urls = self.stream_schema.validate(res.text)
         self.logger.debug("Found {0} stream URL{1}", len(stream_urls),
                           "" if len(stream_urls) == 1 else "s")
 
         for stream_url in stream_urls:
-            if "m3u8" in stream_url:
-                for _, s in HLSStream.parse_variant_playlist(self.session, stream_url).items():
-                    yield "live", s
-            if stream_url.startswith("rtmp://"):
-                a = stream_url.split("///")
-                s = RTMPStream(self.session, {
-                    "rtmp": a[0],
-                    "playpath": "live",
-                    "swfVfy": "http://www.tvr.by/plugines/uppod/uppod.swf",
-                    "pageUrl": self.url
-                })
-                yield "live", s
+            for s in HLSStream.parse_variant_playlist(self.session, stream_url).items():
+                yield s
 
 
 __plugin__ = TVRBy
diff --git a/src/streamlink/plugins/zdf_mediathek.py b/src/streamlink/plugins/zdf_mediathek.py
index 4db51ddb..8aaa959b 100644
--- a/src/streamlink/plugins/zdf_mediathek.py
+++ b/src/streamlink/plugins/zdf_mediathek.py
@@ -4,6 +4,7 @@ from streamlink.plugin import Plugin
 from streamlink.plugin.api import validate
 from streamlink.stream import HDSStream, HLSStream
 from streamlink.utils import parse_json
+from streamlink.utils.url import url_concat
 
 API_URL = "https://api.zdf.de"
 
@@ -48,7 +49,7 @@ _documents_schema = validate.Schema(
     {
         "mainVideoContent": {
             "http://zdf.de/rels/target": {
-                "http://zdf.de/rels/streams/ptmd": validate.text
+                "http://zdf.de/rels/streams/ptmd-template": validate.text
             },
         },
     }
@@ -128,6 +129,7 @@ class zdf_mediathek(Plugin):
             return
 
         headers = {
+            "Accept": "application/vnd.de.zdf.v1.0+json",
             "Api-Auth": "Bearer {0}".format(zdf_json['apiToken']),
             "Referer": self.url
         }
@@ -135,8 +137,8 @@ class zdf_mediathek(Plugin):
         res = self.session.http.get(zdf_json['content'], headers=headers)
         document = self.session.http.json(res, schema=_documents_schema)
 
-        stream_request_url = document["mainVideoContent"]["http://zdf.de/rels/target"]["http://zdf.de/rels/streams/ptmd"]
-        stream_request_url = API_URL + stream_request_url
+        stream_request_url = document["mainVideoContent"]["http://zdf.de/rels/target"]["http://zdf.de/rels/streams/ptmd-template"]
+        stream_request_url = url_concat(API_URL, stream_request_url.format(playerId="ngplayer_2_3").replace(" ", ""))
 
         res = self.session.http.get(stream_request_url, headers=headers)
         res = self.session.http.json(res, schema=_schema)
diff --git a/src/streamlink_cli/main.py b/src/streamlink_cli/main.py
index efbe790c..992c2119 100644
--- a/src/streamlink_cli/main.py
+++ b/src/streamlink_cli/main.py
@@ -338,7 +338,7 @@ def read_stream(stream, output, prebuffer, chunk_size=8192):
     is_http = isinstance(output, HTTPServer)
     is_fifo = is_player and output.namedpipe
     show_progress = isinstance(output, FileOutput) and output.fd is not stdout and sys.stdout.isatty()
-    show_record_progress = isinstance(output.record, FileOutput) and output.record.fd is not stdout and sys.stdout.isatty()
+    show_record_progress = hasattr(output, "record") and isinstance(output.record, FileOutput) and output.record.fd is not stdout and sys.stdout.isatty()
 
     stream_iterator = chain(
         [prebuffer],


Test patch:
diff --git a/tests/plugins/test_trt.py b/tests/plugins/test_trt.py
index c79d2923..ab16c975 100644
--- a/tests/plugins/test_trt.py
+++ b/tests/plugins/test_trt.py
@@ -7,6 +7,7 @@ class TestPluginTRT(unittest.TestCase):
     def test_can_handle_url(self):
         should_match = [
             'http://www.trt.net.tr/anasayfa/canli.aspx?y=tv&k=trt1',
+            'https://www.trt.net.tr/anasayfa/canli.aspx?y=tv&k=trtworld',
         ]
         for url in should_match:
             self.assertTrue(TRT.can_handle_url(url))
@@ -14,6 +15,7 @@ class TestPluginTRT(unittest.TestCase):
     def test_can_handle_url_negative(self):
         should_not_match = [
             'https://example.com/index.html',
+            'https://trt.net.tr/anasayfa/canli.aspx?y=tv&k=trtworld',
         ]
         for url in should_not_match:
             self.assertFalse(TRT.can_handle_url(url))
diff --git a/tests/plugins/test_turkuvaz.py b/tests/plugins/test_turkuvaz.py
index cc4ae0fb..8098cb04 100644
--- a/tests/plugins/test_turkuvaz.py
+++ b/tests/plugins/test_turkuvaz.py
@@ -10,6 +10,11 @@ class TestPluginTurkuvaz(unittest.TestCase):
             'http://www.a2tv.com.tr/webtv/canli-yayin',
             'https://www.ahaber.com.tr/webtv/canli-yayin',
             'https://www.aspor.com.tr/webtv/canli-yayin',
+            'http://www.anews.com.tr/webtv/live-broadcast',
+            'http://www.atvavrupa.tv/webtv/canli-yayin',
+            'http://www.minikacocuk.com.tr/webtv/canli-yayin',
+            'http://www.minikago.com.tr/webtv/canli-yayin',
+            'https://www.sabah.com.tr/apara/canli-yayin',
         ]
         for url in should_match:
             self.assertTrue(Turkuvaz.can_handle_url(url))