Repository: streamlink/streamlink

Issue:
USTVnow plugin stopped working
## Plugin Issue

- [x] This is a plugin issue and I have read the contribution guidelines.


### Description

USTVnow plugin stopped working; it reports "invalid station code" / empty list of Available Stations


### Log output

-- start without station code --
[cli][debug] OS:         Linux-4.15.0-51-generic-x86_64-with-Ubuntu-18.04-bionic
[cli][debug] Python:     3.6.8
[cli][debug] Streamlink: 1.1.1+28.gd655cbd
[cli][debug] Requests(2.22.0), Socks(1.7.0), Websocket(0.56.0)
[cli][info] Found matching plugin ustvnow for URL http://watch.ustvnow.com/watch/
[cli][debug] Plugin specific arguments:
[cli][debug]  --ustvnow-username=******** (username)
[cli][debug]  --ustvnow-password=******** (password)
[plugin.ustvnow][debug] CSRF: None
[plugin.ustvnow][error] Station code not provided, use --ustvnow-station-code.
[plugin.ustvnow][info] Available stations are: 
 
error: No playable streams found on this URL: http://watch.ustvnow.com/watch/
-- stop --

-- start with station code --
[cli][debug] OS:         Linux-4.15.0-51-generic-x86_64-with-Ubuntu-18.04-bionic
[cli][debug] Python:     3.6.8
[cli][debug] Streamlink: 1.1.1+28.gd655cbd
[cli][debug] Requests(2.22.0), Socks(1.7.0), Websocket(0.56.0)
[cli][info] Found matching plugin ustvnow for URL http://watch.ustvnow.com/watch/whtm
[cli][debug] Plugin specific arguments:
[cli][debug]  --ustvnow-username=******** (username)
[cli][debug]  --ustvnow-password=******** (password)
[plugin.ustvnow][debug] CSRF: None
[plugin.ustvnow][error] Invalid station-code: whtm
error: No playable streams found on this URL: http://watch.ustvnow.com/watch/whtm
-- stop --

## Additional Information
They seem to have changed the urls, on watch.* i'm getting a not-available-in-your-region-error, but i.e. https://www.ustvnow.com/live/cwwpix/-53 seems to work



Discussion:
They completely changed the website layout - look much better now. But will require a significant rewrite to make it work again. 

Resolving patch:
diff --git a/src/streamlink/plugins/ustvnow.py b/src/streamlink/plugins/ustvnow.py
--- a/src/streamlink/plugins/ustvnow.py
+++ b/src/streamlink/plugins/ustvnow.py
@@ -1,24 +1,33 @@
 from __future__ import unicode_literals
 
+import argparse
+import base64
+import json
 import logging
 import re
-from collections import OrderedDict
+from uuid import uuid4
 
+from Crypto.Cipher import AES
+from Crypto.Hash import SHA256
+from Crypto.Util.Padding import pad, unpad
+
+from streamlink import PluginError
+from streamlink.compat import urljoin, urlparse
 from streamlink.plugin import Plugin, PluginArguments, PluginArgument
-from streamlink.plugin.api import useragents
-from streamlink.plugin.api.utils import itertags
 from streamlink.stream import HLSStream
 
 log = logging.getLogger(__name__)
 
 
 class USTVNow(Plugin):
-    _url_re = re.compile(r"https?://(?:watch\.)?ustvnow\.com(?:/(?:watch|guide)/(?P<scode>\w+))?")
-    _token_re = re.compile(r'''var\s+token\s*=\s*"(.*?)";''')
-    _login_url = "https://watch.ustvnow.com/account/login"
-    _signin_url = "https://watch.ustvnow.com/account/signin"
-    _guide_url = "http://m.ustvnow.com/gtv/1/live/channelguidehtml"
-    _stream_url = "http://m.ustvnow.com/stream/1/live/view"
+    _url_re = re.compile(r"https?://(?:www\.)?ustvnow\.com/live/(?P<scode>\w+)/-(?P<id>\d+)")
+    _main_js_re = re.compile(r"""src=['"](main\..*\.js)['"]""")
+    _enc_key_re = re.compile(r'(?P<key>AES_(?:Key|IV))\s*:\s*"(?P<value>[^"]+)"')
+
+    TENANT_CODE = "ustvnow"
+    _api_url = "https://teleupapi.revlet.net/service/api/v1/"
+    _token_url = _api_url + "get/token"
+    _signin_url = "https://www.ustvnow.com/signin"
 
     arguments = PluginArguments(
         PluginArgument(
@@ -38,65 +47,135 @@ class USTVNow(Plugin):
         PluginArgument(
             "station-code",
             metavar="CODE",
-            help="USTV Now station code"
+            help=argparse.SUPPRESS
         ),
     )
 
+    def __init__(self, url):
+        super(USTVNow, self).__init__(url)
+        self._encryption_config = {}
+        self._token = None
+
     @classmethod
     def can_handle_url(cls, url):
         return cls._url_re.match(url) is not None
 
-    def login(self, username, password):
-        r = self.session.http.get(self._signin_url)
-        csrf = None
+    @classmethod
+    def encrypt_data(cls, data, key, iv):
+        rkey = "".join(reversed(key)).encode('utf8')
+        riv = "".join(reversed(iv)).encode('utf8')
+
+        fkey = SHA256.new(rkey).hexdigest()[:32].encode("utf8")
+
+        cipher = AES.new(fkey, AES.MODE_CBC, riv)
+        encrypted = cipher.encrypt(pad(data, 16, 'pkcs7'))
+        return base64.b64encode(encrypted)
 
-        for input in itertags(r.text, "input"):
-            if input.attributes['name'] == "csrf_ustvnow":
-                csrf = input.attributes['value']
+    @classmethod
+    def decrypt_data(cls, data, key, iv):
+        rkey = "".join(reversed(key)).encode('utf8')
+        riv = "".join(reversed(iv)).encode('utf8')
 
-        log.debug("CSRF: {0}", csrf)
+        fkey = SHA256.new(rkey).hexdigest()[:32].encode("utf8")
 
-        r = self.session.http.post(self._login_url, data={'csrf_ustvnow': csrf,
-                                             'signin_email': username,
-                                             'signin_password': password,
-                                             'signin_remember': '1'})
-        m = self._token_re.search(r.text)
-        return m and m.group(1)
+        cipher = AES.new(fkey, AES.MODE_CBC, riv)
+        decrypted = cipher.decrypt(base64.b64decode(data))
+        if decrypted:
+            return unpad(decrypted, 16, 'pkcs7')
+        else:
+            return decrypted
+
+    def _get_encryption_config(self, url):
+        # find the path to the main.js
+        # load the main.js and extract the config
+        if not self._encryption_config:
+            res = self.session.http.get(url)
+            m = self._main_js_re.search(res.text)
+            main_js_path = m and m.group(1)
+            if main_js_path:
+                res = self.session.http.get(urljoin(url, main_js_path))
+                self._encryption_config = dict(self._enc_key_re.findall(res.text))
+
+        return self._encryption_config.get("AES_Key"), self._encryption_config.get("AES_IV")
+
+    @property
+    def box_id(self):
+        if not self.cache.get("box_id"):
+            self.cache.set("box_id", str(uuid4()))
+        return self.cache.get("box_id")
+
+    def get_token(self):
+        """
+        Get the token for USTVNow
+        :return: a valid token
+        """
+
+        if not self._token:
+            log.debug("Getting new session token")
+            res = self.session.http.get(self._token_url, params={
+                "tenant_code": self.TENANT_CODE,
+                "box_id": self.box_id,
+                "product": self.TENANT_CODE,
+                "device_id": 5,
+                "display_lang_code": "ENG",
+                "device_sub_type": "",
+                "timezone": "UTC"
+            })
+
+            data = res.json()
+            if data['status']:
+                self._token = data['response']['sessionId']
+                log.debug("New token: {}".format(self._token))
+            else:
+                log.error("Token acquisition failed: {details} ({detail})".format(**data['error']))
+                raise PluginError("could not obtain token")
+
+        return self._token
+
+    def api_request(self, path, data, metadata=None):
+        key, iv = self._get_encryption_config(self._signin_url)
+        post_data = {
+            "data": self.encrypt_data(json.dumps(data).encode('utf8'), key, iv).decode("utf8"),
+            "metadata": self.encrypt_data(json.dumps(metadata).encode('utf8'), key, iv).decode("utf8")
+        }
+        headers = {"box-id": self.box_id,
+                   "session-id": self.get_token(),
+                   "tenant-code": self.TENANT_CODE,
+                   "content-type": "application/json"}
+        res = self.session.http.post(self._api_url + path, data=json.dumps(post_data), headers=headers).json()
+        data = dict((k, v and json.loads(self.decrypt_data(v, key, iv)))for k, v in res.items())
+        return data
+
+    def login(self, username, password):
+        log.debug("Trying to login...")
+        resp = self.api_request("send",
+                         {
+                             "login_id": username,
+                             "login_key": password,
+                             "login_mode": "1",
+                             "manufacturer": "123"
+                         },
+                         {"request": "signin"})
+
+        return resp['data']['status']
 
     def _get_streams(self):
         """
-        Finds the streams from tvcatchup.com.
+        Finds the streams from ustvnow.com.
         """
-        token = self.login(self.get_option("username"), self.get_option("password"))
-        m = self._url_re.match(self.url)
-        scode = m and m.group("scode") or self.get_option("station_code")
-
-        res = self.session.http.get(self._guide_url, params=dict(token=token))
-
-        channels = OrderedDict()
-        for t in itertags(res.text, "a"):
-            if t.attributes.get('cs'):
-                channels[t.attributes.get('cs').lower()] = t.attributes.get('title').replace("Watch ", "").strip()
-
-        if not scode:
-            log.error("Station code not provided, use --ustvnow-station-code.")
-            log.info("Available stations are: \n{0} ".format('\n'.join('    {0} ({1})'.format(c, n) for c, n in channels.items())))
-            return
-
-        if scode in channels:
-            log.debug("Finding streams for: {0}", channels.get(scode))
-
-            r = self.session.http.get(self._stream_url, params={"scode": scode,
-                                                   "token": token,
-                                                   "br_n": "Firefox",
-                                                   "br_v": "52",
-                                                   "br_d": "desktop"},
-                         headers={"User-Agent": useragents.FIREFOX})
-
-            data = self.session.http.json(r)
-            return HLSStream.parse_variant_playlist(self.session, data["stream"])
+        if self.login(self.get_option("username"), self.get_option("password")):
+            path = urlparse(self.url).path.strip("/")
+            resp = self.api_request("send", {"path": path}, {"request": "page/stream"})
+            if resp['data']['status']:
+                for stream in resp['data']['response']['streams']:
+                    if stream['keys']['licenseKey']:
+                        log.warning("Stream possibly protected by DRM")
+                    for q, s in HLSStream.parse_variant_playlist(self.session, stream['url']).items():
+                        yield (q, s)
+            else:
+                log.error("Could not find any streams: {code}: {message}".format(**resp['data']['error']))
         else:
-            log.error("Invalid station-code: {0}", scode)
+            log.error("Failed to login, check username and password")
 
 
 __plugin__ = USTVNow


Test patch:
diff --git a/tests/plugins/test_ustvnow.py b/tests/plugins/test_ustvnow.py
--- a/tests/plugins/test_ustvnow.py
+++ b/tests/plugins/test_ustvnow.py
@@ -5,12 +5,27 @@
 
 class TestPluginUSTVNow(unittest.TestCase):
     def test_can_handle_url(self):
-        self.assertTrue(USTVNow.can_handle_url("http://watch.ustvnow.com"))
-        self.assertTrue(USTVNow.can_handle_url("https://watch.ustvnow.com/"))
-        self.assertTrue(USTVNow.can_handle_url("http://watch.ustvnow.com/watch"))
-        self.assertTrue(USTVNow.can_handle_url("https://watch.ustvnow.com/watch"))
-        self.assertTrue(USTVNow.can_handle_url("https://watch.ustvnow.com/watch/syfy"))
-        self.assertTrue(USTVNow.can_handle_url("https://watch.ustvnow.com/guide/foxnews"))
+        self.assertTrue(USTVNow.can_handle_url("http://www.ustvnow.com/live/foo/-65"))
 
     def test_can_not_handle_url(self):
         self.assertFalse(USTVNow.can_handle_url("http://www.tvplayer.com"))
+
+    def test_encrypt_data(self):
+
+        key = "80035ad42d7d-bb08-7a14-f726-78403b29"
+        iv = "3157b5680927cc4a"
+
+        self.assertEqual(
+            b"uawIc5n+TnmsmR+aP2iEDKG/eMKji6EKzjI4mE+zMhlyCbHm7K4hz7IDJDWwM3aE+Ro4ydSsgJf4ZInnoW6gqvXvG0qB/J2WJeypTSt4W124zkJpvfoJJmGAvBg2t0HT",
+            USTVNow.encrypt_data(b'{"login_id":"test@test.com","login_key":"testtest1234","login_mode":"1","manufacturer":"123"}', key, iv)
+        )
+
+    def test_decrypt_data(self):
+
+        key = "80035ad42d7d-bb08-7a14-f726-78403b29"
+        iv = "3157b5680927cc4a"
+
+        self.assertEqual(
+            b'{"status":false,"error":{"code":-2,"type":"","message":"Invalid credentials.","details":{}}}',
+            USTVNow.decrypt_data(b"KcRLETVAmHlosM0OyUd5hdTQ6WhBRTe/YRAHiLJWrzf94OLkSueXTtQ9QZ1fjOLCbpX2qteEPUWVnzvvSgVDkQmRUttN/royoxW2aL0gYQSoH1NWoDV8sIgvS5vDiQ85", key, iv)
+        )