Repository: streamlink/streamlink
Pull request #3503 (2021-01-22 13:55:51)

plugins.oneplusone: site changes: fix iframe url pattern
<!--
Thanks for opening a pull request!

Before you continue, please make sure that you have read and understood the contribution guidelines, otherwise your changes may be rejected:
https://github.com/streamlink/streamlink/blob/master/CONTRIBUTING.md#contributing-to-streamlink

If possible, run the tests, perform code linting and build the documentation locally on your system first to avoid unnecessary build failures:
https://streamlink.github.io/latest/developing.html#validating-changes

Also don't forget to add a meaningful description of your changes, so that the reviewing process is as simple as possible for the maintainers.

Thank you very much!
-->



Patch:
diff --git a/MANIFEST.in b/MANIFEST.in
index b43d3bbb..d056e641 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -11,5 +11,7 @@ recursive-include examples *
 recursive-include tests *
 
 prune docs/_build
+include docs/_build/man/*
+
 prune */__pycache__
 global-exclude *.pyc *~ *.bak *.swp *.pyo
diff --git a/docs/_man.rst b/docs/_man.rst
new file mode 100644
index 00000000..fe9cdb37
--- /dev/null
+++ b/docs/_man.rst
@@ -0,0 +1,29 @@
+:orphan:
+
+
+Streamlink
+==========
+
+Synopsis
+--------
+
+.. code-block:: console
+
+   streamlink [OPTIONS] <URL> [STREAM]
+
+   streamlink --loglevel debug youtu.be/VIDEO-ID best
+   streamlink --player mpv --player-args '--no-border --no-keepaspect-window' twitch.tv/CHANNEL 1080p60
+   streamlink --player-external-http --player-external-http-port 8888 URL STREAM
+   streamlink --output /path/to/file --http-timeout 60 URL STREAM
+   streamlink --stdout URL STREAM | ffmpeg -i pipe:0 ...
+   streamlink --http-header 'Authorization=OAuth TOKEN' --http-header 'Referer=URL' URL STREAM
+   streamlink --hls-live-edge 5 --hls-segment-threads 5 'hls://https://host/playlist.m3u8' best
+   streamlink --twitch-low-latency -p mpv -a '--cache=yes --demuxer-max-bytes=750k' twitch.tv/CHANNEL best
+
+
+Options
+=======
+
+.. argparse::
+    :module: streamlink_cli.main
+    :attr: parser_helper
diff --git a/docs/conf.py b/docs/conf.py
index 9e407363..9ad9b0f1 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -182,7 +182,7 @@ htmlhelp_basename = 'streamlinkdoc'
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('cli', 'streamlink', 'extracts streams from various services and pipes them into a video player of choice', ['Streamlink Contributors'], 1)
+    ('_man', 'streamlink', 'extracts streams from various services and pipes them into a video player of choice', ['Streamlink Contributors'], 1)
 ]
 
 # If true, show URL addresses after external links.
diff --git a/src/streamlink/plugins/bloomberg.py b/src/streamlink/plugins/bloomberg.py
index 31026aca..eea41dcc 100644
--- a/src/streamlink/plugins/bloomberg.py
+++ b/src/streamlink/plugins/bloomberg.py
@@ -1,111 +1,87 @@
 import logging
 import re
-from functools import partial
+from urllib.parse import urlparse
 
 from streamlink.plugin import Plugin
 from streamlink.plugin.api import useragents, validate
 from streamlink.stream import HDSStream, HLSStream, HTTPStream
-from streamlink.utils import parse_json, update_scheme
+from streamlink.utils import parse_json
 
 log = logging.getLogger(__name__)
 
 
 class Bloomberg(Plugin):
-    VOD_API_URL = 'https://www.bloomberg.com/api/embed?id={0}'
-    PLAYER_URL = 'https://cdn.gotraffic.net/projector/latest/bplayer.js'
-    CHANNEL_MAP = {
-        'audio': 'BBG_RADIO',
-        'live/europe': 'EU',
-        'live/us': 'US',
-        'live/asia': 'ASIA',
-        'live/stream': 'EVENT',
-        'live/emea': 'EMEA_EVENT',
-        'live/asia_stream': 'ASIA_EVENT'
-    }
+    LIVE_API_URL = "https://cdn.gotraffic.net/projector/latest/assets/config/config.min.json?v=1"
+    VOD_API_URL = "https://www.bloomberg.com/api/embed?id={0}"
 
     _url_re = re.compile(r'''
         https?://(?:www\.)?bloomberg\.com/
         (?:
-            news/videos/[^/]+/[^/]+|
+            news/videos/[^/]+/[^/]+
+            |
             live/(?P<channel>.+)/?
         )
-''', re.VERBOSE)
-    _live_player_re = re.compile(r'{APP_BUNDLE:"(?P<live_player_url>.+?/app.js)"')
-    _js_to_json_re = partial(re.compile(r'(\w+):(["\']|\d?\.?\d+,|true|false|\[|{)').sub, r'"\1":\2')
-    _mp4_bitrate_re = re.compile(r'.*_(?P<bitrate>[0-9]+)\.mp4')
-    _preload_state_re = re.compile(r'<script>\s*window.__PRELOADED_STATE__\s*=\s*({.+});?\s*</script>')
-    _live_stream_info_module_id_re = re.compile(
-        r'{(?:(?:"[^"]+"|\w+):(?:\d+|"[^"]+"),)*"(?:[^"]*/)?config/livestreams":(\d+)(?:,(?:"[^"]+"|\w+):(?:\d+|"[^"]+"))*}'
+    ''', re.VERBOSE)
+
+    _re_preload_state = re.compile(r'<script>\s*window.__PRELOADED_STATE__\s*=\s*({.+});?\s*</script>')
+    _re_mp4_bitrate = re.compile(r'.*_(?P<bitrate>[0-9]+)\.mp4')
+
+    _schema_url = validate.all(
+        {"url": validate.url()},
+        validate.get("url")
     )
-    _live_stream_info_re = r'],{id}:\[function\(.+var n=({{.+}});r.default=n}},{{}}],{next}:\['
 
-    _live_streams_schema = validate.Schema(
-        validate.transform(_js_to_json_re),
-        validate.transform(lambda x: x.replace(':.', ':0.')),
+    _schema_live_list = validate.Schema(
         validate.transform(parse_json),
-        validate.Schema({
-            validate.text: {
-                validate.optional('cdns'): validate.all(
+        validate.get("live"),
+        validate.get("channels"),
+        validate.get("byChannelId"),
+        {
+            str: validate.all({"liveId": str}, validate.get("liveId"))
+        }
+    )
+    _schema_live_streams = validate.Schema(
+        validate.get("livestreams"),
+        {
+            str: {
+                validate.optional("cdns"): validate.all(
                     [
-                        validate.Schema(
-                            {
-                                'streams': validate.all([
-                                    validate.Schema(
-                                        {'url': validate.transform(lambda x: re.sub(r'(https?:/)([^/])', r'\1/\2', x))},
-                                        validate.get('url'),
-                                        validate.url()
-                                    )
-                                ]),
-                            },
-                            validate.get('streams')
-                        )
+                        validate.all({"streams": [_schema_url]}, validate.get("streams"))
                     ],
                     validate.transform(lambda x: [i for y in x for i in y])
                 )
             }
-        },
-        )
+        }
     )
 
-    _channel_list_schema = validate.Schema(
+    _schema_vod_list = validate.Schema(
         validate.transform(parse_json),
-        {"live": {"channels": {"byChannelId": {
-            validate.text: validate.all({"liveId": validate.text}, validate.get("liveId"))
-        }}}},
-        validate.get("live"),
-        validate.get("channels"),
-        validate.get("byChannelId"),
-    )
-
-    _vod_list_schema = validate.Schema(
-        validate.transform(parse_json),
-        validate.get("video"),
-        validate.get("videoList"),
-        validate.all([
-            validate.Schema({
-                "slug": validate.text,
-                "video": validate.Schema({"bmmrId": validate.text}, validate.get("bmmrId"))
-            })
-        ])
+        validate.any(
+            validate.all(
+                {"video": {"videoList": list}},
+                validate.get("video"),
+                validate.get("videoList")
+            ),
+            validate.all(
+                {"quicktakeVideo": {"videoStory": dict}},
+                validate.get("quicktakeVideo"),
+                validate.get("videoStory"),
+                validate.transform(lambda x: [x])
+            )
+        ),
+        [
+            {
+                "slug": str,
+                "video": validate.all({"bmmrId": str}, validate.get("bmmrId"))
+            }
+        ]
     )
-
-    _vod_api_schema = validate.Schema(
+    _schema_vod_streams = validate.Schema(
         {
-            'secureStreams': validate.all([
-                validate.Schema(
-                    {'url': validate.url()},
-                    validate.get('url')
-                )
-            ]),
-            'streams': validate.all([
-                validate.Schema(
-                    {'url': validate.url()},
-                    validate.get('url')
-                )
-            ]),
-            'contentLoc': validate.url(),
+            "secureStreams": validate.all([_schema_url]),
+            "streams": validate.all([_schema_url])
         },
-        validate.transform(lambda x: list(set(x['secureStreams'] + x['streams'] + [x['contentLoc']])))
+        validate.transform(lambda x: list(set(x["secureStreams"] + x["streams"])))
     )
 
     _headers = {
@@ -124,94 +100,66 @@ class Bloomberg(Plugin):
     def can_handle_url(cls, url):
         return Bloomberg._url_re.match(url)
 
-    def _get_live_streams(self):
-        # Get channel id
-        match = self._url_re.match(self.url)
-        channel = match.group('channel')
-
+    def _get_live_streams(self, channel):
         res = self.session.http.get(self.url, headers=self._headers)
         if "Are you a robot?" in res.text:
             log.error("Are you a robot?")
 
-        match = self._preload_state_re.search(res.text)
+        match = self._re_preload_state.search(res.text)
         if match is None:
             return
 
-        live_ids = self._channel_list_schema.validate(match.group(1))
+        live_ids = self._schema_live_list.validate(match.group(1))
         live_id = live_ids.get(channel)
         if not live_id:
-            log.error("Could not find liveId for channel '{0}'".format(channel))
-            return
-
-        log.debug("Found liveId: {0}".format(live_id))
-        # Retrieve live player URL
-        res = self.session.http.get(self.PLAYER_URL)
-        match = self._live_player_re.search(res.text)
-        if match is None:
-            return
-        live_player_url = update_scheme(self.url, match.group('live_player_url'))
-
-        # Extract streams from the live player page
-        log.debug("Player URL: {0}".format(live_player_url))
-        res = self.session.http.get(live_player_url)
-
-        # Get the ID of the webpack module containing the streams config from another module's dependency list
-        match = self._live_stream_info_module_id_re.search(res.text)
-        if match is None:
+            log.error(f"Could not find liveId for channel '{channel}'")
             return
-        webpack_module_id = int(match.group(1))
-        log.debug("Webpack module ID: {0}".format(webpack_module_id))
 
-        # Finally get the streams JSON data from the config/livestreams webpack module
-        regex = re.compile(self._live_stream_info_re.format(id=webpack_module_id, next=webpack_module_id + 1))
-        match = regex.search(res.text)
-        if match is None:
-            return
-        stream_info = self._live_streams_schema.validate(match.group(1))
-        data = stream_info.get(live_id, {})
+        log.debug(f"Found liveId: {live_id}")
+        res = self.session.http.get(self.LIVE_API_URL)
+        streams = self.session.http.json(res, schema=self._schema_live_streams)
+        data = streams.get(live_id, {})
 
-        return data.get('cdns')
+        return data.get("cdns")
 
     def _get_vod_streams(self):
-        # Retrieve URL page and search for video ID
         res = self.session.http.get(self.url, headers=self._headers)
 
-        match = self._preload_state_re.search(res.text)
+        match = self._re_preload_state.search(res.text)
         if match is None:
             return
 
-        videos = self._vod_list_schema.validate(match.group(1))
+        videos = self._schema_vod_list.validate(match.group(1))
         video_id = next((v["video"] for v in videos if v["slug"] in self.url), None)
         if video_id is None:
             return
 
-        log.debug("Found videoId: {0}".format(video_id))
+        log.debug(f"Found videoId: {video_id}")
         res = self.session.http.get(self.VOD_API_URL.format(video_id), headers=self._headers)
-        streams = self.session.http.json(res, schema=self._vod_api_schema)
+        streams = self.session.http.json(res, schema=self._schema_vod_streams)
 
         return streams
 
     def _get_streams(self):
+        match = self._url_re.match(self.url)
+        channel = match.group("channel")
+
         self.session.http.headers.update({"User-Agent": useragents.CHROME})
-        if '/news/videos/' in self.url:
-            # VOD
-            streams = self._get_vod_streams() or []
+        if channel:
+            streams = self._get_live_streams(channel) or []
         else:
-            # Live
-            streams = self._get_live_streams() or []
+            streams = self._get_vod_streams() or []
 
         for video_url in streams:
-            log.debug("Found stream: {0}".format(video_url))
-            if '.f4m' in video_url:
+            log.debug(f"Found stream: {video_url}")
+            parsed = urlparse(video_url)
+            if parsed.path.endswith(".f4m"):
                 yield from HDSStream.parse_manifest(self.session, video_url).items()
-            elif '.m3u8' in video_url:
+            elif parsed.path.endswith(".m3u8"):
                 yield from HLSStream.parse_variant_playlist(self.session, video_url).items()
-            if '.mp4' in video_url:
-                match = self._mp4_bitrate_re.match(video_url)
-                if match is not None:
-                    bitrate = match.group('bitrate') + 'k'
-                else:
-                    bitrate = 'vod'
+            elif parsed.path.endswith(".mp4"):
+                match = self._re_mp4_bitrate.match(video_url)
+                bitrate = "vod" if match is None else f"{match.group('bitrate')}k"
                 yield bitrate, HTTPStream(self.session, video_url)
 
 
diff --git a/src/streamlink/plugins/oneplusone.py b/src/streamlink/plugins/oneplusone.py
index 4ac04e7e..df8f0379 100644
--- a/src/streamlink/plugins/oneplusone.py
+++ b/src/streamlink/plugins/oneplusone.py
@@ -55,9 +55,12 @@ class OnePlusOne(Plugin):
         parser = Online_Parser()
         parser.feed(res.text)
         url = parser.iframe_url
-        if url.startswith("//"):
+        log.trace(f"find_iframe url: {url}")
+        if url.startswith("/"):
             p = urlparse(self.url)
-            return "{0}:{1}".format(p.scheme, url)
+            if url.startswith("//"):
+                return "{0}:{1}".format(p.scheme, url)
+            return "{0}://{1}{2}".format(p.scheme, p.netloc, url)
         else:
             return url
 
diff --git a/src/streamlink/plugins/twitcasting.py b/src/streamlink/plugins/twitcasting.py
index b33ab678..15f344f7 100644
--- a/src/streamlink/plugins/twitcasting.py
+++ b/src/streamlink/plugins/twitcasting.py
@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import re
 from threading import Event, Thread
@@ -7,15 +8,25 @@ import websocket
 
 from streamlink import logger
 from streamlink.buffers import RingBuffer
-from streamlink.plugin import Plugin, PluginError
+from streamlink.plugin import Plugin, PluginArgument, PluginArguments, PluginError
 from streamlink.plugin.api import useragents, validate
 from streamlink.stream.stream import Stream
 from streamlink.stream.stream import StreamIO
+from streamlink.utils.url import update_qsd
+
 
 log = logging.getLogger(__name__)
 
 
 class TwitCasting(Plugin):
+    arguments = PluginArguments(
+        PluginArgument(
+            "password",
+            sensitive=True,
+            metavar="PASSWORD",
+            help="Password for private Twitcasting streams."
+        )
+    )
     _url_re = re.compile(r"http(s)?://twitcasting.tv/(?P<channel>[^/]+)", re.VERBOSE)
     _STREAM_INFO_URL = "https://twitcasting.tv/streamserver.php?target={channel}&mode=client"
     _STREAM_REAL_URL = "{proto}://{host}/ws.app/stream/{movie_id}/fmp4/bd/1/1500?mode={mode}"
@@ -66,6 +77,12 @@ class TwitCasting(Plugin):
             raise PluginError("No stream available for user {}".format(self.channel))
 
         real_stream_url = self._STREAM_REAL_URL.format(proto=proto, host=host, movie_id=movie_id, mode=mode)
+
+        password = self.options.get("password")
+        if password is not None:
+            password_hash = hashlib.md5(password.encode()).hexdigest()
+            real_stream_url = update_qsd(real_stream_url, {"word": password_hash})
+
         log.debug("Real stream url: {}".format(real_stream_url))
 
         return {mode: TwitCastingStream(session=self.session, url=real_stream_url)}
diff --git a/src/streamlink/plugins/wwenetwork.py b/src/streamlink/plugins/wwenetwork.py
index 2f9d489a..e1753c89 100644
--- a/src/streamlink/plugins/wwenetwork.py
+++ b/src/streamlink/plugins/wwenetwork.py
@@ -1,13 +1,13 @@
 import json
 import logging
 import re
+from functools import lru_cache
 from urllib.parse import parse_qsl, urlparse
 
 from streamlink import PluginError
 from streamlink.plugin import Plugin, PluginArgument, PluginArguments
 from streamlink.plugin.api import useragents
 from streamlink.stream import HLSStream
-from streamlink.utils import memoize
 from streamlink.utils.times import seconds_to_hhmmss
 
 log = logging.getLogger(__name__)
@@ -91,7 +91,7 @@ class WWENetwork(Plugin):
         return self.auth_token
 
     @property
-    @memoize
+    @lru_cache(maxsize=128)
     def item_config(self):
         log.debug("Loading page config")
         p = urlparse(self.url)
diff --git a/src/streamlink/session.py b/src/streamlink/session.py
index 2183c034..09856260 100644
--- a/src/streamlink/session.py
+++ b/src/streamlink/session.py
@@ -1,6 +1,7 @@
 import logging
 import pkgutil
 from collections import OrderedDict
+from functools import lru_cache
 
 import requests
 
@@ -10,7 +11,7 @@ from streamlink.exceptions import NoPluginError, PluginError
 from streamlink.logger import StreamlinkLogger
 from streamlink.options import Options
 from streamlink.plugin import Plugin, api
-from streamlink.utils import load_module, memoize, update_scheme
+from streamlink.utils import load_module, update_scheme
 from streamlink.utils.l10n import Localization
 
 # Ensure that the Logger class returned is Streamslink's for using the API (for backwards compatibility)
@@ -337,7 +338,7 @@ class Streamlink:
             plugin = self.plugins[plugin]
             return plugin.get_option(key)
 
-    @memoize
+    @lru_cache(maxsize=128)
     def resolve_url(self, url, follow_redirect=True):
         """Attempts to find a plugin that can use this URL.
 
diff --git a/src/streamlink/utils/__init__.py b/src/streamlink/utils/__init__.py
index 0403c394..f189a5f8 100644
--- a/src/streamlink/utils/__init__.py
+++ b/src/streamlink/utils/__init__.py
@@ -1,4 +1,3 @@
-import functools
 import json
 import re
 import xml.etree.ElementTree as ET
@@ -151,18 +150,6 @@ def rtmpparse(url):
     return tcurl, playpath
 
 
-def memoize(obj):
-    cache = obj.cache = {}
-
-    @functools.wraps(obj)
-    def memoizer(*args, **kwargs):
-        key = str(args) + str(kwargs)
-        if key not in cache:
-            cache[key] = obj(*args, **kwargs)
-        return cache[key]
-    return memoizer
-
-
 def search_dict(data, key):
     """
     Search for a key in a nested dict, or list of nested dicts, and return the values.
diff --git a/src/streamlink/utils/l10n.py b/src/streamlink/utils/l10n.py
index de2c8892..86e2f109 100644
--- a/src/streamlink/utils/l10n.py
+++ b/src/streamlink/utils/l10n.py
@@ -1,6 +1,5 @@
 import locale
 import logging
-import re
 
 
 try:
@@ -68,9 +67,19 @@ class Language:
     def get(cls, language):
         try:
             if PYCOUNTRY:
-                # lookup workaround for alpha_2 language codes
-                lang = languages.get(alpha_2=language) if re.match(r"^[a-z]{2}$", language) else languages.lookup(language)
-                return Language(lang.alpha_2, lang.alpha_3, lang.name, getattr(lang, "bibliographic", None))
+                lang = (languages.get(alpha_2=language)
+                        or languages.get(alpha_3=language)
+                        or languages.get(bibliographic=language)
+                        or languages.get(name=language))
+                if not lang:
+                    raise KeyError(language)
+                return Language(
+                    # some languages don't have an alpha_2 code
+                    getattr(lang, "alpha_2", ""),
+                    lang.alpha_3,
+                    lang.name,
+                    getattr(lang, "bibliographic", "")
+                )
             else:
                 lang = None
                 if len(language) == 2:


Test patch:
diff --git a/tests/test_localization.py b/tests/test_localization.py
index 8f6cbde7..b9896a77 100644
--- a/tests/test_localization.py
+++ b/tests/test_localization.py
@@ -109,6 +109,14 @@ class LocalizationTestsMixin:
         b = l10n.Language("AA", None, "Test")
         self.assertNotEqual(a, b)
 
+    # issue #3517: language lookups without alpha2 but with alpha3 codes should not raise
+    def test_language_a3_no_a2(self):
+        a = l10n.Localization.get_language("des")
+        self.assertEqual(a.alpha2, "")
+        self.assertEqual(a.alpha3, "des")
+        self.assertEqual(a.name, "Desano")
+        self.assertEqual(a.bibliographic, "")
+
 
 @unittest.skipIf(not ISO639, "iso639+iso3166 modules are required to test iso639+iso3166 Localization")
 class TestLocalization(LocalizationTestsMixin, unittest.TestCase):
@@ -131,3 +139,14 @@ class TestLocalizationPyCountry(LocalizationTestsMixin, unittest.TestCase):
 
     def test_pycountry(self):
         self.assertEqual(True, l10n.PYCOUNTRY)
+
+    # issue #3057: generic "en" lookups via pycountry yield the "En" language, but not "English"
+    def test_language_en(self):
+        english_a = l10n.Localization.get_language("en")
+        english_b = l10n.Localization.get_language("eng")
+        english_c = l10n.Localization.get_language("English")
+        for lang in [english_a, english_b, english_c]:
+            self.assertEqual(lang.alpha2, "en")
+            self.assertEqual(lang.alpha3, "eng")
+            self.assertEqual(lang.name, "English")
+            self.assertEqual(lang.bibliographic, "")
diff --git a/tests/test_session.py b/tests/test_session.py
index 61bdd64f..5ce51008 100644
--- a/tests/test_session.py
+++ b/tests/test_session.py
@@ -68,6 +68,7 @@ class TestSession(unittest.TestCase):
         plugin = session.resolve_url("http://test.se/channel")
         self.assertTrue(isinstance(plugin, Plugin))
         self.assertTrue(isinstance(plugin, plugins["testplugin"]))
+        self.assertTrue(hasattr(session.resolve_url, "cache_info"), "resolve_url has a lookup cache")
 
     def test_resolve_url_priority(self):
         from tests.plugin.testplugin import TestPlugin