Repository: streamlink/streamlink
Pull request #3499 (2021-01-21 22:19:24)

docs: remove CLI tutorial from man page
and add synopsis with examples

----

I'd like to embed some sections from the CLI page in the man page, but that would require rewriting the CLI page and making it modular, so that the sections can be included without duplicating them. At least the tutorial does not get included with this PR.

The examples could probably be improved, so if you have suggestions, please tell.


```
$ make --directory=docs clean man
$ man ./docs/_build/man/streamlink.1
```

gravyboat: Looks fine to me. Let's leave this open for a day and see if anyone else has feedback.

Patch:
diff --git a/MANIFEST.in b/MANIFEST.in
index b43d3bbb..d056e641 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -11,5 +11,7 @@ recursive-include examples *
 recursive-include tests *
 
 prune docs/_build
+include docs/_build/man/*
+
 prune */__pycache__
 global-exclude *.pyc *~ *.bak *.swp *.pyo
diff --git a/docs/_man.rst b/docs/_man.rst
new file mode 100644
index 00000000..fe9cdb37
--- /dev/null
+++ b/docs/_man.rst
@@ -0,0 +1,29 @@
+:orphan:
+
+
+Streamlink
+==========
+
+Synopsis
+--------
+
+.. code-block:: console
+
+   streamlink [OPTIONS] <URL> [STREAM]
+
+   streamlink --loglevel debug youtu.be/VIDEO-ID best
+   streamlink --player mpv --player-args '--no-border --no-keepaspect-window' twitch.tv/CHANNEL 1080p60
+   streamlink --player-external-http --player-external-http-port 8888 URL STREAM
+   streamlink --output /path/to/file --http-timeout 60 URL STREAM
+   streamlink --stdout URL STREAM | ffmpeg -i pipe:0 ...
+   streamlink --http-header 'Authorization=OAuth TOKEN' --http-header 'Referer=URL' URL STREAM
+   streamlink --hls-live-edge 5 --hls-segment-threads 5 'hls://https://host/playlist.m3u8' best
+   streamlink --twitch-low-latency -p mpv -a '--cache=yes --demuxer-max-bytes=750k' twitch.tv/CHANNEL best
+
+
+Options
+=======
+
+.. argparse::
+    :module: streamlink_cli.main
+    :attr: parser_helper
diff --git a/docs/conf.py b/docs/conf.py
index 9e407363..9ad9b0f1 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -182,7 +182,7 @@ htmlhelp_basename = 'streamlinkdoc'
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('cli', 'streamlink', 'extracts streams from various services and pipes them into a video player of choice', ['Streamlink Contributors'], 1)
+    ('_man', 'streamlink', 'extracts streams from various services and pipes them into a video player of choice', ['Streamlink Contributors'], 1)
 ]
 
 # If true, show URL addresses after external links.
diff --git a/docs/install.rst b/docs/install.rst
index bdb6c6bc..7130ab05 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -136,7 +136,7 @@ Distribution                         Installing
 .. _Arch Linux (aur, git): https://aur.archlinux.org/packages/streamlink-git/
 .. _Debian (sid, testing): https://packages.debian.org/unstable/streamlink
 .. _Debian (stable): https://packages.debian.org/unstable/streamlink
-.. _Fedora: https://apps.fedoraproject.org/packages/python-streamlink
+.. _Fedora: https://src.fedoraproject.org/rpms/python-streamlink
 .. _Gentoo Linux: https://packages.gentoo.org/package/net-misc/streamlink
 .. _NetBSD (pkgsrc): https://pkgsrc.se/multimedia/streamlink
 .. _NixOS: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/video/streamlink
diff --git a/src/streamlink/plugins/bloomberg.py b/src/streamlink/plugins/bloomberg.py
index 31026aca..eea41dcc 100644
--- a/src/streamlink/plugins/bloomberg.py
+++ b/src/streamlink/plugins/bloomberg.py
@@ -1,111 +1,87 @@
 import logging
 import re
-from functools import partial
+from urllib.parse import urlparse
 
 from streamlink.plugin import Plugin
 from streamlink.plugin.api import useragents, validate
 from streamlink.stream import HDSStream, HLSStream, HTTPStream
-from streamlink.utils import parse_json, update_scheme
+from streamlink.utils import parse_json
 
 log = logging.getLogger(__name__)
 
 
 class Bloomberg(Plugin):
-    VOD_API_URL = 'https://www.bloomberg.com/api/embed?id={0}'
-    PLAYER_URL = 'https://cdn.gotraffic.net/projector/latest/bplayer.js'
-    CHANNEL_MAP = {
-        'audio': 'BBG_RADIO',
-        'live/europe': 'EU',
-        'live/us': 'US',
-        'live/asia': 'ASIA',
-        'live/stream': 'EVENT',
-        'live/emea': 'EMEA_EVENT',
-        'live/asia_stream': 'ASIA_EVENT'
-    }
+    LIVE_API_URL = "https://cdn.gotraffic.net/projector/latest/assets/config/config.min.json?v=1"
+    VOD_API_URL = "https://www.bloomberg.com/api/embed?id={0}"
 
     _url_re = re.compile(r'''
         https?://(?:www\.)?bloomberg\.com/
         (?:
-            news/videos/[^/]+/[^/]+|
+            news/videos/[^/]+/[^/]+
+            |
             live/(?P<channel>.+)/?
         )
-''', re.VERBOSE)
-    _live_player_re = re.compile(r'{APP_BUNDLE:"(?P<live_player_url>.+?/app.js)"')
-    _js_to_json_re = partial(re.compile(r'(\w+):(["\']|\d?\.?\d+,|true|false|\[|{)').sub, r'"\1":\2')
-    _mp4_bitrate_re = re.compile(r'.*_(?P<bitrate>[0-9]+)\.mp4')
-    _preload_state_re = re.compile(r'<script>\s*window.__PRELOADED_STATE__\s*=\s*({.+});?\s*</script>')
-    _live_stream_info_module_id_re = re.compile(
-        r'{(?:(?:"[^"]+"|\w+):(?:\d+|"[^"]+"),)*"(?:[^"]*/)?config/livestreams":(\d+)(?:,(?:"[^"]+"|\w+):(?:\d+|"[^"]+"))*}'
+    ''', re.VERBOSE)
+
+    _re_preload_state = re.compile(r'<script>\s*window.__PRELOADED_STATE__\s*=\s*({.+});?\s*</script>')
+    _re_mp4_bitrate = re.compile(r'.*_(?P<bitrate>[0-9]+)\.mp4')
+
+    _schema_url = validate.all(
+        {"url": validate.url()},
+        validate.get("url")
     )
-    _live_stream_info_re = r'],{id}:\[function\(.+var n=({{.+}});r.default=n}},{{}}],{next}:\['
 
-    _live_streams_schema = validate.Schema(
-        validate.transform(_js_to_json_re),
-        validate.transform(lambda x: x.replace(':.', ':0.')),
+    _schema_live_list = validate.Schema(
         validate.transform(parse_json),
-        validate.Schema({
-            validate.text: {
-                validate.optional('cdns'): validate.all(
+        validate.get("live"),
+        validate.get("channels"),
+        validate.get("byChannelId"),
+        {
+            str: validate.all({"liveId": str}, validate.get("liveId"))
+        }
+    )
+    _schema_live_streams = validate.Schema(
+        validate.get("livestreams"),
+        {
+            str: {
+                validate.optional("cdns"): validate.all(
                     [
-                        validate.Schema(
-                            {
-                                'streams': validate.all([
-                                    validate.Schema(
-                                        {'url': validate.transform(lambda x: re.sub(r'(https?:/)([^/])', r'\1/\2', x))},
-                                        validate.get('url'),
-                                        validate.url()
-                                    )
-                                ]),
-                            },
-                            validate.get('streams')
-                        )
+                        validate.all({"streams": [_schema_url]}, validate.get("streams"))
                     ],
                     validate.transform(lambda x: [i for y in x for i in y])
                 )
             }
-        },
-        )
+        }
     )
 
-    _channel_list_schema = validate.Schema(
+    _schema_vod_list = validate.Schema(
         validate.transform(parse_json),
-        {"live": {"channels": {"byChannelId": {
-            validate.text: validate.all({"liveId": validate.text}, validate.get("liveId"))
-        }}}},
-        validate.get("live"),
-        validate.get("channels"),
-        validate.get("byChannelId"),
-    )
-
-    _vod_list_schema = validate.Schema(
-        validate.transform(parse_json),
-        validate.get("video"),
-        validate.get("videoList"),
-        validate.all([
-            validate.Schema({
-                "slug": validate.text,
-                "video": validate.Schema({"bmmrId": validate.text}, validate.get("bmmrId"))
-            })
-        ])
+        validate.any(
+            validate.all(
+                {"video": {"videoList": list}},
+                validate.get("video"),
+                validate.get("videoList")
+            ),
+            validate.all(
+                {"quicktakeVideo": {"videoStory": dict}},
+                validate.get("quicktakeVideo"),
+                validate.get("videoStory"),
+                validate.transform(lambda x: [x])
+            )
+        ),
+        [
+            {
+                "slug": str,
+                "video": validate.all({"bmmrId": str}, validate.get("bmmrId"))
+            }
+        ]
     )
-
-    _vod_api_schema = validate.Schema(
+    _schema_vod_streams = validate.Schema(
         {
-            'secureStreams': validate.all([
-                validate.Schema(
-                    {'url': validate.url()},
-                    validate.get('url')
-                )
-            ]),
-            'streams': validate.all([
-                validate.Schema(
-                    {'url': validate.url()},
-                    validate.get('url')
-                )
-            ]),
-            'contentLoc': validate.url(),
+            "secureStreams": validate.all([_schema_url]),
+            "streams": validate.all([_schema_url])
         },
-        validate.transform(lambda x: list(set(x['secureStreams'] + x['streams'] + [x['contentLoc']])))
+        validate.transform(lambda x: list(set(x["secureStreams"] + x["streams"])))
     )
 
     _headers = {
@@ -124,94 +100,66 @@ class Bloomberg(Plugin):
     def can_handle_url(cls, url):
         return Bloomberg._url_re.match(url)
 
-    def _get_live_streams(self):
-        # Get channel id
-        match = self._url_re.match(self.url)
-        channel = match.group('channel')
-
+    def _get_live_streams(self, channel):
         res = self.session.http.get(self.url, headers=self._headers)
         if "Are you a robot?" in res.text:
             log.error("Are you a robot?")
 
-        match = self._preload_state_re.search(res.text)
+        match = self._re_preload_state.search(res.text)
         if match is None:
             return
 
-        live_ids = self._channel_list_schema.validate(match.group(1))
+        live_ids = self._schema_live_list.validate(match.group(1))
         live_id = live_ids.get(channel)
         if not live_id:
-            log.error("Could not find liveId for channel '{0}'".format(channel))
-            return
-
-        log.debug("Found liveId: {0}".format(live_id))
-        # Retrieve live player URL
-        res = self.session.http.get(self.PLAYER_URL)
-        match = self._live_player_re.search(res.text)
-        if match is None:
-            return
-        live_player_url = update_scheme(self.url, match.group('live_player_url'))
-
-        # Extract streams from the live player page
-        log.debug("Player URL: {0}".format(live_player_url))
-        res = self.session.http.get(live_player_url)
-
-        # Get the ID of the webpack module containing the streams config from another module's dependency list
-        match = self._live_stream_info_module_id_re.search(res.text)
-        if match is None:
+            log.error(f"Could not find liveId for channel '{channel}'")
             return
-        webpack_module_id = int(match.group(1))
-        log.debug("Webpack module ID: {0}".format(webpack_module_id))
 
-        # Finally get the streams JSON data from the config/livestreams webpack module
-        regex = re.compile(self._live_stream_info_re.format(id=webpack_module_id, next=webpack_module_id + 1))
-        match = regex.search(res.text)
-        if match is None:
-            return
-        stream_info = self._live_streams_schema.validate(match.group(1))
-        data = stream_info.get(live_id, {})
+        log.debug(f"Found liveId: {live_id}")
+        res = self.session.http.get(self.LIVE_API_URL)
+        streams = self.session.http.json(res, schema=self._schema_live_streams)
+        data = streams.get(live_id, {})
 
-        return data.get('cdns')
+        return data.get("cdns")
 
     def _get_vod_streams(self):
-        # Retrieve URL page and search for video ID
         res = self.session.http.get(self.url, headers=self._headers)
 
-        match = self._preload_state_re.search(res.text)
+        match = self._re_preload_state.search(res.text)
         if match is None:
             return
 
-        videos = self._vod_list_schema.validate(match.group(1))
+        videos = self._schema_vod_list.validate(match.group(1))
         video_id = next((v["video"] for v in videos if v["slug"] in self.url), None)
         if video_id is None:
             return
 
-        log.debug("Found videoId: {0}".format(video_id))
+        log.debug(f"Found videoId: {video_id}")
         res = self.session.http.get(self.VOD_API_URL.format(video_id), headers=self._headers)
-        streams = self.session.http.json(res, schema=self._vod_api_schema)
+        streams = self.session.http.json(res, schema=self._schema_vod_streams)
 
         return streams
 
     def _get_streams(self):
+        match = self._url_re.match(self.url)
+        channel = match.group("channel")
+
         self.session.http.headers.update({"User-Agent": useragents.CHROME})
-        if '/news/videos/' in self.url:
-            # VOD
-            streams = self._get_vod_streams() or []
+        if channel:
+            streams = self._get_live_streams(channel) or []
         else:
-            # Live
-            streams = self._get_live_streams() or []
+            streams = self._get_vod_streams() or []
 
         for video_url in streams:
-            log.debug("Found stream: {0}".format(video_url))
-            if '.f4m' in video_url:
+            log.debug(f"Found stream: {video_url}")
+            parsed = urlparse(video_url)
+            if parsed.path.endswith(".f4m"):
                 yield from HDSStream.parse_manifest(self.session, video_url).items()
-            elif '.m3u8' in video_url:
+            elif parsed.path.endswith(".m3u8"):
                 yield from HLSStream.parse_variant_playlist(self.session, video_url).items()
-            if '.mp4' in video_url:
-                match = self._mp4_bitrate_re.match(video_url)
-                if match is not None:
-                    bitrate = match.group('bitrate') + 'k'
-                else:
-                    bitrate = 'vod'
+            elif parsed.path.endswith(".mp4"):
+                match = self._re_mp4_bitrate.match(video_url)
+                bitrate = "vod" if match is None else f"{match.group('bitrate')}k"
                 yield bitrate, HTTPStream(self.session, video_url)
 
 
diff --git a/src/streamlink/plugins/twitcasting.py b/src/streamlink/plugins/twitcasting.py
index b33ab678..15f344f7 100644
--- a/src/streamlink/plugins/twitcasting.py
+++ b/src/streamlink/plugins/twitcasting.py
@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import re
 from threading import Event, Thread
@@ -7,15 +8,25 @@ import websocket
 
 from streamlink import logger
 from streamlink.buffers import RingBuffer
-from streamlink.plugin import Plugin, PluginError
+from streamlink.plugin import Plugin, PluginArgument, PluginArguments, PluginError
 from streamlink.plugin.api import useragents, validate
 from streamlink.stream.stream import Stream
 from streamlink.stream.stream import StreamIO
+from streamlink.utils.url import update_qsd
+
 
 log = logging.getLogger(__name__)
 
 
 class TwitCasting(Plugin):
+    arguments = PluginArguments(
+        PluginArgument(
+            "password",
+            sensitive=True,
+            metavar="PASSWORD",
+            help="Password for private Twitcasting streams."
+        )
+    )
     _url_re = re.compile(r"http(s)?://twitcasting.tv/(?P<channel>[^/]+)", re.VERBOSE)
     _STREAM_INFO_URL = "https://twitcasting.tv/streamserver.php?target={channel}&mode=client"
     _STREAM_REAL_URL = "{proto}://{host}/ws.app/stream/{movie_id}/fmp4/bd/1/1500?mode={mode}"
@@ -66,6 +77,12 @@ class TwitCasting(Plugin):
             raise PluginError("No stream available for user {}".format(self.channel))
 
         real_stream_url = self._STREAM_REAL_URL.format(proto=proto, host=host, movie_id=movie_id, mode=mode)
+
+        password = self.options.get("password")
+        if password is not None:
+            password_hash = hashlib.md5(password.encode()).hexdigest()
+            real_stream_url = update_qsd(real_stream_url, {"word": password_hash})
+
         log.debug("Real stream url: {}".format(real_stream_url))
 
         return {mode: TwitCastingStream(session=self.session, url=real_stream_url)}
diff --git a/src/streamlink/plugins/wwenetwork.py b/src/streamlink/plugins/wwenetwork.py
index 2f9d489a..e1753c89 100644
--- a/src/streamlink/plugins/wwenetwork.py
+++ b/src/streamlink/plugins/wwenetwork.py
@@ -1,13 +1,13 @@
 import json
 import logging
 import re
+from functools import lru_cache
 from urllib.parse import parse_qsl, urlparse
 
 from streamlink import PluginError
 from streamlink.plugin import Plugin, PluginArgument, PluginArguments
 from streamlink.plugin.api import useragents
 from streamlink.stream import HLSStream
-from streamlink.utils import memoize
 from streamlink.utils.times import seconds_to_hhmmss
 
 log = logging.getLogger(__name__)
@@ -91,7 +91,7 @@ class WWENetwork(Plugin):
         return self.auth_token
 
     @property
-    @memoize
+    @lru_cache(maxsize=128)
     def item_config(self):
         log.debug("Loading page config")
         p = urlparse(self.url)
diff --git a/src/streamlink/session.py b/src/streamlink/session.py
index 2183c034..09856260 100644
--- a/src/streamlink/session.py
+++ b/src/streamlink/session.py
@@ -1,6 +1,7 @@
 import logging
 import pkgutil
 from collections import OrderedDict
+from functools import lru_cache
 
 import requests
 
@@ -10,7 +11,7 @@ from streamlink.exceptions import NoPluginError, PluginError
 from streamlink.logger import StreamlinkLogger
 from streamlink.options import Options
 from streamlink.plugin import Plugin, api
-from streamlink.utils import load_module, memoize, update_scheme
+from streamlink.utils import load_module, update_scheme
 from streamlink.utils.l10n import Localization
 
 # Ensure that the Logger class returned is Streamslink's for using the API (for backwards compatibility)
@@ -337,7 +338,7 @@ class Streamlink:
             plugin = self.plugins[plugin]
             return plugin.get_option(key)
 
-    @memoize
+    @lru_cache(maxsize=128)
     def resolve_url(self, url, follow_redirect=True):
         """Attempts to find a plugin that can use this URL.
 
diff --git a/src/streamlink/utils/__init__.py b/src/streamlink/utils/__init__.py
index 0403c394..f189a5f8 100644
--- a/src/streamlink/utils/__init__.py
+++ b/src/streamlink/utils/__init__.py
@@ -1,4 +1,3 @@
-import functools
 import json
 import re
 import xml.etree.ElementTree as ET
@@ -151,18 +150,6 @@ def rtmpparse(url):
     return tcurl, playpath
 
 
-def memoize(obj):
-    cache = obj.cache = {}
-
-    @functools.wraps(obj)
-    def memoizer(*args, **kwargs):
-        key = str(args) + str(kwargs)
-        if key not in cache:
-            cache[key] = obj(*args, **kwargs)
-        return cache[key]
-    return memoizer
-
-
 def search_dict(data, key):
     """
     Search for a key in a nested dict, or list of nested dicts, and return the values.
diff --git a/src/streamlink/utils/l10n.py b/src/streamlink/utils/l10n.py
index de2c8892..86e2f109 100644
--- a/src/streamlink/utils/l10n.py
+++ b/src/streamlink/utils/l10n.py
@@ -1,6 +1,5 @@
 import locale
 import logging
-import re
 
 
 try:
@@ -68,9 +67,19 @@ class Language:
     def get(cls, language):
         try:
             if PYCOUNTRY:
-                # lookup workaround for alpha_2 language codes
-                lang = languages.get(alpha_2=language) if re.match(r"^[a-z]{2}$", language) else languages.lookup(language)
-                return Language(lang.alpha_2, lang.alpha_3, lang.name, getattr(lang, "bibliographic", None))
+                lang = (languages.get(alpha_2=language)
+                        or languages.get(alpha_3=language)
+                        or languages.get(bibliographic=language)
+                        or languages.get(name=language))
+                if not lang:
+                    raise KeyError(language)
+                return Language(
+                    # some languages don't have an alpha_2 code
+                    getattr(lang, "alpha_2", ""),
+                    lang.alpha_3,
+                    lang.name,
+                    getattr(lang, "bibliographic", "")
+                )
             else:
                 lang = None
                 if len(language) == 2:


Test patch:
diff --git a/tests/test_localization.py b/tests/test_localization.py
index 8f6cbde7..b9896a77 100644
--- a/tests/test_localization.py
+++ b/tests/test_localization.py
@@ -109,6 +109,14 @@ class LocalizationTestsMixin:
         b = l10n.Language("AA", None, "Test")
         self.assertNotEqual(a, b)
 
+    # issue #3517: language lookups without alpha2 but with alpha3 codes should not raise
+    def test_language_a3_no_a2(self):
+        a = l10n.Localization.get_language("des")
+        self.assertEqual(a.alpha2, "")
+        self.assertEqual(a.alpha3, "des")
+        self.assertEqual(a.name, "Desano")
+        self.assertEqual(a.bibliographic, "")
+
 
 @unittest.skipIf(not ISO639, "iso639+iso3166 modules are required to test iso639+iso3166 Localization")
 class TestLocalization(LocalizationTestsMixin, unittest.TestCase):
@@ -131,3 +139,14 @@ class TestLocalizationPyCountry(LocalizationTestsMixin, unittest.TestCase):
 
     def test_pycountry(self):
         self.assertEqual(True, l10n.PYCOUNTRY)
+
+    # issue #3057: generic "en" lookups via pycountry yield the "En" language, but not "English"
+    def test_language_en(self):
+        english_a = l10n.Localization.get_language("en")
+        english_b = l10n.Localization.get_language("eng")
+        english_c = l10n.Localization.get_language("English")
+        for lang in [english_a, english_b, english_c]:
+            self.assertEqual(lang.alpha2, "en")
+            self.assertEqual(lang.alpha3, "eng")
+            self.assertEqual(lang.name, "English")
+            self.assertEqual(lang.bibliographic, "")
diff --git a/tests/test_session.py b/tests/test_session.py
index 61bdd64f..5ce51008 100644
--- a/tests/test_session.py
+++ b/tests/test_session.py
@@ -68,6 +68,7 @@ class TestSession(unittest.TestCase):
         plugin = session.resolve_url("http://test.se/channel")
         self.assertTrue(isinstance(plugin, Plugin))
         self.assertTrue(isinstance(plugin, plugins["testplugin"]))
+        self.assertTrue(hasattr(session.resolve_url, "cache_info"), "resolve_url has a lookup cache")
 
     def test_resolve_url_priority(self):
         from tests.plugin.testplugin import TestPlugin