Repository: streamlink/streamlink
Pull request #3972 (2021-08-31 22:04:09)

plugin.api.validate: add parse_{json,html,xml,qsd}
1. plugin.api.validate: add args+kwargs to transform
2. plugin.api.validate: add parse_{json,html,xml,qsd}
   - add parse_{json,html,xml,qsd} validate.transform wrapper methods
   - refactor utils module and move parse* methods into utils.parse
   - refactor parse* methods and use common parsing + exception handling
   - reformat all-export list in utils

bastimeyer: ???
https://codecov.io/gh/streamlink/streamlink/src/3d101763688f919863d11170d8b620aba194b216/tests/test_api_validate.py#L74
bastimeyer: `streamlink.plugin.api.utils` still re-exports `parse_json`, `parse_xml` and `parse_qsd` (as `parse_query`):
https://github.com/streamlink/streamlink/blob/5c035fceb8f50e7cae548e34786ca61b1a3630a7/src/streamlink/plugin/api/utils.py#L5-L7

I didn't add the new methods here, because the goal is to remove this module eventually, as `itertags` is not needed with properly parsed HTML pages via lxml and XPath queries. The new methods make it especially easy to use.

I have already finished refactoring a couple of plugins where I removed the itertags imports and switched to XPath queries, but I don't think I want to refactor every plugin (at least not any time soon). Once the remaining plugins have switched to XPath queries though, we can deprecate itertags and the entire api.utils module, and then remove it in a major version bump.
gravyboat: Sounds like a solid plan to me and this looks good. Thanks @bastimeyer.

Patch:
diff --git a/src/streamlink/plugin/api/validate.py b/src/streamlink/plugin/api/validate.py
index 21ac3824..d2734de1 100644
--- a/src/streamlink/plugin/api/validate.py
+++ b/src/streamlink/plugin/api/validate.py
@@ -24,11 +24,19 @@ from urllib.parse import urlparse
 from lxml.etree import Element, iselement
 
 from streamlink.exceptions import PluginError
+from streamlink.utils.parse import (
+    parse_html as _parse_html,
+    parse_json as _parse_json,
+    parse_qsd as _parse_qsd,
+    parse_xml as _parse_xml
+)
+
 
 __all__ = [
     "any", "all", "filter", "get", "getattr", "hasattr", "length", "optional",
     "transform", "text", "union", "union_get", "url", "startswith", "endswith", "contains",
     "xml_element", "xml_find", "xml_findall", "xml_findtext", "xml_xpath", "xml_xpath_string",
+    "parse_json", "parse_html", "parse_xml", "parse_qsd",
     "validate", "Schema", "SchemaContainer"
 ]
 
@@ -69,8 +77,10 @@ class SchemaContainer:
 class transform:
     """Applies function to value to transform it."""
 
-    def __init__(self, func):
+    def __init__(self, func, *args, **kwargs):
         self.func = func
+        self.args = args
+        self.kwargs = kwargs
 
 
 class optional:
@@ -103,6 +113,9 @@ class xml_element:
         self.attrib = attrib
 
 
+# ----
+
+
 def length(length):
     """Checks value for minimum length using len()."""
     def min_len(value):
@@ -315,6 +328,25 @@ def xml_xpath_string(xpath):
     return xml_xpath(f"string({xpath})")
 
 
+def parse_json(*args, **kwargs):
+    return transform(_parse_json, *args, **kwargs, exception=ValueError, schema=None)
+
+
+def parse_html(*args, **kwargs):
+    return transform(_parse_html, *args, **kwargs, exception=ValueError, schema=None)
+
+
+def parse_xml(*args, **kwargs):
+    return transform(_parse_xml, *args, **kwargs, exception=ValueError, schema=None)
+
+
+def parse_qsd(*args, **kwargs):
+    return transform(_parse_qsd, *args, **kwargs, exception=ValueError, schema=None)
+
+
+# ----
+
+
 @singledispatch
 def validate(schema, value):
     if callable(schema):
@@ -351,9 +383,9 @@ def validate_all(schemas, value):
 
 
 @validate.register(transform)
-def validate_transform(schema, value):
+def validate_transform(schema: transform, value):
     validate(callable, schema.func)
-    return schema.func(value)
+    return schema.func(value, *schema.args, **schema.kwargs)
 
 
 @validate.register(list)
diff --git a/src/streamlink/plugins/atresplayer.py b/src/streamlink/plugins/atresplayer.py
index d7becef6..70852b66 100644
--- a/src/streamlink/plugins/atresplayer.py
+++ b/src/streamlink/plugins/atresplayer.py
@@ -1,6 +1,5 @@
 import logging
 import re
-from functools import partial
 
 from streamlink.plugin import Plugin, pluginmatcher
 from streamlink.plugin.api import validate
@@ -22,7 +21,7 @@ class AtresPlayer(Plugin):
             validate.all(
                 validate.get(1),
                 validate.transform(parse_json),
-                validate.transform(partial(search_dict, key="href")),
+                validate.transform(search_dict, key="href"),
             )
         )
     )
@@ -31,7 +30,7 @@ class AtresPlayer(Plugin):
             None,
             validate.all(
                 validate.transform(parse_json),
-                validate.transform(partial(search_dict, key="urlVideo")),
+                validate.transform(search_dict, key="urlVideo"),
             )
         )
     )
diff --git a/src/streamlink/plugins/rtve.py b/src/streamlink/plugins/rtve.py
index e30b7782..802c2185 100644
--- a/src/streamlink/plugins/rtve.py
+++ b/src/streamlink/plugins/rtve.py
@@ -1,7 +1,6 @@
 import base64
 import logging
 import re
-from functools import partial
 from urllib.parse import urlparse
 
 from Crypto.Cipher import Blowfish
@@ -57,7 +56,7 @@ class Rtve(Plugin):
     _re_idAsset = re.compile(r"\"idAsset\":\"(\d+)\"")
     secret_key = base64.b64decode("eWVMJmRhRDM=")
     cdn_schema = validate.Schema(
-        validate.transform(partial(parse_xml, invalid_char_entities=True)),
+        validate.transform(parse_xml, invalid_char_entities=True),
         validate.xml_findall(".//preset"),
         [
             validate.union({
diff --git a/src/streamlink/utils/__init__.py b/src/streamlink/utils/__init__.py
index f2ff849f..be690f2a 100644
--- a/src/streamlink/utils/__init__.py
+++ b/src/streamlink/utils/__init__.py
@@ -1,17 +1,14 @@
-import json
-import re
 import zlib
 from collections import OrderedDict
 from importlib.machinery import FileFinder, SOURCE_SUFFIXES, SourceFileLoader
 from importlib.util import module_from_spec
 from typing import Dict, Generic, Optional, TypeVar
-from urllib.parse import parse_qsl, urljoin, urlparse
-
-from lxml.etree import XML
+from urllib.parse import urljoin, urlparse
 
 from streamlink.exceptions import PluginError
 from streamlink.utils.lazy_formatter import LazyFormatter
 from streamlink.utils.named_pipe import NamedPipe
+from streamlink.utils.parse import parse_html, parse_json, parse_qsd, parse_xml
 from streamlink.utils.url import update_scheme, url_equal
 
 
@@ -61,74 +58,6 @@ def prepend_www(url):
         return url
 
 
-def parse_json(data, name="JSON", exception=PluginError, schema=None):
-    """Wrapper around json.loads.
-
-    Wraps errors in custom exception with a snippet of the data in the message.
-    """
-    try:
-        json_data = json.loads(data)
-    except ValueError as err:
-        snippet = repr(data)
-        if len(snippet) > 35:
-            snippet = snippet[:35] + " ..."
-        else:
-            snippet = data
-
-        raise exception("Unable to parse {0}: {1} ({2})".format(name, err, snippet))
-
-    if schema:
-        json_data = schema.validate(json_data, name=name, exception=exception)
-
-    return json_data
-
-
-def parse_xml(data, name="XML", ignore_ns=False, exception=PluginError, schema=None, invalid_char_entities=False):
-    """Wrapper around ElementTree.fromstring with some extras.
-
-    Provides these extra features:
-     - Handles incorrectly encoded XML
-     - Allows stripping namespace information
-     - Wraps errors in custom exception with a snippet of the data in the message
-    """
-    if isinstance(data, str):
-        data = bytes(data, "utf8")
-
-    if ignore_ns:
-        data = re.sub(br"[\t ]xmlns=\"(.+?)\"", b"", data)
-
-    if invalid_char_entities:
-        data = re.sub(br"&(?!(?:#(?:[0-9]+|[Xx][0-9A-Fa-f]+)|[A-Za-z0-9]+);)", b"&amp;", data)
-
-    try:
-        tree = XML(data)
-    except Exception as err:
-        snippet = repr(data)
-        if len(snippet) > 35:
-            snippet = snippet[:35] + " ..."
-
-        raise exception(f"Unable to parse {name}: {err} ({snippet})")
-
-    if schema:
-        tree = schema.validate(tree, name=name, exception=exception)
-
-    return tree
-
-
-def parse_qsd(data, name="query string", exception=PluginError, schema=None, **params):
-    """Parses a query string into a dict.
-
-    Unlike parse_qs and parse_qsl, duplicate keys are not preserved in
-    favor of a simpler return value.
-    """
-
-    value = dict(parse_qsl(data, **params))
-    if schema:
-        value = schema.validate(value, name=name, exception=exception)
-
-    return value
-
-
 def rtmpparse(url):
     parse = urlparse(url)
     netloc = "{hostname}:{port}".format(hostname=parse.hostname,
@@ -210,7 +139,15 @@ class LRUCache(Generic[TCacheKey, TCacheValue]):
             self.cache.popitem(last=False)
 
 
-__all__ = ["load_module", "swfdecompress", "update_scheme", "url_equal",
-           "verifyjson", "absolute_url", "parse_qsd", "parse_json",
-           "parse_xml", "rtmpparse", "prepend_www", "NamedPipe",
-           "escape_librtmp", "LRUCache", "LazyFormatter"]
+__all__ = [
+    "load_module",
+    "escape_librtmp", "rtmpparse", "swfdecompress",
+    "verifyjson",
+    "absolute_url", "prepend_www",
+    "search_dict",
+    "LRUCache",
+    "LazyFormatter",
+    "NamedPipe",
+    "parse_html", "parse_json", "parse_qsd", "parse_xml",
+    "update_scheme", "url_equal",
+]
diff --git a/src/streamlink/utils/parse.py b/src/streamlink/utils/parse.py
new file mode 100644
index 00000000..8165a2c3
--- /dev/null
+++ b/src/streamlink/utils/parse.py
@@ -0,0 +1,99 @@
+import json
+import re
+from urllib.parse import parse_qsl
+
+from lxml.etree import HTML, XML
+
+from streamlink.plugin import PluginError
+
+
+def _parse(parser, data, name, exception, schema, *args, **kwargs):
+    try:
+        parsed = parser(data, *args, **kwargs)
+    except Exception as err:
+        snippet = repr(data)
+        if len(snippet) > 35:
+            snippet = f"{snippet[:35]} ..."
+
+        raise exception(f"Unable to parse {name}: {err} ({snippet})")
+
+    if schema:
+        parsed = schema.validate(parsed, name=name, exception=exception)
+
+    return parsed
+
+
+def parse_json(
+    data,
+    name="JSON",
+    exception=PluginError,
+    schema=None,
+    *args, **kwargs
+):
+    """Wrapper around json.loads.
+
+    Provides these extra features:
+     - Wraps errors in custom exception with a snippet of the data in the message
+    """
+    return _parse(json.loads, data, name, exception, schema, *args, **kwargs)
+
+
+def parse_html(
+    data,
+    name="HTML",
+    exception=PluginError,
+    schema=None,
+    *args, **kwargs
+):
+    """Wrapper around lxml.etree.HTML with some extras.
+
+    Provides these extra features:
+     - Handles incorrectly encoded HTML
+     - Wraps errors in custom exception with a snippet of the data in the message
+    """
+    if isinstance(data, str):
+        data = bytes(data, "utf8")
+
+    return _parse(HTML, data, name, exception, schema, *args, **kwargs)
+
+
+def parse_xml(
+    data,
+    ignore_ns=False,
+    invalid_char_entities=False,
+    name="XML",
+    exception=PluginError,
+    schema=None,
+    *args, **kwargs
+):
+    """Wrapper around lxml.etree.XML with some extras.
+
+    Provides these extra features:
+     - Handles incorrectly encoded XML
+     - Allows stripping namespace information
+     - Wraps errors in custom exception with a snippet of the data in the message
+    """
+    if isinstance(data, str):
+        data = bytes(data, "utf8")
+    if ignore_ns:
+        data = re.sub(br"[\t ]xmlns=\"(.+?)\"", b"", data)
+    if invalid_char_entities:
+        data = re.sub(br"&(?!(?:#(?:[0-9]+|[Xx][0-9A-Fa-f]+)|[A-Za-z0-9]+);)", b"&amp;", data)
+
+    return _parse(XML, data, name, exception, schema, *args, **kwargs)
+
+
+def parse_qsd(
+    data,
+    name="query string",
+    exception=PluginError,
+    schema=None,
+    *args, **kwargs
+):
+    """Parses a query string into a dict.
+
+    Provides these extra features:
+     - Unlike parse_qs and parse_qsl, duplicate keys are not preserved in favor of a simpler return value
+     - Wraps errors in custom exception with a snippet of the data in the message
+    """
+    return _parse(lambda d: dict(parse_qsl(d, *args, **kwargs)), data, name, exception, schema)


Test patch:
diff --git a/tests/test_api_validate.py b/tests/test_api_validate.py
index a8663f7a..19dcc691 100644
--- a/tests/test_api_validate.py
+++ b/tests/test_api_validate.py
@@ -4,9 +4,34 @@ import unittest
 from lxml.etree import Element
 
 from streamlink.plugin.api.validate import (
-    all, any, attr, endswith, filter, get, getattr, hasattr,
-    length, map, optional, startswith, text, transform, union, union_get, url,
-    validate, xml_element, xml_find, xml_findall, xml_findtext, xml_xpath, xml_xpath_string
+    all,
+    any,
+    attr,
+    endswith,
+    filter,
+    get,
+    getattr,
+    hasattr,
+    length,
+    map,
+    optional,
+    parse_html,
+    parse_json,
+    parse_qsd,
+    parse_xml,
+    startswith,
+    text,
+    transform,
+    union,
+    union_get,
+    url,
+    validate,
+    xml_element,
+    xml_find,
+    xml_findall,
+    xml_findtext,
+    xml_xpath,
+    xml_xpath_string,
 )
 
 
@@ -16,11 +41,8 @@ class TestPluginAPIValidate(unittest.TestCase):
 
         assert validate(int, 1) == 1
 
-        assert validate(transform(int), "1") == 1
-
         assert validate(text, "abc") == "abc"
         assert validate(text, "日本語") == "日本語"
-        assert validate(transform(text), 1) == "1"
 
         assert validate(list, ["a", 1]) == ["a", 1]
         assert validate(dict, {"a": 1}) == {"a": 1}
@@ -38,6 +60,23 @@ class TestPluginAPIValidate(unittest.TestCase):
 
         assert validate(any(int), 4) == 4
 
+    def test_transform(self):
+        assert validate(transform(int), "1") == 1
+        assert validate(transform(str), 1) == "1"
+        assert validate(
+            transform(
+                lambda value, *args, **kwargs: f"{value}{args}{kwargs}",
+                *("b", "c"),
+                **dict(d="d", e="e")
+            ),
+            "a"
+        ) == "a('b', 'c'){'d': 'd', 'e': 'e'}"
+
+        def no_args():
+            pass  # pragma: no cover
+
+        self.assertRaises(TypeError, validate, transform(no_args), "some value")
+
     def test_union(self):
         assert validate(union((get("foo"), get("bar"))),
                         {"foo": "alpha", "bar": "beta"}) == ("alpha", "beta")
@@ -238,3 +277,27 @@ class TestPluginAPIValidate(unittest.TestCase):
 
     def test_endswith(self):
         assert validate(endswith("åäö"), "xyzåäö")
+
+    def test_parse_json(self):
+        assert validate(parse_json(), '{"a": ["b", true, false, null, 1, 2.3]}') == {"a": ["b", True, False, None, 1, 2.3]}
+        with self.assertRaises(ValueError) as cm:
+            validate(parse_json(), "invalid")
+        assert str(cm.exception) == "Unable to parse JSON: Expecting value: line 1 column 1 (char 0) ('invalid')"
+
+    def test_parse_html(self):
+        assert validate(parse_html(), '<!DOCTYPE html><body>&quot;perfectly&quot;<a>valid<div>HTML').tag == "html"
+        with self.assertRaises(ValueError) as cm:
+            validate(parse_html(), None)
+        assert str(cm.exception) == "Unable to parse HTML: can only parse strings (None)"
+
+    def test_parse_xml(self):
+        assert validate(parse_xml(), '<?xml version="1.0" encoding="utf-8"?><root></root>').tag == "root"
+        with self.assertRaises(ValueError) as cm:
+            validate(parse_xml(), None)
+        assert str(cm.exception) == "Unable to parse XML: can only parse strings (None)"
+
+    def test_parse_qsd(self):
+        assert validate(parse_qsd(), 'foo=bar&foo=baz') == {"foo": "baz"}
+        with self.assertRaises(ValueError) as cm:
+            validate(parse_qsd(), 123)
+        assert str(cm.exception) == "Unable to parse query string: 'int' object has no attribute 'decode' (123)"