Repository: streamlink/streamlink

Issue:
tests: `tests.utils.test_parse.TestUtilsParse.test_parse_html_xhtml5` failure with libxml2 2.12.0
### Checklist

- [X] This is a bug report and not [a different kind of issue](https://github.com/streamlink/streamlink/issues/new/choose)
- [X] [I have read the contribution guidelines](https://github.com/streamlink/streamlink/blob/master/CONTRIBUTING.md#contributing-to-streamlink)
- [X] [I have checked the list of open and recently closed bug reports](https://github.com/streamlink/streamlink/issues?q=is%3Aissue+label%3A%22bug%22)
- [X] [I have checked the commit log of the master branch](https://github.com/streamlink/streamlink/commits/master)

### Streamlink version

6.4.0

### Description

While trying to build the Arch package, I run into this test failure, which seems to be an encoding problem if I'm not mistaken.

### Debug log

```text
=================================== FAILURES ===================================
____________________ TestUtilsParse.test_parse_html_xhtml5 _____________________

self = <tests.utils.test_parse.TestUtilsParse object at 0x7fa710eb4b90>

    def test_parse_html_xhtml5(self):
        tree = parse_html("""<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>ä?></body></html>""")
        assert tree.xpath(".//body/text()") == ["ä?>"]
        tree = parse_html(b"""<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>\xC3\xA4?></body></html>""")
>       assert tree.xpath(".//body/text()") == ["ä?>"]
E       AssertionError: assert ['Ã¤?>'] == ['ä?>']
E         At index 0 diff: 'Ã¤?>' != 'ä?>'
E         Use -v to get more diff

tests/utils/test_parse.py:101: AssertionError
```


Discussion:

Patch:
diff --git a/src/streamlink/compat.py b/src/streamlink/compat.py
index c7520154..993bce64 100644
--- a/src/streamlink/compat.py
+++ b/src/streamlink/compat.py
@@ -2,11 +2,22 @@ import os
 import sys
 
 
+# compatibility import of charset_normalizer/chardet via requests<3.0
+try:
+    from requests.compat import chardet as charset_normalizer  # type: ignore
+except ImportError:  # pragma: no cover
+    import charset_normalizer
+
+
 is_darwin = sys.platform == "darwin"
 is_win32 = os.name == "nt"
 
 
+detect_encoding = charset_normalizer.detect
+
+
 __all__ = [
     "is_darwin",
     "is_win32",
+    "detect_encoding",
 ]
diff --git a/src/streamlink/utils/parse.py b/src/streamlink/utils/parse.py
index 8c9f79c8..17479b81 100644
--- a/src/streamlink/utils/parse.py
+++ b/src/streamlink/utils/parse.py
@@ -4,6 +4,7 @@ from urllib.parse import parse_qsl
 
 from lxml.etree import HTML, XML
 
+from streamlink.compat import detect_encoding
 from streamlink.plugin import PluginError
 
 
@@ -51,7 +52,21 @@ def parse_html(
      - Removes XML declarations of invalid XHTML5 documents
      - Wraps errors in custom exception with a snippet of the data in the message
     """
-    if isinstance(data, str) and data.lstrip().startswith("<?xml"):
+    # strip XML text declarations from XHTML5 documents which were incorrectly defined as HTML5
+    is_bytes = isinstance(data, bytes)
+    if data and data.lstrip()[:5].lower() == (b"<?xml" if is_bytes else "<?xml"):
+        if is_bytes:
+            # get the document's encoding using the "encoding" attribute value of the XML text declaration
+            match = re.match(rb"^\s*<\?xml\s.*?encoding=(?P<q>[\'\"])(?P<encoding>.+?)(?P=q).*?\?>", data, re.IGNORECASE)
+            if match:
+                encoding_value = detect_encoding(match["encoding"])["encoding"]
+                encoding = match["encoding"].decode(encoding_value)
+            else:
+                # no "encoding" attribute: try to figure out encoding from the document's content
+                encoding = detect_encoding(data)["encoding"]
+
+            data = data.decode(encoding)
+
         data = re.sub(r"^\s*<\?xml.+?\?>", "", data)
 
     return _parse(HTML, data, name, exception, schema, *args, **kwargs)


Test patch:
diff --git a/tests/utils/test_parse.py b/tests/utils/test_parse.py
index aedae7d4..69c16f28 100644
--- a/tests/utils/test_parse.py
+++ b/tests/utils/test_parse.py
@@ -74,31 +74,93 @@ class TestUtilsParse:
         assert actual.tag == expected.tag
         assert actual.attrib == expected.attrib
 
-    def test_parse_xml_encoding(self):
-        tree = parse_xml("""<?xml version="1.0" encoding="UTF-8"?><test>ä</test>""")
-        assert tree.xpath(".//text()") == ["ä"]
-        tree = parse_xml("""<test>ä</test>""")
-        assert tree.xpath(".//text()") == ["ä"]
-        tree = parse_xml(b"""<?xml version="1.0" encoding="UTF-8"?><test>\xC3\xA4</test>""")
-        assert tree.xpath(".//text()") == ["ä"]
-        tree = parse_xml(b"""<test>\xC3\xA4</test>""")
-        assert tree.xpath(".//text()") == ["ä"]
-
-    def test_parse_html_encoding(self):
-        tree = parse_html("""<!DOCTYPE html><html><head><meta charset="utf-8"/></head><body>ä</body></html>""")
-        assert tree.xpath(".//body/text()") == ["ä"]
-        tree = parse_html("""<!DOCTYPE html><html><body>ä</body></html>""")
-        assert tree.xpath(".//body/text()") == ["ä"]
-        tree = parse_html(b"""<!DOCTYPE html><html><meta charset="utf-8"/><body>\xC3\xA4</body></html>""")
-        assert tree.xpath(".//body/text()") == ["ä"]
-        tree = parse_html(b"""<!DOCTYPE html><html><body>\xC3\xA4</body></html>""")
-        assert tree.xpath(".//body/text()") == ["Ã¤"]
-
-    def test_parse_html_xhtml5(self):
-        tree = parse_html("""<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>ä?></body></html>""")
-        assert tree.xpath(".//body/text()") == ["ä?>"]
-        tree = parse_html(b"""<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>\xC3\xA4?></body></html>""")
-        assert tree.xpath(".//body/text()") == ["ä?>"]
+    @pytest.mark.parametrize(("content", "expected"), [
+        pytest.param(
+            """<?xml version="1.0" encoding="UTF-8"?><test>ä</test>""",
+            "ä",
+            id="string-utf-8",
+        ),
+        pytest.param(
+            """<test>ä</test>""",
+            "ä",
+            id="string-unknown",
+        ),
+        pytest.param(
+            b"""<?xml version="1.0" encoding="UTF-8"?><test>\xC3\xA4</test>""",
+            "ä",
+            id="bytes-utf-8",
+        ),
+        pytest.param(
+            b"""<?xml version="1.0" encoding="ISO-8859-1"?><test>\xE4</test>""",
+            "ä",
+            id="bytes-iso-8859-1",
+        ),
+        pytest.param(
+            b"""<test>\xC3\xA4</test>""",
+            "ä",
+            id="bytes-unknown",
+        ),
+    ])
+    def test_parse_xml_encoding(self, content, expected):
+        tree = parse_xml(content)
+        assert tree.xpath(".//text()") == [expected]
+
+    @pytest.mark.parametrize(("content", "expected"), [
+        pytest.param(
+            """<!DOCTYPE html><html><head><meta charset="utf-8"/></head><body>ä</body></html>""",
+            "ä",
+            id="string-utf-8",
+        ),
+        pytest.param(
+            """<!DOCTYPE html><html><body>ä</body></html>""",
+            "ä",
+            id="string-unknown",
+        ),
+        pytest.param(
+            b"""<!DOCTYPE html><html><head><meta charset="utf-8"/></head><body>\xC3\xA4</body></html>""",
+            "ä",
+            id="bytes-utf-8",
+        ),
+        pytest.param(
+            b"""<!DOCTYPE html><html><head><meta charset="ISO-8859-1"/></head><body>\xE4</body></html>""",
+            "ä",
+            id="bytes-iso-8859-1",
+        ),
+        pytest.param(
+            b"""<!DOCTYPE html><html><body>\xC3\xA4</body></html>""",
+            "Ã¤",
+            id="bytes-unknown",
+        ),
+    ])
+    def test_parse_html_encoding(self, content, expected):
+        tree = parse_html(content)
+        assert tree.xpath(".//body/text()") == [expected]
+
+    @pytest.mark.parametrize(("content", "expected"), [
+        pytest.param(
+            """<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>ä?></body></html>""",
+            "ä?>",
+            id="string",
+        ),
+        pytest.param(
+            b"""<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>\xC3\xA4?></body></html>""",
+            "ä?>",
+            id="bytes-utf-8",
+        ),
+        pytest.param(
+            b"""<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE html><html><body>\xE4?></body></html>""",
+            "ä?>",
+            id="bytes-iso-8859-1",
+        ),
+        pytest.param(
+            b"""<?xml version="1.0"?><!DOCTYPE html><html><body>\xC3\xA4?></body></html>""",
+            "ä?>",
+            id="bytes-unknown",
+        ),
+    ])
+    def test_parse_html_xhtml5(self, content, expected):
+        tree = parse_html(content)
+        assert tree.xpath(".//body/text()") == [expected]
 
     def test_parse_qsd(self):
         assert parse_qsd("test=1&foo=bar", schema=validate.Schema({"test": str, "foo": "bar"})) == {"test": "1", "foo": "bar"}