Repository: mozilla/bleach

Issue:
AttributeError: 'InputStreamWithMemory' object has no attribute 'charEncoding'
Using this code:

```
import bleach

TEXT = """\
<html>
<head>
    <meta charset="utf-8">
</head>
<body>
</body>
</html>
"""
print(bleach.clean(TEXT, tags=['html', 'head', 'meta', 'body']))
```

we get this traceback:

```
Traceback (most recent call last):
  File "s.py", line 12, in <module>
    print(bleach.clean(TEXT, tags=['html', 'head', 'meta', 'body']))
  File "/home/willkg/mozilla/bleach/bleach/__init__.py", line 84, in clean
    return cleaner.clean(text)
  File "/home/willkg/mozilla/bleach/bleach/sanitizer.py", line 169, in clean
    dom = self.parser.parseFragment(text)
  File "/home/willkg/mozilla/bleach/bleach/_vendor/html5lib/html5parser.py", line 317, in parseFragment
    self._parse(stream, True, *args, **kwargs)
  File "/home/willkg/mozilla/bleach/bleach/html5lib_shim.py", line 385, in _parse
    self.mainLoop()
  File "/home/willkg/mozilla/bleach/bleach/_vendor/html5lib/html5parser.py", line 241, in mainLoop
    new_token = phase.processStartTag(new_token)
  File "/home/willkg/mozilla/bleach/bleach/_vendor/html5lib/html5parser.py", line 472, in processStartTag
    return self.startTagHandler[token["name"]](token)
  File "/home/willkg/mozilla/bleach/bleach/_vendor/html5lib/html5parser.py", line 1067, in startTagProcessInHead
    return self.parser.phases["inHead"].processStartTag(token)
  File "/home/willkg/mozilla/bleach/bleach/_vendor/html5lib/html5parser.py", line 472, in processStartTag
    return self.startTagHandler[token["name"]](token)
  File "/home/willkg/mozilla/bleach/bleach/_vendor/html5lib/html5parser.py", line 745, in startTagMeta
    if self.parser.tokenizer.stream.charEncoding[1] == "tentative":
AttributeError: 'InputStreamWithMemory' object has no attribute 'charEncoding'
```

This is with Bleach from master tip (3.0.3.dev0) and Python 2.7.15.

This issue covers fixing that.

Comments:

Patch:
diff --git a/CHANGES b/CHANGES
index eb3dc23..4fe065e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,7 +1,7 @@
 Bleach changes
 ==============
 
-Version 3.0.3 (In development)
+Version 3.1.0 (In development)
 ------------------------------
 
 **Security fixes**
@@ -25,6 +25,12 @@ None
 * Fix cases where attribute names could have invalid characters in them.
   (#419)
 
+* Fix problems with ``LinkifyFilter`` not being able to match links
+  across ``&amp;``. (#422)
+
+* Fix ``InputStreamWithMemory`` when the ``BleachHTMLParser`` is
+  parsing ``meta`` tags. (#431)
+
 
 Version 3.0.2 (October 11th, 2018)
 ----------------------------------
diff --git a/bleach/__init__.py b/bleach/__init__.py
index a6445d0..6249bf8 100644
--- a/bleach/__init__.py
+++ b/bleach/__init__.py
@@ -20,7 +20,7 @@ from bleach.sanitizer import (
 # yyyymmdd
 __releasedate__ = ''
 # x.y.z or x.y.z.dev0 -- semver
-__version__ = '3.0.3.dev0'
+__version__ = '3.1.0.dev0'
 VERSION = parse_version(__version__)
 
 
diff --git a/bleach/html5lib_shim.py b/bleach/html5lib_shim.py
index 8887667..25e3e95 100644
--- a/bleach/html5lib_shim.py
+++ b/bleach/html5lib_shim.py
@@ -181,6 +181,14 @@ class InputStreamWithMemory(object):
     def errors(self):
         return self._inner_stream.errors
 
+    @property
+    def charEncoding(self):
+        return self._inner_stream.charEncoding
+
+    @property
+    def changeEncoding(self):
+        return self._inner_stream.changeEncoding
+
     def char(self):
         c = self._inner_stream.char()
         # char() can return None if EOF, so ignore that


Test patch:
diff --git a/tests/test_html5lib_shim.py b/tests/test_html5lib_shim.py
index 5712d33..ce15de7 100644
--- a/tests/test_html5lib_shim.py
+++ b/tests/test_html5lib_shim.py
@@ -80,3 +80,65 @@ def test_serializer(data, expected):
     serialized = serializer.render(walker(dom))
 
     assert serialized == expected
+
+
+@pytest.mark.parametrize('parser_args, data, expected', [
+    # Make sure InputStreamWithMemory has charEncoding and changeEncoding
+    (
+        {},
+        '<meta charset="utf-8">',
+        '<meta charset="utf-8">'
+    ),
+    # Handle consume entities False--all entities are passed along and then
+    # escaped when serialized
+    (
+        {'consume_entities': False},
+        'text &amp;&gt;&quot;',
+        'text &amp;amp;&amp;gt;&amp;quot;'
+    ),
+    # Handle consume entities True--all entities are consumed and converted
+    # to their character equivalents and then &, <, and > are escaped when
+    # serialized
+    (
+        {'consume_entities': True},
+        'text &amp;&gt;&quot;',
+        'text &amp;&gt;"'
+    ),
+    # Test that "invalid-character-in-attribute-name" errors in tokenizing
+    # result in attributes with invalid names getting dropped
+    (
+        {},
+        '<a href="http://example.com"">',
+        '<a href="http://example.com"></a>'
+    ),
+    (
+        {},
+        '<a href=\'http://example.com\'\'>',
+        '<a href="http://example.com"></a>'
+    )
+])
+def test_bleach_html_parser(parser_args, data, expected):
+    args = {
+        'tags': None,
+        'strip': True,
+        'consume_entities': True
+    }
+    args.update(parser_args)
+
+    # Build a parser, walker, and serializer just like we do in clean()
+    parser = html5lib_shim.BleachHTMLParser(**args)
+    walker = html5lib_shim.getTreeWalker('etree')
+    serializer = html5lib_shim.BleachHTMLSerializer(
+        quote_attr_values='always',
+        omit_optional_tags=False,
+        escape_lt_in_attrs=True,
+        resolve_entities=False,
+        sanitize=False,
+        alphabetical_attributes=False,
+    )
+
+    # Parse, walk, and then serialize the output
+    dom = parser.parseFragment(data)
+    serialized = serializer.render(walker(dom))
+
+    assert serialized == expected
diff --git a/tests/test_linkify.py b/tests/test_linkify.py
index ab1c513..f121189 100644
--- a/tests/test_linkify.py
+++ b/tests/test_linkify.py
@@ -69,17 +69,6 @@ def test_mangle_text():
     )
 
 
-def test_invalid_attribute_names():
-    """Test that "invalid-character-in-attribute-name" errors in tokenizing
-    result in attributes with invalid names get dropped.
-
-    """
-    assert (
-        linkify('<a href="http://example.com/"">') ==
-        '<a href="http://example.com/" rel="nofollow"></a>'
-    )
-
-
 @pytest.mark.parametrize('data,parse_email,expected', [
     (
         'a james@example.com mailto',