Repository: mozilla/bleach

Issue:
LinkifyFilter not working for URLs with ampersands
Example with bleach 2.1.3:

```
from bleach import Cleaner
from bleach.linkifier import LinkifyFilter

url1 = 'http://a.co?b=1&c=2'
url2 = 'http://a.co?b=1&amp;c=2'

cleaner = Cleaner(filters=[LinkifyFilter])
cleaner.clean(url1)
cleaner.clean(url2)
```

Both result in `<a href="http://a.co?b=1">http://a.co?b=1</a>&amp;c=2`


Discussion:

Patch:
diff --git a/bleach/linkifier.py b/bleach/linkifier.py
index 6394c03..5d815f8 100644
--- a/bleach/linkifier.py
+++ b/bleach/linkifier.py
@@ -499,13 +499,11 @@ class LinkifyFilter(html5lib_shim.Filter):
                     # the tokens we're going to yield
                     in_a = False
                     token_buffer = []
-                    continue
-
                 else:
                     token_buffer.append(token)
-                    continue
+                continue
 
-            elif token['type'] in ['StartTag', 'EmptyTag']:
+            if token['type'] in ['StartTag', 'EmptyTag']:
                 if token['name'] in self.skip_tags:
                     # Skip tags start a "special mode" where we don't linkify
                     # anything until the end tag.
diff --git a/bleach/sanitizer.py b/bleach/sanitizer.py
index 262915a..9ba4c57 100644
--- a/bleach/sanitizer.py
+++ b/bleach/sanitizer.py
@@ -267,8 +267,8 @@ class BleachSanitizerFilter(html5lib_shim.SanitizerFilter):
 
         return super(BleachSanitizerFilter, self).__init__(source, **kwargs)
 
-    def __iter__(self):
-        for token in html5lib_shim.Filter.__iter__(self):
+    def sanitize_stream(self, token_iterator):
+        for token in token_iterator:
             ret = self.sanitize_token(token)
 
             if not ret:
@@ -280,6 +280,40 @@ class BleachSanitizerFilter(html5lib_shim.SanitizerFilter):
             else:
                 yield ret
 
+    def merge_characters(self, token_iterator):
+        """Merge consecutive Characters tokens in a stream"""
+        characters_buffer = []
+
+        for token in token_iterator:
+            if characters_buffer:
+                if token['type'] == 'Characters':
+                    characters_buffer.append(token)
+                    continue
+                else:
+                    # Merge all the characters tokens together into one and then
+                    # operate on it.
+                    new_token = {
+                        'data': ''.join([char_token['data'] for char_token in characters_buffer]),
+                        'type': 'Characters'
+                    }
+                    characters_buffer = []
+                    yield new_token
+
+            elif token['type'] == 'Characters':
+                characters_buffer.append(token)
+                continue
+
+            yield token
+
+        new_token = {
+            'data': ''.join([char_token['data'] for char_token in characters_buffer]),
+            'type': 'Characters'
+        }
+        yield new_token
+
+    def __iter__(self):
+        return self.merge_characters(self.sanitize_stream(html5lib_shim.Filter.__iter__(self)))
+
     def sanitize_token(self, token):
         """Sanitize a token either by HTML-encoding or dropping.
 


Test patch:
diff --git a/tests/test_clean.py b/tests/test_clean.py
index b543cdf..5322767 100644
--- a/tests/test_clean.py
+++ b/tests/test_clean.py
@@ -58,6 +58,7 @@ def test_html_is_lowercased():
         '<a href="http://example.com">foo</a>'
     )
 
+
 def test_invalid_uri_does_not_raise_error():
     assert clean('<a href="http://example.com]">text</a>') == '<a>text</a>'
 
diff --git a/tests/test_linkify.py b/tests/test_linkify.py
index 876cb84..eeea3e3 100644
--- a/tests/test_linkify.py
+++ b/tests/test_linkify.py
@@ -4,7 +4,8 @@ import pytest
 from six.moves.urllib_parse import quote_plus
 
 from bleach import linkify, DEFAULT_CALLBACKS as DC
-from bleach.linkifier import Linker
+from bleach.linkifier import Linker, LinkifyFilter
+from bleach.sanitizer import Cleaner
 
 
 def test_empty():
@@ -656,3 +657,20 @@ class TestLinkify:
 
         with pytest.raises(TypeError):
             linkify(no_type)
+
+
+@pytest.mark.parametrize('text, expected', [
+    ('abc', 'abc'),
+    ('example.com', '<a href="http://example.com">example.com</a>'),
+    (
+        'http://example.com?b=1&c=2',
+        '<a href="http://example.com?b=1&amp;c=2">http://example.com?b=1&amp;c=2</a>'
+    ),
+    (
+        'link: https://example.com/watch#anchor',
+        'link: <a href="https://example.com/watch#anchor">https://example.com/watch#anchor</a>'
+    )
+])
+def test_linkify_filter(text, expected):
+    cleaner = Cleaner(filters=[LinkifyFilter])
+    assert cleaner.clean(text) == expected