Repository: mozilla/bleach

Issue:
linkify feature request - extend skip_pre
Hi, 

Can the skip_pre be extended to support a list of html tags? 
IMHO, `code` tag should be skipped as well.

https://github.com/mozilla/bleach/blob/master/bleach/__init__.py#L318

Thx


Discussion:

Patch:
diff --git a/CHANGES b/CHANGES
index 79f56a9..050f4fc 100644
--- a/CHANGES
+++ b/CHANGES
@@ -60,6 +60,23 @@ Version 2.0 (in development)
 
   All linkify filters will need to be updated.
 
+* ``bleach.linkify`` and friends had a ``skip_pre`` argument--that's been
+  replaced with a more general ``skip_tags`` argument.
+
+  Before, you might do::
+
+      bleach.linkify(some_text, skip_pre=True)
+
+  The equivalent with Bleach 2.0 is::
+
+      bleach.linkify(some_text, skip_tags=['pre'])
+
+  You can skip other tags, too, like ``style`` or ``script`` or other places
+  where you don't want linkification happening.
+
+  All uses of linkify that use ``skip_pre`` will need to be updated.
+
+
 **Changes**
 
 * Supports Python 3.6.
diff --git a/README.rst b/README.rst
index 403ff9b..08dd886 100644
--- a/README.rst
+++ b/README.rst
@@ -8,7 +8,7 @@ Bleach
 .. image:: https://badge.fury.io/py/bleach.svg
    :target: http://badge.fury.io/py/bleach
 
-Bleach is a whitelist-based HTML sanitizing library that escapes or strips
+Bleach is a allowed-list-based HTML sanitizing library that escapes or strips
 markup and attributes.
 
 Bleach can also linkify text safely, applying filters that Django's ``urlize``
diff --git a/bleach/__init__.py b/bleach/__init__.py
index 07b5075..a231f13 100644
--- a/bleach/__init__.py
+++ b/bleach/__init__.py
@@ -47,16 +47,16 @@ def clean(text, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES,
 
     :arg str text: the text to clean
 
-    :arg list tags: whitelist of allowed tags; defaults to
+    :arg list tags: allowed list of tags; defaults to
         ``bleach.ALLOWED_TAGS``
 
-    :arg dict attributes: whitelist of allowed attributes; defaults to
-        ``bleach.ALLOWED_ATTRIBUTES``
+    :arg dict attributes: allowed attributes; can be a callable, list or dict;
+        defaults to ``bleach.ALLOWED_ATTRIBUTES``
 
-    :arg list styles: whitelist of allowed css; defaults to
+    :arg list styles: allowed list of css styles; defaults to
         ``bleach.ALLOWED_STYLES``
 
-    :arg list protocols: whitelist of allowed protocols for links; defaults
+    :arg list protocols: allowed list of protocols for links; defaults
         to ``bleach.ALLOWED_PROTOCOLS``
 
     :arg bool strip: whether or not to strip disallowed elements
@@ -77,7 +77,7 @@ def clean(text, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES,
     return cleaner.clean(text)
 
 
-def linkify(text, callbacks=DEFAULT_CALLBACKS, skip_pre=False, parse_email=False):
+def linkify(text, callbacks=DEFAULT_CALLBACKS, skip_tags=None, parse_email=False):
     """Convert URL-like strings in an HTML fragment to links
 
     This function converts strings that look like URLs, domain names and email
@@ -106,7 +106,9 @@ def linkify(text, callbacks=DEFAULT_CALLBACKS, skip_pre=False, parse_email=False
 
     :arg list callbacks: list of callbacks to run when adjusting tag attributes
 
-    :arg bool skip_pre: whether or not to skip linkifying text in a ``pre`` tag
+    :arg list skip_tags: list of tags that you don't want to linkify the
+        contents of; for example, you could set this to ``['pre']`` to skip
+        linkifying contents of ``pre`` tags
 
     :arg bool parse_email: whether or not to linkify email addresses
 
@@ -115,7 +117,7 @@ def linkify(text, callbacks=DEFAULT_CALLBACKS, skip_pre=False, parse_email=False
     """
     linker = Linker(
         callbacks=callbacks,
-        skip_pre=skip_pre,
+        skip_tags=skip_tags,
         parse_email=parse_email
     )
     return linker.linkify(text)
diff --git a/bleach/linkifier.py b/bleach/linkifier.py
index 92351be..6103e81 100644
--- a/bleach/linkifier.py
+++ b/bleach/linkifier.py
@@ -74,10 +74,40 @@ EMAIL_RE = re.compile(
 
 
 class Linker(object):
-    def __init__(self, callbacks=DEFAULT_CALLBACKS, skip_pre=False, parse_email=False,
+    """Convert URL-like strings in an HTML fragment to links
+
+    This function converts strings that look like URLs, domain names and email
+    addresses in text that may be an HTML fragment to links, while preserving:
+
+    1. links already in the string
+    2. urls found in attributes
+    3. email addresses
+
+    linkify does a best-effort approach and tries to recover from bad
+    situations due to crazy text.
+
+    """
+    def __init__(self, callbacks=DEFAULT_CALLBACKS, skip_tags=None, parse_email=False,
                  url_re=URL_RE, email_re=EMAIL_RE):
+        """Creates a Linker instance
+
+        :arg list callbacks: list of callbacks to run when adjusting tag attributes
+
+        :arg list skip_tags: list of tags that you don't want to linkify the
+            contents of; for example, you could set this to ``['pre']`` to skip
+            linkifying contents of ``pre`` tags
+
+        :arg bool parse_email: whether or not to linkify email addresses
+
+        :arg re url_re: url matching regex
+
+        :arg email_re: email matching regex
+
+        :returns: linkified text as unicode
+
+        """
         self.callbacks = callbacks
-        self.skip_pre = skip_pre
+        self.skip_tags = skip_tags
         self.parse_email = parse_email
         self.url_re = url_re
         self.email_re = email_re
@@ -105,7 +135,7 @@ class Linker(object):
         filtered = LinkifyFilter(
             source=self.walker(dom),
             callbacks=self.callbacks,
-            skip_pre=self.skip_pre,
+            skip_tags=self.skip_tags,
             parse_email=self.parse_email,
             url_re=self.url_re,
             email_re=self.email_re,
@@ -126,12 +156,31 @@ class LinkifyFilter(Filter):
     This filter can be used anywhere html5lib filters can be used.
 
     """
-    def __init__(self, source, callbacks=None, skip_pre=False, parse_email=False,
+    def __init__(self, source, callbacks=None, skip_tags=None, parse_email=False,
                  url_re=URL_RE, email_re=EMAIL_RE):
+        """Creates a LinkifyFilter instance
+
+        :arg TreeWalker source: stream
+
+        :arg list callbacks: list of callbacks to run when adjusting tag attributes
+
+        :arg list skip_tags: list of tags that you don't want to linkify the
+            contents of; for example, you could set this to ``['pre']`` to skip
+            linkifying contents of ``pre`` tags
+
+        :arg bool parse_email: whether or not to linkify email addresses
+
+        :arg re url_re: url matching regex
+
+        :arg email_re: email matching regex
+
+        :returns: linkified text as unicode
+
+        """
         super(LinkifyFilter, self).__init__(source)
 
         self.callbacks = callbacks or []
-        self.skip_pre = skip_pre
+        self.skip_tags = skip_tags or []
         self.parse_email = parse_email
 
         self.url_re = url_re
@@ -140,9 +189,15 @@ class LinkifyFilter(Filter):
     def apply_callbacks(self, attrs, is_new):
         """Given an attrs dict and an is_new bool, runs through callbacks
 
-        Callbacks can return an adjusted attrs dict or None. In the case of
-        None, we stop going through callbacks and return that and the link gets
-        dropped.
+        Callbacks can return an adjusted attrs dict or ``None``. In the case of
+        ``None``, we stop going through callbacks and return that and the link
+        gets dropped.
+
+        :arg dict attrs: map of ``(namespace, name)`` -> ``value``
+
+        :arg bool is_new: whether or not this link was added by linkify
+
+        :returns: adjusted attrs dict or ``None``
 
         """
         for cb in self.callbacks:
@@ -399,7 +454,7 @@ class LinkifyFilter(Filter):
 
     def __iter__(self):
         in_a = False
-        in_pre = False
+        in_skip_tag = None
 
         token_buffer = []
 
@@ -425,10 +480,10 @@ class LinkifyFilter(Filter):
                     continue
 
             elif token['type'] in ['StartTag', 'EmptyTag']:
-                if token['name'] == 'pre' and self.skip_pre:
-                    # The "pre" tag starts a "special mode" where we don't linkify
-                    # anything.
-                    in_pre = True
+                if token['name'] in self.skip_tags:
+                    # Skip tags start a "special mode" where we don't linkify
+                    # anything until the end tag.
+                    in_skip_tag = token['name']
 
                 elif token['name'] == 'a':
                     # The "a" tag is special--we switch to a slurp mode and
@@ -441,13 +496,13 @@ class LinkifyFilter(Filter):
                     # yet
                     continue
 
-            elif in_pre and self.skip_pre:
+            elif in_skip_tag and self.skip_tags:
                 # NOTE(willkg): We put this clause here since in_a and
                 # switching in and out of in_a takes precedence.
-                if token['type'] == 'EndTag' and token['name'] == 'pre':
-                    in_pre = False
+                if token['type'] == 'EndTag' and token['name'] == in_skip_tag:
+                    in_skip_tag = None
 
-            elif not in_a and not in_pre and token['type'] == 'Characters':
+            elif not in_a and not in_skip_tag and token['type'] == 'Characters':
                 new_stream = iter([token])
                 if self.parse_email:
                     new_stream = self.handle_email_addresses(new_stream)
diff --git a/bleach/sanitizer.py b/bleach/sanitizer.py
index 1223e79..b5c2fe9 100644
--- a/bleach/sanitizer.py
+++ b/bleach/sanitizer.py
@@ -63,16 +63,16 @@ class Cleaner(object):
                  strip_comments=True, filters=None):
         """Initializes a Cleaner
 
-        :arg tags: whitelist of allowed tags; defaults to
+        :arg list tags: allowed list of tags; defaults to
             ``bleach.ALLOWED_TAGS``
 
-        :arg attributes: whitelist of allowed attributes; defaults to
-            ``bleach.ALLOWED_ATTRIBUTES``
+        :arg dict attributes: allowed attributes; can be a callable, list or dict;
+            defaults to ``bleach.ALLOWED_ATTRIBUTES``
 
-        :arg styles: whitelist of allowed css; defaults to
+        :arg list styles: allowed list of css styles; defaults to
             ``bleach.ALLOWED_STYLES``
 
-        :arg protocols: whitelist of allowed protocols for links; defaults
+        :arg list protocols: allowed list of protocols for links; defaults
             to ``bleach.ALLOWED_PROTOCOLS``
 
         :arg strip: whether or not to strip disallowed elements
@@ -196,7 +196,27 @@ class BleachSanitizerFilter(sanitizer.Filter):
     def __init__(self, source, attributes=ALLOWED_ATTRIBUTES,
                  strip_disallowed_elements=False, strip_html_comments=True,
                  **kwargs):
+        """Creates a BleachSanitizerFilter instance
 
+        :arg Treewalker source: stream
+
+        :arg list tags: allowed list of tags; defaults to
+            ``bleach.ALLOWED_TAGS``
+
+        :arg dict attributes: allowed attributes; can be a callable, list or dict;
+            defaults to ``bleach.ALLOWED_ATTRIBUTES``
+
+        :arg list styles: allowed list of css styles; defaults to
+            ``bleach.ALLOWED_STYLES``
+
+        :arg list protocols: allowed list of protocols for links; defaults
+            to ``bleach.ALLOWED_PROTOCOLS``
+
+        :arg strip_disallowed_elements: whether or not to strip disallowed elements
+
+        :arg strip_html_comments: whether or not to strip HTML comments
+
+        """
         self.attr_filter = attribute_filter_factory(attributes)
 
         self.strip_disallowed_elements = strip_disallowed_elements
diff --git a/docs/goals.rst b/docs/goals.rst
index 632c222..015bc56 100644
--- a/docs/goals.rst
+++ b/docs/goals.rst
@@ -13,15 +13,15 @@ Goals
 =====
 
 
-Always take a whitelist-based approach
---------------------------------------
+Always take a allowed-list-based approach
+-----------------------------------------
 
-Bleach should always take a whitelist-based approach to allowing any kind of
-content or markup. Blacklisting is error-prone and not future proof.
+Bleach should always take a allowed-list-based approach to markup filtering.
+Specifying disallowed lists is error-prone and not future proof.
 
 For example, you should have to opt-in to allowing the ``onclick`` attribute,
-not blacklist all the other ``on*`` attributes. Future versions of HTML may add
-new event handlers, like ``ontouch``, that old blacklists would not prevent.
+not opt-out of all the other ``on*`` attributes. Future versions of HTML may add
+new event handlers, like ``ontouch``, that old disallow would not prevent.
 
 
 Main goal is to sanitize input of malicious content
@@ -39,8 +39,8 @@ Examples might include:
 
 These examples, and others, are traditionally prone to security issues like XSS
 or other script injection, or annoying issues like unclosed tags and invalid
-markup. Bleach will take a proactive, whitelist-only approach to allowing HTML
-content, and will use the HTML5 parsing algorithm to handle invalid markup.
+markup. Bleach will take a proactive, allowed-list-only approach to allowing
+HTML content, and will use the HTML5 parsing algorithm to handle invalid markup.
 
 See the :ref:`chapter on clean() <clean-chapter>` for more info.
 
@@ -52,7 +52,7 @@ The secondary goal of Bleach is to provide a mechanism for finding or altering
 links (``<a>`` tags with ``href`` attributes, or things that look like URLs or
 email addresses) in text.
 
-While Bleach itself will always operate on a whitelist-based security model,
+While Bleach itself will always operate on a allowed-list-based security model,
 the :ref:`linkify() method <linkify-chapter>` is flexible enough to allow the
 creation, alteration, and removal of links based on an extremely wide range of
 use cases.
@@ -69,8 +69,8 @@ Sanitize complete HTML documents
 --------------------------------
 
 Once you're creating whole documents, you have to allow so many tags that a
-blacklist approach (e.g. forbidding ``<script>`` or ``<object>``) may be more
-appropriate.
+disallow-list approach (e.g. forbidding ``<script>`` or ``<object>``) may be
+more appropriate.
 
 
 Remove all HTML or transforming content for some non-web-page purpose
diff --git a/docs/linkify.rst b/docs/linkify.rst
index a468830..71c2e7f 100644
--- a/docs/linkify.rst
+++ b/docs/linkify.rst
@@ -248,15 +248,13 @@ For example, this removes any ``mailto:`` links:
    u'mail janet!'
 
 
-Skipping links in pre blocks (``skip_pre``)
-===========================================
+Skipping links in specified tag blocks (``skip_tags``)
+======================================================
 
 ``<pre>`` tags are often special, literal sections. If you don't want to create
-any new links within a ``<pre>`` section, pass ``skip_pre=True``.
+any new links within a ``<pre>`` section, pass ``skip_tags=['pre']``.
 
-.. note::
-   Though new links will not be created, existing links created with ``<a>``
-   tags will still be passed through all the callbacks.
+This works for ``code``, ``div`` and any other blocks you want to skip over.
 
 
 Linkifying email addresses (``parse_email``)
@@ -281,7 +279,7 @@ instance.
 
    >>> from bleach.linkifier import Linker
 
-   >>> linker = Linker(skip_pre=True)
+   >>> linker = Linker(skip_tags=['pre'])
    >>> linker.linkify('a b c http://example.com d e f')
    u'a b c <a href="http://example.com" rel="nofollow">http://example.com</a> d e f'
 
@@ -328,7 +326,7 @@ And passing parameters to ``LinkifyFilter``:
 
    >>> cleaner = Cleaner(
    ...     tags=['pre'],
-   ...     filters=[partial(LinkifyFilter, skip_pre=True)]
+   ...     filters=[partial(LinkifyFilter, skip_tags=['pre'])]
    ... )
    ...
    >>> cleaner.clean('<pre>http://example.com</pre>')
diff --git a/setup.py b/setup.py
index 39fbb37..bf95efc 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@ if 'test' in sys.argv:
 tests_require = [
     'pytest>=3.0.0',
 ]
-    
+
 install_requires = [
     'six',
     # >= 8 9s because of breaking API change
@@ -42,7 +42,7 @@ def get_version():
 setup(
     name='bleach',
     version=get_version(),
-    description='An easy whitelist-based HTML-sanitizing tool.',
+    description='An easy safelist-based HTML-sanitizing tool.',
     long_description=get_long_desc(),
     maintainer='Will Kahn-Greene',
     url='http://github.com/mozilla/bleach',


Test patch:
diff --git a/tests/test_links.py b/tests/test_links.py
index 99b30b8..a3cb973 100644
--- a/tests/test_links.py
+++ b/tests/test_links.py
@@ -357,24 +357,24 @@ def test_unsafe_url():
     )
 
 
-def test_skip_pre():
-    """Skip linkification in <pre> tags."""
+def test_skip_tags():
+    """Skip linkification in skip tags"""
     simple = 'http://xx.com <pre>http://xx.com</pre>'
     linked = ('<a href="http://xx.com" rel="nofollow">http://xx.com</a> '
               '<pre>http://xx.com</pre>')
     all_linked = ('<a href="http://xx.com" rel="nofollow">http://xx.com</a> '
                   '<pre><a href="http://xx.com" rel="nofollow">http://xx.com'
                   '</a></pre>')
-    assert linkify(simple, skip_pre=True) == linked
+    assert linkify(simple, skip_tags=['pre']) == linked
     assert linkify(simple) == all_linked
 
     already_linked = '<pre><a href="http://xx.com">xx</a></pre>'
     nofollowed = '<pre><a href="http://xx.com" rel="nofollow">xx</a></pre>'
     assert linkify(already_linked) == nofollowed
-    assert linkify(already_linked, skip_pre=True) == nofollowed
+    assert linkify(already_linked, skip_tags=['pre']) == nofollowed
 
     assert (
-        linkify('<pre><code>http://example.com</code></pre>http://example.com', skip_pre=True) ==
+        linkify('<pre><code>http://example.com</code></pre>http://example.com', skip_tags=['pre']) ==
         (
             '<pre><code>http://example.com</code></pre>'
             '<a href="http://example.com" rel="nofollow">http://example.com</a>'