Repository: fedora-infra/bodhi

Issue:
Markdown linkified links without http:// produce relative href= attr
if you add a link to a markdown field (update description or a comment) without the http:// (or an email address), the custom fedora-markdown code linkifies this, but produces a relative link. For example, including the string `the fedora project website is getfedora.org` produces the following html markup:

```
the fedora project website is <a href="getfedora.org">getfedora.org</a>
```
which tries to link to `http://bodhi.fedoraproject.org/getfedora.org`




Discussion:

Resolving patch:
diff --git a/bodhi/server/ffmarkdown.py b/bodhi/server/ffmarkdown.py
--- a/bodhi/server/ffmarkdown.py
+++ b/bodhi/server/ffmarkdown.py
@@ -50,15 +50,7 @@ def bug_url(tracker, idx):
 def inject():
     """ Hack out python-markdown to do the autolinking that we want. """
 
-    # First, make it so that bare links get automatically linkified.
-    markdown.inlinepatterns.AUTOLINK_RE = '(%s)' % '|'.join([
-        r'<(?:f|ht)tps?://[^>]*>',
-        r'\b(?:f|ht)tps?://[^)<>\s]+[^.,)<>\s]',
-        r'\bwww\.[^)<>\s]+[^.,)<>\s]',
-        r'[^(<\s]+\.(?:com|net|org)\b',
-    ])
-
-    # Second, build some Pattern objects for @mentions, #bugs, etc...
+    # build some Pattern objects for @mentions, #bugs, etc...
     class MentionPattern(markdown.inlinepatterns.Pattern):
         def handleMatch(self, m):
             el = markdown.util.etree.Element("a")
@@ -81,14 +73,14 @@ def handleMatch(self, m):
             el.text = idx
             return el
 
-    MENTION_RE = r'(@\w+)'
+    MENTION_RE = r'(?<!\S)(@\w+)'
     BUGZILLA_RE = r'([\S]+)(#[0-9]{5,})'
 
     class SurroundProcessor(markdown.postprocessors.Postprocessor):
         def run(self, text):
             return "<div class='markdown'>" + text + "</div>"
 
-    # Lastly, monkey-patch the build_inlinepatterns func to insert our patterns
+    # monkey-patch the build_inlinepatterns func to insert our patterns
     original_pattern_builder = markdown.build_inlinepatterns
 
     def extended_pattern_builder(md_instance, **kwargs):
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -296,12 +296,47 @@ def markup(context, text):
         basestring: HTML representation of the markdown text
     """
 
+    # determine the major component of the bleach version installed.
+    # this is similar to the approach that Pagure uses to determine the bleach version
+    # https://pagure.io/pagure/pull-request/2269#request_diff
+    bleach_major_v = int(bleach.__version__.split('.')[0])
+
+    # the only difference in the bleach API that we use between v1 and v2 is
+    # the formatting of the attributes parameter. Bleach 1 only allowed you
+    # to specify attributes to be whitelisted for all whitelisted tags.
+    # Bleach 2 requires you to specify the list of attributes whitelisted for
+    # specific tags.
+    if bleach_major_v >= 2:
+        markdown_attrs = {
+            "img": ["src", "alt", "title"],
+            "a": ["href", "alt", "title"],
+            "div": ["class"],
+        }
+    else:
+        markdown_attrs = [
+            "src", "href", "alt", "title", "class"
+        ]
+
+    markdown_tags = [
+        "h1", "h2", "h3", "h4", "h5", "h6",
+        "b", "i", "strong", "em", "tt",
+        "p", "br",
+        "span", "div", "blockquote", "code", "hr", "pre",
+        "ul", "ol", "li", "dd", "dt",
+        "img",
+        "a",
+    ]
+
+    markdown_text = markdown.markdown(text, extensions=['markdown.extensions.fenced_code'])
+
+    # previously, we linkified text in ffmarkdown.py, but this was causing issues like #1721
+    # so now we use the bleach linkifier to do this for us.
+    markdown_text = bleach.linkify(markdown_text, parse_email=True)
+
     # previously, we used the Safe Mode in python-markdown to strip all HTML
     # tags. Safe Mode is deprecated, so we now use Bleach to sanitize all HTML
-    # tags before running it through the markdown parser
-    sanitized_text = bleach.clean(text, tags=[])
-
-    return markdown.markdown(sanitized_text, extensions=['markdown.extensions.fenced_code'])
+    # tags after running it through the markdown parser
+    return bleach.clean(markdown_text, tags=markdown_tags, attributes=markdown_attrs)
 
 
 def status2html(context, status):


Test patch:
diff --git a/bodhi/tests/server/functional/test_generic.py b/bodhi/tests/server/functional/test_generic.py
--- a/bodhi/tests/server/functional/test_generic.py
+++ b/bodhi/tests/server/functional/test_generic.py
@@ -111,18 +111,49 @@ def test_markdown(self):
         res = self.app.get('/markdown', {'text': 'wat'}, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
-            "<p>wat</p>"
-            "</div>"
+            '<div class="markdown">'
+            '<p>wat</p>'
+            '</div>'
         )
 
-    def test_markdown_with_html(self):
-        res = self.app.get('/markdown', {'text': '<b>bold</b>'}, status=200)
+    def test_markdown_with_html_blocked_tag(self):
+        res = self.app.get('/markdown', {'text': '<script>bold</script>'}, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
-            '<p>&lt;b&gt;bold&lt;/b&gt;</p>'
-            "</div>"
+            '<div class="markdown">'
+            '&lt;script&gt;bold&lt;/script&gt;\n'
+            '</div>'
+        )
+
+    def test_markdown_with_html_whitelisted_tag(self):
+        res = self.app.get('/markdown', {'text': '<pre>sudo dnf install pants</pre>'}, status=200)
+        self.assertEquals(
+            res.json_body['html'],
+            '<div class="markdown">'
+            '<pre>sudo dnf install pants</pre>\n'
+            '</div>'
+        )
+
+    def test_markdown_with_html_blocked_attribute(self):
+        res = self.app.get('/markdown',
+                           {'text': '<b onclick="alert(\'pants\')">bold</b>'},
+                           status=200)
+        self.assertEquals(
+            res.json_body['html'],
+            '<div class="markdown">'
+            '<p><b>bold</b></p>'
+            '</div>'
+        )
+
+    def test_markdown_with_html_whitelisted_attribute(self):
+        res = self.app.get('/markdown',
+                           {'text': '<img src="pants.png">'},
+                           status=200)
+        self.assertEquals(
+            res.json_body['html'],
+            '<div class="markdown">'
+            '<p><img src="pants.png"></p>'
+            '</div>'
         )
 
     def test_markdown_with_mention(self):
@@ -131,10 +162,10 @@ def test_markdown_with_mention(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<p>my <a href="http://localhost/users/colleague">@colleague</a>'
             ' is distinguished</p>'
-            "</div>"
+            '</div>'
         )
 
     def test_markdown_with_mention_at_start(self):
@@ -143,7 +174,7 @@ def test_markdown_with_mention_at_start(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<p><a href="http://localhost/users/pingou">@pingou</a>'
             ' is on it</p>'
             "</div>"
@@ -155,7 +186,7 @@ def test_markdown_with_mention_at_start_with_comma(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<p><a href="http://localhost/users/kevin">@kevin</a>'
             ', thanks for that</p>'
             "</div>"
@@ -167,7 +198,7 @@ def test_markdown_with_mention_with_numbers(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<p>I vote for '
             '<a href="http://localhost/users/number80">@number80</a></p>'
             "</div>"
@@ -179,7 +210,7 @@ def test_markdown_with_unprefixed_bugzilla(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<p>Crazy.  #12345 is still busted.</p>'
             "</div>"
         )
@@ -190,7 +221,7 @@ def test_markdown_with_prefixed_bugzilla(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<p>Crazy.  '
             '<a href="https://bugzilla.redhat.com/show_bug.cgi?id=12345">'
             '#12345</a> is still busted.</p>'
@@ -203,7 +234,7 @@ def test_markdown_with_unknown_prefixed_bugzilla(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<p>Crazy.  upstream#12345 is still busted.</p>'
             "</div>"
         )
@@ -214,11 +245,55 @@ def test_markdown_with_fenced_code_block(self):
         }, status=200)
         self.assertEquals(
             res.json_body['html'],
-            "<div class='markdown'>"
+            '<div class="markdown">'
             '<pre><code>sudo dnf install bodhi\n</code></pre>\n'
             "</div>"
         )
 
+    def test_markdown_with_email_autolink(self):
+        res = self.app.get('/markdown', {
+            'text': 'email me at dude@mcpants.org',
+        }, status=200)
+        self.assertEquals(
+            res.json_body['html'],
+            '<div class="markdown">'
+            '<p>email me at <a href="mailto:dude@mcpants.org">dude@mcpants.org</a></p>'
+            "</div>"
+        )
+
+    def test_markdown_with_email_in_lt_gt(self):
+        res = self.app.get('/markdown', {
+            'text': 'email me at <dude@mcpants.org>',
+        }, status=200)
+        self.assertEquals(
+            res.json_body['html'],
+            '<div class="markdown">'
+            '<p>email me at <a href="mailto:dude@mcpants.org">dude@mcpants.org</a></p>'
+            "</div>"
+        )
+
+    def test_markdown_with_autolink(self):
+        res = self.app.get('/markdown', {
+            'text': 'http://getfedora.org',
+        }, status=200)
+        self.assertEquals(
+            res.json_body['html'],
+            '<div class="markdown">'
+            '<p><a href="http://getfedora.org">http://getfedora.org</a></p>'
+            "</div>"
+        )
+
+    def test_markdown_with_autolink_without_http(self):
+        res = self.app.get('/markdown', {
+            'text': 'getfedora.org',
+        }, status=200)
+        self.assertEquals(
+            res.json_body['html'],
+            '<div class="markdown">'
+            '<p><a href="http://getfedora.org">getfedora.org</a></p>'
+            "</div>"
+        )
+
     def test_metrics(self):
         res = self.app.get('/metrics')
         self.assertIn('$.plot', res)
diff --git a/bodhi/tests/server/test_utils.py b/bodhi/tests/server/test_utils.py
--- a/bodhi/tests/server/test_utils.py
+++ b/bodhi/tests/server/test_utils.py
@@ -357,16 +357,60 @@ def test_get_nvr(self):
         for element in result:
             assert isinstance(element, unicode)
 
-    def test_markup(self):
-        """Ensure we escape HTML"""
-        text = '<b>bold</b>'
+    def test_markup_escapes(self):
+        """Ensure we correctly parse markdown & escape HTML"""
+        text = (
+            '# this is a header\n'
+            'this is some **text**\n'
+            '<script>alert("pants")</script>'
+        )
         html = util.markup(None, text)
         assert html == (
-            "<div class='markdown'>"
-            '<p>&lt;b&gt;bold&lt;/b&gt;</p>'
-            "</div>"
+            '<div class="markdown"><h1>this is a header</h1>\n'
+            '<p>this is some <strong>text</strong>\n'
+            '&lt;script&gt;alert("pants")&lt;/script&gt;</p></div>'
         ), html
 
+    @mock.patch('bodhi.server.util.bleach.clean', return_value='cleaned text')
+    @mock.patch.object(util.bleach, '__version__', u'1.4.3')
+    def test_markup_with_bleach_1(self, clean):
+        """Use mocking to ensure we correctly use the bleach 1 API."""
+        text = '# this is a header\nthis is some **text**'
+
+        result = util.markup(None, text)
+
+        self.assertEqual(result, 'cleaned text')
+        expected_text = (
+            u'<div class="markdown"><h1>this is a header</h1>\n<p>this is some <strong>text'
+            u'</strong></p></div>')
+        expected_tags = [
+            "h1", "h2", "h3", "h4", "h5", "h6", "b", "i", "strong", "em", "tt", "p", "br", "span",
+            "div", "blockquote", "code", "hr", "pre", "ul", "ol", "li", "dd", "dt", "img", "a"]
+        # The bleach 1 API shoudl get these attrs passed.
+        clean.assert_called_once_with(expected_text, tags=expected_tags,
+                                      attributes=["src", "href", "alt", "title", "class"])
+
+    @mock.patch('bodhi.server.util.bleach.clean', return_value='cleaned text')
+    @mock.patch.object(util.bleach, '__version__', u'2.0')
+    def test_markup_with_bleach_2(self, clean):
+        """Use mocking to ensure we correctly use the bleach 2 API."""
+        text = '# this is a header\nthis is some **text**'
+
+        result = util.markup(None, text)
+
+        self.assertEqual(result, 'cleaned text')
+        expected_text = (
+            u'<div class="markdown"><h1>this is a header</h1>\n<p>this is some <strong>text'
+            u'</strong></p></div>')
+        expected_tags = [
+            "h1", "h2", "h3", "h4", "h5", "h6", "b", "i", "strong", "em", "tt", "p", "br", "span",
+            "div", "blockquote", "code", "hr", "pre", "ul", "ol", "li", "dd", "dt", "img", "a"]
+        expected_attributes = {
+            "img": ["src", "alt", "title"], "a": ["href", "alt", "title"], "div": ["class"]}
+        # The bleach 2 API shoudl get these attrs passed.
+        clean.assert_called_once_with(expected_text, tags=expected_tags,
+                                      attributes=expected_attributes)
+
     def test_rpm_header(self):
         h = util.get_rpm_header('libseccomp')
         assert h['name'] == 'libseccomp', h