Repository: liberapay/liberapay.com

Issue:
add support for xmpp: uri in markdown syntax
When adding and XMPP uri in the following form:
`[sat@chat.jabberfr.org](xmpp:sat@chat.jabberfr.org?join)`
the uri syntax is shown raw instead of linking to the room as expected.
add support for xmpp: uri in markdown syntax
When adding and XMPP uri in the following form:
`[sat@chat.jabberfr.org](xmpp:sat@chat.jabberfr.org?join)`
the uri syntax is shown raw instead of linking to the room as expected.


Discussion:
Okay, so here's the Markdown situation:

We're currently using Misaka 1.0.2, an obsolete version based on the abandoned Sundown lib.

We have two possibilities:

- upgrade [Misaka](http://misaka.61924.nl/) to 2.0.0
- switch to [Mistune](http://mistune.readthedocs.io/en/latest/)

According to [this benchmark](https://github.com/lepture/mistune/issues/1) Mistune is around 20 times slower than Misaka, but it's pure python so there are no dependency issues, and it allows customizing the lexer in addition to the renderer (we don't need that feature right now though).

In both cases we need a Renderer subclass to sanitize links. We've been using the `HTML_SAFELINK` flag of Misaka 1, but Misaka 2 and Mistune don't have this feature built-in, and anyway we need to have sufficient control over the filter to allow XMPP links and close this issue.
Okay, so here's the Markdown situation:

We're currently using Misaka 1.0.2, an obsolete version based on the abandoned Sundown lib.

We have two possibilities:

- upgrade [Misaka](http://misaka.61924.nl/) to 2.0.0
- switch to [Mistune](http://mistune.readthedocs.io/en/latest/)

According to [this benchmark](https://github.com/lepture/mistune/issues/1) Mistune is around 20 times slower than Misaka, but it's pure python so there are no dependency issues, and it allows customizing the lexer in addition to the renderer (we don't need that feature right now though).

In both cases we need a Renderer subclass to sanitize links. We've been using the `HTML_SAFELINK` flag of Misaka 1, but Misaka 2 and Mistune don't have this feature built-in, and anyway we need to have sufficient control over the filter to allow XMPP links and close this issue.

Resolving patch:
diff --git a/liberapay/utils/markdown.py b/liberapay/utils/markdown.py
--- a/liberapay/utils/markdown.py
+++ b/liberapay/utils/markdown.py
@@ -1,9 +1,41 @@
-from markupsafe import Markup
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import re
+
+from markupsafe import Markup, escape
 import misaka as m  # http://misaka.61924.nl/
 
+
+url_re = re.compile(r'^(https?|xmpp):')
+
+
+class CustomRenderer(m.HtmlRenderer):
+
+    def image(self, link, title='', alt=''):
+        if url_re.match(link):
+            maybe_alt = Markup(' alt="%s"') % alt if alt else ''
+            maybe_title = Markup(' title="%s"') % title if title else ''
+            return Markup('<img src="%s"%s%s />') % (link, maybe_alt, maybe_title)
+        else:
+            return escape("![%s](%s)" % (alt, link))
+
+    def link(self, content, link, title=''):
+        if url_re.match(link):
+            maybe_title = Markup(' title="%s"') % title if title else ''
+            return Markup('<a href="%s"%s>%s</a>') % (link, maybe_title, content)
+        else:
+            return escape("[%s](%s)" % (content, link))
+
+    def autolink(self, link, is_email):
+        if url_re.match(link):
+            return Markup('<a href="%s">%s</a>') % (link, link)
+        else:
+            return escape('<%s>' % link)
+
+
+renderer = CustomRenderer(flags=m.HTML_SKIP_HTML)
+md = m.Markdown(renderer, extensions=('autolink', 'strikethrough', 'no-intra-emphasis'))
+
+
 def render(markdown):
-    return Markup(m.html(
-        markdown,
-        extensions=m.EXT_AUTOLINK | m.EXT_STRIKETHROUGH | m.EXT_NO_INTRA_EMPHASIS,
-        render_flags=m.HTML_SKIP_HTML | m.HTML_TOC | m.HTML_SMARTYPANTS | m.HTML_SAFELINK
-    ))
+    return Markup(md(markdown))


Test patch:
diff --git a/tests/py/test_utils.py b/tests/py/test_utils.py
--- a/tests/py/test_utils.py
+++ b/tests/py/test_utils.py
@@ -72,34 +72,63 @@ def test_format_currency_defaults_to_trailing_zeroes(self):
         actual = i18n.format_money(16, 'USD', locale='en')
         assert actual == expected
 
+    # Markdown
+    # ========
 
     def test_markdown_render_does_render(self):
         expected = "<p>Example</p>\n"
         actual = markdown.render('Example')
         assert expected == actual
 
-    def test_markdown_render_escapes_scripts(self):
-        expected = '<p>Example alert &ldquo;hi&rdquo;;</p>\n'
-        actual = markdown.render('Example <script>alert "hi";</script>')
+    def test_markdown_render_discards_scripts(self):
+        expected = '<p>Example alert(1);</p>\n'
+        actual = markdown.render('Example <script>alert(1);</script>')
         assert expected == actual
-
-    def test_markdown_render_renders_http_links(self):
-        expected = '<p><a href="http://example.com/">foo</a></p>\n'
-        assert markdown.render('[foo](http://example.com/)') == expected
-        expected = '<p><a href="http://example.com/">http://example.com/</a></p>\n'
-        assert markdown.render('<http://example.com/>') == expected
-
-    def test_markdown_render_renders_https_links(self):
-        expected = '<p><a href="https://example.com/">foo</a></p>\n'
-        assert markdown.render('[foo](https://example.com/)') == expected
-        expected = '<p><a href="https://example.com/">https://example.com/</a></p>\n'
-        assert markdown.render('<https://example.com/>') == expected
-
-    def test_markdown_render_escapes_javascript_links(self):
-        expected = '<p>[foo](javascript:foo)</p>\n'
-        assert markdown.render('[foo](javascript:foo)') == expected
+        payload = '<sc<script>ript>alert(123)</sc</script>ript>'
+        html = markdown.render(payload)
+        assert '<script>' not in html
+
+    def test_markdown_autolink_filtering(self):
+        # Nice data
+        for url in ('http://a', 'https://b', 'xmpp:c'):
+            expected = '<p><a href="{0}">{0}</a></p>\n'.format(url)
+            actual = markdown.render('<%s>' % url)
+            assert actual == expected
+        # Naughty data
         expected = '<p>&lt;javascript:foo&gt;</p>\n'
         assert markdown.render('<javascript:foo>') == expected
+        link = 'javascript:0'
+        encoded_link = ''.join('&x{0:x};'.format(ord(c)) for c in link)
+        html = markdown.render('<%s>' % encoded_link)
+        assert link not in html
+
+    def test_markdown_link_filtering(self):
+        # Nice data
+        for url in ('http://a', 'https://b', 'xmpp:c'):
+            expected = '<p><a href="{0}">foo</a></p>\n'.format(url)
+            actual = markdown.render('[foo](%s)' % url)
+            assert actual == expected
+        # Naughty data
+        html = markdown.render('[foo](javascript:xss)')
+        assert html == '<p>[foo](javascript:xss)</p>\n'
+        html = markdown.render('[foo](unknown:bar)')
+        assert html == '<p>[foo](unknown:bar)</p>\n'
+        html = markdown.render('[" xss><xss>]("><xss>)')
+        assert '<xss>' not in html
+        assert '" xss' not in html
+        html = markdown.render('[" xss><xss>](https:"><xss>)')
+        assert '<xss>' not in html
+        assert '" xss' not in html
+
+    def test_markdown_image_src_filtering(self):
+        # Nice data
+        expected = '<p><img src="http:foo" /></p>\n'
+        assert markdown.render('![](http:foo)') == expected
+        expected = '<p><img src="https://example.org/" alt="bar" /></p>\n'
+        assert markdown.render('![bar](https://example.org/)') == expected
+        # Naughty data
+        expected = '<p>![foo](javascript:foo)</p>\n'
+        assert markdown.render('![foo](javascript:foo)') == expected
 
     def test_markdown_render_doesnt_allow_any_explicit_anchors(self):
         expected = '<p>foo</p>\n'
@@ -119,7 +148,6 @@ def test_markdown_render_no_intra_emphasis(self):
         actual = markdown.render('Examples like this_one and this other_one.')
         assert expected == actual
 
-
     # Base64 encoding/decoding
     # ========================