Repository: Python-Markdown/markdown (python)

Issue:
Supporting namespaces in extension-output
I want to implement a TeX to MathML parser as extension. It works fine if I use plain `<math></math>`-tags as extension-output. However, if I understood the specs correctly, MathML2 requires an `xmlns="http://www.w3.org/1998/Math/MathML"` attribute:
https://www.w3.org/TR/MathML2/chapter7.html
(It is never stated specifically weather the attribute is *required* or just *allowed*. MathML3 does not require this attribute in HTML-only context.)

The moment I add this attribute I get the following error:

    Traceback (most recent call last):
      File "test.py", line 3, in <module>
        print(markdown("$x^2$", extensions=[TeXExtension()]))
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/__init__.py", line 494, in markdown
        return md.convert(text)
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/__init__.py", line 380, in convert
        output = self.serializer(root)
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/serializers.py", line 282, in to_xhtml_string
        return _write_html(ElementTree(element).getroot(), format="xhtml")
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/serializers.py", line 203, in _write_html
        _serialize_html(write, root, qnames, namespaces, format)
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/serializers.py", line 173, in _serialize_html
        items.sort(key=lambda x: x[1])  # sort on prefix
    AttributeError: 'dict_items' object has no attribute 'sort'

This is a [Python3-issue](https://stackoverflow.com/questions/28039155/dict-items-object-has-no-attribute-sortreplace-min), so I changed `items.sort( ` to `items=sorted(items, `, but now I get this error:

    Traceback (most recent call last):
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/__init__.py", line 384, in convert
        '<%s>' % self.doc_tag) + len(self.doc_tag) + 2
    ValueError: substring not found
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "test.py", line 3, in <module>
        print(markdown("$x^2$", extensions=[TeXExtension()]))
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/__init__.py", line 494, in markdown
        return md.convert(text)
      File "/home/user/Documents/projects/rocketcollider/venv/lib64/python3.6/site-packages/markdown/__init__.py", line 394, in convert
        'tags. Document=%r' % output.strip())
    ValueError: Markdown failed to strip top-level tags. Document='<div xmlns:ns0="http://www.w3.org/1998/Math/MathML">\n<p><ns0:math display="block">\n<ns0:semantics>\n  <ns0:msup>\n    <ns0:mi>x</ns0:mi>\n    <ns0:mn>2</ns0:mn>\n  </ns0:msup>\n<ns0:annotations>x^2</ns0:annotations>\n</ns0:semantics>\n</ns0:math></p>\n</div>'

Since the `item.sort` call happens after an `if namespaces:`, I'm assuming there is a bigger issue regarding handling namespaces

For comparison, the MathML I'm returning from `handleMatch` (using `etree.fromstring(math)`) is:

    <math xmlns="http://www.w3.org/1998/Math/MathML" display="block">
    <semantics>
      <msup>
        <mi>x</mi>
        <mn>2</mn>
      </msup>
    <annotations>x^2</annotations>
    </semantics>
    </math>


Pull request description:
Simplify namespace support in serializer.
Fixes #679.

I'm still not 100% sure this is the correct approach. But I like the output we get in the included test better than what we got before.

Patch:
diff --git a/markdown/serializers.py b/markdown/serializers.py
index 1e8d9dd..187d755 100644
--- a/markdown/serializers.py
+++ b/markdown/serializers.py
@@ -39,6 +39,7 @@
 
 from __future__ import absolute_import
 from __future__ import unicode_literals
+from xml.etree.ElementTree import ProcessingInstruction
 from . import util
 ElementTree = util.etree.ElementTree
 QName = util.etree.QName
@@ -46,8 +47,6 @@ if hasattr(util.etree, 'test_comment'):  # pragma: no cover
     Comment = util.etree.test_comment
 else:  # pragma: no cover
     Comment = util.etree.Comment
-PI = util.etree.PI
-ProcessingInstruction = util.etree.ProcessingInstruction
 
 __all__ = ['to_html_string', 'to_xhtml_string']
 
@@ -59,19 +58,6 @@ try:
 except NameError:  # pragma: no cover
     pass
 
-_namespace_map = {
-    # "well-known" namespace prefixes
-    "http://www.w3.org/XML/1998/namespace": "xml",
-    "http://www.w3.org/1999/xhtml": "html",
-    "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf",
-    "http://schemas.xmlsoap.org/wsdl/": "wsdl",
-    # xml schema
-    "http://www.w3.org/2001/XMLSchema": "xs",
-    "http://www.w3.org/2001/XMLSchema-instance": "xsi",
-    # dublic core
-    "http://purl.org/dc/elements/1.1/": "dc",
-}
-
 
 def _raise_serialization_error(text):  # pragma: no cover
     raise TypeError(
@@ -79,13 +65,6 @@ def _raise_serialization_error(text):  # pragma: no cover
         )
 
 
-def _encode(text, encoding):
-    try:
-        return text.encode(encoding, "xmlcharrefreplace")
-    except (TypeError, AttributeError):  # pragma: no cover
-        _raise_serialization_error(text)
-
-
 def _escape_cdata(text):
     # escape character data
     try:
@@ -137,142 +116,73 @@ def _escape_attrib_html(text):
         _raise_serialization_error(text)
 
 
-def _serialize_html(write, elem, qnames, namespaces, format):
+def _serialize_html(write, elem, format):
     tag = elem.tag
     text = elem.text
     if tag is Comment:
         write("<!--%s-->" % _escape_cdata(text))
     elif tag is ProcessingInstruction:
         write("<?%s?>" % _escape_cdata(text))
+    elif tag is None:
+        if text:
+            write(_escape_cdata(text))
+        for e in elem:
+            _serialize_html(write, e, format)
     else:
-        tag = qnames[tag]
-        if tag is None:
+        namespace_uri = None
+        if isinstance(tag, QName):
+            # QNAME objects store their data as a string: `{uri}tag`
+            if tag.text[:1] == "{":
+                namespace_uri, tag = tag.text[1:].split("}", 1)
+            else:
+                raise ValueError('QName objects must define a tag.')
+        write("<" + tag)
+        items = elem.items()
+        if items:
+            items = sorted(items)  # lexical order
+            for k, v in items:
+                if isinstance(k, QName):
+                    # Assume a text only QName
+                    k = k.text
+                if isinstance(v, QName):
+                    # Assume a text only QName
+                    v = v.text
+                else:
+                    v = _escape_attrib_html(v)
+                if k == v and format == 'html':
+                    # handle boolean attributes
+                    write(" %s" % v)
+                else:
+                    write(' %s="%s"' % (k, v))
+        if namespace_uri:
+            write(' xmlns="%s"' % (_escape_attrib(namespace_uri)))
+        if format == "xhtml" and tag.lower() in HTML_EMPTY:
+            write(" />")
+        else:
+            write(">")
             if text:
-                write(_escape_cdata(text))
+                if tag.lower() in ["script", "style"]:
+                    write(text)
+                else:
+                    write(_escape_cdata(text))
             for e in elem:
-                _serialize_html(write, e, qnames, None, format)
-        else:
-            write("<" + tag)
-            items = elem.items()
-            if items or namespaces:
-                items = sorted(items)  # lexical order
-                for k, v in items:
-                    if isinstance(k, QName):
-                        k = k.text
-                    if isinstance(v, QName):
-                        v = qnames[v.text]
-                    else:
-                        v = _escape_attrib_html(v)
-                    if qnames[k] == v and format == 'html':
-                        # handle boolean attributes
-                        write(" %s" % v)
-                    else:
-                        write(" %s=\"%s\"" % (qnames[k], v))
-                if namespaces:
-                    items = namespaces.items()
-                    items.sort(key=lambda x: x[1])  # sort on prefix
-                    for v, k in items:
-                        if k:
-                            k = ":" + k
-                        write(" xmlns%s=\"%s\"" % (k, _escape_attrib(v)))
-            if format == "xhtml" and tag.lower() in HTML_EMPTY:
-                write(" />")
-            else:
-                write(">")
-                if text:
-                    if tag.lower() in ["script", "style"]:
-                        write(text)
-                    else:
-                        write(_escape_cdata(text))
-                for e in elem:
-                    _serialize_html(write, e, qnames, None, format)
-                if tag.lower() not in HTML_EMPTY:
-                    write("</" + tag + ">")
+                _serialize_html(write, e, format)
+            if tag.lower() not in HTML_EMPTY:
+                write("</" + tag + ">")
     if elem.tail:
         write(_escape_cdata(elem.tail))
 
 
-def _write_html(root,
-                encoding=None,
-                default_namespace=None,
-                format="html"):
+def _write_html(root, format="html"):
     assert root is not None
     data = []
     write = data.append
-    qnames, namespaces = _namespaces(root, default_namespace)
-    _serialize_html(write, root, qnames, namespaces, format)
-    if encoding is None:
-        return "".join(data)
-    else:
-        return _encode("".join(data))
+    _serialize_html(write, root, format)
+    return "".join(data)
 
 
 # --------------------------------------------------------------------
-# serialization support
-
-def _namespaces(elem, default_namespace=None):
-    # identify namespaces used in this tree
-
-    # maps qnames to *encoded* prefix:local names
-    qnames = {None: None}
-
-    # maps uri:s to prefixes
-    namespaces = {}
-    if default_namespace:
-        namespaces[default_namespace] = ""
-
-    def add_qname(qname):
-        # calculate serialized qname representation
-        try:
-            if qname[:1] == "{":
-                uri, tag = qname[1:].split("}", 1)
-                prefix = namespaces.get(uri)
-                if prefix is None:
-                    prefix = _namespace_map.get(uri)
-                    if prefix is None:
-                        prefix = "ns%d" % len(namespaces)
-                    if prefix != "xml":
-                        namespaces[uri] = prefix
-                if prefix:
-                    qnames[qname] = "%s:%s" % (prefix, tag)
-                else:
-                    qnames[qname] = tag  # default element
-            else:
-                if default_namespace:
-                    raise ValueError(
-                        "cannot use non-qualified names with "
-                        "default_namespace option"
-                        )
-                qnames[qname] = qname
-        except TypeError:  # pragma: no cover
-            _raise_serialization_error(qname)
-
-    # populate qname and namespaces table
-    try:
-        iterate = elem.iter
-    except AttributeError:
-        iterate = elem.getiterator  # cET compatibility
-    for elem in iterate():
-        tag = elem.tag
-        if isinstance(tag, QName) and tag.text not in qnames:
-            add_qname(tag.text)
-        elif isinstance(tag, util.string_type):
-            if tag not in qnames:
-                add_qname(tag)
-        elif tag is not None and tag is not Comment and tag is not PI:
-            _raise_serialization_error(tag)
-        for key, value in elem.items():
-            if isinstance(key, QName):
-                key = key.text
-            if key not in qnames:
-                add_qname(key)
-            if isinstance(value, QName) and value.text not in qnames:
-                add_qname(value.text)
-        text = elem.text
-        if isinstance(text, QName) and text.text not in qnames:
-            add_qname(text.text)
-    return qnames, namespaces
-
+# public functions
 
 def to_html_string(element):
     return _write_html(ElementTree(element).getroot(), format="html")


Test patch:
diff --git a/tests/test_apis.py b/tests/test_apis.py
index 251657b..2875c85 100644
--- a/tests/test_apis.py
+++ b/tests/test_apis.py
@@ -18,6 +18,7 @@ from logging import DEBUG, WARNING, CRITICAL
 import yaml
 import tempfile
 from io import BytesIO
+from xml.etree.ElementTree import ProcessingInstruction
 
 
 PY3 = sys.version_info[0] == 3
@@ -465,23 +466,47 @@ class testSerializers(unittest.TestCase):
     def testHtml(self):
         """ Test HTML serialization. """
         el = markdown.util.etree.Element('div')
+        el.set('id', 'foo<&">')
         p = markdown.util.etree.SubElement(el, 'p')
-        p.text = 'foo'
+        p.text = 'foo <&escaped>'
+        p.set('hidden', 'hidden')
         markdown.util.etree.SubElement(el, 'hr')
+        non_element = markdown.util.etree.SubElement(el, None)
+        non_element.text = 'non-element text'
+        script = markdown.util.etree.SubElement(non_element, 'script')
+        script.text = '<&"test\nescaping">'
+        el.tail = "tail text"
         self.assertEqual(
             markdown.serializers.to_html_string(el),
-            '<div><p>foo</p><hr></div>'
+            '<div id="foo&lt;&amp;&quot;&gt;">'
+            '<p hidden>foo &lt;&amp;escaped&gt;</p>'
+            '<hr>'
+            'non-element text'
+            '<script><&"test\nescaping"></script>'
+            '</div>tail text'
         )
 
     def testXhtml(self):
         """" Test XHTML serialization. """
         el = markdown.util.etree.Element('div')
+        el.set('id', 'foo<&">')
         p = markdown.util.etree.SubElement(el, 'p')
-        p.text = 'foo'
+        p.text = 'foo<&escaped>'
+        p.set('hidden', 'hidden')
         markdown.util.etree.SubElement(el, 'hr')
+        non_element = markdown.util.etree.SubElement(el, None)
+        non_element.text = 'non-element text'
+        script = markdown.util.etree.SubElement(non_element, 'script')
+        script.text = '<&"test\nescaping">'
+        el.tail = "tail text"
         self.assertEqual(
             markdown.serializers.to_xhtml_string(el),
-            '<div><p>foo</p><hr /></div>'
+            '<div id="foo&lt;&amp;&quot;&gt;">'
+            '<p hidden="hidden">foo&lt;&amp;escaped&gt;</p>'
+            '<hr />'
+            'non-element text'
+            '<script><&"test\nescaping"></script>'
+            '</div>tail text'
         )
 
     def testMixedCaseTags(self):
@@ -496,6 +521,68 @@ class testSerializers(unittest.TestCase):
             '<MixedCase>not valid <EMPHASIS>html</EMPHASIS><HR /></MixedCase>'
         )
 
+    def testProsessingInstruction(self):
+        """ Test serialization of ProcessignInstruction. """
+        pi = ProcessingInstruction('foo', text='<&"test\nescaping">')
+        self.assertIs(pi.tag, ProcessingInstruction)
+        self.assertEqual(
+            markdown.serializers.to_xhtml_string(pi),
+            '<?foo &lt;&amp;"test\nescaping"&gt;?>'
+        )
+
+    def testQNameTag(self):
+        """ Test serialization of QName tag. """
+        div = markdown.util.etree.Element('div')
+        qname = markdown.util.etree.QName('http://www.w3.org/1998/Math/MathML', 'math')
+        math = markdown.util.etree.SubElement(div, qname)
+        math.set('display', 'block')
+        sem = markdown.util.etree.SubElement(math, 'semantics')
+        msup = markdown.util.etree.SubElement(sem, 'msup')
+        mi = markdown.util.etree.SubElement(msup, 'mi')
+        mi.text = 'x'
+        mn = markdown.util.etree.SubElement(msup, 'mn')
+        mn.text = '2'
+        ann = markdown.util.etree.SubElement(sem, 'annotations')
+        ann.text = 'x^2'
+        self.assertEqual(
+            markdown.serializers.to_xhtml_string(div),
+            '<div>'
+            '<math display="block" xmlns="http://www.w3.org/1998/Math/MathML">'
+            '<semantics>'
+            '<msup>'
+            '<mi>x</mi>'
+            '<mn>2</mn>'
+            '</msup>'
+            '<annotations>x^2</annotations>'
+            '</semantics>'
+            '</math>'
+            '</div>'
+        )
+
+    def testQNameAttribute(self):
+        """ Test serialization of QName attribute. """
+        div = markdown.util.etree.Element('div')
+        div.set(markdown.util.etree.QName('foo'), markdown.util.etree.QName('bar'))
+        self.assertEqual(
+            markdown.serializers.to_xhtml_string(div),
+            '<div foo="bar"></div>'
+        )
+
+    def testBadQNameTag(self):
+        """ Test serialization of QName with no tag. """
+        qname = markdown.util.etree.QName('http://www.w3.org/1998/Math/MathML')
+        el = markdown.util.etree.Element(qname)
+        self.assertRaises(ValueError, markdown.serializers.to_xhtml_string, el)
+
+    def testQNameEscaping(self):
+        """ Test QName escaping. """
+        qname = markdown.util.etree.QName('<&"test\nescaping">', 'div')
+        el = markdown.util.etree.Element(qname)
+        self.assertEqual(
+            markdown.serializers.to_xhtml_string(el),
+            '<div xmlns="&lt;&amp;&quot;test&#10;escaping&quot;&gt;"></div>'
+        )
+
     def buildExtension(self):
         """ Build an extension which registers fakeSerializer. """
         def fakeSerializer(elem):