Repository: Python-Markdown/markdown
Pull request #907 (2020-02-07 22:17:22)

Fix escaping of HTML special chars (<, >, &) in `.toc_tokens`
This should fix the problem I found in https://github.com/mkdocs/mkdocs/pull/1970. I think the tests cover all the necessary cases. (I'm a little surprised that escaping where I did works though; I guess the `etree.Element.text` setter used for making the HTML TOC accepts escaped *or* unescaped HTML special chars.)

Sorry I didn't catch this the first time around!

waylan: Looks good. As per the [contributing guide][1], this just needs a comment in the [change log][2] under 3.2.1. 

[1]: https://python-markdown.github.io/contributing/#pull-requests
[2]: https://github.com/Python-Markdown/markdown/blob/1de3083d82a5245015f0f0b9471a965916134d6d/docs/change_log/index.md

Patch:
diff --git a/docs/change_log/index.md b/docs/change_log/index.md
index 6c9ca9d..afe8e0c 100644
--- a/docs/change_log/index.md
+++ b/docs/change_log/index.md
@@ -3,7 +3,10 @@ title: Change Log
 Python-Markdown Change Log
 =========================
 
-Under Development: Released version 3.1.1 (a bug-fix release).
+Under Development: Released version 3.2.1 (a bug-fix release). 
+
+* The `name` property in `toc_tokens` from the TOC extension now
+  escapes HTML special characters (`<`, `>`, and `&`).
 
 Feb 7, 2020: Released version 3.2 ([Notes](release-3.2.md)).
 
diff --git a/markdown/extensions/toc.py b/markdown/extensions/toc.py
index dc80c7e..8f2b13f 100644
--- a/markdown/extensions/toc.py
+++ b/markdown/extensions/toc.py
@@ -15,7 +15,7 @@ License: [BSD](https://opensource.org/licenses/bsd-license.php)
 
 from . import Extension
 from ..treeprocessors import Treeprocessor
-from ..util import parseBoolValue, AMP_SUBSTITUTE, HTML_PLACEHOLDER_RE
+from ..util import code_escape, parseBoolValue, AMP_SUBSTITUTE, HTML_PLACEHOLDER_RE
 from ..postprocessors import UnescapePostprocessor
 import re
 import unicodedata
@@ -264,7 +264,8 @@ class TocTreeprocessor(Treeprocessor):
                     'level': int(el.tag[-1]),
                     'id': el.attrib["id"],
                     'name': unescape(stashedHTML2text(
-                        el.attrib.get('data-toc-label', text), self.md, strip_entities=False
+                        code_escape(el.attrib.get('data-toc-label', text)),
+                        self.md, strip_entities=False
                     ))
                 })
 


Test patch:
diff --git a/tests/test_extensions.py b/tests/test_extensions.py
index c787886..8ecbe0d 100644
--- a/tests/test_extensions.py
+++ b/tests/test_extensions.py
@@ -827,6 +827,25 @@ class TestTOC(TestCaseWithAssertStartsWith):
             {'level': 1, 'id': 'foo-bar', 'name': 'Foo &amp; bar', 'children': []},
         ])
 
+    def testHtmlSpecialChars(self):
+        """ Test Headers with HTML special characters. """
+        text = '# Foo > & bar'
+        self.assertEqual(
+            self.md.convert(text),
+            '<h1 id="foo-bar">Foo &gt; &amp; bar</h1>'
+        )
+        self.assertEqual(
+            self.md.toc,
+            '<div class="toc">\n'
+              '<ul>\n'                                                  # noqa
+                '<li><a href="#foo-bar">Foo &gt; &amp; bar</a></li>\n'  # noqa
+              '</ul>\n'                                                 # noqa
+            '</div>\n'
+        )
+        self.assertEqual(self.md.toc_tokens, [
+            {'level': 1, 'id': 'foo-bar', 'name': 'Foo &gt; &amp; bar', 'children': []},
+        ])
+
     def testRawHtml(self):
         """ Test Headers with raw HTML. """
         text = '# Foo <b>Bar</b> Baz.'
@@ -1000,27 +1019,31 @@ class TestTOC(TestCaseWithAssertStartsWith):
         md = markdown.Markdown(extensions=['toc', 'attr_list'])
         text = ('# Header 1\n\n'
                 '## Header 2 { #foo }\n\n'
-                '## Header 3 { data-toc-label="Foo Bar"}\n\n'
-                '# Header 4 { data-toc-label="Foo <b>Baz</b>" }')
+                '## Header 3 { data-toc-label="Foo Bar" }\n\n'
+                '# Header 4 { data-toc-label="Foo > Baz" }\n\n'
+                '# Header 5 { data-toc-label="Foo <b>Quux</b>" }')
+
         self.assertEqual(
             md.convert(text),
             '<h1 id="header-1">Header 1</h1>\n'
             '<h2 id="foo">Header 2</h2>\n'
             '<h2 id="header-3">Header 3</h2>\n'
-            '<h1 id="header-4">Header 4</h1>'
+            '<h1 id="header-4">Header 4</h1>\n'
+            '<h1 id="header-5">Header 5</h1>'
         )
         self.assertEqual(
             md.toc,
             '<div class="toc">\n'
-              '<ul>\n'                                            # noqa
-                '<li><a href="#header-1">Header 1</a>'            # noqa
-                  '<ul>\n'                                        # noqa
-                    '<li><a href="#foo">Header 2</a></li>\n'      # noqa
-                    '<li><a href="#header-3">Foo Bar</a></li>\n'  # noqa
-                  '</ul>\n'                                       # noqa
-                '</li>\n'                                         # noqa
-                '<li><a href="#header-4">Foo Baz</a></li>\n'      # noqa
-              '</ul>\n'                                           # noqa
+              '<ul>\n'                                             # noqa
+                '<li><a href="#header-1">Header 1</a>'             # noqa
+                  '<ul>\n'                                         # noqa
+                    '<li><a href="#foo">Header 2</a></li>\n'       # noqa
+                    '<li><a href="#header-3">Foo Bar</a></li>\n'   # noqa
+                  '</ul>\n'                                        # noqa
+                '</li>\n'                                          # noqa
+                '<li><a href="#header-4">Foo &gt; Baz</a></li>\n'  # noqa
+                '<li><a href="#header-5">Foo Quux</a></li>\n'      # noqa
+              '</ul>\n'                                            # noqa
             '</div>\n'
         )
         self.assertEqual(md.toc_tokens, [
@@ -1028,7 +1051,8 @@ class TestTOC(TestCaseWithAssertStartsWith):
                 {'level': 2, 'id': 'foo', 'name': 'Header 2', 'children': []},
                 {'level': 2, 'id': 'header-3', 'name': 'Foo Bar', 'children': []}
             ]},
-            {'level': 1, 'id': 'header-4', 'name': 'Foo Baz', 'children': []},
+            {'level': 1, 'id': 'header-4', 'name': 'Foo &gt; Baz', 'children': []},
+            {'level': 1, 'id': 'header-5', 'name': 'Foo Quux', 'children': []},
         ])
 
     def testUniqueFunc(self):