Repository: Python-Markdown/markdown (2019-11-25 ~ 2019-11-26)

problem_statement: Backslash escape not unescaped in ID attributes
I'm using the `TocExtension` to generate table of contens for my `.md. files (which could be relevant/irrelevant to the issue below).

In my original text, I'm escaping the single underscore `_` with a backslash:

```markdown
### select\_related
```

After running the text through `markdown`, this is the HTML result:

```HTML
<h3 id="select95related">select_related</h3>
```

And the generated TOC points to the ID above as well. I'm wondering if this is the expected behaviour? I would expect `markdown` to simply render:

```HTML
<h3 id="select_related">select_related</h3>
```

At least according to [one online converter](https://www.browserling.com/tools/markdown-to-html), the result should be: `<...id="select_related"...>`

 

Am I missing something?
date: 2019-11-25
patch: diff --git a/markdown/extensions/toc.py b/markdown/extensions/toc.py
index 6d45239..7629ee9 100644
--- a/markdown/extensions/toc.py
+++ b/markdown/extensions/toc.py
@@ -16,6 +16,7 @@ License: [BSD](https://opensource.org/licenses/bsd-license.php)
 from . import Extension
 from ..treeprocessors import Treeprocessor
 from ..util import etree, parseBoolValue, AMP_SUBSTITUTE, HTML_PLACEHOLDER_RE
+from ..postprocessors import UnescapePostprocessor
 import re
 import unicodedata
 
@@ -56,6 +57,12 @@ def stashedHTML2text(text, md):
     return HTML_PLACEHOLDER_RE.sub(_html_sub, text)
 
 
+def unescape(text):
+    """ Unescape escaped text. """
+    c = UnescapePostprocessor()
+    return c.run(text)
+
+
 def nest_toc_tokens(toc_list):
     """Given an unsorted list with errors and skips, return a nested one.
     [{'level': 1}, {'level': 2}]
@@ -242,7 +249,7 @@ class TocTreeprocessor(Treeprocessor):
 
                 # Do not override pre-existing ids
                 if "id" not in el.attrib:
-                    innertext = stashedHTML2text(text, self.md)
+                    innertext = unescape(stashedHTML2text(text, self.md))
                     el.attrib["id"] = unique(self.slugify(innertext, self.sep), used_ids)
 
                 toc_tokens.append({

test_patch: diff --git a/tests/test_extensions.py b/tests/test_extensions.py
index 3ff4f7f..c787886 100644
--- a/tests/test_extensions.py
+++ b/tests/test_extensions.py
@@ -780,6 +780,7 @@ class TestTOC(TestCaseWithAssertStartsWith):
         self.assertStartsWith('<div class="toc">', self.md.toc)
         self.md.reset()
         self.assertEqual(self.md.toc, '')
+        self.assertEqual(self.md.toc_tokens, [])
 
     def testUniqueIds(self):
         """ Test Unique IDs. """
@@ -791,6 +792,21 @@ class TestTOC(TestCaseWithAssertStartsWith):
             '<h1 id="header_1">Header</h1>\n'
             '<h1 id="header_2">Header</h1>'
         )
+        self.assertEqual(
+            self.md.toc,
+            '<div class="toc">\n'
+              '<ul>\n'                                       # noqa
+                '<li><a href="#header">Header</a></li>\n'    # noqa
+                '<li><a href="#header_1">Header</a></li>\n'  # noqa
+                '<li><a href="#header_2">Header</a></li>\n'  # noqa
+              '</ul>\n'                                      # noqa
+            '</div>\n'
+        )
+        self.assertEqual(self.md.toc_tokens, [
+            {'level': 1, 'id': 'header', 'name': 'Header', 'children': []},
+            {'level': 1, 'id': 'header_1', 'name': 'Header', 'children': []},
+            {'level': 1, 'id': 'header_2', 'name': 'Header', 'children': []},
+        ])
 
     def testHtmlEntities(self):
         """ Test Headers with HTML Entities. """
@@ -799,6 +815,17 @@ class TestTOC(TestCaseWithAssertStartsWith):
             self.md.convert(text),
             '<h1 id="foo-bar">Foo &amp; bar</h1>'
         )
+        self.assertEqual(
+            self.md.toc,
+            '<div class="toc">\n'
+              '<ul>\n'                                             # noqa
+                '<li><a href="#foo-bar">Foo &amp; bar</a></li>\n'  # noqa
+              '</ul>\n'                                            # noqa
+            '</div>\n'
+        )
+        self.assertEqual(self.md.toc_tokens, [
+            {'level': 1, 'id': 'foo-bar', 'name': 'Foo &amp; bar', 'children': []},
+        ])
 
     def testRawHtml(self):
         """ Test Headers with raw HTML. """
@@ -807,6 +834,17 @@ class TestTOC(TestCaseWithAssertStartsWith):
             self.md.convert(text),
             '<h1 id="foo-bar-baz">Foo <b>Bar</b> Baz.</h1>'
         )
+        self.assertEqual(
+            self.md.toc,
+            '<div class="toc">\n'
+              '<ul>\n'                                                # noqa
+                '<li><a href="#foo-bar-baz">Foo Bar Baz.</a></li>\n'  # noqa
+              '</ul>\n'                                               # noqa
+            '</div>\n'
+        )
+        self.assertEqual(self.md.toc_tokens, [
+            {'level': 1, 'id': 'foo-bar-baz', 'name': 'Foo Bar Baz.', 'children': []},
+        ])
 
     def testBaseLevel(self):
         """ Test Header Base Level. """
@@ -833,6 +871,12 @@ class TestTOC(TestCaseWithAssertStartsWith):
               '</ul>\n'                                                # noqa
             '</div>\n'
         )
+        self.assertEqual(md.toc_tokens, [
+            {'level': 5, 'id': 'some-header', 'name': 'Some Header', 'children': [
+                {'level': 6, 'id': 'next-level', 'name': 'Next Level', 'children': []},
+                {'level': 6, 'id': 'too-high', 'name': 'Too High', 'children': []},
+            ]},
+        ])
 
     def testHeaderInlineMarkup(self):
         """ Test Headers with inline markup. """
@@ -843,6 +887,18 @@ class TestTOC(TestCaseWithAssertStartsWith):
             '<h1 id="some-header-with-markup">Some <em>Header</em> with '
             '<a href="http://example.com">markup</a>.</h1>'
         )
+        self.assertEqual(
+            self.md.toc,
+            '<div class="toc">\n'
+              '<ul>\n'                                     # noqa
+                '<li><a href="#some-header-with-markup">'  # noqa
+                  'Some Header with markup.</a></li>\n'    # noqa
+              '</ul>\n'                                    # noqa
+            '</div>\n'
+        )
+        self.assertEqual(self.md.toc_tokens, [
+            {'level': 1, 'id': 'some-header-with-markup', 'name': 'Some Header with markup.', 'children': []},
+        ])
 
     def testAnchorLink(self):
         """ Test TOC Anchorlink. """
@@ -942,40 +998,38 @@ class TestTOC(TestCaseWithAssertStartsWith):
     def testWithAttrList(self):
         """ Test TOC with attr_list Extension. """
         md = markdown.Markdown(extensions=['toc', 'attr_list'])
-        text = '# Header 1\n\n## Header 2 { #foo }\n\n## Header 3 { data-toc-label="Foo Bar"}'
+        text = ('# Header 1\n\n'
+                '## Header 2 { #foo }\n\n'
+                '## Header 3 { data-toc-label="Foo Bar"}\n\n'
+                '# Header 4 { data-toc-label="Foo <b>Baz</b>" }')
         self.assertEqual(
             md.convert(text),
             '<h1 id="header-1">Header 1</h1>\n'
             '<h2 id="foo">Header 2</h2>\n'
-            '<h2 id="header-3">Header 3</h2>'
+            '<h2 id="header-3">Header 3</h2>\n'
+            '<h1 id="header-4">Header 4</h1>'
         )
         self.assertEqual(
             md.toc,
             '<div class="toc">\n'
-              '<ul>\n'                                           # noqa
-                '<li><a href="#header-1">Header 1</a>'           # noqa
-                  '<ul>\n'                                       # noqa
-                    '<li><a href="#foo">Header 2</a></li>\n'     # noqa
-                    '<li><a href="#header-3">Foo Bar</a></li>\n' # noqa
-                  '</ul>\n'                                      # noqa
-                '</li>\n'                                        # noqa
-              '</ul>\n'                                          # noqa
+              '<ul>\n'                                            # noqa
+                '<li><a href="#header-1">Header 1</a>'            # noqa
+                  '<ul>\n'                                        # noqa
+                    '<li><a href="#foo">Header 2</a></li>\n'      # noqa
+                    '<li><a href="#header-3">Foo Bar</a></li>\n'  # noqa
+                  '</ul>\n'                                       # noqa
+                '</li>\n'                                         # noqa
+                '<li><a href="#header-4">Foo Baz</a></li>\n'      # noqa
+              '</ul>\n'                                           # noqa
             '</div>\n'
         )
-        self.assertEqual(
-            md.toc_tokens,
-            [
-                {
-                    'level': 1,
-                    'id': 'header-1',
-                    'name': 'Header 1',
-                    'children': [
-                        {'level': 2, 'id': 'foo', 'name': 'Header 2', 'children': []},
-                        {'level': 2, 'id': 'header-3', 'name': 'Foo Bar', 'children': []}
-                    ]
-                }
-            ]
-        )
+        self.assertEqual(md.toc_tokens, [
+            {'level': 1, 'id': 'header-1', 'name': 'Header 1', 'children': [
+                {'level': 2, 'id': 'foo', 'name': 'Header 2', 'children': []},
+                {'level': 2, 'id': 'header-3', 'name': 'Foo Bar', 'children': []}
+            ]},
+            {'level': 1, 'id': 'header-4', 'name': 'Foo Baz', 'children': []},
+        ])
 
     def testUniqueFunc(self):
         """ Test 'unique' function. """
diff --git a/tests/test_syntax/extensions/test_toc.py b/tests/test_syntax/extensions/test_toc.py
new file mode 100644
index 0000000..5b9ad92
--- /dev/null
+++ b/tests/test_syntax/extensions/test_toc.py
@@ -0,0 +1,121 @@
+"""
+Python Markdown
+
+A Python implementation of John Gruber's Markdown.
+
+Documentation: https://python-markdown.github.io/
+GitHub: https://github.com/Python-Markdown/markdown/
+PyPI: https://pypi.org/project/Markdown/
+
+Started by Manfred Stienstra (http://www.dwerg.net/).
+Maintained for a few years by Yuri Takhteyev (http://www.freewisdom.org).
+Currently maintained by Waylan Limberg (https://github.com/waylan),
+Dmitry Shachnev (https://github.com/mitya57) and Isaac Muse (https://github.com/facelessuser).
+
+Copyright 2007-2019 The Python Markdown Project (v. 1.7 and later)
+Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b)
+Copyright 2004 Manfred Stienstra (the original version)
+
+License: BSD (see LICENSE.md for details).
+"""
+
+from markdown.test_tools import TestCase
+from markdown.extensions.toc import TocExtension
+
+
+class TestTOC(TestCase):
+
+    # TODO: Move the rest of the TOC tests here.
+
+    def test_escaped_char_in_id(self):
+        self.assertMarkdownRenders(
+            r'# escaped\_character',
+            '<h1 id="escaped_character">escaped_character</h1>',
+            extensions=['toc']
+        )
+
+    def testAnchorLinkWithCustomClass(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                '''
+                # Header 1
+
+                ## Header *2*
+                '''
+            ),
+            self.dedent(
+                '''
+                <h1 id="header-1"><a class="custom" href="#header-1">Header 1</a></h1>
+                <h2 id="header-2"><a class="custom" href="#header-2">Header <em>2</em></a></h2>
+                '''
+            ),
+            extensions=[TocExtension(anchorlink=True, anchorlink_class="custom")]
+        )
+
+    def testAnchorLinkWithCustomClasses(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                '''
+                # Header 1
+
+                ## Header *2*
+                '''
+            ),
+            self.dedent(
+                '''
+                <h1 id="header-1"><a class="custom1 custom2" href="#header-1">Header 1</a></h1>
+                <h2 id="header-2"><a class="custom1 custom2" href="#header-2">Header <em>2</em></a></h2>
+                '''
+            ),
+            extensions=[TocExtension(anchorlink=True, anchorlink_class="custom1 custom2")]
+        )
+
+    def testPermalinkWithEmptyText(self):
+        self.assertMarkdownRenders(
+            '# Header',
+            '<h1 id="header">'                                                      # noqa
+                'Header'                                                            # noqa
+                '<a class="headerlink" href="#header" title="Permanent link"></a>'  # noqa
+            '</h1>',                                                                # noqa
+            extensions=[TocExtension(permalink="")]
+        )
+
+    def testPermalinkWithCustomClass(self):
+        self.assertMarkdownRenders(
+            '# Header',
+            '<h1 id="header">'                                                        # noqa
+                'Header'                                                              # noqa
+                '<a class="custom" href="#header" title="Permanent link">&para;</a>'  # noqa
+            '</h1>',                                                                  # noqa
+            extensions=[TocExtension(permalink=True, permalink_class="custom")]
+        )
+
+    def testPermalinkWithCustomClasses(self):
+        self.assertMarkdownRenders(
+            '# Header',
+            '<h1 id="header">'                                                                 # noqa
+                'Header'                                                                       # noqa
+                '<a class="custom1 custom2" href="#header" title="Permanent link">&para;</a>'  # noqa
+            '</h1>',                                                                           # noqa
+            extensions=[TocExtension(permalink=True, permalink_class="custom1 custom2")]
+        )
+
+    def testPermalinkWithCustomTitle(self):
+        self.assertMarkdownRenders(
+            '# Header',
+            '<h1 id="header">'                                                    # noqa
+                'Header'                                                          # noqa
+                '<a class="headerlink" href="#header" title="custom">&para;</a>'  # noqa
+            '</h1>',                                                              # noqa
+            extensions=[TocExtension(permalink=True, permalink_title="custom")]
+        )
+
+    def testPermalinkWithEmptyTitle(self):
+        self.assertMarkdownRenders(
+            '# Header',
+            '<h1 id="header">'                                                    # noqa
+                'Header'                                                          # noqa
+                '<a class="headerlink" href="#header">&para;</a>'                 # noqa
+            '</h1>',                                                              # noqa
+            extensions=[TocExtension(permalink=True, permalink_title="")]
+        )


----

problem_statement: TOC: Configurable permalink title attribute
Would be nice to have an ability to set a custom title text for permalink title attribute. For example:

```yml
- toc:
    title: 'Ссылка на этот раздел' #Custom title here
    permalink: '#'
    slugify: !!python/name:pymdownx.slugs.uslugify
```

For now, "Permament link" text is simply hardcoded within `toc.py`:
https://github.com/Python-Markdown/markdown/blob/9ec83c4c414f27bc8aa490dbac1fe30b49058ae6/markdown/extensions/toc.py#L194

Related: #657
date: 2019-11-26
patch: diff --git a/markdown/extensions/toc.py b/markdown/extensions/toc.py
index 7629ee9..d490a06 100644
--- a/markdown/extensions/toc.py
+++ b/markdown/extensions/toc.py
@@ -136,9 +136,12 @@ class TocTreeprocessor(Treeprocessor):
         self.slugify = config["slugify"]
         self.sep = config["separator"]
         self.use_anchors = parseBoolValue(config["anchorlink"])
+        self.anchorlink_class = config["anchorlink_class"]
         self.use_permalinks = parseBoolValue(config["permalink"], False)
         if self.use_permalinks is None:
             self.use_permalinks = config["permalink"]
+        self.permalink_class = config["permalink_class"]
+        self.permalink_title = config["permalink_title"]
         self.header_rgx = re.compile("[Hh][123456]")
         if isinstance(config["toc_depth"], str) and '-' in config["toc_depth"]:
             self.toc_top, self.toc_bottom = [int(x) for x in config["toc_depth"].split('-')]
@@ -184,7 +187,7 @@ class TocTreeprocessor(Treeprocessor):
         anchor = etree.Element("a")
         anchor.text = c.text
         anchor.attrib["href"] = "#" + elem_id
-        anchor.attrib["class"] = "toclink"
+        anchor.attrib["class"] = self.anchorlink_class
         c.text = ""
         for elem in c:
             anchor.append(elem)
@@ -198,8 +201,9 @@ class TocTreeprocessor(Treeprocessor):
                           if self.use_permalinks is True
                           else self.use_permalinks)
         permalink.attrib["href"] = "#" + elem_id
-        permalink.attrib["class"] = "headerlink"
-        permalink.attrib["title"] = "Permanent link"
+        permalink.attrib["class"] = self.permalink_class
+        if self.permalink_title:
+            permalink.attrib["title"] = self.permalink_title
         c.append(permalink)
 
     def build_toc_div(self, toc_list):
@@ -264,7 +268,7 @@ class TocTreeprocessor(Treeprocessor):
 
                 if self.use_anchors:
                     self.add_anchor(el, el.attrib["id"])
-                if self.use_permalinks:
+                if self.use_permalinks not in [False, None]:
                     self.add_permalink(el, el.attrib["id"])
 
         toc_tokens = nest_toc_tokens(toc_tokens)
@@ -295,9 +299,18 @@ class TocExtension(Extension):
             "anchorlink": [False,
                            "True if header should be a self link - "
                            "Defaults to False"],
+            "anchorlink_class": ['toclink',
+                                 'CSS class(es) used for the link. '
+                                 'Defaults to "toclink"'],
             "permalink": [0,
                           "True or link text if a Sphinx-style permalink should "
                           "be added - Defaults to False"],
+            "permalink_class": ['headerlink',
+                                'CSS class(es) used for the link. '
+                                'Defaults to "headerlink"'],
+            "permalink_title": ["Permanent link",
+                                "Title attribute of the permalink - "
+                                "Defaults to 'Permanent link'"],
             "baselevel": ['1', 'Base level for headers.'],
             "slugify": [slugify,
                         "Function to generate anchors based on header text - "


----

problem_statement: Heading names in toc_tokens contain stashed HTML placeholders
If a Markdown heading contains HTML, the corresponding entry in the `.toc_tokens` property ends up with HTML placeholders when returned to the user. The following example should illustrate the problem:

```
>>> import markdown
>>> md = markdown.Markdown(extensions=['toc'])
>>> md.convert('# <code>Heading</code>\n')
'<h1 id="heading"><code>Heading</code></h1>'
>>> md.toc_tokens
[{'level': 1, 'id': 'heading', 'name': '\x02wzxhzdk:0\x03Heading\x02wzxhzdk:1\x03', 'children': []}]
```

While this isn't *too* hard to fix (we could just un-stash the HTML immediately before returning it to the user), it does raise a bigger question: what should the data format of the `name` field in `toc_tokens` be? Is it...

* Markdown (so the value would be exactly as in the source file: `<code>Heading</code>`)
* Plain text (so the value would strip HTML: `Heading`)
* HTML (similar to the Markdown format, but with HTML entities replaced, so `<code>a>b</code>` becomes `<code>a&gt;b</code>`)

In particular, this is relevant for https://github.com/mkdocs/mkdocs/pull/1970. Prior to that PR, MkDocs would build an internal representation of the TOC by parsing the HTML from `.toc`. With the change, it (tries to) use `.toc_tokens`, but fails due to this issue.

FWIW, I think MkDocs wants this to be plain text in the end, but HTML makes the most sense to me in general: after all, the purpose of this lib is to convert Markdown to HTML. (MkDocs would then just need to parse the HTML fragment and strip out the tags.)
date: 2019-11-26
patch: diff --git a/markdown/extensions/toc.py b/markdown/extensions/toc.py
index d490a06..e42836a 100644
--- a/markdown/extensions/toc.py
+++ b/markdown/extensions/toc.py
@@ -43,7 +43,7 @@ def unique(id, ids):
     return id
 
 
-def stashedHTML2text(text, md):
+def stashedHTML2text(text, md, strip_entities=True):
     """ Extract raw HTML from stash, reduce to plain text and swap with placeholder. """
     def _html_sub(m):
         """ Substitute raw html with plain text. """
@@ -51,8 +51,11 @@ def stashedHTML2text(text, md):
             raw = md.htmlStash.rawHtmlBlocks[int(m.group(1))]
         except (IndexError, TypeError):  # pragma: no cover
             return m.group(0)
-        # Strip out tags and entities - leaveing text
-        return re.sub(r'(<[^>]+>)|(&[\#a-zA-Z0-9]+;)', '', raw)
+        # Strip out tags and/or entities - leaving text
+        res = re.sub(r'(<[^>]+>)', '', raw)
+        if strip_entities:
+            res = re.sub(r'(&[\#a-zA-Z0-9]+;)', '', res)
+        return res
 
     return HTML_PLACEHOLDER_RE.sub(_html_sub, text)
 
@@ -259,7 +262,9 @@ class TocTreeprocessor(Treeprocessor):
                 toc_tokens.append({
                     'level': int(el.tag[-1]),
                     'id': el.attrib["id"],
-                    'name': el.attrib.get('data-toc-label', text)
+                    'name': unescape(stashedHTML2text(
+                        el.attrib.get('data-toc-label', text), self.md, strip_entities=False
+                    ))
                 })
 
                 # Remove the data-toc-label attribute as it is no longer needed