Repository: mkdocs/mkdocs

Issue:
Header and navigation sidebars display incorrectly arbitrary markup
Hello!

## Since when does the bug occur?

I recently upgraded my `mkdocs` package from `1.4.2` to `1.5.2`, and I noticed this bug when...
* I don't specify a page title in `mkdocs.yml` file or in the page metadata (in yaml).
* I use an emoji (HTML in general) in the first level section title in the markdown file:

```md
# Intégration et Déploiement Continu sur :simple-gitlab: Gitlab
```

For example (with `mkdocs-material` theme and `readthedocs` theme):

![263000285-555a71e9-3530-4a63-906a-17617c1d0be3](https://github.com/mkdocs/mkdocs/assets/46783975/fafe7715-678b-4813-9535-c6ddf3df9acd)
![263459179-2afb6262-bde4-4727-b0e3-c5c4da035873](https://github.com/mkdocs/mkdocs/assets/46783975/90edf72c-61df-4d96-a8ba-aed2e35332a0)

... while it was displayed like that before:

![262992157-dcebad1a-82b1-445d-b250-8ef7aa34bafb](https://github.com/mkdocs/mkdocs/assets/46783975/8e25e34a-036d-4358-8929-f08d473e6004)

More generally, some parts like the search page of `readthedocs` theme don't handle well the HTML in title sections.

## External links:

[`readthedocs` theme pictures and bug explanations](https://github.com/squidfunk/mkdocs-material/issues/5893) are from @squidfunk.

Discussion:

Patch:
diff --git a/mkdocs/structure/pages.py b/mkdocs/structure/pages.py
index ef13e512..8b0a642d 100644
--- a/mkdocs/structure/pages.py
+++ b/mkdocs/structure/pages.py
@@ -10,6 +10,7 @@ from urllib.parse import unquote as urlunquote
 from urllib.parse import urljoin, urlsplit, urlunsplit
 
 import markdown
+import markdown.extensions.toc
 import markdown.htmlparser  # type: ignore
 import markdown.postprocessors
 import markdown.treeprocessors
@@ -549,7 +550,7 @@ class _HTMLHandler(markdown.htmlparser.htmlparser.HTMLParser):  # type: ignore[n
 
 class _ExtractTitleTreeprocessor(markdown.treeprocessors.Treeprocessor):
     title: str | None = None
-    postprocessors: Sequence[markdown.postprocessors.Postprocessor] = ()
+    md: markdown.Markdown
 
     def run(self, root: etree.Element) -> etree.Element:
         for el in root:
@@ -561,14 +562,15 @@ class _ExtractTitleTreeprocessor(markdown.treeprocessors.Treeprocessor):
                 # Extract the text only, recursively.
                 title = ''.join(el.itertext())
                 # Unescape per Markdown implementation details.
-                for pp in self.postprocessors:
-                    title = pp.run(title)
-                self.title = title
+                title = markdown.extensions.toc.stashedHTML2text(
+                    title, self.md, strip_entities=False
+                )
+                self.title = title.strip()
             break
         return root
 
     def _register(self, md: markdown.Markdown) -> None:
-        self.postprocessors = tuple(md.postprocessors)
+        self.md = md
         md.treeprocessors.register(self, "mkdocs_extract_title", priority=-1)  # After the end.
 
 
diff --git a/mkdocs/structure/toc.py b/mkdocs/structure/toc.py
index 6d09867b..e1df40be 100644
--- a/mkdocs/structure/toc.py
+++ b/mkdocs/structure/toc.py
@@ -33,7 +33,7 @@ class AnchorLink:
         self.children = []
 
     title: str
-    """The text of the item."""
+    """The text of the item, as HTML."""
 
     @property
     def url(self) -> str:
diff --git a/pyproject.toml b/pyproject.toml
index 28355b73..71bc189e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -36,7 +36,7 @@ dependencies = [
     "click >=7.0",
     "Jinja2 >=2.11.1",
     "markupsafe >=2.0.1",
-    "Markdown >=3.3.6",
+    "Markdown >=3.4.1",
     "PyYAML >=5.1",
     "watchdog >=2.0",
     "ghp-import >=1.0",
@@ -57,7 +57,7 @@ min-versions = [
     "click ==7.0",
     "Jinja2 ==2.11.1",
     "markupsafe ==2.0.1",
-    "Markdown ==3.3.6",
+    "Markdown ==3.4.1",
     "PyYAML ==5.1",
     "watchdog ==2.0",
     "ghp-import ==1.0",


Test patch:
diff --git a/mkdocs/tests/structure/page_tests.py b/mkdocs/tests/structure/page_tests.py
index ee6aa159..77d8542b 100644
--- a/mkdocs/tests/structure/page_tests.py
+++ b/mkdocs/tests/structure/page_tests.py
@@ -6,9 +6,11 @@ import textwrap
 import unittest
 from unittest import mock
 
+import markdown
+
 from mkdocs.config.defaults import MkDocsConfig
 from mkdocs.structure.files import File, Files
-from mkdocs.structure.pages import Page, _RelativePathTreeprocessor
+from mkdocs.structure.pages import Page, _ExtractTitleTreeprocessor, _RelativePathTreeprocessor
 from mkdocs.tests.base import dedent, tempdir
 
 DOCS_DIR = os.path.join(
@@ -315,9 +317,16 @@ class PageTests(unittest.TestCase):
         self.assertEqual(pg.parent, None)
         self.assertEqual(pg.previous_page, None)
         self.assertEqual(pg.title, 'Welcome to MkDocs')
-        pg.render(cfg, fl)
+        pg.render(cfg, Files([fl]))
         self.assertEqual(pg.title, 'Welcome to MkDocs')
 
+    def _test_extract_title(self, content, expected, extensions={}):
+        md = markdown.Markdown(extensions=list(extensions.keys()), extension_configs=extensions)
+        extract_title_ext = _ExtractTitleTreeprocessor()
+        extract_title_ext._register(md)
+        md.convert(content)
+        self.assertEqual(extract_title_ext.title, expected)
+
     _SETEXT_CONTENT = dedent(
         '''
         Welcome to MkDocs Setext
@@ -327,46 +336,37 @@ class PageTests(unittest.TestCase):
         '''
     )
 
-    @tempdir(files={'testing_setext_title.md': _SETEXT_CONTENT})
-    def test_page_title_from_setext_markdown(self, docs_dir):
-        cfg = load_config()
-        fl = File('testing_setext_title.md', docs_dir, docs_dir, use_directory_urls=True)
-        pg = Page(None, fl, cfg)
-        self.assertIsNone(pg.title)
-        pg.read_source(cfg)
-        self.assertEqual(pg.title, 'Testing setext title')
-        pg.render(cfg, fl)
-        self.assertEqual(pg.title, 'Welcome to MkDocs Setext')
-
-    @tempdir(files={'testing_setext_title.md': _SETEXT_CONTENT})
-    def test_page_title_from_markdown_stripped_anchorlinks(self, docs_dir):
-        cfg = MkDocsConfig()
-        cfg.site_name = 'example'
-        cfg.markdown_extensions = {'toc': {'permalink': '&'}}
-        self.assertEqual(cfg.validate(), ([], []))
-        fl = File('testing_setext_title.md', docs_dir, docs_dir, use_directory_urls=True)
-        pg = Page(None, fl, cfg)
-        pg.read_source(cfg)
-        pg.render(cfg, fl)
-        self.assertEqual(pg.title, 'Welcome to MkDocs Setext')
+    def test_page_title_from_setext_markdown(self):
+        self._test_extract_title(
+            self._SETEXT_CONTENT,
+            expected='Welcome to MkDocs Setext',
+        )
 
-    _FORMATTING_CONTENT = dedent(
-        '''
-        # \\*Hello --- *beautiful* `world`
+    def test_page_title_from_markdown_stripped_anchorlinks(self):
+        self._test_extract_title(
+            self._SETEXT_CONTENT,
+            extensions={'toc': {'permalink': '&'}},
+            expected='Welcome to MkDocs Setext',
+        )
 
-        Hi.
-        '''
-    )
+    def test_page_title_from_markdown_strip_formatting(self):
+        self._test_extract_title(
+            '''# \\*Hello --- *beautiful* `wor<dl>`''',
+            extensions={'smarty': {}},
+            expected='*Hello &mdash; beautiful wor&lt;dl&gt;',
+        )
 
-    @tempdir(files={'testing_formatting.md': _FORMATTING_CONTENT})
-    def test_page_title_from_markdown_strip_formatting(self, docs_dir):
-        cfg = load_config()
-        cfg.markdown_extensions.append('smarty')
-        fl = File('testing_formatting.md', docs_dir, docs_dir, use_directory_urls=True)
-        pg = Page(None, fl, cfg)
-        pg.read_source(cfg)
-        pg.render(cfg, fl)
-        self.assertEqual(pg.title, '*Hello &mdash; beautiful world')
+    def test_page_title_from_markdown_strip_raw_html(self):
+        self._test_extract_title(
+            '''# Hello <b>world</b>''',
+            expected='Hello world',
+        )
+
+    def test_page_title_from_markdown_strip_image(self):
+        self._test_extract_title(
+            '''# Hi ![😄](hah.png)''',
+            expected='Hi',  # TODO: Should the alt text of the image be extracted?
+        )
 
     _ATTRLIST_CONTENT = dedent(
         '''
@@ -376,24 +376,18 @@ class PageTests(unittest.TestCase):
         '''
     )
 
-    @tempdir(files={'testing_attr_list.md': _ATTRLIST_CONTENT})
-    def test_page_title_from_markdown_stripped_attr_list(self, docs_dir):
-        cfg = load_config()
-        cfg.markdown_extensions.append('attr_list')
-        fl = File('testing_attr_list.md', docs_dir, docs_dir, use_directory_urls=True)
-        pg = Page(None, fl, cfg)
-        pg.read_source(cfg)
-        pg.render(cfg, fl)
-        self.assertEqual(pg.title, 'Welcome to MkDocs Attr')
+    def test_page_title_from_markdown_stripped_attr_list(self):
+        self._test_extract_title(
+            self._ATTRLIST_CONTENT,
+            extensions={'attr_list': {}},
+            expected='Welcome to MkDocs Attr',
+        )
 
-    @tempdir(files={'testing_attr_list.md': _ATTRLIST_CONTENT})
-    def test_page_title_from_markdown_preserved_attr_list(self, docs_dir):
-        cfg = load_config()
-        fl = File('testing_attr_list.md', docs_dir, docs_dir, use_directory_urls=True)
-        pg = Page(None, fl, cfg)
-        pg.read_source(cfg)
-        pg.render(cfg, fl)
-        self.assertEqual(pg.title, 'Welcome to MkDocs Attr { #welcome }')
+    def test_page_title_from_markdown_preserved_attr_list(self):
+        self._test_extract_title(
+            self._ATTRLIST_CONTENT,
+            expected='Welcome to MkDocs Attr { #welcome }',
+        )
 
     def test_page_title_from_meta(self):
         cfg = load_config(docs_dir=DOCS_DIR)
@@ -418,7 +412,7 @@ class PageTests(unittest.TestCase):
         self.assertEqual(pg.previous_page, None)
         self.assertEqual(pg.title, 'A Page Title')
         self.assertEqual(pg.toc, [])
-        pg.render(cfg, fl)
+        pg.render(cfg, Files([fl]))
         self.assertEqual(pg.title, 'A Page Title')
 
     def test_page_title_from_filename(self):
@@ -443,7 +437,7 @@ class PageTests(unittest.TestCase):
         self.assertEqual(pg.parent, None)
         self.assertEqual(pg.previous_page, None)
         self.assertEqual(pg.title, 'Page title')
-        pg.render(cfg, fl)
+        pg.render(cfg, Files([fl]))
         self.assertEqual(pg.title, 'Page title')
 
     def test_page_title_from_capitalized_filename(self):
@@ -704,7 +698,7 @@ class PageTests(unittest.TestCase):
         pg.read_source(cfg)
         self.assertEqual(pg.content, None)
         self.assertEqual(pg.toc, [])
-        pg.render(cfg, [fl])
+        pg.render(cfg, Files([fl]))
         self.assertTrue(
             pg.content.startswith('<h1 id="welcome-to-mkdocs">Welcome to MkDocs</h1>\n')
         )