Repository: mkdocs/mkdocs

Issue:
Add support to extract underline-ish title (setext-style headings)
The util function [`get_markdown_title`](https://github.com/mkdocs/mkdocs/blob/master/mkdocs/utils/__init__.py#L332) supports only ATX-style headers. It should also support:
```markdown
A level-one heading
===================

A level-two heading
-------------------
```
Fix support of stripping attribute list from determined page title if attr_list extension enabled
Add or fix support of stripping attribute list from through markdown file determined page title if attr_list markdown extension enabled

Closes #3136


Discussion:
I'm closing this as a duplicate of #1826. As the discussion there is long and covers multiple issues, I've copied the relevant [comment][0] below. Also note that in #1843 we have explicitly documented that setext style headers are not supported.

> The page title is set in [`mkdocs.structure.pages.Page._set_title`][1].  The [`mkdocs.utils.get_markdown_title`][2] function actually extracts the title from the Markdown source and would need to be significantly refactored as it currently assumes the header is contained on one line only (it doesn't support setext-style headers because they take 2 lines). Note that the title is set on the page before the Markdown source is rendered to HTML and this is non-negotiable. It would be easier to get the header from the rendered HTML (which would eliminate any concern about which format was used), but MkDocs needs the title to be set at an earlier stage than when it renders the Markdown to HTML. This is documented behavior in the Plugin API which we have promised to continue supporting. Also, I will only consider a change which enforces the use of an H1. If the first header is an H2, then the page title should be defined elsewhere (such as in meta-data) and a custom theme template would presumably be inserting the page title as an H1 separate from the page body.
> 
> Finally, this is very much something which could be addressed by a plugin. I expect either the [on_page_markdown][3] or [on_page_content][4] events could be used to overwrite the page title based on the Markdown source or rendered HTML respectively. As a third-party plugin, you could then use whatever criteria you want to define a page title.

[3]: https://www.mkdocs.org/user-guide/plugins/#on_page_markdown
[4]: https://www.mkdocs.org/user-guide/plugins/#on_page_content
[1]: https://github.com/mkdocs/mkdocs/blob/master/mkdocs/structure/pages.py#L143
[2]: https://github.com/mkdocs/mkdocs/blob/master/mkdocs/utils/__init__.py#L332

[0]: https://github.com/mkdocs/mkdocs/issues/1826#issuecomment-511497060


Resolving patch:
diff --git a/mkdocs/structure/pages.py b/mkdocs/structure/pages.py
--- a/mkdocs/structure/pages.py
+++ b/mkdocs/structure/pages.py
@@ -1,27 +1,36 @@
 from __future__ import annotations
 
+import copy
 import logging
 import os
 import posixpath
-from typing import TYPE_CHECKING, Any, Mapping, MutableMapping
+import warnings
+from typing import TYPE_CHECKING, Any, Callable, Mapping, MutableMapping
 from urllib.parse import unquote as urlunquote
 from urllib.parse import urljoin, urlsplit, urlunsplit
-from xml.etree.ElementTree import Element
+from xml.etree import ElementTree as etree
 
 import markdown
-from markdown.extensions import Extension
-from markdown.treeprocessors import Treeprocessor
+import markdown.extensions
+import markdown.postprocessors
+import markdown.treeprocessors
 from markdown.util import AMP_SUBSTITUTE
 
 from mkdocs.structure.files import File, Files
 from mkdocs.structure.toc import get_toc
-from mkdocs.utils import get_build_date, get_markdown_title, meta
+from mkdocs.utils import get_build_date, get_markdown_title, meta, weak_property
 
 if TYPE_CHECKING:
     from mkdocs.config.defaults import MkDocsConfig
     from mkdocs.structure.nav import Section
     from mkdocs.structure.toc import TableOfContents
 
+_unescape: Callable[[str], str]
+try:
+    _unescape = markdown.treeprocessors.UnescapeTreeprocessor().unescape  # type: ignore
+except AttributeError:
+    _unescape = markdown.postprocessors.UnescapePostprocessor().run
+
 
 log = logging.getLogger(__name__)
 
@@ -32,7 +41,8 @@ def __init__(
     ) -> None:
         file.page = self
         self.file = file
-        self.title = title
+        if title is not None:
+            self.title = title
 
         # Navigation attributes
         self.parent = None
@@ -50,6 +60,7 @@ def __init__(
 
         # Placeholders to be filled in later in the build process.
         self.markdown = None
+        self._title_from_render: str | None = None
         self.content = None
         self.toc = []  # type: ignore
         self.meta = {}
@@ -69,9 +80,6 @@ def __repr__(self):
     def _indent_print(self, depth=0):
         return '{}{}'.format('    ' * depth, repr(self))
 
-    title: str | None
-    """Contains the Title for the current page."""
-
     markdown: str | None
     """The original Markdown content from the file."""
 
@@ -226,11 +234,18 @@ def read_source(self, config: MkDocsConfig) -> None:
                 raise
 
         self.markdown, self.meta = meta.get_data(source)
-        self._set_title()
 
     def _set_title(self) -> None:
+        warnings.warn(
+            "_set_title is no longer used in MkDocs and will be removed soon.", DeprecationWarning
+        )
+
+    @weak_property
+    def title(self) -> str | None:
         """
-        Set the title for a Markdown document.
+        Returns the title for the current page.
+
+        Before calling `read_source()`, this value is empty. It can also be updated by `render()`.
 
         Check these in order and use the first that returns a valid title:
         - value provided on init (passed in from config)
@@ -238,48 +253,56 @@ def _set_title(self) -> None:
         - content of the first H1 in Markdown content
         - convert filename to title
         """
-        if self.title is not None:
-            return
+        if self.markdown is None:
+            return None
 
         if 'title' in self.meta:
-            self.title = self.meta['title']
-            return
+            return self.meta['title']
 
-        assert self.markdown is not None
-        title = get_markdown_title(self.markdown)
+        if self._title_from_render:
+            return self._title_from_render
+        elif self.content is None:  # Preserve legacy behavior only for edge cases in plugins.
+            title_from_md = get_markdown_title(self.markdown)
+            if title_from_md is not None:
+                return title_from_md
 
-        if title is None:
-            if self.is_homepage:
-                title = 'Home'
-            else:
-                title = self.file.name.replace('-', ' ').replace('_', ' ')
-                # Capitalize if the filename was all lowercase, otherwise leave it as-is.
-                if title.lower() == title:
-                    title = title.capitalize()
+        if self.is_homepage:
+            return 'Home'
 
-        self.title = title
+        title = self.file.name.replace('-', ' ').replace('_', ' ')
+        # Capitalize if the filename was all lowercase, otherwise leave it as-is.
+        if title.lower() == title:
+            title = title.capitalize()
+        return title
 
     def render(self, config: MkDocsConfig, files: Files) -> None:
         """
         Convert the Markdown source file to HTML as per the config.
         """
-        extensions = [_RelativePathExtension(self.file, files), *config['markdown_extensions']]
+        if self.markdown is None:
+            raise RuntimeError("`markdown` field hasn't been set (via `read_source`)")
 
+        relative_path_extension = _RelativePathExtension(self.file, files)
+        extract_title_extension = _ExtractTitleExtension()
         md = markdown.Markdown(
-            extensions=extensions,
+            extensions=[
+                relative_path_extension,
+                extract_title_extension,
+                *config['markdown_extensions'],
+            ],
             extension_configs=config['mdx_configs'] or {},
         )
-        assert self.markdown is not None
         self.content = md.convert(self.markdown)
         self.toc = get_toc(getattr(md, 'toc_tokens', []))
+        self._title_from_render = extract_title_extension.title
 
 
-class _RelativePathTreeprocessor(Treeprocessor):
+class _RelativePathTreeprocessor(markdown.treeprocessors.Treeprocessor):
     def __init__(self, file: File, files: Files) -> None:
         self.file = file
         self.files = files
 
-    def run(self, root: Element) -> Element:
+    def run(self, root: etree.Element) -> etree.Element:
         """
         Update urls on anchors and images to make them relative
 
@@ -335,7 +358,7 @@ def path_to_url(self, url: str) -> str:
         return urlunsplit(components)
 
 
-class _RelativePathExtension(Extension):
+class _RelativePathExtension(markdown.extensions.Extension):
     """
     The Extension class is what we pass to markdown, it then
     registers the Treeprocessor.
@@ -348,3 +371,32 @@ def __init__(self, file: File, files: Files) -> None:
     def extendMarkdown(self, md: markdown.Markdown) -> None:
         relpath = _RelativePathTreeprocessor(self.file, self.files)
         md.treeprocessors.register(relpath, "relpath", 0)
+
+
+class _ExtractTitleExtension(markdown.extensions.Extension):
+    def __init__(self) -> None:
+        self.title: str | None = None
+
+    def extendMarkdown(self, md: markdown.Markdown) -> None:
+        md.treeprocessors.register(
+            _ExtractTitleTreeprocessor(self),
+            "mkdocs_extract_title",
+            priority=1,  # Close to the end.
+        )
+
+
+class _ExtractTitleTreeprocessor(markdown.treeprocessors.Treeprocessor):
+    def __init__(self, ext: _ExtractTitleExtension) -> None:
+        self.ext = ext
+
+    def run(self, root: etree.Element) -> etree.Element:
+        for el in root:
+            if el.tag == 'h1':
+                # Drop anchorlink from the element, if present.
+                if len(el) > 0 and el[-1].tag == 'a' and not (el.tail or '').strip():
+                    el = copy.copy(el)
+                    del el[-1]
+                # Extract the text only, recursively.
+                self.ext.title = _unescape(''.join(el.itertext()))
+            break
+        return root
diff --git a/mkdocs/utils/__init__.py b/mkdocs/utils/__init__.py
--- a/mkdocs/utils/__init__.py
+++ b/mkdocs/utils/__init__.py
@@ -385,13 +385,7 @@ def dirname_to_title(dirname: str) -> str:
 
 
 def get_markdown_title(markdown_src: str) -> str | None:
-    """
-    Get the title of a Markdown document. The title in this case is considered
-    to be a H1 that occurs before any other content in the document.
-    The procedure is then to iterate through the lines, stopping at the first
-    non-whitespace content. If it is a title, return that, otherwise return
-    None.
-    """
+    """Soft-deprecated, do not use."""
     lines = markdown_src.replace('\r\n', '\n').replace('\r', '\n').split('\n')
     while lines:
         line = lines.pop(0).strip()
@@ -460,6 +454,19 @@ def get_counts(self) -> list[tuple[str, int]]:
         return [(logging.getLevelName(k), v) for k, v in sorted(self.counts.items(), reverse=True)]
 
 
+class weak_property:
+    """Same as a read-only property, but allows overwriting the field for good."""
+
+    def __init__(self, func):
+        self.func = func
+        self.__doc__ = func.__doc__
+
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            return self
+        return self.func(instance)
+
+
 def __getattr__(name: str):
     if name == 'warning_filter':
         warnings.warn(


Test patch:
diff --git a/mkdocs/tests/structure/page_tests.py b/mkdocs/tests/structure/page_tests.py
--- a/mkdocs/tests/structure/page_tests.py
+++ b/mkdocs/tests/structure/page_tests.py
@@ -4,6 +4,7 @@
 import unittest
 from unittest import mock
 
+from mkdocs.config.defaults import MkDocsConfig
 from mkdocs.structure.files import File, Files
 from mkdocs.structure.pages import Page
 from mkdocs.tests.base import dedent, load_config, tempdir
@@ -299,7 +300,84 @@ def test_page_title_from_markdown(self):
         self.assertEqual(pg.parent, None)
         self.assertEqual(pg.previous_page, None)
         self.assertEqual(pg.title, 'Welcome to MkDocs')
-        self.assertEqual(pg.toc, [])
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'Welcome to MkDocs')
+
+    _SETEXT_CONTENT = dedent(
+        '''
+        Welcome to MkDocs Setext
+        ========================
+
+        This tests extracting a setext style title.
+        '''
+    )
+
+    @tempdir(files={'testing_setext_title.md': _SETEXT_CONTENT})
+    def test_page_title_from_setext_markdown(self, docs_dir):
+        cfg = load_config()
+        fl = File('testing_setext_title.md', docs_dir, docs_dir, use_directory_urls=True)
+        pg = Page(None, fl, cfg)
+        self.assertIsNone(pg.title)
+        pg.read_source(cfg)
+        self.assertEqual(pg.title, 'Testing setext title')
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'Welcome to MkDocs Setext')
+
+    @tempdir(files={'testing_setext_title.md': _SETEXT_CONTENT})
+    def test_page_title_from_markdown_stripped_anchorlinks(self, docs_dir):
+        cfg = MkDocsConfig()
+        cfg.site_name = 'example'
+        cfg.markdown_extensions = {'toc': {'permalink': '&'}}
+        self.assertEqual(cfg.validate(), ([], []))
+        fl = File('testing_setext_title.md', docs_dir, docs_dir, use_directory_urls=True)
+        pg = Page(None, fl, cfg)
+        pg.read_source(cfg)
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'Welcome to MkDocs Setext')
+
+    _FORMATTING_CONTENT = dedent(
+        '''
+        # Hello *beautiful* `world`
+
+        Hi.
+        '''
+    )
+
+    @tempdir(files={'testing_formatting.md': _FORMATTING_CONTENT})
+    def test_page_title_from_markdown_strip_formatting(self, docs_dir):
+        cfg = load_config()
+        fl = File('testing_formatting.md', docs_dir, docs_dir, use_directory_urls=True)
+        pg = Page(None, fl, cfg)
+        pg.read_source(cfg)
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'Hello beautiful world')
+
+    _ATTRLIST_CONTENT = dedent(
+        '''
+        # Welcome to MkDocs Attr { #welcome }
+
+        This tests extracting the title, with enabled attr_list markdown_extension.
+        '''
+    )
+
+    @tempdir(files={'testing_attr_list.md': _ATTRLIST_CONTENT})
+    def test_page_title_from_markdown_stripped_attr_list(self, docs_dir):
+        cfg = load_config()
+        cfg.markdown_extensions.append('attr_list')
+        fl = File('testing_attr_list.md', docs_dir, docs_dir, use_directory_urls=True)
+        pg = Page(None, fl, cfg)
+        pg.read_source(cfg)
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'Welcome to MkDocs Attr')
+
+    @tempdir(files={'testing_attr_list.md': _ATTRLIST_CONTENT})
+    def test_page_title_from_markdown_preserved_attr_list(self, docs_dir):
+        cfg = load_config()
+        fl = File('testing_attr_list.md', docs_dir, docs_dir, use_directory_urls=True)
+        pg = Page(None, fl, cfg)
+        pg.read_source(cfg)
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'Welcome to MkDocs Attr { #welcome }')
 
     def test_page_title_from_meta(self):
         cfg = load_config(docs_dir=self.DOCS_DIR)
@@ -324,6 +402,8 @@ def test_page_title_from_meta(self):
         self.assertEqual(pg.previous_page, None)
         self.assertEqual(pg.title, 'A Page Title')
         self.assertEqual(pg.toc, [])
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'A Page Title')
 
     def test_page_title_from_filename(self):
         cfg = load_config(docs_dir=self.DOCS_DIR)
@@ -347,7 +427,8 @@ def test_page_title_from_filename(self):
         self.assertEqual(pg.parent, None)
         self.assertEqual(pg.previous_page, None)
         self.assertEqual(pg.title, 'Page title')
-        self.assertEqual(pg.toc, [])
+        pg.render(cfg, fl)
+        self.assertEqual(pg.title, 'Page title')
 
     def test_page_title_from_capitalized_filename(self):
         cfg = load_config(docs_dir=self.DOCS_DIR)
@@ -371,7 +452,6 @@ def test_page_title_from_capitalized_filename(self):
         self.assertEqual(pg.parent, None)
         self.assertEqual(pg.previous_page, None)
         self.assertEqual(pg.title, 'pageTitle')
-        self.assertEqual(pg.toc, [])
 
     def test_page_title_from_homepage_filename(self):
         cfg = load_config(docs_dir=self.DOCS_DIR)