Repository: hynek/doc2dash (language: python)

Issue:
Can't find anchors to patch TOC for pydoctor docs (Twisted)
Hi Hynek. I'm trying to generate the docset for Twisted 22.8.0 and doc2dash is not able to add the TOC anchors for Dash.

I'm getting errors like this:

```
Can't find anchor 'func_UP_ARROW' (EntryType.METHOD) in 'twisted.conch.insults.window.ScrolledArea.html'.
Can't find anchor 'func_DOWN_ARROW' (EntryType.METHOD) in 'twisted.conch.insults.window.ScrolledArea.html'.
Can't find anchor 'func_LEFT_ARROW' (EntryType.METHOD) in 'twisted.conch.insults.window.ScrolledArea.html'.
Can't find anchor 'func_RIGHT_ARROW' (EntryType.METHOD) in 'twisted.conch.insults.window.ScrolledArea.html'.
Can't find anchor 'filthy' (EntryType.METHOD) in 'twisted.conch.insults.window.ScrolledArea.html'.
```

You can download the docs I'm running doc2dash over from https://www.dropbox.com/s/415yxsar09cije8/twisted_apidocs.zip?dl=0.

Thanks for the help!


Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2db44f3..60d7799 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://github.com/hynek/doc2dash/compare/3.0.0...HEAD)
 
+### Fixed
+
+- The table of contents-generation for *pydoctor*-based documentation has been restored. [#133](https://github.com/hynek/doc2dash/pull/133)
+
 
 ## [3.0.0](https://github.com/hynek/doc2dash/compare/2.4.1...3.0.0) - 2022-09-13
 
diff --git a/src/doc2dash/parsers/intersphinx.py b/src/doc2dash/parsers/intersphinx.py
index cf23a55..52078a2 100644
--- a/src/doc2dash/parsers/intersphinx.py
+++ b/src/doc2dash/parsers/intersphinx.py
@@ -176,6 +176,8 @@ def _find_entry_and_add_ref(
             or soup.find("span", id=anchor)
             # mkdocs / mkdocstrings
             or soup.find("a", {"class": "md-nav__link"}, href="#" + anchor)
+            # pydoctor
+            or soup.find("a", {"name": name})
         )
 
     if not pos:


Test patch:
diff --git a/tests/parsers/intersphinx/pydoctor_example.html b/tests/parsers/intersphinx/pydoctor_example.html
new file mode 100644
index 0000000..af5701e
--- /dev/null
+++ b/tests/parsers/intersphinx/pydoctor_example.html
@@ -0,0 +1,458 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+          "DTD/xhtml1-strict.dtd">
+<html>
+
+
+  <head>
+
+    <title>twisted._threads._convenience.Quit</title>
+    <meta name="generator" content="pydoctor 22.9.1">
+
+    </meta>
+    <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1 maximum-scale=1" />
+    <link rel="stylesheet" type="text/css" href="apidocs.css" />
+    <link rel="stylesheet" type="text/css" href="readthedocstheme.css" />
+    <link rel="stylesheet" type="text/css" href="extra.css" />
+</head>
+
+  <body>
+
+
+
+    <nav class="navbar navbar-default mainnavbar">
+
+
+  <div class="container-fluid">
+
+
+    <div class="navbar-header">
+
+      <div class="navlinks">
+        <span class="navbar-brand">
+          <a href="https://twisted.org/" class="projecthome">Twisted</a> <a href="index.html">API Documentation</a>
+        </span>
+
+        <a href="moduleIndex.html">
+          Modules
+        </a>
+
+        <a href="classIndex.html">
+          Classes
+        </a>
+
+        <a href="nameIndex.html">
+          Names
+        </a>
+
+        <div id="search-box-container">
+          <div class="input-group">
+            <input id="search-box" type="search" name="search-query" placeholder="Search..." aria-label="Search" minlength="2" class="form-control" autocomplete="off" />
+
+            <span class="input-group-btn">
+              <a style="display: none;" class="btn btn-default" id="search-clear-button" title="Clear" onclick="clearSearch()"><img src="fonts/x-circle.svg" alt="Clear" /></a>
+              <a class="btn btn-default" id="search-help-button" title="Help" onclick="toggleSearchHelpText()"><img src="fonts/info.svg" alt="Help" /></a>
+            </span>
+          </div>
+        </div>
+
+      </div>
+
+    </div>
+
+  </div>
+
+
+        <!-- Side navigation -->
+      <div class="sidebarcontainer">
+        <nav class="sidebar">
+
+
+  <div>
+
+      <div class="thingTitle">
+        <span>Class</span>
+        <code class="thisobject"><a href="twisted._threads._convenience.Quit.html" class="internal-link" title="This class"><wbr></wbr>Quit</a></code>
+      </div>
+
+      <div>
+
+
+
+
+
+
+
+
+        <div class="childrenKindTitle">Methods</div>
+        <ul>
+
+    <li class="">
+
+
+
+
+
+            <div class="itemName"><code><a href="#__init__" class="internal-link" title="twisted._threads._convenience.Quit.__init__">__init__</a></code>
+            </div>
+
+
+    </li><li class="">
+
+
+
+
+
+            <div class="itemName"><code><a href="#check" class="internal-link" title="twisted._threads._convenience.Quit.check">check</a></code>
+            </div>
+
+
+    </li><li class="">
+
+
+
+
+
+            <div class="itemName"><code><a href="#set" class="internal-link" title="twisted._threads._convenience.Quit.set">set</a></code>
+            </div>
+
+
+    </li>
+</ul>
+
+
+
+
+        <div class="childrenKindTitle">Attributes</div>
+        <ul>
+
+    <li class="">
+
+
+
+
+
+            <div class="itemName"><code><a href="#isSet" class="internal-link" title="twisted._threads._convenience.Quit.isSet">is<wbr></wbr>Set</a></code>
+            </div>
+
+
+    </li>
+</ul>
+
+
+
+
+      </div>
+  </div><div>
+
+      <div class="thingTitle">
+        <span>Module</span>
+        <code><a href="twisted._threads._convenience.html" class="internal-link" title="The parent of this class">_convenience</a></code>
+      </div>
+
+      <div>
+
+
+
+
+
+        <div class="childrenKindTitle">Classes</div>
+        <ul>
+
+    <li class=" thisobject">
+
+
+
+
+
+            <div class="itemName"><code><a href="twisted._threads._convenience.Quit.html" class="internal-link" title="twisted._threads._convenience.Quit"><wbr></wbr>Quit</a></code>
+            </div>
+
+
+    </li>
+</ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+      </div>
+  </div>
+
+</nav>
+        <!-- No sidebar toggle for read the docs theme, the sidebar is always
+          visible when the screen is width enough.
+        -->
+      </div>
+
+    </nav>
+
+    <div class="container-fluid">
+
+      <div id="main" class="">
+
+
+
+
+        <div id="search-results-container" style="display: none;">
+
+          <div id="search-buttons">
+            <span class="label label-default" id="search-docstrings-button">
+            <label class="checkbox-inline">
+              <input type="checkbox" id="toggle-search-in-docstrings-checkbox" value="false" onclick="toggleSearchInDocstrings()">
+                search in docstrings
+              </input>
+            </label>
+            </span>
+
+          </div>
+
+          <noscript>
+              <h1>Cannot search: JavaScript is not supported/enabled in your browser.</h1>
+          </noscript>
+
+          <div class="hint" id="search-help-box">
+            <p class="rst-last">
+
+              Search bar offers the following options:
+              <ul>
+                  <li>
+                    <strong>Term presence.</strong> The below example searches for documents that
+                      must contain “foo”, might contain “bar” and must not contain “baz”: <code>+foo bar -baz</code>
+                  </li>
+
+                  <li>
+                    <strong>Wildcards.</strong> The below example searches for documents with words beginning with “foo”: <code>foo*</code>
+                  </li>
+
+                  <li>
+                    <strong>Search in specific fields.</strong> The following search matches all objects
+                    in "twisted.mail" that matches “search”: <code>+qname:twisted.mail.* +search</code>
+
+                    <p>
+                      Possible fields: 'name', 'qname' (fully qualified name), 'docstring', and 'kind'.
+                      Last two fields are only applicable if "search in docstrings" is enabled.
+                    </p>
+                  </li>
+
+                  <li>
+                    <strong>Fuzzy matches.</strong> The following search matches all documents
+                    that have a word within 1 edit distance of “foo”: <code>foo~1</code>
+                  </li>
+              </ul>
+
+            </p>
+          </div>
+
+          <div id="search-status"> </div>
+
+          <div class="warning" id="search-warn-box" style="display: none;">
+            <p class="rst-last"><span id="search-warn"></span></p>
+          </div>
+
+          <table id="search-results">
+            <!-- Filled dynamically by JS -->
+          </table>
+
+          <div style="margin-top: 8px;">
+            <p>Results provided by <a href="https://lunrjs.com">Lunr.js</a></p>
+          </div>
+        </div>
+
+        <div class="page-header">
+          <h1 class="class"><code><code><a href="index.html" class="internal-link">twisted</a></code><wbr></wbr>.<code><a href="twisted._threads.html" class="internal-link" title="twisted._threads">_threads</a></code><wbr></wbr>.<code><a href="twisted._threads._convenience.html" class="internal-link" title="twisted._threads._convenience">_convenience</a></code><wbr></wbr>.<code><a href="twisted._threads._convenience.Quit.html" class="internal-link" title="twisted._threads._convenience.Quit">Quit</a></code></code></h1>
+          <div id="showPrivate">
+            <button class="btn btn-link" onclick="togglePrivate()">Toggle Private API</button>
+          </div>
+        </div>
+
+        <div class="categoryHeader">
+          class documentation
+        </div>
+
+        <div class="extrasDocstring">
+          <p><code><span class="py-keyword">class</span> <span class="py-defname">Quit</span>: <a href="https://github.com/twisted/twisted/tree/trunk/src/_threads/_convenience.py#L13" class="sourceLink">(source)</a></code></p>
+          <p><a href="classIndex.html#twisted._threads._convenience.Quit">View In Hierarchy</a></p>
+        </div>
+
+        <div class="moduleDocstring">
+          <div><p>A flag representing whether a worker has been quit.</p>
+</div>
+        </div>
+
+        <div id="splitTables">
+          <table class="children sortable" id="id5">
+
+
+  <tr class="method">
+
+    <td>Method</td>
+    <td><code><a href="#__init__" class="internal-link" title="twisted._threads._convenience.Quit.__init__">__init__</a></code></td>
+    <td>Create a <code><a href="twisted._threads._convenience.Quit.html" class="internal-link" title="twisted._threads._convenience.Quit">Quit</a></code> un-set.</td>
+  </tr><tr class="method">
+
+    <td>Method</td>
+    <td><code><a href="#check" class="internal-link" title="twisted._threads._convenience.Quit.check">check</a></code></td>
+    <td>Check if the flag has been set.</td>
+  </tr><tr class="method">
+
+    <td>Method</td>
+    <td><code><a href="#set" class="internal-link" title="twisted._threads._convenience.Quit.set">set</a></code></td>
+    <td>Set the flag if it has not been set.</td>
+  </tr><tr class="instancevariable">
+
+    <td>Instance Variable</td>
+    <td><code><a href="#isSet" class="internal-link" title="twisted._threads._convenience.Quit.isSet">is<wbr></wbr>Set</a></code></td>
+    <td>Whether this flag is set.</td>
+  </tr>
+</table>
+
+
+
+        </div>
+
+        <div id="childList">
+          <div class="basemethod">
+
+
+  <a name="twisted._threads._convenience.Quit.__init__">
+
+  </a>
+  <a name="__init__">
+
+  </a>
+  <div class="functionHeader">
+
+    <span class="py-keyword">def</span> <span class="py-defname">__init__</span>(self):
+    <a class="sourceLink" href="https://github.com/twisted/twisted/tree/trunk/src/_threads/_convenience.py#L21">
+
+      (source)
+    </a>
+    <a class="headerLink" href="#__init__" title="twisted._threads._convenience.Quit.__init__">
+
+
+      ¶
+    </a>
+  </div>
+  <div class="docstring functionBody">
+
+    <div><p>Create a <code><a href="twisted._threads._convenience.Quit.html" class="internal-link" title="twisted._threads._convenience.Quit">Quit</a></code> un-set.</p>
+</div>
+  </div>
+</div><div class="basemethod">
+
+
+  <a name="twisted._threads._convenience.Quit.check">
+
+  </a>
+  <a name="check">
+
+  </a>
+  <div class="functionHeader">
+
+    <span class="py-keyword">def</span> <span class="py-defname">check</span>(self):
+    <a class="sourceLink" href="https://github.com/twisted/twisted/tree/trunk/src/_threads/_convenience.py#L36">
+
+      (source)
+    </a>
+    <a class="headerLink" href="#check" title="twisted._threads._convenience.Quit.check">
+
+
+      ¶
+    </a>
+  </div>
+  <div class="docstring functionBody">
+
+    <div><p>Check if the flag has been set.</p>
+<table class="fieldTable"><tr class="fieldStart"><td class="fieldName" colspan="2">Raises</td></tr><tr><td class="fieldArgContainer"><code>AlreadyQuit</code></td><td>If it has been set.</td></tr></table></div>
+  </div>
+</div><div class="basemethod">
+
+
+  <a name="twisted._threads._convenience.Quit.set">
+
+  </a>
+  <a name="set">
+
+  </a>
+  <div class="functionHeader">
+
+    <span class="py-keyword">def</span> <span class="py-defname">set</span>(self):
+    <a class="sourceLink" href="https://github.com/twisted/twisted/tree/trunk/src/_threads/_convenience.py#L27">
+
+      (source)
+    </a>
+    <a class="headerLink" href="#set" title="twisted._threads._convenience.Quit.set">
+
+
+      ¶
+    </a>
+  </div>
+  <div class="docstring functionBody">
+
+    <div><p>Set the flag if it has not been set.</p>
+<table class="fieldTable"><tr class="fieldStart"><td class="fieldName" colspan="2">Raises</td></tr><tr><td class="fieldArgContainer"><code>AlreadyQuit</code></td><td>If it has been set.</td></tr></table></div>
+  </div>
+</div><div class="baseinstancevariable">
+
+
+  <a name="twisted._threads._convenience.Quit.isSet">
+
+  </a>
+  <a name="isSet">
+
+  </a>
+  <div class="functionHeader">
+
+    <span class="py-defname">isSet</span>: <code><a href="https://docs.python.org/3/library/functions.html#bool" class="intersphinx-link">bool</a></code> =
+    <a class="sourceLink" href="https://github.com/twisted/twisted/tree/trunk/src/_threads/_convenience.py#L17">
+
+      (source)
+    </a>
+    <a class="headerLink" href="#isSet" title="twisted._threads._convenience.Quit.isSet">
+
+
+      ¶
+    </a>
+  </div>
+  <div class="functionBody">
+
+    <div><p>Whether this flag is set.</p>
+</div>
+
+  </div>
+</div>
+        </div>
+
+      </div>
+    </div>
+
+    <footer class="navbar navbar-default">
+    <hr />
+
+
+    <div class="container">
+      <a href="index.html">API Documentation</a> for <a href="https://twisted.org/" class="projecthome">Twisted</a>,
+    generated by <a href="https://github.com/twisted/pydoctor/">pydoctor</a>
+      22.9.1 at 2022-09-30 12:50:42.
+    </div>
+
+    <script src="ajax.js" type="text/javascript"></script>
+    <script src="searchlib.js" type="text/javascript"></script>
+    <script src="search.js" type="text/javascript"></script>
+
+  </footer>
+
+    <script src="pydoctor.js" type="text/javascript"></script>
+
+  </body>
+</html>
diff --git a/tests/parsers/intersphinx/test_intersphinx.py b/tests/parsers/intersphinx/test_intersphinx.py
index eb425ac..8376de9 100644
--- a/tests/parsers/intersphinx/test_intersphinx.py
+++ b/tests/parsers/intersphinx/test_intersphinx.py
@@ -170,7 +170,15 @@ def create_entry(self, dash_type, key, inv_entry):
 @pytest.fixture(name="soup")
 def _soup():
     return BeautifulSoup(
-        (Path(HERE) / "function_example.html").read_text(),
+        (Path(HERE) / "function_example.html").read_text(encoding="utf-8"),
+        "html.parser",
+    )
+
+
+@pytest.fixture(name="pydoctor_soup")
+def _pydoctor_soup():
+    return BeautifulSoup(
+        (Path(HERE) / "pydoctor_example.html").read_text(encoding="utf-8"),
         "html.parser",
     )
 
@@ -261,6 +269,27 @@ def test_patch_section(self, soup):
             '<section id="chains">' in str(soup)
         )
 
+    def test_pydoctor_class(self, pydoctor_soup):
+        """
+        Pydoctor classes are found.
+        """
+        ref = "//apple_ref/cpp/Word/twisted._threads._convenience.Quit.isSet"
+
+        patched = _find_entry_and_add_ref(
+            pydoctor_soup,
+            name="twisted._threads._convenience.Quit.isSet",
+            type=EntryType.WORD,
+            anchor="isSet",
+            ref=ref,
+        )
+
+        assert patched
+        assert (
+            f'<a class="dashAnchor" name="{ref}"></a>'
+            '<a name="twisted._threads._convenience.Quit.isSet">'
+            in str(pydoctor_soup)
+        )
+
 
 class TestIntersphinxDetect:
     def test_does_not_exist(self, tmp_path):