Repository: twisted/pydoctor

Issue:
Crash when building numpy docs
Pydoctor crashes when building numpy docs. 

This must be a regression introduced by #488 :/ 

I'll work on a fix and release a bugfix version of pydoctor, as well as a system test for numpy docs

```
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/pydoctor/driver.py", line 431, in main
    system.process()
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/pydoctor/model.py", line 1024, in process
    self.processModule(mod)
  File "/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/pydoctor/model.py", line 1013, in processModule
    del self.unprocessed_modules[mod.fullName()]
KeyError: 'numpy.core.char'
```

Comments:

Patch:
diff --git a/.github/workflows/system.yaml b/.github/workflows/system.yaml
index f98c369d..f30105c6 100644
--- a/.github/workflows/system.yaml
+++ b/.github/workflows/system.yaml
@@ -12,7 +12,11 @@ jobs:
 
     strategy:
       matrix:
-        tox_target: [twisted-apidoc, cpython-summary, python-igraph-apidocs, cpython-apidocs]
+        tox_target: [twisted-apidoc, 
+                     cpython-summary, 
+                     python-igraph-apidocs, 
+                     cpython-apidocs,
+                     numpy-apidocs,]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/README.rst b/README.rst
index 6c62894b..bfb1de3e 100644
--- a/README.rst
+++ b/README.rst
@@ -71,7 +71,11 @@ What's New?
 ~~~~~~~~~~~
 
 in development
-^^^^^^^^^^^^^^ 
+^^^^^^^^^^^^^^
+
+pydoctor 22.2.1
+^^^^^^^^^^^^^^^
+* Fix crash of pydoctor when processing a reparented module.
 
 pydoctor 22.2.0
 ^^^^^^^^^^^^^^^
diff --git a/pydoctor/model.py b/pydoctor/model.py
index 2e7ceaa9..f6423992 100644
--- a/pydoctor/model.py
+++ b/pydoctor/model.py
@@ -626,7 +626,11 @@ class System:
         self.verboselevel = 0
         self.needsnl = False
         self.once_msgs: Set[Tuple[str, str]] = set()
-        self.unprocessed_modules: Dict[str, _ModuleT] = OrderedDict()
+
+        # We're using the id() of the modules as key, and not the fullName becaue modules can
+        # be reparented, generating KeyError.
+        self.unprocessed_modules: Dict[int, _ModuleT] = OrderedDict()
+
         self.module_count = 0
         self.processing_modules: List[str] = []
         self.buildtime = datetime.datetime.now()
@@ -826,10 +830,13 @@ class System:
         module to the system.
         """
         assert mod.state is ProcessingState.UNPROCESSED
-        first = self.unprocessed_modules.setdefault(mod.fullName(), mod)
-        if mod is not first:
+        first = self.allobjects.get(mod.fullName())
+        if first is not None:
+            # At this step of processing only modules exists
+            assert isinstance(first, Module)
             self._handleDuplicateModule(first, mod)
         else:
+            self.unprocessed_modules[id(mod)] = mod
             self.addObject(mod)
             self.progress(
                 "analyzeModule", len(self.allobjects),
@@ -855,7 +862,7 @@ class System:
             return
         else:
             # Else, the last added module wins
-            del self.unprocessed_modules[dup.fullName()]
+            del self.unprocessed_modules[id(dup)]
             self._addUnprocessedModule(dup)
 
     def _introspectThing(self, thing: object, parent: Documentable, parentMod: _ModuleT) -> None:
@@ -1010,7 +1017,7 @@ class System:
                 mod.state = ProcessingState.PROCESSED
                 head = self.processing_modules.pop()
                 assert head == mod.fullName()
-        del self.unprocessed_modules[mod.fullName()]
+        del self.unprocessed_modules[id(mod)]
         self.progress(
             'process',
             self.module_count - len(self.unprocessed_modules),
diff --git a/setup.cfg b/setup.cfg
index d3237ed1..796b8a0c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = pydoctor
-version = 22.2.0.dev0
+version = 22.2.1.dev0
 author = Michael Hudson-Doyle
 author_email = micahel@gmail.com
 maintainer = Maarten ter Huurne
diff --git a/tox.ini b/tox.ini
index 56e3cab5..0f388a6c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -82,6 +82,23 @@ commands =
         {toxworkdir}/cpython/Lib
     pytest -v docs/tests/test_standard_library_docs.py
 
+[testenv:numpy-apidocs]
+description = Build numpy API documentation. For now we don't check for any warnings or other errors. The only purpose of this test is to make sure pydoctor doesn't crash.
+deps = 
+    pytest
+commands =
+    sh -c "if [ ! -d {toxworkdir}/numpy ]; then \
+        git clone --depth 1 https://github.com/numpy/numpy.git {toxworkdir}/numpy; \
+        fi"
+    sh -c "cd {toxworkdir}/numpy && git pull"
+    rm -rf {toxworkdir}/numpy-output
+    python3 -c "from pydoctor.driver import main; \
+        code = main(['--html-summary-pages', '--quiet', \
+        '--html-output={toxworkdir}/numpy-output', \
+        '{toxworkdir}/numpy/numpy']); \
+        # Code 2 error means bad docstrings, which is OK for this test.
+        assert code==2, 'pydoctor exited with code %s, expected code 2.'%code"
+
 # Requires cmake
 [testenv:python-igraph-apidocs]
 description = Build python-igraph API documentation
@@ -134,7 +151,9 @@ deps =
     sphinx>=3.4.0
     git+https://github.com/twisted/twisted.git
     types-requests
-    types-docutils
+    # FIXME: https://github.com/twisted/pydoctor/issues/504
+    # This is pinned for now as newer versions are breaking our static checks.
+    types-docutils==0.17.5
 
 commands =
     mypy                                       \


Test patch:
diff --git a/pydoctor/test/test_packages.py b/pydoctor/test/test_packages.py
index 9d2a3aa6..125fb396 100644
--- a/pydoctor/test/test_packages.py
+++ b/pydoctor/test/test_packages.py
@@ -89,4 +89,24 @@ def test_package_module_name_clash() -> None:
     """
     system = processPackage('package_module_name_clash')
     pack = system.allobjects['package_module_name_clash.pack']
-    assert 'package' == pack.contents.popitem()[0]
\ No newline at end of file
+    assert 'package' == pack.contents.popitem()[0]
+
+def test_reparented_module() -> None:
+    """
+    A module that is imported in a package as a different name and exported
+    in that package under the new name via C{__all__} is presented using the
+    new name.
+    """
+    system = processPackage('reparented_module')
+
+    mod = system.allobjects['reparented_module.module']
+    top = system.allobjects['reparented_module']
+
+    assert mod.fullName() == 'reparented_module.module'
+    assert top.resolveName('module') is top.contents['module']
+    assert top.resolveName('module.f') is mod.contents['f']
+
+    # The module old name is not in allobjects
+    assert 'reparented_module.mod' not in system.allobjects
+    # But can still be resolved with it's old name
+    assert top.resolveName('mod') is top.contents['module']
\ No newline at end of file
diff --git a/pydoctor/test/testpackages/reparented_module/__init__.py b/pydoctor/test/testpackages/reparented_module/__init__.py
new file mode 100644
index 00000000..a7c376d0
--- /dev/null
+++ b/pydoctor/test/testpackages/reparented_module/__init__.py
@@ -0,0 +1,6 @@
+"""
+Here the module C{mod} is made available under an alias name
+that is explicitly advertised under the alias name. 
+"""
+from . import mod as module
+__all__=('module',)
diff --git a/pydoctor/test/testpackages/reparented_module/mod.py b/pydoctor/test/testpackages/reparented_module/mod.py
new file mode 100644
index 00000000..ef076130
--- /dev/null
+++ b/pydoctor/test/testpackages/reparented_module/mod.py
@@ -0,0 +1,5 @@
+"""
+This is the "origin" module which for testing purpose is used from the C{reparented_module} package.
+"""
+def f():
+    pass