Repository: lukasgeiter/mkdocs-awesome-pages-plugin

Issue:
Allow `mkdocs-gen-files` generated content
:wave: Loving the plugin! 

Currently also loving https://github.com/oprypin/mkdocs-gen-files, but noticed that if I generate navs, or pages referenced by my static navs, they don't end up making it to the generated docsite.

E.g.
`mkdocs.yml`
```yaml
plugins:
  - gen-files:
      scripts:
        - generate_reference_docs.py
  - awesome-pages:
      filename: .nav.yaml
```

`generate_reference_docs.py`
```python
with mkdocs_gen_files.open(f"reference/global-options.md", "w") as f:
    f.write("Hello world!\n")
with mkdocs_gen_files.open(f"reference/goals/index.md", "w") as f:
    f.write("# Goals!\n")
with mkdocs_gen_files.open(f"reference/goals/.nav.yaml", "w") as f:
    f.write("nav:\n  - index.md\n")
```

along with:

```console
$ tree -a docs/markdown/reference
docs/markdown/reference
└── .nav.yaml
$ cat docs/markdown/reference/.nav.yaml 
nav:
  - Global Options: global-options.md
  - Goals: goals
```

Results in `Reference` being in the nav, but is "flat". No expand arrow.

Additionally, the pages _do_ exist, `/reference/global-options` is a valid URL and gives what I expect.

Discussion:
thejcannon: @lukasgeiter This is ready to review.

The je ne sais quoi is boiled down to:
- Try to refer to everything by a "relative path"
- Synthesizing a `File` per config, so there isn't a split codepath
thejcannon: And to report, this is working as expected on my project :tada: 
lukasgeiter: Thanks for creating this PR! Before I dive into reviewing, I've noticed some formatting changes that shouldn't be there. Please make sure to format the files using [black](https://black.readthedocs.io/en/stable/index.html).
lukasgeiter: Please disregard the failing tests for pypy, they have nothing to do with your PR. I've actually just removed them on `master` so a rebase should get rid of them. The same does not apply for the failing tests on windows, that seems to be an actual issue 😉  
thejcannon: Thanks for taking a look!

Yeah looks like editor ran black with a different line length (shorter) leading to some spurious reformats. I'll fix those 
thejcannon: @lukasgeiter All tests are passing! Ready for review when you are.
thejcannon: @lukasgeiter I think that should cover those comments. Happy for another look

Patch:
diff --git a/mkdocs_awesome_pages_plugin/meta.py b/mkdocs_awesome_pages_plugin/meta.py
index 52e0bf9..3acc1da 100644
--- a/mkdocs_awesome_pages_plugin/meta.py
+++ b/mkdocs_awesome_pages_plugin/meta.py
@@ -4,6 +4,7 @@ from enum import Enum
 from pathlib import PurePath
 from typing import Optional, List, Union, Any, Iterator
 
+from mkdocs.structure.files import Files
 import yaml
 from wcmatch import glob
 
@@ -147,13 +148,20 @@ class Meta:
         self.order_by = order_by
 
     @staticmethod
-    def try_load_from(path: Optional[str]) -> "Meta":
-        if path is None:
+    def try_load_from_files(rel_path: Optional[str], files: "Files") -> "Meta":
+        if rel_path is None:
             return Meta()
+
+        file = files.src_paths.get(rel_path)
+        if file is None:
+            return Meta(path=rel_path)
+
         try:
-            return Meta.load_from(path)
+            meta = Meta.load_from(file.abs_src_path)
+            meta.path = file.src_path  # Use the relative path
+            return meta
         except FileNotFoundError:
-            return Meta(path=path)
+            return Meta(path=file.src_path)
 
     @staticmethod
     def load_from(path: str) -> "Meta":
diff --git a/mkdocs_awesome_pages_plugin/navigation.py b/mkdocs_awesome_pages_plugin/navigation.py
index 3847827..ee4f716 100644
--- a/mkdocs_awesome_pages_plugin/navigation.py
+++ b/mkdocs_awesome_pages_plugin/navigation.py
@@ -1,5 +1,4 @@
 import warnings
-from pathlib import Path
 
 import mkdocs.utils
 import mkdocs.utils.meta
@@ -13,6 +12,7 @@ from mkdocs.structure.nav import (
     _add_parent_links,
     _add_previous_and_next_links,
 )
+from mkdocs.structure.files import Files
 from mkdocs.structure.pages import Page
 
 from .meta import Meta, MetaNavItem, MetaNavRestItem, RestItemList
@@ -50,13 +50,13 @@ class AwesomeNavigation:
         self,
         items: List[NavigationItem],
         options: Options,
-        docs_dir: str,
+        files: Files,
         explicit_sections: Set[Section],
     ):
         self.options = options
         self.explicit_sections = explicit_sections
 
-        self.meta = NavigationMeta(items, options, docs_dir, explicit_sections)
+        self.meta = NavigationMeta(items, options, files, explicit_sections)
 
         if self.meta.root.title is not None:
             warnings.warn(TitleInRootHasNoEffect(self.options.filename))
@@ -193,7 +193,7 @@ class AwesomeNavigation:
         if isinstance(item, Section):
             return dirname(self.meta.sections[item].path)
         elif isinstance(item, Page):
-            return item.file.abs_src_path
+            return item.file.src_path
 
     def _get_item_title(self, item: NavigationItem) -> str:
         # Handle custom section titles in the ".pages" file
@@ -261,12 +261,12 @@ class NavigationMeta:
         self,
         items: List[NavigationItem],
         options: Options,
-        docs_dir: str,
+        files: Files,
         explicit_sections: Set[Section],
     ):
         self.options = options
         self.sections: Dict[Section, Meta] = {}
-        self.docs_dir = docs_dir
+        self.files = files
         self.explicit_sections = explicit_sections
 
         self.root: Meta = self._gather_metadata(items)
@@ -275,8 +275,7 @@ class NavigationMeta:
         paths: List[str] = []
         for item in items:
             if isinstance(item, Page):
-                if Path(self.docs_dir) in Path(item.file.abs_src_path).parents:
-                    paths.append(item.file.abs_src_path)
+                paths.append(item.file.src_path)
             elif isinstance(item, Section):
                 section_meta = self._gather_metadata(item.children)
 
@@ -289,7 +288,8 @@ class NavigationMeta:
 
                 self.sections[item] = section_meta
 
-        return Meta.try_load_from(join_paths(self._common_dirname(paths), self.options.filename))
+        rel_config_path = join_paths(self._common_dirname(paths), self.options.filename)
+        return Meta.try_load_from_files(rel_config_path, self.files)
 
     @staticmethod
     def _common_dirname(paths: List[str]) -> Optional[str]:
diff --git a/mkdocs_awesome_pages_plugin/plugin.py b/mkdocs_awesome_pages_plugin/plugin.py
index bb83193..8824e36 100644
--- a/mkdocs_awesome_pages_plugin/plugin.py
+++ b/mkdocs_awesome_pages_plugin/plugin.py
@@ -1,4 +1,6 @@
+import os.path
 import warnings
+import glob
 from typing import List, Dict
 
 from mkdocs.config import config_options, Config
@@ -45,6 +47,22 @@ class AwesomePagesPlugin(BasePlugin):
         self.rest_items = RestItemList()
         self.rest_blocks = {}
 
+    def on_files(self, files: Files, config: Config):
+        # Add config files to Files, so we can unconditionally load files/configs from Files
+        config_files = glob.glob(os.path.join(config["docs_dir"], f"**/{self.config['filename']}"), recursive=True)
+        initial_src_paths = files.src_paths
+        for filename in config_files:
+            config_path = os.path.relpath(filename, config["docs_dir"])
+            if config_path not in initial_src_paths:
+                file = File(
+                    os.path.relpath(filename, config["docs_dir"]),
+                    src_dir=config["docs_dir"],
+                    dest_dir=config["site_dir"],
+                    use_directory_urls=config["use_directory_urls"],
+                )
+                files.append(file)
+        return files
+
     def on_nav(self, nav: MkDocsNavigation, config: Config, files: Files):
         explicit_nav = nav if config["nav"] else None
 
@@ -60,7 +78,12 @@ class AwesomePagesPlugin(BasePlugin):
             self._insert_rest(explicit_nav.items)
             nav = explicit_nav
 
-        return AwesomeNavigation(nav.items, Options(**self.config), config["docs_dir"], explicit_sections).to_mkdocs()
+        return AwesomeNavigation(
+            nav.items,
+            Options(**self.config),
+            files,
+            explicit_sections,
+        ).to_mkdocs()
 
     def on_config(self, config: Config):
         for name, plugin in config["plugins"].items():


Test patch:
diff --git a/mkdocs_awesome_pages_plugin/tests/e2e/base.py b/mkdocs_awesome_pages_plugin/tests/e2e/base.py
index 2bb5ef0..065cf82 100644
--- a/mkdocs_awesome_pages_plugin/tests/e2e/base.py
+++ b/mkdocs_awesome_pages_plugin/tests/e2e/base.py
@@ -1,7 +1,7 @@
 import os
 import tempfile
 import warnings
-from typing import Optional, List, Tuple, Union, Dict
+from typing import Iterable, Optional, List, Tuple, Union, Dict, TypeVar
 from unittest import TestCase
 
 import yaml
@@ -13,10 +13,13 @@ from mkdocs.config import load_config
 
 from ...utils import cd
 
+PluginBaseT = TypeVar("PluginBaseT", bound=plugins.BasePlugin)
+
 
 class E2ETestCase(TestCase):
     MIN_ROOT_ITEMS = 2
     DUMMY_NAME = "__dummy__"
+    PLUGINS: Tuple[PluginBaseT] = ()
 
     def setUp(self):
         self.config = self.createConfig()
@@ -72,13 +75,17 @@ class E2ETestCase(TestCase):
         plugins_entry = "awesome-pages"
         if plugin_options:
             plugins_entry = {plugins_entry: plugin_options}
+        plugins = [
+            plugins_entry,
+            *[class_plugin.__name__.lower() for class_plugin in getattr(type(self), "PLUGINS", [])],
+        ]
 
         if mkdocs_nav is not None:
             # mkdocs requires a minimum amount of top-level items to render the navigation properly
             # ensure that this requirement is met by adding dummy pages
             self._addDummyPages(mkdocs_nav, self.MIN_ROOT_ITEMS - len(mkdocs_nav))
 
-        return {"plugins": [plugins_entry], "nav": mkdocs_nav}
+        return {"plugins": plugins, "nav": mkdocs_nav}
 
     def mkdocs(
         self,
@@ -165,12 +172,20 @@ class E2ETestCase(TestCase):
 
         return pages
 
-    @staticmethod
-    def _patchGetPlugins():
+    @classmethod
+    def _patchGetPlugins(cls):
         _original_get_plugins = plugins.get_plugins
 
         def _patched_get_plugins():
             result = _original_get_plugins()
+
+            for class_plugin in cls.PLUGINS:
+                name = class_plugin.__name__.lower()
+                result[name] = EntryPoint(
+                    name=name,
+                    value=f"{class_plugin.__module__}:{class_plugin.__name__}",
+                    group="mkdocs.plugins",
+                )
             result["awesome-pages"] = EntryPoint(
                 name="awesome-pages",
                 value="mkdocs_awesome_pages_plugin.plugin:AwesomePagesPlugin",
diff --git a/mkdocs_awesome_pages_plugin/tests/e2e/test_gen_files.py b/mkdocs_awesome_pages_plugin/tests/e2e/test_gen_files.py
new file mode 100644
index 0000000..5e53263
--- /dev/null
+++ b/mkdocs_awesome_pages_plugin/tests/e2e/test_gen_files.py
@@ -0,0 +1,92 @@
+import os.path
+import tempfile
+from pathlib import Path
+
+import yaml
+from mkdocs.plugins import BasePlugin
+from mkdocs.structure.files import File
+
+from .base import E2ETestCase
+
+
+class GeneratedFiles(BasePlugin):
+    def on_pre_build(self, *args, **kwargs):
+        self.src_dir = tempfile.TemporaryDirectory()
+
+    def on_post_build(self, *args, **kwargs):
+        self.src_dir.cleanup()
+
+    def on_files(self, files, config):
+        docs_dir = Path(self.src_dir.name)
+        section_dir = docs_dir / "section"
+        section_dir.mkdir()
+        (section_dir / "2.md").touch()
+        (section_dir / "3.md").touch()
+        (section_dir / ".pages").write_text(yaml.dump({"arrange": ["3.md", "sub", "..."]}))
+        subsection_dir = section_dir / "sub"
+        subsection_dir.mkdir()
+        (subsection_dir / "a.md").touch()
+        (subsection_dir / ".pages").write_text(yaml.dump({"nav": ["a.md"]}))
+
+        def write_files(path):
+            for entry in path.iterdir():
+                if entry.is_file():
+                    files.append(
+                        File(
+                            os.path.join(path.relative_to(docs_dir), entry.name),
+                            src_dir=self.src_dir.name,
+                            dest_dir=config["site_dir"],
+                            use_directory_urls=config["use_directory_urls"],
+                        )
+                    )
+                else:
+                    write_files(path / entry.name)
+
+        write_files(section_dir)
+        return files
+
+
+class TestGeneratedFiles(E2ETestCase):
+    """See https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin/issues/78."""
+
+    PLUGINS = (GeneratedFiles,)
+
+    def test_mixed(self):
+        navigation = self.mkdocs(
+            self.config,
+            [
+                (
+                    "section",
+                    ["1.md"],
+                )
+            ],
+        )
+        self.assertEqual(
+            navigation,
+            [
+                (
+                    "Section",
+                    [
+                        ("3", "/section/3"),
+                        ("Sub", [("A", "/section/sub/a")]),
+                        ("1", "/section/1"),
+                        ("2", "/section/2"),
+                    ],
+                )
+            ],
+        )
+
+    def test_all_virtual(self):
+        navigation = self.mkdocs(
+            self.config,
+            [],
+        )
+        self.assertEqual(
+            navigation,
+            [
+                (
+                    "Section",
+                    [("3", "/section/3"), ("Sub", [("A", "/section/sub/a")]), ("2", "/section/2")],
+                )
+            ],
+        )
diff --git a/mkdocs_awesome_pages_plugin/tests/navigation/base.py b/mkdocs_awesome_pages_plugin/tests/navigation/base.py
index 78a72a4..cf65f9c 100644
--- a/mkdocs_awesome_pages_plugin/tests/navigation/base.py
+++ b/mkdocs_awesome_pages_plugin/tests/navigation/base.py
@@ -1,8 +1,8 @@
-import os
+import os.path
 from typing import List, Union, Optional
 from unittest import TestCase, mock
 
-from mkdocs.structure.files import File
+from mkdocs.structure.files import File, Files
 from mkdocs.structure.nav import (
     Navigation as MkDocsNavigation,
     Section,
@@ -25,8 +25,8 @@ class NavigationMetaMock:
 
 class NavigationTestCase(TestCase):
     @staticmethod
-    def page(title: str, path: Optional[str] = None, docs_dir: str = "") -> Page:
-        return Page(title, File(path or title + ".md", docs_dir, "", False), {})
+    def page(title: str, path: Optional[str] = None) -> Page:
+        return Page(title, File(path or title + ".md", os.path.abspath("docs"), "", False), {})
 
     @staticmethod
     def link(title: str, url: Optional[str] = None):
@@ -75,7 +75,7 @@ class NavigationTestCase(TestCase):
                 collapse_single_pages=collapse_single_pages,
                 strict=strict,
             ),
-            docs_dir="",
+            files=Files([]),
             explicit_sections=set(),
         )
 
diff --git a/mkdocs_awesome_pages_plugin/tests/navigation/test_meta.py b/mkdocs_awesome_pages_plugin/tests/navigation/test_meta.py
index b3a3b58..805a443 100644
--- a/mkdocs_awesome_pages_plugin/tests/navigation/test_meta.py
+++ b/mkdocs_awesome_pages_plugin/tests/navigation/test_meta.py
@@ -1,9 +1,14 @@
-from typing import Optional
+import os.path
+from typing import List, Optional, Set
 from unittest import TestCase
 
+from mkdocs.structure.files import File, Files
+from mkdocs.structure.nav import Section
+from mkdocs.structure.pages import Page
+
 from .base import NavigationTestCase
 from ...meta import Meta, MetaNavRestItem, RestType
-from ...navigation import NavigationMeta
+from ...navigation import NavigationItem, NavigationMeta
 from ...options import Options
 from ...utils import normpath
 
@@ -45,18 +50,52 @@ class TestMeta(NavigationTestCase):
         super(TestMeta, self).setUp()
         self.options = Options(filename=".pages", collapse_single_pages=False, strict=True)
 
+    def _make_nav_meta(
+        self,
+        items: List[NavigationItem],
+        options: Options,
+        explicit_sections: Set[Section] = frozenset(),
+    ):
+        files = Files([])
+        config_dirs = set()
+
+        def populate_config_dirs(items: List[NavigationItem]):
+            for item in items:
+                if isinstance(item, Page):
+                    config_dirs.add(
+                        (
+                            os.path.dirname(item.file.src_path),
+                            item.file.abs_src_path[: -len(item.file.src_path)],
+                        )
+                    )
+                    files.append(item.file)
+                elif isinstance(item, Section):
+                    populate_config_dirs(item.children)
+
+        populate_config_dirs(items)
+
+        for rel_config_dir, abs_docs_dir in config_dirs:
+            files.append(
+                File(
+                    os.path.join(rel_config_dir, options.filename),
+                    src_dir=abs_docs_dir,
+                    dest_dir="",
+                    use_directory_urls=False,
+                )
+            )
+
+        return NavigationMeta(items, options, files, explicit_sections)
+
     def test_empty(self):
-        meta = NavigationMeta([], self.options, docs_dir="", explicit_sections=set())
+        meta = self._make_nav_meta([], self.options)
 
         self.assertEqual(len(meta.sections), 0)
         self.assertEmptyMeta(meta.root)
 
     def test_page_in_root(self):
-        meta = NavigationMeta(
+        meta = self._make_nav_meta(
             [self.page("Page", "page.md")],
             self.options,
-            docs_dir="",
-            explicit_sections=set(),
         )
 
         self.assertEqual(len(meta.sections), 0)
@@ -64,7 +103,7 @@ class TestMeta(NavigationTestCase):
 
     def test_empty_section(self):
         section = self.section("Section", [])
-        meta = NavigationMeta([section], self.options, docs_dir="", explicit_sections=set())
+        meta = self._make_nav_meta([section], self.options)
 
         self.assertEqual(len(meta.sections), 1)
         self.assertEmptyMeta(meta.sections[section])
@@ -72,7 +111,7 @@ class TestMeta(NavigationTestCase):
 
     def test_section(self):
         section = self.section("Section", [self.page("Page", "section/page.md")])
-        meta = NavigationMeta([section], self.options, docs_dir="", explicit_sections=set())
+        meta = self._make_nav_meta([section], self.options)
 
         self.assertEqual(len(meta.sections), 1)
         self.assertMeta(meta.sections[section], path="section/.pages")
@@ -86,7 +125,7 @@ class TestMeta(NavigationTestCase):
         e = self.section("E", [self.page("2", "c/e/2.md")])
         c = self.section("C", [d, e])
 
-        meta = NavigationMeta([a, c], self.options, docs_dir="", explicit_sections=set())
+        meta = self._make_nav_meta([a, c], self.options)
 
         self.assertEqual(len(meta.sections), 5)
 
@@ -101,50 +140,26 @@ class TestMeta(NavigationTestCase):
 
     def test_filename_option(self):
         section = self.section("Section", [self.page("Page", "section/page.md")])
-        meta = NavigationMeta(
-            [section],
-            Options(filename=".index", collapse_single_pages=False, strict=True),
-            docs_dir="",
-            explicit_sections=set(),
-        )
+        meta = self._make_nav_meta([section], Options(filename=".index", collapse_single_pages=False, strict=True))
 
         self.assertEqual(len(meta.sections), 1)
         self.assertMeta(meta.sections[section], path="section/.index")
         self.assertMeta(meta.root, path=".index")
 
     def test_links(self):
-        meta = NavigationMeta(
-            [self.page("Page", "page.md"), self.link("Link")],
-            self.options,
-            docs_dir="",
-            explicit_sections=set(),
-        )
+        meta = self._make_nav_meta([self.page("Page", "page.md"), self.link("Link")], self.options)
 
         self.assertEqual(len(meta.sections), 0)
         self.assertMeta(meta.root, path=".pages")
 
     def test_no_common_dirname(self):
         section = self.section("Section", [self.page("1", "a/1.md"), self.page("2", "b/2.md")])
-        meta = NavigationMeta([section], self.options, docs_dir="", explicit_sections=set())
+        meta = self._make_nav_meta([section], self.options)
 
         self.assertEqual(len(meta.sections), 1)
         self.assertEmptyMeta(meta.sections[section])
         self.assertEmptyMeta(meta.root)
 
-    def test_path_outside_docs(self):
-        meta = NavigationMeta(
-            [
-                self.page("Page", "page.md", docs_dir="/docs"),
-                self.page("Outside", "/outside", docs_dir="/docs"),
-            ],
-            self.options,
-            docs_dir="/docs",
-            explicit_sections=set(),
-        )
-
-        self.assertEqual(len(meta.sections), 0)
-        self.assertMeta(meta.root, path="/docs/.pages")
-
 
 class TestRestParsing(NavigationTestCase):
     def test_all(self):
diff --git a/mkdocs_awesome_pages_plugin/tests/test_meta.py b/mkdocs_awesome_pages_plugin/tests/test_meta.py
index 63c9c8b..ca3a490 100644
--- a/mkdocs_awesome_pages_plugin/tests/test_meta.py
+++ b/mkdocs_awesome_pages_plugin/tests/test_meta.py
@@ -1,5 +1,8 @@
+from pathlib import Path
 from unittest import TestCase, mock
 
+from mkdocs.structure.files import File, Files
+
 from ..meta import Meta, DuplicateRestItemError, MetaNavItem, MetaNavRestItem
 from .file_mock import FileMock
 
@@ -190,16 +193,19 @@ class TestLoadFrom(TestCase):
 
 @mock.patch("builtins.open", new_callable=FileMock)
 class TestTryLoadFrom(TestCase):
-    def test(self, file_mock: FileMock):
-        file_mock[".pages"].read_data = "title: Section Title\n"
+    def test_in_docs_dir(self, file_mock: FileMock):
+        docs_path = Path("docs").resolve()
+        file_mock[str(docs_path / ".pages")].read_data = "title: Section Title\n"
+        files = Files([File(".pages", str(docs_path), "", False)])
 
-        meta = Meta.try_load_from(".pages")
+        meta = Meta.try_load_from_files(".pages", files)
         self.assertEqual(meta.title, "Section Title")
+        self.assertEqual(meta.path, ".pages")
 
     def test_file_not_found(self, file_mock: FileMock):
-        meta = Meta.try_load_from(".pages")
+        meta = Meta.try_load_from_files(".pages", Files([]))
         self.assertIsInstance(meta, Meta)
 
     def test_none_path(self, file_mock: FileMock):
-        meta = Meta.try_load_from(None)
+        meta = Meta.try_load_from_files(None, Files([]))
         self.assertIsInstance(meta, Meta)