Repository: PrefectHQ/prefect

Issue:
API docs generation: don't require all dependencies to be installed
Currently, the docs generation imports each file to introspect it. This means that all possible Prefect dependencies must be installed to generate docs. It would be great to find a way around it -- possibly by patching the import -- so docs can be generated with only the base imports available.


Discussion:

Resolving patch:
diff --git a/conftest.py b/conftest.py
new file mode 100644
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,53 @@
+import sys
+
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--airflow",
+        action="store_true",
+        dest="airflow",
+        help="including this flag will attempt to ONLY run airflow compatibility tests",
+    )
+    parser.addoption(
+        "--skip-formatting",
+        action="store_true",
+        dest="formatting",
+        help="including this flag will skip all formatting tests",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "airflow: mark test to run only when --airflow flag is provided."
+    )
+    config.addinivalue_line(
+        "markers",
+        "formatting: mark test as formatting to skip when --skip-formatting flag is provided.",
+    )
+
+
+def pytest_runtest_setup(item):
+    air_mark = item.get_closest_marker("airflow")
+
+    # if a test IS marked as "airflow" and the airflow flag IS NOT set, skip it
+    if air_mark is not None and item.config.getoption("--airflow") is False:
+        pytest.skip(
+            "Airflow tests skipped by default unless --airflow flag provided to pytest."
+        )
+
+    # if a test IS NOT marked as airflow and the airflow flag IS set, skip it
+    elif air_mark is None and item.config.getoption("--airflow") is True:
+        pytest.skip("Non-Airflow tests skipped because --airflow flag was provided.")
+
+    formatting_mark = item.get_closest_marker("formatting")
+
+    # if a test IS marked as "formatting" and the --skip-formatting flag IS set, skip it
+    if (
+        formatting_mark is not None
+        and item.config.getoption("--skip-formatting") is True
+    ):
+        pytest.skip(
+            "Formatting tests skipped because --skip-formatting flag was provided."
+        )
diff --git a/docs/generate_docs.py b/docs/generate_docs.py
--- a/docs/generate_docs.py
+++ b/docs/generate_docs.py
@@ -11,6 +11,7 @@
 
 On a development installation of Prefect, simply run `python generate_docs.py` from inside the `docs/` folder.
 """
+import builtins
 import importlib
 import inspect
 import os
@@ -19,9 +20,10 @@
 import subprocess
 import textwrap
 import warnings
+from contextlib import contextmanager
 from functools import partial
+from unittest.mock import MagicMock
 
-import nbformat as nbf
 import pendulum
 import toml
 import toolz
@@ -34,6 +36,23 @@
 outline_config = toml.load(OUTLINE_PATH)
 
 
+@contextmanager
+def patch_imports():
+    try:
+
+        def patched_import(*args, **kwargs):
+            try:
+                return real_import(*args, **kwargs)
+            except Exception:
+                return MagicMock(name=args[0])
+
+        # swap
+        real_import, builtins.__import__ = builtins.__import__, patched_import
+        yield
+    finally:
+        builtins.__import__ = real_import
+
+
 def load_outline(
     outline=outline_config["pages"],
     ext=outline_config.get("extension", ".md"),
@@ -62,9 +81,6 @@ def load_outline(
     return OUTLINE
 
 
-OUTLINE = load_outline()
-
-
 @toolz.curry
 def preprocess(f, remove_partial=True):
     def wrapped(*args, **kwargs):
@@ -141,7 +157,7 @@ def format_doc(obj, in_table=False):
             block = block[block.startswith("python") and 6 :].lstrip("\n")
             block = (
                 '<pre class="language-python"><code class="language-python">'
-                + format_code(block).replace("\n", "<br>").replace("*", "\*")
+                + format_code(block).replace("\n", "<br>").replace("*", r"\*")
                 + "</code></pre>"
             )
         cleaned = cleaned.replace(f"$CODEBLOCK{num}", block.rstrip(" "))
@@ -278,6 +294,8 @@ def create_tutorial_notebooks(tutorial):
         os.path.basename(os.getcwd()) == "docs"
     ), "Only run this utility from inside the docs/ directory!"
 
+    import nbformat as nbf
+
     os.makedirs(".vuepress/public/notebooks", exist_ok=True)
     text = open(tutorial, "r").read()
     code_blocks = re.findall(r"```(.*?)```", text, re.DOTALL)
@@ -294,139 +312,141 @@ def create_tutorial_notebooks(tutorial):
 
 if __name__ == "__main__":
 
-    assert (
-        os.path.basename(os.getcwd()) == "docs"
-    ), "Only run this script from inside the docs/ directory!"
-
-    GIT_SHA = os.getenv("GIT_SHA", "n/a")
-    SHORT_SHA = GIT_SHA[:7]
-    auto_generated_footer = (
-        '<p class="auto-gen">This documentation was auto-generated from commit '
-        "<a href='https://github.com/PrefectHQ/prefect/commit/{git_sha}'>{short_sha}</a> "
-        "</br>by Prefect {version} on {timestamp}</p>".format(
-            short_sha=SHORT_SHA,
-            git_sha=GIT_SHA,
-            version=prefect.__version__,
-            timestamp=pendulum.now("utc").format("MMMM D, YYYY [at] HH:mm [UTC]"),
+    with patch_imports():
+        OUTLINE = load_outline()
+        assert (
+            os.path.basename(os.getcwd()) == "docs"
+        ), "Only run this script from inside the docs/ directory!"
+
+        GIT_SHA = os.getenv("GIT_SHA", "n/a")
+        SHORT_SHA = GIT_SHA[:7]
+        auto_generated_footer = (
+            '<p class="auto-gen">This documentation was auto-generated from commit '
+            "<a href='https://github.com/PrefectHQ/prefect/commit/{git_sha}'>{short_sha}</a> "
+            "</br>by Prefect {version} on {timestamp}</p>".format(
+                short_sha=SHORT_SHA,
+                git_sha=GIT_SHA,
+                version=prefect.__version__,
+                timestamp=pendulum.now("utc").format("MMMM D, YYYY [at] HH:mm [UTC]"),
+            )
         )
-    )
 
-    front_matter = textwrap.dedent(
-        """
-        ---
-        sidebarDepth: 2
-        editLink: false
-        ---
-        """
-    ).lstrip()
-
-    shutil.rmtree("api/unreleased", ignore_errors=True)
-    os.makedirs("api/unreleased", exist_ok=True)
-
-    ## write link to hosted coverage reports
-    with open("api/unreleased/coverage.md", "w+") as f:
-        f.write(
-            textwrap.dedent(
-                """
+        front_matter = textwrap.dedent(
+            """
             ---
-            title: Test Coverage
+            sidebarDepth: 2
+            editLink: false
             ---
+            """
+        ).lstrip()
 
-            # Unit test coverage report
+        shutil.rmtree("api/unreleased", ignore_errors=True)
+        os.makedirs("api/unreleased", exist_ok=True)
 
-            To view test coverage reports, <a href="https://codecov.io/gh/PrefectHQ/prefect">click here</a>.
-            """
-            ).lstrip()
-        )
+        ## write link to hosted coverage reports
+        with open("api/unreleased/coverage.md", "w+") as f:
+            f.write(
+                textwrap.dedent(
+                    """
+                ---
+                title: Test Coverage
+                ---
+
+                # Unit test coverage report
 
-    ## UPDATE README
-    with open("api/unreleased/README.md", "w+") as f:
-        f.write(
-            textwrap.dedent(
+                To view test coverage reports, <a href="https://codecov.io/gh/PrefectHQ/prefect">click here</a>.
                 """
-            ---
-            sidebarDepth: 0
-            editLink: false
-            ---
-            """
-            ).lstrip()
-        )
+                ).lstrip()
+            )
 
-        api_reference_section = textwrap.dedent(
-            """
+        ## UPDATE README
+        with open("api/unreleased/README.md", "w+") as f:
+            f.write(
+                textwrap.dedent(
+                    """
+                ---
+                sidebarDepth: 0
+                editLink: false
+                ---
+                """
+                ).lstrip()
+            )
 
-            <div align="center" style="margin-bottom:40px;">
-            <img src="/assets/wordmark-color-horizontal.svg"  width=600 >
-            </div>
+            api_reference_section = textwrap.dedent(
+                """
 
-            # API Reference
+                <div align="center" style="margin-bottom:40px;">
+                <img src="/assets/wordmark-color-horizontal.svg"  width=600 >
+                </div>
 
-            This API reference is automatically generated from Prefect's source code and unit-tested to ensure it's up to date.
+                # API Reference
 
-            """
-        )
+                This API reference is automatically generated from Prefect's source code and unit-tested to ensure it's up to date.
 
-        with open("../README.md", "r") as g:
-            readme = g.read()
-            index = readme.index("## Hello, world!")
-            readme = "\n".join([api_reference_section, readme[index:]])
-            f.write(readme)
-            f.write(auto_generated_footer)
-
-    ## UPDATE CHANGELOG
-    with open("api/unreleased/changelog.md", "w+") as f:
-        f.write(
-            textwrap.dedent(
                 """
-            ---
-            sidebarDepth: 1
-            editLink: false
-            ---
-            """
-            ).lstrip()
-        )
-        with open("../CHANGELOG.md", "r") as g:
-            changelog = g.read()
-            f.write(changelog)
-            f.write(auto_generated_footer)
-
-    for page in OUTLINE:
-        # collect what to document
-        fname, classes, fns = (
-            page["page"],
-            page.get("classes", []),
-            page.get("functions", []),
-        )
-        fname = f"api/unreleased/{fname}"
-        directory = os.path.dirname(fname)
-        if directory:
-            os.makedirs(directory, exist_ok=True)
-        with open(fname, "w") as f:
-            # PAGE TITLE / SETUP
-            f.write(front_matter)
-            title = page.get("title")
-            if title:  # this would be a good place to have assignments
-                f.write(f"# {title}\n---\n")
-
-            top_doc_obj = page.get("top-level-doc")
-            if top_doc_obj is not None:
-                top_doc = inspect.getdoc(top_doc_obj)
-                if top_doc is not None:
-                    f.write(top_doc + "\n")
-            for obj in classes:
-                f.write(format_subheader(obj))
-
-                f.write(format_doc(obj) + "\n\n")
-                if type(obj) == toolz.functoolz.curry:
-                    f.write("\n")
-                    continue
-
-                public_members = get_class_methods(obj)
-                f.write(create_methods_table(public_members, title="methods:"))
-                f.write("\n---\n<br>\n\n")
-
-            if fns:
-                f.write("\n## Functions\n")
-            f.write(create_methods_table(fns, title="top-level functions:"))
-            f.write("\n")
-            f.write(auto_generated_footer)
+            )
+
+            with open("../README.md", "r") as g:
+                readme = g.read()
+                index = readme.index("## Hello, world!")
+                readme = "\n".join([api_reference_section, readme[index:]])
+                f.write(readme)
+                f.write(auto_generated_footer)
+
+        ## UPDATE CHANGELOG
+        with open("api/unreleased/changelog.md", "w+") as f:
+            f.write(
+                textwrap.dedent(
+                    """
+                ---
+                sidebarDepth: 1
+                editLink: false
+                ---
+                """
+                ).lstrip()
+            )
+            with open("../CHANGELOG.md", "r") as g:
+                changelog = g.read()
+                f.write(changelog)
+                f.write(auto_generated_footer)
+
+        for page in OUTLINE:
+            # collect what to document
+            fname, classes, fns = (
+                page["page"],
+                page.get("classes", []),
+                page.get("functions", []),
+            )
+            fname = f"api/unreleased/{fname}"
+            directory = os.path.dirname(fname)
+            if directory:
+                os.makedirs(directory, exist_ok=True)
+            with open(fname, "w") as f:
+                # PAGE TITLE / SETUP
+                f.write(front_matter)
+                title = page.get("title")
+                if title:  # this would be a good place to have assignments
+                    f.write(f"# {title}\n---\n")
+
+                top_doc_obj = page.get("top-level-doc")
+                if top_doc_obj is not None:
+                    top_doc = inspect.getdoc(top_doc_obj)
+                    if top_doc is not None:
+                        f.write(top_doc + "\n")
+                for obj in classes:
+                    f.write(format_subheader(obj))
+
+                    f.write(format_doc(obj) + "\n\n")
+                    if type(obj) == toolz.functoolz.curry:
+                        f.write("\n")
+                        continue
+
+                    public_members = get_class_methods(obj)
+                    f.write(create_methods_table(public_members, title="methods:"))
+                    f.write("\n---\n<br>\n\n")
+
+                if fns:
+                    f.write("\n## Functions\n")
+                f.write(create_methods_table(fns, title="top-level functions:"))
+                f.write("\n")
+                f.write(auto_generated_footer)


Test patch:
diff --git a/docs/test_generate_docs.py b/docs/test_generate_docs.py
--- a/docs/test_generate_docs.py
+++ b/docs/test_generate_docs.py
@@ -13,7 +13,7 @@
 
 try:
     from generate_docs import (
-        OUTLINE,
+        load_outline,
         create_absolute_path,
         format_doc,
         format_lists,
@@ -22,7 +22,11 @@
         get_call_signature,
         get_class_methods,
         get_source,
+        patch_imports,
     )
+
+    with patch_imports():
+        OUTLINE = load_outline()
 except ImportError:
     pytestmark = pytest.skip(
         "Documentation requirements not installed.", allow_module_level=True
@@ -34,7 +38,7 @@
 
 
 def consistency_check(obj, obj_name):
-    patt = re.compile("(?<=>`)(.*?)(?=[\(|`:])")
+    patt = re.compile(r"(?<=>`)(.*?)(?=[\(|`:])")
     doc = format_doc(obj)
     try:
         arg_list_index = doc.index("**Args**:")
@@ -427,7 +431,7 @@ def my_doc():
         pass
 
     res = format_doc(my_doc, in_table=True)
-    assert res.count(">\*<") == 2
+    assert res.count(r">\*<") == 2
 
 
 @pytest.mark.parametrize(
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -65,53 +65,3 @@ def executor(request, _switch):
     or with some subset of executors that you want to use.
     """
     return _switch(request.param)
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--airflow",
-        action="store_true",
-        dest="airflow",
-        help="including this flag will attempt to ONLY run airflow compatibility tests",
-    )
-    parser.addoption(
-        "--skip-formatting",
-        action="store_true",
-        dest="formatting",
-        help="including this flag will skip all formatting tests",
-    )
-
-
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "airflow: mark test to run only when --airflow flag is provided."
-    )
-    config.addinivalue_line(
-        "markers",
-        "formatting: mark test as formatting to skip when --skip-formatting flag is provided.",
-    )
-
-
-def pytest_runtest_setup(item):
-    air_mark = item.get_closest_marker("airflow")
-
-    # if a test IS marked as "airflow" and the airflow flag IS NOT set, skip it
-    if air_mark is not None and item.config.getoption("--airflow") is False:
-        pytest.skip(
-            "Airflow tests skipped by default unless --airflow flag provided to pytest."
-        )
-
-    # if a test IS NOT marked as airflow and the airflow flag IS set, skip it
-    elif air_mark is None and item.config.getoption("--airflow") is True:
-        pytest.skip("Non-Airflow tests skipped because --airflow flag was provided.")
-
-    formatting_mark = item.get_closest_marker("formatting")
-
-    # if a test IS marked as "formatting" and the --skip-formatting flag IS set, skip it
-    if (
-        formatting_mark is not None
-        and item.config.getoption("--skip-formatting") is True
-    ):
-        pytest.skip(
-            "Formatting tests skipped because --skip-formatting flag was provided."
-        )
diff --git a/tests/core/test_flow.py b/tests/core/test_flow.py
--- a/tests/core/test_flow.py
+++ b/tests/core/test_flow.py
@@ -935,7 +935,7 @@ def test_skip_validation_in_init_with_kwarg():
     assert Flow(name="test", edges=[e1, e2], validate=False)
 
 
-@pytest.mark.xfail(raises=ImportError)
+@pytest.mark.xfail(raises=ImportError, reason="viz extras not installed.")
 class TestFlowVisualize:
     def test_visualize_raises_informative_importerror_without_graphviz(
         self, monkeypatch
diff --git a/tests/engine/result_handlers/test_result_handlers.py b/tests/engine/result_handlers/test_result_handlers.py
--- a/tests/engine/result_handlers/test_result_handlers.py
+++ b/tests/engine/result_handlers/test_result_handlers.py
@@ -110,7 +110,7 @@ def read(self, val):
     assert "abstract methods write" in str(exc.value)
 
 
-@pytest.mark.xfail(raises=ImportError)
+@pytest.mark.xfail(raises=ImportError, reason="google extras not installed.")
 class TestGCSResultHandler:
     @pytest.fixture
     def google_client(self, monkeypatch):
@@ -170,7 +170,7 @@ def __getstate__(self):
         assert isinstance(res, GCSResultHandler)
 
 
-@pytest.mark.xfail(raises=ImportError)
+@pytest.mark.xfail(raises=ImportError, reason="aws extras not installed.")
 class TestS3ResultHandler:
     @pytest.fixture
     def s3_client(self, monkeypatch):
diff --git a/tests/serialization/test_result_handlers.py b/tests/serialization/test_result_handlers.py
--- a/tests/serialization/test_result_handlers.py
+++ b/tests/serialization/test_result_handlers.py
@@ -166,7 +166,7 @@ def test_deserialize_cloud_result_handler_with_None_populates_from_config(self):
         assert obj._client is None
 
 
-@pytest.mark.xfail(raises=ImportError)
+@pytest.mark.xfail(raises=ImportError, reason="google extras not installed.")
 class TestGCSResultHandler:
     @pytest.fixture
     def google_client(self, monkeypatch):
@@ -214,7 +214,7 @@ def test_roundtrip(self):
         assert handler.write(3) == "3"
 
 
-@pytest.mark.xfail(raises=ImportError)
+@pytest.mark.xfail(raises=ImportError, reason="aws extras not installed.")
 class TestS3ResultHandler:
     @pytest.fixture
     def s3_client(self, monkeypatch):