Repository: scikit-build/scikit-build-core
Pull request #553 (2023-11-10 17:42:45)

feat: add inplace mode for editable installs
This adds support for editable.mode=inplace, which does an inplace CMake build, along with a simple editable mode .pth file. This has a lot of caveats (just like CMake inplace builds), but works better with some workflows and tools. It's certainly better than copying the output file back into the source, which is what some projects are currently doing. :)



Patch:
diff --git a/README.md b/README.md
index 3182092..4c1eabf 100644
--- a/README.md
+++ b/README.md
@@ -237,7 +237,7 @@ wheel.platlib = ""
 # disable this, or the empty string.
 backport.find-python = "3.26.1"
 
-# Select the editable mode to use. Currently only "redirect" is supported.
+# Select the editable mode to use. Can be "redirect" (default) or "inplace".
 editable.mode = "redirect"
 
 # Turn on verbose output for the editable mode rebuilds.
diff --git a/docs/configuration.md b/docs/configuration.md
index d48082a..797c114 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -523,14 +523,15 @@ configuration. Recommendations:
 - Use `--no-build-isolation` when doing an editable install is recommended; you
   should preinstall your dependencies.
 - Automatic rebuilds do not have the original isolated build dir (pip deletes
-  it).
-- Select a `build-dir` when using editable installs, especially if you also
-  enable automatic rebuilds.
+  it), so select a `build-dir` when using editable installs, especially if you
+  also enable automatic rebuilds.
 - You need to reinstall to pick up new files.
 
 Known limitations:
 
 - Resources (via `importlib.resources`) are not properly supported (yet).
+  Currently experimentally supported except on Python 3.9 (3.7, 3.8, 3.10, 3.11,
+  and 3.12 work). `importlib_resources` may work on Python 3.9.
 
 ```console
 # Very experimental rebuild on initial import feature
@@ -545,11 +546,36 @@ You can disable the verbose rebuild output with `editable.verbose=false` if you
 want. (Also available as the `SKBUILD_EDITABLE_VERBOSE` envvar when importing;
 this will override if non-empty, and `"0"` will disable verbose output).
 
-Currently one `editable.mode` is provided, `"redirect"`, which uses a custom
-redirecting finder to combine the static CMake install dir with the original
-source code. Python code added via scikit-build-core's package discovery will be
-found in the original location, so changes there are picked up on import,
-regardless of the `editable.rebuild` setting.
+The default `editable.mode`, `"redirect"`, uses a custom redirecting finder to
+combine the static CMake install dir with the original source code. Python code
+added via scikit-build-core's package discovery will be found in the original
+location, so changes there are picked up on import, regardless of the
+`editable.rebuild` setting.
+
+:::{note}
+
+A second experimental mode, `"inplace"`, is also available. This does an
+in-place CMake build, so all the caveats there apply too -- only one build per
+source directory, you can't change to an out-of-source builds without removing
+the build artifacts, your source directory will be littered with build
+artifacts, etc. Also, to make your binaries importable, you should set
+`LIBRARY_OUTPUT_DIRECTORY` (include a generator expression, like the empty one
+`$<0:>` for multi-config generator support, like MSVC, so you don't have to set
+all possible `*_<CONFIG>` variations) to make sure they are placed inside your
+source directory inside the Python packages; this will be run from the build
+directory, rather than installed. This will also not support automatic rebuilds.
+The build directory setting must be empty/unset to use this. You can detect this
+mode by checking for an in-place build and checking `SKBUILD` being set.
+
+With all the caveats, this is very logically simple (one directory) and a near
+identical replacement for `python setup.py build_ext --inplace`. Some third
+party tooling might work better with this mode. Scikit-build-core will simply
+install a `.pth` file that points at your source package(s) and do an inplace
+CMake build.
+
+On the command line, you can pass `-Ceditable.mode=inplace` to enable this mode.
+
+:::
 
 ## Other options
 
diff --git a/noxfile.py b/noxfile.py
index 4ec6c4e..c41d677 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -136,6 +136,7 @@ def docs(session: nox.Session) -> None:
     extra_installs = ["sphinx-autobuild"] if args.serve else []
 
     session.install("-e.[docs,pyproject]", *extra_installs)
+    session.run("pip", "list")
     session.chdir("docs")
 
     if args.builder == "linkcheck":
diff --git a/src/scikit_build_core/build/wheel.py b/src/scikit_build_core/build/wheel.py
index b3896de..3ba7638 100644
--- a/src/scikit_build_core/build/wheel.py
+++ b/src/scikit_build_core/build/wheel.py
@@ -260,6 +260,8 @@ def _build_wheel_impl(
         install_options = []
 
         if cmake is not None:
+            if editable and settings.editable.mode == "inplace":
+                build_dir = settings.cmake.source_dir
             config = CMaker(
                 cmake,
                 source_dir=settings.cmake.source_dir,
@@ -300,8 +302,9 @@ def _build_wheel_impl(
             build_args: list[str] = []
             builder.build(build_args=build_args)
 
-            rich_print("[green]***[/green] [bold]Installing project into wheel...")
-            builder.install(install_dir)
+            if not (editable and settings.editable.mode == "inplace"):
+                rich_print("[green]***[/green] [bold]Installing project into wheel...")
+                builder.install(install_dir)
 
             if not builder.config.single_config and builder.config.build_type:
                 build_options += ["--config", builder.config.build_type]
@@ -338,7 +341,7 @@ def _build_wheel_impl(
         ) as wheel:
             wheel.build(wheel_dirs)
 
-            if editable:
+            if editable and settings.editable.mode == "redirect":
                 reload_dir = build_dir.resolve() if settings.build_dir else None
 
                 _make_editable(
@@ -351,6 +354,18 @@ def _build_wheel_impl(
                     wheel=wheel,
                     name=normalized_name,
                 )
+            elif editable and settings.editable.mode == "inplace":
+                if not packages:
+                    msg = "Editable inplace mode requires at least one package"
+                    raise AssertionError(msg)
+
+                str_pkgs = (
+                    str(Path.cwd().joinpath(p).parent.resolve()) for p in packages
+                )
+                wheel.writestr(
+                    f"_{normalized_name}_editable.pth",
+                    "\n".join(str_pkgs).encode(),
+                )
 
     if metadata_directory is not None:
         dist_info_contents = wheel.dist_info_contents()
diff --git a/src/scikit_build_core/cmake.py b/src/scikit_build_core/cmake.py
index 6db0da0..f901fc3 100644
--- a/src/scikit_build_core/cmake.py
+++ b/src/scikit_build_core/cmake.py
@@ -85,28 +85,25 @@ class CMaker:
             raise CMakeConfigError(msg)
 
         # If these were the same, the following check could wipe the source directory!
-        if self.build_dir.resolve() == self.source_dir.resolve():
-            msg = "build directory must be different from source directory"
-            raise CMakeConfigError(msg)
-
-        skbuild_info = self.build_dir / ".skbuild-info.json"
-        # If building via SDist, this could be pre-filled, so delete it if it exists
-        with contextlib.suppress(FileNotFoundError):
-            with skbuild_info.open("r", encoding="utf-8") as f:
-                info = json.load(f)
-
-            cached_source_dir = Path(info["source_dir"])
-            if cached_source_dir.resolve() != self.source_dir.resolve():
-                logger.warning(
-                    "Original src {} != {}, wiping build directory",
-                    cached_source_dir,
-                    self.source_dir,
-                )
-                shutil.rmtree(self.build_dir)
-                self.build_dir.mkdir()
-
-        with skbuild_info.open("w", encoding="utf-8") as f:
-            json.dump(self._info_dict(), f, indent=2)
+        if self.build_dir.resolve() != self.source_dir.resolve():
+            skbuild_info = self.build_dir / ".skbuild-info.json"
+            # If building via SDist, this could be pre-filled, so delete it if it exists
+            with contextlib.suppress(FileNotFoundError):
+                with skbuild_info.open("r", encoding="utf-8") as f:
+                    info = json.load(f)
+
+                cached_source_dir = Path(info["source_dir"])
+                if cached_source_dir.resolve() != self.source_dir.resolve():
+                    logger.warning(
+                        "Original src {} != {}, wiping build directory",
+                        cached_source_dir,
+                        self.source_dir,
+                    )
+                    shutil.rmtree(self.build_dir)
+                    self.build_dir.mkdir()
+
+            with skbuild_info.open("w", encoding="utf-8") as f:
+                json.dump(self._info_dict(), f, indent=2)
 
     def _info_dict(self) -> dict[str, str]:
         """
diff --git a/src/scikit_build_core/resources/scikit-build.schema.json b/src/scikit_build_core/resources/scikit-build.schema.json
index 9559eec..8b71ae4 100644
--- a/src/scikit_build_core/resources/scikit-build.schema.json
+++ b/src/scikit_build_core/resources/scikit-build.schema.json
@@ -186,10 +186,11 @@
       "properties": {
         "mode": {
           "enum": [
-            "redirect"
+            "redirect",
+            "inplace"
           ],
           "default": "redirect",
-          "description": "Select the editable mode to use. Currently only \"redirect\" is supported."
+          "description": "Select the editable mode to use. Can be \"redirect\" (default) or \"inplace\"."
         },
         "verbose": {
           "type": "boolean",
diff --git a/src/scikit_build_core/settings/skbuild_model.py b/src/scikit_build_core/settings/skbuild_model.py
index 1e569c4..085977a 100644
--- a/src/scikit_build_core/settings/skbuild_model.py
+++ b/src/scikit_build_core/settings/skbuild_model.py
@@ -189,9 +189,9 @@ class BackportSettings:
 
 @dataclasses.dataclass
 class EditableSettings:
-    mode: Literal["redirect"] = "redirect"
+    mode: Literal["redirect", "inplace"] = "redirect"
     """
-    Select the editable mode to use. Currently only "redirect" is supported.
+    Select the editable mode to use. Can be "redirect" (default) or "inplace".
     """
 
     verbose: bool = True
diff --git a/src/scikit_build_core/settings/skbuild_read_settings.py b/src/scikit_build_core/settings/skbuild_read_settings.py
index 0daeda8..8e7af37 100644
--- a/src/scikit_build_core/settings/skbuild_read_settings.py
+++ b/src/scikit_build_core/settings/skbuild_read_settings.py
@@ -197,6 +197,19 @@ class SettingsReader:
                 )
                 raise CMakeConfigError(msg)
 
+        if self.settings.editable.mode == "inplace":
+            if self.settings.editable.rebuild:
+                rich_print(
+                    "[red][bold]ERROR:[/bold] editable rebuild is incompatible with inplace mode"
+                )
+                raise SystemExit(7)
+
+            if self.settings.build_dir:
+                rich_print(
+                    "[red][bold]ERROR:[/bold] build-dir must be empty for editable inplace mode"
+                )
+                raise SystemExit(7)
+
         if self.settings.editable.rebuild and not self.settings.build_dir:
             rich_print(
                 "[red][bold]ERROR:[/bold] editable mode with rebuild requires build_dir"


Test patch:
diff --git a/tests/packages/simplest_c/CMakeLists.txt b/tests/packages/simplest_c/CMakeLists.txt
index 9905b7b..61105f9 100644
--- a/tests/packages/simplest_c/CMakeLists.txt
+++ b/tests/packages/simplest_c/CMakeLists.txt
@@ -14,6 +14,16 @@ install(
   DESTINATION ${SKBUILD_PROJECT_NAME}
   COMPONENT PythonModule)
 
+if("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}"
+   AND DEFINED SKBUILD)
+  # Editable in-place builds. THe empty generator expression ensures
+  # multi-config enerators keeps us from having to set
+  # LIBRARY_OUTPUT_DIRECTORY_<CONFIG> too.
+  set_target_properties(
+    _module PROPERTIES LIBRARY_OUTPUT_DIRECTORY
+                       "${CMAKE_BINARY_DIR}/src/${SKBUILD_PROJECT_NAME}$<0:>")
+endif()
+
 # Testing artifacts
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/generated.txt "Testing")
 install(
diff --git a/tests/test_pyproject_pep660.py b/tests/test_pyproject_pep660.py
index c66ff3f..c4ed403 100644
--- a/tests/test_pyproject_pep660.py
+++ b/tests/test_pyproject_pep660.py
@@ -1,5 +1,6 @@
 import sys
 import sysconfig
+import typing
 import zipfile
 from pathlib import Path
 
@@ -8,6 +9,11 @@ import pytest
 from scikit_build_core.build import build_editable
 
 
+@pytest.fixture(params=["redirect", "inplace"])
+def editable_mode(request: pytest.FixtureRequest) -> str:
+    return typing.cast(str, request.param)
+
+
 # TODO: figure out why gmake is reporting no rule to make simple_pure.cpp
 @pytest.mark.compile()
 @pytest.mark.configure()
@@ -17,9 +23,9 @@ from scikit_build_core.build import build_editable
     reason="No idea why this fails on Cygwin",
 )
 @pytest.mark.usefixtures("package_simplest_c")
-def test_pep660_wheel():
+def test_pep660_wheel(editable_mode: str):
     dist = Path("dist")
-    out = build_editable("dist")
+    out = build_editable("dist", {"editable.mode": editable_mode})
     (wheel,) = dist.glob("simplest-0.0.1-*.whl")
     assert wheel == dist / out
 
@@ -29,10 +35,14 @@ def test_pep660_wheel():
             file_names = [p.name for p in p.iterdir()]
             metadata = p.joinpath("simplest-0.0.1.dist-info/METADATA").read_text()
 
-        assert len(file_names) == 4
+        assert len(file_names) == 4 if editable_mode == "redirect" else 2
         assert "simplest-0.0.1.dist-info" in file_names
-        assert "simplest" in file_names
-        assert "_simplest_editable.py" in file_names
+        if editable_mode == "redirect":
+            assert "simplest" in file_names
+            assert "_simplest_editable.py" in file_names
+        else:
+            assert "simplest" not in file_names
+            assert "_simplest_editable.py" not in file_names
         assert "_simplest_editable.pth" in file_names
 
         assert "Metadata-Version: 2.1" in metadata
@@ -43,16 +53,23 @@ def test_pep660_wheel():
 @pytest.mark.compile()
 @pytest.mark.configure()
 @pytest.mark.integration()
-@pytest.mark.parametrize("isolate", [True, False])
+@pytest.mark.parametrize("isolate", [True, False], ids=["isolated", "not_isolated"])
 @pytest.mark.usefixtures("package_simplest_c")
-def test_pep660_pip_isolated(isolated, isolate):
+def test_pep660_pip_isolated(isolated, isolate, editable_mode: str):
     isolate_args = ["--no-build-isolation"] if not isolate else []
     isolated.install("pip>=23")
     if not isolate:
         isolated.install("scikit-build-core[pyproject]")
 
+    build_dir = "" if editable_mode == "inplace" else "build/{wheel_tag}"
+
     isolated.install(
-        "-v", "--config-settings=build-dir=build/{wheel_tag}", *isolate_args, "-e", "."
+        "-v",
+        f"--config-settings=build-dir={build_dir}",
+        f"--config-settings=editable.mode={editable_mode}",
+        *isolate_args,
+        "-e",
+        ".",
     )
 
     value = isolated.execute("import simplest; print(simplest.square(2))")
@@ -67,9 +84,10 @@ def test_pep660_pip_isolated(isolated, isolate):
     python_source = Path("src/simplest").resolve()
     assert any(x.samefile(python_source) for x in locations)
 
-    # Second path is from the CMake install
     cmake_install = isolated.platlib.joinpath("simplest").resolve()
-    assert any(x.samefile(cmake_install) for x in locations)
+    if editable_mode == "redirect":
+        # Second path is from the CMake install
+        assert any(x.samefile(cmake_install) for x in locations)
 
     location = isolated.execute("import simplest; print(simplest.__file__)")
     # The package file is defined in the python source and __file__ must point to it
@@ -78,6 +96,7 @@ def test_pep660_pip_isolated(isolated, isolate):
     location = isolated.execute(
         "import simplest._module; print(simplest._module.__file__)"
     )
+
     if sys.version_info < (3, 8, 7):
         import distutils.sysconfig  # pylint: disable=deprecated-module
 
@@ -85,13 +104,15 @@ def test_pep660_pip_isolated(isolated, isolate):
     else:
         ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
 
-    module_file = cmake_install / f"_module{ext_suffix}"
+    module_source = python_source if editable_mode == "inplace" else cmake_install
+    module_file = module_source / f"_module{ext_suffix}"
+
     # Windows FindPython may produce the wrong extension
     if (
         sys.version_info < (3, 8, 7)
         and sys.platform.startswith("win")
         and not module_file.is_file()
     ):
-        module_file = cmake_install / "_module.pyd"
+        module_file = module_source / "_module.pyd"
 
     assert module_file.samefile(Path(location).resolve())