Repository: scikit-build/scikit-build-core
Pull request #565 (2023-12-04 10:32:22)

feat: Setting CMake variables though environment variables
This PR allows reading environment variables to define CMake variable. This is especially useful when the set of valid values which a CMake variable can assume is either not known a priori, or is not finite, and so `overrides` is not really helpful. 

Simple example:

```toml
[tool.scikit-build]
cmake.define.FOO = {env = "FOO"}
```

Bumps `typing_extensions` to being required for Python 3.9+.

henryiii: What about `cmake.define.FOO = {env = "FOO"}`?
stubbiali: I'm pretty open about the syntax to use. My suggestion were inspired by f-strings in Python, but I also like the idea of having a dictionary. Do you think that specifying a default value for when the env variable is not set might be useful? Something like `cmake.define.FOO = {env = "FOO", default = 0}`.
stubbiali: The logic to support the syntax `cmake.define.FOO = {env = "FOO", default = 0}` has been implemented.
henryiii: FYI, I'm taking a look at this slowly, playing with some ideas to integrate it a bit more fully (we'll want it to be part of the schema too). Should hopefully get somewhere later this week.

Patch:
diff --git a/docs/configuration.md b/docs/configuration.md
index ad292f9..f04ff8c 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -386,6 +386,14 @@ SKBUILD_CMAKE_DEFINE: SOME_DEFINE=ON
 
 ````
 
+You can also (`pyproject.toml` only) specify a dict, with `env=` to load a
+define from an environment variable, with optional `default=`.
+
+```toml
+[tool.scikit-build.cmake.define]
+SOME_DEFINE = {env="SOME_DEFINE", default="EMPTY"}
+```
+
 You can also manually specify the exact CMake args. Beyond the normal
 `SKBUILD_CMAKE_ARGS`, the `CMAKE_ARGS` space-separated environment variable is
 also supported (with some filtering for options scikit-build-core doesn't
diff --git a/pyproject.toml b/pyproject.toml
index bebdc52..d19b180 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,7 +38,7 @@ dependencies = [
     "importlib-resources >=1.3; python_version<'3.9'",
     "packaging >=20.9",
     "tomli >=1.1; python_version<'3.11'",
-    "typing-extensions >=3.10.0; python_version<'3.8'",
+    "typing-extensions >=3.10.0; python_version<'3.9'",
 ]
 # Note: for building wheels and sdists, there are also additional dependencies
 # in the pyproject extra. And cmake and possibly ninja if those are not already
diff --git a/src/scikit_build_core/_compat/typing.py b/src/scikit_build_core/_compat/typing.py
index 171dea7..05e0747 100644
--- a/src/scikit_build_core/_compat/typing.py
+++ b/src/scikit_build_core/_compat/typing.py
@@ -7,11 +7,14 @@ if sys.version_info < (3, 8):
     from typing_extensions import (
         Literal,
         Protocol,
-        get_args,
-        get_origin,
     )
 else:
-    from typing import Literal, Protocol, get_args, get_origin
+    from typing import Literal, Protocol
+
+if sys.version_info < (3, 9):
+    from typing_extensions import Annotated, get_args, get_origin
+else:
+    from typing import Annotated, get_args, get_origin
 
 if sys.version_info < (3, 11):
     if typing.TYPE_CHECKING:
@@ -26,7 +29,15 @@ if sys.version_info < (3, 11):
 else:
     from typing import Self, assert_never
 
-__all__ = ["Protocol", "Literal", "Self", "get_origin", "get_args", "assert_never"]
+__all__ = [
+    "Annotated",
+    "Protocol",
+    "Literal",
+    "Self",
+    "get_origin",
+    "get_args",
+    "assert_never",
+]
 
 
 def __dir__() -> list[str]:
diff --git a/src/scikit_build_core/build/wheel.py b/src/scikit_build_core/build/wheel.py
index fd8ddc4..d562ecf 100644
--- a/src/scikit_build_core/build/wheel.py
+++ b/src/scikit_build_core/build/wheel.py
@@ -148,7 +148,7 @@ def _build_wheel_impl(
     normalized_name = metadata.name.replace("-", "_").replace(".", "_")
 
     if settings.wheel.cmake:
-        cmake = CMake.default_search(version=settings.cmake.version)
+        cmake = CMake.default_search(version=settings.cmake.version, env=os.environ)
         cmake_msg = [f"using [blue]CMake {cmake.version}[/blue]"]
     else:
         cmake = None
diff --git a/src/scikit_build_core/builder/get_requires.py b/src/scikit_build_core/builder/get_requires.py
index 0de8c9b..0f7d751 100644
--- a/src/scikit_build_core/builder/get_requires.py
+++ b/src/scikit_build_core/builder/get_requires.py
@@ -59,6 +59,9 @@ class GetRequires:
         return self._settings
 
     def cmake(self) -> Generator[str, None, None]:
+        if os.environ.get("CMAKE_EXECUTABLE", ""):
+            return
+
         cmake_verset = self.settings.cmake.version
 
         # If the module is already installed (via caching the build
diff --git a/src/scikit_build_core/cmake.py b/src/scikit_build_core/cmake.py
index e3893e3..1f73a8d 100644
--- a/src/scikit_build_core/cmake.py
+++ b/src/scikit_build_core/cmake.py
@@ -10,16 +10,16 @@ import sys
 import sysconfig
 import textwrap
 from pathlib import Path
-from typing import TYPE_CHECKING, Generator
+from typing import TYPE_CHECKING, Any
 
 from . import __version__
 from ._logging import logger
 from ._shutil import Run
 from .errors import CMakeConfigError, CMakeNotFoundError, FailedLiveProcessError
-from .program_search import best_program, get_cmake_programs
+from .program_search import Program, best_program, get_cmake_program, get_cmake_programs
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping, Sequence
+    from collections.abc import Generator, Iterable, Mapping, Sequence
 
     from packaging.specifiers import SpecifierSet
     from packaging.version import Version
@@ -43,9 +43,19 @@ class CMake:
 
     @classmethod
     def default_search(
-        cls, *, version: SpecifierSet | None = None, module: bool = True
+        cls,
+        *,
+        version: SpecifierSet | None = None,
+        module: bool = True,
+        env: Mapping[str, Any] | None = None,
     ) -> Self:
-        candidates = get_cmake_programs(module=module)
+        env = env or {}
+        cmake_executable = env.get("CMAKE_EXECUTABLE", "")
+        candidates: Iterable[Program] = (
+            [get_cmake_program(Path(cmake_executable))]
+            if cmake_executable
+            else get_cmake_programs(module=module)
+        )
         cmake_program = best_program(candidates, version=version)
 
         if cmake_program is None:
diff --git a/src/scikit_build_core/program_search.py b/src/scikit_build_core/program_search.py
index 25e2210..7982d6a 100644
--- a/src/scikit_build_core/program_search.py
+++ b/src/scikit_build_core/program_search.py
@@ -16,7 +16,13 @@ if TYPE_CHECKING:
 
     from packaging.specifiers import SpecifierSet
 
-__all__ = ["get_cmake_programs", "get_ninja_programs", "best_program", "Program"]
+__all__ = [
+    "get_cmake_programs",
+    "get_ninja_programs",
+    "get_cmake_program",
+    "best_program",
+    "Program",
+]
 
 
 def __dir__() -> list[str]:
@@ -65,6 +71,26 @@ def _get_ninja_path(*, module: bool = True) -> Generator[Path, None, None]:
             yield Path(ninja_path)
 
 
+def get_cmake_program(cmake_path: Path) -> Program:
+    """
+    Get the Program (with version) for CMake given a path. The version will be
+    None if it cannot be determined.
+    """
+    try:
+        result = Run().capture(cmake_path, "--version")
+    except subprocess.CalledProcessError:
+        return Program(cmake_path, None)
+
+    try:
+        version = Version(result.stdout.splitlines()[0].split()[-1].split("-")[0])
+    except (IndexError, InvalidVersion):
+        logger.warning(f"Could not determine CMake version, got {result.stdout!r}")
+        return Program(cmake_path, None)
+
+    logger.info("CMake version: {}", version)
+    return Program(cmake_path, version)
+
+
 def get_cmake_programs(*, module: bool = True) -> Generator[Program, None, None]:
     """
     Get the path and version for CMake. If the version cannot be determined,
@@ -72,21 +98,7 @@ def get_cmake_programs(*, module: bool = True) -> Generator[Program, None, None]
     yielded first.
     """
     for cmake_path in _get_cmake_path(module=module):
-        try:
-            result = Run().capture(cmake_path, "--version")
-        except subprocess.CalledProcessError:
-            yield Program(cmake_path, None)
-            continue
-
-        try:
-            version = Version(result.stdout.splitlines()[0].split()[-1].split("-")[0])
-        except (IndexError, InvalidVersion):
-            logger.warning(f"Could not determine CMake version, got {result.stdout!r}")
-            yield Program(cmake_path, None)
-            continue
-
-        logger.info("CMake version: {}", version)
-        yield Program(cmake_path, version)
+        yield get_cmake_program(cmake_path)
 
 
 def get_ninja_programs(*, module: bool = True) -> Generator[Program, None, None]:
diff --git a/src/scikit_build_core/resources/scikit-build.schema.json b/src/scikit_build_core/resources/scikit-build.schema.json
index 0033346..e286954 100644
--- a/src/scikit_build_core/resources/scikit-build.schema.json
+++ b/src/scikit_build_core/resources/scikit-build.schema.json
@@ -32,10 +32,37 @@
             ".+": {
               "oneOf": [
                 {
-                  "type": "string"
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "boolean"
+                    }
+                  ]
                 },
                 {
-                  "type": "boolean"
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "env"
+                  ],
+                  "properties": {
+                    "env": {
+                      "type": "string",
+                      "minLength": 1
+                    },
+                    "default": {
+                      "oneOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "boolean"
+                        }
+                      ]
+                    }
+                  }
                 }
               ]
             }
diff --git a/src/scikit_build_core/settings/json_schema.py b/src/scikit_build_core/settings/json_schema.py
index 22b3721..260024f 100644
--- a/src/scikit_build_core/settings/json_schema.py
+++ b/src/scikit_build_core/settings/json_schema.py
@@ -9,7 +9,7 @@ from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 
 from .._compat.builtins import ExceptionGroup
-from .._compat.typing import Literal, get_args, get_origin
+from .._compat.typing import Annotated, Literal, get_args, get_origin
 from .documentation import pull_docs
 
 __all__ = ["to_json_schema", "convert_type", "FailedConversionError"]
@@ -35,6 +35,31 @@ def to_json_schema(dclass: type[Any], *, normalize_keys: bool) -> dict[str, Any]
             )
             continue
 
+        if get_origin(field.type) is Annotated:
+            if get_args(field.type)[1] == "EnvVar":
+                full = convert_type(
+                    get_args(field.type)[0], normalize_keys=normalize_keys
+                )
+                types = full["patternProperties"][".+"]
+                full["patternProperties"][".+"] = {
+                    "oneOf": [
+                        types,
+                        {
+                            "type": "object",
+                            "additionalProperties": False,
+                            "required": ["env"],
+                            "properties": {
+                                "env": {"type": "string", "minLength": 1},
+                                "default": types,
+                            },
+                        },
+                    ]
+                }
+                props[field.name] = full
+                continue
+            msg = "Only EnvVar is supported for Annotated"
+            raise FailedConversionError(msg)
+
         try:
             props[field.name] = convert_type(field.type, normalize_keys=normalize_keys)
         except FailedConversionError as err:
diff --git a/src/scikit_build_core/settings/skbuild_model.py b/src/scikit_build_core/settings/skbuild_model.py
index bfd6dc1..0e60aa7 100644
--- a/src/scikit_build_core/settings/skbuild_model.py
+++ b/src/scikit_build_core/settings/skbuild_model.py
@@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Union
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 
-from .._compat.typing import Literal
+from .._compat.typing import Annotated, Literal
 
 __all__ = [
     "BackportSettings",
@@ -45,7 +45,9 @@ class CMakeSettings:
     in config or envvar will override toml. See also ``cmake.define``.
     """
 
-    define: Dict[str, Union[str, bool]] = dataclasses.field(default_factory=dict)
+    define: Annotated[Dict[str, Union[str, bool]], "EnvVar"] = dataclasses.field(
+        default_factory=dict
+    )
     """
     A table of defines to pass to CMake when configuring the project. Additive.
     """
diff --git a/src/scikit_build_core/settings/sources.py b/src/scikit_build_core/settings/sources.py
index 395fde4..76bba50 100644
--- a/src/scikit_build_core/settings/sources.py
+++ b/src/scikit_build_core/settings/sources.py
@@ -37,6 +37,7 @@ When setting up your dataclasses, these types are handled:
 - ``List[T]``: A list of items. `;` separated supported in EnvVar/config forms. T can be a dataclass (TOML only).
 - ``Dict[str, T]``: A table of items. TOML supports a layer of nesting. Any is supported as an item type.
 - ``Literal[...]``: A list of strings, the result must be in the list.
+- ``Annotated[Dict[...], "EnvVar"]``: A dict of items, where each item can be a string or a dict with "env" and "default" keys.
 
 These are supported for JSON schema generation for the TOML, as well.
 
@@ -51,7 +52,7 @@ import typing
 from typing import Any, TypeVar, Union
 
 from .._compat.builtins import ExceptionGroup
-from .._compat.typing import Literal, Protocol, get_args, get_origin
+from .._compat.typing import Annotated, Literal, Protocol, get_args, get_origin
 
 if typing.TYPE_CHECKING:
     from collections.abc import Generator, Iterator, Mapping, Sequence
@@ -71,6 +72,10 @@ def _dig_strict(__dict: Mapping[str, Any], *names: str) -> Any:
     return __dict
 
 
+def _process_bool(value: str) -> bool:
+    return value.strip().lower() not in {"0", "false", "off", "no", ""}
+
+
 def _dig_not_strict(__dict: Mapping[str, Any], *names: str) -> Any:
     for name in names:
         __dict = __dict.get(name, {})
@@ -104,13 +109,26 @@ def _process_union(target: type[Any]) -> Any:
     return target
 
 
+def _process_annotated(target: type[Any]) -> tuple[Any, tuple[Any, ...]]:
+    """
+    Splits annotated into raw type and annotations. If not annotated, the annotations will be empty.
+    """
+
+    origin = get_origin(target)
+    if origin is Annotated:
+        return get_args(target)[0], get_args(target)[1:]
+
+    return target, ()
+
+
 def _get_target_raw_type(target: type[Any]) -> Any:
     """
     Takes a type like ``Optional[str]`` and returns str, or ``Optional[Dict[str,
     int]]`` and returns dict. Returns Union for a Union with more than one
-    non-none type. Literal is also a valid return.
+    non-none type. Literal is also a valid return. Works through Annotated.
     """
 
+    target, _ = _process_annotated(target)
     target = _process_union(target)
     origin = get_origin(target)
     return origin or target
@@ -144,6 +162,20 @@ def _nested_dataclass_to_names(__target: type[Any], *inner: str) -> Iterator[lis
         yield list(inner)
 
 
+def _dict_with_envvar(target: Any) -> Any:
+    """
+    This produces values. Supports "env" and "default" keys.
+    """
+    if not isinstance(target, dict):
+        return target
+    env = target["env"]
+    default = target.get("default", None)
+    value = os.environ.get(env, default)
+    if isinstance(default, bool) and isinstance(value, str):
+        return _process_bool(value)
+    return value
+
+
 class Source(Protocol):
     def has_item(self, *fields: str, is_dict: bool) -> bool:
         """
@@ -214,6 +246,7 @@ class EnvSource:
 
     @classmethod
     def convert(cls, item: str, target: type[Any]) -> object:
+        target, _ = _process_annotated(target)
         raw_target = _get_target_raw_type(target)
         if dataclasses.is_dataclass(raw_target):
             msg = f"Array of dataclasses are not supported in configuration settings ({raw_target})"
@@ -227,7 +260,7 @@ class EnvSource:
             return {k: cls.convert(v, _get_inner_type(target)) for k, v in items}
 
         if raw_target is bool:
-            return item.strip().lower() not in {"0", "false", "off", "no", ""}
+            return _process_bool(item)
 
         if raw_target is Union and str in get_args(target):
             return item
@@ -329,6 +362,7 @@ class ConfSource:
     def convert(
         cls, item: str | list[str] | dict[str, str], target: type[Any]
     ) -> object:
+        target, _ = _process_annotated(target)
         raw_target = _get_target_raw_type(target)
         if dataclasses.is_dataclass(raw_target):
             msg = f"Array of dataclasses are not supported in configuration settings ({raw_target})"
@@ -349,7 +383,7 @@ class ConfSource:
             msg = f"Expected {target}, got {type(item).__name__}"
             raise TypeError(msg)
         if raw_target is bool:
-            return item.strip().lower() not in {"0", "false", "off", "no", ""}
+            return _process_bool(item)
         if raw_target is Union and str in get_args(target):
             return item
         if raw_target is Literal:
@@ -414,6 +448,7 @@ class TOMLSource:
 
     @classmethod
     def convert(cls, item: Any, target: type[Any]) -> object:
+        target, annotations = _process_annotated(target)
         raw_target = _get_target_raw_type(target)
         if dataclasses.is_dataclass(raw_target):
             fields = dataclasses.fields(raw_target)
@@ -433,6 +468,12 @@ class TOMLSource:
             if not isinstance(item, dict):
                 msg = f"Expected {target}, got {type(item).__name__}"
                 raise TypeError(msg)
+            if annotations == ("EnvVar",):
+                return {
+                    k: cls.convert(_dict_with_envvar(v), _get_inner_type(target))
+                    for k, v in item.items()
+                    if _dict_with_envvar(v) is not None
+                }
             return {k: cls.convert(v, _get_inner_type(target)) for k, v in item.items()}
         if raw_target is Any:
             return item


Test patch:
diff --git a/tests/test_cmake_config.py b/tests/test_cmake_config.py
index 0b41f6c..d08f45e 100644
--- a/tests/test_cmake_config.py
+++ b/tests/test_cmake_config.py
@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+import os
 import shutil
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import pytest
 from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 
 from scikit_build_core.cmake import CMake, CMaker
 from scikit_build_core.errors import CMakeNotFoundError
@@ -117,3 +119,13 @@ def test_cmake_paths(tmp_path, fp):
     config.configure()
 
     assert len(fp.calls) == 2
+
+
+def test_get_cmake_via_envvar(monkeypatch: pytest.MonkeyPatch, fp):
+    monkeypatch.setattr("shutil.which", lambda x: x)
+    cmake_path = Path("some-prog")
+    fp.register([cmake_path, "--version"], stdout="cmake version 3.20.0\n")
+    monkeypatch.setenv("CMAKE_EXECUTABLE", str(cmake_path))
+    result = CMake.default_search(env=os.environ)
+    assert result.cmake_path == cmake_path
+    assert result.version == Version("3.20.0")
diff --git a/tests/test_schema.py b/tests/test_schema.py
index 899030e..9be8897 100644
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -61,6 +61,8 @@ def test_valid_schemas_files(filepath: Path) -> None:
         {"metadata": {"version": {"provider-path": True}}},
         {"metadata": {"version": {"provider": 2}}},
         {"metadata": {"invalid": {"provider": "correct"}}},
+        {"cmake": {"define": {"FOO": {"env": ""}}}},
+        {"cmake": {"define": {"FOO": {"default": False}}}},
     ],
 )
 def test_invalid_schemas(addition: dict[str, Any]) -> None:
@@ -100,6 +102,9 @@ def test_invalid_schemas(addition: dict[str, Any]) -> None:
         },
         {"metadata": {"version": {"provider-path": "string"}}},
         {"metadata": {"description": {"anything": True}}},
+        {"cmake": {"define": {"FOO": "BAR"}}},
+        {"cmake": {"define": {"FOO": {"env": "FOO"}}}},
+        {"cmake": {"define": {"FOO": {"env": "FOO", "default": False}}}},
     ],
 )
 def test_valid_schemas(addition: dict[str, Any]) -> None:
diff --git a/tests/test_skbuild_settings.py b/tests/test_skbuild_settings.py
index 8df7d15..2ad4a41 100644
--- a/tests/test_skbuild_settings.py
+++ b/tests/test_skbuild_settings.py
@@ -462,3 +462,46 @@ def test_skbuild_settings_version_too_old(
 
     with pytest.raises(SystemExit):
         SettingsReader.from_file(pyproject_toml, {"minimum-version": "0.7"})
+
+
+def test_skbuild_settings_pyproject_toml_envvar_defines(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+):
+    pyproject_toml = tmp_path / "pyproject.toml"
+    pyproject_toml.write_text(
+        textwrap.dedent(
+            """\
+            [tool.scikit-build.cmake.define]
+            a = "1"
+            b = {env = "SIMPLE"}
+            c = {env = "DEFAULT", default="empty"}
+            d = false
+            e = {env = "BOOL", default = false}
+            f = {env = "NOTSET"}
+            """
+        ),
+        encoding="utf-8",
+    )
+
+    config_settings: dict[str, list[str] | str] = {}
+
+    monkeypatch.setenv("SIMPLE", "2")
+    settings_reader = SettingsReader.from_file(pyproject_toml, config_settings)
+    assert settings_reader.settings.cmake.define == {
+        "a": "1",
+        "b": "2",
+        "c": "empty",
+        "d": False,
+        "e": False,
+    }
+
+    monkeypatch.setenv("DEFAULT", "3")
+    monkeypatch.setenv("BOOL", "ON")
+    settings_reader = SettingsReader.from_file(pyproject_toml, config_settings)
+    assert settings_reader.settings.cmake.define == {
+        "a": "1",
+        "b": "2",
+        "c": "3",
+        "d": False,
+        "e": True,
+    }