Repository: scikit-build/scikit-build-core
Pull request #457 (2023-08-10 20:10:34)

feat: regex and dynamic-metadata rewrite
Based on https://github.com/scikit-build/dynamic-metadata/pull/1 (with some updates). Fixes https://github.com/scikit-build/scikit-build-core/issues/435.

This doesn't start using `tool.dynamic-metadata` yet, to give us room to change that. Plus we have to back-compat support `tool.scikit-build.metadata`, so might as well keep that as the only way to set this for now.

henryiii: A few thoughts: (A) Should we have a write_to / template pair of options for the regex plugin? (B) Or should we put metadata file writing into scikit-build-core generally? (C) Or should we build in something to dynamic-metadata to handle this generally? One benefit of the last one is that you could probably keep from writing the file out into the source tree - except probably with editable installs.

I think I'm pretty strongly in favor of B, maybe along with a mention in dynamic-metadata of how backends could do this.
henryiii: @bennyrowland - I'm making this small change to the interface for plugins following https://github.com/scikit-build/dynamic-metadata/pull/1. Let me know if there are any issues. I've been moving ahead with option B above, so will likely merge this soon and then open a PR for option B.

Patch:
diff --git a/docs/api/scikit_build_core.metadata.rst b/docs/api/scikit_build_core.metadata.rst
index 8350c49..0abd7e0 100644
--- a/docs/api/scikit_build_core.metadata.rst
+++ b/docs/api/scikit_build_core.metadata.rst
@@ -17,6 +17,14 @@ scikit\_build\_core.metadata.fancy\_pypi\_readme module
    :undoc-members:
    :show-inheritance:
 
+scikit\_build\_core.metadata.regex module
+-----------------------------------------
+
+.. automodule:: scikit_build_core.metadata.regex
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 scikit\_build\_core.metadata.setuptools\_scm module
 ---------------------------------------------------
 
diff --git a/docs/configuration.md b/docs/configuration.md
index 81be30f..d09bfeb 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -445,6 +445,31 @@ metadata.readme.provider = "scikit_build_core.metadata.fancy_pypi_readme"
 
 :::
 
+:::{tab} Regex
+
+If you want to pull a string-valued expression (usually version) from an
+existing file, you can the integrated `regex` plugin to pull the information.
+
+```toml
+name = "mypackage"
+dynamic = ["version"]
+
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "src/mypackage/__init__.py"
+```
+
+You can set a custom regex with `regex=`; use `(?p<value>...)` to capture the
+value you want to use. By default when targeting version, you get a reasonable
+regex for python files,
+`'(?i)^(__version__|VERSION) *= *([\'"])v?(?P<value>.+?)\2'`.
+
+```{versionadded} 0.5
+
+```
+
+:::
+
 ## Editable installs
 
 Experimental support for editable installs is provided, with some caveats and
diff --git a/src/scikit_build_core/metadata/fancy_pypi_readme.py b/src/scikit_build_core/metadata/fancy_pypi_readme.py
index 1040ee9..ad9b4cc 100644
--- a/src/scikit_build_core/metadata/fancy_pypi_readme.py
+++ b/src/scikit_build_core/metadata/fancy_pypi_readme.py
@@ -4,7 +4,7 @@ from pathlib import Path
 
 from .._compat import tomllib
 
-__all__ = ["dynamic_metadata"]
+__all__ = ["dynamic_metadata", "get_requires_for_dynamic_metadata"]
 
 
 def __dir__() -> list[str]:
@@ -12,13 +12,13 @@ def __dir__() -> list[str]:
 
 
 def dynamic_metadata(
-    fields: frozenset[str],
+    field: str,
     settings: dict[str, list[str] | str] | None = None,
-) -> dict[str, str | dict[str, str | None]]:
+) -> str | dict[str, str]:
     from hatch_fancy_pypi_readme._builder import build_text
     from hatch_fancy_pypi_readme._config import load_and_validate_config
 
-    if fields != {"readme"}:
+    if field != "readme":
         msg = "Only the 'readme' field is supported"
         raise ValueError(msg)
 
@@ -35,13 +35,11 @@ def dynamic_metadata(
 
     # Version 22.3 does not have fragment support
     return {
-        "readme": {
-            "content-type": config.content_type,
-            "text": build_text(config.fragments, config.substitutions)
-            if hasattr(config, "substitutions")
-            # pylint: disable-next=no-value-for-parameter
-            else build_text(config.fragments),  # type: ignore[call-arg]
-        }
+        "content-type": config.content_type,
+        "text": build_text(config.fragments, config.substitutions)
+        if hasattr(config, "substitutions")
+        # pylint: disable-next=no-value-for-parameter
+        else build_text(config.fragments),  # type: ignore[call-arg]
     }
 
 
diff --git a/src/scikit_build_core/metadata/regex.py b/src/scikit_build_core/metadata/regex.py
new file mode 100644
index 0000000..2feae1e
--- /dev/null
+++ b/src/scikit_build_core/metadata/regex.py
@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+__all__ = ["dynamic_metadata"]
+
+
+def __dir__() -> list[str]:
+    return __all__
+
+
+def dynamic_metadata(
+    field: str,
+    settings: Mapping[str, Any],
+) -> str:
+    # Input validation
+    if field not in {"version", "description", "requires-python"}:
+        msg = "Only string fields supported by this plugin"
+        raise RuntimeError(msg)
+    if settings.keys() > {"input", "regex"}:
+        msg = "Only 'input' and 'regex' settings allowed by this plugin"
+        raise RuntimeError(msg)
+    if "input" not in settings:
+        msg = "Must contain the 'input' setting to perform a regex on"
+        raise RuntimeError(msg)
+    if field != "version" and "regex" not in settings:
+        msg = "Must contain the 'regex' setting if not getting version"
+        raise RuntimeError(msg)
+    if not all(isinstance(x, str) for x in settings.values()):
+        msg = "Must set 'input' and/or 'regex' to strings"
+        raise RuntimeError(msg)
+
+    input_filename = settings["input"]
+    regex = settings.get(
+        "regex", r'(?i)^(__version__|VERSION) *= *([\'"])v?(?P<value>.+?)\2'
+    )
+
+    with Path(input_filename).open(encoding="utf-8") as f:
+        match = re.search(regex, f.read(), re.MULTILINE)
+
+    if not match:
+        msg = f"Couldn't find {regex!r} in {input_filename}"
+        raise RuntimeError(msg)
+
+    return match.group("value")
diff --git a/src/scikit_build_core/metadata/setuptools_scm.py b/src/scikit_build_core/metadata/setuptools_scm.py
index 2f902ef..6efd79a 100644
--- a/src/scikit_build_core/metadata/setuptools_scm.py
+++ b/src/scikit_build_core/metadata/setuptools_scm.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["dynamic_metadata"]
+__all__ = ["dynamic_metadata", "get_requires_for_dynamic_metadata"]
 
 
 def __dir__() -> list[str]:
@@ -8,13 +8,13 @@ def __dir__() -> list[str]:
 
 
 def dynamic_metadata(
-    fields: frozenset[str],
+    field: str,
     settings: dict[str, object] | None = None,
-) -> dict[str, str | dict[str, str | None]]:
+) -> str:
     # this is a classic implementation, waiting for the release of
     # vcs-versioning and an improved public interface
 
-    if fields != {"version"}:
+    if field != "version":
         msg = "Only the 'version' field is supported"
         raise ValueError(msg)
 
@@ -27,7 +27,7 @@ def dynamic_metadata(
     config = Configuration.from_file("pyproject.toml")
     version: str = _get_version(config)
 
-    return {"version": version}
+    return version
 
 
 def get_requires_for_dynamic_metadata(
diff --git a/src/scikit_build_core/settings/_load_provider.py b/src/scikit_build_core/settings/_load_provider.py
index b735a2b..43fa73d 100644
--- a/src/scikit_build_core/settings/_load_provider.py
+++ b/src/scikit_build_core/settings/_load_provider.py
@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import importlib
 import sys
-from collections.abc import Iterable
+from collections.abc import Generator, Iterable, Mapping
 from pathlib import Path
-from typing import Any
+from typing import Any, Union
 
 from .._compat.typing import Protocol
 
-__all__ = ["load_provider"]
+__all__ = ["load_provider", "load_dynamic_metadata"]
 
 
 def __dir__() -> list[str]:
@@ -27,10 +27,31 @@ class DynamicMetadataRequirementsProtocol(DynamicMetadataProtocol, Protocol):
         ...
 
 
+class DynamicMetadataWheelProtocol(DynamicMetadataProtocol, Protocol):
+    def dynamic_wheel(
+        self, field: str, settings: Mapping[str, Any] | None = None
+    ) -> bool:
+        ...
+
+
+class DynamicMetadataRequirementsWheelProtocol(
+    DynamicMetadataRequirementsProtocol, DynamicMetadataWheelProtocol, Protocol
+):
+    ...
+
+
+DMProtocols = Union[
+    DynamicMetadataProtocol,
+    DynamicMetadataRequirementsProtocol,
+    DynamicMetadataWheelProtocol,
+    DynamicMetadataRequirementsWheelProtocol,
+]
+
+
 def load_provider(
     provider: str,
     provider_path: str | None = None,
-) -> DynamicMetadataProtocol | DynamicMetadataRequirementsProtocol:
+) -> DMProtocols:
     if provider_path is None:
         return importlib.import_module(provider)
 
@@ -43,3 +64,16 @@ def load_provider(
         return importlib.import_module(provider)
     finally:
         sys.path.pop(0)
+
+
+def load_dynamic_metadata(
+    metadata: Mapping[str, Mapping[str, str]]
+) -> Generator[tuple[str, DMProtocols | None, dict[str, str]], None, None]:
+    for field, orig_config in metadata.items():
+        if "provider" in orig_config:
+            config = dict(orig_config)
+            provider = config.pop("provider")
+            provider_path = config.pop("provider-path", None)
+            yield field, load_provider(provider, provider_path), config
+        else:
+            yield field, None, dict(orig_config)
diff --git a/src/scikit_build_core/settings/metadata.py b/src/scikit_build_core/settings/metadata.py
index 33c424e..2a41c01 100644
--- a/src/scikit_build_core/settings/metadata.py
+++ b/src/scikit_build_core/settings/metadata.py
@@ -1,12 +1,13 @@
 from __future__ import annotations
 
+from collections.abc import Mapping
 from typing import Any
 
 from packaging.version import Version
 from pyproject_metadata import StandardMetadata
 
 from ..settings.skbuild_model import ScikitBuildSettings
-from ._load_provider import load_provider
+from ._load_provider import load_dynamic_metadata
 
 __all__ = ["get_standard_metadata"]
 
@@ -17,37 +18,24 @@ def __dir__() -> list[str]:
 
 # If pyproject-metadata eventually supports updates, this can be simplified
 def get_standard_metadata(
-    pyproject_dict: dict[str, Any],
+    pyproject_dict: Mapping[str, Any],
     settings: ScikitBuildSettings,
 ) -> StandardMetadata:
+    new_pyproject_dict = dict(pyproject_dict)
     # Handle any dynamic metadata
-    calls: dict[frozenset[tuple[str, Any]], set[str]] = {}
-    for field, raw_settings in settings.metadata.items():
-        if field not in pyproject_dict.get("project", {}).get("dynamic", []):
-            msg = f"{field} is not in project.dynamic"
-            raise KeyError(msg)
-        if "provider" not in raw_settings:
+    for field, provider, config in load_dynamic_metadata(settings.metadata):
+        if provider is None:
             msg = f"{field} is missing provider"
             raise KeyError(msg)
-        calls.setdefault(frozenset(raw_settings.items()), set()).add(field)
-
-    for call, fields in calls.items():
-        args = dict(call)
-        provider = args.pop("provider")
-        provider_path = args.pop("provider-path", None)
-        computed = load_provider(provider, provider_path).dynamic_metadata(
-            frozenset(fields), args
-        )
-        if set(computed) != fields:
-            msg = f"{provider} did not return requested fields"
+        if field not in pyproject_dict.get("project", {}).get("dynamic", []):
+            msg = f"{field} is not in project.dynamic"
             raise KeyError(msg)
-        pyproject_dict["project"].update(computed)
-        for field in fields:
-            pyproject_dict["project"]["dynamic"].remove(field)
+        new_pyproject_dict["project"][field] = provider.dynamic_metadata(field, config)
+        new_pyproject_dict["project"]["dynamic"].remove(field)
 
-    metadata = StandardMetadata.from_pyproject(pyproject_dict)
+    metadata = StandardMetadata.from_pyproject(new_pyproject_dict)
     # pyproject-metadata normalizes the name - see https://github.com/FFY00/python-pyproject-metadata/pull/65
     # For scikit-build-core 0.5+, we keep the un-normalized name, and normalize it when using it for filenames
     if settings.minimum_version is None or settings.minimum_version >= Version("0.5"):
-        metadata.name = pyproject_dict["project"]["name"]
+        metadata.name = new_pyproject_dict["project"]["name"]
     return metadata


Test patch:
diff --git a/tests/packages/custom_cmake/CMakeLists.txt b/tests/packages/custom_cmake/CMakeLists.txt
index b6b3699..5642e0b 100644
--- a/tests/packages/custom_cmake/CMakeLists.txt
+++ b/tests/packages/custom_cmake/CMakeLists.txt
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.15...3.26)
 project(
   ${SKBUILD_PROJECT_NAME}
   LANGUAGES
-  VERSION ${SKBUILD_PROJECT_VERSION})
+  VERSION 2.3.4)
 
 find_package(ExamplePkg REQUIRED)
 
diff --git a/tests/packages/custom_cmake/pyproject.toml b/tests/packages/custom_cmake/pyproject.toml
index c72e803..7be9455 100644
--- a/tests/packages/custom_cmake/pyproject.toml
+++ b/tests/packages/custom_cmake/pyproject.toml
@@ -4,4 +4,13 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "custom_modules"
-version = "0.0.1"
+dynamic = ["version"]
+
+[tool.scikit-build]
+wheel.packages = []
+wheel.license-files = []
+
+[tool.scikit-build.metadata.version]
+provider = "scikit_build_core.metadata.regex"
+input = "CMakeLists.txt"
+regex = 'project\([^)]+ VERSION (?P<value>[0-9.]+)'
diff --git a/tests/packages/dynamic_metadata/plugins/local/version/nested/__init__.py b/tests/packages/dynamic_metadata/plugins/local/version/nested/__init__.py
index f68ad03..d26c09c 100644
--- a/tests/packages/dynamic_metadata/plugins/local/version/nested/__init__.py
+++ b/tests/packages/dynamic_metadata/plugins/local/version/nested/__init__.py
@@ -8,10 +8,10 @@ def __dir__() -> list[str]:
 
 
 def dynamic_metadata(
-    fields: frozenset[str],
+    field: str,
     settings: dict[str, object] | None = None,
-) -> dict[str, str | dict[str, str | None]]:
-    if fields != {"version"}:
+) -> str:
+    if field != "version":
         msg = "Only the 'version' field is supported"
         raise ValueError(msg)
 
@@ -19,4 +19,4 @@ def dynamic_metadata(
         msg = "No inline configuration is supported"
         raise ValueError(msg)
 
-    return {"version": "3.2.1"}
+    return "3.2.1"
diff --git a/tests/test_custom_modules.py b/tests/test_custom_modules.py
index 5deae17..13c70db 100644
--- a/tests/test_custom_modules.py
+++ b/tests/test_custom_modules.py
@@ -22,3 +22,11 @@ def test_ep(isolated):
     pysys = isolated.execute("import sys; print(sys.executable)").strip()
     contents = Path(script).read_text()
     assert contents.startswith(f"#!{pysys}")
+
+    if sys.version_info >= (3, 8):
+        assert (
+            isolated.execute(
+                "from importlib import metadata; print(metadata.version('custom_modules'), end='')"
+            )
+            == "2.3.4"
+        )
diff --git a/tests/test_dynamic_metadata.py b/tests/test_dynamic_metadata.py
index 9663aa8..1359a30 100644
--- a/tests/test_dynamic_metadata.py
+++ b/tests/test_dynamic_metadata.py
@@ -16,6 +16,7 @@ from packaging.version import Version
 from scikit_build_core._compat import tomllib
 from scikit_build_core.build import build_wheel
 from scikit_build_core.builder.get_requires import GetRequires
+from scikit_build_core.metadata import regex
 from scikit_build_core.settings.metadata import get_standard_metadata
 from scikit_build_core.settings.skbuild_read_settings import SettingsReader
 
@@ -24,43 +25,43 @@ from scikit_build_core.settings.skbuild_read_settings import SettingsReader
 # it turns out to be easier to create EntryPoint objects pointing to real
 # functions than to mock them.
 def ep_version(
-    fields: frozenset[str],
+    field: str,
     _settings: dict[str, object] | None = None,
-) -> dict[str, str | dict[str, str | None]]:
-    assert fields == {"version"}
-    return {"version": "0.0.2"}
+) -> str:
+    assert field == "version"
+    return "0.0.2"
 
 
 def ep_readme(
-    fields: frozenset[str],
+    field: str,
     _settings: dict[str, object] | None = None,
-) -> dict[str, str | dict[str, str | None]]:
-    assert fields == {"readme"}
+) -> str | dict[str, str | None]:
+    assert field == "readme"
     return {
-        "readme": {
-            "content-type": "text/x-rst",
-            "text": "Some text",
-        }
+        "content-type": "text/x-rst",
+        "text": "Some text",
     }
 
 
 def ep_license(
-    fields: frozenset[str],
+    field: str,
     _settings: dict[str, object] | None = None,
-) -> dict[str, str | dict[str, str | None]]:
-    assert fields == {"license"}
-    return {"license": {"text": "MIT License"}}
+) -> dict[str, str | None]:
+    assert field == "license"
+    return {"text": "MIT License"}
 
 
 def ep_dual(
-    _fields: list[str],
+    _field: str,
     _settings: dict[str, object] | None = None,
-) -> dict[str, str | dict[str, str | None]]:
+) -> str | dict[str, str | None]:
     # Fields intentionally not checked to verify backend error thrown
-    return {
-        "version": "0.3",
-        "license": {"text": "BSD License"},
-    }
+    if _field == "version":
+        return "0.3"
+    if _field == "license":
+        return {"text": "BSD License"}
+    msg = f"Invalid field {_field}"
+    raise KeyError(msg)
 
 
 original_loader = importlib.import_module
@@ -254,3 +255,21 @@ def test_pep517_wheel(virtualenv):
 
     version = virtualenv.execute("from dynamic import square; print(square(2))")
     assert version == "4.0"
+
+
+def test_regex(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    d = tmp_path / "test_regex"
+    d.mkdir()
+    monkeypatch.chdir(d)
+
+    with Path("__init__.py").open("w") as f:
+        f.write("__version__ = '0.1.0'")
+
+    regex.dynamic_metadata("version", {"input": "__init__.py"})
+
+
+def test_regex_errors() -> None:
+    with pytest.raises(RuntimeError):
+        regex.dynamic_metadata("version", {})
+    with pytest.raises(RuntimeError, match="Only string fields supported"):
+        regex.dynamic_metadata("author", {"input": "x", "regex": "x"})