Repository: python-wheel-build/fromager

Issue:
add a command to list all packages with overrides
When setting up a build environment with overrides, it will be useful to be able to establish tests for each customized build. To do that, it would be useful to be able to know exactly which packages have overrides, since that would let someone create a "linter" for their CI config to ensure that as a new override is added, a new test job is also added.

We should create a command to list the available overrides, including all of the plugins that can be loaded, any packages that have patches, and any packages that have env vars. The list should not include packages listed as pre-built.

The output should be just the list of packages, without versions, one name per line written to the console. We could optionally include versions, but since plugins don't have versions that may be less useful, so let's consider that as an enhancement for later.

The function used to generate the list of names should be a stable API so that a linter can import it and invoke it to get an iterable of the same names that are printed by the new command.

Discussion:
dhellmann: The way the patch filenames are being parsed is not correct.

Patch:
diff --git a/src/fromager/commands/__init__.py b/src/fromager/commands/__init__.py
index 05d92b8..aee7a37 100644
--- a/src/fromager/commands/__init__.py
+++ b/src/fromager/commands/__init__.py
@@ -1,10 +1,19 @@
-from . import bootstrap, build, build_order, canonicalize, download_sequence, step
+from . import (
+    bootstrap,
+    build,
+    build_order,
+    canonicalize,
+    download_sequence,
+    list_overrides,
+    step,
+)
 
 commands = [
     bootstrap.bootstrap,
     build.build,
     build.build_sequence,
     build_order.build_order,
+    list_overrides.list_overrides,
     step.step,
     canonicalize.canonicalize,
     download_sequence.download_sequence,
diff --git a/src/fromager/commands/list_overrides.py b/src/fromager/commands/list_overrides.py
new file mode 100644
index 0000000..1d4c78b
--- /dev/null
+++ b/src/fromager/commands/list_overrides.py
@@ -0,0 +1,13 @@
+import click
+
+from fromager import context, overrides
+
+
+@click.command()
+@click.pass_obj
+def list_overrides(
+    wkctx: context.WorkContext,
+):
+    """List all of the packages with overrides in the current configuration."""
+    for name in overrides.list_all(wkctx.patches_dir, wkctx.envs_dir):
+        print(name)
diff --git a/src/fromager/overrides.py b/src/fromager/overrides.py
index b89bc53..4adaf79 100644
--- a/src/fromager/overrides.py
+++ b/src/fromager/overrides.py
@@ -5,7 +5,8 @@ import pathlib
 import string
 import typing
 
-from packaging.utils import canonicalize_name
+from packaging.utils import canonicalize_name, parse_sdist_filename
+from packaging.version import Version
 from stevedore import extension
 
 # An interface for reretrieving per-package information which influences
@@ -128,3 +129,54 @@ def find_override_method(distname: str, method: str) -> typing.Callable:
         return None
     logger.info("found %s override for %s", method, distname)
     return getattr(mod, method)
+
+
+def list_all(patches_dir: pathlib.Path, envs_dir: pathlib.Path, test: bool = False):
+    exts = _get_extensions()
+
+    def patched_projects():
+        for item in patches_dir.glob("*"):
+            if not item.is_dir():
+                continue
+            fake_sdist = item.name + ".tar.gz"
+            name, _ = parse_sdist_filename(fake_sdist)
+            yield name
+
+    def patched_projects_legacy():
+        for item in patches_dir.glob("*.patch"):
+            parts = []
+            for p in item.stem.split("-"):
+                parts.append(p)
+                try:
+                    Version(p)
+                    # Stop when we get something we can parse as a version string.
+                    break
+                except Exception:
+                    pass
+            fake_sdist = ("-".join(parts)) + ".tar.gz"
+            try:
+                name, _ = parse_sdist_filename(fake_sdist)
+            except Exception as err:
+                logger.warning(f"could not extract package name from {item}: {err}")
+                continue
+            yield name
+
+    def env_projects():
+        for item in envs_dir.glob("*/*.env"):
+            yield item.stem
+
+    # Use canonicalize_name() to ensure we can correctly remove duplicate
+    # entries from the return list.
+    return sorted(
+        set(
+            canonicalize_name(n)
+            for n in itertools.chain(
+                exts.names(),
+                patched_projects(),
+                patched_projects_legacy(),
+                env_projects(),
+            )
+            if not test
+            or n != "fromager_test"  # filter out test package except in test mode
+        )
+    )


Test patch:
diff --git a/tests/test_overrides.py b/tests/test_overrides.py
index 08420a4..32a1164 100644
--- a/tests/test_overrides.py
+++ b/tests/test_overrides.py
@@ -96,3 +96,52 @@ def test_extra_environ_for_pkg_expansion(tmp_path: pathlib.Path):
 
     with pytest.raises(ValueError):
         extra_environ = overrides.extra_environ_for_pkg(tmp_path, pkg_name, variant)
+
+
+def test_list_all(tmp_path):
+    patches_dir = tmp_path / "patches"
+    patches_dir.mkdir()
+
+    project_patch_dir = patches_dir / "project-with-patch-1.2.3"
+    project_patch_dir.mkdir()
+
+    # legacy form
+    p1 = patches_dir / "legacy-project-1.2.3-001.patch"
+    np1 = patches_dir / "legacy-project-1.2.3.txt"
+    p2 = patches_dir / "fromager_test-1.2.3.patch"  # duplicate
+
+    # new form with project dir
+    p3 = project_patch_dir / "003.patch"
+    p4 = project_patch_dir / "004.patch"
+    np2 = project_patch_dir / "not-a-patch.txt"
+
+    # Create all of the test files
+    for p in [p1, p2, p3, p4]:
+        p.write_text("this is a patch file")
+    for f in [np1, np2]:
+        f.write_text("this is not a patch file")
+
+    env_dir = tmp_path / "env"
+    env_dir.mkdir()
+    variant_dir = env_dir / "variant"
+    variant_dir.mkdir()
+    project_env = variant_dir / "project-with-env.env"
+    project_env.write_text("VAR1=VALUE1\nVAR2=VALUE2")
+    project_env2 = variant_dir / "fromager_test.env"
+    project_env2.write_text("VAR1=VALUE1\nVAR2=VALUE2")  # duplicate
+
+    expected = [
+        "project-with-patch",
+        "legacy-project",
+        "project-with-env",
+        "fromager-test",
+    ]
+    expected.sort()
+
+    packages = overrides.list_all(
+        patches_dir=patches_dir,
+        envs_dir=env_dir,
+        test=True,
+    )
+
+    assert expected == packages