Repository: scikit-build/scikit-build-core
Pull request #827 (2024-07-25 19:46:00)

fix: support nested and local gitignores
Supports nested gitignores and `.git/info/exclude`. Close https://github.com/scikit-build/scikit-build-core/issues/726.

As a followup, we could add a `sdist.ignore-system = ["git", "local"]` or similar, which would allow scikit-build-core to directly use git.


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/scikit-build/scikit-build-core/pull/827?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scikit-build) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 83.57%. Comparing base [(`d977a08`)](https://app.codecov.io/gh/scikit-build/scikit-build-core/commit/d977a084ff2716bd66e84ac2c5248618b069d6b5?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scikit-build) to head [(`9d20bc3`)](https://app.codecov.io/gh/scikit-build/scikit-build-core/commit/9d20bc34a6e8d9b3d6107ebaeaa954599a632556?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scikit-build).
> Report is 2 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #827      +/-   ##
==========================================
- Coverage   83.76%   83.57%   -0.20%     
==========================================
  Files          74       74              
  Lines        4139     4200      +61     
==========================================
+ Hits         3467     3510      +43     
- Misses        672      690      +18     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/scikit-build/scikit-build-core/pull/827?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scikit-build).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=scikit-build).


Patch:
diff --git a/src/scikit_build_core/build/_file_processor.py b/src/scikit_build_core/build/_file_processor.py
index e4dbf3b..c396718 100644
--- a/src/scikit_build_core/build/_file_processor.py
+++ b/src/scikit_build_core/build/_file_processor.py
@@ -37,18 +37,38 @@ def each_unignored_file(
     Runs through all non-ignored files. Must be run from the root directory.
     """
     exclude_lines = EXCLUDE_LINES + list(exclude)
-    gi = Path(".gitignore")
-    with contextlib.suppress(FileNotFoundError), gi.open(encoding="utf-8") as f:
-        exclude_lines += f.readlines()
+
+    for gi in [Path(".git/info/exclude"), Path(".gitignore")]:
+        with contextlib.suppress(FileNotFoundError), gi.open(encoding="utf-8") as f:
+            exclude_lines += f.readlines()
+
+    nested_excludes = {
+        p.parent: pathspec.GitIgnoreSpec.from_lines(p.read_text().splitlines())
+        for p in Path().rglob("**/.gitignore")
+        if p != Path(".gitignore")
+    }
 
     exclude_spec = pathspec.GitIgnoreSpec.from_lines(exclude_lines)
     include_spec = pathspec.GitIgnoreSpec.from_lines(include)
 
-    for dirpath, _, filenames in os.walk(str(starting_path), followlinks=True):
-        all_paths = (Path(dirpath) / fn for fn in filenames)
-        paths = (
-            p
-            for p in all_paths
-            if not exclude_spec.match_file(p) or include_spec.match_file(p)
-        )
-        yield from paths
+    for dirstr, _, filenames in os.walk(str(starting_path), followlinks=True):
+        dirpath = Path(dirstr)
+        all_paths = (dirpath / fn for fn in filenames)
+        for p in all_paths:
+            # Always include something included
+            if include_spec.match_file(p):
+                yield p
+
+            # Ignore from global ignore
+            if exclude_spec.match_file(p):
+                continue
+
+            # Check relative ignores (Python 3.9's is_relative_to workaround)
+            if any(
+                nex.match_file(p.relative_to(np))
+                for np, nex in nested_excludes.items()
+                if dirpath == np or np in dirpath.parents
+            ):
+                continue
+
+            yield p


Test patch:
diff --git a/tests/test_file_processor.py b/tests/test_file_processor.py
index 83aa65c..f1f993e 100644
--- a/tests/test_file_processor.py
+++ b/tests/test_file_processor.py
@@ -38,6 +38,18 @@ def test_on_each_with_symlink(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -
     exposed_symlink = dir / "exposed_symlink"
     exposed_symlink.symlink_to("../hidden_dir")
 
+    local_ignored_file = Path("local_ignored_file.txt")
+    Path(".git/info").mkdir(parents=True)
+    Path(".git/info/exclude").write_text(f"{local_ignored_file}\n")
+
+    nested_dir = Path("nested_dir")
+    nested_dir.mkdir()
+    nested_dir.joinpath("not_ignored.txt").write_text("content")
+    nested_dir.joinpath("ignored.txt").write_text("content")
+    nested_dir.joinpath(".gitignore").write_text("ignored.txt")
+    nested_dir.joinpath("more").mkdir()
+    nested_dir.joinpath("more/ignored.txt").write_text("content")
+
     if (
         sys.platform.startswith("win")
         and not exposed_symlink.joinpath("file2").is_file()
@@ -45,9 +57,11 @@ def test_on_each_with_symlink(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -
         pytest.skip("Windows symlink support not available")
 
     # Test that each_unignored_file() follows the symlink
-    assert sorted(each_unignored_file(Path())) == [
+    assert set(each_unignored_file(Path())) == {
         gitignore,
         exposed_symlink / "file2",
         file1,
         file2,
-    ]
+        nested_dir / "not_ignored.txt",
+        nested_dir / ".gitignore",
+    }