Repository: pantsbuild/pants

Issue:
Hook up new BUILD file formatters to `lint`
See https://github.com/pantsbuild/pants/pull/16650


Discussion:

Resolving patch:
diff --git a/src/python/pants/core/goals/lint.py b/src/python/pants/core/goals/lint.py
--- a/src/python/pants/core/goals/lint.py
+++ b/src/python/pants/core/goals/lint.py
@@ -5,11 +5,12 @@
 
 import itertools
 import logging
+import os
 from dataclasses import dataclass
-from typing import Any, ClassVar, Iterable, Iterator, Type, TypeVar, cast
+from typing import Any, Callable, ClassVar, Iterable, Iterator, TypeVar, cast
 
 from pants.base.specs import Specs
-from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
+from pants.core.goals.fmt import FmtResult, FmtTargetsRequest, _FmtBuildFilesRequest
 from pants.core.goals.style_request import (
     StyleRequest,
     determine_specified_tool_names,
@@ -21,8 +22,10 @@
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.console import Console
 from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType
-from pants.engine.fs import EMPTY_DIGEST, Digest, SpecsPaths, Workspace
+from pants.engine.fs import EMPTY_DIGEST, Digest, PathGlobs, Snapshot, SpecsPaths, Workspace
 from pants.engine.goal import Goal, GoalSubsystem
+from pants.engine.internals.build_files import BuildFileOptions
+from pants.engine.internals.native_engine import FilespecMatcher
 from pants.engine.process import FallibleProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule
 from pants.engine.target import FieldSet, FilteredTargets, SourcesField
@@ -38,13 +41,14 @@
 logger = logging.getLogger(__name__)
 
 _SR = TypeVar("_SR", bound=StyleRequest)
+_T = TypeVar("_T")
 
 
 class AmbiguousRequestNamesError(Exception):
     def __init__(
         self,
         ambiguous_name: str,
-        requests: set[Type[StyleRequest] | Type[LintFilesRequest]],
+        requests: set[type],
     ):
         request_names = {
             f"{request_target.__module__}.{request_target.__qualname__}"
@@ -216,9 +220,12 @@ class Lint(Goal):
 
 
 def _check_ambiguous_request_names(
-    *requests: Type[LintFilesRequest] | Type[StyleRequest],
+    *requests: type,
 ) -> None:
-    for name, request_group in itertools.groupby(requests, key=lambda target: target.name):
+    def key(target: type) -> str:
+        return target.name  # type: ignore[attr-defined,no-any-return]
+
+    for name, request_group in itertools.groupby(requests, key=key):
         request_group_set = set(request_group)
 
         if len(request_group_set) > 1:
@@ -261,6 +268,7 @@ async def lint(
     console: Console,
     workspace: Workspace,
     specs: Specs,
+    build_file_options: BuildFileOptions,
     lint_subsystem: LintSubsystem,
     union_membership: UnionMembership,
     dist_dir: DistDir,
@@ -271,26 +279,43 @@ async def lint(
     fmt_target_request_types = cast(
         "Iterable[type[FmtTargetsRequest]]", union_membership.get(FmtTargetsRequest)
     )
+    fmt_build_request_types = cast(
+        "Iterable[type[_FmtBuildFilesRequest]]", union_membership.get(_FmtBuildFilesRequest)
+    )
     file_request_types = cast(
         "Iterable[type[LintFilesRequest]]", union_membership[LintFilesRequest]
     )
 
+    # NB: Target formatters and build file formatters can share a name, so we can't check them both
+    # for ambiguity at the same time.
+    _check_ambiguous_request_names(
+        *lint_target_request_types,
+        *fmt_target_request_types,
+        *file_request_types,
+    )
+
     _check_ambiguous_request_names(
-        *lint_target_request_types, *fmt_target_request_types, *file_request_types
+        *lint_target_request_types,
+        *fmt_build_request_types,
+        *file_request_types,
     )
 
     specified_names = determine_specified_tool_names(
         "lint",
         lint_subsystem.only,
         [*lint_target_request_types, *fmt_target_request_types],
-        extra_valid_names={request.name for request in file_request_types},
+        extra_valid_names={
+            request.name  # type: ignore[attr-defined]
+            for request in [*file_request_types, *fmt_build_request_types]
+        },
     )
 
-    def is_specified(request_type: type[StyleRequest] | type[LintFilesRequest]):
-        return request_type.name in specified_names
+    def is_specified(request_type: type):
+        return request_type.name in specified_names  # type: ignore[attr-defined]
 
     lint_target_request_types = filter(is_specified, lint_target_request_types)
     fmt_target_request_types = filter(is_specified, fmt_target_request_types)
+    fmt_build_request_types = filter(is_specified, fmt_build_request_types)
     file_request_types = filter(is_specified, file_request_types)
 
     _get_targets = Get(
@@ -298,13 +323,22 @@ def is_specified(request_type: type[StyleRequest] | type[LintFilesRequest]):
         Specs,
         specs if lint_target_request_types or fmt_target_request_types else Specs.empty(),
     )
-    _get_specs_paths = Get(SpecsPaths, Specs, specs if file_request_types else Specs.empty())
+    _get_specs_paths = Get(
+        SpecsPaths, Specs, specs if file_request_types or fmt_build_request_types else Specs.empty()
+    )
     targets, specs_paths = await MultiGet(_get_targets, _get_specs_paths)
 
-    def batch(field_sets: Iterable[FieldSet]) -> Iterator[list[FieldSet]]:
+    specified_build_files = FilespecMatcher(
+        includes=[os.path.join("**", p) for p in build_file_options.patterns],
+        excludes=build_file_options.ignores,
+    ).matches(specs_paths.files)
+
+    def batch(
+        iterable: Iterable[_T], key: Callable[[_T], str] = lambda x: str(x)
+    ) -> Iterator[list[_T]]:
         partitions = partition_sequentially(
-            field_sets,
-            key=lambda fs: fs.address.spec,
+            iterable,
+            key=key,
             size_target=lint_subsystem.batch_size,
             size_max=4 * lint_subsystem.batch_size,
         )
@@ -314,13 +348,19 @@ def batch(field_sets: Iterable[FieldSet]) -> Iterator[list[FieldSet]]:
     def batch_by_type(
         request_types: Iterable[type[_SR]],
     ) -> tuple[tuple[type[_SR], list[FieldSet]], ...]:
+        def key(fs: FieldSet) -> str:
+            return fs.address.spec
+
         return tuple(
             (request_type, field_set_batch)
             for request_type in request_types
             for field_set_batch in batch(
-                request_type.field_set_type.create(target)
-                for target in targets
-                if request_type.field_set_type.is_applicable(target)
+                (
+                    request_type.field_set_type.create(target)
+                    for target in targets
+                    if request_type.field_set_type.is_applicable(target)
+                ),
+                key=key,
             )
         )
 
@@ -328,9 +368,10 @@ def batch_by_type(
         request_type(batch) for request_type, batch in batch_by_type(lint_target_request_types)
     )
 
-    fmt_requests: Iterable[FmtTargetsRequest] = ()
+    fmt_target_requests: Iterable[FmtTargetsRequest] = ()
+    fmt_build_requests: Iterable[_FmtBuildFilesRequest] = ()
     if not lint_subsystem.skip_formatters:
-        batched_fmt_request_pairs = batch_by_type(fmt_target_request_types)
+        batched_fmt_target_request_pairs = batch_by_type(fmt_target_request_types)
         all_fmt_source_batches = await MultiGet(
             Get(
                 SourceFiles,
@@ -342,18 +383,27 @@ def batch_by_type(
                     for field_set in batch
                 ),
             )
-            for _, batch in batched_fmt_request_pairs
+            for _, batch in batched_fmt_target_request_pairs
         )
-        fmt_requests = (
+        fmt_target_requests = (
             request_type(
                 batch,
                 snapshot=source_files_snapshot.snapshot,
             )
             for (request_type, batch), source_files_snapshot in zip(
-                batched_fmt_request_pairs, all_fmt_source_batches
+                batched_fmt_target_request_pairs, all_fmt_source_batches
             )
         )
 
+        build_file_batch_snapshots = await MultiGet(
+            Get(Snapshot, PathGlobs(paths_batch)) for paths_batch in batch(specified_build_files)
+        )
+        fmt_build_requests = (
+            fmt_build_request_type(snapshot)
+            for fmt_build_request_type in fmt_build_request_types
+            for snapshot in build_file_batch_snapshots
+        )
+
     file_requests = (
         tuple(request_type(specs_paths.files) for request_type in file_request_types)
         if specs_paths.files
@@ -362,7 +412,8 @@ def batch_by_type(
 
     all_requests = [
         *(Get(LintResults, LintTargetsRequest, request) for request in lint_target_requests),
-        *(Get(FmtResult, FmtTargetsRequest, request) for request in fmt_requests),
+        *(Get(FmtResult, FmtTargetsRequest, request) for request in fmt_target_requests),
+        *(Get(FmtResult, _FmtBuildFilesRequest, request) for request in fmt_build_requests),
         *(Get(LintResults, LintFilesRequest, request) for request in file_requests),
     ]
     all_batch_results = cast(


Test patch:
diff --git a/src/python/pants/core/goals/lint_test.py b/src/python/pants/core/goals/lint_test.py
--- a/src/python/pants/core/goals/lint_test.py
+++ b/src/python/pants/core/goals/lint_test.py
@@ -12,7 +12,7 @@
 import pytest
 
 from pants.base.specs import Specs
-from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
+from pants.core.goals.fmt import FmtResult, FmtTargetsRequest, _FmtBuildFilesRequest
 from pants.core.goals.lint import (
     AmbiguousRequestNamesError,
     Lint,
@@ -26,7 +26,8 @@
 from pants.core.util_rules.distdir import DistDir
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.addresses import Address
-from pants.engine.fs import SpecsPaths, Workspace
+from pants.engine.fs import PathGlobs, SpecsPaths, Workspace
+from pants.engine.internals.build_files import BuildFileOptions
 from pants.engine.internals.native_engine import EMPTY_DIGEST, EMPTY_SNAPSHOT, Digest, Snapshot
 from pants.engine.target import FieldSet, FilteredTargets, MultipleSourcesField, Target
 from pants.engine.unions import UnionMembership
@@ -149,6 +150,14 @@ def fmt_result(self) -> FmtResult:
         return FmtResult(before, after, "", "", formatter_name=self.name)
 
 
+class BuildFileFormatter(_FmtBuildFilesRequest):
+    name = "BobTheBUILDer"
+
+    @property
+    def fmt_result(self) -> FmtResult:
+        return FmtResult(EMPTY_SNAPSHOT, EMPTY_SNAPSHOT, "", "", formatter_name=self.name)
+
+
 @pytest.fixture
 def rule_runner() -> RuleRunner:
     return RuleRunner()
@@ -165,6 +174,7 @@ def run_lint_rule(
     fmt_request_types: Sequence[Type[FmtTargetsRequest]] = (),
     targets: list[Target],
     run_files_linter: bool = False,
+    run_build_formatter: bool = False,
     batch_size: int = 128,
     only: list[str] | None = None,
     skip_formatters: bool = False,
@@ -173,6 +183,7 @@ def run_lint_rule(
         {
             LintTargetsRequest: lint_request_types,
             LintFilesRequest: [MockFilesRequest] if run_files_linter else [],
+            _FmtBuildFilesRequest: [BuildFileFormatter] if run_build_formatter else [],
             FmtTargetsRequest: fmt_request_types,
         }
     )
@@ -189,6 +200,7 @@ def run_lint_rule(
                 console,
                 Workspace(rule_runner.scheduler, _enforce_effects=False),
                 Specs.empty(),
+                BuildFileOptions(("BUILD",)),
                 lint_subsystem,
                 union_membership,
                 DistDir(relpath=Path("dist")),
@@ -214,6 +226,11 @@ def run_lint_rule(
                     input_type=FmtTargetsRequest,
                     mock=lambda mock_request: mock_request.fmt_result,
                 ),
+                MockGet(
+                    output_type=FmtResult,
+                    input_type=_FmtBuildFilesRequest,
+                    mock=lambda mock_request: mock_request.fmt_result,
+                ),
                 MockGet(
                     output_type=FilteredTargets,
                     input_type=Specs,
@@ -222,7 +239,12 @@ def run_lint_rule(
                 MockGet(
                     output_type=SpecsPaths,
                     input_type=Specs,
-                    mock=lambda _: SpecsPaths(("f.txt",), ()),
+                    mock=lambda _: SpecsPaths(("f.txt", "BUILD"), ()),
+                ),
+                MockGet(
+                    output_type=Snapshot,
+                    input_type=PathGlobs,
+                    mock=lambda _: EMPTY_SNAPSHOT,
                 ),
             ],
             union_membership=union_membership,
@@ -267,11 +289,13 @@ def test_summary(rule_runner: RuleRunner) -> None:
         fmt_request_types=fmt_request_types,
         targets=targets,
         run_files_linter=True,
+        run_build_formatter=True,
     )
     assert exit_code == FailingRequest.exit_code([bad_address])
     assert stderr == dedent(
         """\
 
+        ✓ BobTheBUILDer succeeded.
         ✕ ConditionallySucceedsLinter failed.
         ✕ FailingFormatter failed.
         ✕ FailingLinter failed.
@@ -289,11 +313,18 @@ def test_summary(rule_runner: RuleRunner) -> None:
         fmt_request_types=fmt_request_types,
         targets=targets,
         run_files_linter=True,
-        only=[FailingRequest.name, MockFilesRequest.name, FailingFormatter.name],
+        run_build_formatter=True,
+        only=[
+            FailingRequest.name,
+            MockFilesRequest.name,
+            FailingFormatter.name,
+            BuildFileFormatter.name,
+        ],
     )
     assert stderr == dedent(
         """\
 
+        ✓ BobTheBUILDer succeeded.
         ✕ FailingFormatter failed.
         ✕ FailingLinter failed.
         ✓ FilesLinter succeeded.
@@ -308,6 +339,7 @@ def test_summary(rule_runner: RuleRunner) -> None:
         fmt_request_types=fmt_request_types,
         targets=targets,
         run_files_linter=True,
+        run_build_formatter=True,
         skip_formatters=True,
     )
     assert stderr == dedent(
@@ -406,9 +438,30 @@ class AmbiguousLintTargetsRequest(LintTargetsRequest):
     with pytest.raises(AmbiguousRequestNamesError):
         run_lint_rule(
             rule_runner,
-            lint_request_types=[
-                AmbiguousLintTargetsRequest,
-            ],
+            lint_request_types=[AmbiguousLintTargetsRequest],
             run_files_linter=True,  # needed for MockFilesRequest
             targets=[],
         )
+
+    class BuildAndLintTargetType(LintTargetsRequest):
+        name = BuildFileFormatter.name
+
+    with pytest.raises(AmbiguousRequestNamesError):
+        run_lint_rule(
+            rule_runner,
+            lint_request_types=[BuildAndLintTargetType],
+            targets=[],
+            run_build_formatter=True,
+        )
+
+    class BuildAndFmtTargetType(FmtTargetsRequest):
+        name = BuildFileFormatter.name
+
+    # Ambiguity between a target formatter and BUILD formatter are OK
+    run_lint_rule(
+        rule_runner,
+        lint_request_types=[],
+        fmt_request_types=[BuildAndFmtTargetType],
+        run_build_formatter=True,
+        targets=[],
+    )