Repository: pantsbuild/pants

Issue:
Synthetic targets should not live outside of existing address namespaces
In order to avoid confusion, synthetic targets must exist within known address namespaces so they are included in `AllTargets`.



Discussion:

Resolving patch:
diff --git a/src/python/pants/base/specs.py b/src/python/pants/base/specs.py
--- a/src/python/pants/base/specs.py
+++ b/src/python/pants/base/specs.py
@@ -8,6 +8,8 @@
 from dataclasses import dataclass
 from typing import ClassVar, Iterable, Iterator, cast
 
+from typing_extensions import Protocol
+
 from pants.base.deprecated import warn_or_error
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.engine.fs import GlobExpansionConjunction, PathGlobs
@@ -23,6 +25,11 @@ def __str__(self) -> str:
         """The normalized string representation of this spec."""
 
 
+class GlobSpecsProtocol(Protocol):
+    def matches_target_residence_dir(self, residence_dir: str) -> bool:
+        pass
+
+
 @dataclass(frozen=True)
 class AddressLiteralSpec(Spec):
     """A single target address.
diff --git a/src/python/pants/engine/internals/specs_rules.py b/src/python/pants/engine/internals/specs_rules.py
--- a/src/python/pants/engine/internals/specs_rules.py
+++ b/src/python/pants/engine/internals/specs_rules.py
@@ -33,6 +33,10 @@
     _TargetParametrizationsRequest,
 )
 from pants.engine.internals.selectors import Get, MultiGet
+from pants.engine.internals.synthetic_targets import (
+    SyntheticTargetsSpecPaths,
+    SyntheticTargetsSpecPathsRequest,
+)
 from pants.engine.rules import collect_rules, rule, rule_helper
 from pants.engine.target import (
     FieldSet,
@@ -129,6 +133,7 @@ async def addresses_from_raw_specs_without_file_owners(
     build_file_options: BuildFileOptions,
     specs_filter: SpecsFilter,
     local_environment_name: ChosenLocalEnvironmentName,
+    union_membership: UnionMembership,
 ) -> Addresses:
     matched_addresses: OrderedSet[Address] = OrderedSet()
     filtering_disabled = specs.filter_by_global_options is False
@@ -155,8 +160,12 @@ async def addresses_from_raw_specs_without_file_owners(
         Get(Paths, PathGlobs, build_file_globs),
         Get(Paths, PathGlobs, validation_globs),
     )
-
-    dirnames = {os.path.dirname(f) for f in build_file_paths.files}
+    dirnames = set(
+        await Get(
+            SyntheticTargetsSpecPaths, SyntheticTargetsSpecPathsRequest(tuple(specs.glob_specs()))
+        )
+    )
+    dirnames.update(os.path.dirname(f) for f in build_file_paths.files)
     address_families = await MultiGet(Get(AddressFamily, AddressFamilyDir(d)) for d in dirnames)
     base_addresses = Addresses(
         itertools.chain.from_iterable(
diff --git a/src/python/pants/engine/internals/synthetic_targets.py b/src/python/pants/engine/internals/synthetic_targets.py
--- a/src/python/pants/engine/internals/synthetic_targets.py
+++ b/src/python/pants/engine/internals/synthetic_targets.py
@@ -12,18 +12,39 @@
     from pants.engine.internals.synthetic_targets import (
         SyntheticAddressMaps,
         SyntheticTargetsRequest,
+        SyntheticTargetsSpecPaths,
     )
     from pants.engine.internals.target_adaptor import TargetAdaptor
-    from pants.engine.unions import UnionRule
     from pants.engine.rules import collect_rules, rule
 
 
     @dataclass(frozen=True)
     class SyntheticExampleTargetsRequest(SyntheticTargetsRequest):
-        path: str = SyntheticTargetsRequest.REQUEST_TARGETS_PER_DIRECTORY
         path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS
 
 
+    class SyntheticExampleTargetsPerDirectorySpecPathsRequest:
+        pass
+
+
+    @dataclass(frozen=True)
+    class SyntheticExampleTargetsPerDirectoryRequest(SyntheticTargetsRequest):
+        path: str = SyntheticTargetsRequest.REQUEST_TARGETS_PER_DIRECTORY
+
+        # Optional: (without it, only paths with BUILD files will be consistently considered)
+        spec_paths_request = SyntheticExampleTargetsPerDirectorySpecPathsRequest
+
+
+    @rule
+    def example_synthetic_targets_per_directory_spec_paths(
+        request: SyntheticExampleTargetsPerDirectorySpecPathsRequest,
+    ) -> SyntheticTargetsSpecPaths:
+        # Return all paths we have targets for.
+        # This may involve using GlobPaths etc to discover files in the project source tree.
+        known_paths = ["src/a/dir1", "src/a/dir2", ...]
+        return SyntheticTargetsSpecPaths.from_paths(known_paths)
+
+
     @rule
     async def example_synthetic_targets(request: SyntheticExampleTargetsRequest) -> SyntheticAddressMaps:
         # Return targets for `request.path`.
@@ -45,7 +66,8 @@ async def example_synthetic_targets(request: SyntheticExampleTargetsRequest) ->
     def rules():
         return (
             *collect_rules(),
-            UnionRule(SyntheticTargetsRequest, SyntheticExampleTargetsRequest),
+            SyntheticExampleTargetsRequest.rules(),
+            SyntheticExampleTargetsPerDirectoryRequest.rules(),
             ...
         )
 """
@@ -54,19 +76,42 @@ def rules():
 import itertools
 import os.path
 from dataclasses import dataclass
-from typing import ClassVar, Iterable, Sequence
+from typing import ClassVar, Iterable, Iterator, Sequence
 
+from pants.base.specs import GlobSpecsProtocol
 from pants.engine.collection import Collection
 from pants.engine.internals.defaults import BuildFileDefaults
 from pants.engine.internals.mapper import AddressMap
 from pants.engine.internals.target_adaptor import TargetAdaptor
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import InvalidTargetException
-from pants.engine.unions import UnionMembership, union
+from pants.engine.unions import UnionMembership, UnionRule, union
 from pants.util.frozendict import FrozenDict
 from pants.util.strutil import softwrap
 
 
+@dataclass(frozen=True)
+class SyntheticTargetsSpecPathsRequest:
+    specs: tuple[GlobSpecsProtocol, ...]
+
+
+class SyntheticTargetsSpecPaths(Collection[str]):
+    @staticmethod
+    def from_paths(paths: Iterable[str]) -> SyntheticTargetsSpecPaths:
+        return SyntheticTargetsSpecPaths(sorted(set(paths)))
+
+    @staticmethod
+    def from_request(
+        request: SyntheticTargetsSpecPathsRequest, paths: Iterable[str]
+    ) -> SyntheticTargetsSpecPaths:
+        return SyntheticTargetsSpecPaths.from_paths(
+            filter(
+                lambda path: any(spec.matches_target_residence_dir(path) for spec in request.specs),
+                paths,
+            )
+        )
+
+
 @union
 @dataclass(frozen=True)
 class SyntheticTargetsRequest:
@@ -89,6 +134,22 @@ class SyntheticTargetsRequest:
     which case _all_ synthetic targets should be returned.
     """
 
+    spec_paths_request: ClassVar[type | None] = None
+    """Request class for providing paths in addition to those where BUILD files are found.
+
+    Implement a rule that takes `spec_paths_request` and returns an `SyntheticTargetsSpecPaths`.
+    """
+
+    @union
+    class _SpecPathsRequest:
+        """Protected union type."""
+
+    @classmethod
+    def rules(cls) -> Iterator[UnionRule]:
+        yield UnionRule(SyntheticTargetsRequest, cls)
+        if cls.spec_paths_request is not None:
+            yield UnionRule(SyntheticTargetsRequest._SpecPathsRequest, cls.spec_paths_request)
+
 
 class SyntheticAddressMap(AddressMap):
     def process_declared_targets(self, address_map: AddressMap) -> None:
@@ -171,12 +232,14 @@ class AllSyntheticAddressMaps:
 
     address_maps: FrozenDict[str, SyntheticAddressMaps]
     path_request_types: FrozenDict[str, Sequence[type[SyntheticTargetsRequest]]]
+    spec_paths: Sequence[str]
 
     @classmethod
     def create(
         cls,
         address_maps: Iterable[SyntheticAddressMap],
         requests: Iterable[SyntheticTargetsRequest],
+        spec_paths: Iterable[str],
     ) -> AllSyntheticAddressMaps:
         def address_map_key(address_map: SyntheticAddressMap) -> str:
             return os.path.dirname(address_map.path)
@@ -202,6 +265,7 @@ def requests_key(request: SyntheticAddressMapsRequest) -> str:
                     if path != SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS
                 }
             ),
+            spec_paths=tuple(sorted(spec_paths)),
         )
 
     def targets_request_types(self, path: str) -> Iterable[type[SyntheticTargetsRequest]]:
@@ -236,9 +300,28 @@ async def all_synthetic_targets(union_membership: UnionMembership) -> AllSynthet
         for request in requests
         if request.path == SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS
     )
+    all_spec_paths = await MultiGet(
+        Get(
+            SyntheticTargetsSpecPaths,
+            SyntheticTargetsRequest._SpecPathsRequest,
+            spec_paths_request(),
+        )
+        for spec_paths_request in union_membership.get(SyntheticTargetsRequest._SpecPathsRequest)
+    )
     return AllSyntheticAddressMaps.create(
         address_maps=itertools.chain.from_iterable(all_synthetic),
         requests=requests,
+        spec_paths=set(itertools.chain.from_iterable(all_spec_paths)),
+    )
+
+
+@rule
+def get_synthetic_targets_spec_paths(
+    request: SyntheticTargetsSpecPathsRequest, all_synthetic: AllSyntheticAddressMaps
+) -> SyntheticTargetsSpecPaths:
+    """Return all known paths for synthetic targets."""
+    return SyntheticTargetsSpecPaths.from_request(
+        request, itertools.chain(all_synthetic.address_maps, all_synthetic.spec_paths)
     )
 
 


Test patch:
diff --git a/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem_test.py b/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem_test.py
--- a/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem_test.py
+++ b/src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem_test.py
@@ -38,7 +38,12 @@ def test_find_protobuf_python_requirement() -> None:
         {"codegen/dir/f.proto": "", "codegen/dir/BUILD": "protobuf_sources(grpc=True)"}
     )
     rule_runner.set_options(
-        ["--python-resolves={'python-default': '', 'another': ''}", "--python-enable-resolves"]
+        [
+            "--python-resolves={'python-default': '', 'another': ''}",
+            "--python-enable-resolves",
+            # Turn off python synthetic lockfile targets to make the test simpler.
+            "--no-python-enable-lockfile-targets",
+        ]
     )
     proto_tgt = rule_runner.get_target(Address("codegen/dir", relative_file_path="f.proto"))
     request = InferPythonProtobufDependencies(
diff --git a/src/python/pants/backend/python/goals/export_test.py b/src/python/pants/backend/python/goals/export_test.py
--- a/src/python/pants/backend/python/goals/export_test.py
+++ b/src/python/pants/backend/python/goals/export_test.py
@@ -68,6 +68,8 @@ def test_export_venv_old_codepath(
             f"--python-enable-resolves={enable_resolves}",
             # Turn off lockfile validation to make the test simpler.
             "--python-invalid-lockfile-behavior=ignore",
+            # Turn off python synthetic lockfile targets to make the test simpler.
+            "--no-python-enable-lockfile-targets",
             symlink_flag,
         ],
         env_inherit={"PATH", "PYENV_ROOT"},
diff --git a/src/python/pants/engine/internals/synthetic_targets_test.py b/src/python/pants/engine/internals/synthetic_targets_test.py
--- a/src/python/pants/engine/internals/synthetic_targets_test.py
+++ b/src/python/pants/engine/internals/synthetic_targets_test.py
@@ -12,12 +12,15 @@
 
 from pants.backend.python.macros import python_requirements
 from pants.backend.python.macros.python_requirements import PythonRequirementsTargetGenerator
+from pants.base.specs import RawSpecsWithoutFileOwners, RecursiveGlobSpec
 from pants.core.target_types import FileTarget, GenericTarget, LockfilesGeneratorTarget
 from pants.engine.addresses import Address, Addresses
 from pants.engine.environment import EnvironmentName
+from pants.engine.internals.specs_rules_test import resolve_raw_specs_without_file_owners
 from pants.engine.internals.synthetic_targets import (
     SyntheticAddressMaps,
     SyntheticTargetsRequest,
+    SyntheticTargetsSpecPaths,
     rules,
 )
 from pants.engine.internals.target_adaptor import TargetAdaptor
@@ -32,7 +35,6 @@
     WrappedTarget,
     WrappedTargetRequest,
 )
-from pants.engine.unions import UnionRule
 from pants.testutil.rule_runner import RuleRunner, engine_error
 from pants.util.strutil import softwrap
 
@@ -42,11 +44,6 @@ class SyntheticExampleTargetsRequest(SyntheticTargetsRequest):
     path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS
 
 
-@dataclass(frozen=True)
-class SyntheticExampleTargetsPerDirectoryRequest(SyntheticTargetsRequest):
-    path: str = SyntheticTargetsRequest.REQUEST_TARGETS_PER_DIRECTORY
-
-
 @rule
 async def example_synthetic_targets(
     request: SyntheticExampleTargetsRequest,
@@ -69,57 +66,86 @@ async def example_synthetic_targets(
                 ),
             ),
         ),
+        (
+            "src/synthetic/BUILD.synthetic",
+            (TargetAdaptor("target", "generic-synth", description="Additional target"),),
+        ),
     ]
     return SyntheticAddressMaps.for_targets_request(request, targets)
 
 
-example_synthetic_targets_per_directory_counts: DefaultDict[str, int] = defaultdict(int)
+class SyntheticExampleTargetsPerDirectorySpecPathsRequest:
+    pass
 
 
-@rule
-async def example_synthetic_targets_per_directory(
-    request: SyntheticExampleTargetsPerDirectoryRequest,
-) -> SyntheticAddressMaps:
-    assert request.path != SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS
-    example_synthetic_targets_per_directory_counts[request.path] += 1
-    targets = {
-        "src/test": [
+@dataclass(frozen=True)
+class SyntheticExampleTargetsPerDirectoryRequest(SyntheticTargetsRequest):
+    path: str = SyntheticTargetsRequest.REQUEST_TARGETS_PER_DIRECTORY
+    spec_paths_request = SyntheticExampleTargetsPerDirectorySpecPathsRequest
+
+
+example_synthetic_targets_per_directory_counts: DefaultDict[str, int] = defaultdict(int)
+example_synthetic_targets_per_directory_targets = {
+    "src/test": [
+        (
+            "BUILD.dir-a",
             (
-                "BUILD.dir-a",
-                (
-                    TargetAdaptor(
-                        "target",
-                        "generic3",
-                        description="Target 3",
-                    ),
+                TargetAdaptor(
+                    "target",
+                    "generic3",
+                    description="Target 3",
                 ),
             ),
+        ),
+        (
+            "BUILD.dir-b",
             (
-                "BUILD.dir-b",
-                (
-                    TargetAdaptor(
-                        "target",
-                        "generic4",
-                        description="Target 4",
-                        tags=["synthetic", "tags"],
-                    ),
+                TargetAdaptor(
+                    "target",
+                    "generic4",
+                    description="Target 4",
+                    tags=["synthetic", "tags"],
                 ),
             ),
-        ],
-        "src/issues/17343": [
+        ),
+    ],
+    "src/issues/17343": [
+        (
+            "BUILD.issue",
             (
-                "BUILD.issue",
-                (
-                    TargetAdaptor(
-                        "_lockfiles",
-                        "python-default",
-                        sources=["lockfile"],
-                    ),
+                TargetAdaptor(
+                    "_lockfiles",
+                    "python-default",
+                    sources=["lockfile"],
                 ),
             ),
-        ],
-    }
-    return SyntheticAddressMaps.for_targets_request(request, targets.get(request.path, ()))
+        ),
+    ],
+    "src/bare/tree": [
+        (
+            "BUILD.synthetic-targets",
+            (TargetAdaptor("target", "bare-tree"),),
+        ),
+    ],
+}
+
+
+@rule
+def example_synthetic_targets_per_directory_spec_paths(
+    request: SyntheticExampleTargetsPerDirectorySpecPathsRequest,
+) -> SyntheticTargetsSpecPaths:
+    # Return all paths we have targets for.
+    return SyntheticTargetsSpecPaths.from_paths(example_synthetic_targets_per_directory_targets)
+
+
+@rule
+async def example_synthetic_targets_per_directory(
+    request: SyntheticExampleTargetsPerDirectoryRequest,
+) -> SyntheticAddressMaps:
+    assert request.path != SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS
+    example_synthetic_targets_per_directory_counts[request.path] += 1
+    targets = example_synthetic_targets_per_directory_targets.get(request.path, ())
+    return SyntheticAddressMaps.for_targets_request(request, targets)
 
 
 @pytest.fixture
@@ -127,11 +153,13 @@ def rule_runner() -> RuleRunner:
     rule_runner = RuleRunner(
         rules=[
             *rules(),
+            *python_requirements.rules(),
+            *SyntheticExampleTargetsRequest.rules(),
+            *SyntheticExampleTargetsPerDirectoryRequest.rules(),
             example_synthetic_targets,
             example_synthetic_targets_per_directory,
-            *python_requirements.rules(),
-            UnionRule(SyntheticTargetsRequest, SyntheticExampleTargetsRequest),
-            UnionRule(SyntheticTargetsRequest, SyntheticExampleTargetsPerDirectoryRequest),
+            example_synthetic_targets_per_directory_spec_paths,
+            QueryRule(Addresses, [RawSpecsWithoutFileOwners]),
             QueryRule(Addresses, (DependenciesRequest, EnvironmentName)),
         ],
         target_types=[
@@ -253,6 +281,19 @@ def test_extend_missing_synthetic_target(rule_runner: RuleRunner) -> None:
         )
 
 
+def test_additional_spec_path(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            # We need this to avoid:
+            # Exception: Unmatched glob from tests: "src/**"
+            "src/BUILD": "",
+        }
+    )
+    addresses = resolve_raw_specs_without_file_owners(rule_runner, [RecursiveGlobSpec("src")])
+    assert Address("src/synthetic", target_name="generic-synth") in addresses
+    assert Address("src/bare/tree", target_name="bare-tree") in addresses
+
+
 def test_target_name_collision_issue_17343(rule_runner: RuleRunner) -> None:
     # The issue was that the synthesized _lockfiles target is replaced by the python_requirements
     # target with the same name so the injected dependency pointed to itself rather than the