Repository: pantsbuild/pants

Issue:
Parametrized `python_tests` and `python_sources` in same package not lining up when using explicit `dependencies`
**Describe the bug**
I have a `python_source` target in the same directory as a `python_test` target. Both are parametrized for multiple resolves/environments/interpreter constraints. The `python_test` target in question does not import the `python_source` target but instead calls it via the `subprocess` module, so I have the `python_source` target as an explicit dependency of the `python_test` one. This is causing an error because the `python_test` target is grabbing the first parametrization as the dependency instead of the matching one. `pants peek` output below:
```json
{
    "address": "my/python/tests/test_the_thing.py@parametrize=py311",
    "target_type": "python_test",
    "batch_compatibility_tag": null,
    "dependencies": [
      "my/inferred_dep/__init__.py@parametrize=py311",
      "my/python/tests/__init__.py:sources_for_tests@parametrize=py38",
      "my/python/tests/source_for_test.py:sources_for_tests@parametrize=py38"
    ],
    "dependencies_raw": [
      ":sources_for_tests"
    ],
    "description": null,
    "environment": "py311-test",
    "extra_env_vars": null,
    "interpreter_constraints": [
      "==3.11.*"
    ],
    "resolve": "py311",
    "run_goal_use_sandbox": null,
    "runtime_package_dependencies": null,
    "skip_black": false,
    "skip_docformatter": false,
    "skip_isort": false,
    "skip_mypy": true,
    "skip_ruff": true,
    "skip_tests": false,
    "source_raw": "test_the_thing.py",
    "sources": [
      "my/python/tests/test_the_thing.py"
    ],
    "sources_fingerprint": "b1274e36d37387cc0fca9f18cd77e456483663e6ac869ec1ecce9aaa502a8d83",
    "tags": null,
    "timeout": null,
    "xdist_concurrency": null
 }
```

**Pants version**
2.19.1

**OS**
Intel Mac



Discussion:
Sorry for the trouble; can you create a reduced example as a demonstration?
I understand the issue here.. will see if I can come up with a fix for it.
@ndellosa95  Can you share the relevant parts of your BUILD files for this (redacted as needed), just to make sure I test the right thing? Thanks :)
Or, if you can confirm that this test matches what you have..
```python
    assert_generated(
        generated_targets_rule_runner,
        Address("demo", target_name="tst"),
        dedent(
            """\
            generator(
              name='src',
              sources=['src1.ext'],
              **parametrize('b1', resolve='a'),
              **parametrize('b2', resolve='b'),
            )
            generator(
              name='tst',
              sources=['tst1.ext'],
              dependencies=['./src1.ext:src'],
              **parametrize('b1', resolve='a'),
              **parametrize('b2', resolve='b'),
            )
            """
        ),
        ["src1.ext", "tst1.ext"],
        expected_dependencies={
            "demo/src1.ext:src@parametrize=b1": set(),
            "demo/src1.ext:src@parametrize=b2": set(),
            "demo/tst1.ext:tst@parametrize=b1": {
                "demo/src1.ext:src@parametrize=b1",
            },
            "demo/tst1.ext:tst@parametrize=b2": {
                "demo/src1.ext:src@parametrize=b2",
            },
            "demo:src@parametrize=b1": {
                "demo/src1.ext:src@parametrize=b1",
            },
            "demo:src@parametrize=b2": {
                "demo/src1.ext:src@parametrize=b2",
            },
            "demo:tst@parametrize=b1": {
                "demo/tst1.ext:tsr@parametrize=b1",
            },
            "demo:tst@parametrize=b2": {
                "demo/tst1.ext:tst@parametrize=b2",
            },
        },
    )
```

which does fail with a missmatched parametrization for the explicitly provided dependency:
```
E               'demo/tst1.ext:tst@parametriz    'demo/tst1.ext:tst@parametriz
E             e=b2': set([                     e=b2': set([
E                 'demo/src1.ext:src@parametr      'demo/src1.ext:src@parametr
E             ize=b1',                         ize=b2',
E               ]),                              ]),
```

> Or, if you can confirm that this test matches what you have..
> 
> ```python
>     assert_generated(
>         generated_targets_rule_runner,
>         Address("demo", target_name="tst"),
>         dedent(
>             """\
>             generator(
>               name='src',
>               sources=['src1.ext'],
>               **parametrize('b1', resolve='a'),
>               **parametrize('b2', resolve='b'),
>             )
>             generator(
>               name='tst',
>               sources=['tst1.ext'],
>               dependencies=['./src1.ext:src'],
>               **parametrize('b1', resolve='a'),
>               **parametrize('b2', resolve='b'),
>             )
>             """
>         ),
>         ["src1.ext", "tst1.ext"],
>         expected_dependencies={
>             "demo/src1.ext:src@parametrize=b1": set(),
>             "demo/src1.ext:src@parametrize=b2": set(),
>             "demo/tst1.ext:tst@parametrize=b1": {
>                 "demo/src1.ext:src@parametrize=b1",
>             },
>             "demo/tst1.ext:tst@parametrize=b2": {
>                 "demo/src1.ext:src@parametrize=b2",
>             },
>             "demo:src@parametrize=b1": {
>                 "demo/src1.ext:src@parametrize=b1",
>             },
>             "demo:src@parametrize=b2": {
>                 "demo/src1.ext:src@parametrize=b2",
>             },
>             "demo:tst@parametrize=b1": {
>                 "demo/tst1.ext:tsr@parametrize=b1",
>             },
>             "demo:tst@parametrize=b2": {
>                 "demo/tst1.ext:tst@parametrize=b2",
>             },
>         },
>     )
> ```
> 
> which does fail with a missmatched parametrization for the explicitly provided dependency:
> 
> ```
> E               'demo/tst1.ext:tst@parametriz    'demo/tst1.ext:tst@parametriz
> E             e=b2': set([                     e=b2': set([
> E                 'demo/src1.ext:src@parametr      'demo/src1.ext:src@parametr
> E             ize=b1',                         ize=b2',
> E               ]),                              ]),
> ```

Yes, it always selects the first target in the parametrization.

Resolving patch:
diff --git a/src/python/pants/engine/internals/parametrize.py b/src/python/pants/engine/internals/parametrize.py
--- a/src/python/pants/engine/internals/parametrize.py
+++ b/src/python/pants/engine/internals/parametrize.py
@@ -427,12 +427,20 @@ def remaining_fields_match(candidate: Target) -> bool:
             ):
                 return parametrization.original_target
 
-            # Else, see whether any of the generated targets match.
-            for candidate in parametrization.parametrization.values():
-                if address.is_parametrized_subset_of(candidate.address) and remaining_fields_match(
-                    candidate
-                ):
-                    return candidate
+        consumer_parametrize_group = consumer.address.parameters.get("parametrize")
+
+        def matching_parametrize_group(candidate: Target) -> bool:
+            return candidate.address.parameters.get("parametrize") == consumer_parametrize_group
+
+        for candidate in sorted(
+            self.parametrizations.values(), key=matching_parametrize_group, reverse=True
+        ):
+            # Else, see whether any of the generated targets match, preferring a matching
+            # parametrize group when available.
+            if address.is_parametrized_subset_of(candidate.address) and remaining_fields_match(
+                candidate
+            ):
+                return candidate
 
         raise ValueError(
             f"The explicit dependency `{address}` of the target at `{consumer.address}` does "


Test patch:
diff --git a/src/python/pants/engine/internals/graph_test.py b/src/python/pants/engine/internals/graph_test.py
--- a/src/python/pants/engine/internals/graph_test.py
+++ b/src/python/pants/engine/internals/graph_test.py
@@ -1627,6 +1627,55 @@ def test_parametrize_group_on_target_generator_20418(
     )
 
 
+def test_parametrize_explicit_dependencies_20739(
+    generated_targets_rule_runner: RuleRunner,
+) -> None:
+    assert_generated(
+        generated_targets_rule_runner,
+        Address("demo", target_name="tst"),
+        dedent(
+            """\
+            generator(
+              name='src',
+              sources=['src1.ext'],
+              **parametrize('b1', resolve='a'),
+              **parametrize('b2', resolve='b'),
+            )
+            generator(
+              name='tst',
+              sources=['tst1.ext'],
+              dependencies=['./src1.ext:src'],
+              **parametrize('b1', resolve='a'),
+              **parametrize('b2', resolve='b'),
+            )
+            """
+        ),
+        ["src1.ext", "tst1.ext"],
+        expected_dependencies={
+            "demo/src1.ext:src@parametrize=b1": set(),
+            "demo/src1.ext:src@parametrize=b2": set(),
+            "demo/tst1.ext:tst@parametrize=b1": {
+                "demo/src1.ext:src@parametrize=b1",
+            },
+            "demo/tst1.ext:tst@parametrize=b2": {
+                "demo/src1.ext:src@parametrize=b2",
+            },
+            "demo:src@parametrize=b1": {
+                "demo/src1.ext:src@parametrize=b1",
+            },
+            "demo:src@parametrize=b2": {
+                "demo/src1.ext:src@parametrize=b2",
+            },
+            "demo:tst@parametrize=b1": {
+                "demo/tst1.ext:tst@parametrize=b1",
+            },
+            "demo:tst@parametrize=b2": {
+                "demo/tst1.ext:tst@parametrize=b2",
+            },
+        },
+    )
+
+
 # -----------------------------------------------------------------------------------------------
 # Test `SourcesField`. Also see `engine/target_test.py`.
 # -----------------------------------------------------------------------------------------------