Repository: pantsbuild/pants

Issue:
Multi-parametrize doesn't work with defaults or target generators
**Describe the bug**
The new group-parametrization logic added by [this PR](https://github.com/pantsbuild/pants/pull/20065) only works for individual targets and not target generators or `__defaults__`.

**Pants version**
2.19.0rc4

**OS**
Mac

**Additional info**
Below is an example of the error message I see when I try and parametrize generators/defaults.

```
Unrecognized field `parametrize_-8470896455302443883:580806178752=parametrize('py38', interpreter_constraints=('==3.8.*',), resolve='py38')` in target <my target>. Valid fields for the target type `python_sources`: ['dependencies', 'description', 'interpreter_constraints', 'overrides', 'resolve', 'restartable', 'run_goal_use_sandbox', 'skip_black', 'skip_docformatter', 'skip_isort', 'skip_mypy', 'skip_ruff', 'sources', 'tags'].
```

Multi-parametrize doesn't work with defaults or target generators
**Describe the bug**
The new group-parametrization logic added by [this PR](https://github.com/pantsbuild/pants/pull/20065) only works for individual targets and not target generators or `__defaults__`.

**Pants version**
2.19.0rc4

**OS**
Mac

**Additional info**
Below is an example of the error message I see when I try and parametrize generators/defaults.

```
Unrecognized field `parametrize_-8470896455302443883:580806178752=parametrize('py38', interpreter_constraints=('==3.8.*',), resolve='py38')` in target <my target>. Valid fields for the target type `python_sources`: ['dependencies', 'description', 'interpreter_constraints', 'overrides', 'resolve', 'restartable', 'run_goal_use_sandbox', 'skip_black', 'skip_docformatter', 'skip_isort', 'skip_mypy', 'skip_ruff', 'sources', 'tags'].
```



Discussion:
To confirm, the issue with `__defaults__` is also only when applied for target generators, right?
> To confirm, the issue with `__defaults__` is also only when applied for target generators, right?

Just confirmed that is indeed the case.
To confirm, the issue with `__defaults__` is also only when applied for target generators, right?
> To confirm, the issue with `__defaults__` is also only when applied for target generators, right?

Just confirmed that is indeed the case.

Resolving patch:
diff --git a/src/python/pants/engine/internals/graph.py b/src/python/pants/engine/internals/graph.py
--- a/src/python/pants/engine/internals/graph.py
+++ b/src/python/pants/engine/internals/graph.py
@@ -429,6 +429,16 @@ def _parametrized_target_generators_with_templates(
         field_value = generator_fields.pop(field_type.alias, None)
         if field_value is not None:
             template_fields[field_type.alias] = field_value
+
+    # Move parametrize groups over to `template_fields` in order to expand them.
+    parametrize_group_field_names = [
+        name
+        for name, field in generator_fields.items()
+        if isinstance(field, Parametrize) and field.is_group
+    ]
+    for field_name in parametrize_group_field_names:
+        template_fields[field_name] = generator_fields.pop(field_name)
+
     field_type_aliases = target_type._get_field_aliases_to_field_types(
         target_type.class_field_types(union_membership)
     ).keys()


Test patch:
diff --git a/src/python/pants/engine/internals/defaults_test.py b/src/python/pants/engine/internals/defaults_test.py
--- a/src/python/pants/engine/internals/defaults_test.py
+++ b/src/python/pants/engine/internals/defaults_test.py
@@ -306,6 +306,31 @@ def _determenistic_parametrize_group_keys(value: Mapping[str, Any]) -> dict[str,
             ),
             id="overrides value not frozen (issue #18784)",
         ),
+        pytest.param(
+            Scenario(
+                args=(
+                    {
+                        TestGenTargetGenerator.alias: {
+                            "tags": Parametrize(["foo"], ["bar"], baz=["baz"]),
+                            **Parametrize(
+                                "splat", description="splat-desc", dependencies=["splat:dep"]
+                            ),
+                        }
+                    },
+                ),
+                expected_defaults={
+                    "test_gen_targets": _determenistic_parametrize_group_keys(
+                        {
+                            "tags": ParametrizeDefault(("foo",), ("bar",), baz=("baz",)),  # type: ignore[arg-type]
+                            **ParametrizeDefault(
+                                "splat", description="splat-desc", dependencies=["splat:dep"]
+                            ),
+                        }
+                    )
+                },
+            ),
+            id="parametrizations on target generator (issue #20418)",
+        ),
     ],
 )
 def test_set_defaults(
diff --git a/src/python/pants/engine/internals/graph_test.py b/src/python/pants/engine/internals/graph_test.py
--- a/src/python/pants/engine/internals/graph_test.py
+++ b/src/python/pants/engine/internals/graph_test.py
@@ -1506,6 +1506,9 @@ def test_parametrize_partial_exclude(generated_targets_rule_runner: RuleRunner)
 
 
 def test_parametrize_16190(generated_targets_rule_runner: RuleRunner) -> None:
+    """Field subclassing defeats automatic filling of parameters for explicit dependencies on
+    parametrized targets."""
+
     class ParentField(Field):
         alias = "parent"
         help = "foo"
@@ -1558,6 +1561,7 @@ class ChildTarget(Target):
     ],
 )
 def test_parametrize_16910(generated_targets_rule_runner: RuleRunner, field_content: str) -> None:
+    """Misleading errror message for parametrized field that is unknown."""
     with engine_error(
         InvalidTargetException, contains=f"demo/BUILD:1: Unrecognized field `{field_content}`"
     ):
@@ -1589,6 +1593,40 @@ def test_parametrize_single_value_16978(generated_targets_rule_runner: RuleRunne
     )
 
 
+def test_parametrize_group_on_target_generator_20418(
+    generated_targets_rule_runner: RuleRunner,
+) -> None:
+    assert_generated(
+        generated_targets_rule_runner,
+        Address("demo", target_name="t"),
+        dedent(
+            """\
+            generator(
+              name='t',
+              sources=['f1.ext', 'f2.ext'],
+              **parametrize('a1', resolve='a', tags=['1']),
+              **parametrize('b2', resolve='b', tags=['2']),
+            )
+            """
+        ),
+        ["f1.ext", "f2.ext"],
+        expected_dependencies={
+            "demo/f1.ext:t@parametrize=a1": set(),
+            "demo/f1.ext:t@parametrize=b2": set(),
+            "demo/f2.ext:t@parametrize=a1": set(),
+            "demo/f2.ext:t@parametrize=b2": set(),
+            "demo:t@parametrize=a1": {
+                "demo/f1.ext:t@parametrize=a1",
+                "demo/f2.ext:t@parametrize=a1",
+            },
+            "demo:t@parametrize=b2": {
+                "demo/f1.ext:t@parametrize=b2",
+                "demo/f2.ext:t@parametrize=b2",
+            },
+        },
+    )
+
+
 # -----------------------------------------------------------------------------------------------
 # Test `SourcesField`. Also see `engine/target_test.py`.
 # -----------------------------------------------------------------------------------------------