Repository: pantsbuild/pants

Issue:
__defaults__ does not support nested dicts
**Describe the bug**
Using `__defaults__` to [extend](https://www.pantsbuild.org/docs/targets#field-default-values) an `overrides` field is not supported; using
```
__defaults__(
    {
        (python_sources, ): dict(overrides={
            "*_generated.py": {
                "skip_yapf": True
            },
        })
    },
    extend=True,
    ignore_unknown_fields=True,
)
```
yields
```
TypeError: Even though you are using a `FrozenDict`, the underlying values are not hashable. Please use hashable (and preferably immutable) types for the underlying values, e.g. use tuples instead of lists and use FrozenOrderedSet instead of set().

Original error message: unhashable type: 'dict'

Value: FrozenDict({'overrides': {('*_generated.py',): {'skip_yapf': True}}})
```
while fudging the type of the `overrides` yields
```
InvalidFieldTypeException: The 'overrides' field in target my/path#__defaults__ must be dict[str | tuple[str, ...], dict[str, Any]], but was `(('*_generated.py',), (('skip_yapf', True),))` with type `tuple`.
```

**Pants version**
2.15.0

**OS**
Linux

**Additional info**
Expected behavior is that one can provide default override entry values.



Discussion:

Resolving patch:
diff --git a/src/python/pants/engine/internals/defaults.py b/src/python/pants/engine/internals/defaults.py
--- a/src/python/pants/engine/internals/defaults.py
+++ b/src/python/pants/engine/internals/defaults.py
@@ -37,22 +37,13 @@ class BuildFileDefaults(FrozenDict[str, FrozenDict[str, ImmutableValue]]):
     """Map target types to default field values."""
 
 
-@dataclass(frozen=True)
 class ParametrizeDefault(Parametrize):
-    """A frozen version of `Parametrize` for defaults.
+    """Parametrize for default field values.
 
-    This is needed since all defaults must be hashable, which the `Parametrize` class is not nor can
-    it be as it may get unhashable data as input and is unaware of the field type it is being
-    applied to.
+    This is to have eager validation on the field values rather than erroring first when applied on
+    an actual target.
     """
 
-    args: tuple[str, ...]
-    kwargs: FrozenDict[str, ImmutableValue]  # type: ignore[assignment]
-
-    def __init__(self, *args: str, **kwargs: ImmutableValue) -> None:
-        object.__setattr__(self, "args", args)
-        object.__setattr__(self, "kwargs", FrozenDict(kwargs))
-
     @classmethod
     def create(
         cls, freeze: Callable[[Any], ImmutableValue], parametrize: Parametrize
diff --git a/src/python/pants/engine/internals/parametrize.py b/src/python/pants/engine/internals/parametrize.py
--- a/src/python/pants/engine/internals/parametrize.py
+++ b/src/python/pants/engine/internals/parametrize.py
@@ -12,7 +12,13 @@
 from pants.engine.addresses import Address
 from pants.engine.collection import Collection
 from pants.engine.engine_aware import EngineAwareParameter
-from pants.engine.target import Field, FieldDefaults, Target, TargetTypesToGenerateTargetsRequests
+from pants.engine.target import (
+    Field,
+    FieldDefaults,
+    ImmutableValue,
+    Target,
+    TargetTypesToGenerateTargetsRequests,
+)
 from pants.util.frozendict import FrozenDict
 from pants.util.strutil import bullet_list, softwrap
 
@@ -33,11 +39,11 @@ class Parametrize:
     """
 
     args: tuple[str, ...]
-    kwargs: dict[str, Any]
+    kwargs: FrozenDict[str, ImmutableValue]
 
     def __init__(self, *args: str, **kwargs: Any) -> None:
         object.__setattr__(self, "args", args)
-        object.__setattr__(self, "kwargs", kwargs)
+        object.__setattr__(self, "kwargs", FrozenDict.deep_freeze(kwargs))
 
     def to_parameters(self) -> dict[str, Any]:
         """Validates and returns a mapping from aliases to parameter values.
diff --git a/src/python/pants/engine/target.py b/src/python/pants/engine/target.py
--- a/src/python/pants/engine/target.py
+++ b/src/python/pants/engine/target.py
@@ -2837,38 +2837,37 @@ def compute_value(
         cls,
         raw_value: Optional[Dict[Union[str, Tuple[str, ...]], Dict[str, Any]]],
         address: Address,
-    ) -> Optional[Dict[Tuple[str, ...], Dict[str, Any]]]:
+    ) -> Optional[FrozenDict[Tuple[str, ...], FrozenDict[str, ImmutableValue]]]:
         value_or_default = super().compute_value(raw_value, address)
         if value_or_default is None:
             return None
-        invalid_type_exception = InvalidFieldTypeException(
-            address,
-            cls.alias,
-            raw_value,
-            expected_type="dict[str | tuple[str, ...], dict[str, Any]]",
-        )
+
+        def invalid_type_exception() -> InvalidFieldException:
+            return InvalidFieldTypeException(
+                address,
+                cls.alias,
+                raw_value,
+                expected_type="dict[str | tuple[str, ...], dict[str, Any]]",
+            )
+
         if not isinstance(value_or_default, collections.abc.Mapping):
-            raise invalid_type_exception
+            raise invalid_type_exception()
 
-        result: dict[tuple[str, ...], dict[str, Any]] = {}
+        result: dict[tuple[str, ...], FrozenDict[str, ImmutableValue]] = {}
         for outer_key, nested_value in value_or_default.items():
             if isinstance(outer_key, str):
                 outer_key = (outer_key,)
             if not isinstance(outer_key, collections.abc.Sequence) or not all(
                 isinstance(elem, str) for elem in outer_key
             ):
-                raise invalid_type_exception
+                raise invalid_type_exception()
             if not isinstance(nested_value, collections.abc.Mapping):
-                raise invalid_type_exception
+                raise invalid_type_exception()
             if not all(isinstance(inner_key, str) for inner_key in nested_value):
-                raise invalid_type_exception
-            result[tuple(outer_key)] = dict(nested_value)
+                raise invalid_type_exception()
+            result[tuple(outer_key)] = FrozenDict.deep_freeze(cast(Mapping[str, Any], nested_value))
 
-        return result
-
-    def __hash__(self) -> int:
-        # The value might have unhashable elements like `list`, so we stringify it.
-        return hash((self.__class__, repr(self.value)))
+        return FrozenDict(result)
 
     @classmethod
     def to_path_globs(


Test patch:
diff --git a/src/python/pants/engine/internals/defaults_test.py b/src/python/pants/engine/internals/defaults_test.py
--- a/src/python/pants/engine/internals/defaults_test.py
+++ b/src/python/pants/engine/internals/defaults_test.py
@@ -18,6 +18,7 @@
     COMMON_TARGET_FIELDS,
     Dependencies,
     InvalidFieldException,
+    OverridesField,
     RegisteredTargetTypes,
     TargetGenerator,
 )
@@ -41,7 +42,10 @@ class TestGenTarget(GenericTarget):
 class TestGenTargetGenerator(TargetGenerator):
     alias = "test_gen_targets"
     generated_target_cls = TestGenTarget
-    core_fields = COMMON_TARGET_FIELDS
+    core_fields = (
+        OverridesField,
+        *COMMON_TARGET_FIELDS,
+    )
     copied_fields = COMMON_TARGET_FIELDS
     moved_fields = (GenericTargetDependenciesField,)
 
@@ -240,6 +244,25 @@ def test_assumptions(
             ),
             id="parametrize default field value",
         ),
+        pytest.param(
+            Scenario(
+                args=(
+                    {
+                        ("test_gen_targets",): dict(
+                            overrides={"*_generated.py": {"skip_yapf": True}},
+                        ),
+                    },
+                ),
+                expected_defaults={
+                    "test_gen_targets": dict(
+                        overrides=FrozenDict.deep_freeze(
+                            {("*_generated.py",): {"skip_yapf": True}}
+                        ),
+                    ),
+                },
+            ),
+            id="overrides value not frozen (issue #18784)",
+        ),
     ],
 )
 def test_set_defaults(
diff --git a/src/python/pants/engine/target_test.py b/src/python/pants/engine/target_test.py
--- a/src/python/pants/engine/target_test.py
+++ b/src/python/pants/engine/target_test.py
@@ -1414,15 +1414,16 @@ def test_overrides_field_normalization() -> None:
     addr = Address("", target_name="example")
 
     assert OverridesField(None, addr).value is None
-    assert OverridesField({}, addr).value == {}
+    assert OverridesField({}, addr).value == FrozenDict({})
 
-    # Note that `list_field` is not hashable. We have to override `__hash__` for this to work.
     tgt1_override = {"str_field": "value", "list_field": [0, 1, 3]}
     tgt2_override = {"int_field": 0, "dict_field": {"a": 0}}
 
     # Convert a `str` key to `tuple[str, ...]`.
     field = OverridesField({"tgt1": tgt1_override, ("tgt1", "tgt2"): tgt2_override}, addr)
-    assert field.value == {("tgt1",): tgt1_override, ("tgt1", "tgt2"): tgt2_override}
+    assert field.value == FrozenDict.deep_freeze(
+        {("tgt1",): tgt1_override, ("tgt1", "tgt2"): tgt2_override}
+    )
     with no_exception():
         hash(field)
 
@@ -1454,8 +1455,8 @@ def test_overrides_field_normalization() -> None:
         "dir/bar2.ext": tgt2_override,
     }
     assert path_field.flatten() == {
-        "foo.ext": {**tgt2_override, **tgt1_override},
-        "bar*.ext": tgt2_override,
+        "foo.ext": dict(FrozenDict.deep_freeze({**tgt2_override, **tgt1_override})),
+        "bar*.ext": dict(FrozenDict.deep_freeze(tgt2_override)),
     }
     with pytest.raises(InvalidFieldException):
         # Same field is overridden for the same file multiple times, which is an error.