Repository: python-attrs/attrs

Issue:
field checks done too early causing error
Attrs does some checks on fields like no defaulted fields coming before non-defaulted fields before running custom field transformers. This is a problem when a field transformer intends to modify the fields but gets an error before being able to apply their modification.
For example, this field transformer reorders fields so that in fact the defaulted field `x` comes after the non-defaulted field `y`. But attrs errors before the transformer is ever called.

```python
def order_by_metadata(cls, fields):
    fields.sort(key=lambda x: x.metadata["field_order"])
    return fields

@define(field_transformer=order_by_metadata)
class A:
    x: int = field(metadata={"field_order": 1}, default=0)
    y: int = field(metadata={"field_order": 0})
```

```
Traceback (most recent call last):
  File "...", line 426, in <module>
    class A:
  File ".../attr/_next_gen.py", line 146, in wrap
    return do_it(cls, True)
  File ".../attr/_next_gen.py", line 90, in do_it
    return attrs(
  File ".../attr/_make.py", line 1603, in attrs
    return wrap(maybe_cls)
  File ".../attr/_make.py", line 1499, in wrap
    builder = _ClassBuilder(
  File ".../attr/_make.py", line 657, in __init__
    attrs, base_attrs, base_map = _transform_attrs(
  File ".../attr/_make.py", line 566, in _transform_attrs
    raise ValueError(
ValueError: No mandatory attributes allowed after an attribute with a default value or factory.  Attribute in question: Attribute(name='y', default=NOTHING, validator=None, repr=True, eq=True, eq_key=None, order=True, order_key=None, hash=None, init=True, metadata=mappingproxy({'field_order': 0}), type='int', converter=None, kw_only=False, inherited=False, on_setattr=None, alias=None)
```
I think these checks should be done after custom field transformers.


Discussion:
I concur, see also https://github.com/python-attrs/attrs/issues/1169#issuecomment-2240403489
@sscherfke transformers are your labor of love – is this realistic without adding a whole abstraction layer over this?
> is this realistic without adding a whole abstraction layer over this?

Not sure what you mean -- why isn't it sufficient to just move [the order check](https://github.com/python-attrs/attrs/blob/ae7340966c12b9d6525f0c2fc413b997dc294e66/src/attr/_make.py#L449) after the field transformer has been applied?

IIUC, field transformers can already do arbitrary rearranging, as long as the initial field order is valid.
But all that matters should be the final order, not the initial one, right? Conversely, a field transformer that results in an invalid order would currently slip past the check.

That said, why do we need the manual check in the first place? Python already gives us a pretty good exception when attempting to evaluate the incorrect code. Picking up the above example:
```python
def order_by_metadata(cls, fields):
    fields.sort(key=lambda x: x.metadata["field_order"])
    return fields

@define(field_transformer=order_by_metadata)
class A:
    x: int = field(metadata={"field_order": 1})
    y: int = field(metadata={"field_order": 0}, default=0)
```

This yields the following error:
```pytb
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/me/.local/lib/python3.11/site-packages/attr/_next_gen.py", line 144, in wrap
    return do_it(cls, True)
           ^^^^^^^^^^^^^^^^
  File "/home/me/.local/lib/python3.11/site-packages/attr/_next_gen.py", line 90, in do_it
    return attrs(
           ^^^^^^
  File "/home/me/.local/lib/python3.11/site-packages/attr/_make.py", line 1715, in attrs
    return wrap(maybe_cls)
           ^^^^^^^^^^^^^^^
  File "/home/me/.local/lib/python3.11/site-packages/attr/_make.py", line 1694, in wrap
    builder.add_init()
  File "/home/me/.local/lib/python3.11/site-packages/attr/_make.py", line 1090, in add_init
    _make_init(
  File "/home/me/.local/lib/python3.11/site-packages/attr/_make.py", line 2181, in _make_init
    init = _make_method(
           ^^^^^^^^^^^^^
  File "/home/me/.local/lib/python3.11/site-packages/attr/_make.py", line 345, in _make_method
    _compile_and_eval(script, globs, locs, filename)
  File "/home/me/.local/lib/python3.11/site-packages/attr/_make.py", line 317, in _compile_and_eval
    bytecode = compile(script, filename, "exec")
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<attrs generated init __main__.A>", line 1
    def __init__(self, y=attr_dict['y'].default, x):
                                                 ^
SyntaxError: non-default argument follows default argument
```



Patch:
diff --git a/changelog.d/1147.change.md b/changelog.d/1147.change.md
new file mode 100644
index 000000000..17c62638f
--- /dev/null
+++ b/changelog.d/1147.change.md
@@ -0,0 +1,1 @@
+Checking mandatory vs non-mandatory attribute order is now performed after the field transformer, since the field transformer may change attributes and/or their order.
diff --git a/src/attr/_make.py b/src/attr/_make.py
index a7a4075af..b1550eb8a 100644
--- a/src/attr/_make.py
+++ b/src/attr/_make.py
@@ -449,6 +449,10 @@ def _transform_attrs(
 
     attrs = base_attrs + own_attrs
 
+    if field_transformer is not None:
+        attrs = field_transformer(cls, attrs)
+
+    # Check attr order after executing the field_transformer.
     # Mandatory vs non-mandatory attr order only matters when they are part of
     # the __init__ signature and when they aren't kw_only (which are moved to
     # the end and can be mandatory or non-mandatory in any order, as they will
@@ -462,9 +466,6 @@ def _transform_attrs(
         if had_default is False and a.default is not NOTHING:
             had_default = True
 
-    if field_transformer is not None:
-        attrs = field_transformer(cls, attrs)
-
     # Resolve default field alias after executing field_transformer.
     # This allows field_transformer to differentiate between explicit vs
     # default aliases and supply their own defaults.


Test patch:
diff --git a/tests/test_hooks.py b/tests/test_hooks.py
index 9c37a98cd..6b3420639 100644
--- a/tests/test_hooks.py
+++ b/tests/test_hooks.py
@@ -4,6 +4,8 @@
 
 from datetime import datetime
 
+import pytest
+
 import attr
 
 
@@ -30,7 +32,7 @@ class C:
             y = attr.ib(type=int)
             z: float = attr.ib()
 
-        assert results == [("x", None), ("y", int), ("z", float)]
+        assert [("x", None), ("y", int), ("z", float)] == results
 
     def test_hook_applied_auto_attrib(self):
         """
@@ -49,7 +51,7 @@ class C:
             x: int
             y: str = attr.ib()
 
-        assert results == [("x", int), ("y", str)]
+        assert [("x", int), ("y", str)] == results
 
     def test_hook_applied_modify_attrib(self):
         """
@@ -66,7 +68,8 @@ class C:
             y: float
 
         c = C(x="3", y="3.14")
-        assert c == C(x=3, y=3.14)
+
+        assert C(x=3, y=3.14) == c
 
     def test_hook_remove_field(self):
         """
@@ -82,7 +85,7 @@ class C:
             x: int
             y: float
 
-        assert attr.asdict(C(2.7)) == {"y": 2.7}
+        assert {"y": 2.7} == attr.asdict(C(2.7))
 
     def test_hook_add_field(self):
         """
@@ -98,7 +101,7 @@ def hook(cls, attribs):
         class C:
             x: int
 
-        assert attr.asdict(C(1, 2)) == {"x": 1, "new": 2}
+        assert {"x": 1, "new": 2} == attr.asdict(C(1, 2))
 
     def test_hook_override_alias(self):
         """
@@ -118,13 +121,73 @@ class NameCase:
             1, 2, 3
         )
 
+    def test_hook_reorder_fields(self):
+        """
+        It is possible to reorder fields via the hook.
+        """
+
+        def hook(cls, attribs):
+            return sorted(attribs, key=lambda x: x.metadata["field_order"])
+
+        @attr.s(field_transformer=hook)
+        class C:
+            x: int = attr.ib(metadata={"field_order": 1})
+            y: int = attr.ib(metadata={"field_order": 0})
+
+        assert {"x": 0, "y": 1} == attr.asdict(C(1, 0))
+
+    def test_hook_reorder_fields_before_order_check(self):
+        """
+        It is possible to reorder fields via the hook before order-based errors are raised.
+
+        Regression test for #1147.
+        """
+
+        def hook(cls, attribs):
+            return sorted(attribs, key=lambda x: x.metadata["field_order"])
+
+        @attr.s(field_transformer=hook)
+        class C:
+            x: int = attr.ib(metadata={"field_order": 1}, default=0)
+            y: int = attr.ib(metadata={"field_order": 0})
+
+        assert {"x": 0, "y": 1} == attr.asdict(C(1))
+
+    def test_hook_conflicting_defaults_after_reorder(self):
+        """
+        Raises `ValueError` if attributes with defaults are followed by
+        mandatory attributes after the hook reorders fields.
+
+        Regression test for #1147.
+        """
+
+        def hook(cls, attribs):
+            return sorted(attribs, key=lambda x: x.metadata["field_order"])
+
+        with pytest.raises(ValueError) as e:
+
+            @attr.s(field_transformer=hook)
+            class C:
+                x: int = attr.ib(metadata={"field_order": 1})
+                y: int = attr.ib(metadata={"field_order": 0}, default=0)
+
+        assert (
+            "No mandatory attributes allowed after an attribute with a "
+            "default value or factory.  Attribute in question: Attribute"
+            "(name='x', default=NOTHING, validator=None, repr=True, "
+            "eq=True, eq_key=None, order=True, order_key=None, "
+            "hash=None, init=True, "
+            "metadata=mappingproxy({'field_order': 1}), type='int', converter=None, "
+            "kw_only=False, inherited=False, on_setattr=None, alias=None)",
+        ) == e.value.args
+
     def test_hook_with_inheritance(self):
         """
         The hook receives all fields from base classes.
         """
 
         def hook(cls, attribs):
-            assert [a.name for a in attribs] == ["x", "y"]
+            assert ["x", "y"] == [a.name for a in attribs]
             # Remove Base' "x"
             return attribs[1:]
 
@@ -136,7 +199,7 @@ class Base:
         class Sub(Base):
             y: int
 
-        assert attr.asdict(Sub(2)) == {"y": 2}
+        assert {"y": 2} == attr.asdict(Sub(2))
 
     def test_attrs_attrclass(self):
         """
@@ -151,7 +214,7 @@ class C:
             x: int
 
         fields_type = type(attr.fields(C))
-        assert fields_type.__name__ == "CAttributes"
+        assert "CAttributes" == fields_type.__name__
         assert issubclass(fields_type, tuple)
 
 
@@ -187,12 +250,12 @@ class Parent:
         )
 
         result = attr.asdict(inst, value_serializer=hook)
-        assert result == {
+        assert {
             "a": {"x": 1, "y": ["2020-07-01T00:00:00"]},
             "b": [{"x": 2, "y": ["2020-07-02T00:00:00"]}],
             "c": {"spam": {"x": 3, "y": ["2020-07-03T00:00:00"]}},
             "d": {"eggs": "2020-07-04T00:00:00"},
-        }
+        } == result
 
     def test_asdict_calls(self):
         """
@@ -217,7 +280,7 @@ class Parent:
         inst = Parent(a=Child(1), b=[Child(2)], c={"spam": Child(3)})
 
         attr.asdict(inst, value_serializer=hook)
-        assert calls == [
+        assert [
             (inst, "a", inst.a),
             (inst.a, "x", inst.a.x),
             (inst, "b", inst.b),
@@ -225,4 +288,4 @@ class Parent:
             (inst, "c", inst.c),
             (None, None, "spam"),
             (inst.c["spam"], "x", inst.c["spam"].x),
-        ]
+        ] == calls