Repository: encode/django-rest-framework

Issue:
ModelSerializer UniqueTogetherValidator is incompatible with field source
## Checklist

- [x] I have verified that that issue exists against the `master` branch of Django REST framework.
- [x] I have searched for similar issues in both open and closed tickets and cannot find a duplicate.
- [x] This is not a usage question. (Those should be directed to the [discussion group](https://groups.google.com/forum/#!forum/django-rest-framework) instead.)
- [x] This cannot be dealt with as a third party library. (We prefer new functionality to be [in the form of third party libraries](https://www.django-rest-framework.org/community/third-party-packages/#about-third-party-packages) where possible.)
- [x] I have reduced the issue to the simplest possible case.


## Steps to reproduce
In DRF == 3.10.3 and below it was a valid case

```python
# models.py
class MyModel(models.Model):
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE
    )
    date_from = models.DateField()
    date_to = models.DateField()
    
    class Meta:
        unique_together = (
            ('date_from', 'user'),
            ('date_to', 'user'),
        )

# serializers.py
class MyModelSerializer(serializers.ModelSerializer):
    date_start = serializers.DateField(source='date_from')
    date_end = serializers.DateField(source='date_to')
    
    class Meta:
        model = MyModel
        fields = 'date_start', 'date_end', 'user'

data = {
    'user': 1,
    'date_start': '2019-11-13',
    'date_end': '2019-11-14'
}

serializer = MyModelSerializer(data=data)
serializer.is_valid(raise_exception=True)
serializer.save()
```

In DRF == 3.11.0 I get an error in is_valid() method

```
~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/utils/serializer_helpers.py in __getitem__(self, key)
    146 
    147     def __getitem__(self, key):
--> 148         return self.fields[key]
    149 
    150     def __delitem__(self, key):

KeyError: 'date_from'
```

This case is no longer valid?
It was a convenient way to redirect field names.




Discussion:
Why aren't your fields in a list?
@napsterv it's equivalent to a tuple so it's fine anyway.
@hauworkarea could you paste the entire traceback please ?
@xordoquy that's entire traceback

```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-3-7ffae1e9ca9d> in <module>
      6 
      7 serializer = UserVacationSer(data=data)
----> 8 serializer.is_valid(raise_exception=True)
      9 serializer.save()

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/serializers.py in is_valid(self, raise_exception)
    232         if not hasattr(self, '_validated_data'):
    233             try:
--> 234                 self._validated_data = self.run_validation(self.initial_data)
    235             except ValidationError as exc:
    236                 self._validated_data = {}

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/serializers.py in run_validation(self, data)
    433         value = self.to_internal_value(data)
    434         try:
--> 435             self.run_validators(value)
    436             value = self.validate(value)
    437             assert value is not None, '.validate() should return the validated data'

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/serializers.py in run_validators(self, value)
    466         else:
    467             to_validate = value
--> 468         super().run_validators(to_validate)
    469 
    470     def to_internal_value(self, data):

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/fields.py in run_validators(self, value)
    586             try:
    587                 if getattr(validator, 'requires_context', False):
--> 588                     validator(value, self)
    589                 else:
    590                     validator(value)

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/validators.py in __call__(self, attrs, serializer)
    146 
    147     def __call__(self, attrs, serializer):
--> 148         self.enforce_required_fields(attrs, serializer)
    149         queryset = self.queryset
    150         queryset = self.filter_queryset(attrs, queryset, serializer)

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/validators.py in enforce_required_fields(self, attrs, serializer)
    106         missing_items = {
    107             field_name: self.missing_message
--> 108             for field_name in self.fields
    109             if serializer.fields[field_name].source not in attrs
    110         }

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/validators.py in <dictcomp>(.0)
    107             field_name: self.missing_message
    108             for field_name in self.fields
--> 109             if serializer.fields[field_name].source not in attrs
    110         }
    111         if missing_items:

~/PycharmProjects/env/staff.3.7.5/lib/python3.7/site-packages/rest_framework/utils/serializer_helpers.py in __getitem__(self, key)
    146 
    147     def __getitem__(self, key):
--> 148         return self.fields[key]
    149 
    150     def __delitem__(self, key):

KeyError: 'date_from'
```

It seems that the problem occurs when there is a unique_together in the meta class of the model.
Yes, the model contains unique_together.

```python
# models.py
class MyModel(models.Model):
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.CASCADE
    )
    date_from = models.DateField()
    date_to = models.DateField()
    
    class Meta:
        unique_together = (
            ('date_from', 'user'),
            ('date_to', 'user'),
        )
```

Corrected the original description.

I'm having the same issue.
`UniqueTogetherValidator.enforce_required_fields` assumes the presence of its listed fields in the dict comprehension: 
```
missing_items = {
    field_name: self.missing_message
    for field_name in self.fields
    if serializer.fields[field_name].source not in attrs
}
```

I can make my failing tests pass by replacing this with: 
```
fields_from_serializer = [
    field
    for field in serializer.fields.values()
    if field.source in self.fields
]
missing_items = {
    field.field_name: self.missing_message
    for field in fields_from_serializer
    if field.source not in attrs
}

# or just 
missing_items = {
    field.field_name: self.missing_message
    for field in serializer.fields.values()
    if field.source in self.fields
    and field.source not in attrs
}
```
and similarly, in `UniqueTogetherValidator.filter_queryset`, replacing
```
sources = [
    serializer.fields[field_name].source
    for field_name in self.fields
]
# with:
sources = [
    field.source
    for field in serializer.fields.values()
    if field.source in self.fields
]
```

I can't be 100% sure the different way of finding the fields doesn't break any assumptions about when the fields should be present.
I'm getting the same issue with DRF 3.11
I have unique_together constraint where one field is ForeignKey.

Serializer
```
 class SomeSerializer(SomeBaseSerializer):
        xxxx_id = serializers.PrimaryKeyRelatedField(source='xxxx', write_only=True,
                                                        queryset=XXXmodel.objects.all())
        class Meta:
            model = SomeModel
            fields = ('field_1', 'field_2', 'xxxx_id')
```
Model
```
class SomeModel(BaseModel):
    xxxx = models.ForeignKey(
        'app.App',
        null=False,
        blank=False,
        on_delete=models.CASCADE
    )
    field_1 = CleanCharField(max_length=64, null=False, blank=False)
    field_2 = models.DecimalField(max_digits=5, decimal_places=1, null=False, blank=False)
    class Meta:
        app_label = 'app'
        db_table = 'app_table'
        unique_together = ('xxxx', 'field_2')
```
Thanks - I'll look into this. This stems from #7086, which fixed a related bug. It looks like those changes in turn broke `UniqueTogetherValidator`s that are generated by `ModelSerializer`s. 

For now, a workaround would be to declare the `UniqueTogetherValidator`s directly on your serializer class. Using the example the issue description...

```python
class MyModelSerializer(serializers.ModelSerializer):
    date_start = serializers.DateField(source='date_from')
    date_end = serializers.DateField(source='date_to')
    
    class Meta:
        model = MyModel
        fields = ['date_start', 'date_end', 'user']
        validators = [
            UniqueTogetherValidator(queryset=MyModel.objects.all(), fields=['user', 'date_start']),
            UniqueTogetherValidator(queryset=MyModel.objects.all(), fields=['user', 'date_end']),
        ]
```




Resolving patch:
diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py
--- a/rest_framework/serializers.py
+++ b/rest_framework/serializers.py
@@ -13,7 +13,7 @@
 import copy
 import inspect
 import traceback
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from collections.abc import Mapping
 
 from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured
@@ -1508,28 +1508,55 @@ def get_unique_together_validators(self):
         # which may map onto a model field. Any dotted field name lookups
         # cannot map to a field, and must be a traversal, so we're not
         # including those.
-        field_names = {
-            field.source for field in self._writable_fields
+        field_sources = OrderedDict(
+            (field.field_name, field.source) for field in self._writable_fields
             if (field.source != '*') and ('.' not in field.source)
-        }
+        )
 
         # Special Case: Add read_only fields with defaults.
-        field_names |= {
-            field.source for field in self.fields.values()
+        field_sources.update(OrderedDict(
+            (field.field_name, field.source) for field in self.fields.values()
             if (field.read_only) and (field.default != empty) and (field.source != '*') and ('.' not in field.source)
-        }
+        ))
+
+        # Invert so we can find the serializer field names that correspond to
+        # the model field names in the unique_together sets. This also allows
+        # us to check that multiple fields don't map to the same source.
+        source_map = defaultdict(list)
+        for name, source in field_sources.items():
+            source_map[source].append(name)
 
         # Note that we make sure to check `unique_together` both on the
         # base model class, but also on any parent classes.
         validators = []
         for parent_class in model_class_inheritance_tree:
             for unique_together in parent_class._meta.unique_together:
-                if field_names.issuperset(set(unique_together)):
-                    validator = UniqueTogetherValidator(
-                        queryset=parent_class._default_manager,
-                        fields=unique_together
+                # Skip if serializer does not map to all unique together sources
+                if not set(source_map).issuperset(set(unique_together)):
+                    continue
+
+                for source in unique_together:
+                    assert len(source_map[source]) == 1, (
+                        "Unable to create `UniqueTogetherValidator` for "
+                        "`{model}.{field}` as `{serializer}` has multiple "
+                        "fields ({fields}) that map to this model field. "
+                        "Either remove the extra fields, or override "
+                        "`Meta.validators` with a `UniqueTogetherValidator` "
+                        "using the desired field names."
+                        .format(
+                            model=self.Meta.model.__name__,
+                            serializer=self.__class__.__name__,
+                            field=source,
+                            fields=', '.join(source_map[source]),
+                        )
                     )
-                    validators.append(validator)
+
+                field_names = tuple(source_map[f][0] for f in unique_together)
+                validator = UniqueTogetherValidator(
+                    queryset=parent_class._default_manager,
+                    fields=field_names
+                )
+                validators.append(validator)
         return validators
 
     def get_unique_for_date_validators(self):


Test patch:
diff --git a/tests/test_validators.py b/tests/test_validators.py
--- a/tests/test_validators.py
+++ b/tests/test_validators.py
@@ -344,6 +344,49 @@ class Meta:
             ]
         }
 
+    def test_default_validator_with_fields_with_source(self):
+        class TestSerializer(serializers.ModelSerializer):
+            name = serializers.CharField(source='race_name')
+
+            class Meta:
+                model = UniquenessTogetherModel
+                fields = ['name', 'position']
+
+        serializer = TestSerializer()
+        expected = dedent("""
+            TestSerializer():
+                name = CharField(source='race_name')
+                position = IntegerField()
+                class Meta:
+                    validators = [<UniqueTogetherValidator(queryset=UniquenessTogetherModel.objects.all(), fields=('name', 'position'))>]
+        """)
+        assert repr(serializer) == expected
+
+    def test_default_validator_with_multiple_fields_with_same_source(self):
+        class TestSerializer(serializers.ModelSerializer):
+            name = serializers.CharField(source='race_name')
+            other_name = serializers.CharField(source='race_name')
+
+            class Meta:
+                model = UniquenessTogetherModel
+                fields = ['name', 'other_name', 'position']
+
+        serializer = TestSerializer(data={
+            'name': 'foo',
+            'other_name': 'foo',
+            'position': 1,
+        })
+        with pytest.raises(AssertionError) as excinfo:
+            serializer.is_valid()
+
+        expected = (
+            "Unable to create `UniqueTogetherValidator` for "
+            "`UniquenessTogetherModel.race_name` as `TestSerializer` has "
+            "multiple fields (name, other_name) that map to this model field. "
+            "Either remove the extra fields, or override `Meta.validators` "
+            "with a `UniqueTogetherValidator` using the desired field names.")
+        assert str(excinfo.value) == expected
+
     def test_allow_explict_override(self):
         """
         Ensure validators can be explicitly removed..