Repository: encode/django-rest-framework

Issue:
ListField default values not honored in multipart/form-data mode
When sending a `multipart/form-data` request to a serializer with a `ListField`, the default value returned for the field is always the empty list `[]`.  Any `default=` passed to field will be ignored due to the way html input is handled in the `get_value()` function.

* This functionality works properly when posting `json` data
* May be related to #5807 but I don't think so.  

## Steps to reproduce
```python
class SimpleSerializer(Serializer):
    a = IntegerField(required=True)
    c = ListField(default=lambda: [1, 2, 3])

# simulate a multipart/form-data post (querydict is a confusing name here)
s1 = SimpleSerializer(data=QueryDict("a=3"))
s1.is_valid(raise_exception=True)
print("html:", s1.validated_data)

# simulate a JSON post
s2 = SimpleSerializer(data={'a': 3})
s2.is_valid(raise_exception=True)
print("json:", s2.validated_data)

>>> html: OrderedDict([(u'a', 3), (u'c', [])])
>>> json: OrderedDict([(u'a', 3), (u'c', [1, 2, 3])])
```

## Expected behavior

* Default value should be honored in HTML mode

## Actual behavior

* Default value is always returned as `[]`

## Explanation and Workaround

The `ListField.get_value()` function **always** returns the value of `html.parse_html_list` from get_value when that field_name was not passed.  There is a check at the top for a missing key, but that only affects `partial` posts.  Missing keys on regular POSTs will still be processed normally and return `[]`.

I'm not sure if it is OK to just *remove* that check for partial and always return `empty`, but that is what I have done for this version of my workaround.  The list fields I use this way are in pure serializers and are only used for validating input.  They are not used in `ModelSerializers` so for my case this is ok.  

## Initial Workaround
```python
class ListFieldWithSaneDefault(ListField):
    """
    This is used ONLY as a base class for other fields.  When using it, please ensure that you
    always provide a default value (at least `default=lambda: []`) if the field is not required.
    Your derived class should take no parameters to __init__, it should be self contained
    """
    def get_value(self, dictionary):
        """
        When handling html multipart forms input (as opposed to json, which works properly)
        the base list field returns `[]` for _missing_ keys.  This override checks for that specific
        case and returns `empty` so that standard default-value processing takes over
        """
        if self.field_name not in dictionary:
            return empty
        return super(ListFieldWithSaneDefault, self).get_value(dictionary)
```



Discussion:
Okay. One sensible thing to do would be to try that in the repo, and see which (if any) tests fail,
and then take things from there.


Resolving patch:
diff --git a/rest_framework/fields.py b/rest_framework/fields.py
--- a/rest_framework/fields.py
+++ b/rest_framework/fields.py
@@ -1614,7 +1614,8 @@ def get_value(self, dictionary):
             if len(val) > 0:
                 # Support QueryDict lists in HTML input.
                 return val
-            return html.parse_html_list(dictionary, prefix=self.field_name)
+            return html.parse_html_list(dictionary, prefix=self.field_name, default=empty)
+
         return dictionary.get(self.field_name, empty)
 
     def to_internal_value(self, data):
@@ -1622,7 +1623,7 @@ def to_internal_value(self, data):
         List of dicts of native values <- List of dicts of primitive datatypes.
         """
         if html.is_html_input(data):
-            data = html.parse_html_list(data)
+            data = html.parse_html_list(data, default=[])
         if isinstance(data, type('')) or isinstance(data, collections.Mapping) or not hasattr(data, '__iter__'):
             self.fail('not_a_list', input_type=type(data).__name__)
         if not self.allow_empty and len(data) == 0:
diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py
--- a/rest_framework/serializers.py
+++ b/rest_framework/serializers.py
@@ -607,7 +607,7 @@ def get_value(self, dictionary):
         # We override the default field access in order to support
         # lists in HTML forms.
         if html.is_html_input(dictionary):
-            return html.parse_html_list(dictionary, prefix=self.field_name)
+            return html.parse_html_list(dictionary, prefix=self.field_name, default=empty)
         return dictionary.get(self.field_name, empty)
 
     def run_validation(self, data=empty):
@@ -635,7 +635,7 @@ def to_internal_value(self, data):
         List of dicts of native values <- List of dicts of primitive datatypes.
         """
         if html.is_html_input(data):
-            data = html.parse_html_list(data)
+            data = html.parse_html_list(data, default=[])
 
         if not isinstance(data, list):
             message = self.error_messages['not_a_list'].format(
diff --git a/rest_framework/utils/html.py b/rest_framework/utils/html.py
--- a/rest_framework/utils/html.py
+++ b/rest_framework/utils/html.py
@@ -12,7 +12,7 @@ def is_html_input(dictionary):
     return hasattr(dictionary, 'getlist')
 
 
-def parse_html_list(dictionary, prefix=''):
+def parse_html_list(dictionary, prefix='', default=None):
     """
     Used to support list values in HTML forms.
     Supports lists of primitives and/or dictionaries.
@@ -44,6 +44,8 @@ def parse_html_list(dictionary, prefix=''):
         {'foo': 'abc', 'bar': 'def'},
         {'foo': 'hij', 'bar': 'klm'}
     ]
+
+    :returns a list of objects, or the value specified in ``default`` if the list is empty
     """
     ret = {}
     regex = re.compile(r'^%s\[([0-9]+)\](.*)$' % re.escape(prefix))
@@ -59,7 +61,9 @@ def parse_html_list(dictionary, prefix=''):
             ret[index][key] = value
         else:
             ret[index] = MultiValueDict({key: [value]})
-    return [ret[item] for item in sorted(ret)]
+
+    # return the items of the ``ret`` dict, sorted by key, or ``default`` if the dict is empty
+    return [ret[item] for item in sorted(ret)] if ret else default
 
 
 def parse_html_dict(dictionary, prefix=''):


Test patch:
diff --git a/tests/test_fields.py b/tests/test_fields.py
--- a/tests/test_fields.py
+++ b/tests/test_fields.py
@@ -466,6 +466,55 @@ class TestSerializer(serializers.Serializer):
         assert serializer.is_valid()
         assert serializer.validated_data == {'scores': [1]}
 
+    def test_querydict_list_input_no_values_uses_default(self):
+        """
+        When there are no values passed in, and default is set
+        The field should return the default value
+        """
+        class TestSerializer(serializers.Serializer):
+            a = serializers.IntegerField(required=True)
+            scores = serializers.ListField(default=lambda: [1, 3])
+
+        serializer = TestSerializer(data=QueryDict('a=1&'))
+        assert serializer.is_valid()
+        assert serializer.validated_data == {'a': 1, 'scores': [1, 3]}
+
+    def test_querydict_list_input_supports_indexed_keys(self):
+        """
+        When data is passed in the format `scores[0]=1&scores[1]=3`
+        The field should return the correct list, ignoring the default
+        """
+        class TestSerializer(serializers.Serializer):
+            scores = serializers.ListField(default=lambda: [1, 3])
+
+        serializer = TestSerializer(data=QueryDict("scores[0]=5&scores[1]=6"))
+        assert serializer.is_valid()
+        assert serializer.validated_data == {'scores': ['5', '6']}
+
+    def test_querydict_list_input_no_values_no_default_and_not_required(self):
+        """
+        When there are no keys passed, there is no default, and required=False
+        The field should be skipped
+        """
+        class TestSerializer(serializers.Serializer):
+            scores = serializers.ListField(required=False)
+
+        serializer = TestSerializer(data=QueryDict(''))
+        assert serializer.is_valid()
+        assert serializer.validated_data == {}
+
+    def test_querydict_list_input_posts_key_but_no_values(self):
+        """
+        When there are no keys passed, there is no default, and required=False
+        The field should return an array of 1 item, blank
+        """
+        class TestSerializer(serializers.Serializer):
+            scores = serializers.ListField(required=False)
+
+        serializer = TestSerializer(data=QueryDict('scores=&'))
+        assert serializer.is_valid()
+        assert serializer.validated_data == {'scores': ['']}
+
 
 class TestCreateOnlyDefault:
     def setup(self):
diff --git a/tests/test_serializer_lists.py b/tests/test_serializer_lists.py
--- a/tests/test_serializer_lists.py
+++ b/tests/test_serializer_lists.py
@@ -1,3 +1,4 @@
+from django.http import QueryDict
 from django.utils.datastructures import MultiValueDict
 
 from rest_framework import serializers
@@ -532,3 +533,32 @@ class Serializer(serializers.Serializer):
                 assert value == updated_data_list[index][key]
 
         assert serializer.errors == {}
+
+
+class TestEmptyListSerializer:
+    """
+    Tests the behaviour of ListSerializers when there is no data passed to it
+    """
+
+    def setup(self):
+        class ExampleListSerializer(serializers.ListSerializer):
+            child = serializers.IntegerField()
+
+        self.Serializer = ExampleListSerializer
+
+    def test_nested_serializer_with_list_json(self):
+        # pass an empty array to the serializer
+        input_data = []
+
+        serializer = self.Serializer(data=input_data)
+
+        assert serializer.is_valid()
+        assert serializer.validated_data == []
+
+    def test_nested_serializer_with_list_multipart(self):
+        # pass an "empty" QueryDict to the serializer (should be the same as an empty array)
+        input_data = QueryDict('')
+        serializer = self.Serializer(data=input_data)
+
+        assert serializer.is_valid()
+        assert serializer.validated_data == []
diff --git a/tests/test_serializer_nested.py b/tests/test_serializer_nested.py
--- a/tests/test_serializer_nested.py
+++ b/tests/test_serializer_nested.py
@@ -202,3 +202,42 @@ def test_nested_serializer_with_list_multipart(self):
 
         assert serializer.is_valid()
         assert serializer.validated_data['nested']['example'] == {1, 2}
+
+
+class TestNotRequiredNestedSerializerWithMany:
+    def setup(self):
+        class NestedSerializer(serializers.Serializer):
+            one = serializers.IntegerField(max_value=10)
+
+        class TestSerializer(serializers.Serializer):
+            nested = NestedSerializer(required=False, many=True)
+
+        self.Serializer = TestSerializer
+
+    def test_json_validate(self):
+        input_data = {}
+        serializer = self.Serializer(data=input_data)
+
+        # request is empty, therefor 'nested' should not be in serializer.data
+        assert serializer.is_valid()
+        assert 'nested' not in serializer.validated_data
+
+        input_data = {'nested': [{'one': '1'}, {'one': 2}]}
+        serializer = self.Serializer(data=input_data)
+        assert serializer.is_valid()
+        assert 'nested' in serializer.validated_data
+
+    def test_multipart_validate(self):
+        # leave querydict empty
+        input_data = QueryDict('')
+        serializer = self.Serializer(data=input_data)
+
+        # the querydict is empty, therefor 'nested' should not be in serializer.data
+        assert serializer.is_valid()
+        assert 'nested' not in serializer.validated_data
+
+        input_data = QueryDict('nested[0]one=1&nested[1]one=2')
+
+        serializer = self.Serializer(data=input_data)
+        assert serializer.is_valid()
+        assert 'nested' in serializer.validated_data