Repository: MongoEngine/mongoengine

Issue:
Deleting an element from middle of ListField results in an incorrect list.
``` python
import mongoengine as me

class TestList(me.Document):
    test_list = me.ListField(me.IntField())

def test():
    me.connect('test_list', host='mongomock://localhost')

    vals = [1, 2, 3, 4]
    test_list = TestList(test_list=vals)
    test_list.save()

    for tl in TestList.objects():
        print tl.to_json()

    del test_list.test_list[1]
    test_list.save()

    for tl in TestList.objects():
        print tl.to_json()

if __name__ == '__main__':
    test()
```

Running this gives the following result:

```
$ python test_list.py
{"_id": {"$oid": "576506f653c26b60b8f9d93c"}, "test_list": [1, 2, 3, 4]}
{"_id": {"$oid": "576506f653c26b60b8f9d93c"}, "test_list": [1, 3, 3, 4]}
```

I would expect the modified list to be [1, 3, 4].  If I completely replace the list (test_list.test_list = [1, 3, 4]) instead of deleting an element it works fine.



Discussion:
This bug appears to also effecting the subclass EmbeddedDocumentList. I'm doing work-arounds at the moment, but I'd like to get this fixed at some point.

The bug is still in 10.0.6.  So, I'm considering tracing down the source of the error and correcting.

But before I get started: Is anyone else working on this? Any conflicts?

(BTW, the actual list item does get deleted in memory. The error appears to be in actually saving the document with the deletion. Just guessing, it is doing an update rather than a deletion on the list item.)
This is definitely a bug, good catch! FYI, I noticed that using `test_list.update(set__test_list=test_list.test_list)` does the right thing as opposed to using `test_list.save()`.
@JohnAD AFAIK nobody else is working on it. I'd love to see a PR if you can find and fix the issue!
Starting to work on this. But in the meantime, there appears to be an easy workaround; at least for my combination of versions: use the `.pop` method instead. `pop` returns a value, but you can simply ignore it.

So, given the example above, replace:

    del test_list.test_list[1]

with:

    test_list.test_list.pop(1)



Resolving patch:
diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py
--- a/mongoengine/base/datastructures.py
+++ b/mongoengine/base/datastructures.py
@@ -138,10 +138,7 @@ def __setitem__(self, key, value, *args, **kwargs):
         return super(BaseList, self).__setitem__(key, value)
 
     def __delitem__(self, key, *args, **kwargs):
-        if isinstance(key, slice):
-            self._mark_as_changed()
-        else:
-            self._mark_as_changed(key)
+        self._mark_as_changed()
         return super(BaseList, self).__delitem__(key)
 
     def __setslice__(self, *args, **kwargs):


Test patch:
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -1042,6 +1042,7 @@ class BlogPost(Document):
         self.assertEqual(
             BlogPost.objects.filter(info__100__test__exact='test').count(), 0)
 
+        # test queries by list
         post = BlogPost()
         post.info = ['1', '2']
         post.save()
@@ -1053,8 +1054,251 @@ class BlogPost(Document):
         post.info *= 2
         post.save()
         self.assertEqual(BlogPost.objects(info=['1', '2', '3', '4', '1', '2', '3', '4']).count(), 1)
+
+        BlogPost.drop_collection()
+
+
+    def test_list_field_manipulative_operators(self):
+        """Ensure that ListField works with standard list operators that manipulate the list.
+        """
+        class BlogPost(Document):
+            ref = StringField()
+            info = ListField(StringField())
+
+        BlogPost.drop_collection()
+
+        post = BlogPost()
+        post.ref = "1234"
+        post.info = ['0', '1', '2', '3', '4', '5']
+        post.save()
+
+        def reset_post():
+            post.info = ['0', '1', '2', '3', '4', '5']
+            post.save()
+
+        # '__add__(listB)'
+        # listA+listB
+        # operator.add(listA, listB) 
+        reset_post()
+        temp = ['a', 'b']
+        post.info = post.info + temp
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'a', 'b'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'a', 'b'])
+
+        # '__delitem__(index)'
+        # aka 'del list[index]'
+        # aka 'operator.delitem(list, index)'
+        reset_post()
+        del post.info[2] # del from middle ('2')
+        self.assertEqual(post.info, ['0', '1', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '3', '4', '5'])
+
+        # '__delitem__(slice(i, j))'
+        # aka 'del list[i:j]'
+        # aka 'operator.delitem(list, slice(i,j))'
+        reset_post()
+        del post.info[1:3] # removes '1', '2'
+        self.assertEqual(post.info, ['0', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '3', '4', '5'])
+
+        # '__iadd__'
+        # aka 'list += list'
+        reset_post()
+        temp = ['a', 'b']
+        post.info += temp
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'a', 'b'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'a', 'b'])
+
+        # '__imul__'
+        # aka 'list *= number'
+        reset_post()
+        post.info *= 2
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', '0', '1', '2', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', '0', '1', '2', '3', '4', '5'])
+
+        # '__mul__'
+        # aka 'listA*listB'
+        reset_post()
+        post.info = post.info * 2
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', '0', '1', '2', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', '0', '1', '2', '3', '4', '5'])
+
+        # '__rmul__'
+        # aka 'listB*listA'
+        reset_post()
+        post.info = 2 * post.info
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', '0', '1', '2', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', '0', '1', '2', '3', '4', '5'])
+
+        # '__setitem__(index, value)'
+        # aka 'list[index]=value'
+        # aka 'setitem(list, value)'
+        reset_post()
+        post.info[4] = 'a'
+        self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', 'a', '5'])
+
+        # '__setitem__(slice(i, j), listB)'
+        # aka 'listA[i:j] = listB'
+        # aka 'setitem(listA, slice(i, j), listB)'
+        reset_post()
+        post.info[1:3] = ['h', 'e', 'l', 'l', 'o']
+        self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', 'h', 'e', 'l', 'l', 'o', '3', '4', '5'])
+
+        # 'append'
+        reset_post()
+        post.info.append('h')
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'h'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'h'])
+
+        # 'extend'
+        reset_post()
+        post.info.extend(['h', 'e', 'l', 'l', 'o'])
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'h', 'e', 'l', 'l', 'o'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '2', '3', '4', '5', 'h', 'e', 'l', 'l', 'o'])
+        # 'insert'
+
+        # 'pop'
+        reset_post()
+        x = post.info.pop(2)
+        y = post.info.pop()
+        self.assertEqual(post.info, ['0', '1', '3', '4'])
+        self.assertEqual(x, '2')
+        self.assertEqual(y, '5')
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '3', '4'])
+
+        # 'remove'
+        reset_post()
+        post.info.remove('2')
+        self.assertEqual(post.info, ['0', '1', '3', '4', '5'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['0', '1', '3', '4', '5'])
+
+        # 'reverse'
+        reset_post()
+        post.info.reverse()
+        self.assertEqual(post.info, ['5', '4', '3', '2', '1', '0'])
+        post.save()
+        post.reload()
+        self.assertEqual(post.info, ['5', '4', '3', '2', '1', '0'])
+
+        # 'sort': though this operator method does manipulate the list, it is tested in
+        #     the 'test_list_field_lexicograpic_operators' function
+        BlogPost.drop_collection()
+
+    def test_list_field_invalid_operators(self):
+        class BlogPost(Document):
+            ref = StringField()
+            info = ListField(StringField())
+        post = BlogPost()
+        post.ref = "1234"
+        post.info = ['0', '1', '2', '3', '4', '5']
+        # '__hash__'
+        # aka 'hash(list)'
+        # # assert TypeError
+        self.assertRaises(TypeError, lambda: hash(post.info))
+
+    def test_list_field_lexicographic_operators(self):
+        """Ensure that ListField works with standard list operators that do lexigraphic ordering.
+        """
+        class BlogPost(Document):
+            ref = StringField()
+            text_info = ListField(StringField())
+            oid_info = ListField(ObjectIdField())
+            bool_info = ListField(BooleanField())
         BlogPost.drop_collection()
 
+        blogSmall = BlogPost(ref="small")
+        blogSmall.text_info = ["a", "a", "a"]
+        blogSmall.bool_info = [False, False]
+        blogSmall.save()
+        blogSmall.reload()
+
+        blogLargeA = BlogPost(ref="big")
+        blogLargeA.text_info = ["a", "z", "j"]
+        blogLargeA.bool_info = [False, True]
+        blogLargeA.save()
+        blogLargeA.reload()
+
+        blogLargeB = BlogPost(ref="big2")
+        blogLargeB.text_info = ["a", "z", "j"]
+        blogLargeB.oid_info = [
+            "54495ad94c934721ede76f90",
+            "54495ad94c934721ede76d23",
+            "54495ad94c934721ede76d00"
+        ]
+        blogLargeB.bool_info = [False, True]
+        blogLargeB.save()
+        blogLargeB.reload()
+        # '__eq__' aka '=='
+        self.assertEqual(blogLargeA.text_info, blogLargeB.text_info)
+        self.assertEqual(blogLargeA.bool_info, blogLargeB.bool_info)
+        # '__ge__' aka '>='
+        self.assertGreaterEqual(blogLargeA.text_info, blogSmall.text_info)
+        self.assertGreaterEqual(blogLargeA.text_info, blogLargeB.text_info)
+        self.assertGreaterEqual(blogLargeA.bool_info, blogSmall.bool_info)
+        self.assertGreaterEqual(blogLargeA.bool_info, blogLargeB.bool_info)
+        # '__gt__' aka '>'
+        self.assertGreaterEqual(blogLargeA.text_info, blogSmall.text_info)
+        self.assertGreaterEqual(blogLargeA.bool_info, blogSmall.bool_info)
+        # '__le__' aka '<='
+        self.assertLessEqual(blogSmall.text_info, blogLargeB.text_info)
+        self.assertLessEqual(blogLargeA.text_info, blogLargeB.text_info)
+        self.assertLessEqual(blogSmall.bool_info, blogLargeB.bool_info)
+        self.assertLessEqual(blogLargeA.bool_info, blogLargeB.bool_info)
+        # '__lt__' aka '<'
+        self.assertLess(blogSmall.text_info, blogLargeB.text_info)
+        self.assertLess(blogSmall.bool_info, blogLargeB.bool_info)
+        # '__ne__' aka '!='
+        self.assertNotEqual(blogSmall.text_info, blogLargeB.text_info)
+        self.assertNotEqual(blogSmall.bool_info, blogLargeB.bool_info)
+        # 'sort'
+        blogLargeB.bool_info = [True, False, True, False]
+        blogLargeB.text_info.sort()
+        blogLargeB.oid_info.sort()
+        blogLargeB.bool_info.sort()
+        sorted_target_list = [
+            ObjectId("54495ad94c934721ede76d00"),
+            ObjectId("54495ad94c934721ede76d23"),
+            ObjectId("54495ad94c934721ede76f90")
+        ]
+        self.assertEqual(blogLargeB.text_info, ["a", "j", "z"])
+        self.assertEqual(blogLargeB.oid_info, sorted_target_list)
+        self.assertEqual(blogLargeB.bool_info, [False, False, True, True])
+        blogLargeB.save()
+        blogLargeB.reload()
+        self.assertEqual(blogLargeB.text_info, ["a", "j", "z"])
+        self.assertEqual(blogLargeB.oid_info, sorted_target_list)
+        self.assertEqual(blogLargeB.bool_info, [False, False, True, True])
+
+        BlogPost.drop_collection()
+ 
     def test_list_assignment(self):
         """Ensure that list field element assignment and slicing work
         """