Repository: MongoEngine/mongoengine

Issue:
mixin Field dereference fail.
hi, 
I got some trouble in using mixin fields, here is the sample code:

```python
>>> from mongoengine import Document
>>> from mongoengine import MapField, ListField, ReferenceField
>>> class R(Document):
...   pass
...
>>> class T(Document):
...   rs = MapField(field=ListField(ReferenceField(R)))
...
>>> T.objects
[]
>>> t = T()
>>> t.rs
{}
>>> t.rs['rs1'] = [R()]
>>> t.rs
{'rs1': [<R: R object>]}
>>> t.rs['rs1'][0].save()
<R: R object>
>>> t.save()
<T: T object>
>>> t
<T: T object>
>>> T.objects
[<T: T object>]
>>> T.objects[0].rs
Traceback (most recent call last):
  File "<console>", line 1, in <module>
  File "C:\tmp\venv\lib\site-packages\mongoengine\base\fields.py", line 279, in __get__
    value, max_depth=1, instance=instance, name=self.name
  File "C:\tmp\venv\lib\site-packages\mongoengine\dereference.py", line 76, in __call__
    self.reference_map = self._find_references(items)
  File "C:\tmp\venv\lib\site-packages\mongoengine\dereference.py", line 115, in _find_references
    reference_map.setdefault(item.collection, set()).add(item.id)
TypeError: unhashable type: 'list'
```

I want to use `MapField` to mapping `key=String, value=List[Ref(Document)]`, save into mongodb is work, but can't retrieve it.

I tried to trace to figure out something, but I'm failed too. :joy_cat: 


Discussion:
I've confirmed this, but have not yet hunted down the reason. Additional notes:

The problem is also with parent DictField; so this is not a MapField problem per se.
The problem is specifically with a ListField containing ReferenceField. So,

```
class T(Document):
    rs = MapField(field=ListField(StringField()))
```

works fine. And

```
class T(Document):
    rs = MapField(field=ReferenceField(R))
```

Also works fine.
Until this is resolved, here is a temporary work-around:

```
class R(Document):
    pass

class TRMap(EmbeddedDocument):
    rl = ListField(ReferenceField(R))

class T(Document):
    rs = MapField(field=EmbeddedDocumentField(TRMap))

t = T()
m = TRMap(rl=[R()])
t.rs['rs1'] = m
t.rs['rs1'].rl[0].save()
t.save()
print T.objects[0].rs['rs1'].rl[0]
```

Resolving patch:
diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -3,9 +3,7 @@
 from collections import Hashable
 from functools import partial
 
-from bson import ObjectId, json_util
-from bson.dbref import DBRef
-from bson.son import SON
+from bson import DBRef, ObjectId, SON, json_util
 import pymongo
 import six
 
diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py
--- a/mongoengine/dereference.py
+++ b/mongoengine/dereference.py
@@ -52,26 +52,40 @@ def __call__(self, items, max_depth=1, instance=None, name=None):
                         [i.__class__ == doc_type for i in items.values()]):
                     return items
                 elif not field.dbref:
-                    if not hasattr(items, 'items'):
+                    # We must turn the ObjectIds into DBRefs
+
+                    # Recursively dig into the sub items of a list/dict
+                    # to turn the ObjectIds into DBRefs
+                    def _get_items_from_list(items):
+                        new_items = []
+                        for v in items:
+                            value = v
+                            if isinstance(v, dict):
+                                value = _get_items_from_dict(v)
+                            elif isinstance(v, list):
+                                value = _get_items_from_list(v)
+                            elif not isinstance(v, (DBRef, Document)):
+                                value = field.to_python(v)
+                            new_items.append(value)
+                        return new_items
 
-                        def _get_items(items):
-                            new_items = []
-                            for v in items:
-                                if isinstance(v, list):
-                                    new_items.append(_get_items(v))
-                                elif not isinstance(v, (DBRef, Document)):
-                                    new_items.append(field.to_python(v))
-                                else:
-                                    new_items.append(v)
-                            return new_items
-
-                        items = _get_items(items)
+                    def _get_items_from_dict(items):
+                        new_items = {}
+                        for k, v in items.iteritems():
+                            value = v
+                            if isinstance(v, list):
+                                value = _get_items_from_list(v)
+                            elif isinstance(v, dict):
+                                value = _get_items_from_dict(v)
+                            elif not isinstance(v, (DBRef, Document)):
+                                value = field.to_python(v)
+                            new_items[k] = value
+                        return new_items
+
+                    if not hasattr(items, 'items'):
+                        items = _get_items_from_list(items)
                     else:
-                        items = {
-                            k: (v if isinstance(v, (DBRef, Document))
-                                else field.to_python(v))
-                            for k, v in items.iteritems()
-                        }
+                        items = _get_items_from_dict(items)
 
         self.reference_map = self._find_references(items)
         self.object_map = self._fetch_objects(doc_type=doc_type)


Test patch:
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -1187,7 +1187,7 @@ def reset_post():
         # aka 'del list[index]'
         # aka 'operator.delitem(list, index)'
         reset_post()
-        del post.info[2] # del from middle ('2')
+        del post.info[2]  # del from middle ('2')
         self.assertEqual(post.info, ['0', '1', '3', '4', '5'])
         post.save()
         post.reload()
@@ -1197,7 +1197,7 @@ def reset_post():
         # aka 'del list[i:j]'
         # aka 'operator.delitem(list, slice(i,j))'
         reset_post()
-        del post.info[1:3] # removes '1', '2'
+        del post.info[1:3]  # removes '1', '2'
         self.assertEqual(post.info, ['0', '3', '4', '5'])
         post.save()
         post.reload()
@@ -1826,6 +1826,48 @@ class Simple(Document):
         with self.assertRaises(ValueError):
             e.update(set__mapping={"somestrings": ["foo", "bar", ]})
 
+    def test_dictfield_with_referencefield_complex_nesting_cases(self):
+        """Ensure complex nesting inside DictField handles dereferencing of ReferenceField(dbref=True | False)"""
+        # Relates to Issue #1453
+        class Doc(Document):
+            s = StringField()
+
+        class Simple(Document):
+            mapping0 = DictField(ReferenceField(Doc, dbref=True))
+            mapping1 = DictField(ReferenceField(Doc, dbref=False))
+            mapping2 = DictField(ListField(ReferenceField(Doc, dbref=True)))
+            mapping3 = DictField(ListField(ReferenceField(Doc, dbref=False)))
+            mapping4 = DictField(DictField(field=ReferenceField(Doc, dbref=True)))
+            mapping5 = DictField(DictField(field=ReferenceField(Doc, dbref=False)))
+            mapping6 = DictField(ListField(DictField(ReferenceField(Doc, dbref=True))))
+            mapping7 = DictField(ListField(DictField(ReferenceField(Doc, dbref=False))))
+            mapping8 = DictField(ListField(DictField(ListField(ReferenceField(Doc, dbref=True)))))
+            mapping9 = DictField(ListField(DictField(ListField(ReferenceField(Doc, dbref=False)))))
+
+        Doc.drop_collection()
+        Simple.drop_collection()
+
+        d = Doc(s='aa').save()
+        e = Simple()
+        e.mapping0['someint'] = e.mapping1['someint'] = d
+        e.mapping2['someint'] = e.mapping3['someint'] = [d]
+        e.mapping4['someint'] = e.mapping5['someint'] = {'d': d}
+        e.mapping6['someint'] = e.mapping7['someint'] = [{'d': d}]
+        e.mapping8['someint'] = e.mapping9['someint'] = [{'d': [d]}]
+        e.save()
+
+        s = Simple.objects.first()
+        self.assertIsInstance(s.mapping0['someint'], Doc)
+        self.assertIsInstance(s.mapping1['someint'], Doc)
+        self.assertIsInstance(s.mapping2['someint'][0], Doc)
+        self.assertIsInstance(s.mapping3['someint'][0], Doc)
+        self.assertIsInstance(s.mapping4['someint']['d'], Doc)
+        self.assertIsInstance(s.mapping5['someint']['d'], Doc)
+        self.assertIsInstance(s.mapping6['someint'][0]['d'], Doc)
+        self.assertIsInstance(s.mapping7['someint'][0]['d'], Doc)
+        self.assertIsInstance(s.mapping8['someint'][0]['d'][0], Doc)
+        self.assertIsInstance(s.mapping9['someint'][0]['d'][0], Doc)
+
     def test_mapfield(self):
         """Ensure that the MapField handles the declared type."""
         class Simple(Document):
@@ -4348,6 +4390,7 @@ class MyFailingDoc(Document):
             class MyFailingdoc2(Document):
                 emb = EmbeddedDocumentField('MyDoc')
 
+
 class CachedReferenceFieldTest(MongoDBTestCase):
 
     def test_cached_reference_field_get_and_save(self):