Repository: MongoEngine/mongoengine

Issue:
Cannot upsert a document with `GenericEmbeddedDocumentField`
When I use `set__` modifier to upsert a document with `GenericEmbeddedDocumentField`, mongoengine raises a strange error:
`mongoengine.errors.ValidationError: Invalid embedded document instance provided to an GenericEmbeddedDocumentField`

I want to know if this action is not allowed or this is a bug in mongoengine. If this is not allowed, what is the proper way to upsert this kind of document?

Thanks!

**Environment**
mongoengine-0.10.0
pymongo-2.8.1
MongoDB v2.6.7
Python 2.7.8

**Test Script**

```
from mongoengine import *

connect()

class Bar(EmbeddedDocument):
  val = StringField()


class Foo(Document):
  user = StringField()
  bar = GenericEmbeddedDocumentField()

Foo.objects(user='user').modify(set__bar=Bar(val='test'), upsert=True)  # This failed
Foo.objects(user='user').update(set__bar=Bar(val='test'), upsert=True)  # This failed too
```

**Traceback**

```
Traceback (most recent call last):
  File "generic_bug.py", line 13, in <module>
    Foo.objects(user='user').modify(set__bar=Bar(val='test'), upsert=True)  # This failed
  File "*******/virtualenv/lib/python2.7/site-packages/mongoengine/queryset/base.py", line 523, in modify
    update = transform.update(queryset._document, **update)
  File "*******/virtualenv/lib/python2.7/site-packages/mongoengine/queryset/transform.py", line 249, in update
    value = field.prepare_query_value(op, value)
  File "*******/virtualenv/lib/python2.7/site-packages/mongoengine/fields.py", line 587, in prepare_query_value
    return super(GenericEmbeddedDocumentField, self).prepare_query_value(op, self.to_mongo(value))
  File "*******/virtualenv/lib/python2.7/site-packages/mongoengine/base/fields.py", line 160, in prepare_query_value
    self.validate(value)
  File "*******/virtualenv/lib/python2.7/site-packages/mongoengine/fields.py", line 598, in validate
    self.error('Invalid embedded document instance provided to an '
  File "*******/virtualenv/lib/python2.7/site-packages/mongoengine/base/fields.py", line 144, in error
    raise ValidationError(message, errors=errors, field_name=field_name)
mongoengine.errors.ValidationError: Invalid embedded document instance provided to an GenericEmbeddedDocumentField
```



Discussion:
This also happens with `pull` on `ListField(GenericEmbeddedDocumentField)`.  Same error.

```
python 3.5.2
mongo 3.2

pymongo (3.3.0)
mongoengine (0.10.6)
```

```
File "libs/payments/models/__init__.py", line 91, in __call__
File "lib/python3.5/site-packages/mongoengine/queryset/base.py", line 516, in update_one
  upsert=upsert, multi=False, write_concern=write_concern, **update)
File "lib/python3.5/site-packages/mongoengine/queryset/base.py", line 450, in update
  update = transform.update(queryset._document, **update)
File "lib/python3.5/site-packages/mongoengine/queryset/transform.py", line 254, in update
  value = field.prepare_query_value(op, value)
File "lib/python3.5/site-packages/mongoengine/fields.py", line 706, in prepare_query_value
  return self.field.prepare_query_value(op, value)
File "lib/python3.5/site-packages/mongoengine/fields.py", line 588, in prepare_query_value
  return super(GenericEmbeddedDocumentField, self).prepare_query_value(op, self.to_mongo(value))
File "lib/python3.5/site-packages/mongoengine/base/fields.py", line 171, in prepare_query_value
  self.validate(value)
File "lib/python3.5/site-packages/mongoengine/fields.py", line 599, in validate
  self.error('Invalid embedded document instance provided to an '
File "lib/python3.5/site-packages/mongoengine/base/fields.py", line 155, in error
  raise ValidationError(message, errors=errors, field_name=field_name)
mongoengine.errors.ValidationError: Invalid embedded document instance provided to an GenericEmbeddedDocumentField
```

Example

``` python
class Foo(EmbeddedDocument):
   somefield = StringField()

class Bar(Document):
   foos = ListField(GenericEmbeddedDocumentField(choices=[
       Foo,
   ]))
```

``` python
foo = Foo(somefield='hello')
Bar.objects.create(
   foos=[foo],
)
Bar.objects.filter().update_one(pull__foos=foo)
```

anyone looking at this?
I can work on this but does it have to behave like as these guys requested?
What do you think about that? Should I send PR for this? @wojcikstefan 
I did a simple investigation. The problem occurs for any type of update document with generic embedded field when value of that field is changed. The exception raise in validate method of the class GenericEmbeddedDocumentField because the value parameter is of type bson.son.SON.

Resolving patch:
diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -13,6 +13,7 @@
 from mongoengine.base.common import get_document
 from mongoengine.base.datastructures import (BaseDict, BaseList,
                                              EmbeddedDocumentList,
+                                             LazyReference,
                                              StrictDict)
 from mongoengine.base.fields import ComplexBaseField
 from mongoengine.common import _import_class
@@ -488,7 +489,7 @@ def _clear_changed_fields(self):
                 else:
                     data = getattr(data, part, None)
 
-                if hasattr(data, '_changed_fields'):
+                if not isinstance(data, LazyReference) and hasattr(data, '_changed_fields'):
                     if getattr(data, '_is_document', False):
                         continue
 
diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py
--- a/mongoengine/dereference.py
+++ b/mongoengine/dereference.py
@@ -3,6 +3,7 @@
 
 from mongoengine.base import (BaseDict, BaseList, EmbeddedDocumentList,
                               TopLevelDocumentMetaclass, get_document)
+from mongoengine.base.datastructures import LazyReference
 from mongoengine.connection import get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.fields import DictField, ListField, MapField, ReferenceField
@@ -99,7 +100,10 @@ def _find_references(self, items, depth=0):
             if isinstance(item, (Document, EmbeddedDocument)):
                 for field_name, field in item._fields.iteritems():
                     v = item._data.get(field_name, None)
-                    if isinstance(v, DBRef):
+                    if isinstance(v, LazyReference):
+                        # LazyReference inherits DBRef but should not be dereferenced here !
+                        continue
+                    elif isinstance(v, DBRef):
                         reference_map.setdefault(field.document_type, set()).add(v.id)
                     elif isinstance(v, (dict, SON)) and '_ref' in v:
                         reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)
@@ -110,6 +114,9 @@ def _find_references(self, items, depth=0):
                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
                                 key = field_cls
                             reference_map.setdefault(key, set()).update(refs)
+            elif isinstance(item, LazyReference):
+                # LazyReference inherits DBRef but should not be dereferenced here !
+                continue
             elif isinstance(item, DBRef):
                 reference_map.setdefault(item.collection, set()).add(item.id)
             elif isinstance(item, (dict, SON)) and '_ref' in item:
diff --git a/mongoengine/fields.py b/mongoengine/fields.py
--- a/mongoengine/fields.py
+++ b/mongoengine/fields.py
@@ -28,6 +28,7 @@
 from mongoengine.base import (BaseDocument, BaseField, ComplexBaseField,
                               GeoJsonBaseField, LazyReference, ObjectIdField,
                               get_document)
+from mongoengine.common import _import_class
 from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db
 from mongoengine.document import Document, EmbeddedDocument
 from mongoengine.errors import DoesNotExist, InvalidQueryError, ValidationError
@@ -688,6 +689,11 @@ def to_python(self, value):
         return value
 
     def validate(self, value, clean=True):
+        if self.choices and isinstance(value, SON):
+            for choice in self.choices:
+                if value['_cls'] == choice._class_name:
+                    return True
+
         if not isinstance(value, EmbeddedDocument):
             self.error('Invalid embedded document instance provided to an '
                        'GenericEmbeddedDocumentField')
@@ -705,7 +711,6 @@ def lookup_member(self, member_name):
     def to_mongo(self, document, use_db_field=True, fields=None):
         if document is None:
             return None
-
         data = document.to_mongo(use_db_field, fields)
         if '_cls' not in data:
             data['_cls'] = document._class_name
@@ -789,6 +794,17 @@ def __init__(self, field=None, **kwargs):
         kwargs.setdefault('default', lambda: [])
         super(ListField, self).__init__(**kwargs)
 
+    def __get__(self, instance, owner):
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+        value = instance._data.get(self.name)
+        LazyReferenceField = _import_class('LazyReferenceField')
+        GenericLazyReferenceField = _import_class('GenericLazyReferenceField')
+        if isinstance(self.field, (LazyReferenceField, GenericLazyReferenceField)) and value:
+            instance._data[self.name] = [self.field.build_lazyref(x) for x in value]
+        return super(ListField, self).__get__(instance, owner)
+
     def validate(self, value):
         """Make sure that a list of valid fields is being used."""
         if (not isinstance(value, (list, tuple, QuerySet)) or
@@ -2211,17 +2227,10 @@ def document_type(self):
                 self.document_type_obj = get_document(self.document_type_obj)
         return self.document_type_obj
 
-    def __get__(self, instance, owner):
-        """Descriptor to allow lazy dereferencing."""
-        if instance is None:
-            # Document class being used rather than a document object
-            return self
-
-        value = instance._data.get(self.name)
+    def build_lazyref(self, value):
         if isinstance(value, LazyReference):
             if value.passthrough != self.passthrough:
-                instance._data[self.name] = LazyReference(
-                    value.document_type, value.pk, passthrough=self.passthrough)
+                value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
         elif value is not None:
             if isinstance(value, self.document_type):
                 value = LazyReference(self.document_type, value.pk, passthrough=self.passthrough)
@@ -2230,6 +2239,16 @@ def __get__(self, instance, owner):
             else:
                 # value is the primary key of the referenced document
                 value = LazyReference(self.document_type, value, passthrough=self.passthrough)
+        return value
+
+    def __get__(self, instance, owner):
+        """Descriptor to allow lazy dereferencing."""
+        if instance is None:
+            # Document class being used rather than a document object
+            return self
+
+        value = self.build_lazyref(instance._data.get(self.name))
+        if value:
             instance._data[self.name] = value
 
         return super(LazyReferenceField, self).__get__(instance, owner)
@@ -2254,7 +2273,7 @@ def to_mongo(self, value):
 
     def validate(self, value):
         if isinstance(value, LazyReference):
-            if not issubclass(value.document_type, self.document_type):
+            if value.collection != self.document_type._get_collection_name():
                 self.error('Reference must be on a `%s` document.' % self.document_type)
             pk = value.pk
         elif isinstance(value, self.document_type):
@@ -2314,23 +2333,26 @@ def __init__(self, *args, **kwargs):
 
     def _validate_choices(self, value):
         if isinstance(value, LazyReference):
-            value = value.document_type
+            value = value.document_type._class_name
         super(GenericLazyReferenceField, self)._validate_choices(value)
 
-    def __get__(self, instance, owner):
-        if instance is None:
-            return self
-
-        value = instance._data.get(self.name)
+    def build_lazyref(self, value):
         if isinstance(value, LazyReference):
             if value.passthrough != self.passthrough:
-                instance._data[self.name] = LazyReference(
-                    value.document_type, value.pk, passthrough=self.passthrough)
+                value = LazyReference(value.document_type, value.pk, passthrough=self.passthrough)
         elif value is not None:
             if isinstance(value, (dict, SON)):
                 value = LazyReference(get_document(value['_cls']), value['_ref'].id, passthrough=self.passthrough)
             elif isinstance(value, Document):
                 value = LazyReference(type(value), value.pk, passthrough=self.passthrough)
+        return value
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+
+        value = self.build_lazyref(instance._data.get(self.name))
+        if value:
             instance._data[self.name] = value
 
         return super(GenericLazyReferenceField, self).__get__(instance, owner)
@@ -2348,7 +2370,7 @@ def to_mongo(self, document):
         if isinstance(document, LazyReference):
             return SON((
                 ('_cls', document.document_type._class_name),
-                ('_ref', document)
+                ('_ref', DBRef(document.document_type._get_collection_name(), document.pk))
             ))
         else:
             return super(GenericLazyReferenceField, self).to_mongo(document)


Test patch:
diff --git a/tests/fields/fields.py b/tests/fields/fields.py
--- a/tests/fields/fields.py
+++ b/tests/fields/fields.py
@@ -4871,6 +4871,48 @@ class Animal(Document):
         self.assertNotEqual(animal, other_animalref)
         self.assertNotEqual(other_animalref, animal)
 
+    def test_lazy_reference_embedded(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class EmbeddedOcurrence(EmbeddedDocument):
+            in_list = ListField(LazyReferenceField(Animal))
+            direct = LazyReferenceField(Animal)
+
+        class Ocurrence(Document):
+            in_list = ListField(LazyReferenceField(Animal))
+            in_embedded = EmbeddedDocumentField(EmbeddedOcurrence)
+            direct = LazyReferenceField(Animal)
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal1 = Animal('doggo').save()
+        animal2 = Animal('cheeta').save()
+
+        def check_fields_type(occ):
+            self.assertIsInstance(occ.direct, LazyReference)
+            for elem in occ.in_list:
+                self.assertIsInstance(elem, LazyReference)
+            self.assertIsInstance(occ.in_embedded.direct, LazyReference)
+            for elem in occ.in_embedded.in_list:
+                self.assertIsInstance(elem, LazyReference)
+
+        occ = Ocurrence(
+            in_list=[animal1, animal2],
+            in_embedded={'in_list': [animal1, animal2], 'direct': animal1},
+            direct=animal1
+        ).save()
+        check_fields_type(occ)
+        occ.reload()
+        check_fields_type(occ)
+        occ.direct = animal1.id
+        occ.in_list = [animal1.id, animal2.id]
+        occ.in_embedded.direct = animal1.id
+        occ.in_embedded.in_list = [animal1.id, animal2.id]
+        check_fields_type(occ)
+
 
 class GenericLazyReferenceFieldTest(MongoDBTestCase):
     def test_generic_lazy_reference_simple(self):
@@ -5051,6 +5093,50 @@ class Ocurrence(Document):
         p = Ocurrence.objects.get()
         self.assertIs(p.animal, None)
 
+    def test_generic_lazy_reference_embedded(self):
+        class Animal(Document):
+            name = StringField()
+            tag = StringField()
+
+        class EmbeddedOcurrence(EmbeddedDocument):
+            in_list = ListField(GenericLazyReferenceField())
+            direct = GenericLazyReferenceField()
+
+        class Ocurrence(Document):
+            in_list = ListField(GenericLazyReferenceField())
+            in_embedded = EmbeddedDocumentField(EmbeddedOcurrence)
+            direct = GenericLazyReferenceField()
+
+        Animal.drop_collection()
+        Ocurrence.drop_collection()
+
+        animal1 = Animal('doggo').save()
+        animal2 = Animal('cheeta').save()
+
+        def check_fields_type(occ):
+            self.assertIsInstance(occ.direct, LazyReference)
+            for elem in occ.in_list:
+                self.assertIsInstance(elem, LazyReference)
+            self.assertIsInstance(occ.in_embedded.direct, LazyReference)
+            for elem in occ.in_embedded.in_list:
+                self.assertIsInstance(elem, LazyReference)
+
+        occ = Ocurrence(
+            in_list=[animal1, animal2],
+            in_embedded={'in_list': [animal1, animal2], 'direct': animal1},
+            direct=animal1
+        ).save()
+        check_fields_type(occ)
+        occ.reload()
+        check_fields_type(occ)
+        animal1_ref = {'_cls': 'Animal', '_ref': DBRef(animal1._get_collection_name(), animal1.pk)}
+        animal2_ref = {'_cls': 'Animal', '_ref': DBRef(animal2._get_collection_name(), animal2.pk)}
+        occ.direct = animal1_ref
+        occ.in_list = [animal1_ref, animal2_ref]
+        occ.in_embedded.direct = animal1_ref
+        occ.in_embedded.in_list = [animal1_ref, animal2_ref]
+        check_fields_type(occ)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py
--- a/tests/queryset/queryset.py
+++ b/tests/queryset/queryset.py
@@ -2086,6 +2086,23 @@ class Site(Document):
             Site.objects(id=s.id).update_one(
                 pull_all__collaborators__helpful__user=['Ross'])
 
+    def test_pull_in_genericembedded_field(self):
+
+        class Foo(EmbeddedDocument):
+            name = StringField()
+
+        class Bar(Document):
+            foos = ListField(GenericEmbeddedDocumentField(
+                choices=[Foo, ]))
+
+        Bar.drop_collection()
+
+        foo = Foo(name="bar")
+        bar = Bar(foos=[foo]).save()
+        Bar.objects(id=bar.id).update(pull__foos=foo)
+        bar.reload()
+        self.assertEqual(len(bar.foos), 0)
+
     def test_update_one_pop_generic_reference(self):
 
         class BlogTag(Document):
@@ -2179,6 +2196,24 @@ class Message(Document):
         self.assertEqual(message.authors[1].name, "Ross")
         self.assertEqual(message.authors[2].name, "Adam")
 
+    def test_set_generic_embedded_documents(self):
+
+        class Bar(EmbeddedDocument):
+            name = StringField()
+
+        class User(Document):
+            username = StringField()
+            bar = GenericEmbeddedDocumentField(choices=[Bar,])
+
+        User.drop_collection()
+
+        User(username='abc').save()
+        User.objects(username='abc').update(
+            set__bar=Bar(name='test'), upsert=True)
+
+        user = User.objects(username='abc').first()
+        self.assertEqual(user.bar.name, "test")
+
     def test_reload_embedded_docs_instance(self):
 
         class SubDoc(EmbeddedDocument):