Repository: googleapis/google-cloud-python

Issue:
Automagically convert dicts to datastore entities.
I just ran into the following error for the X'th time:

```python
    client.put(entity)
  File "lib/python3.6/site-packages/google/cloud/datastore/client.py", line 384, in put
    self.put_multi(entities=[entity])
  File "lib/python3.6/site-packages/google/cloud/datastore/client.py", line 408, in put_multi
    current.put(entity)
  File "lib/python3.6/site-packages/google/cloud/datastore/batch.py", line 199, in put
    _assign_entity_to_pb(entity_pb, entity)
  File "lib/python3.6/site-packages/google/cloud/datastore/batch.py", line 319, in _assign_entity_to_pb
    bare_entity_pb = helpers.entity_to_protobuf(entity)
  File "lib/python3.6/site-packages/google/cloud/datastore/helpers.py", line 219, in entity_to_protobuf
    _set_protobuf_value(value_pb, value)
  File "lib/python3.6/site-packages/google/cloud/datastore/helpers.py", line 420, in _set_protobuf_value
    _set_protobuf_value(i_pb, item)
  File "lib/python3.6/site-packages/google/cloud/datastore/helpers.py", line 408, in _set_protobuf_value
    attr, val = _pb_attr_value(val)
  File "lib/python3.6/site-packages/google/cloud/datastore/helpers.py", line 325, in _pb_attr_value
    raise ValueError("Unknown protobuf attr type %s" % type(val))
ValueError: Unknown protobuf attr type <class 'dict'>
```

The cause is pretty obvious, the following code snippet triggers this error:

```python
entity = datastore.Entity(client.key('Resource'))
entity.update(foo={'bar': 'baz'})
client.put(entity)
```

The code crashes, because `dict` type properties are not supported. Instead they should be converted to an `Entity`.

I think it's cumbersome having to convert `dict` types to `Entity` types every time. I've written a function that does the conversion recursively for production code, but for simple scripts this is still an issue.

I think it would be nice if this library would just do the conversion from `dict` to `Entity` automagically.


Discussion:
@remcohaszing It's definitely worth considering! Thanks for filing.

@lukesneeringer @jonparrott WDYT?
I am supportive.
Working on it now. Hopefully it'll be quick.

Resolving patch:
diff --git a/datastore/google/cloud/datastore/entity.py b/datastore/google/cloud/datastore/entity.py
--- a/datastore/google/cloud/datastore/entity.py
+++ b/datastore/google/cloud/datastore/entity.py
@@ -83,10 +83,12 @@ class Entity(dict):
     * :class:`~google.cloud.datastore.helpers.GeoPoint`
     * :data:`None`
 
-    In addition, two container types are supported:
+    In addition, three container types are supported:
 
     * :class:`list`
     * :class:`~google.cloud.datastore.entity.Entity`
+    * :class:`dict` (will just be treated like an ``Entity`` without
+      a key or ``exclude_from_indexes``)
 
     Each entry in a list must be one of the value types (basic or
     container) and each value in an
diff --git a/datastore/google/cloud/datastore/helpers.py b/datastore/google/cloud/datastore/helpers.py
--- a/datastore/google/cloud/datastore/helpers.py
+++ b/datastore/google/cloud/datastore/helpers.py
@@ -290,8 +290,11 @@ def _pb_attr_value(val):
     >>> _pb_attr_value('my_string')
     ('string_value', 'my_string')
 
-    :type val: `datetime.datetime`, :class:`google.cloud.datastore.key.Key`,
-               bool, float, integer, string
+    :type val:
+        :class:`datetime.datetime`, :class:`google.cloud.datastore.key.Key`,
+        bool, float, integer, bytes, str, unicode,
+        :class:`google.cloud.datastore.entity.Entity`, dict, list,
+        :class:`google.cloud.datastore.helpers.GeoPoint`, NoneType
     :param val: The value to be scrutinized.
 
     :rtype: tuple
@@ -315,6 +318,10 @@ def _pb_attr_value(val):
         name, value = 'blob', val
     elif isinstance(val, Entity):
         name, value = 'entity', val
+    elif isinstance(val, dict):
+        entity_val = Entity(key=None)
+        entity_val.update(val)
+        name, value = 'entity', entity_val
     elif isinstance(val, list):
         name, value = 'array', val
     elif isinstance(val, GeoPoint):
@@ -322,7 +329,7 @@ def _pb_attr_value(val):
     elif val is None:
         name, value = 'null', struct_pb2.NULL_VALUE
     else:
-        raise ValueError("Unknown protobuf attr type %s" % type(val))
+        raise ValueError('Unknown protobuf attr type', type(val))
 
     return name + '_value', value
 


Test patch:
diff --git a/datastore/tests/unit/test_helpers.py b/datastore/tests/unit/test_helpers.py
--- a/datastore/tests/unit/test_helpers.py
+++ b/datastore/tests/unit/test_helpers.py
@@ -199,7 +199,7 @@ def _call_fut(self, entity):
 
         return entity_to_protobuf(entity)
 
-    def _compareEntityProto(self, entity_pb1, entity_pb2):
+    def _compare_entity_proto(self, entity_pb1, entity_pb2):
         from google.cloud.datastore.helpers import _property_tuples
 
         self.assertEqual(entity_pb1.key, entity_pb2.key)
@@ -212,8 +212,8 @@ def _compareEntityProto(self, entity_pb1, entity_pb2):
             self.assertEqual(name1, name2)
             if val1.HasField('entity_value'):  # Message field (Entity)
                 self.assertEqual(val1.meaning, val2.meaning)
-                self._compareEntityProto(val1.entity_value,
-                                         val2.entity_value)
+                self._compare_entity_proto(
+                    val1.entity_value, val2.entity_value)
             else:
                 self.assertEqual(val1, val2)
 
@@ -223,7 +223,7 @@ def test_empty(self):
 
         entity = Entity()
         entity_pb = self._call_fut(entity)
-        self._compareEntityProto(entity_pb, entity_pb2.Entity())
+        self._compare_entity_proto(entity_pb, entity_pb2.Entity())
 
     def test_key_only(self):
         from google.cloud.proto.datastore.v1 import entity_pb2
@@ -242,7 +242,7 @@ def test_key_only(self):
         path_elt.kind = kind
         path_elt.name = name
 
-        self._compareEntityProto(entity_pb, expected_pb)
+        self._compare_entity_proto(entity_pb, expected_pb)
 
     def test_simple_fields(self):
         from google.cloud.proto.datastore.v1 import entity_pb2
@@ -262,7 +262,7 @@ def test_simple_fields(self):
         val_pb2 = _new_value_pb(expected_pb, name2)
         val_pb2.string_value = value2
 
-        self._compareEntityProto(entity_pb, expected_pb)
+        self._compare_entity_proto(entity_pb, expected_pb)
 
     def test_with_empty_list(self):
         from google.cloud.proto.datastore.v1 import entity_pb2
@@ -272,7 +272,7 @@ def test_with_empty_list(self):
         entity['foo'] = []
         entity_pb = self._call_fut(entity)
 
-        self._compareEntityProto(entity_pb, entity_pb2.Entity())
+        self._compare_entity_proto(entity_pb, entity_pb2.Entity())
 
     def test_inverts_to_protobuf(self):
         from google.cloud.proto.datastore.v1 import entity_pb2
@@ -325,7 +325,7 @@ def test_inverts_to_protobuf(self):
 
         # NOTE: entity_to_protobuf() strips the project so we "cheat".
         new_pb.key.partition_id.project_id = project
-        self._compareEntityProto(original_pb, new_pb)
+        self._compare_entity_proto(original_pb, new_pb)
 
     def test_meaning_with_change(self):
         from google.cloud.proto.datastore.v1 import entity_pb2
@@ -343,7 +343,7 @@ def test_meaning_with_change(self):
         value_pb.integer_value = value
         # NOTE: No meaning is used since the value differs from the
         #       value stored.
-        self._compareEntityProto(entity_pb, expected_pb)
+        self._compare_entity_proto(entity_pb, expected_pb)
 
     def test_variable_meanings(self):
         from google.cloud.proto.datastore.v1 import entity_pb2
@@ -369,7 +369,78 @@ def test_variable_meanings(self):
         value2 = value_pb.array_value.values.add()
         value2.integer_value = values[2]
 
-        self._compareEntityProto(entity_pb, expected_pb)
+        self._compare_entity_proto(entity_pb, expected_pb)
+
+    def test_dict_to_entity(self):
+        from google.cloud.proto.datastore.v1 import entity_pb2
+        from google.cloud.datastore.entity import Entity
+
+        entity = Entity()
+        entity['a'] = {'b': u'c'}
+        entity_pb = self._call_fut(entity)
+
+        expected_pb = entity_pb2.Entity(
+            properties={
+                'a': entity_pb2.Value(
+                    entity_value=entity_pb2.Entity(
+                        properties={
+                            'b': entity_pb2.Value(
+                                string_value='c',
+                            ),
+                        },
+                    ),
+                ),
+            },
+        )
+        self.assertEqual(entity_pb, expected_pb)
+
+    def test_dict_to_entity_recursive(self):
+        from google.cloud.proto.datastore.v1 import entity_pb2
+        from google.cloud.datastore.entity import Entity
+
+        entity = Entity()
+        entity['a'] = {
+            'b': {
+                'c': {
+                    'd': 1.25,
+                },
+                'e': True,
+            },
+            'f': 10,
+        }
+        entity_pb = self._call_fut(entity)
+
+        b_entity_pb = entity_pb2.Entity(
+            properties={
+                'c': entity_pb2.Value(
+                    entity_value=entity_pb2.Entity(
+                        properties={
+                            'd': entity_pb2.Value(
+                                double_value=1.25,
+                            ),
+                        },
+                    ),
+                ),
+                'e': entity_pb2.Value(boolean_value=True),
+            }
+        )
+        expected_pb = entity_pb2.Entity(
+            properties={
+                'a': entity_pb2.Value(
+                    entity_value=entity_pb2.Entity(
+                        properties={
+                            'b': entity_pb2.Value(
+                                entity_value=b_entity_pb,
+                            ),
+                            'f': entity_pb2.Value(
+                                integer_value=10,
+                            ),
+                        },
+                    ),
+                ),
+            },
+        )
+        self.assertEqual(entity_pb, expected_pb)
 
 
 class Test_key_from_protobuf(unittest.TestCase):
@@ -516,6 +587,18 @@ def test_entity(self):
         self.assertEqual(name, 'entity_value')
         self.assertIs(value, entity)
 
+    def test_dict(self):
+        from google.cloud.datastore.entity import Entity
+
+        orig_value = {'richard': b'feynman'}
+        name, value = self._call_fut(orig_value)
+        self.assertEqual(name, 'entity_value')
+        self.assertIsInstance(value, Entity)
+        self.assertIsNone(value.key)
+        self.assertEqual(value._meanings, {})
+        self.assertEqual(value.exclude_from_indexes, set())
+        self.assertEqual(dict(value), orig_value)
+
     def test_array(self):
         values = ['a', 0, 3.14]
         name, value = self._call_fut(values)