Repository: googleapis/google-cloud-python

Issue:
Entity indexed=False not preserved in helpers.entity_from_protobuf
In `entity_from_protobuf` a key is [created](https://github.com/GoogleCloudPlatform/gcloud-python/blob/6bde37d1162749eea72a1b177f99750d575739d1/gcloud/datastore/helpers.py#L48) and the values are added via

``` python
    for property_pb in pb.property:
        value = _get_value_from_property_pb(property_pb)
        entity[property_pb.name] = value
```

However, `property_pb.value.indexed` is ignored and `entity` has the default empty tuple for `exclude_from_indexes`.

---

To see this, first save with `exclude_from_indexes` non-empty and inspect the protobuf values:

``` python
>>> from gcloud import datastore
>>> from gcloud.datastore.entity import Entity
>>> from gcloud.datastore.key import Key
>>>
>>> datastore.set_default_connection()
>>> datastore.set_default_dataset_id('foo')
>>>
>>> e = Entity(key=Key('Foo', 1), exclude_from_indexes=('foo', 'bar'))
>>> e.update({
...     'foo': 10,
...     'bar': 11,
...     'baz': 12,
... })
>>> e.save()
>>>
>>> entity_pb, = datastore.get_connection().lookup(
...         dataset_id=e.key.dataset_id,
...         key_pbs=[e.key.to_protobuf()],
... )
>>> for p in entity_pb.property:
...     print p.value.indexed
...
True
False
False
```

then use the key to `get` (which calls `get_entities` -> `entity_from_protobuf`), save the newly created (supposedly identical) `Entity`. After saving, check the same protobuf values:

``` python
>>> e_retrieved = e.key.get()
>>> e_retrieved.save()
>>>
>>> entity_pb, = datastore.get_connection().lookup(
...         dataset_id=e_retrieved.key.dataset_id,
...         key_pbs=[e_retrieved.key.to_protobuf()],
... )
>>> for p in entity_pb.property:
...     print p.value.indexed
...
True
True
True
```



Discussion:

Resolving patch:
diff --git a/gcloud/datastore/helpers.py b/gcloud/datastore/helpers.py
--- a/gcloud/datastore/helpers.py
+++ b/gcloud/datastore/helpers.py
@@ -16,6 +16,7 @@
 
 The non-private functions are part of the API.
 """
+
 __all__ = ('entity_from_protobuf', 'key_from_protobuf')
 
 import calendar
@@ -45,12 +46,30 @@ def entity_from_protobuf(pb):
     :returns: The entity derived from the protobuf.
     """
     key = key_from_protobuf(pb.key)
-    entity = Entity(key=key)
+    entity_props = {}
+    exclude_from_indexes = []
 
     for property_pb in pb.property:
         value = _get_value_from_property_pb(property_pb)
-        entity[property_pb.name] = value
-
+        entity_props[property_pb.name] = value
+
+        # Check if property_pb.value was indexed. Lists need to be
+        # special-cased and we require all `indexed` values in a list agree.
+        if isinstance(value, list):
+            indexed_values = set(value_pb.indexed
+                                 for value_pb in property_pb.value.list_value)
+            if len(indexed_values) != 1:
+                raise ValueError('For a list_value, subvalues must either all '
+                                 'be indexed or all excluded from indexes.')
+
+            if not indexed_values.pop():
+                exclude_from_indexes.append(property_pb.name)
+        else:
+            if not property_pb.value.indexed:
+                exclude_from_indexes.append(property_pb.name)
+
+    entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
+    entity.update(entity_props)
     return entity
 
 


Test patch:
diff --git a/gcloud/datastore/test_helpers.py b/gcloud/datastore/test_helpers.py
--- a/gcloud/datastore/test_helpers.py
+++ b/gcloud/datastore/test_helpers.py
@@ -42,15 +42,68 @@ def test_it(self):
         prop_pb = entity_pb.property.add()
         prop_pb.name = 'foo'
         prop_pb.value.string_value = 'Foo'
+
+        unindexed_prop_pb = entity_pb.property.add()
+        unindexed_prop_pb.name = 'bar'
+        unindexed_prop_pb.value.integer_value = 10
+        unindexed_prop_pb.value.indexed = False
+
+        list_prop_pb1 = entity_pb.property.add()
+        list_prop_pb1.name = 'baz'
+        list_pb1 = list_prop_pb1.value.list_value
+
+        unindexed_value_pb = list_pb1.add()
+        unindexed_value_pb.integer_value = 11
+        unindexed_value_pb.indexed = False
+
+        list_prop_pb2 = entity_pb.property.add()
+        list_prop_pb2.name = 'qux'
+        list_pb2 = list_prop_pb2.value.list_value
+
+        indexed_value_pb = list_pb2.add()
+        indexed_value_pb.integer_value = 12
+        indexed_value_pb.indexed = True
+
         entity = self._callFUT(entity_pb)
         self.assertEqual(entity.kind, _KIND)
-        self.assertEqual(entity['foo'], 'Foo')
+        self.assertEqual(entity.exclude_from_indexes,
+                         frozenset(['bar', 'baz']))
+        entity_props = dict(entity)
+        self.assertEqual(entity_props,
+                         {'foo': 'Foo', 'bar': 10, 'baz': [11], 'qux': [12]})
+
+        # Also check the key.
         key = entity.key
         self.assertEqual(key.dataset_id, _DATASET_ID)
         self.assertEqual(key.namespace, None)
         self.assertEqual(key.kind, _KIND)
         self.assertEqual(key.id, _ID)
 
+    def test_mismatched_value_indexed(self):
+        from gcloud.datastore import _datastore_v1_pb2 as datastore_pb
+
+        _DATASET_ID = 'DATASET'
+        _KIND = 'KIND'
+        _ID = 1234
+        entity_pb = datastore_pb.Entity()
+        entity_pb.key.partition_id.dataset_id = _DATASET_ID
+        entity_pb.key.path_element.add(kind=_KIND, id=_ID)
+
+        list_prop_pb = entity_pb.property.add()
+        list_prop_pb.name = 'baz'
+        list_pb = list_prop_pb.value.list_value
+
+        unindexed_value_pb1 = list_pb.add()
+        unindexed_value_pb1.integer_value = 10
+        unindexed_value_pb1.indexed = False
+
+        unindexed_value_pb2 = list_pb.add()
+        unindexed_value_pb2.integer_value = 11
+        unindexed_value_pb2.indexed = True
+
+        with self.assertRaises(ValueError):
+            self._callFUT(entity_pb)
+
 
 class Test_key_from_protobuf(unittest2.TestCase):