Repository: googleapis/google-cloud-python

Issue:
FireStore: Saving items that have both empty object values and SERVER_TIMESTAMP results in missing attributes.
When saving data to firestore with an attribute that is an empty `{}`:



if the date fields are `firestore.SERVER_TIMESTAMP` then the field with an empty `{}` is not persisted to the db but if the date fields are `Datetime` instances then the empty object fields are persisted. 

### OS type and version
1) MacOS 10.13.6 (17G2307)
2) DOCKER container: Linux version 4.9.93-linuxkit-aufs (root@856d34d1168e) (gcc version 6.4.0 (Alpin
e 6.4.0) ) #1 SMP Wed Jun 6 16:55:56 UTC 2018 

### Python version and virtual environment information `python --version`
Python 3.6.6

### google-cloud-python versions

google-api-core==1.3.0
google-auth==1.5.1
google-cloud-core==0.28.1
google-cloud-firestore==0.29.0
google-cloud-storage==1.11.0
google-resumable-media==0.3.1


### Stacktrace if available

NA

### Steps to reproduce

1) save to the db:
 
```python
data = {
     'date_created': firestore.SERVER_TIMESTAMP,
     'date_modified': firestore.SERVER_TIMESTAMP,
     'resources': {}
}

transaction.set(
    reference,
    data,
    option=db.write_option(exists=False),
)
```
2) look in firestore and notice `resources` attribute is not present.

save with `date_created` and `date_modified` = `datetime.datetime.utcnow()` and `resources` attributes is set with an empty object value.




Discussion:
I can verify that the ~~error is present~~ empty dict value is omitted in `google.cloud.firestore_v1._helpers.pbs_for_set` (i.e, we are not copying the empty dict *before* we send it on the wire to the back-end).
@lukesneeringer added an explicit skip of empty dicts in `_helpers.process_server_timestamp` in 8d03ef88.

@jba Can you confirm one way or another whether skipping empty dict values is correct?
@schmidt-sebastian
You can implement this in a couple different ways, but in order to get the conformance tests to pass, you need to send the absolute minimum requests that produces the desired outcome:

- For set(empty:{}), you need to send an write request that sets 'empty' to an empty map.
- For set(nonEmpty:{time:ServerTimestamp}), you only need to send a server transform, since the backend will implicitly create or convert `nonEmpty` to a map.

This cases are covered well by the conformance tests, and I am a little surprised that this is broken. Let me know if I am not understanding this correctly.
@schmidt-sebastian The reported case is a combination of the two.  The code in Python which generates the server transform ends up munging the data (to comb out the server timestamp fields) and ends up dropping empty maps.
I think the intent of the code in place is to avoid sending an empty map if the original map contained only server transforms:  it overlooks the case that the map was originally empty.
@jadekler to add this case to the conformance tests.

Resolving patch:
diff --git a/firestore/google/cloud/firestore_v1beta1/_helpers.py b/firestore/google/cloud/firestore_v1beta1/_helpers.py
--- a/firestore/google/cloud/firestore_v1beta1/_helpers.py
+++ b/firestore/google/cloud/firestore_v1beta1/_helpers.py
@@ -851,6 +851,9 @@ def process_server_timestamp(document_data, split_on_dots=True):
         else:
             top_level_path = FieldPath.from_string(field_name)
         if isinstance(value, dict):
+            if len(value) == 0:
+                actual_data[field_name] = value
+                continue
             sub_transform_paths, sub_data, sub_field_paths = (
                 process_server_timestamp(value, False))
             for sub_transform_path in sub_transform_paths:


Test patch:
diff --git a/firestore/tests/unit/test__helpers.py b/firestore/tests/unit/test__helpers.py
--- a/firestore/tests/unit/test__helpers.py
+++ b/firestore/tests/unit/test__helpers.py
@@ -1377,6 +1377,27 @@ def test_field_updates(self):
         expected_data = {'a': {'b': data['a']['b']}}
         self.assertEqual(actual_data, expected_data)
 
+    def test_field_updates_w_empty_value(self):
+        import collections
+        from google.cloud.firestore_v1beta1 import _helpers
+        from google.cloud.firestore_v1beta1.constants import SERVER_TIMESTAMP
+
+        # "Cheat" and use OrderedDict-s so that iteritems() is deterministic.
+        data = collections.OrderedDict((
+            ('a', {'b': 10}),
+            ('c.d', {'e': SERVER_TIMESTAMP}),
+            ('f.g', SERVER_TIMESTAMP),
+            ('h', {}),
+        ))
+        transform_paths, actual_data, field_paths = self._call_fut(data)
+        self.assertEqual(
+            transform_paths,
+            [_helpers.FieldPath('c', 'd', 'e'),
+             _helpers.FieldPath('f', 'g')])
+
+        expected_data = {'a': {'b': data['a']['b']}, 'h': {}}
+        self.assertEqual(actual_data, expected_data)
+
 
 class Test_canonicalize_field_paths(unittest.TestCase):
 
@@ -1460,78 +1481,108 @@ def test_it(self):
 class Test_pbs_for_set(unittest.TestCase):
 
     @staticmethod
-    def _call_fut(document_path, document_data, option):
+    def _call_fut(document_path, document_data, merge=False, exists=None):
         from google.cloud.firestore_v1beta1._helpers import pbs_for_set
 
-        return pbs_for_set(document_path, document_data, option)
+        return pbs_for_set(
+            document_path, document_data, merge=merge, exists=exists)
 
-    def _helper(self, merge=False, do_transform=False, **write_kwargs):
-        from google.cloud.firestore_v1beta1.constants import SERVER_TIMESTAMP
-        from google.cloud.firestore_v1beta1.gapic import enums
-        from google.cloud.firestore_v1beta1.proto import common_pb2
+    @staticmethod
+    def _make_write_w_document(document_path, **data):
         from google.cloud.firestore_v1beta1.proto import document_pb2
         from google.cloud.firestore_v1beta1.proto import write_pb2
+        from google.cloud.firestore_v1beta1._helpers import encode_dict
 
-        document_path = _make_ref_string(
-            u'little', u'town', u'of', u'ham')
-        field_name1 = 'cheese'
-        value1 = 1.5
-        field_name2 = 'crackers'
-        value2 = True
-        field_name3 = 'butter'
+        return write_pb2.Write(
+            update=document_pb2.Document(
+                name=document_path,
+                fields=encode_dict(data),
+            ),
+        )
+
+    @staticmethod
+    def _make_write_w_transform(document_path, fields):
+        from google.cloud.firestore_v1beta1.proto import write_pb2
+        from google.cloud.firestore_v1beta1.gapic import enums
+
+        server_val = enums.DocumentTransform.FieldTransform.ServerValue
+        transforms = [
+            write_pb2.DocumentTransform.FieldTransform(
+                field_path=field, set_to_server_value=server_val.REQUEST_TIME)
+            for field in fields
+        ]
+
+        return write_pb2.Write(
+            transform=write_pb2.DocumentTransform(
+                document=document_path,
+                field_transforms=transforms,
+            ),
+        )
+
+    def _helper(self, merge=False, do_transform=False, exists=None,
+                empty_val=False):
+        from google.cloud.firestore_v1beta1.constants import SERVER_TIMESTAMP
+        from google.cloud.firestore_v1beta1.proto import common_pb2
 
+        document_path = _make_ref_string(u'little', u'town', u'of', u'ham')
         document_data = {
-            field_name1: value1,
-            field_name2: value2,
+            'cheese': 1.5,
+            'crackers': True,
         }
+
         if do_transform:
-            document_data[field_name3] = SERVER_TIMESTAMP
+            document_data['butter'] = SERVER_TIMESTAMP
 
-        write_pbs = self._call_fut(document_path, document_data, merge)
+        if empty_val:
+            document_data['mustard'] = {}
 
-        expected_update_pb = write_pb2.Write(
-            update=document_pb2.Document(
-                name=document_path,
-                fields={
-                    field_name1: _value_pb(double_value=value1),
-                    field_name2: _value_pb(boolean_value=value2),
-                },
-            ),
-            **write_kwargs
-        )
-        expected_pbs = [expected_update_pb]
+        write_pbs = self._call_fut(
+            document_path, document_data, merge, exists)
+
+        if empty_val:
+            update_pb = self._make_write_w_document(
+                document_path, cheese=1.5, crackers=True, mustard={},
+            )
+        else:
+            update_pb = self._make_write_w_document(
+                document_path, cheese=1.5, crackers=True,
+            )
+        expected_pbs = [update_pb]
 
         if merge:
-            field_paths = [field_name1, field_name2]
-            mask = common_pb2.DocumentMask(field_paths=sorted(field_paths))
-            expected_pbs[0].update_mask.CopyFrom(mask)
+            field_paths = sorted(['cheese', 'crackers'])
+            update_pb.update_mask.CopyFrom(
+                common_pb2.DocumentMask(field_paths=field_paths))
+
+        if exists is not None:
+            update_pb.current_document.CopyFrom(
+                common_pb2.Precondition(exists=exists))
 
         if do_transform:
-            server_val = enums.DocumentTransform.FieldTransform.ServerValue
-            expected_transform_pb = write_pb2.Write(
-                transform=write_pb2.DocumentTransform(
-                    document=document_path,
-                    field_transforms=[
-                        write_pb2.DocumentTransform.FieldTransform(
-                            field_path=field_name3,
-                            set_to_server_value=server_val.REQUEST_TIME,
-                        ),
-                    ],
-                ),
-            )
-            expected_pbs.append(expected_transform_pb)
+            expected_pbs.append(
+                self._make_write_w_transform(document_path, fields=['butter']))
 
         self.assertEqual(write_pbs, expected_pbs)
 
-    def test_without_option(self):
+    def test_without_merge(self):
         self._helper()
 
-    def test_with_merge_option(self):
+    def test_with_merge(self):
         self._helper(merge=True)
 
-    def test_update_and_transform(self):
+    def test_with_exists_false(self):
+        self._helper(exists=False)
+
+    def test_with_exists_true(self):
+        self._helper(exists=True)
+
+    def test_w_transform(self):
         self._helper(do_transform=True)
 
+    def test_w_transform_and_empty_value(self):
+        # Exercise #5944
+        self._helper(do_transform=True, empty_val=True)
+
 
 class Test_pbs_for_update(unittest.TestCase):