Repository: googleapis/google-cloud-python

Issue:
Firestore SERVER_TIMESTAMP only works with "set" and not "add"
Mac OSX Mojave 10.14
Python 3.7.0
google-cloud-firestore==0.30.0
#### Steps to reproduce

  1. Run the two code example below. `set` works but `add` doesn't. The expected behavior is that they both parse the firestore.SERVER_TIMESTAMP constant the same way.

#### Code example

this fails
```python
rows = firestore.client().collection('rows')
rows.add(
                dict(
                    created_at=firestore.SERVER_TIMESTAMP,
                    updated_at=firestore.SERVER_TIMESTAMP))
```

this works
```python
rows = firestore.client().collection('rows')
rows.document('test_id').set(
                dict(
                    created_at=firestore.SERVER_TIMESTAMP,
                    updated_at=firestore.SERVER_TIMESTAMP))
```

Seems like it was forgotten to add SERVER_TIMESTAMP parsing to the `add` method and it's only on the `set` method.

#### Stack trace
```
17:30:32 tables.1    |   File "app.py", line 1337, in mutate
17:30:32 tables.1    |     updated_at=firestore.SERVER_TIMESTAMP))
17:30:32 tables.1    |   File "env/lib/python3.7/site-packages/google/cloud/firestore_v1beta1/collection.py", line 160, in add
17:30:32 tables.1    |     fields=_helpers.encode_dict(document_data))
17:30:32 tables.1    |   File "env/lib/python3.7/site-packages/google/cloud/firestore_v1beta1/_helpers.py", line 536, in encode_dict
17:30:32 tables.1    |     for key, value in six.iteritems(values_dict)
17:30:32 tables.1    |   File "env/lib/python3.7/site-packages/google/cloud/firestore_v1beta1/_helpers.py", line 536, in <dictcomp>
17:30:32 tables.1    |     for key, value in six.iteritems(values_dict)
17:30:32 tables.1    |   File "env/lib/python3.7/site-packages/google/cloud/firestore_v1beta1/_helpers.py", line 520, in encode_value
17:30:32 tables.1    |     'Invalid type', type(value))
17:30:32 tables.1    | TypeError: ('Cannot convert to a Firestore Value', <object object at 0x104fdebe0>, 'Invalid type', <class 'object'>)
```


Discussion:
The bug in `CollectionReference.add` is in the branch where no document ID is passed: it doesn't separate out the transforms from the "real" document data.  Fixing it will involve using `_helpers.DocumentExtractor` (to do the separation) and making a separate `set()` call, passing the transformed values after the initial document creation.

Resolving patch:
diff --git a/firestore/google/cloud/firestore_v1beta1/collection.py b/firestore/google/cloud/firestore_v1beta1/collection.py
--- a/firestore/google/cloud/firestore_v1beta1/collection.py
+++ b/firestore/google/cloud/firestore_v1beta1/collection.py
@@ -159,9 +159,8 @@ def add(self, document_data, document_id=None):
         """
         if document_id is None:
             parent_path, expected_prefix = self._parent_info()
-            document_pb = document_pb2.Document(
-                fields=_helpers.encode_dict(document_data)
-            )
+
+            document_pb = document_pb2.Document()
 
             created_document_pb = self._client._firestore_api.create_document(
                 parent_path,
@@ -174,7 +173,8 @@ def add(self, document_data, document_id=None):
 
             new_document_id = _helpers.get_doc_id(created_document_pb, expected_prefix)
             document_ref = self.document(new_document_id)
-            return created_document_pb.update_time, document_ref
+            set_result = document_ref.set(document_data)
+            return set_result.update_time, document_ref
         else:
             document_ref = self.document(document_id)
             write_result = document_ref.create(document_data)


Test patch:
diff --git a/firestore/tests/system.py b/firestore/tests/system.py
--- a/firestore/tests/system.py
+++ b/firestore/tests/system.py
@@ -409,7 +409,6 @@ def test_collection_add(client, cleanup):
     cleanup(document_ref1)
     snapshot1 = document_ref1.get()
     assert snapshot1.to_dict() == data1
-    assert snapshot1.create_time == update_time1
     assert snapshot1.update_time == update_time1
     assert RANDOM_ID_REGEX.match(document_ref1.id)
 
@@ -429,7 +428,6 @@ def test_collection_add(client, cleanup):
     cleanup(document_ref3)
     snapshot3 = document_ref3.get()
     assert snapshot3.to_dict() == data3
-    assert snapshot3.create_time == update_time3
     assert snapshot3.update_time == update_time3
     assert RANDOM_ID_REGEX.match(document_ref3.id)
 
diff --git a/firestore/tests/unit/test_collection.py b/firestore/tests/unit/test_collection.py
--- a/firestore/tests/unit/test_collection.py
+++ b/firestore/tests/unit/test_collection.py
@@ -191,11 +191,21 @@ def test__parent_info_nested(self):
 
     def test_add_auto_assigned(self):
         from google.cloud.firestore_v1beta1.proto import document_pb2
-        from google.cloud.firestore_v1beta1 import _helpers
         from google.cloud.firestore_v1beta1.document import DocumentReference
+        from google.cloud.firestore_v1beta1 import SERVER_TIMESTAMP
+        from google.cloud.firestore_v1beta1._helpers import pbs_for_set_no_merge
 
         # Create a minimal fake GAPIC add attach it to a real client.
-        firestore_api = mock.Mock(spec=["create_document"])
+        firestore_api = mock.Mock(spec=["create_document", "commit"])
+        write_result = mock.Mock(
+            update_time=mock.sentinel.update_time, spec=["update_time"]
+        )
+        commit_response = mock.Mock(
+            write_results=[write_result],
+            spec=["write_results", "commit_time"],
+            commit_time=mock.sentinel.commit_time,
+        )
+        firestore_api.commit.return_value = commit_response
         create_doc_response = document_pb2.Document()
         firestore_api.create_document.return_value = create_doc_response
         client = _make_client()
@@ -212,20 +222,19 @@ def test_add_auto_assigned(self):
         create_doc_response.update_time.FromDatetime(datetime.datetime.utcnow())
         firestore_api.create_document.return_value = create_doc_response
 
-        # Actually call add() on our collection.
-        document_data = {"been": "here"}
+        # Actually call add() on our collection; include a transform to make
+        # sure transforms during adds work.
+        document_data = {"been": "here", "now": SERVER_TIMESTAMP}
         update_time, document_ref = collection.add(document_data)
 
         # Verify the response and the mocks.
-        self.assertIs(update_time, create_doc_response.update_time)
+        self.assertIs(update_time, mock.sentinel.update_time)
         self.assertIsInstance(document_ref, DocumentReference)
         self.assertIs(document_ref._client, client)
         expected_path = collection._path + (auto_assigned_id,)
         self.assertEqual(document_ref._path, expected_path)
 
-        expected_document_pb = document_pb2.Document(
-            fields=_helpers.encode_dict(document_data)
-        )
+        expected_document_pb = document_pb2.Document()
         firestore_api.create_document.assert_called_once_with(
             parent_path,
             collection_id=collection.id,
@@ -234,6 +243,13 @@ def test_add_auto_assigned(self):
             mask=None,
             metadata=client._rpc_metadata,
         )
+        write_pbs = pbs_for_set_no_merge(document_ref._document_path, document_data)
+        firestore_api.commit.assert_called_once_with(
+            client._database_string,
+            write_pbs,
+            transaction=None,
+            metadata=client._rpc_metadata,
+        )
 
     @staticmethod
     def _write_pb_for_create(document_path, document_data):