Repository: googleapis/google-cloud-python

Issue:
Firestore DocumentReference.update fails with integer path
### OS
Ubuntu 14.04

### Python version
Python 2.7.6

### Library version
```
pip show google-cloud-firestore
---
Metadata-Version: 2.0
Name: google-cloud-firestore
Version: 0.27.0
Summary: Python Client for Google Cloud Firestore
Home-page: https://github.com/GoogleCloudPlatform/google-cloud-python
Author: Google Cloud Platform
Author-email: googleapis-publisher@google.com
Installer: pip
License: Apache 2.0
Location: /home/vagrant/venv/app_tophat/lib/python2.7/site-packages
Requires: google-gax, google-cloud-core
Classifiers:
  Development Status :: 4 - Beta
  Intended Audience :: Developers
  License :: OSI Approved :: Apache Software License
  Operating System :: OS Independent
  Programming Language :: Python :: 2
  Programming Language :: Python :: 2.7
  Programming Language :: Python :: 3
  Programming Language :: Python :: 3.4
  Programming Language :: Python :: 3.5
  Programming Language :: Python :: 3.6
  Topic :: Internet
```

### Stack trace
```
In [55]: status_doc.update({u'14': {u'status': u'active'}}, firestore.CreateIfMissingOption(True))
---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
<ipython-input-55-15a7a489a7f1> in <module>()
----> 1 status_doc.update({u'14': {u'status': u'active'}}, firestore.CreateIfMissingOption(True))

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/cloud/firestore_v1beta1/document.pyc in update(self, field_updates, option)
    369         batch = self._client.batch()
    370         batch.update(self, field_updates, option=option)
--> 371         write_results = batch.commit()
    372         return _first_write_result(write_results)
    373

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/cloud/firestore_v1beta1/batch.pyc in commit(self)
    133             commit_response = self._client._firestore_api.commit(
    134                 self._client._database_string, self._write_pbs,
--> 135                 transaction=None, options=self._client._call_options)
    136
    137         self._write_pbs = []

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/cloud/firestore_v1beta1/gapic/firestore_client.pyc in commit(self, database, writes, transaction, options)
    849         request = firestore_pb2.CommitRequest(
    850             database=database, writes=writes, transaction=transaction)
--> 851         return self._commit(request, options)
    852
    853     def rollback(self, database, transaction, options=None):

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/gax/api_callable.pyc in inner(request, options)
    450                 func, this_settings.timeout, **this_settings.kwargs)
    451         api_call = _catch_errors(api_call, gax.config.API_ERRORS)
--> 452         return api_caller(api_call, this_settings, request)
    453
    454     if settings.page_descriptor:

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/gax/api_callable.pyc in base_caller(api_call, _, *args)
    436     def base_caller(api_call, _, *args):
    437         """Simply call api_call and ignore settings."""
--> 438         return api_call(*args)
    439
    440     def inner(request, options=None):

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/gax/api_callable.pyc in inner(*args, **kwargs)
    378         except tuple(to_catch) as exception:
    379             utils.raise_with_traceback(
--> 380                 gax.errors.create_error('RPC failed', cause=exception))
    381
    382     return inner

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/gax/api_callable.pyc in inner(*args, **kwargs)
    374         """Wraps specified exceptions"""
    375         try:
--> 376             return a_func(*args, **kwargs)
    377         # pylint: disable=catching-non-exception
    378         except tuple(to_catch) as exception:

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/google/gax/retry.pyc in inner(*args)
     66         """Updates args with the timeout."""
     67         updated_args = args + (timeout,)
---> 68         return a_func(*updated_args, **kwargs)
     69
     70     return inner

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/grpc/_channel.pyc in __call__(self, request, timeout, metadata, credentials)
    505         state, call, deadline = self._blocking(request, timeout, metadata,
    506                                                credentials)
--> 507         return _end_unary_response_blocking(state, call, False, deadline)
    508
    509     def with_call(self, request, timeout=None, metadata=None, credentials=None):

/home/vagrant/venv/app_tophat/local/lib/python2.7/site-packages/grpc/_channel.pyc in _end_unary_response_blocking(state, call, with_call, deadline)
    453             return state.response
    454     else:
--> 455         raise _Rendezvous(state, None, None, deadline)
    456
    457

InvalidArgumentError: InvalidArgumentError(RPC failed, caused by <_Rendezvous of RPC that terminated with (StatusCode.INVALID_ARGUMENT, Invalid property path "14".)>)
```

### Repro Steps
- Given a document, run the code below

### Code sample
Given a `DocumentReference`
```
status_doc = db.collection(u'status').document(u'us-{}'.format(9))
```
Try to update it with the `CreateIfMissingOption` set to `True` with a new top level key that is an integer cast to a string. Expectation is that it creates a new field entry under the document.
```
status_doc.update({u'14': {u'status': u'active'}}, firestore.CreateIfMissingOption(True))
```
This results in the above stack trace.

Running a set command works
```
In [57]: status_doc.set({u'14': {u'status': u'active'}})
Out[57]:
update_time {
  seconds: 1509564296
  nanos: 432172000
}
```
Running an update command with a non-integer string works
```
In [56]: status_doc.update({u'fourteen': {u'status': u'active'}}, firestore.CreateIfMissingOption(True))
Out[56]:
update_time {
  seconds: 1509564250
  nanos: 341559000
}
```
Performing the same action in the JS library also works:
```
const statusSet = statusDoc.set(
            {'14': {'status': 'active'}},
            {merge: true}
        )

```



Discussion:
`StatusCode.INVALID_ARGUMENT` means it's a backend issue, **not** an issue with the library. With this in mind, I'm going to close this issue.

However, it's still relevant, so we should continue the discussion here.

@samtstern is more knowledgeable about Firestore than I am, so hopefully he can lend some insight / redirect to the backend team.
@dhermes this still seems to be an issue and I don't see any followup here. can I direct this issue somewhere else instead?
@dhermes are we sure this is a backend issue and not an issue with how the Python library is preparing the request?  @jdubinsky has shown that it works in the JS client library which means that it's may not truly be an "invalid argument"
@lukesneeringer @jonparrott can one of you have a poke into this please?  Thanks!
@danoscarmike This problem might be changed with #4111.  The problem with #4111 was that I was stuck not being able to merge two protobufs and it didn't seem like it should be done with a fix in our client.  I could try to make workarounds, but it wouldn't be the right place to put it.
Let me take a look again.
@danoscarmike So I have found the issue that causes the problem, but I have no idea how to fix this yet and I will probably need someone who is more familiar with protobuf.

[update](https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/firestore/google/cloud/firestore_v1beta1/_helpers.py#L865-L868) returns
```
name: "projects/firestorecredential-3a5dc/databases/(default)/documents/i-did-it/for-set-1517905333256"
fields {
  key: "14"
  value {
    map_value {
      fields {
        key: "city"
        value {
          bytes_value: "LA"
        }
      }
    }
  }
}
```
whereas
`encode_dict(update_values)`
returns 
```
{u'14': map_value {
  fields {
    key: "city"
    value {
      bytes_value: "LA"
    }
  }
}
}
```

The protobuf field is [here](https://github.com/googleapis/googleapis/blob/master/google/firestore/v1beta1/document.proto#L65).

and the problem is that unicode was lost in the string representation and I guess the protobuf sees it as an int.  I have also tried replacing `u'14'` with `u'\u0031\u0034'` and I still see the same problem.
The protobuf `Map` is a `<string, Value>` (it is [here](https://github.com/googleapis/googleapis/blob/master/google/firestore/v1beta1/document.proto#L65)), so I am reasonably certain a stealth integer encoding by protobuf is not the actual problem.

_We_ might be doing it, but protobuf is probably not.
The field names cannot start with numbers unless it is in utf8.  So if it worked in node, it might be an encoding issue?

// A simple field name contains only characters `a` to `z`, `A` to `Z`,
  // `0` to `9`, or `_`, and must not start with `0` to `9` or `_`. For example,
  // `foo_bar_17`.

 The map
  // keys, represented as UTF-8, must not exceed 1,500 bytes and cannot be
  // empty.
_Ohhhhhhh._ Okay, I see. And the OP is on Python 2.7.
@lukesneeringer Did you fix this somehow?  I don't seem to be reproducing it any more.
never mind, I still see it.

by the way, it also has problems in python 3
found the fix, wrote the solution, tested system, just need to write the unit test for it.

Resolving patch:
diff --git a/firestore/google/cloud/firestore_v1beta1/_helpers.py b/firestore/google/cloud/firestore_v1beta1/_helpers.py
--- a/firestore/google/cloud/firestore_v1beta1/_helpers.py
+++ b/firestore/google/cloud/firestore_v1beta1/_helpers.py
@@ -18,6 +18,7 @@
 import collections
 import contextlib
 import datetime
+import re
 import sys
 
 import google.gax
@@ -67,6 +68,7 @@
     grpc.StatusCode.ALREADY_EXISTS: exceptions.Conflict,
     grpc.StatusCode.NOT_FOUND: exceptions.NotFound,
 }
+_UNESCAPED_FIELD_NAME_RE = re.compile('^[_a-zA-Z][_a-zA-Z0-9]*$')
 
 
 class GeoPoint(object):
@@ -837,6 +839,38 @@ def pbs_for_set(document_path, document_data, option):
     return write_pbs
 
 
+def canonicalize_field_paths(field_paths):
+    """Converts simple field path with integer beginnings to quoted field path
+
+    Args:
+        field_paths (Sequence[str]): A list of field paths
+
+    Returns:
+        Sequence[str]:
+            The same list of field paths except non-simple field names
+            in the `.` delimited field path have been converted
+            into quoted unicode field paths. Simple field paths match
+            the regex ^[_a-zA-Z][_a-zA-Z0-9]*$.  See `Document`_ page for
+            more information.
+
+    .. _Document: https://cloud.google.com/firestore/docs/reference/rpc/google.firestore.v1beta1#google.firestore.v1beta1.Document  # NOQA
+    """
+    canonical_strings = []
+    for field_path in field_paths:
+        escaped_names = []
+        field_names = field_path.split('.')
+        for field_name in field_names:
+            if re.match(_UNESCAPED_FIELD_NAME_RE, field_name):
+                escaped_name = field_name
+            else:
+                escaped_name = u"`{}`".format(
+                    field_name.replace('\\', '\\\\').replace('`', '``'))
+            escaped_names.append(escaped_name)
+        new_field_path = '.'.join(escaped_names)
+        canonical_strings.append(new_field_path)
+    return canonical_strings
+
+
 def pbs_for_update(client, document_path, field_updates, option):
     """Make ``Write`` protobufs for ``update()`` methods.
 
@@ -860,6 +894,7 @@ def pbs_for_update(client, document_path, field_updates, option):
 
     transform_paths, actual_updates = remove_server_timestamp(field_updates)
     update_values, field_paths = FieldPathHelper.to_field_paths(actual_updates)
+    field_paths = canonicalize_field_paths(field_paths)
 
     update_pb = write_pb2.Write(
         update=document_pb2.Document(
diff --git a/firestore/google/cloud/firestore_v1beta1/client.py b/firestore/google/cloud/firestore_v1beta1/client.py
--- a/firestore/google/cloud/firestore_v1beta1/client.py
+++ b/firestore/google/cloud/firestore_v1beta1/client.py
@@ -434,7 +434,6 @@ class CreateIfMissingOption(WriteOption):
         create_if_missing (bool): Indicates if the document should be created
             if it doesn't already exist.
     """
-
     def __init__(self, create_if_missing):
         self._create_if_missing = create_if_missing
 


Test patch:
diff --git a/firestore/tests/system.py b/firestore/tests/system.py
--- a/firestore/tests/system.py
+++ b/firestore/tests/system.py
@@ -194,6 +194,38 @@ def test_document_set(client, cleanup):
     assert exc_to_code(exc_info.value.cause) == StatusCode.FAILED_PRECONDITION
 
 
+def test_document_integer_field(client, cleanup):
+    document_id = 'for-set' + unique_resource_id('-')
+    document = client.document('i-did-it', document_id)
+    # Add to clean-up before API request (in case ``set()`` fails).
+    cleanup(document)
+
+    data1 = {
+        '1a': {
+            '2b': '3c',
+            'ab': '5e'},
+        '6f': {
+            '7g': '8h',
+            'cd': '0j'}
+    }
+    option1 = client.write_option(exists=False)
+    document.set(data1, option=option1)
+
+    data2 = {'1a.ab': '4d', '6f.7g': '9h'}
+    option2 = client.write_option(create_if_missing=True)
+    document.update(data2, option=option2)
+    snapshot = document.get()
+    expected = {
+        '1a': {
+            '2b': '3c',
+            'ab': '4d'},
+        '6f': {
+            '7g': '9h',
+            'cd': '0j'}
+    }
+    assert snapshot.to_dict() == expected
+
+
 def test_update_document(client, cleanup):
     document_id = 'for-update' + unique_resource_id('-')
     document = client.document('made', document_id)
diff --git a/firestore/tests/unit/test__helpers.py b/firestore/tests/unit/test__helpers.py
--- a/firestore/tests/unit/test__helpers.py
+++ b/firestore/tests/unit/test__helpers.py
@@ -1186,6 +1186,20 @@ def test_update_and_transform(self):
         self._helper(do_transform=True)
 
 
+class Test_canonicalize_field_paths(unittest.TestCase):
+
+    def test_canonicalize_field_paths(self):
+        from google.cloud.firestore_v1beta1 import _helpers
+        field_paths = ['0abc.deq', 'abc.654', '321.0deq._321',
+                       u'0abc.deq', u'abc.654', u'321.0deq._321']
+        convert = _helpers.canonicalize_field_paths(field_paths)
+        self.assertListEqual(
+            convert,
+            ['`0abc`.deq', 'abc.`654`', '`321`.`0deq`._321',
+             '`0abc`.deq', 'abc.`654`', '`321`.`0deq`._321']
+        )
+
+
 class Test_pbs_for_update(unittest.TestCase):
 
     @staticmethod