Repository: googleapis/google-cloud-python

Issue:
datastore: default to empty dataset
When working with datasets, the Datastore backend will normalize any empty datasets to the dataset of the calling application. This should be preferred to prepending "s~" to dataset ids as that is not always a valid operation.



Discussion:
The 'to_protobuf' method of gcloud.datastore.key.Key jams a 's~' prefix onto the key's dataset_id, if the ID does not already have a 's~' or 'e~' prefix:

https://github.com/GoogleCloudPlatform/gcloud-python/blob/1f2aaf3606802f4f02eeb30e22e1082ef7f4594e/gcloud/datastore/key.py#L60

@pcostell  Is that the normalization you are thinking of?

Yes that's the normalization that is _not_ needed. The datastore backend is smart about looking up the correct dataset for the application.

The fix in #259 is not correct / complete.  See:
- [my report on breakage to the demo](https://github.com/GoogleCloudPlatform/gcloud-python/pull/259#issuecomment-59850038)
- [@pcostell's follow up](https://github.com/GoogleCloudPlatform/gcloud-python/pull/259#issuecomment-59850761)


Resolving patch:
diff --git a/gcloud/datastore/key.py b/gcloud/datastore/key.py
--- a/gcloud/datastore/key.py
+++ b/gcloud/datastore/key.py
@@ -95,8 +95,13 @@ def to_protobuf(self):
         # but we shouldn't throw a cryptic error if one isn't provided
         # in the initializer.
         if self.dataset():
+            # Apparently 's~' is a prefix for High-Replication and is necessary
+            # here. Another valid preflix is 'e~' indicating EU datacenters.
             dataset_id = self.dataset().id()
             if dataset_id:
+                if dataset_id[:2] not in ['s~', 'e~']:
+                    dataset_id = 's~' + dataset_id
+
                 key.partition_id.dataset_id = dataset_id
 
         if self._namespace:


Test patch:
diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py
--- a/gcloud/datastore/test_connection.py
+++ b/gcloud/datastore/test_connection.py
@@ -425,7 +425,7 @@ def test_lookup_single_key_empty_response(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '24',
+            'Content-Length': '26',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
@@ -466,7 +466,7 @@ def test_lookup_single_key_nonempty_response(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '24',
+            'Content-Length': '26',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
@@ -504,7 +504,7 @@ def test_lookup_multiple_keys_empty_response(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '48',
+            'Content-Length': '52',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
@@ -549,7 +549,7 @@ def test_commit_wo_transaction(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '45',
+            'Content-Length': '47',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
@@ -597,7 +597,7 @@ def id(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '51',
+            'Content-Length': '53',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
@@ -634,7 +634,7 @@ def test_save_entity_wo_transaction_w_upsert(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '45',
+            'Content-Length': '47',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
@@ -687,7 +687,7 @@ def test_save_entity_wo_transaction_w_auto_id(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '42',
+            'Content-Length': '44',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
@@ -784,7 +784,7 @@ def test_delete_entities_wo_transaction(self):
         self.assertEqual(cw['method'], 'POST')
         expected_headers = {
             'Content-Type': 'application/x-protobuf',
-            'Content-Length': '28',
+            'Content-Length': '30',
             'User-Agent': conn.USER_AGENT,
         }
         self.assertEqual(cw['headers'], expected_headers)
diff --git a/gcloud/datastore/test_key.py b/gcloud/datastore/test_key.py
--- a/gcloud/datastore/test_key.py
+++ b/gcloud/datastore/test_key.py
@@ -133,12 +133,28 @@ def test_to_protobuf_w_explicit_dataset_empty_id(self):
         pb = key.to_protobuf()
         self.assertEqual(pb.partition_id.dataset_id, '')
 
-    def test_to_protobuf_w_explicit_dataset(self):
+    def test_to_protobuf_w_explicit_dataset_no_prefix(self):
         from gcloud.datastore.dataset import Dataset
         _DATASET = 'DATASET'
         dataset = Dataset(_DATASET)
         key = self._makeOne(dataset)
         pb = key.to_protobuf()
+        self.assertEqual(pb.partition_id.dataset_id, 's~%s' % _DATASET)
+
+    def test_to_protobuf_w_explicit_dataset_w_s_prefix(self):
+        from gcloud.datastore.dataset import Dataset
+        _DATASET = 's~DATASET'
+        dataset = Dataset(_DATASET)
+        key = self._makeOne(dataset)
+        pb = key.to_protobuf()
+        self.assertEqual(pb.partition_id.dataset_id, _DATASET)
+
+    def test_to_protobuf_w_explicit_dataset_w_e_prefix(self):
+        from gcloud.datastore.dataset import Dataset
+        _DATASET = 'e~DATASET'
+        dataset = Dataset(_DATASET)
+        key = self._makeOne(dataset)
+        pb = key.to_protobuf()
         self.assertEqual(pb.partition_id.dataset_id, _DATASET)
 
     def test_to_protobuf_w_explicit_namespace(self):