Repository: googleapis/google-cloud-python

Issue:
Storage: hmac operations should support userProject
Even though as it stands currently, the field is ignored by the GCS backend, as noted in the JSON API documentation. However, there is expectation that the field will start to be respected in later roll-outs.


Discussion:

Resolving patch:
diff --git a/storage/google/cloud/storage/client.py b/storage/google/cloud/storage/client.py
--- a/storage/google/cloud/storage/client.py
+++ b/storage/google/cloud/storage/client.py
@@ -582,7 +582,9 @@ def list_buckets(
             extra_params=extra_params,
         )
 
-    def create_hmac_key(self, service_account_email, project_id=None):
+    def create_hmac_key(
+        self, service_account_email, project_id=None, user_project=None
+    ):
         """Create an HMAC key for a service account.
 
         :type service_account_email: str
@@ -592,6 +594,9 @@ def create_hmac_key(self, service_account_email, project_id=None):
         :param project_id: (Optional) explicit project ID for the key.
             Defaults to the client's project.
 
+        :type user_project: str
+        :param user_project: (Optional) This parameter is currently ignored.
+
         :rtype:
             Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
         :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
@@ -601,6 +606,10 @@ def create_hmac_key(self, service_account_email, project_id=None):
 
         path = "/projects/{}/hmacKeys".format(project_id)
         qs_params = {"serviceAccountEmail": service_account_email}
+
+        if user_project is not None:
+            qs_params["userProject"] = user_project
+
         api_response = self._connection.api_request(
             method="POST", path=path, query_params=qs_params
         )
@@ -615,6 +624,7 @@ def list_hmac_keys(
         service_account_email=None,
         show_deleted_keys=None,
         project_id=None,
+        user_project=None,
     ):
         """List HMAC keys for a project.
 
@@ -635,6 +645,9 @@ def list_hmac_keys(
         :param project_id: (Optional) explicit project ID for the key.
             Defaults to the client's project.
 
+        :type user_project: str
+        :param user_project: (Optional) This parameter is currently ignored.
+
         :rtype:
             Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str]
         :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string.
@@ -651,6 +664,9 @@ def list_hmac_keys(
         if show_deleted_keys is not None:
             extra_params["showDeletedKeys"] = show_deleted_keys
 
+        if user_project is not None:
+            extra_params["userProject"] = user_project
+
         return page_iterator.HTTPIterator(
             client=self,
             api_request=self._connection.api_request,
@@ -660,7 +676,7 @@ def list_hmac_keys(
             extra_params=extra_params,
         )
 
-    def get_hmac_key_metadata(self, access_id, project_id=None):
+    def get_hmac_key_metadata(self, access_id, project_id=None, user_project=None):
         """Return a metadata instance for the given HMAC key.
 
         :type access_id: str
@@ -669,8 +685,11 @@ def get_hmac_key_metadata(self, access_id, project_id=None):
         :type project_id: str
         :param project_id: (Optional) project ID of an existing key.
             Defaults to client's project.
+
+        :type user_project: str
+        :param user_project: (Optional) This parameter is currently ignored.
         """
-        metadata = HMACKeyMetadata(self, access_id, project_id)
+        metadata = HMACKeyMetadata(self, access_id, project_id, user_project)
         metadata.reload()  # raises NotFound for missing key
         return metadata
 
diff --git a/storage/google/cloud/storage/hmac_key.py b/storage/google/cloud/storage/hmac_key.py
--- a/storage/google/cloud/storage/hmac_key.py
+++ b/storage/google/cloud/storage/hmac_key.py
@@ -28,6 +28,9 @@ class HMACKeyMetadata(object):
     :type project_id: str
     :param project_id: (Optional) project ID of an existing key.
         Defaults to client's project.
+
+    :type user_project: str
+    :param user_project: (Optional) This parameter is currently ignored.
     """
 
     ACTIVE_STATE = "ACTIVE"
@@ -42,7 +45,7 @@ class HMACKeyMetadata(object):
 
     _SETTABLE_STATES = (ACTIVE_STATE, INACTIVE_STATE)
 
-    def __init__(self, client, access_id=None, project_id=None):
+    def __init__(self, client, access_id=None, project_id=None, user_project=None):
         self._client = client
         self._properties = {}
 
@@ -52,6 +55,8 @@ def __init__(self, client, access_id=None, project_id=None):
         if project_id is not None:
             self._properties["projectId"] = project_id
 
+        self._user_project = user_project
+
     def __eq__(self, other):
         if not isinstance(other, self.__class__):
             return NotImplemented
@@ -170,6 +175,16 @@ def path(self):
 
         return "/projects/{}/hmacKeys/{}".format(project, self.access_id)
 
+    @property
+    def user_project(self):
+        """Project ID to be billed for API requests made via this bucket.
+
+        This property is currently ignored by the server.
+
+        :rtype: str
+        """
+        return self._user_project
+
     def exists(self):
         """Determine whether or not the key for this metadata exists.
 
@@ -177,7 +192,14 @@ def exists(self):
         :returns: True if the key exists in Cloud Storage.
         """
         try:
-            self._client._connection.api_request(method="GET", path=self.path)
+            qs_params = {}
+
+            if self.user_project is not None:
+                qs_params["userProject"] = self.user_project
+
+            self._client._connection.api_request(
+                method="GET", path=self.path, query_params=qs_params
+            )
         except NotFound:
             return False
         else:
@@ -189,8 +211,13 @@ def reload(self):
         :raises :class:`~google.api_core.exceptions.NotFound`:
             if the key does not exist on the back-end.
         """
+        qs_params = {}
+
+        if self.user_project is not None:
+            qs_params["userProject"] = self.user_project
+
         self._properties = self._client._connection.api_request(
-            method="GET", path=self.path
+            method="GET", path=self.path, query_params=qs_params
         )
 
     def update(self):
@@ -199,9 +226,13 @@ def update(self):
         :raises :class:`~google.api_core.exceptions.NotFound`:
             if the key does not exist on the back-end.
         """
+        qs_params = {}
+        if self.user_project is not None:
+            qs_params["userProject"] = self.user_project
+
         payload = {"state": self.state}
         self._properties = self._client._connection.api_request(
-            method="PUT", path=self.path, data=payload
+            method="PUT", path=self.path, data=payload, query_params=qs_params
         )
 
     def delete(self):
@@ -213,4 +244,10 @@ def delete(self):
         if self.state != self.INACTIVE_STATE:
             raise ValueError("Cannot delete key if not in 'INACTIVE' state.")
 
-        self._client._connection.api_request(method="DELETE", path=self.path)
+        qs_params = {}
+        if self.user_project is not None:
+            qs_params["userProject"] = self.user_project
+
+        self._client._connection.api_request(
+            method="DELETE", path=self.path, query_params=qs_params
+        )


Test patch:
diff --git a/storage/tests/unit/test_client.py b/storage/tests/unit/test_client.py
--- a/storage/tests/unit/test_client.py
+++ b/storage/tests/unit/test_client.py
@@ -954,7 +954,7 @@ def dummy_response():
         self.assertIsInstance(bucket, Bucket)
         self.assertEqual(bucket.name, blob_name)
 
-    def _create_hmac_key_helper(self, explicit_project=None):
+    def _create_hmac_key_helper(self, explicit_project=None, user_project=None):
         import datetime
         from pytz import UTC
         from six.moves.urllib.parse import urlencode
@@ -996,6 +996,9 @@ def _create_hmac_key_helper(self, explicit_project=None):
         if explicit_project is not None:
             kwargs["project_id"] = explicit_project
 
+        if user_project is not None:
+            kwargs["user_project"] = user_project
+
         metadata, secret = client.create_hmac_key(service_account_email=EMAIL, **kwargs)
 
         self.assertIsInstance(metadata, HMACKeyMetadata)
@@ -1013,8 +1016,12 @@ def _create_hmac_key_helper(self, explicit_project=None):
                 "hmacKeys",
             ]
         )
-        QS_PARAMS = {"serviceAccountEmail": EMAIL}
-        FULL_URI = "{}?{}".format(URI, urlencode(QS_PARAMS))
+        qs_params = {"serviceAccountEmail": EMAIL}
+
+        if user_project is not None:
+            qs_params["userProject"] = user_project
+
+        FULL_URI = "{}?{}".format(URI, urlencode(qs_params))
         http.request.assert_called_once_with(
             method="POST", url=FULL_URI, data=None, headers=mock.ANY
         )
@@ -1025,6 +1032,9 @@ def test_create_hmac_key_defaults(self):
     def test_create_hmac_key_explicit_project(self):
         self._create_hmac_key_helper(explicit_project="other-project-456")
 
+    def test_create_hmac_key_user_project(self):
+        self._create_hmac_key_helper(user_project="billed-project")
+
     def test_list_hmac_keys_defaults_empty(self):
         PROJECT = "PROJECT"
         CREDENTIALS = _make_credentials()
@@ -1060,6 +1070,7 @@ def test_list_hmac_keys_explicit_non_empty(self):
         MAX_RESULTS = 3
         EMAIL = "storage-user-123@example.com"
         ACCESS_ID = "ACCESS-ID"
+        USER_PROJECT = "billed-project"
         CREDENTIALS = _make_credentials()
         client = self._make_one(project=PROJECT, credentials=CREDENTIALS)
 
@@ -1083,6 +1094,7 @@ def test_list_hmac_keys_explicit_non_empty(self):
                 service_account_email=EMAIL,
                 show_deleted_keys=True,
                 project_id=OTHER_PROJECT,
+                user_project=USER_PROJECT,
             )
         )
 
@@ -1107,6 +1119,7 @@ def test_list_hmac_keys_explicit_non_empty(self):
             "maxResults": str(MAX_RESULTS),
             "serviceAccountEmail": EMAIL,
             "showDeletedKeys": "True",
+            "userProject": USER_PROJECT,
         }
         http.request.assert_called_once_with(
             method="GET", url=mock.ANY, data=None, headers=mock.ANY
@@ -1160,12 +1173,14 @@ def test_get_hmac_key_metadata_wo_project(self):
         )
 
     def test_get_hmac_key_metadata_w_project(self):
+        from six.moves.urllib.parse import urlencode
         from google.cloud.storage.hmac_key import HMACKeyMetadata
 
         PROJECT = "PROJECT"
         OTHER_PROJECT = "other-project-456"
         EMAIL = "storage-user-123@example.com"
         ACCESS_ID = "ACCESS-ID"
+        USER_PROJECT = "billed-project"
         CREDENTIALS = _make_credentials()
         client = self._make_one(project=PROJECT, credentials=CREDENTIALS)
 
@@ -1179,7 +1194,9 @@ def test_get_hmac_key_metadata_w_project(self):
         http = _make_requests_session([_make_json_response(resource)])
         client._http_internal = http
 
-        metadata = client.get_hmac_key_metadata(ACCESS_ID, project_id=OTHER_PROJECT)
+        metadata = client.get_hmac_key_metadata(
+            ACCESS_ID, project_id=OTHER_PROJECT, user_project=USER_PROJECT
+        )
 
         self.assertIsInstance(metadata, HMACKeyMetadata)
         self.assertIs(metadata._client, client)
@@ -1197,6 +1214,10 @@ def test_get_hmac_key_metadata_w_project(self):
                 ACCESS_ID,
             ]
         )
+
+        qs_params = {"userProject": USER_PROJECT}
+        FULL_URI = "{}?{}".format(URI, urlencode(qs_params))
+
         http.request.assert_called_once_with(
-            method="GET", url=URI, data=None, headers=mock.ANY
+            method="GET", url=FULL_URI, data=None, headers=mock.ANY
         )
diff --git a/storage/tests/unit/test_hmac_key.py b/storage/tests/unit/test_hmac_key.py
--- a/storage/tests/unit/test_hmac_key.py
+++ b/storage/tests/unit/test_hmac_key.py
@@ -46,12 +46,19 @@ def test_ctor_defaults(self):
     def test_ctor_explicit(self):
         OTHER_PROJECT = "other-project-456"
         ACCESS_ID = "access-id-123456789"
+        USER_PROJECT = "billed-project"
         client = _Client()
-        metadata = self._make_one(client, access_id=ACCESS_ID, project_id=OTHER_PROJECT)
+        metadata = self._make_one(
+            client,
+            access_id=ACCESS_ID,
+            project_id=OTHER_PROJECT,
+            user_project=USER_PROJECT,
+        )
         self.assertIs(metadata._client, client)
         expected = {"accessId": ACCESS_ID, "projectId": OTHER_PROJECT}
         self.assertEqual(metadata._properties, expected)
         self.assertEqual(metadata.access_id, ACCESS_ID)
+        self.assertEqual(metadata.user_project, USER_PROJECT)
         self.assertIsNone(metadata.etag)
         self.assertIsNone(metadata.id)
         self.assertEqual(metadata.project, OTHER_PROJECT)
@@ -217,12 +224,13 @@ def test_exists_miss_no_project_set(self):
         expected_path = "/projects/{}/hmacKeys/{}".format(
             client.DEFAULT_PROJECT, access_id
         )
-        expected_kwargs = {"method": "GET", "path": expected_path}
+        expected_kwargs = {"method": "GET", "path": expected_path, "query_params": {}}
         connection.api_request.assert_called_once_with(**expected_kwargs)
 
     def test_exists_hit_w_project_set(self):
         project = "PROJECT-ID"
         access_id = "ACCESS-ID"
+        user_project = "billed-project"
         email = "service-account@example.com"
         resource = {
             "kind": "storage#hmacKeyMetadata",
@@ -232,14 +240,18 @@ def test_exists_hit_w_project_set(self):
         connection = mock.Mock(spec=["api_request"])
         connection.api_request.return_value = resource
         client = _Client(connection)
-        metadata = self._make_one(client)
+        metadata = self._make_one(client, user_project=user_project)
         metadata._properties["accessId"] = access_id
         metadata._properties["projectId"] = project
 
         self.assertTrue(metadata.exists())
 
         expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id)
-        expected_kwargs = {"method": "GET", "path": expected_path}
+        expected_kwargs = {
+            "method": "GET",
+            "path": expected_path,
+            "query_params": {"userProject": user_project},
+        }
         connection.api_request.assert_called_once_with(**expected_kwargs)
 
     def test_reload_miss_no_project_set(self):
@@ -258,12 +270,13 @@ def test_reload_miss_no_project_set(self):
         expected_path = "/projects/{}/hmacKeys/{}".format(
             client.DEFAULT_PROJECT, access_id
         )
-        expected_kwargs = {"method": "GET", "path": expected_path}
+        expected_kwargs = {"method": "GET", "path": expected_path, "query_params": {}}
         connection.api_request.assert_called_once_with(**expected_kwargs)
 
     def test_reload_hit_w_project_set(self):
         project = "PROJECT-ID"
         access_id = "ACCESS-ID"
+        user_project = "billed-project"
         email = "service-account@example.com"
         resource = {
             "kind": "storage#hmacKeyMetadata",
@@ -273,7 +286,7 @@ def test_reload_hit_w_project_set(self):
         connection = mock.Mock(spec=["api_request"])
         connection.api_request.return_value = resource
         client = _Client(connection)
-        metadata = self._make_one(client)
+        metadata = self._make_one(client, user_project=user_project)
         metadata._properties["accessId"] = access_id
         metadata._properties["projectId"] = project
 
@@ -282,7 +295,11 @@ def test_reload_hit_w_project_set(self):
         self.assertEqual(metadata._properties, resource)
 
         expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id)
-        expected_kwargs = {"method": "GET", "path": expected_path}
+        expected_kwargs = {
+            "method": "GET",
+            "path": expected_path,
+            "query_params": {"userProject": user_project},
+        }
         connection.api_request.assert_called_once_with(**expected_kwargs)
 
     def test_update_miss_no_project_set(self):
@@ -306,12 +323,14 @@ def test_update_miss_no_project_set(self):
             "method": "PUT",
             "path": expected_path,
             "data": {"state": "INACTIVE"},
+            "query_params": {},
         }
         connection.api_request.assert_called_once_with(**expected_kwargs)
 
     def test_update_hit_w_project_set(self):
         project = "PROJECT-ID"
         access_id = "ACCESS-ID"
+        user_project = "billed-project"
         email = "service-account@example.com"
         resource = {
             "kind": "storage#hmacKeyMetadata",
@@ -322,7 +341,7 @@ def test_update_hit_w_project_set(self):
         connection = mock.Mock(spec=["api_request"])
         connection.api_request.return_value = resource
         client = _Client(connection)
-        metadata = self._make_one(client)
+        metadata = self._make_one(client, user_project=user_project)
         metadata._properties["accessId"] = access_id
         metadata._properties["projectId"] = project
         metadata.state = "ACTIVE"
@@ -336,6 +355,7 @@ def test_update_hit_w_project_set(self):
             "method": "PUT",
             "path": expected_path,
             "data": {"state": "ACTIVE"},
+            "query_params": {"userProject": user_project},
         }
         connection.api_request.assert_called_once_with(**expected_kwargs)
 
@@ -364,16 +384,21 @@ def test_delete_miss_no_project_set(self):
         expected_path = "/projects/{}/hmacKeys/{}".format(
             client.DEFAULT_PROJECT, access_id
         )
-        expected_kwargs = {"method": "DELETE", "path": expected_path}
+        expected_kwargs = {
+            "method": "DELETE",
+            "path": expected_path,
+            "query_params": {},
+        }
         connection.api_request.assert_called_once_with(**expected_kwargs)
 
     def test_delete_hit_w_project_set(self):
         project = "PROJECT-ID"
         access_id = "ACCESS-ID"
+        user_project = "billed-project"
         connection = mock.Mock(spec=["api_request"])
         connection.api_request.return_value = {}
         client = _Client(connection)
-        metadata = self._make_one(client)
+        metadata = self._make_one(client, user_project=user_project)
         metadata._properties["accessId"] = access_id
         metadata._properties["projectId"] = project
         metadata.state = "INACTIVE"
@@ -381,7 +406,11 @@ def test_delete_hit_w_project_set(self):
         metadata.delete()
 
         expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id)
-        expected_kwargs = {"method": "DELETE", "path": expected_path}
+        expected_kwargs = {
+            "method": "DELETE",
+            "path": expected_path,
+            "query_params": {"userProject": user_project},
+        }
         connection.api_request.assert_called_once_with(**expected_kwargs)