Repository: googleapis/google-cloud-python

Issue:
Add missing generation arguments for bucket and blob methods.
Allow setting generation for getting and deleting blobs, and respect generation for rewriting.


Discussion:

Resolving patch:
diff --git a/storage/google/cloud/storage/_helpers.py b/storage/google/cloud/storage/_helpers.py
--- a/storage/google/cloud/storage/_helpers.py
+++ b/storage/google/cloud/storage/_helpers.py
@@ -43,8 +43,9 @@ class _PropertyMixin(object):
     """Abstract mixin for cloud storage classes with associated properties.
 
     Non-abstract subclasses should implement:
-      - client
       - path
+      - client
+      - user_project
 
     :type name: str
     :param name: The name of the object. Bucket names must start and end with a
@@ -98,6 +99,14 @@ def _encryption_headers(self):
         """
         return {}
 
+    @property
+    def _query_params(self):
+        """Default query parameters."""
+        params = {}
+        if self.user_project is not None:
+            params["userProject"] = self.user_project
+        return params
+
     def reload(self, client=None):
         """Reload properties from Cloud Storage.
 
@@ -109,17 +118,16 @@ def reload(self, client=None):
                        ``client`` stored on the current object.
         """
         client = self._require_client(client)
+        query_params = self._query_params
         # Pass only '?projection=noAcl' here because 'acl' and related
         # are handled via custom endpoints.
-        query_params = {"projection": "noAcl"}
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params["projection"] = "noAcl"
         api_response = client._connection.api_request(
             method="GET",
             path=self.path,
             query_params=query_params,
             headers=self._encryption_headers(),
-            _target_object=self
+            _target_object=self,
         )
         self._set_properties(api_response)
 
@@ -164,11 +172,10 @@ def patch(self, client=None):
                        ``client`` stored on the current object.
         """
         client = self._require_client(client)
+        query_params = self._query_params
         # Pass '?projection=full' here because 'PATCH' documented not
         # to work properly w/ 'noAcl'.
-        query_params = {"projection": "full"}
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params["projection"] = "full"
         update_properties = {key: self._properties[key] for key in self._changes}
 
         # Make the API call.
@@ -194,9 +201,8 @@ def update(self, client=None):
                        ``client`` stored on the current object.
         """
         client = self._require_client(client)
-        query_params = {"projection": "full"}
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params = self._query_params
+        query_params["projection"] = "full"
         api_response = client._connection.api_request(
             method="PUT",
             path=self.path,
diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py
--- a/storage/google/cloud/storage/blob.py
+++ b/storage/google/cloud/storage/blob.py
@@ -159,7 +159,13 @@ class Blob(_PropertyMixin):
     """
 
     def __init__(
-        self, name, bucket, chunk_size=None, encryption_key=None, kms_key_name=None
+        self,
+        name,
+        bucket,
+        chunk_size=None,
+        encryption_key=None,
+        kms_key_name=None,
+        generation=None,
     ):
         name = _bytes_to_unicode(name)
         super(Blob, self).__init__(name=name)
@@ -177,6 +183,9 @@ def __init__(
         if kms_key_name is not None:
             self._properties["kmsKeyName"] = kms_key_name
 
+        if generation is not None:
+            self._properties["generation"] = generation
+
     @property
     def chunk_size(self):
         """Get the blob's default chunk size.
@@ -265,6 +274,16 @@ def _encryption_headers(self):
         """
         return _get_encryption_headers(self._encryption_key)
 
+    @property
+    def _query_params(self):
+        """Default query parameters."""
+        params = {}
+        if self.generation is not None:
+            params["generation"] = self.generation
+        if self.user_project is not None:
+            params["userProject"] = self.user_project
+        return params
+
     @property
     def public_url(self):
         """The public URL for this blob.
@@ -397,10 +416,8 @@ def exists(self, client=None):
         client = self._require_client(client)
         # We only need the status code (200 or not) so we seek to
         # minimize the returned payload.
-        query_params = {"fields": "name"}
-
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        query_params = self._query_params
+        query_params["fields"] = "name"
 
         try:
             # We intentionally pass `_target_object=None` since fields=name
@@ -435,7 +452,9 @@ def delete(self, client=None):
                  (propagated from
                  :meth:`google.cloud.storage.bucket.Bucket.delete_blob`).
         """
-        return self.bucket.delete_blob(self.name, client=client)
+        return self.bucket.delete_blob(
+            self.name, client=client, generation=self.generation
+        )
 
     def _get_transport(self, client):
         """Return the client's transport.
@@ -1492,13 +1511,15 @@ def rewrite(self, source, token=None, client=None):
         headers = _get_encryption_headers(self._encryption_key)
         headers.update(_get_encryption_headers(source._encryption_key, source=True))
 
-        query_params = {}
+        query_params = self._query_params
+        if "generation" in query_params:
+            del query_params["generation"]
 
         if token:
             query_params["rewriteToken"] = token
 
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
+        if source.generation:
+            query_params["sourceGeneration"] = source.generation
 
         if self.kms_key_name is not None:
             query_params["destinationKmsKeyName"] = self.kms_key_name
diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py
--- a/storage/google/cloud/storage/bucket.py
+++ b/storage/google/cloud/storage/bucket.py
@@ -434,7 +434,14 @@ def user_project(self):
         """
         return self._user_project
 
-    def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=None):
+    def blob(
+        self,
+        blob_name,
+        chunk_size=None,
+        encryption_key=None,
+        kms_key_name=None,
+        generation=None,
+    ):
         """Factory constructor for blob object.
 
         .. note::
@@ -457,6 +464,10 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non
         :param kms_key_name:
             Optional resource name of KMS key used to encrypt blob's content.
 
+        :type generation: long
+        :param generation: Optional. If present, selects a specific revision of
+                           this object.
+
         :rtype: :class:`google.cloud.storage.blob.Blob`
         :returns: The blob object created.
         """
@@ -466,6 +477,7 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non
             chunk_size=chunk_size,
             encryption_key=encryption_key,
             kms_key_name=kms_key_name,
+            generation=generation,
         )
 
     def notification(
@@ -638,7 +650,9 @@ def path(self):
 
         return self.path_helper(self.name)
 
-    def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
+    def get_blob(
+        self, blob_name, client=None, encryption_key=None, generation=None, **kwargs
+    ):
         """Get a blob object by name.
 
         This will return None if the blob doesn't exist:
@@ -663,6 +677,10 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
             See
             https://cloud.google.com/storage/docs/encryption#customer-supplied.
 
+        :type generation: long
+        :param generation: Optional. If present, selects a specific revision of
+                           this object.
+
         :type kwargs: dict
         :param kwargs: Keyword arguments to pass to the
                        :class:`~google.cloud.storage.blob.Blob` constructor.
@@ -671,7 +689,11 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
         :returns: The blob object if it exists, otherwise None.
         """
         blob = Blob(
-            bucket=self, name=blob_name, encryption_key=encryption_key, **kwargs
+            bucket=self,
+            name=blob_name,
+            encryption_key=encryption_key,
+            generation=generation,
+            **kwargs
         )
         try:
             # NOTE: This will not fail immediately in a batch. However, when
@@ -867,7 +889,7 @@ def delete(self, force=False, client=None):
             _target_object=None,
         )
 
-    def delete_blob(self, blob_name, client=None):
+    def delete_blob(self, blob_name, client=None, generation=None):
         """Deletes a blob from the current bucket.
 
         If the blob isn't found (backend 404), raises a
@@ -889,6 +911,10 @@ def delete_blob(self, blob_name, client=None):
         :param client: Optional. The client to use.  If not passed, falls back
                        to the ``client`` stored on the current bucket.
 
+        :type generation: long
+        :param generation: Optional. If present, permanently deletes a specific
+                           revision of this object.
+
         :raises: :class:`google.cloud.exceptions.NotFound` (to suppress
                  the exception, call ``delete_blobs``, passing a no-op
                  ``on_error`` callback, e.g.:
@@ -899,19 +925,15 @@ def delete_blob(self, blob_name, client=None):
 
         """
         client = self._require_client(client)
-        query_params = {}
+        blob = Blob(blob_name, bucket=self, generation=generation)
 
-        if self.user_project is not None:
-            query_params["userProject"] = self.user_project
-
-        blob_path = Blob.path_helper(self.path, blob_name)
         # We intentionally pass `_target_object=None` since a DELETE
         # request has no response value (whether in a standard request or
         # in a batch request).
         client._connection.api_request(
             method="DELETE",
-            path=blob_path,
-            query_params=query_params,
+            path=blob.path,
+            query_params=blob._query_params,
             _target_object=None,
         )
 


Test patch:
diff --git a/storage/tests/system.py b/storage/tests/system.py
--- a/storage/tests/system.py
+++ b/storage/tests/system.py
@@ -44,7 +44,9 @@ def _bad_copy(bad_request):
 
 
 retry_429 = RetryErrors(exceptions.TooManyRequests, max_tries=6)
-retry_429_503 = RetryErrors([exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=6)
+retry_429_503 = RetryErrors(
+    [exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=6
+)
 retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
 
 
@@ -78,6 +80,7 @@ def setUpModule():
     # In the **very** rare case the bucket name is reserved, this
     # fails with a ConnectionError.
     Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name)
+    Config.TEST_BUCKET.versioning_enabled = True
     retry_429(Config.TEST_BUCKET.create)()
 
 
@@ -414,6 +417,20 @@ def test_crud_blob_w_user_project(self):
 
         # Exercise 'objects.insert' w/ userProject.
         blob.upload_from_filename(file_data["path"])
+        gen0 = blob.generation
+
+        # Upload a second generation of the blob
+        blob.upload_from_string(b"gen1")
+        gen1 = blob.generation
+
+        blob0 = with_user_project.blob("SmallFile", generation=gen0)
+        blob1 = with_user_project.blob("SmallFile", generation=gen1)
+
+        # Exercise 'objects.get' w/ generation
+        self.assertEqual(with_user_project.get_blob(blob.name).generation, gen1)
+        self.assertEqual(
+            with_user_project.get_blob(blob.name, generation=gen0).generation, gen0
+        )
 
         try:
             # Exercise 'objects.get' (metadata) w/ userProject.
@@ -421,22 +438,31 @@ def test_crud_blob_w_user_project(self):
             blob.reload()
 
             # Exercise 'objects.get' (media) w/ userProject.
-            downloaded = blob.download_as_string()
-            self.assertEqual(downloaded, file_contents)
+            self.assertEqual(blob0.download_as_string(), file_contents)
+            self.assertEqual(blob1.download_as_string(), b"gen1")
 
             # Exercise 'objects.patch' w/ userProject.
-            blob.content_language = "en"
-            blob.patch()
-            self.assertEqual(blob.content_language, "en")
+            blob0.content_language = "en"
+            blob0.patch()
+            self.assertEqual(blob0.content_language, "en")
+            self.assertIsNone(blob1.content_language)
 
             # Exercise 'objects.update' w/ userProject.
             metadata = {"foo": "Foo", "bar": "Bar"}
-            blob.metadata = metadata
-            blob.update()
-            self.assertEqual(blob.metadata, metadata)
+            blob0.metadata = metadata
+            blob0.update()
+            self.assertEqual(blob0.metadata, metadata)
+            self.assertIsNone(blob1.metadata)
         finally:
             # Exercise 'objects.delete' (metadata) w/ userProject.
-            blob.delete()
+            blobs = with_user_project.list_blobs(prefix=blob.name, versions=True)
+            self.assertEqual([each.generation for each in blobs], [gen0, gen1])
+
+            blob0.delete()
+            blobs = with_user_project.list_blobs(prefix=blob.name, versions=True)
+            self.assertEqual([each.generation for each in blobs], [gen1])
+
+            blob1.delete()
 
     @unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.")
     def test_blob_acl_w_user_project(self):
diff --git a/storage/tests/unit/test__helpers.py b/storage/tests/unit/test__helpers.py
--- a/storage/tests/unit/test__helpers.py
+++ b/storage/tests/unit/test__helpers.py
@@ -59,6 +59,15 @@ def test__encryption_headers(self):
         mixin = self._make_one()
         self.assertEqual(mixin._encryption_headers(), {})
 
+    def test__query_params_wo_user_project(self):
+        derived = self._derivedClass("/path", None)()
+        self.assertEqual(derived._query_params, {})
+
+    def test__query_params_w_user_project(self):
+        user_project = "user-project-123"
+        derived = self._derivedClass("/path", user_project)()
+        self.assertEqual(derived._query_params, {"userProject": user_project})
+
     def test_reload(self):
         connection = _Connection({"foo": "Foo"})
         client = _Client(connection)
diff --git a/storage/tests/unit/test_blob.py b/storage/tests/unit/test_blob.py
--- a/storage/tests/unit/test_blob.py
+++ b/storage/tests/unit/test_blob.py
@@ -101,6 +101,13 @@ def test_ctor_w_kms_key_name(self):
         self.assertEqual(blob._encryption_key, None)
         self.assertEqual(blob.kms_key_name, KMS_RESOURCE)
 
+    def test_ctor_with_generation(self):
+        BLOB_NAME = "blob-name"
+        GENERATION = 12345
+        bucket = _Bucket()
+        blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION)
+        self.assertEqual(blob.generation, GENERATION)
+
     def _set_properties_helper(self, kms_key_name=None):
         import datetime
         from google.cloud._helpers import UTC
@@ -301,6 +308,26 @@ def test__encryption_headers_w_encryption_key(self):
         }
         self.assertEqual(blob._encryption_headers(), expected)
 
+    def test__query_params_default(self):
+        BLOB_NAME = "blob-name"
+        bucket = _Bucket()
+        blob = self._make_one(BLOB_NAME, bucket=bucket)
+        self.assertEqual(blob._query_params, {})
+
+    def test__query_params_w_user_project(self):
+        user_project = "user-project-123"
+        BLOB_NAME = "BLOB"
+        bucket = _Bucket(user_project=user_project)
+        blob = self._make_one(BLOB_NAME, bucket=bucket)
+        self.assertEqual(blob._query_params, {"userProject": user_project})
+
+    def test__query_params_w_generation(self):
+        generation = 123456
+        BLOB_NAME = "BLOB"
+        bucket = _Bucket()
+        blob = self._make_one(BLOB_NAME, bucket=bucket, generation=generation)
+        self.assertEqual(blob._query_params, {"generation": generation})
+
     def test_public_url(self):
         BLOB_NAME = "blob-name"
         bucket = _Bucket()
@@ -587,7 +614,28 @@ def test_exists_hit_w_user_project(self):
             },
         )
 
-    def test_delete(self):
+    def test_exists_hit_w_generation(self):
+        BLOB_NAME = "blob-name"
+        GENERATION = 123456
+        found_response = ({"status": http_client.OK}, b"")
+        connection = _Connection(found_response)
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION)
+        bucket._blobs[BLOB_NAME] = 1
+        self.assertTrue(blob.exists())
+        self.assertEqual(len(connection._requested), 1)
+        self.assertEqual(
+            connection._requested[0],
+            {
+                "method": "GET",
+                "path": "/b/name/o/{}".format(BLOB_NAME),
+                "query_params": {"fields": "name", "generation": GENERATION},
+                "_target_object": None,
+            },
+        )
+
+    def test_delete_wo_generation(self):
         BLOB_NAME = "blob-name"
         not_found_response = ({"status": http_client.NOT_FOUND}, b"")
         connection = _Connection(not_found_response)
@@ -597,7 +645,20 @@ def test_delete(self):
         bucket._blobs[BLOB_NAME] = 1
         blob.delete()
         self.assertFalse(blob.exists())
-        self.assertEqual(bucket._deleted, [(BLOB_NAME, None)])
+        self.assertEqual(bucket._deleted, [(BLOB_NAME, None, None)])
+
+    def test_delete_w_generation(self):
+        BLOB_NAME = "blob-name"
+        GENERATION = 123456
+        not_found_response = ({"status": http_client.NOT_FOUND}, b"")
+        connection = _Connection(not_found_response)
+        client = _Client(connection)
+        bucket = _Bucket(client)
+        blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION)
+        bucket._blobs[BLOB_NAME] = 1
+        blob.delete()
+        self.assertFalse(blob.exists())
+        self.assertEqual(bucket._deleted, [(BLOB_NAME, None, GENERATION)])
 
     def test__get_transport(self):
         client = mock.Mock(spec=[u"_credentials", "_http"])
@@ -2316,6 +2377,48 @@ def test_rewrite_response_without_resource(self):
         self.assertEqual(rewritten, 33)
         self.assertEqual(size, 42)
 
+    def test_rewrite_w_generations(self):
+        SOURCE_BLOB = "source"
+        SOURCE_GENERATION = 42
+        DEST_BLOB = "dest"
+        DEST_BUCKET = "other-bucket"
+        DEST_GENERATION = 43
+        TOKEN = "TOKEN"
+        RESPONSE = {
+            "totalBytesRewritten": 33,
+            "objectSize": 42,
+            "done": False,
+            "rewriteToken": TOKEN,
+        }
+        response = ({"status": http_client.OK}, RESPONSE)
+        connection = _Connection(response)
+        client = _Client(connection)
+        source_bucket = _Bucket(client=client)
+        source_blob = self._make_one(
+            SOURCE_BLOB, bucket=source_bucket, generation=SOURCE_GENERATION
+        )
+        dest_bucket = _Bucket(client=client, name=DEST_BUCKET)
+        dest_blob = self._make_one(
+            DEST_BLOB, bucket=dest_bucket, generation=DEST_GENERATION
+        )
+
+        token, rewritten, size = dest_blob.rewrite(source_blob)
+
+        self.assertEqual(token, TOKEN)
+        self.assertEqual(rewritten, 33)
+        self.assertEqual(size, 42)
+
+        kw, = connection._requested
+        self.assertEqual(kw["method"], "POST")
+        self.assertEqual(
+            kw["path"],
+            "/b/%s/o/%s/rewriteTo/b/%s/o/%s"
+            % (
+                (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name)
+            ),
+        )
+        self.assertEqual(kw["query_params"], {"sourceGeneration": SOURCE_GENERATION})
+
     def test_rewrite_other_bucket_other_name_no_encryption_partial(self):
         SOURCE_BLOB = "source"
         DEST_BLOB = "dest"
@@ -3196,9 +3299,9 @@ def __init__(self, client=None, name="name", user_project=None):
         self.path = "/b/" + name
         self.user_project = user_project
 
-    def delete_blob(self, blob_name, client=None):
+    def delete_blob(self, blob_name, client=None, generation=None):
         del self._blobs[blob_name]
-        self._deleted.append((blob_name, client))
+        self._deleted.append((blob_name, client, generation))
 
 
 class _Signer(object):
diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py
--- a/storage/tests/unit/test_bucket.py
+++ b/storage/tests/unit/test_bucket.py
@@ -360,6 +360,21 @@ def test_blob_w_encryption_key(self):
         self.assertEqual(blob._encryption_key, KEY)
         self.assertIsNone(blob.kms_key_name)
 
+    def test_blob_w_generation(self):
+        from google.cloud.storage.blob import Blob
+
+        BUCKET_NAME = "BUCKET_NAME"
+        BLOB_NAME = "BLOB_NAME"
+        GENERATION = 123
+
+        bucket = self._make_one(name=BUCKET_NAME)
+        blob = bucket.blob(BLOB_NAME, generation=GENERATION)
+        self.assertIsInstance(blob, Blob)
+        self.assertIs(blob.bucket, bucket)
+        self.assertIs(blob.client, bucket.client)
+        self.assertEqual(blob.name, BLOB_NAME)
+        self.assertEqual(blob.generation, GENERATION)
+
     def test_blob_w_kms_key_name(self):
         from google.cloud.storage.blob import Blob
 
@@ -669,10 +684,24 @@ def test_get_blob_hit_w_user_project(self):
         self.assertIs(blob.bucket, bucket)
         self.assertEqual(blob.name, BLOB_NAME)
         kw, = connection._requested
-        expected_qp = {
-            "userProject": USER_PROJECT,
-            "projection": "noAcl",
-        }
+        expected_qp = {"userProject": USER_PROJECT, "projection": "noAcl"}
+        self.assertEqual(kw["method"], "GET")
+        self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME))
+        self.assertEqual(kw["query_params"], expected_qp)
+
+    def test_get_blob_hit_w_generation(self):
+        NAME = "name"
+        BLOB_NAME = "blob-name"
+        GENERATION = 1512565576797178
+        connection = _Connection({"name": BLOB_NAME, "generation": GENERATION})
+        client = _Client(connection)
+        bucket = self._make_one(name=NAME)
+        blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION)
+        self.assertIs(blob.bucket, bucket)
+        self.assertEqual(blob.name, BLOB_NAME)
+        self.assertEqual(blob.generation, GENERATION)
+        kw, = connection._requested
+        expected_qp = {"generation": GENERATION, "projection": "noAcl"}
         self.assertEqual(kw["method"], "GET")
         self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME))
         self.assertEqual(kw["query_params"], expected_qp)
@@ -945,6 +974,20 @@ def test_delete_blob_hit_with_user_project(self):
         self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME))
         self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT})
 
+    def test_delete_blob_hit_with_generation(self):
+        NAME = "name"
+        BLOB_NAME = "blob-name"
+        GENERATION = 1512565576797178
+        connection = _Connection({})
+        client = _Client(connection)
+        bucket = self._make_one(client=client, name=NAME)
+        result = bucket.delete_blob(BLOB_NAME, generation=GENERATION)
+        self.assertIsNone(result)
+        kw, = connection._requested
+        self.assertEqual(kw["method"], "DELETE")
+        self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME))
+        self.assertEqual(kw["query_params"], {"generation": GENERATION})
+
     def test_delete_blobs_empty(self):
         NAME = "name"
         connection = _Connection()