Repository: PrefectHQ/prefect

Issue:
Google Cloud Storage <= 1.31 no longer supported
#3741 changed the GCS result to use a function that was only released a couple of months ago in 1.31.0 (see https://github.com/googleapis/python-storage/commits/master?after=9d5bb5622f17a0143fa6670406596e53d0068286+34&branch=master).

In my opinion it's a bit too aggressive to require a very new version of a low-level requirement like this, but regardless the listed requirement should be bumped correspondingly (right now it's at 1.13.0 https://github.com/PrefectHQ/prefect/blob/b133b44cffab79735a8fe86e16c81d3e6c74d1ab/setup.py#L35 which is from October 2018). Would a `try: ... except AttributeError:` block be acceptable as a way to maintain backwards compatibility?

Discussion:

Patch:
diff --git a/changes/issue3870.yaml b/changes/issue3870.yaml
new file mode 100644
index 0000000000..6167f76816
--- /dev/null
+++ b/changes/issue3870.yaml
@@ -0,0 +1,2 @@
+fix:
+  - "Add support for `google-cloud-storage` < 1.31.0 - [#3875](https://github.com/PrefectHQ/prefect/pull/3875)"
diff --git a/src/prefect/engine/results/gcs_result.py b/src/prefect/engine/results/gcs_result.py
index c6ae22cb78..ab5c9a5b92 100644
--- a/src/prefect/engine/results/gcs_result.py
+++ b/src/prefect/engine/results/gcs_result.py
@@ -94,7 +94,13 @@ class GCSResult(Result):
 
         try:
             self.logger.debug("Starting to download result from {}...".format(location))
-            serialized_value = self.gcs_bucket.blob(location).download_as_bytes()
+            blob = self.gcs_bucket.blob(location)
+            # Support GCS < 1.31
+            serialized_value = (
+                blob.download_as_bytes()
+                if hasattr(blob, "download_as_bytes")
+                else blob.download_as_string()
+            )
             try:
                 new.value = new.serializer.deserialize(serialized_value)
             except EOFError:
diff --git a/src/prefect/storage/gcs.py b/src/prefect/storage/gcs.py
index 7a65ac0cda..bd878b9954 100644
--- a/src/prefect/storage/gcs.py
+++ b/src/prefect/storage/gcs.py
@@ -99,7 +99,12 @@ class GCS(Storage):
                     flow_location, self.bucket
                 )
             )
-        content = blob.download_as_string()
+        # Support GCS < 1.31
+        content = (
+            blob.download_as_bytes()
+            if hasattr(blob, "download_as_bytes")
+            else blob.download_as_string()
+        )
 
         if self.stored_as_script:
             return extract_flow_from_file(file_contents=content)
diff --git a/src/prefect/tasks/gcp/storage.py b/src/prefect/tasks/gcp/storage.py
index 11da317ab6..0e9d758acf 100644
--- a/src/prefect/tasks/gcp/storage.py
+++ b/src/prefect/tasks/gcp/storage.py
@@ -168,14 +168,18 @@ class GCSDownload(GCSBaseTask):
         )
 
         # identify blob name
-        gcs_blob = self._get_blob(
+        blob = self._get_blob(
             bucket,
             blob,
             encryption_key=encryption_key,
             encryption_key_secret=encryption_key_secret,
         )
-        data = gcs_blob.download_as_string(timeout=request_timeout)
-        return data
+        # Support GCS < 1.31
+        return (
+            blob.download_as_bytes(timeout=request_timeout)
+            if hasattr(blob, "download_as_bytes")
+            else blob.download_as_string(timeout=request_timeout)
+        )
 
 
 class GCSUpload(GCSBaseTask):
diff --git a/src/prefect/utilities/filesystems.py b/src/prefect/utilities/filesystems.py
index 266a1efa70..91b090ff92 100644
--- a/src/prefect/utilities/filesystems.py
+++ b/src/prefect/utilities/filesystems.py
@@ -66,7 +66,12 @@ def read_bytes_from_path(path: str) -> bytes:
         blob = bucket.get_blob(parsed.path.lstrip("/"))
         if blob is None:
             raise ValueError(f"Job template doesn't exist at {path}")
-        return blob.download_as_bytes()
+        # Support GCS < 1.31
+        return (
+            blob.download_as_bytes()
+            if hasattr(blob, "download_as_bytes")
+            else blob.download_as_string()
+        )
     elif parsed.scheme == "s3":
         from prefect.utilities.aws import get_boto_client
 


Test patch:
diff --git a/tests/storage/test_gcs_storage.py b/tests/storage/test_gcs_storage.py
index 862e165a8f..dbaf160d25 100644
--- a/tests/storage/test_gcs_storage.py
+++ b/tests/storage/test_gcs_storage.py
@@ -92,7 +92,7 @@ class TestGCSStorage:
         f = Flow("awesome-flow")
         flow_content = cloudpickle.dumps(f)
 
-        blob_mock = MagicMock(download_as_string=MagicMock(return_value=flow_content))
+        blob_mock = MagicMock(download_as_bytes=MagicMock(return_value=flow_content))
         bucket_mock = MagicMock(get_blob=MagicMock(return_value=blob_mock))
         google_client.return_value.get_bucket = MagicMock(return_value=bucket_mock)
 
@@ -107,13 +107,13 @@ class TestGCSStorage:
 
         assert fetched_flow.name == f.name
         bucket_mock.get_blob.assert_called_with("a-place")
-        assert blob_mock.download_as_string.call_count == 1
+        assert blob_mock.download_as_bytes.call_count == 1
 
     def test_get_flow_from_gcs_as_file(self, google_client):
         f = Flow("awesome-flow")
         flow_content = """from prefect import Flow\nf=Flow('awesome-flow')"""
 
-        blob_mock = MagicMock(download_as_string=MagicMock(return_value=flow_content))
+        blob_mock = MagicMock(download_as_bytes=MagicMock(return_value=flow_content))
         bucket_mock = MagicMock(get_blob=MagicMock(return_value=blob_mock))
         google_client.return_value.get_bucket = MagicMock(return_value=bucket_mock)
 
@@ -124,7 +124,7 @@ class TestGCSStorage:
 
         assert fetched_flow.name == f.name
         bucket_mock.get_blob.assert_called_with("a-place")
-        assert blob_mock.download_as_string.call_count == 1
+        assert blob_mock.download_as_bytes.call_count == 1
 
     def test_build_no_upload_if_file_and_no_local_script_path(self, google_client):
         storage = GCS(bucket="awesome-bucket", stored_as_script=True)