Repository: mabel-dev/opteryx

Issue:
✨ GCS improvements
Create the client object once and reuse

List blobs should only return the name of the blob and not any other details 


Discussion:
On a test dataset of 49 files on GCS - before the change, planning took 1.45s (fastest of four) and reading 40.3s (fastest of four). following the changes, planning took 1.2s (fastest of four) and reading took 20.5s (fastest of four)

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 189
+__build__ = 193
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,8 +28,8 @@ class VersionStatus(Enum):
 
 _major = 0
 _minor = 12
-_revision = 2
-_status = VersionStatus.RELEASE
+_revision = 3
+_status = VersionStatus.BETA
 
 __version__ = f"{_major}.{_minor}.{_revision}" + (
     f"-{_status.value}.{__build__}" if _status != VersionStatus.RELEASE else ""
diff --git a/opteryx/connectors/gcp_cloudstorage_connector.py b/opteryx/connectors/gcp_cloudstorage_connector.py
--- a/opteryx/connectors/gcp_cloudstorage_connector.py
+++ b/opteryx/connectors/gcp_cloudstorage_connector.py
@@ -64,6 +64,7 @@ def __init__(self, credentials=None, **kwargs):
         # we're going to cache the first blob as the schema and dataset reader
         # sometimes both start here
         self.cached_first_blob = None
+        self.client = self._get_storage_client()
 
     def _get_storage_client(self):
         from google.cloud import storage
@@ -76,9 +77,7 @@ def _get_storage_client(self):
             return storage.Client()
 
     def _get_blob(self, bucket: str, blob_name: str):
-        client = self._get_storage_client()
-
-        gcs_bucket = client.get_bucket(bucket)
+        gcs_bucket = self.client.get_bucket(bucket)
         blob = gcs_bucket.get_blob(blob_name)
         return blob
 
@@ -100,10 +99,8 @@ def get_list_of_blob_names(self, *, prefix: str) -> List[str]:
         bucket = bucket.replace("va_data", "va-data")
         bucket = bucket.replace("data_", "data-")
 
-        client = self._get_storage_client()
-
-        gcs_bucket = client.get_bucket(bucket)
-        blobs = client.list_blobs(bucket_or_name=gcs_bucket, prefix=object_path)
+        gcs_bucket = self.client.get_bucket(bucket)
+        blobs = self.client.list_blobs(bucket_or_name=gcs_bucket, prefix=object_path, fields="items(name)")
         blobs = (bucket + "/" + blob.name for blob in blobs if not blob.name.endswith("/"))
         return [blob for blob in blobs if ("." + blob.split(".")[-1].lower()) in VALID_EXTENSIONS]
 
@@ -117,12 +114,6 @@ def read_dataset(
             prefix=self.dataset,
         )
 
-        # Check if the first blob was cached earlier
-        #        if self.cached_first_blob is not None:
-        #            yield self.cached_first_blob  # Use cached blob
-        #            blob_names = blob_names[1:]  # Skip first blob
-        #        self.cached_first_blob = None
-
         for blob_name in blob_names:
             try:
                 decoder = get_decoder(blob_name)


Test patch: