Repository: googleapis/java-storage
Pull request #2469 (2024-03-25 20:44:40)

chore: Transfer Manager Parallel Composite Uploads Prework
Performing a chunked upload vs a chunked download are very different operations and should be able to be configured separately. This renames the previous allowDivideAndConquer to be allowDivideAndConquerDownload and introduces a new field specific to ParallelCompositeUploads.

BenWhitehead: Changed the title from `feat` to `chore` since this is pre-work. When TM is leveraging the PCU implementation is when we'd want the `feat` in the title.

Patch:
diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml
index 7fffdb6b..d2272068 100644
--- a/google-cloud-storage/clirr-ignored-differences.xml
+++ b/google-cloud-storage/clirr-ignored-differences.xml
@@ -61,4 +61,14 @@
     <className>com/google/cloud/storage/ParallelCompositeUploadBlobWriteSessionConfig$PartNamingStrategy</className>
     <method>java.lang.String fmtFields(*)</method>
   </difference>
+  <difference>
+    <differenceType>7002</differenceType>
+    <className>com/google/cloud/storage/transfermanager/TransferManagerConfig</className>
+    <method>boolean isAllowDivideAndConquer()</method>
+  </difference>
+  <difference>
+    <differenceType>7002</differenceType>
+    <className>com/google/cloud/storage/transfermanager/TransferManagerConfig$Builder</className>
+    <method>* setAllowDivideAndConquer(boolean)</method>
+  </difference>
 </differences>
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerConfig.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerConfig.java
index 835d5271..b638e46f 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerConfig.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerConfig.java
@@ -30,7 +30,8 @@ import java.util.Objects;
 public final class TransferManagerConfig {
   private final int maxWorkers;
   private final int perWorkerBufferSize;
-  private final boolean allowDivideAndConquer;
+  private final boolean allowDivideAndConquerDownload;
+  private final boolean allowParallelCompositeUpload;
 
   private final StorageOptions storageOptions;
   private final Qos qos;
@@ -38,12 +39,14 @@ public final class TransferManagerConfig {
   TransferManagerConfig(
       int maxWorkers,
       int perWorkerBufferSize,
-      boolean allowDivideAndConquer,
+      boolean allowDivideAndConquerDownload,
+      boolean allowParallelCompositeUpload,
       StorageOptions storageOptions,
       Qos qos) {
     this.maxWorkers = maxWorkers;
     this.perWorkerBufferSize = perWorkerBufferSize;
-    this.allowDivideAndConquer = allowDivideAndConquer;
+    this.allowDivideAndConquerDownload = allowDivideAndConquerDownload;
+    this.allowParallelCompositeUpload = allowParallelCompositeUpload;
     this.storageOptions = storageOptions;
     this.qos = qos;
   }
@@ -72,11 +75,21 @@ public final class TransferManagerConfig {
    * Whether to allow Transfer Manager to perform chunked Uploads/Downloads if it determines
    * chunking will be beneficial
    *
-   * @see Builder#setAllowDivideAndConquer(boolean)
+   * @see Builder#setAllowDivideAndConquerDownload(boolean)
    */
   @BetaApi
-  public boolean isAllowDivideAndConquer() {
-    return allowDivideAndConquer;
+  public boolean isAllowDivideAndConquerDownload() {
+    return allowDivideAndConquerDownload;
+  }
+  /**
+   * Whether to allow Transfer Manager to perform Parallel Composite Uploads if it determines
+   * chunking will be beneficial
+   *
+   * @see Builder#setAllowParallelCompositeUpload(boolean)
+   */
+  @BetaApi
+  public boolean isAllowParallelCompositeUpload() {
+    return allowParallelCompositeUpload;
   }
 
   /**
@@ -98,7 +111,8 @@ public final class TransferManagerConfig {
   @BetaApi
   public Builder toBuilder() {
     return new Builder()
-        .setAllowDivideAndConquer(allowDivideAndConquer)
+        .setAllowDivideAndConquerDownload(allowDivideAndConquerDownload)
+        .setAllowParallelCompositeUpload(allowParallelCompositeUpload)
         .setMaxWorkers(maxWorkers)
         .setPerWorkerBufferSize(perWorkerBufferSize)
         .setQos(qos)
@@ -120,13 +134,19 @@ public final class TransferManagerConfig {
     TransferManagerConfig that = (TransferManagerConfig) o;
     return maxWorkers == that.maxWorkers
         && perWorkerBufferSize == that.perWorkerBufferSize
-        && allowDivideAndConquer == that.allowDivideAndConquer
+        && allowDivideAndConquerDownload == that.allowDivideAndConquerDownload
+        && allowParallelCompositeUpload == that.allowParallelCompositeUpload
         && Objects.equals(storageOptions, that.storageOptions);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(maxWorkers, perWorkerBufferSize, allowDivideAndConquer, storageOptions);
+    return Objects.hash(
+        maxWorkers,
+        perWorkerBufferSize,
+        allowDivideAndConquerDownload,
+        allowParallelCompositeUpload,
+        storageOptions);
   }
 
   @Override
@@ -134,7 +154,8 @@ public final class TransferManagerConfig {
     return MoreObjects.toStringHelper(this)
         .add("maxWorkers", maxWorkers)
         .add("perWorkerBufferSize", perWorkerBufferSize)
-        .add("allowChunking", allowDivideAndConquer)
+        .add("allowDivideAndConquerDownload", allowDivideAndConquerDownload)
+        .add("allowParallelCompositeUpload", allowParallelCompositeUpload)
         .add("storageOptions", storageOptions)
         .toString();
   }
@@ -154,7 +175,8 @@ public final class TransferManagerConfig {
 
     private int maxWorkers;
     private int perWorkerBufferSize;
-    private boolean allowDivideAndConquer;
+    private boolean allowDivideAndConquerDownload;
+    private boolean allowParallelCompositeUpload;
 
     private StorageOptions storageOptions;
     private Qos qos;
@@ -162,7 +184,8 @@ public final class TransferManagerConfig {
     private Builder() {
       this.perWorkerBufferSize = 16 * 1024 * 1024;
       this.maxWorkers = 2 * Runtime.getRuntime().availableProcessors();
-      this.allowDivideAndConquer = false;
+      this.allowDivideAndConquerDownload = false;
+      this.allowParallelCompositeUpload = false;
       this.storageOptions = StorageOptions.getDefaultInstance();
       this.qos = DefaultQos.of();
     }
@@ -202,12 +225,27 @@ public final class TransferManagerConfig {
      *
      * <p><i>Default Value:</i> false
      *
-     * @return the instance of Builder with the value for allowDivideAndConquer modified.
-     * @see TransferManagerConfig#isAllowDivideAndConquer()
+     * @return the instance of Builder with the value for allowDivideAndConquerDownload modified.
+     * @see TransferManagerConfig#isAllowDivideAndConquerDownload()
+     */
+    @BetaApi
+    public Builder setAllowDivideAndConquerDownload(boolean allowDivideAndConquerDownload) {
+      this.allowDivideAndConquerDownload = allowDivideAndConquerDownload;
+      return this;
+    }
+
+    /**
+     * Whether to allow Transfer Manager to perform Parallel Composite Uploads if it determines
+     * chunking will be beneficial
+     *
+     * <p><i>Default Value:</i> false
+     *
+     * @return the instance of Builder with the value for allowDivideAndConquerDownload modified.
+     * @see TransferManagerConfig#isAllowDivideAndConquerDownload()
      */
     @BetaApi
-    public Builder setAllowDivideAndConquer(boolean allowDivideAndConquer) {
-      this.allowDivideAndConquer = allowDivideAndConquer;
+    public Builder setAllowParallelCompositeUpload(boolean allowDivideAndConquerDownload) {
+      this.allowDivideAndConquerDownload = allowDivideAndConquerDownload;
       return this;
     }
 
@@ -239,7 +277,12 @@ public final class TransferManagerConfig {
     @BetaApi
     public TransferManagerConfig build() {
       return new TransferManagerConfig(
-          maxWorkers, perWorkerBufferSize, allowDivideAndConquer, storageOptions, qos);
+          maxWorkers,
+          perWorkerBufferSize,
+          allowDivideAndConquerDownload,
+          allowParallelCompositeUpload,
+          storageOptions,
+          qos);
     }
   }
 }
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerImpl.java
index 2807d792..b731c2c3 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerImpl.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerImpl.java
@@ -103,7 +103,7 @@ final class TransferManagerImpl implements TransferManager {
     Storage.BlobSourceOption[] opts =
         config.getOptionsPerRequest().toArray(new Storage.BlobSourceOption[0]);
     List<ApiFuture<DownloadResult>> downloadTasks = new ArrayList<>();
-    if (!transferManagerConfig.isAllowDivideAndConquer()) {
+    if (!transferManagerConfig.isAllowDivideAndConquerDownload()) {
       for (BlobInfo blob : blobs) {
         DirectDownloadCallable callable = new DirectDownloadCallable(storage, blob, config, opts);
         downloadTasks.add(convert(executor.submit(callable)));


Test patch:
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java
index 7416109b..4f345af0 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java
@@ -303,7 +303,7 @@ public class ITTransferManagerTest {
     TransferManagerConfig config =
         TransferManagerConfigTestingInstances.defaults(storage.getOptions())
             .toBuilder()
-            .setAllowDivideAndConquer(true)
+            .setAllowDivideAndConquerDownload(true)
             .setPerWorkerBufferSize(128 * 1024)
             .build();
     try (TransferManager transferManager = config.getService()) {
@@ -367,7 +367,7 @@ public class ITTransferManagerTest {
     TransferManagerConfig config =
         TransferManagerConfigTestingInstances.defaults(storage.getOptions())
             .toBuilder()
-            .setAllowDivideAndConquer(true)
+            .setAllowDivideAndConquerDownload(true)
             .setPerWorkerBufferSize(128 * 1024)
             .build();
     try (TransferManager transferManager = config.getService()) {
@@ -393,7 +393,7 @@ public class ITTransferManagerTest {
     TransferManagerConfig config =
         TransferManagerConfigTestingInstances.defaults(storage.getOptions())
             .toBuilder()
-            .setAllowDivideAndConquer(true)
+            .setAllowDivideAndConquerDownload(true)
             .setPerWorkerBufferSize(128 * 1024)
             .build();
     try (TransferManager transferManager = config.getService()) {
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/transfermanager/TransferManagerConfigTestingInstances.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/transfermanager/TransferManagerConfigTestingInstances.java
index 1c5e67b9..e9ef86c3 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/transfermanager/TransferManagerConfigTestingInstances.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/transfermanager/TransferManagerConfigTestingInstances.java
@@ -27,7 +27,7 @@ public final class TransferManagerConfigTestingInstances {
 
   public static TransferManagerConfig defaults(StorageOptions options) {
     return TransferManagerConfig.newBuilder()
-        .setAllowDivideAndConquer(false)
+        .setAllowDivideAndConquerDownload(false)
         .setMaxWorkers(1)
         .setPerWorkerBufferSize(512 * 1024)
         .setQos(DefaultQos.of(2 * 1024 * 1024))