Repository: GoogleCloudPlatform/cloud-bigtable-client (java)

Issue:
Client will retry until max timeout is reached, even if deadline is exceeded
I've been experimenting with using gRPC itself to set my per-call timeouts.  I've noticed that if I set my per-call deadline low, lets say in this example to 1 second, I can end up in a situation where the client submits "requests" even though they'll never actually be sent to bigtable (since the deadline is already exceeded).

Example log:
```
2018-06-24 19:22:36,317 INF AbstractRetryingOperation Retrying failed call. Failure #1, got: Status{code=DEADLINE_EXCEEDED, description=deadline exceeded after 739333109ns, cause=null} on channel 14
2018-06-24 19:22:36,325 INF AbstractRetryingOperation Retrying failed call. Failure #2, got: Status{code=DEADLINE_EXCEEDED, description=deadline exceeded: -16434248 ns from now, cause=null} on channel 15
2018-06-24 19:22:36,341 INF AbstractRetryingOperation Retrying failed call. Failure #3, got: Status{code=DEADLINE_EXCEEDED, description=deadline exceeded: -31615089 ns from now, cause=null} on channel 16
2018-06-24 19:22:36,366 INF AbstractRetryingOperation Retrying failed call. Failure #4, got: Status{code=DEADLINE_EXCEEDED, description=deadline exceeded: -56484939 ns from now, cause=null} on channel 17
2018-06-24 19:22:36,396 INF AbstractRetryingOperation Retrying failed call. Failure #5, got: Status{code=DEADLINE_EXCEEDED, description=deadline exceeded: -86724983 ns from now, cause=null} on channel 18
```

It'll then retry until it reaches the max-retry limit based on the retry policy.  It seems like the client should stop retrying once the deadline has been exceeded here.

Pull request:
Moving the creation of Backoff to AbstractRetryingOperation
RetryOptions had the creation of a Backoff for historical reasons.  The only place that its used is AbstractRetryingOperation.  This refactoring will be part of a larger set of PRs that work to fix #1858.

Patch:
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/config/RetryOptions.java b/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/config/RetryOptions.java
index af5dd1fec..ed311c5d8 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/config/RetryOptions.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/config/RetryOptions.java
@@ -343,28 +343,6 @@ public class RetryOptions implements Serializable {
   public boolean isRetryable(Status.Code code) {
     return statusToRetryOn.contains(code);
   }
-
-  /**
-   * <p>createBackoff.</p>
-   *
-   * @return a {@link com.google.api.client.util.BackOff} object.
-   */
-  public BackOff createBackoff() {
-    return createBackoffBuilder().build();
-  }
-
-  /**
-   * <p>createBackoffBuilder.</p>
-   *
-   * @return a {@link com.google.api.client.util.ExponentialBackOff.Builder} object.
-   */
-  @VisibleForTesting
-  protected ExponentialBackOff.Builder createBackoffBuilder() {
-    return new ExponentialBackOff.Builder()
-        .setInitialIntervalMillis(getInitialBackoffMillis())
-        .setMaxElapsedTimeMillis(getMaxElapsedBackoffMillis())
-        .setMultiplier(getBackoffMultiplier());
-  }
   
   /** {@inheritDoc} */
   @Override
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/async/AbstractRetryingOperation.java b/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/async/AbstractRetryingOperation.java
index 6906c03a0..8e0652506 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/async/AbstractRetryingOperation.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/async/AbstractRetryingOperation.java
@@ -25,8 +25,8 @@ import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
 
 import com.google.api.client.util.BackOff;
+import com.google.api.client.util.ExponentialBackOff;
 import com.google.api.client.util.Sleeper;
-import com.google.bigtable.v2.BigtableGrpc;
 import com.google.bigtable.v2.ReadRowsRequest;
 import com.google.cloud.bigtable.config.Logger;
 import com.google.cloud.bigtable.config.RetryOptions;
@@ -43,8 +43,6 @@ import com.google.common.util.concurrent.ListenableFuture;
 import io.grpc.CallOptions;
 import io.grpc.ClientCall;
 import io.grpc.Metadata;
-import io.grpc.MethodDescriptor;
-import io.grpc.MethodDescriptor.MethodType;
 import io.grpc.Status;
 import io.grpc.Status.Code;
 import io.opencensus.contrib.grpc.util.StatusConverter;
@@ -216,8 +214,8 @@ public abstract class AbstractRetryingOperation<RequestT, ResponseT, ResultT>
     // Unauthenticated is special because the request never made it to
     // to the server, so all requests are retryable
         || !(isRequestRetryable() || code == Code.UNAUTHENTICATED || code == Code.UNAVAILABLE)) {
-      LOG.info("Could not complete RPC. Failure #%d, got: %s on channel %s", status.getCause(),
-        failedCount, status, channelId);
+      LOG.info("Could not complete RPC. Failure #%d, got: %s on channel %s.\nTrailers: %s",
+          status.getCause(), failedCount, status, channelId, trailers);
       rpc.getRpcMetrics().markFailure();
       finalizeStats(status);
       setException(status.asRuntimeException());
@@ -230,12 +228,12 @@ public abstract class AbstractRetryingOperation<RequestT, ResponseT, ResultT>
 
     // Backoffs timed out.
     if (nextBackOff == BackOff.STOP) {
-      LOG.info("All retries were exhausted. Failure #%d, got: %s on channel %s", status.getCause(),
-        failedCount, status, channelId);
+      LOG.info("All retries were exhausted. Failure #%d, got: %s on channel %s.\nTrailers: %s",
+          status.getCause(), failedCount, status, channelId, trailers);
       setException(getExhaustedRetriesException(status));
     } else {
-      LOG.info("Retrying failed call. Failure #%d, got: %s on channel %s", status.getCause(),
-        failedCount, status, channelId);
+      LOG.info("Retrying failed call. Failure #%d, got: %s on channel %s.\nTrailers: %s",
+          status.getCause(), failedCount, status, channelId, trailers);
       performRetry(nextBackOff);
     }
   }
@@ -282,7 +280,7 @@ public abstract class AbstractRetryingOperation<RequestT, ResponseT, ResultT>
 
   protected long getNextBackoff() {
     if (currentBackoff == null) {
-      currentBackoff = retryOptions.createBackoff();
+      currentBackoff = createBackoff();
     }
     try {
       return currentBackoff.nextBackOffMillis();
@@ -291,6 +289,20 @@ public abstract class AbstractRetryingOperation<RequestT, ResponseT, ResultT>
     }
   }
 
+  /**
+   * <p>createBackoff.</p>
+   *
+   * @return a {@link com.google.api.client.util.ExponentialBackOff} object.
+   */
+  @VisibleForTesting
+  protected ExponentialBackOff createBackoff() {
+    return new ExponentialBackOff.Builder()
+            .setInitialIntervalMillis(retryOptions.getInitialBackoffMillis())
+            .setMaxElapsedTimeMillis(retryOptions.getMaxElapsedBackoffMillis())
+            .setMultiplier(retryOptions.getBackoffMultiplier())
+            .build();
+  }
+
   /**
    * Calls {@link BigtableAsyncRpc#newCall(CallOptions)} and
    * {@link BigtableAsyncRpc#start(Object, io.grpc.ClientCall.Listener, Metadata, ClientCall)} }
diff --git a/pom.xml b/pom.xml
index 1fcffeb25..c59df3e49 100644
--- a/pom.xml
+++ b/pom.xml
@@ -59,8 +59,8 @@ limitations under the License.
         <hamcrest.version>1.3</hamcrest.version>
 
         <!-- google-cloud-java related dependency versions -->
-        <google-cloud.version>0.53.0-alpha</google-cloud.version>
-        <opencensus.version>0.11.1</opencensus.version>
+        <google-cloud.version>0.54.0-alpha</google-cloud.version>
+        <opencensus.version>0.12.3</opencensus.version>
         <guava.version>20.0</guava.version>
 
         <!-- hbase dependency versions -->


Test patch:
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/config/RetryOptionsUtil.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/config/RetryOptionsUtil.java
deleted file mode 100644
index 398c84fee..000000000
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/config/RetryOptionsUtil.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.cloud.bigtable.config;
-
-import static com.google.cloud.bigtable.config.RetryOptions.DEFAULT_BACKOFF_MULTIPLIER;
-import static com.google.cloud.bigtable.config.RetryOptions.DEFAULT_INITIAL_BACKOFF_MILLIS;
-import static com.google.cloud.bigtable.config.RetryOptions.DEFAULT_MAX_ELAPSED_BACKOFF_MILLIS;
-import static com.google.cloud.bigtable.config.RetryOptions.DEFAULT_MAX_SCAN_TIMEOUT_RETRIES;
-import static com.google.cloud.bigtable.config.RetryOptions.DEFAULT_READ_PARTIAL_ROW_TIMEOUT_MS;
-import static com.google.cloud.bigtable.config.RetryOptions.DEFAULT_STREAMING_BUFFER_SIZE;
-import static com.google.cloud.bigtable.config.RetryOptions.DEFAULT_ENABLE_GRPC_RETRIES_SET;
-
-import com.google.api.client.util.ExponentialBackOff;
-import com.google.api.client.util.NanoClock;
-
-/**
- * Testing utility for creating RetryOptions
- */
-public class RetryOptionsUtil {
-  public static RetryOptions createTestRetryOptions(final NanoClock nanoClock) {
-    return createTestRetryOptions(nanoClock, false);
-  }
-
-  public static RetryOptions createTestRetryOptions(final NanoClock nanoClock,
-      boolean allowRetriesWithoutTimestamp) {
-    return new RetryOptions(true, allowRetriesWithoutTimestamp, DEFAULT_INITIAL_BACKOFF_MILLIS,
-        DEFAULT_BACKOFF_MULTIPLIER, DEFAULT_MAX_ELAPSED_BACKOFF_MILLIS,
-        DEFAULT_STREAMING_BUFFER_SIZE, DEFAULT_READ_PARTIAL_ROW_TIMEOUT_MS,
-        DEFAULT_MAX_SCAN_TIMEOUT_RETRIES, DEFAULT_ENABLE_GRPC_RETRIES_SET) {
-      private static final long serialVersionUID = 1L;
-
-      @Override
-      protected ExponentialBackOff.Builder createBackoffBuilder() {
-        return super.createBackoffBuilder().setNanoClock(nanoClock);
-      }
-    };
-  }
-}
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableDataGrpcClient.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableDataGrpcClient.java
index 2eb672b82..9c00c28d2 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableDataGrpcClient.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableDataGrpcClient.java
@@ -51,7 +51,6 @@ import com.google.bigtable.v2.ReadRowsResponse;
 import com.google.bigtable.v2.RowRange;
 import com.google.cloud.bigtable.config.BigtableOptions;
 import com.google.cloud.bigtable.config.RetryOptions;
-import com.google.cloud.bigtable.config.RetryOptionsUtil;
 import com.google.cloud.bigtable.grpc.io.GoogleCloudResourcePrefixInterceptor;
 import com.google.cloud.bigtable.grpc.scanner.FlatRow;
 import com.google.cloud.bigtable.grpc.scanner.ResultScanner;
@@ -79,9 +78,6 @@ public class TestBigtableDataGrpcClient {
   @Mock
   ClientCall mockClientCall;
 
-  @Mock
-  NanoClock nanoClock;
-
   BigtableDataGrpcClient defaultClient;
 
   @Before
@@ -93,8 +89,8 @@ public class TestBigtableDataGrpcClient {
   }
 
   protected BigtableDataGrpcClient createClient(boolean allowRetriesWithoutTimestamp) {
-    RetryOptions retryOptions =
-        RetryOptionsUtil.createTestRetryOptions(nanoClock, allowRetriesWithoutTimestamp);
+    RetryOptions retryOptions = new RetryOptions.Builder()
+            .setAllowRetriesWithoutTimestamp(allowRetriesWithoutTimestamp).build();
     BigtableOptions options = new BigtableOptions.Builder().setRetryOptions(retryOptions).build();
     doAnswer(new Answer<Void>(){
       @Override
@@ -107,7 +103,7 @@ public class TestBigtableDataGrpcClient {
   }
 
   @Test
-  public void testRetyableMutateRow() throws Exception {
+  public void testRetryableMutateRow() {
     MutateRowRequest request = MutateRowRequest.newBuilder().setTableName(TABLE_NAME).build();
     setResponse(MutateRowResponse.getDefaultInstance());
     defaultClient.mutateRow(request);
@@ -115,14 +111,14 @@ public class TestBigtableDataGrpcClient {
   }
 
   @Test
-  public void testRetyableMutateRowAsync() {
+  public void testRetryableMutateRowAsync() {
     MutateRowRequest request = MutateRowRequest.newBuilder().setTableName(TABLE_NAME).build();
     defaultClient.mutateRowAsync(request);
     verifyRequestCalled(request);
   }
 
   @Test
-  public void testRetyableCheckAndMutateRow() throws Exception {
+  public void testRetryableCheckAndMutateRow() {
     CheckAndMutateRowRequest request =
         CheckAndMutateRowRequest.newBuilder().setTableName(TABLE_NAME).build();
     setResponse(CheckAndMutateRowResponse.getDefaultInstance());
@@ -131,7 +127,7 @@ public class TestBigtableDataGrpcClient {
   }
 
   @Test
-  public void testRetyableCheckAndMutateRowAsync() {
+  public void testRetryableCheckAndMutateRowAsync() {
     CheckAndMutateRowRequest request =
         CheckAndMutateRowRequest.newBuilder().setTableName(TABLE_NAME).build();
     defaultClient.checkAndMutateRowAsync(request);
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableTableAdminGrpcClient.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableTableAdminGrpcClient.java
index 052de1a5c..0dc1d52ef 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableTableAdminGrpcClient.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/TestBigtableTableAdminGrpcClient.java
@@ -26,7 +26,6 @@ import com.google.bigtable.admin.v2.GenerateConsistencyTokenRequest;
 import com.google.bigtable.admin.v2.GenerateConsistencyTokenResponse;
 import com.google.bigtable.admin.v2.GetSnapshotRequest;
 import com.google.bigtable.admin.v2.GetTableRequest;
-import com.google.bigtable.admin.v2.Instance;
 import com.google.bigtable.admin.v2.ListSnapshotsRequest;
 import com.google.bigtable.admin.v2.ModifyColumnFamiliesRequest;
 import com.google.bigtable.admin.v2.SnapshotTableRequest;
@@ -34,7 +33,6 @@ import com.google.bigtable.admin.v2.Table;
 
 import com.google.cloud.bigtable.config.BigtableOptions;
 import com.google.cloud.bigtable.config.RetryOptions;
-import com.google.cloud.bigtable.config.RetryOptionsUtil;
 import io.grpc.CallOptions;
 import io.grpc.Channel;
 import io.grpc.ClientCall;
@@ -52,7 +50,6 @@ import org.mockito.MockitoAnnotations;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -94,9 +91,7 @@ public class TestBigtableTableAdminGrpcClient {
   }
 
   protected BigtableTableAdminGrpcClient createClient(boolean allowRetriesWithoutTimestamp) {
-    RetryOptions retryOptions =
-        RetryOptionsUtil.createTestRetryOptions(nanoClock, allowRetriesWithoutTimestamp);
-    BigtableOptions options = new BigtableOptions.Builder().setRetryOptions(retryOptions).build();
+    BigtableOptions options = new BigtableOptions.Builder().build();
     return new BigtableTableAdminGrpcClient(mockChannel, null, options);
   }
 
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestMutateRowsRequestManager.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestMutateRowsRequestManager.java
index 5988ce175..598220a9c 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestMutateRowsRequestManager.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestMutateRowsRequestManager.java
@@ -27,14 +27,12 @@ import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.MockitoAnnotations;
 
-import com.google.api.client.util.NanoClock;
 import com.google.bigtable.v2.MutateRowsRequest;
 import com.google.bigtable.v2.MutateRowsRequest.Entry;
 import com.google.bigtable.v2.MutateRowsResponse;
 import com.google.bigtable.v2.Mutation;
 import com.google.bigtable.v2.Mutation.SetCell;
 import com.google.cloud.bigtable.config.RetryOptions;
-import com.google.cloud.bigtable.config.RetryOptionsUtil;
 import com.google.cloud.bigtable.grpc.async.MutateRowsRequestManager.ProcessingStatus;
 import com.google.rpc.Status;
 
@@ -91,19 +89,13 @@ public class TestMutateRowsRequestManager {
   }
 
   private AtomicLong time = new AtomicLong();
-  private NanoClock nanoClock = new NanoClock() {
-    @Override
-    public long nanoTime() {
-      return time.get();
-    }
-  };
 
   private RetryOptions retryOptions;
 
   @Before
   public void setup() {
     MockitoAnnotations.initMocks(this);
-    retryOptions = RetryOptionsUtil.createTestRetryOptions(nanoClock);
+    retryOptions = new RetryOptions.Builder().build();
   }
 
   @Test
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingMutateRowsOperation.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingMutateRowsOperation.java
index 16fb8c0c7..45433b75d 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingMutateRowsOperation.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingMutateRowsOperation.java
@@ -41,7 +41,6 @@ import com.google.bigtable.v2.MutateRowsResponse;
 import com.google.bigtable.v2.Mutation;
 import com.google.bigtable.v2.Mutation.SetCell;
 import com.google.cloud.bigtable.config.RetryOptions;
-import com.google.cloud.bigtable.config.RetryOptionsUtil;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.rpc.Status;
 
@@ -129,7 +128,7 @@ public class TestRetryingMutateRowsOperation {
     MockitoAnnotations.initMocks(this);
     when(mutateRows.getRpcMetrics()).thenReturn(metrics);
     when(mutateRows.getMethodDescriptor()).thenReturn(BigtableGrpc.getMutateRowsMethod());
-    retryOptions = RetryOptionsUtil.createTestRetryOptions(nanoClock);
+    retryOptions = new RetryOptions.Builder().build();
   }
 
   @Test
diff --git a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingUnaryOperation.java b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingUnaryOperation.java
index e2980f870..284158c5e 100644
--- a/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingUnaryOperation.java
+++ b/bigtable-client-core-parent/bigtable-client-core/src/test/java/com/google/cloud/bigtable/grpc/async/TestRetryingUnaryOperation.java
@@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.api.client.util.ExponentialBackOff;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
@@ -45,7 +46,6 @@ import com.google.bigtable.v2.BigtableGrpc;
 import com.google.bigtable.v2.ReadRowsRequest;
 import com.google.bigtable.v2.ReadRowsResponse;
 import com.google.cloud.bigtable.config.RetryOptions;
-import com.google.cloud.bigtable.config.RetryOptionsUtil;
 import com.google.cloud.bigtable.grpc.scanner.BigtableRetriesExhaustedException;
 import com.google.common.util.concurrent.ListenableFuture;
 import io.grpc.CallOptions;
@@ -88,14 +88,11 @@ public class TestRetryingUnaryOperation {
   @Before
   public void setup() {
     MockitoAnnotations.initMocks(this);
-    retryOptions = RetryOptionsUtil.createTestRetryOptions(nanoClock);
+    retryOptions = new RetryOptions.Builder().build();
 
     when(readAsync.getRpcMetrics()).thenReturn(metrics);
     when(readAsync.getMethodDescriptor()).thenReturn(BigtableGrpc.getReadRowsMethod());
 
-    underTest = new RetryingUnaryOperation<>(retryOptions, ReadRowsRequest.getDefaultInstance(),
-        readAsync, CallOptions.DEFAULT, executorService, new Metadata());
-
     totalSleep = new AtomicLong();
 
     doAnswer(new Answer<Void>() {
@@ -123,6 +120,21 @@ public class TestRetryingUnaryOperation {
       }
     });
     when(readAsync.isRetryable(any(ReadRowsRequest.class))).thenReturn(true);
+
+    underTest = new RetryingUnaryOperation<ReadRowsRequest, ReadRowsResponse>(retryOptions,
+            ReadRowsRequest.getDefaultInstance(), readAsync, CallOptions.DEFAULT, executorService,
+            new Metadata()) {
+        @Override
+        protected ExponentialBackOff createBackoff() {
+            return new ExponentialBackOff.Builder()
+                    .setNanoClock(nanoClock)
+                    .setInitialIntervalMillis(retryOptions.getInitialBackoffMillis())
+                    .setMaxElapsedTimeMillis(retryOptions.getMaxElapsedBackoffMillis())
+                    .setMultiplier(retryOptions.getBackoffMultiplier())
+                    .build();
+        }
+    };
+
   }
 
   @Test