Repository: DataBiosphere/terra-drs-hub
Pull request #147: CORE-447 retry drs api calls
Author: login: dvoet
Created: 2025-04-22T16:06:00Z | Merged: 2025-04-22T19:10:17Z

https://broadworkbench.atlassian.net/browse/CORE-447

The underlying problem is that when using passport authentication and ECM returns a 502 to TDR, TDR propagates the 502 to DrsHub, then DrsHub treats that a "passport doesn't work try something else" which returns a 403 which is the symptom the caller sees. The scale test shows just a few of these so we think a retry strategy should fix the problem instead of scaling up the infrastructure which costs $$$. Retrying is probably a good idea anyway.

I looked into using Spring's retry features but it was too much spring magic to make `DrsApi` a bean and other approaches take too much refactoring for not much benefit.


Diff:
diff --git a/service/src/main/java/bio/terra/drshub/models/DrsApi.java b/service/src/main/java/bio/terra/drshub/models/DrsApi.java
index b06550f0..1be5db22 100644
--- a/service/src/main/java/bio/terra/drshub/models/DrsApi.java
+++ b/service/src/main/java/bio/terra/drshub/models/DrsApi.java
@@ -1,20 +1,88 @@
 package bio.terra.drshub.models;
 
+import com.google.common.annotations.VisibleForTesting;
 import io.github.ga4gh.drs.api.ObjectsApi;
 import io.github.ga4gh.drs.client.ApiClient;
 import io.github.ga4gh.drs.client.auth.OAuth;
+import io.github.ga4gh.drs.model.AccessURL;
+import io.github.ga4gh.drs.model.Authorizations;
+import io.github.ga4gh.drs.model.DrsObject;
+import java.util.function.Supplier;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.client.HttpServerErrorException.BadGateway;
+import org.springframework.web.client.HttpServerErrorException.GatewayTimeout;
+import org.springframework.web.client.HttpServerErrorException.InternalServerError;
+import org.springframework.web.client.HttpServerErrorException.ServiceUnavailable;
+import org.springframework.web.client.ResourceAccessException;
+import org.springframework.web.client.RestClientException;
 
-public class DrsApi extends ObjectsApi {
+@Slf4j
+public class DrsApi {
+  static final int MAX_TRIALS = 4;
+
+  private ObjectsApi objectsApi;
 
   public DrsApi(ApiClient apiClient) {
-    super(apiClient);
+    this.objectsApi = new ObjectsApi(apiClient);
+  }
+
+  @VisibleForTesting
+  public ApiClient getApiClient() {
+    return objectsApi.getApiClient();
   }
 
   public void setBearerToken(String bearerToken) {
-    ((OAuth) this.getApiClient().getAuthentication("BearerAuth")).setAccessToken(bearerToken);
+    ((OAuth) objectsApi.getApiClient().getAuthentication("BearerAuth")).setAccessToken(bearerToken);
   }
 
   public void setHeader(String name, String value) {
-    this.getApiClient().addDefaultHeader(name, value);
+    objectsApi.getApiClient().addDefaultHeader(name, value);
+  }
+
+  public DrsObject postObject(Object body, String objectId) throws RestClientException {
+    return retry(() -> objectsApi.postObject(body, objectId));
+  }
+
+  public AccessURL postAccessURL(Object body, String objectId, String accessId)
+      throws RestClientException {
+    return retry(() -> objectsApi.postAccessURL(body, objectId, accessId));
+  }
+
+  public Authorizations optionsObject(String objectId) throws RestClientException {
+    return retry(() -> objectsApi.optionsObject(objectId));
+  }
+
+  public DrsObject getObject(String objectId, Boolean expand) throws RestClientException {
+    return retry(() -> objectsApi.getObject(objectId, expand));
+  }
+
+  public AccessURL getAccessURL(String objectId, String accessId) throws RestClientException {
+    return retry(() -> objectsApi.getAccessURL(objectId, accessId));
+  }
+
+  @VisibleForTesting
+  static <T> T retry(Supplier<T> supplier) {
+    for (int trial = 1; true; trial++) {
+      try {
+        return supplier.get();
+      } catch (InternalServerError
+          | BadGateway
+          | ServiceUnavailable
+          | GatewayTimeout
+          | ResourceAccessException e) {
+        if (trial == MAX_TRIALS) {
+          throw e;
+        } else {
+          log.info(
+              "retrying DRS call, attempt {} of {}, error {}", trial, MAX_TRIALS, e.getMessage());
+          try {
+            Thread.sleep(10L * trial);
+          } catch (InterruptedException ex) {
+            Thread.currentThread().interrupt();
+            throw e;
+          }
+        }
+      }
+    }
   }
 }
diff --git a/service/src/test/java/bio/terra/drshub/models/DrsApiTest.java b/service/src/test/java/bio/terra/drshub/models/DrsApiTest.java
index 6ee10851..b23fc5a2 100644
--- a/service/src/test/java/bio/terra/drshub/models/DrsApiTest.java
+++ b/service/src/test/java/bio/terra/drshub/models/DrsApiTest.java
@@ -1,17 +1,24 @@
 package bio.terra.drshub.models;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import io.github.ga4gh.drs.client.ApiClient;
 import io.github.ga4gh.drs.client.auth.OAuth;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.client.HttpServerErrorException;
 
 @Tag("Unit")
 @ExtendWith(MockitoExtension.class)
@@ -46,4 +53,52 @@ void testSetHeader() {
 
     verify(apiClient).addDefaultHeader(name, value);
   }
+
+  @ParameterizedTest
+  @ValueSource(
+      ints = {
+        500, 502, 503, 504,
+      }) // HttpStatus.INTERNAL_SERVER_ERROR, BAD_GATEWAY, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT
+  void testRetriesBadGateway(int status) {
+    AtomicInteger callCount = new AtomicInteger();
+    assertThrows(
+        HttpServerErrorException.class,
+        () ->
+            DrsApi.retry(
+                () -> {
+                  callCount.getAndIncrement();
+                  throw HttpServerErrorException.create(
+                      HttpStatus.valueOf(status), "error", null, null, null);
+                }));
+    assertEquals(DrsApi.MAX_TRIALS, callCount.get());
+  }
+
+  @Test
+  void testDoesNotRetryNotFound() {
+    AtomicInteger callCount = new AtomicInteger();
+    assertThrows(
+        HttpServerErrorException.class,
+        () ->
+            DrsApi.retry(
+                () -> {
+                  callCount.getAndIncrement();
+                  throw HttpServerErrorException.create(
+                      HttpStatus.NOT_FOUND, "Not Found", null, null, null);
+                }));
+    assertEquals(1, callCount.get());
+  }
+
+  @Test
+  void testRetryReturnsValue() {
+    AtomicInteger callCount = new AtomicInteger();
+    var expected = "result";
+    var actual =
+        DrsApi.retry(
+            () -> {
+              callCount.getAndIncrement();
+              return expected;
+            });
+    assertEquals(1, callCount.get());
+    assertEquals(expected, actual);
+  }
 }


Reviews:
user: login: rjohanek
body: adding retries looks good to me
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T17:13:40Z
user: login: pshapiro4broad
body: This looks OK but I would prefer to see an embedded object approach for `DrsApi`, to avoid a maintainer needing to know about this feature when/if new DRS APIs are added.

Also, this adds retries for all DRS API calls, not just TDR. We are hoping/assuming that retries on these errors would be good for other DRS providers too.
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-22T17:30:22Z
user: login: dvoet
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T18:47:27Z

Review comments:
diff_hunk: @@ -46,4 +53,52 @@ void testSetHeader() {
 
     verify(apiClient).addDefaultHeader(name, value);
   }
+
+  @ParameterizedTest
+  @ValueSource(
+      ints = {
+        500, 502, 503, 504,
+      }) // HttpStatus.INTERNAL_SERVER_ERROR, BAD_GATEWAY, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT
path: service/src/test/java/bio/terra/drshub/models/DrsApiTest.java
user: login: pshapiro4broad
body: I think you can use java constants in this context
```suggestion
        HttpStatus.INTERNAL_SERVER_ERROR,
        HttpStatus.BAD_GATEWAY,
        HttpStatus.SERVICE_UNAVAILABLE,
        HttpStatus.GATEWAY_TIMEOUT
      })
```
created_at: 2025-04-22T17:07:19Z
author_association: MEMBER
diff_hunk: @@ -17,4 +30,54 @@ public void setBearerToken(String bearerToken) {
   public void setHeader(String name, String value) {
     this.getApiClient().addDefaultHeader(name, value);
   }
+
+  @Override
+  public DrsObject postObject(Object body, String objectId) throws RestClientException {
path: service/src/main/java/bio/terra/drshub/models/DrsApi.java
user: login: pshapiro4broad
body: The main downside with this approach is that new APIs won't have this retry and we won't get a compile error when they're added. Changing this to embed `ObjectsApi` instead of subclassing is one way to fix this.
created_at: 2025-04-22T17:23:49Z
author_association: MEMBER
diff_hunk: @@ -46,4 +53,52 @@ void testSetHeader() {
 
     verify(apiClient).addDefaultHeader(name, value);
   }
+
+  @ParameterizedTest
+  @ValueSource(
+      ints = {
+        500, 502, 503, 504,
+      }) // HttpStatus.INTERNAL_SERVER_ERROR, BAD_GATEWAY, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT
path: service/src/test/java/bio/terra/drshub/models/DrsApiTest.java
user: login: dvoet
body: unfortunately no because `ValueSource` only supports primitives and those constants are not
created_at: 2025-04-22T18:47:27Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-04-22T18:56:44Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=DataBiosphere_terra-drs-hub&pullRequest=147) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=DataBiosphere_terra-drs-hub&pullRequest=147&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=DataBiosphere_terra-drs-hub&pullRequest=147&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=DataBiosphere_terra-drs-hub&pullRequest=147&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=DataBiosphere_terra-drs-hub&pullRequest=147&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=DataBiosphere_terra-drs-hub&pullRequest=147&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=DataBiosphere_terra-drs-hub&pullRequest=147)


author_association: NONE