Code review comment: Nit: let's pad this string in the front and back to better check "e.getMessage().contains".

Code under review:
public void testInsertFailsGracefully() throws Exception {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(wrapValue(new TableRow()), wrapValue(new TableRow()));

    ErrorProto errorProto = new ErrorProto().setReason("schemaMismatch");
    final TableDataInsertAllResponse row1Failed =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors().setIndex(1L).setErrors(ImmutableList.of(errorProto))));

    final TableDataInsertAllResponse row0Failed =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors().setIndex(0L).setErrors(ImmutableList.of(errorProto))));

    MockSetupFunction row0FailureResponseFunction =
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenAnswer(invocation -> toStream(row0Failed));
        };

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          
          when(response.getStatusCode()).thenReturn(200);
          
          when(response.getContent()).thenReturn(toStream(row1Failed));
        },
        
        row0FailureResponseFunction,
        row0FailureResponseFunction,
        row0FailureResponseFunction);

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    
    try {
      dataService.insertAll(
          ref,
          rows,
          null,
          BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
          TEST_BACKOFF,
          new MockSleeper(),
          InsertRetryPolicy.alwaysRetry(),
          null,
          null,
          false,
          false,
          false,
          null);
      fail();
    } catch (IOException e) {
      assertThat(e, instanceOf(IOException.class));
      assertThat(e.getMessage(), containsString("Insert failed:"));
      assertThat(e.getMessage(), containsString("[{\"errors\":[{\"reason\":\"schemaMismatch\"}]"));
    }

    
    verifyAllResponsesAreRead();
    expectedLogs.verifyInfo("Retrying 1 failed inserts to BigQuery");

    verifyWriteMetricWasSet("project", "dataset", "table", "schemamismatch", 4);
  }

  /**
   * Tests that {@link DatasetServiceImpl
   * non-quota-exceeded attempts.
   */
  @Test
  public void testFailInsertOtherRetry() throws Exception {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = new ArrayList<>();
    rows.add(wrapValue(new TableRow()));

    final TableDataInsertAllResponse allRowsSucceeded =
        new TableDataInsertAllResponse().setInsertErrors(ImmutableList.of());
    
    
    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(403);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(toStream(errorWithReasonAndStatus("actually forbidden", 403)));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(new TableDataInsertAllResponse()));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("actually forbidden");
    try {
      dataService.insertAll(
          ref,
          rows,
          null,
          BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
          TEST_BACKOFF,
          new MockSleeper(),
          InsertRetryPolicy.alwaysRetry(),
          null,
          null,
          false,
          false,
          false,
          null);
    } finally {
      verify(responses[0], atLeastOnce()).getStatusCode();
      verify(responses[0]).getContent();
      verify(responses[0]).getContentType();
      
      verify(responses[1], never()).getStatusCode();
      verify(responses[1], never()).getContent();
      verify(responses[1], never()).getContentType();
    }

    verifyWriteMetricWasSet("project", "dataset", "table", "actually forbidden", 1);
  }

  /** Tests that {@link DatasetServiceImpl
  @Test
  public void testInsertTimeoutLog() throws Exception {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = new ArrayList<>();
    rows.add(wrapValue(new TableRow()));

    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(400);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(
                  toStream(errorWithReasonAndStatus("No rows present in the request.", 400)));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
    RuntimeException e =
        assertThrows(
            RuntimeException.class,
            () ->
                dataService.insertAll(
                    ref,
                    rows,
                    null,
                    BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
                    TEST_BACKOFF,
                    new MockSleeper(),
                    InsertRetryPolicy.alwaysRetry(),
                    null,
                    null,
                    false,
                    false,
                    false,
                    null));

    assertThat(e.getCause().getMessage(), containsString("No rows present in the request."));

    verifyAllResponsesAreRead();
    expectedLogs.verifyError("No rows present in the request error likely caused by");

    verifyWriteMetricWasSet("project", "dataset", "table", "no rows present in the request.", 1);
  }

  /**
   * Tests that {@link DatasetServiceImpl
   * and returns the list of rows not retried.
   */
  @Test
  public void testInsertRetryPolicy() throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(wrapValue(new TableRow()), wrapValue(new TableRow()));

    
    final TableDataInsertAllResponse firstFailure =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout"))),
                    new InsertErrors()
                        .setIndex(1L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("invalid")))));

    
    final TableDataInsertAllResponse secondFialure =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout")))));

    
    final TableDataInsertAllResponse allRowsSucceeded = new TableDataInsertAllResponse();

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(firstFailure));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(secondFialure));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(allRowsSucceeded));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.retryTransientErrors(),
        failedInserts,
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        false,
        false,
        false,
        null);
    assertEquals(1, failedInserts.size());
    expectedLogs.verifyInfo("Retrying 1 failed inserts to BigQuery");

    verifyWriteMetricWasSet("project", "dataset", "table", "timeout", 2);
  }

  /**
   * Tests that {@link DatasetServiceImpl
   * ignoreUnknownValues and ignoreInsertIds parameters.
   */
  @Test
  public void testSkipInvalidRowsIgnoreUnknownIgnoreInsertIdsValuesStreaming()
      throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(wrapValue(new TableRow()), wrapValue(new TableRow()));

    final TableDataInsertAllResponse allRowsSucceeded = new TableDataInsertAllResponse();

    
    MockSetupFunction allRowsSucceededResponseFunction =
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(allRowsSucceeded));
        };
    setupMockResponses(allRowsSucceededResponseFunction, allRowsSucceededResponseFunction);

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        Lists.newArrayList(),
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        false,
        false,
        false,
        null);

    TableDataInsertAllRequest parsedRequest =
        fromString(request.getContentAsString(), TableDataInsertAllRequest.class);

    assertFalse(parsedRequest.getSkipInvalidRows());
    assertFalse(parsedRequest.getIgnoreUnknownValues());

    
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        Lists.newArrayList(),
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        true,
        true,
        true,
        null);

    parsedRequest = fromString(request.getContentAsString(), TableDataInsertAllRequest.class);

    assertTrue(parsedRequest.getSkipInvalidRows());
    assertTrue(parsedRequest.getIgnoreUnknownValues());
    assertNull(parsedRequest.getRows().get(0).getInsertId());
    assertNull(parsedRequest.getRows().get(1).getInsertId());

    verifyWriteMetricWasSet("project", "dataset", "table", "ok", 2);
  }

  /** A helper to convert a string response back to a {@link GenericJson} subclass. */
  private static <T extends GenericJson> T fromString(String content, Class<T> clazz)
      throws IOException {
    return JacksonFactory.getDefaultInstance().fromString(content, clazz);
  }

  /** A helper to wrap a {@link GenericJson} object in a content stream. */
  private static InputStream toStream(GenericJson content) throws IOException {
    return new ByteArrayInputStream(JacksonFactory.getDefaultInstance().toByteArray(content));
  }

  /** A helper that generates the error JSON payload that Google APIs produce. */
  private static GoogleJsonErrorContainer errorWithReasonAndStatus(String reason, int status) {
    ErrorInfo info = new ErrorInfo();
    info.setReason(reason);
    info.setDomain("global");
    
    GoogleJsonError error = new GoogleJsonError();
    error.setErrors(ImmutableList.of(info));
    error.setCode(status);
    error.setMessage(reason);
    
    GoogleJsonErrorContainer container = new GoogleJsonErrorContainer();
    container.setError(error);
    return container;
  }

  @Test
  public void testGetErrorInfo() throws IOException {
    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
    ErrorInfo info = new ErrorInfo();
    List<ErrorInfo> infoList = new ArrayList<>();
    infoList.add(info);
    info.setReason("QuotaExceeded");
    GoogleJsonError error = new GoogleJsonError();
    error.setErrors(infoList);
    HttpResponseException.Builder builder = mock(HttpResponseException.Builder.class);
    IOException validException = new GoogleJsonResponseException(builder, error);
    IOException invalidException = new IOException();
    assertEquals(info.getReason(), DatasetServiceImpl.getErrorInfo(validException).getReason());
    assertNull(DatasetServiceImpl.getErrorInfo(invalidException));
  }

  @Test
  public void testCreateTableSucceeds() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(testTable));
        });

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    Table ret =
        services.tryCreateTable(
            testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 0), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verifyAllResponsesAreRead();
  }

  /** Tests that {@link BigQueryServicesImpl} does not retry non-rate-limited attempts. */
  @Test
  public void testCreateTableDoesNotRetry() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);
    
    
    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(403);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(toStream(errorWithReasonAndStatus("actually forbidden", 403)));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(testTable));
        });

    thrown.expect(GoogleJsonResponseException.class);
    thrown.expectMessage("actually forbidden");

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    try {
      services.tryCreateTable(
          testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 3), Sleeper.DEFAULT);
      fail();
    } catch (IOException e) {
      verify(responses[0], atLeastOnce()).getStatusCode();
      verify(responses[0]).getContent();
      verify(responses[0]).getContentType();
      
      verify(responses[1], never()).getStatusCode();
      verify(responses[1], never()).getContent();
      verify(responses[1], never()).getContentType();
      throw e;
    }
  }

  /** Tests that table creation succeeds when the table already exists. */
  @Test
  public void testCreateTableSucceedsAlreadyExists() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    TableSchema schema =
        new TableSchema()
            .setFields(
                ImmutableList.of(
                    new TableFieldSchema().setName("column1").setType("String"),
                    new TableFieldSchema().setName("column2").setType("Integer")));
    Table testTable = new Table().setTableReference(ref).setSchema(schema);

    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(409); 
        });

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    Table ret =
        services.tryCreateTable(
            testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 0), Sleeper.DEFAULT);

    assertNull(ret);
    verifyAllResponsesAreRead();
  }

  /** Tests that {@link BigQueryServicesImpl} retries quota rate limited attempts. */
  @Test
  public void testCreateTableRetry() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);

    
    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(403);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(testTable));
        });

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    Table ret =
        services.tryCreateTable(
            testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 3), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verifyAllResponsesAreRead();

    assertNotNull(ret.getTableReference());

    expectedLogs.verifyInfo(
        "Quota limit reached when creating table project:dataset.table, "
            + "retrying up to 5 minutes");
  }

  /** Tests that {@link DatasetServiceImpl
  @Test
  public void testSimpleErrorRetrieval() throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(
            wrapValue(new TableRow().set("a", 1)), wrapValue(new TableRow().set("b", 2)));

    final List<ValueInSingleWindow<TableRow>> expected =
        ImmutableList.of(
            wrapErrorValue(new TableRow().set("a", 1)), wrapErrorValue(new TableRow().set("b", 2)));

    final TableDataInsertAllResponse failures =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout"))),
                    new InsertErrors()
                        .setIndex(1L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("invalid")))));

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(failures));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        failedInserts,
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        false,
        false,
        false,
        null);

    assertThat(failedInserts, is(expected));
  }

  /** Tests that {@link DatasetServiceImpl
  @Test
  public void testExtendedErrorRetrieval() throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(
            wrapValue(new TableRow().set("a", 1)), wrapValue(new TableRow().set("b", 2)));

    final TableDataInsertAllResponse failures =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout"))),
                    new InsertErrors()
                        .setIndex(1L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("invalid")))));

    final List<ValueInSingleWindow<BigQueryInsertError>> expected =
        ImmutableList.of(
            wrapErrorValue(
                new BigQueryInsertError(
                    rows.get(0).getValue(), failures.getInsertErrors().get(0), ref)),
            wrapErrorValue(
                new BigQueryInsertError(
                    rows.get(1).getValue(), failures.getInsertErrors().get(1), ref)));

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);

          when(response.getContent()).thenReturn(toStream(failures));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    List<ValueInSingleWindow<BigQueryInsertError>> failedInserts = Lists.newArrayList();
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        failedInserts,
        ErrorContainer.BIG_QUERY_INSERT_ERROR_ERROR_CONTAINER,
        false,
        false,
        false,
        null);

    assertThat(failedInserts, is(expected));
  }

  @Test
  public void testCreateReadSessionSetsRequestCountMetric()
      throws InterruptedException, IOException {
    BigQueryServicesImpl.StorageClientImpl client =
        mock(BigQueryServicesImpl.StorageClientImpl.class);

    CreateReadSessionRequest.Builder builder = CreateReadSessionRequest.newBuilder();
    builder.getReadSessionBuilder().setTable("myproject:mydataset.mytable");
    CreateReadSessionRequest request = builder.build();
    when(client.callCreateReadSession(request))
        .thenReturn(ReadSession.newBuilder().build()); 
    when(client.createReadSession(any())).thenCallRealMethod(); 

    client.createReadSession(request);
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "ok", 1);
  }

  @Test
  public void testCreateReadSessionSetsRequestCountMetricOnError()
      throws InterruptedException, IOException {
    BigQueryServicesImpl.StorageClientImpl client =
        mock(BigQueryServicesImpl.StorageClientImpl.class);

    CreateReadSessionRequest.Builder builder = CreateReadSessionRequest.newBuilder();
    builder.getReadSessionBuilder().setTable("myproject:mydataset.mytable");
    CreateReadSessionRequest request = builder.build();
    StatusCode statusCode =
        new StatusCode() {
          @Override
          public Code getCode() {
            return Code.NOT_FOUND;
          }

          @Override
          public Object getTransportCode() {
            return null;
          }
        };
    when(client.callCreateReadSession(request))
        .thenThrow(new ApiException("Not Found", null, statusCode, false)); 
    when(client.createReadSession(any())).thenCallRealMethod(); 

    thrown.expect(ApiException.class);
    thrown.expectMessage("Not Found");

    client.createReadSession(request);
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "not_found", 1);
  }

  @Test
  public void testReadRowsSetsRequestCountMetric() throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    ReadRowsRequest request = null;
    BigQueryServices.BigQueryServerStream<ReadRowsResponse> response =
        new BigQueryServices.BigQueryServerStream<ReadRowsResponse>() {
          @Override
          public Iterator<ReadRowsResponse> iterator() {
            return null;
          }

          @Override
          public void cancel() {}
        };

    when(client.readRows(request)).thenReturn(response); 
    when(client.readRows(any(), any())).thenCallRealMethod(); 

    client.readRows(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "ok", 1);
  }

  @Test
  public void testReadRowsSetsRequestCountMetricOnError() throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    ReadRowsRequest request = null;
    StatusCode statusCode =
        new StatusCode() {
          @Override
          public Code getCode() {
            return Code.INTERNAL;
          }

          @Override
          public Object getTransportCode() {
            return null;
          }
        };
    when(client.readRows(request))
        .thenThrow(new ApiException("Internal", null, statusCode, false)); 
    when(client.readRows(any(), any())).thenCallRealMethod(); 

    thrown.expect(ApiException.class);
    thrown.expectMessage("Internal");

    client.readRows(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "internal", 1);
  }

  @Test
  public void testSplitReadStreamSetsRequestCountMetric() throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);

    SplitReadStreamRequest request = null;
    when(client.splitReadStream(request))
        .thenReturn(SplitReadStreamResponse.newBuilder().build()); 
    when(client.splitReadStream(any(), any())).thenCallRealMethod(); 

    client.splitReadStream(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "ok", 1);
  }

  @Test
  public void testSplitReadStreamSetsRequestCountMetricOnError()
      throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    SplitReadStreamRequest request = null;
    StatusCode statusCode =
        new StatusCode() {
          @Override
          public Code getCode() {
            return Code.RESOURCE_EXHAUSTED;
          }

          @Override
          public Object getTransportCode() {
            return null;
          }
        };
    when(client.splitReadStream(request))
        .thenThrow(
            new ApiException(
                "Resource Exhausted", null, statusCode, false)); 
    when(client.splitReadStream(any(), any())).thenCallRealMethod(); 

    thrown.expect(ApiException.class);
    thrown.expectMessage("Resource Exhausted");

    client.splitReadStream(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "resource_exhausted", 1);
  }

  @Test
  public void testRetryAttemptCounter() {
    BigQueryServicesImpl.StorageClientImpl.RetryAttemptCounter counter =
        new BigQueryServicesImpl.StorageClientImpl.RetryAttemptCounter();

    RetryInfo retryInfo =
        RetryInfo.newBuilder()
            .setRetryDelay(
                com.google.protobuf.Duration.newBuilder()
                    .setSeconds(123)
                    .setNanos(456000000)
                    .build())
            .build();

    Metadata metadata = new Metadata();
    metadata.put(
        Metadata.Key.of(
            "google.rpc.retryinfo-bin",
            new Metadata.BinaryMarshaller<RetryInfo>() {
              @Override
              public byte[] toBytes(RetryInfo value) {
                return value.toByteArray();
              }

              @Override
              public RetryInfo parseBytes(byte[] serialized) {
                try {
                  Parser<RetryInfo> parser = RetryInfo.newBuilder().build().getParserForType();
                  return parser.parseFrom(serialized);
                } catch (Exception e) {
                  return null;
                }
              }
            }),
        retryInfo);

    MetricName metricName =
        MetricName.named(
            "org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl$StorageClientImpl",
            "throttling-msecs");
    MetricsContainerImpl container =
        (MetricsContainerImpl) MetricsEnvironment.getCurrentContainer();

    
    counter.onRetryAttempt(null, null);
    assertEquals(0, (long) container.getCounter(metricName).getCumulative());

    
    counter.onRetryAttempt(
        Status.RESOURCE_EXHAUSTED.withDescription("You have consumed some quota"), new Metadata());
    assertEquals(0, (long) container.getCounter(metricName).getCumulative());

    
    counter.onRetryAttempt(Status.RESOURCE_EXHAUSTED.withDescription("Stop for a while"), metadata);
    assertEquals(123456, (long) container.getCounter(metricName).getCumulative());

    
    counter.onRetryAttempt(Status.UNAVAILABLE.withDescription("Server is gone"), metadata);
    assertEquals(123456, (long) container.getCounter(metricName).getCumulative());
  }
}

Line commented on: toStream(errorWithReasonAndStatus("No rows present in the request.", 400)));

Revised code:
public void testInsertFailsGracefully() throws Exception {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(wrapValue(new TableRow()), wrapValue(new TableRow()));

    ErrorProto errorProto = new ErrorProto().setReason("schemaMismatch");
    final TableDataInsertAllResponse row1Failed =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors().setIndex(1L).setErrors(ImmutableList.of(errorProto))));

    final TableDataInsertAllResponse row0Failed =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors().setIndex(0L).setErrors(ImmutableList.of(errorProto))));

    MockSetupFunction row0FailureResponseFunction =
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenAnswer(invocation -> toStream(row0Failed));
        };

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          
          when(response.getStatusCode()).thenReturn(200);
          
          when(response.getContent()).thenReturn(toStream(row1Failed));
        },
        
        row0FailureResponseFunction,
        row0FailureResponseFunction,
        row0FailureResponseFunction);

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    
    try {
      dataService.insertAll(
          ref,
          rows,
          null,
          BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
          TEST_BACKOFF,
          new MockSleeper(),
          InsertRetryPolicy.alwaysRetry(),
          null,
          null,
          false,
          false,
          false,
          null);
      fail();
    } catch (IOException e) {
      assertThat(e, instanceOf(IOException.class));
      assertThat(e.getMessage(), containsString("Insert failed:"));
      assertThat(e.getMessage(), containsString("[{\"errors\":[{\"reason\":\"schemaMismatch\"}]"));
    }

    
    verifyAllResponsesAreRead();
    expectedLogs.verifyInfo("Retrying 1 failed inserts to BigQuery");

    verifyWriteMetricWasSet("project", "dataset", "table", "schemamismatch", 4);
  }

  /**
   * Tests that {@link DatasetServiceImpl
   * non-quota-exceeded attempts.
   */
  @Test
  public void testFailInsertOtherRetry() throws Exception {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = new ArrayList<>();
    rows.add(wrapValue(new TableRow()));

    final TableDataInsertAllResponse allRowsSucceeded =
        new TableDataInsertAllResponse().setInsertErrors(ImmutableList.of());
    
    
    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(403);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(toStream(errorWithReasonAndStatus("actually forbidden", 403)));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(new TableDataInsertAllResponse()));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("actually forbidden");
    try {
      dataService.insertAll(
          ref,
          rows,
          null,
          BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
          TEST_BACKOFF,
          new MockSleeper(),
          InsertRetryPolicy.alwaysRetry(),
          null,
          null,
          false,
          false,
          false,
          null);
    } finally {
      verify(responses[0], atLeastOnce()).getStatusCode();
      verify(responses[0]).getContent();
      verify(responses[0]).getContentType();
      
      verify(responses[1], never()).getStatusCode();
      verify(responses[1], never()).getContent();
      verify(responses[1], never()).getContentType();
    }

    verifyWriteMetricWasSet("project", "dataset", "table", "actually forbidden", 1);
  }

  /** Tests that {@link DatasetServiceImpl
  @Test
  public void testInsertTimeoutLog() throws Exception {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = new ArrayList<>();
    rows.add(wrapValue(new TableRow()));

    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(400);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(
                  toStream(errorWithReasonAndStatus(" No rows present in the request. ", 400)));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
    RuntimeException e =
        assertThrows(
            RuntimeException.class,
            () ->
                dataService.insertAll(
                    ref,
                    rows,
                    null,
                    BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
                    TEST_BACKOFF,
                    new MockSleeper(),
                    InsertRetryPolicy.alwaysRetry(),
                    null,
                    null,
                    false,
                    false,
                    false,
                    null));

    assertThat(e.getCause().getMessage(), containsString("No rows present in the request."));

    verifyAllResponsesAreRead();
    expectedLogs.verifyError("No rows present in the request error likely caused by");

    verifyWriteMetricWasSet("project", "dataset", "table", " no rows present in the request. ", 1);
  }

  /**
   * Tests that {@link DatasetServiceImpl
   * and returns the list of rows not retried.
   */
  @Test
  public void testInsertRetryPolicy() throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(wrapValue(new TableRow()), wrapValue(new TableRow()));

    
    final TableDataInsertAllResponse firstFailure =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout"))),
                    new InsertErrors()
                        .setIndex(1L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("invalid")))));

    
    final TableDataInsertAllResponse secondFialure =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout")))));

    
    final TableDataInsertAllResponse allRowsSucceeded = new TableDataInsertAllResponse();

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(firstFailure));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(secondFialure));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(allRowsSucceeded));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.retryTransientErrors(),
        failedInserts,
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        false,
        false,
        false,
        null);
    assertEquals(1, failedInserts.size());
    expectedLogs.verifyInfo("Retrying 1 failed inserts to BigQuery");

    verifyWriteMetricWasSet("project", "dataset", "table", "timeout", 2);
  }

  /**
   * Tests that {@link DatasetServiceImpl
   * ignoreUnknownValues and ignoreInsertIds parameters.
   */
  @Test
  public void testSkipInvalidRowsIgnoreUnknownIgnoreInsertIdsValuesStreaming()
      throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(wrapValue(new TableRow()), wrapValue(new TableRow()));

    final TableDataInsertAllResponse allRowsSucceeded = new TableDataInsertAllResponse();

    
    MockSetupFunction allRowsSucceededResponseFunction =
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(allRowsSucceeded));
        };
    setupMockResponses(allRowsSucceededResponseFunction, allRowsSucceededResponseFunction);

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        Lists.newArrayList(),
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        false,
        false,
        false,
        null);

    TableDataInsertAllRequest parsedRequest =
        fromString(request.getContentAsString(), TableDataInsertAllRequest.class);

    assertFalse(parsedRequest.getSkipInvalidRows());
    assertFalse(parsedRequest.getIgnoreUnknownValues());

    
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        Lists.newArrayList(),
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        true,
        true,
        true,
        null);

    parsedRequest = fromString(request.getContentAsString(), TableDataInsertAllRequest.class);

    assertTrue(parsedRequest.getSkipInvalidRows());
    assertTrue(parsedRequest.getIgnoreUnknownValues());
    assertNull(parsedRequest.getRows().get(0).getInsertId());
    assertNull(parsedRequest.getRows().get(1).getInsertId());

    verifyWriteMetricWasSet("project", "dataset", "table", "ok", 2);
  }

  /** A helper to convert a string response back to a {@link GenericJson} subclass. */
  private static <T extends GenericJson> T fromString(String content, Class<T> clazz)
      throws IOException {
    return JacksonFactory.getDefaultInstance().fromString(content, clazz);
  }

  /** A helper to wrap a {@link GenericJson} object in a content stream. */
  private static InputStream toStream(GenericJson content) throws IOException {
    return new ByteArrayInputStream(JacksonFactory.getDefaultInstance().toByteArray(content));
  }

  /** A helper that generates the error JSON payload that Google APIs produce. */
  private static GoogleJsonErrorContainer errorWithReasonAndStatus(String reason, int status) {
    ErrorInfo info = new ErrorInfo();
    info.setReason(reason);
    info.setDomain("global");
    
    GoogleJsonError error = new GoogleJsonError();
    error.setErrors(ImmutableList.of(info));
    error.setCode(status);
    error.setMessage(reason);
    
    GoogleJsonErrorContainer container = new GoogleJsonErrorContainer();
    container.setError(error);
    return container;
  }

  @Test
  public void testGetErrorInfo() throws IOException {
    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
    ErrorInfo info = new ErrorInfo();
    List<ErrorInfo> infoList = new ArrayList<>();
    infoList.add(info);
    info.setReason("QuotaExceeded");
    GoogleJsonError error = new GoogleJsonError();
    error.setErrors(infoList);
    HttpResponseException.Builder builder = mock(HttpResponseException.Builder.class);
    IOException validException = new GoogleJsonResponseException(builder, error);
    IOException invalidException = new IOException();
    assertEquals(info.getReason(), DatasetServiceImpl.getErrorInfo(validException).getReason());
    assertNull(DatasetServiceImpl.getErrorInfo(invalidException));
  }

  @Test
  public void testCreateTableSucceeds() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(testTable));
        });

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    Table ret =
        services.tryCreateTable(
            testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 0), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verifyAllResponsesAreRead();
  }

  /** Tests that {@link BigQueryServicesImpl} does not retry non-rate-limited attempts. */
  @Test
  public void testCreateTableDoesNotRetry() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);
    
    
    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(403);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(toStream(errorWithReasonAndStatus("actually forbidden", 403)));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(testTable));
        });

    thrown.expect(GoogleJsonResponseException.class);
    thrown.expectMessage("actually forbidden");

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    try {
      services.tryCreateTable(
          testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 3), Sleeper.DEFAULT);
      fail();
    } catch (IOException e) {
      verify(responses[0], atLeastOnce()).getStatusCode();
      verify(responses[0]).getContent();
      verify(responses[0]).getContentType();
      
      verify(responses[1], never()).getStatusCode();
      verify(responses[1], never()).getContent();
      verify(responses[1], never()).getContentType();
      throw e;
    }
  }

  /** Tests that table creation succeeds when the table already exists. */
  @Test
  public void testCreateTableSucceedsAlreadyExists() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    TableSchema schema =
        new TableSchema()
            .setFields(
                ImmutableList.of(
                    new TableFieldSchema().setName("column1").setType("String"),
                    new TableFieldSchema().setName("column2").setType("Integer")));
    Table testTable = new Table().setTableReference(ref).setSchema(schema);

    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(409); 
        });

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    Table ret =
        services.tryCreateTable(
            testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 0), Sleeper.DEFAULT);

    assertNull(ret);
    verifyAllResponsesAreRead();
  }

  /** Tests that {@link BigQueryServicesImpl} retries quota rate limited attempts. */
  @Test
  public void testCreateTableRetry() throws IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);

    
    setupMockResponses(
        response -> {
          when(response.getStatusCode()).thenReturn(403);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getContent())
              .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)));
        },
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(testTable));
        });

    BigQueryServicesImpl.DatasetServiceImpl services =
        new BigQueryServicesImpl.DatasetServiceImpl(
            bigquery, null, PipelineOptionsFactory.create());
    Table ret =
        services.tryCreateTable(
            testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 3), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verifyAllResponsesAreRead();

    assertNotNull(ret.getTableReference());

    expectedLogs.verifyInfo(
        "Quota limit reached when creating table project:dataset.table, "
            + "retrying up to 5 minutes");
  }

  /** Tests that {@link DatasetServiceImpl
  @Test
  public void testSimpleErrorRetrieval() throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(
            wrapValue(new TableRow().set("a", 1)), wrapValue(new TableRow().set("b", 2)));

    final List<ValueInSingleWindow<TableRow>> expected =
        ImmutableList.of(
            wrapErrorValue(new TableRow().set("a", 1)), wrapErrorValue(new TableRow().set("b", 2)));

    final TableDataInsertAllResponse failures =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout"))),
                    new InsertErrors()
                        .setIndex(1L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("invalid")))));

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContent()).thenReturn(toStream(failures));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        failedInserts,
        ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
        false,
        false,
        false,
        null);

    assertThat(failedInserts, is(expected));
  }

  /** Tests that {@link DatasetServiceImpl
  @Test
  public void testExtendedErrorRetrieval() throws InterruptedException, IOException {
    TableReference ref =
        new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows =
        ImmutableList.of(
            wrapValue(new TableRow().set("a", 1)), wrapValue(new TableRow().set("b", 2)));

    final TableDataInsertAllResponse failures =
        new TableDataInsertAllResponse()
            .setInsertErrors(
                ImmutableList.of(
                    new InsertErrors()
                        .setIndex(0L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("timeout"))),
                    new InsertErrors()
                        .setIndex(1L)
                        .setErrors(ImmutableList.of(new ErrorProto().setReason("invalid")))));

    final List<ValueInSingleWindow<BigQueryInsertError>> expected =
        ImmutableList.of(
            wrapErrorValue(
                new BigQueryInsertError(
                    rows.get(0).getValue(), failures.getInsertErrors().get(0), ref)),
            wrapErrorValue(
                new BigQueryInsertError(
                    rows.get(1).getValue(), failures.getInsertErrors().get(1), ref)));

    setupMockResponses(
        response -> {
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
          when(response.getStatusCode()).thenReturn(200);
          when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);

          when(response.getContent()).thenReturn(toStream(failures));
        });

    DatasetServiceImpl dataService =
        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());

    List<ValueInSingleWindow<BigQueryInsertError>> failedInserts = Lists.newArrayList();
    dataService.insertAll(
        ref,
        rows,
        null,
        BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()),
        TEST_BACKOFF,
        new MockSleeper(),
        InsertRetryPolicy.neverRetry(),
        failedInserts,
        ErrorContainer.BIG_QUERY_INSERT_ERROR_ERROR_CONTAINER,
        false,
        false,
        false,
        null);

    assertThat(failedInserts, is(expected));
  }

  @Test
  public void testCreateReadSessionSetsRequestCountMetric()
      throws InterruptedException, IOException {
    BigQueryServicesImpl.StorageClientImpl client =
        mock(BigQueryServicesImpl.StorageClientImpl.class);

    CreateReadSessionRequest.Builder builder = CreateReadSessionRequest.newBuilder();
    builder.getReadSessionBuilder().setTable("myproject:mydataset.mytable");
    CreateReadSessionRequest request = builder.build();
    when(client.callCreateReadSession(request))
        .thenReturn(ReadSession.newBuilder().build()); 
    when(client.createReadSession(any())).thenCallRealMethod(); 

    client.createReadSession(request);
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "ok", 1);
  }

  @Test
  public void testCreateReadSessionSetsRequestCountMetricOnError()
      throws InterruptedException, IOException {
    BigQueryServicesImpl.StorageClientImpl client =
        mock(BigQueryServicesImpl.StorageClientImpl.class);

    CreateReadSessionRequest.Builder builder = CreateReadSessionRequest.newBuilder();
    builder.getReadSessionBuilder().setTable("myproject:mydataset.mytable");
    CreateReadSessionRequest request = builder.build();
    StatusCode statusCode =
        new StatusCode() {
          @Override
          public Code getCode() {
            return Code.NOT_FOUND;
          }

          @Override
          public Object getTransportCode() {
            return null;
          }
        };
    when(client.callCreateReadSession(request))
        .thenThrow(new ApiException("Not Found", null, statusCode, false)); 
    when(client.createReadSession(any())).thenCallRealMethod(); 

    thrown.expect(ApiException.class);
    thrown.expectMessage("Not Found");

    client.createReadSession(request);
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "not_found", 1);
  }

  @Test
  public void testReadRowsSetsRequestCountMetric() throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    ReadRowsRequest request = null;
    BigQueryServices.BigQueryServerStream<ReadRowsResponse> response =
        new BigQueryServices.BigQueryServerStream<ReadRowsResponse>() {
          @Override
          public Iterator<ReadRowsResponse> iterator() {
            return null;
          }

          @Override
          public void cancel() {}
        };

    when(client.readRows(request)).thenReturn(response); 
    when(client.readRows(any(), any())).thenCallRealMethod(); 

    client.readRows(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "ok", 1);
  }

  @Test
  public void testReadRowsSetsRequestCountMetricOnError() throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    ReadRowsRequest request = null;
    StatusCode statusCode =
        new StatusCode() {
          @Override
          public Code getCode() {
            return Code.INTERNAL;
          }

          @Override
          public Object getTransportCode() {
            return null;
          }
        };
    when(client.readRows(request))
        .thenThrow(new ApiException("Internal", null, statusCode, false)); 
    when(client.readRows(any(), any())).thenCallRealMethod(); 

    thrown.expect(ApiException.class);
    thrown.expectMessage("Internal");

    client.readRows(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "internal", 1);
  }

  @Test
  public void testSplitReadStreamSetsRequestCountMetric() throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);

    SplitReadStreamRequest request = null;
    when(client.splitReadStream(request))
        .thenReturn(SplitReadStreamResponse.newBuilder().build()); 
    when(client.splitReadStream(any(), any())).thenCallRealMethod(); 

    client.splitReadStream(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "ok", 1);
  }

  @Test
  public void testSplitReadStreamSetsRequestCountMetricOnError()
      throws InterruptedException, IOException {
    BigQueryServices.StorageClient client = mock(BigQueryServicesImpl.StorageClientImpl.class);
    SplitReadStreamRequest request = null;
    StatusCode statusCode =
        new StatusCode() {
          @Override
          public Code getCode() {
            return Code.RESOURCE_EXHAUSTED;
          }

          @Override
          public Object getTransportCode() {
            return null;
          }
        };
    when(client.splitReadStream(request))
        .thenThrow(
            new ApiException(
                "Resource Exhausted", null, statusCode, false)); 
    when(client.splitReadStream(any(), any())).thenCallRealMethod(); 

    thrown.expect(ApiException.class);
    thrown.expectMessage("Resource Exhausted");

    client.splitReadStream(request, "myproject:mydataset.mytable");
    verifyReadMetricWasSet("myproject", "mydataset", "mytable", "resource_exhausted", 1);
  }

  @Test
  public void testRetryAttemptCounter() {
    BigQueryServicesImpl.StorageClientImpl.RetryAttemptCounter counter =
        new BigQueryServicesImpl.StorageClientImpl.RetryAttemptCounter();

    RetryInfo retryInfo =
        RetryInfo.newBuilder()
            .setRetryDelay(
                com.google.protobuf.Duration.newBuilder()
                    .setSeconds(123)
                    .setNanos(456000000)
                    .build())
            .build();

    Metadata metadata = new Metadata();
    metadata.put(
        Metadata.Key.of(
            "google.rpc.retryinfo-bin",
            new Metadata.BinaryMarshaller<RetryInfo>() {
              @Override
              public byte[] toBytes(RetryInfo value) {
                return value.toByteArray();
              }

              @Override
              public RetryInfo parseBytes(byte[] serialized) {
                try {
                  Parser<RetryInfo> parser = RetryInfo.newBuilder().build().getParserForType();
                  return parser.parseFrom(serialized);
                } catch (Exception e) {
                  return null;
                }
              }
            }),
        retryInfo);

    MetricName metricName =
        MetricName.named(
            "org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl$StorageClientImpl",
            "throttling-msecs");
    MetricsContainerImpl container =
        (MetricsContainerImpl) MetricsEnvironment.getCurrentContainer();

    
    counter.onRetryAttempt(null, null);
    assertEquals(0, (long) container.getCounter(metricName).getCumulative());

    
    counter.onRetryAttempt(
        Status.RESOURCE_EXHAUSTED.withDescription("You have consumed some quota"), new Metadata());
    assertEquals(0, (long) container.getCounter(metricName).getCumulative());

    
    counter.onRetryAttempt(Status.RESOURCE_EXHAUSTED.withDescription("Stop for a while"), metadata);
    assertEquals(123456, (long) container.getCounter(metricName).getCumulative());

    
    counter.onRetryAttempt(Status.UNAVAILABLE.withDescription("Server is gone"), metadata);
    assertEquals(123456, (long) container.getCounter(metricName).getCumulative());
  }
}