TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) {
  enableHedgeOnPerTryTimeout();

  NiceMock<Http::MockRequestEncoder> encoder1;
  Http::ResponseDecoder* response_decoder1 = nullptr;
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _))
      .WillOnce(Invoke(
          [&](Http::ResponseDecoder& decoder,
              Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* {
            response_decoder1 = &decoder;
            EXPECT_CALL(*router_.retry_state_, onHostAttempted(_));
            callbacks.onPoolReady(encoder1, cm_.thread_local_cluster_.conn_pool_.host_,
                                  upstream_stream_info_, Http::Protocol::Http10);
            return nullptr;
          }));
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess,
                        absl::optional<uint64_t>(absl::nullopt)));
  expectPerTryTimerCreate();
  expectResponseTimerCreate();

  Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-per-try-timeout-ms", "5"}};
  HttpTestUtility::addDefaultHeaders(headers);
  router_.decodeHeaders(headers, true);
  EXPECT_EQ(1U,
            callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value());

  EXPECT_CALL(
      cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
      putResult(Upstream::Outlier::Result::LocalOriginTimeout, absl::optional<uint64_t>(504)));
  EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0);
  EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0);
  router_.retry_state_->expectHedgedPerTryTimeoutRetry();
  per_try_timeout_->invokeCallback();

  NiceMock<Http::MockRequestEncoder> encoder2;
  Http::ResponseDecoder* response_decoder2 = nullptr;
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _))
      .WillOnce(Invoke(
          [&](Http::ResponseDecoder& decoder,
              Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* {
            response_decoder2 = &decoder;
            EXPECT_CALL(*router_.retry_state_, onHostAttempted(_));
            callbacks.onPoolReady(encoder2, cm_.thread_local_cluster_.conn_pool_.host_,
                                  upstream_stream_info_, Http::Protocol::Http10);
            return nullptr;
          }));
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess,
                        absl::optional<uint64_t>(absl::nullopt)));
  expectPerTryTimerCreate();
  router_.retry_state_->callback_();
  EXPECT_EQ(2U,
            callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value());

  EXPECT_TRUE(verifyHostUpstreamStats(0, 0));

  // Now trigger a 503 in response to the second request.
  Http::ResponseHeaderMapPtr bad_response_headers1(
      new Http::TestResponseHeaderMapImpl{{":status", "503"}});
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putHttpResponseCode(503));

  EXPECT_CALL(*router_.retry_state_, shouldRetryHeaders(_, _))
      .WillOnce(Return(RetryStatus::NoRetryLimitExceeded));
  response_decoder2->decodeHeaders(std::move(bad_response_headers1), true);

  EXPECT_TRUE(verifyHostUpstreamStats(0, 1));

  // Now trigger a 503 in response to the first request.
  Http::ResponseHeaderMapPtr bad_response_headers2(
      new Http::TestResponseHeaderMapImpl{{":status", "503"}});
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putHttpResponseCode(503));

  // We should not call shouldRetryHeaders() because you never retry the same
  // request twice.
  EXPECT_CALL(*router_.retry_state_, shouldRetryHeaders(_, _)).Times(0);

  EXPECT_CALL(callbacks_, encodeHeaders_(_, _))
      .WillOnce(Invoke([&](Http::ResponseHeaderMap& headers, bool) -> void {
        EXPECT_EQ(headers.Status()->value(), "503");
      }));
  response_decoder1->decodeHeaders(std::move(bad_response_headers2), true);

  EXPECT_TRUE(verifyHostUpstreamStats(0, 2));
}
TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) {
  enableHedgeOnPerTryTimeout();

  NiceMock<Http::MockRequestEncoder> encoder1;
  Http::ResponseDecoder* response_decoder1 = nullptr;
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _))
      .WillOnce(Invoke(
          [&](Http::ResponseDecoder& decoder,
              Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* {
            response_decoder1 = &decoder;
            EXPECT_CALL(*router_.retry_state_, onHostAttempted(_));
            callbacks.onPoolReady(encoder1, cm_.thread_local_cluster_.conn_pool_.host_,
                                  upstream_stream_info_, Http::Protocol::Http10);
            return nullptr;
          }));
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess,
                        absl::optional<uint64_t>(absl::nullopt)));
  expectPerTryTimerCreate();
  expectResponseTimerCreate();

  Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-rq-per-try-timeout-ms", "5"}};
  HttpTestUtility::addDefaultHeaders(headers);
  router_.decodeHeaders(headers, true);
  EXPECT_EQ(1U,
            callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value());

  EXPECT_CALL(
      cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
      putResult(Upstream::Outlier::Result::LocalOriginTimeout, absl::optional<uint64_t>(504)));
  EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0);
  EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0);
  router_.retry_state_->expectHedgedPerTryTimeoutRetry();
  per_try_timeout_->invokeCallback();

  NiceMock<Http::MockRequestEncoder> encoder2;
  Http::ResponseDecoder* response_decoder2 = nullptr;
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _))
      .WillOnce(Invoke(
          [&](Http::ResponseDecoder& decoder,
              Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* {
            response_decoder2 = &decoder;
            EXPECT_CALL(*router_.retry_state_, onHostAttempted(_));
            callbacks.onPoolReady(encoder2, cm_.thread_local_cluster_.conn_pool_.host_,
                                  upstream_stream_info_, Http::Protocol::Http10);
            return nullptr;
          }));
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putResult(Upstream::Outlier::Result::LocalOriginConnectSuccess,
                        absl::optional<uint64_t>(absl::nullopt)));
  expectPerTryTimerCreate();
  router_.retry_state_->callback_();
  EXPECT_EQ(2U,
            callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value());

  EXPECT_TRUE(verifyHostUpstreamStats(0, 0));

   
  Http::ResponseHeaderMapPtr bad_response_headers1(
      new Http::TestResponseHeaderMapImpl{{":status", "503"}});
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putHttpResponseCode(503));

  EXPECT_CALL(*router_.retry_state_, shouldRetryHeaders(_, _))
      .WillOnce(Return(RetryStatus::NoRetryLimitExceeded));
  response_decoder2->decodeHeaders(std::move(bad_response_headers1), true);

  EXPECT_TRUE(verifyHostUpstreamStats(0, 1));

   
  Http::ResponseHeaderMapPtr bad_response_headers2(
      new Http::TestResponseHeaderMapImpl{{":status", "503"}});
  EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_.host_->outlier_detector_,
              putHttpResponseCode(503));

   
   
  EXPECT_CALL(*router_.retry_state_, shouldRetryHeaders(_, _)).Times(0);

  EXPECT_CALL(callbacks_, encodeHeaders_(_, _))
      .WillOnce(Invoke([&](Http::ResponseHeaderMap& headers, bool) -> void {
        EXPECT_EQ(headers.Status()->value(), "503");
      }));
  response_decoder1->decodeHeaders(std::move(bad_response_headers2), true);

  EXPECT_TRUE(verifyHostUpstreamStats(0, 2));
}