CVE-2021-41130: Authentication Bypass by Spoofing (CWE-290)
value: Extensible Service Proxy, a.k.a. ESP is a proxy which enables API management capabilities for JSON/REST or gRPC API services. ESPv1 can be configured to authenticate a JWT token. Its verified JWT claim is passed to the application by HTTP header "X-Endpoint-API-UserInfo", the application can use it to do authorization. But if there are two "X-Endpoint-API-UserInfo" headers from the client, ESPv1 only replaces the first one, the 2nd one will be passed to the application. An attacker can send two "X-Endpoint-API-UserInfo" headers, the second one with a fake JWT claim. Application may use the fake JWT claim to do the authorization. This impacts following ESPv1 usages: 1) Users have configured ESPv1 to do JWT authentication with Google ID Token as described in the referenced google endpoint document. 2) Users backend application is using the info in the "X-Endpoint-API-UserInfo" header to do the authorization. It has been fixed by v1.58.0. You need to patch it in the following ways: * If your docker image is using tag ":1", needs to re-start the container to pick up the new version. The tag ":1" will automatically point to the latest version. * If your docker image tag pings to a specific minor version, e.g. ":1.57". You need to update it to ":1.58" and re-start the container. There are no workaround for this issue.
CWE description: This attack-focused weakness is caused by incorrectly implemented authentication schemes that are subject to spoofing attacks.
CVSS v3 base score: 5.4
Published: 2021-10-07T19:15Z
Repository: https://github.com/cloudendpoints/esp
Language: C++
Files changed: include/api_manager/request.h, src/api_manager/check_auth.cc, src/api_manager/check_auth_test.cc, src/api_manager/check_service_control.cc, src/api_manager/context/client_ip_extraction_test.cc, src/api_manager/context/request_context.cc, src/api_manager/mock_request.h, src/api_manager/request_handler.cc, src/nginx/request.cc, src/nginx/request.h, src/nginx/t/ApiManager.pm, src/nginx/t/auth_remove_user_info.t
Commit date: 2021-08-30 22:05:31 -0700
Commit message: Remove X-Endpoint-API-UserInfo for all header occurrences. (#845)


Diff: diff --git a/include/api_manager/request.h b/include/api_manager/request.h
index 6382a63..7dc71a3 100644
--- a/include/api_manager/request.h
+++ b/include/api_manager/request.h
@@ -72,11 +72,12 @@ class Request {
   // need to use it compose error message if authentication fails.
   virtual void SetAuthToken(const std::string &auth_token) = 0;
 
   // Adds a header to backend. If the header exists, overwrite its value
   virtual utils::Status AddHeaderToBackend(const std::string &key,
-                                           const std::string &value) = 0;
+                                           const std::string &value,
+                                           bool changeAllOccurrence) = 0;
 };
 
 }  // namespace api_manager
 }  // namespace google
 
diff --git a/src/api_manager/check_auth.cc b/src/api_manager/check_auth.cc
index a49b6cf..ea0dbd2 100644
--- a/src/api_manager/check_auth.cc
+++ b/src/api_manager/check_auth.cc
@@ -391,11 +391,11 @@ void AuthChecker::PassUserInfoOnSuccess() {
     return;
   }
   char *base64_json_buf = auth::esp_base64_encode(
       json_buf, strlen(json_buf), true, false, true /*padding*/);
   context_->request()->AddHeaderToBackend(auth::kEndpointApiUserInfo,
-                                          base64_json_buf);
+                                          base64_json_buf, false);
   auth::esp_grpc_free(json_buf);
   auth::esp_grpc_free(base64_json_buf);
 
   TRACE(trace_span_) << "Authenticated.";
   trace_span_.reset();
diff --git a/src/api_manager/check_auth_test.cc b/src/api_manager/check_auth_test.cc
index dcd338c..f9d8833 100644
--- a/src/api_manager/check_auth_test.cc
+++ b/src/api_manager/check_auth_test.cc
@@ -456,11 +456,11 @@ void CheckAuthTest::TestValidToken(const std::string &auth_token,
         std::map<std::string, std::string> empty;
         req->OnComplete(Status::OK, std::move(empty), std::move(body));
       }));
   std::cout << "need be replaced: " << user_info << std::endl;
   EXPECT_CALL(*raw_request_,
-              AddHeaderToBackend(kEndpointApiUserInfo, user_info))
+              AddHeaderToBackend(kEndpointApiUserInfo, user_info, false))
       .WillOnce(Return(utils::Status::OK));
 
   CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); });
 }
 
@@ -488,12 +488,12 @@ TEST_F(CheckAuthTest, TestOKAuth) {
         *token = std::string(kBearer) + std::string(kToken);
         return true;
       }));
   EXPECT_CALL(*raw_request_, SetAuthToken(kToken)).Times(1);
   EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0);
-  EXPECT_CALL(*raw_request_,
-              AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub_kIss))
+  EXPECT_CALL(*raw_request_, AddHeaderToBackend(kEndpointApiUserInfo,
+                                                kUserInfo_kSub_kIss, false))
       .WillOnce(Return(utils::Status::OK));
 
   CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); });
 
   EXPECT_TRUE(Mock::VerifyAndClearExpectations(raw_request_));
@@ -511,12 +511,12 @@ TEST_F(CheckAuthTest, TestOKAuth) {
         *token = std::string(kToken2);
         return true;
       }));
   EXPECT_CALL(*raw_request_, SetAuthToken(kToken2)).Times(1);
   EXPECT_CALL(*raw_env_, DoRunHTTPRequest(_)).Times(0);
-  EXPECT_CALL(*raw_request_,
-              AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub2_kIss2))
+  EXPECT_CALL(*raw_request_, AddHeaderToBackend(kEndpointApiUserInfo,
+                                                kUserInfo_kSub2_kIss2, false))
       .WillOnce(Return(utils::Status::OK));
 
   CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); });
 }
 
@@ -603,12 +603,12 @@ TEST_F(CheckAuthTest, TestNoOpenId) {
         EXPECT_EQ(req->url(), kIssuer2PubkeyUrl);
         std::string body(kPubkey);
         std::map<std::string, std::string> empty;
         req->OnComplete(Status::OK, std::move(empty), std::move(body));
       }));
-  EXPECT_CALL(*raw_request_,
-              AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub_kIss2))
+  EXPECT_CALL(*raw_request_, AddHeaderToBackend(kEndpointApiUserInfo,
+                                                kUserInfo_kSub_kIss2, false))
       .WillOnce(Return(utils::Status::OK));
 
   CheckAuth(context_, [](Status status) { ASSERT_TRUE(status.ok()); });
 }
 
diff --git a/src/api_manager/check_service_control.cc b/src/api_manager/check_service_control.cc
index f80fbd1..3e2e68d 100644
--- a/src/api_manager/check_service_control.cc
+++ b/src/api_manager/check_service_control.cc
@@ -13,10 +13,11 @@
 // limitations under the License.
 //
 ////////////////////////////////////////////////////////////////////////////////
 // includes should be ordered. This seems like a bug in clang-format?
 #include "src/api_manager/check_service_control.h"
+
 #include "google/protobuf/stubs/status.h"
 #include "src/api_manager/cloud_trace/cloud_trace.h"
 
 using ::google::api_manager::utils::Status;
 using ::google::protobuf::util::error::Code;
@@ -83,12 +84,12 @@ void CheckServiceControl(std::shared_ptr<context::RequestContext> context,
         // info is valid regardless status.
         context->set_check_response_info(info);
 
         // update consumer_project_id to service context
         if (!info.consumer_project_id.empty()) {
-          context->request()->AddHeaderToBackend(kConsumerProjecId,
-                                                 info.consumer_project_id);
+          context->request()->AddHeaderToBackend(
+              kConsumerProjecId, info.consumer_project_id, false);
         }
 
         continuation(status);
       });
 }
diff --git a/src/api_manager/context/client_ip_extraction_test.cc b/src/api_manager/context/client_ip_extraction_test.cc
index 026b39f..9d94e2a 100644
--- a/src/api_manager/context/client_ip_extraction_test.cc
+++ b/src/api_manager/context/client_ip_extraction_test.cc
@@ -115,12 +115,12 @@ class MockRequest : public Request {
 
   bool FindQuery(const std::string &name, std::string *query) override {
     return false;
   }
 
-  MOCK_METHOD2(AddHeaderToBackend,
-               utils::Status(const std::string &, const std::string &));
+  MOCK_METHOD3(AddHeaderToBackend,
+               utils::Status(const std::string &, const std::string &, bool));
   MOCK_METHOD1(SetAuthToken, void(const std::string &));
   MOCK_METHOD0(GetFrontendProtocol,
                ::google::api_manager::protocol::Protocol());
   MOCK_METHOD0(GetBackendProtocol, ::google::api_manager::protocol::Protocol());
   MOCK_METHOD0(GetRequestPath, std::string());
diff --git a/src/api_manager/context/request_context.cc b/src/api_manager/context/request_context.cc
index 53b597d..ab91e2b 100644
--- a/src/api_manager/context/request_context.cc
+++ b/src/api_manager/context/request_context.cc
@@ -14,19 +14,21 @@
 //
 ////////////////////////////////////////////////////////////////////////////////
 //
 
 #include "src/api_manager/context/request_context.h"
-#include "google/api/backend.pb.h"
-#include "src/api_manager/auth/lib/json_util.h"
-#include "src/api_manager/utils/str_util.h"
 
 #include <uuid/uuid.h>
+
 #include <numeric>
 #include <sstream>
 #include <vector>
 
+#include "google/api/backend.pb.h"
+#include "src/api_manager/auth/lib/json_util.h"
+#include "src/api_manager/utils/str_util.h"
+
 using ::google::api_manager::auth::GetPrimitiveFieldValue;
 using ::google::api_manager::cloud_trace::HeaderType;
 using ::google::api_manager::utils::Status;
 
 namespace google {
@@ -206,11 +208,11 @@ void RequestContext::ExtractApiKey() {
     request_->FindHeader(kDefaultApiKeyHeaderName, &api_key_);
   }
 }
 
 void RequestContext::SetApiKeyHeader() {
-  request_->AddHeaderToBackend(kDefaultApiKeyHeaderName, api_key_);
+  request_->AddHeaderToBackend(kDefaultApiKeyHeaderName, api_key_, false);
 }
 
 void RequestContext::CompleteCheck(Status status) {
   // Makes sure set_check_continuation() is called.
   // Only making sure CompleteCheck() is NOT called twice.
@@ -432,11 +434,11 @@ void RequestContext::StartBackendSpanAndSetTraceContext() {
   // Set trace context header to backend.
   Status status = request()->AddHeaderToBackend(
       cloud_trace()->header_type() == HeaderType::CLOUD_TRACE_CONTEXT
           ? kCloudTraceContextHeader
           : kGRpcTraceContextHeader,
-      trace_context_header);
+      trace_context_header, false);
   if (!status.ok()) {
     service_context()->env()->LogError(
         "Failed to set trace context header to backend.");
   }
 }
@@ -533,18 +535,18 @@ void RequestContext::AddInstanceIdentityToken() {
                       ->GetAuthToken();
     if (!token.empty()) {
       std::string origin_auth_header;
       if (request()->FindHeader(kAuthorizationHeader, &origin_auth_header)) {
         Status status = request()->AddHeaderToBackend(
-            kXForwardedAuthorizationHeader, origin_auth_header);
+            kXForwardedAuthorizationHeader, origin_auth_header, false);
         if (!status.ok()) {
           service_context()->env()->LogError(
               "Failed to set X-Forwarded-Authorization header to backend.");
         }
       }
-      Status status = request()->AddHeaderToBackend(kAuthorizationHeader,
-                                                    kBearerPrefix + token);
+      Status status = request()->AddHeaderToBackend(
+          kAuthorizationHeader, kBearerPrefix + token, false);
       if (!status.ok()) {
         service_context()->env()->LogError(
             "Failed to set authorization header to backend.");
       }
     }
diff --git a/src/api_manager/mock_request.h b/src/api_manager/mock_request.h
index 908591e..49aadb9 100644
--- a/src/api_manager/mock_request.h
+++ b/src/api_manager/mock_request.h
@@ -23,12 +23,12 @@ namespace api_manager {
 
 class MockRequest : public Request {
  public:
   MOCK_METHOD2(FindQuery, bool(const std::string &, std::string *));
   MOCK_METHOD2(FindHeader, bool(const std::string &, std::string *));
-  MOCK_METHOD2(AddHeaderToBackend,
-               utils::Status(const std::string &, const std::string &));
+  MOCK_METHOD3(AddHeaderToBackend,
+               utils::Status(const std::string &, const std::string &, bool));
   MOCK_METHOD1(SetAuthToken, void(const std::string &));
   MOCK_METHOD0(GetRequestHTTPMethod, std::string());
   MOCK_METHOD0(GetQueryParameters, std::string());
   MOCK_METHOD0(GetFrontendProtocol,
                ::google::api_manager::protocol::Protocol());
diff --git a/src/api_manager/request_handler.cc b/src/api_manager/request_handler.cc
index 24536a7..eedcff8 100644
--- a/src/api_manager/request_handler.cc
+++ b/src/api_manager/request_handler.cc
@@ -41,11 +41,11 @@ RequestHandler::RequestHandler(
   // It should be set by the last Endpoint proxy to prevent users spoofing.
   std::string buffer;
   if (context_->request()->FindHeader(
           google::api_manager::auth::kEndpointApiUserInfo, &buffer)) {
     context_->request()->AddHeaderToBackend(
-        google::api_manager::auth::kEndpointApiUserInfo, "");
+        google::api_manager::auth::kEndpointApiUserInfo, "", true);
   }
 }
 
 void RequestHandler::Check(std::function<void(Status status)> continuation) {
   auto interception = [continuation, this](Status status) {
diff --git a/src/nginx/request.cc b/src/nginx/request.cc
index c66ae61..09da19d 100644
--- a/src/nginx/request.cc
+++ b/src/nginx/request.cc
@@ -144,22 +144,25 @@ void NgxEspRequest::SetAuthToken(const std::string &auth_token) {
   ngx_esp_request_ctx_t *ctx = ngx_http_esp_get_module_ctx(r_);
   ngx_str_copy_from_std(r_->pool, auth_token, &ctx->auth_token);
 }
 
 utils::Status NgxEspRequest::AddHeaderToBackend(const std::string &key,
-                                                const std::string &value) {
-  ngx_table_elt_t *h = nullptr;
+                                                const std::string &value,
+                                                bool changeAllOccurrence) {
+  std::vector<ngx_table_elt_t *> headers;
   for (auto &h_in : r_->headers_in) {
     if (key.size() == h_in.key.len &&
         strncasecmp(key.c_str(), reinterpret_cast<const char *>(h_in.key.data),
                     h_in.key.len) == 0) {
-      h = &h_in;
-      break;
+      headers.push_back(&h_in);
+      if (!changeAllOccurrence) {
+        break;
+      }
     }
   }
-  if (h == nullptr) {
-    h = reinterpret_cast<ngx_table_elt_t *>(
+  if (headers.empty()) {
+    ngx_table_elt_t *h = reinterpret_cast<ngx_table_elt_t *>(
         ngx_list_push(&r_->headers_in.headers));
     if (h == nullptr) {
       return utils::Status(Code::INTERNAL, "Out of memory");
     }
 
@@ -170,18 +173,25 @@ utils::Status NgxEspRequest::AddHeaderToBackend(const std::string &key,
     }
     h->hash = ngx_hash_strlow(
         h->lowcase_key,
         reinterpret_cast<u_char *>(const_cast<char *>(key.c_str())),
         key.size());
+    headers.push_back(h);
   }
 
-  if (ngx_str_copy_from_std(r_->pool, key, &h->key) != NGX_OK ||
-      ngx_str_copy_from_std(r_->pool, value, &h->value) != NGX_OK) {
-    return utils::Status(Code::INTERNAL, "Out of memory");
+  for (size_t i = 0; i < headers.size(); ++i) {
+    ngx_table_elt_t *it = headers[i];
+    if (ngx_str_copy_from_std(r_->pool, key, &it->key) != NGX_OK ||
+        ngx_str_copy_from_std(r_->pool, value, &it->value) != NGX_OK) {
+      return utils::Status(Code::INTERNAL, "Out of memory");
+    }
   }
-  ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r_->connection->log, 0,
-                 "updates header to backend: \"%V: %V\"", &h->key, &h->value);
+
+  ngx_log_debug3(
+      NGX_LOG_DEBUG_HTTP, r_->connection->log, 0,
+      "updates header to backend, changeAllOccurrence: '%t', \"%V: %V\"",
+      changeAllOccurrence, &h->key, &h->value);
   return utils::Status::OK;
 }
 
 }  // namespace nginx
 }  // namespace api_manager
diff --git a/src/nginx/request.h b/src/nginx/request.h
index ba22fe2..f1e6d6f 100644
--- a/src/nginx/request.h
+++ b/src/nginx/request.h
@@ -55,11 +55,12 @@ class NgxEspRequest : public Request {
   virtual int64_t GetGrpcRequestMessageCounts();
   virtual int64_t GetGrpcResponseMessageCounts();
 
   virtual void SetAuthToken(const std::string &auth_token);
   virtual utils::Status AddHeaderToBackend(const std::string &key,
-                                           const std::string &value);
+                                           const std::string &value,
+                                           bool changeAllOccurrence);
   virtual bool FindQuery(const std::string &name, std::string *query);
   virtual bool FindHeader(const std::string &name, std::string *header);
 
  private:
   ngx_http_request_t *r_;
diff --git a/src/nginx/t/ApiManager.pm b/src/nginx/t/ApiManager.pm
index e34928f..24ce55a 100644
--- a/src/nginx/t/ApiManager.pm
+++ b/src/nginx/t/ApiManager.pm
@@ -454,10 +454,11 @@ sub read_http_stream {
     push @requests, {
       'verb' => $verb,
       'path' => $path,
       'uri' => $uri,
       'headers' => \%headers,
+      'header_lines' => \@header_lines,
       'body' => $body
     };
 
     $http = $rest;
   }
diff --git a/src/nginx/t/auth_remove_user_info.t b/src/nginx/t/auth_remove_user_info.t
index 17da599..2f2920a 100644
--- a/src/nginx/t/auth_remove_user_info.t
+++ b/src/nginx/t/auth_remove_user_info.t
@@ -40,11 +40,11 @@ use Data::Dumper;
 # Port assignments
 my $NginxPort = ApiManager::pick_port();
 my $BackendPort = ApiManager::pick_port();
 my $ServiceControlPort = ApiManager::pick_port();
 
-my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(14);
+my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(15);
 
 # Save service name in the service configuration protocol buffer file.
 
 $t->write_file('service.pb.txt', ApiManager::get_bookstore_service_config . <<"EOF");
 control {
@@ -86,10 +86,11 @@ $t->run();
 
 my $response = ApiManager::http($NginxPort,<<'EOF');
 GET /shelves?key=this-is-an-api-key HTTP/1.0
 Host: localhost
 x-endpoint-api-userinfo: Should be removed
+x-endpoint-api-userinfo: Multiple userinfo should all be removed
 
 EOF
 
 $t->stop_daemons();
 
@@ -112,10 +113,18 @@ my $r = shift @requests;
 is($r->{verb}, 'GET', 'Backend request was a get');
 is($r->{uri}, '/shelves?key=this-is-an-api-key', 'Backend uri was /shelves');
 is($r->{headers}->{host}, "127.0.0.1:${BackendPort}", 'Host header was set');
 is($r->{headers}->{'x-endpoint-api-userinfo'}, '',
     'X-Endpoint-API-UserInfo should be removed from the request headers');
+my $arr = $r->{header_lines};
+my @userinfo_list = ();
+
+foreach (@$arr) {
+  push(@userinfo_list, $_) if index($_, 'X-Endpoint-API-UserInfo') != -1;
+}
+is(join(",", @userinfo_list), 'X-Endpoint-API-UserInfo: ,X-Endpoint-API-UserInfo: ',
+   "Multiple userinfo should all be removed");
 
 @requests = ApiManager::read_http_stream($t, 'servicecontrol.log');
 is(scalar @requests, 1, 'Service control received one request');
 
 $r = shift @requests;

Vulnerable code:                                            const std::string &value) = 0;
                                          base64_json_buf);
              AddHeaderToBackend(kEndpointApiUserInfo, user_info))
  EXPECT_CALL(*raw_request_,
              AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub_kIss))
  EXPECT_CALL(*raw_request_,
              AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub2_kIss2))
  EXPECT_CALL(*raw_request_,
              AddHeaderToBackend(kEndpointApiUserInfo, kUserInfo_kSub_kIss2))
          context->request()->AddHeaderToBackend(kConsumerProjecId,
                                                 info.consumer_project_id);
  MOCK_METHOD2(AddHeaderToBackend,
               utils::Status(const std::string &, const std::string &));
#include "google/api/backend.pb.h"
#include "src/api_manager/auth/lib/json_util.h"
#include "src/api_manager/utils/str_util.h"
  request_->AddHeaderToBackend(kDefaultApiKeyHeaderName, api_key_);
      trace_context_header);
            kXForwardedAuthorizationHeader, origin_auth_header);
      Status status = request()->AddHeaderToBackend(kAuthorizationHeader,
                                                    kBearerPrefix + token);
  MOCK_METHOD2(AddHeaderToBackend,
               utils::Status(const std::string &, const std::string &));
        google::api_manager::auth::kEndpointApiUserInfo, "");
                                                const std::string &value) {
  ngx_table_elt_t *h = nullptr;
      h = &h_in;
      break;
  if (h == nullptr) {
    h = reinterpret_cast<ngx_table_elt_t *>(
  if (ngx_str_copy_from_std(r_->pool, key, &h->key) != NGX_OK ||
      ngx_str_copy_from_std(r_->pool, value, &h->value) != NGX_OK) {
    return utils::Status(Code::INTERNAL, "Out of memory");
  ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r_->connection->log, 0,
                 "updates header to backend: \"%V: %V\"", &h->key, &h->value);
                                           const std::string &value);
my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(14);
Fixed code:                                            const std::string &value,
                                           bool changeAllOccurrence) = 0;
                                          base64_json_buf, false);
              AddHeaderToBackend(kEndpointApiUserInfo, user_info, false))
  EXPECT_CALL(*raw_request_, AddHeaderToBackend(kEndpointApiUserInfo,
                                                kUserInfo_kSub_kIss, false))
  EXPECT_CALL(*raw_request_, AddHeaderToBackend(kEndpointApiUserInfo,
                                                kUserInfo_kSub2_kIss2, false))
  EXPECT_CALL(*raw_request_, AddHeaderToBackend(kEndpointApiUserInfo,
                                                kUserInfo_kSub_kIss2, false))

          context->request()->AddHeaderToBackend(
              kConsumerProjecId, info.consumer_project_id, false);
  MOCK_METHOD3(AddHeaderToBackend,
               utils::Status(const std::string &, const std::string &, bool));

#include "google/api/backend.pb.h"
#include "src/api_manager/auth/lib/json_util.h"
#include "src/api_manager/utils/str_util.h"

  request_->AddHeaderToBackend(kDefaultApiKeyHeaderName, api_key_, false);
      trace_context_header, false);
            kXForwardedAuthorizationHeader, origin_auth_header, false);
      Status status = request()->AddHeaderToBackend(
          kAuthorizationHeader, kBearerPrefix + token, false);
  MOCK_METHOD3(AddHeaderToBackend,
               utils::Status(const std::string &, const std::string &, bool));
        google::api_manager::auth::kEndpointApiUserInfo, "", true);
                                                const std::string &value,
                                                bool changeAllOccurrence) {
  std::vector<ngx_table_elt_t *> headers;
      headers.push_back(&h_in);
      if (!changeAllOccurrence) {
        break;
      }
  if (headers.empty()) {
    ngx_table_elt_t *h = reinterpret_cast<ngx_table_elt_t *>(
    headers.push_back(h);
  for (size_t i = 0; i < headers.size(); ++i) {
    ngx_table_elt_t *it = headers[i];
    if (ngx_str_copy_from_std(r_->pool, key, &it->key) != NGX_OK ||
        ngx_str_copy_from_std(r_->pool, value, &it->value) != NGX_OK) {
      return utils::Status(Code::INTERNAL, "Out of memory");
    }

  ngx_log_debug3(
      NGX_LOG_DEBUG_HTTP, r_->connection->log, 0,
      "updates header to backend, changeAllOccurrence: '%t', \"%V: %V\"",
      changeAllOccurrence, &h->key, &h->value);
                                           const std::string &value,
                                           bool changeAllOccurrence);
      'header_lines' => \@header_lines,
my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(15);
x-endpoint-api-userinfo: Multiple userinfo should all be removed
my $arr = $r->{header_lines};
my @userinfo_list = ();

foreach (@$arr) {
  push(@userinfo_list, $_) if index($_, 'X-Endpoint-API-UserInfo') != -1;
}
is(join(",", @userinfo_list), 'X-Endpoint-API-UserInfo: ,X-Endpoint-API-UserInfo: ',
   "Multiple userinfo should all be removed");