Repository: pantsbuild/pants

Issue:
Add support for including arbitrary headers on remote execution gRPC requests
In order to support a proxying setup that Twitter has internally, we need to be able to add a (fixed) header that will hint to the proxy that HTTP2 should be assumed on the backend.

In order to do this, we need the ability to add arbitrary header(s) to all remote execution gRPC traffic, and the ability to configure the added header(s) via a `dict_option` or `list_option` (which we would configure in `pants.ini` for this usecase).

We currently optionally add an authorization header here: https://github.com/pantsbuild/pants/blob/33a6d51db91ea1fe69c39117c8878abcb824cdfb/src/rust/engine/process_execution/src/remote.rs#L498-L508 , and so the new headers would need to be propagated down to this location from their definition site in the pants options somewhere near here: https://github.com/pantsbuild/pants/blob/33a6d51db91ea1fe69c39117c8878abcb824cdfb/src/python/pants/option/global_options.py#L387-L402


Discussion:

Resolving patch:
diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py
--- a/src/python/pants/engine/native.py
+++ b/src/python/pants/engine/native.py
@@ -558,6 +558,12 @@ def utf8_buf_buf(self, strings):
     buf_buf = self._ffi.new('Buffer[]', bufs)
     return (buf_buf, len(bufs), self.to_value(buf_buf))
 
+  def utf8_dict(self, d):
+    """Stores the dict as a list of interleaved keys and values, as utf8 strings."""
+    bufs = [self.utf8_buf(item) for keyvalue in d.items() for item in keyvalue]
+    buf_buf = self._ffi.new('Buffer[]', bufs)
+    return (buf_buf, len(bufs), self.to_value(buf_buf))
+
   def vals_buf(self, vals):
     buf = self._ffi.new('Handle[]', vals)
     return (buf, len(vals), self.to_value(buf))
@@ -892,6 +898,7 @@ def ti(type_obj):
         execution_options.process_execution_speculation_delay,
         self.context.utf8_buf(execution_options.process_execution_speculation_strategy),
         execution_options.process_execution_use_local_cache,
+        self.context.utf8_dict(execution_options.remote_execution_headers),
       )
     if scheduler_result.is_throw:
       value = self.context.from_value(scheduler_result.throw_handle)
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -57,6 +57,7 @@ class ExecutionOptions:
   remote_ca_certs_path: Any
   remote_oauth_bearer_token_path: Any
   remote_execution_extra_platform_properties: Any
+  remote_execution_headers: Any
 
   @classmethod
   def from_bootstrap_options(cls, bootstrap_options):
@@ -80,6 +81,7 @@ def from_bootstrap_options(cls, bootstrap_options):
       remote_ca_certs_path=bootstrap_options.remote_ca_certs_path,
       remote_oauth_bearer_token_path=bootstrap_options.remote_oauth_bearer_token_path,
       remote_execution_extra_platform_properties=bootstrap_options.remote_execution_extra_platform_properties,
+      remote_execution_headers=bootstrap_options.remote_execution_headers,
     )
 
 
@@ -103,6 +105,7 @@ def from_bootstrap_options(cls, bootstrap_options):
     remote_ca_certs_path=None,
     remote_oauth_bearer_token_path=None,
     remote_execution_extra_platform_properties=[],
+    remote_execution_headers={},
   )
 
 
@@ -400,6 +403,10 @@ def register_bootstrap_options(cls, register):
                   'Format: property=value. Multiple values should be specified as multiple '
                   'occurrences of this flag. Pants itself may add additional platform properties.',
                    type=list, default=[])
+    register('--remote-execution-headers', advanced=True,
+             help='Headers to set on remote execution requests. '
+                  'Format: header=value. Pants itself may add additional headers.',
+             type=dict, default={})
     register('--process-execution-local-parallelism', type=int, default=DEFAULT_EXECUTION_OPTIONS.process_execution_local_parallelism,
              advanced=True,
              help='Number of concurrent processes that may be executed locally.')


Test patch: