Repository: localstack/localstack

Issue:
question: Use HTTP proxy when downloading resources
### Is there an existing issue for this?

- [X] I have searched the existing issues and read the documentation

### Question

I'm running Localstack inside minikube on macOS. Due to the networking configuration in macOS, I need to use an HTTP proxy whenever I'm connected to a VPN.

This means that Localstack fails to start inside minikube when the host is connected to a VPN. The issue came up when Localstack tried to download https://github.com/localstack/localstack-artifacts/raw/master/local-certs/server.key.

Is there a way to tell Localstack to use an HTTP proxy when downloading resources?

### Anything else?

_No response_


Discussion:

Resolving patch:
diff --git a/localstack/config.py b/localstack/config.py
--- a/localstack/config.py
+++ b/localstack/config.py
@@ -293,6 +293,12 @@ def is_linux():
 # For example: "my-first-stream:1,my-other-stream:2,my-last-stream:1"
 KINESIS_INITIALIZE_STREAMS = os.environ.get("KINESIS_INITIALIZE_STREAMS", "").strip()
 
+# Equivalent to HTTP_PROXY, but only applicable for external connections
+OUTBOUND_HTTP_PROXY = os.environ.get("OUTBOUND_HTTP_PROXY", "")
+
+# Equivalent to HTTPS_PROXY, but only applicable for external connections
+OUTBOUND_HTTPS_PROXY = os.environ.get("OUTBOUND_HTTPS_PROXY", "")
+
 # list of environment variable names used for configuration.
 # Make sure to keep this in sync with the above!
 # Note: do *not* include DATA_DIR in this list, as it is treated separately
@@ -359,6 +365,9 @@ def is_linux():
     "EXTRA_CORS_EXPOSE_HEADERS",
     "EXTRA_CORS_ALLOWED_ORIGINS",
     "ENABLE_CONFIG_UPDATES",
+    "LOCALSTACK_HTTP_PROXY",
+    "LOCALSTACK_HTTPS_PROXY",
+    "REQUESTS_CA_BUNDLE",
 ]
 
 for key, value in six.iteritems(DEFAULT_SERVICE_PORTS):
diff --git a/localstack/utils/analytics/client.py b/localstack/utils/analytics/client.py
--- a/localstack/utils/analytics/client.py
+++ b/localstack/utils/analytics/client.py
@@ -7,7 +7,7 @@
 import requests
 
 from localstack import config, constants
-from localstack.utils.common import now
+from localstack.utils.common import get_proxies, now
 
 from .events import Event, EventMetadata
 from .metadata import ClientMetadata, get_session_id
@@ -45,7 +45,10 @@ def start_session(self, metadata: ClientMetadata) -> SessionResponse:
         )
 
         response = requests.post(
-            self.endpoint_session, headers=self._create_headers(), json=request.asdict()
+            self.endpoint_session,
+            headers=self._create_headers(),
+            json=request.asdict(),
+            proxies=get_proxies(),
         )
 
         if not response.ok:
@@ -77,7 +80,9 @@ def append_events(self, events: List[Event]):
             LOG.debug("posting to %s events %s", endpoint, docs)
 
         # FIXME: fault tolerance/timeouts
-        response = requests.post(endpoint, json={"events": docs}, headers=headers)
+        response = requests.post(
+            endpoint, json={"events": docs}, headers=headers, proxies=get_proxies()
+        )
 
         if self.debug:
             LOG.debug("response from %s was: %s %s", endpoint, response.status_code, response.text)
diff --git a/localstack/utils/common.py b/localstack/utils/common.py
--- a/localstack/utils/common.py
+++ b/localstack/utils/common.py
@@ -1078,11 +1078,23 @@ def format_bytes(count, default="n/a"):
     return count
 
 
+def get_proxies() -> Dict[str, str]:
+    proxy_map = {}
+    if config.OUTBOUND_HTTP_PROXY:
+        proxy_map["http"] = config.OUTBOUND_HTTP_PROXY
+    if config.OUTBOUND_HTTPS_PROXY:
+        proxy_map["https"] = config.OUTBOUND_HTTPS_PROXY
+    return proxy_map
+
+
 def download(url: str, path: str, verify_ssl=True):
     """Downloads file at url to the given path"""
     # make sure we're creating a new session here to
     # enable parallel file downloads during installation!
     s = requests.Session()
+    proxies = get_proxies()
+    if proxies:
+        s.proxies.update(proxies)
     # Use REQUESTS_CA_BUNDLE path. If it doesn't exist, use the method provided settings.
     # Note that a value that is not False, will result to True and will get the bundle file.
     r = s.get(url, stream=True, verify=os.getenv("REQUESTS_CA_BUNDLE", verify_ssl))


Test patch:
diff --git a/tests/unit/test_common.py b/tests/unit/test_common.py
--- a/tests/unit/test_common.py
+++ b/tests/unit/test_common.py
@@ -12,6 +12,7 @@
 import pytz
 import yaml
 
+from localstack import config
 from localstack.utils import common
 from localstack.utils.bootstrap import extract_port_flags
 from localstack.utils.common import is_empty_dir, mkdir, new_tmp_dir, rm_rf, save_file
@@ -367,6 +368,22 @@ def run_method(*args, **kwargs):
         common.cleanup_threads_and_processes()
         self.assertTrue(done.wait(timeout=2))
 
+    def test_proxy_map(self):
+        old_http_proxy = config.OUTBOUND_HTTP_PROXY
+        old_https_proxy = config.OUTBOUND_HTTPS_PROXY
+        config.OUTBOUND_HTTP_PROXY = "http://localhost"
+        config.OUTBOUND_HTTPS_PROXY = "https://localhost"
+        assert {
+            "http": config.OUTBOUND_HTTP_PROXY,
+            "https": config.OUTBOUND_HTTPS_PROXY,
+        } == common.get_proxies()
+        config.OUTBOUND_HTTP_PROXY = ""
+        assert {"https": config.OUTBOUND_HTTPS_PROXY} == common.get_proxies()
+        config.OUTBOUND_HTTPS_PROXY = ""
+        assert {} == common.get_proxies()
+        config.OUTBOUND_HTTP_PROXY = old_http_proxy
+        config.OUTBOUND_HTTPS_PROXY = old_https_proxy
+
 
 class TestCommandLine(unittest.TestCase):
     def test_extract_port_flags(self):