Repository: boto/botocore
Pull request #2574 (2021-12-10 20:25:24)

Support IPv6 endpoint urls
This patch should expand our support for endpoint URLs to include both ipv4 and ipv6 addresses. This should address boto/boto3#3086.


Patch:
diff --git a/botocore/endpoint.py b/botocore/endpoint.py
index 5ab370354..f209170fd 100644
--- a/botocore/endpoint.py
+++ b/botocore/endpoint.py
@@ -22,7 +22,9 @@ from botocore.vendored import six
 from botocore.awsrequest import create_request_object
 from botocore.exceptions import HTTPClientError
 from botocore.httpsession import URLLib3Session
-from botocore.utils import is_valid_endpoint_url, get_environ_proxies
+from botocore.utils import (
+    is_valid_endpoint_url, is_valid_ipv6_endpoint_url, get_environ_proxies
+)
 from botocore.hooks import first_non_none_response
 from botocore.history import get_global_history_recorder
 from botocore.response import StreamingBody
@@ -272,18 +274,19 @@ class EndpointCreator(object):
     def __init__(self, event_emitter):
         self._event_emitter = event_emitter
 
-    def create_endpoint(self, service_model, region_name, endpoint_url,
-                        verify=None, response_parser_factory=None,
-                        timeout=DEFAULT_TIMEOUT,
-                        max_pool_connections=MAX_POOL_CONNECTIONS,
-                        http_session_cls=URLLib3Session,
-                        proxies=None,
-                        socket_options=None,
-                        client_cert=None,
-                        proxies_config=None):
-        if not is_valid_endpoint_url(endpoint_url):
-
+    def create_endpoint(
+        self, service_model, region_name, endpoint_url,
+        verify=None, response_parser_factory=None,
+        timeout=DEFAULT_TIMEOUT, max_pool_connections=MAX_POOL_CONNECTIONS,
+        http_session_cls=URLLib3Session, proxies=None, socket_options=None,
+        client_cert=None, proxies_config=None
+    ):
+        if (
+            not is_valid_endpoint_url(endpoint_url)
+            and not is_valid_ipv6_endpoint_url(endpoint_url)
+        ):
             raise ValueError("Invalid endpoint: %s" % endpoint_url)
+
         if proxies is None:
             proxies = self._get_proxies(endpoint_url)
         endpoint_prefix = service_model.endpoint_prefix
diff --git a/botocore/exceptions.py b/botocore/exceptions.py
index f2fbcbc71..65784270c 100644
--- a/botocore/exceptions.py
+++ b/botocore/exceptions.py
@@ -142,6 +142,10 @@ class ProxyConnectionError(ConnectionError, requests.exceptions.ProxyError):
     fmt = 'Failed to connect to proxy URL: "{proxy_url}"'
 
 
+class ResponseStreamingError(HTTPClientError):
+    fmt = 'An error occurred while reading from response stream: {error}'
+
+
 class NoCredentialsError(BotoCoreError):
     """
     No credentials could be found.
diff --git a/botocore/response.py b/botocore/response.py
index be32345c4..a381a06c5 100644
--- a/botocore/response.py
+++ b/botocore/response.py
@@ -15,8 +15,11 @@
 import logging
 
 from botocore.compat import set_socket_timeout
-from botocore.exceptions import IncompleteReadError, ReadTimeoutError
+from botocore.exceptions import (
+    IncompleteReadError, ReadTimeoutError, ResponseStreamingError
+)
 from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError
+from urllib3.exceptions import ProtocolError as URLLib3ProtocolError
 from botocore import parsers
 
 # Keep these imported.  There's pre-existing code that uses them.
@@ -80,6 +83,8 @@ class StreamingBody(object):
         except URLLib3ReadTimeoutError as e:
             # TODO: the url will be None as urllib3 isn't setting it yet
             raise ReadTimeoutError(endpoint_url=e.url, error=e)
+        except URLLib3ProtocolError as e:
+            raise ResponseStreamingError(error=e)
         self._amount_read += len(chunk)
         if amt is None or (not chunk and amt > 0):
             # If the server sends empty contents or
diff --git a/botocore/utils.py b/botocore/utils.py
index 6b9e7195d..c7a4d75aa 100644
--- a/botocore/utils.py
+++ b/botocore/utils.py
@@ -1032,8 +1032,8 @@ class ArgumentGenerator(object):
 def is_valid_ipv6_endpoint_url(endpoint_url):
     if UNSAFE_URL_CHARS.intersection(endpoint_url):
         return False
-    netloc = urlparse(endpoint_url).netloc
-    return IPV6_ADDRZ_RE.match(netloc) is not None
+    hostname = '[{}]'.format(urlparse(endpoint_url).hostname)
+    return IPV6_ADDRZ_RE.match(hostname) is not None
 
 
 def is_valid_endpoint_url(endpoint_url):


Test patch:
diff --git a/tests/unit/test_endpoint.py b/tests/unit/test_endpoint.py
index 34b2834e7..8469fd903 100644
--- a/tests/unit/test_endpoint.py
+++ b/tests/unit/test_endpoint.py
@@ -12,6 +12,8 @@
 # language governing permissions and limitations under the License.
 import socket
 
+import pytest
+
 from tests import mock
 from tests import unittest
 
@@ -293,6 +295,25 @@ class TestEndpointCreator(unittest.TestCase):
             endpoint_url='https://endpoint.url')
         self.assertEqual(endpoint.host, 'https://endpoint.url')
 
+    def test_creates_endpoint_with_ipv4_url(self):
+        endpoint = self.creator.create_endpoint(
+            self.service_model, region_name='us-east-1',
+            endpoint_url='https://192.168.0.0')
+        self.assertEqual(endpoint.host, 'https://192.168.0.0')
+
+    def test_creates_endpoint_with_ipv6_url(self):
+        endpoint = self.creator.create_endpoint(
+            self.service_model, region_name='us-east-1',
+            endpoint_url='https://[100:0:2::61]:7480')
+        self.assertEqual(endpoint.host, 'https://[100:0:2::61]:7480')
+
+    def test_raises_error_with_invalid_url(self):
+        with pytest.raises(ValueError):
+            self.creator.create_endpoint(
+                self.service_model, region_name='us-east-1',
+                endpoint_url='https://*.aws.amazon.com/'
+            )
+
     def test_create_endpoint_with_default_timeout(self):
         self.creator.create_endpoint(
             self.service_model, region_name='us-west-2',
diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py
index 29afcd0be..e7ccf8cf5 100644
--- a/tests/unit/test_response.py
+++ b/tests/unit/test_response.py
@@ -16,11 +16,14 @@ import datetime
 
 from dateutil.tz import tzutc
 from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError
+from urllib3.exceptions import ProtocolError as URLLib3ProtocolError
 
 import botocore
 from botocore import response
 from botocore.compat import six
-from botocore.exceptions import IncompleteReadError, ReadTimeoutError
+from botocore.exceptions import (
+    IncompleteReadError, ReadTimeoutError, ResponseStreamingError
+)
 from botocore.awsrequest import AWSResponse
 
 XMLBODY1 = (b'<?xml version="1.0" encoding="UTF-8"?><Error>'
@@ -166,6 +169,20 @@ class TestStreamWrapper(unittest.TestCase):
         with self.assertRaises(ReadTimeoutError):
             stream.read()
 
+    def test_catches_urllib3_protocol_error(self):
+        class ProtocolErrorBody:
+            def read(*args, **kwargs):
+                raise URLLib3ProtocolError(None, None, None)
+
+            def geturl(*args, **kwargs):
+                return 'http://example.com'
+
+        stream = response.StreamingBody(
+            ProtocolErrorBody(), content_length=None
+        )
+        with self.assertRaises(ResponseStreamingError):
+            stream.read()
+
     def test_streaming_line_abstruse_newline_standard(self):
         for chunk_size in range(1, 30):
             body = six.BytesIO(b'1234567890\r\n1234567890\r\n12345\r\n')