Repository: boto/boto

Issue:
Calculating credential scope fails when passing "localhost" as host (endpoint)
Related to #1732

When passing 'localhost as the endpoint for a DynamoDBConnection, calculating the credential scope fails.

https://github.com/boto/boto/blob/develop/boto/auth.py#L431

It looks like your auth code is expecting the standard amazon endpoint syntax to determine which service to use, which in this case is "localhost" instead of something like "dynamodb.us-east-1.amazonaws.com".

File "/Users/jfeatherstone/dev/virtualenvs/deos/lib/python2.7/site-packages/boto/auth.py", line 434, in credential_scope
elif parts[1] == 'us-gov':
IndexError: list index out of range



Discussion:
This is actually a little more complicated than originally thought. Our connection pooling isn't port-aware and our connection pool HTTP/HTTPS connection creation code is only partially port-aware, so I'm working on updating those to support this use case.

Sounds great.  Thanks for your work on this.  It will become a huge part of our workflow to use dynamodb offline for local development.

Any progress on this? I'd love to use the local database for our development, but I need boto for that =)

@danielgtaylor any update here?


Resolving patch:
diff --git a/boto/auth.py b/boto/auth.py
--- a/boto/auth.py
+++ b/boto/auth.py
@@ -431,13 +431,17 @@ def credential_scope(self, http_request):
         parts = http_request.host.split('.')
         if self.region_name is not None:
             region_name = self.region_name
-        elif parts[1] == 'us-gov':
-            region_name = 'us-gov-west-1'
-        else:
-            if len(parts) == 3:
-                region_name = 'us-east-1'
+        elif len(parts) > 1:
+            if parts[1] == 'us-gov':
+                region_name = 'us-gov-west-1'
             else:
-                region_name = parts[1]
+                if len(parts) == 3:
+                    region_name = 'us-east-1'
+                else:
+                    region_name = parts[1]
+        else:
+            region_name = parts[0]
+
         if self.service_name is not None:
             service_name = self.service_name
         else:
diff --git a/boto/connection.py b/boto/connection.py
--- a/boto/connection.py
+++ b/boto/connection.py
@@ -101,7 +101,7 @@
 class HostConnectionPool(object):
 
     """
-    A pool of connections for one remote (host,is_secure).
+    A pool of connections for one remote (host,port,is_secure).
 
     When connections are added to the pool, they are put into a
     pending queue.  The _mexe method returns connections to the pool
@@ -145,7 +145,7 @@ def put(self, conn):
     def get(self):
         """
         Returns the next connection in this pool that is ready to be
-        reused.  Returns None of there aren't any.
+        reused.  Returns None if there aren't any.
         """
         # Discard ready connections that are too old.
         self.clean()
@@ -234,7 +234,7 @@ class ConnectionPool(object):
     STALE_DURATION = 60.0
 
     def __init__(self):
-        # Mapping from (host,is_secure) to HostConnectionPool.
+        # Mapping from (host,port,is_secure) to HostConnectionPool.
         # If a pool becomes empty, it is removed.
         self.host_to_pool = {}
         # The last time the pool was cleaned.
@@ -259,7 +259,7 @@ def size(self):
         """
         return sum(pool.size() for pool in self.host_to_pool.values())
 
-    def get_http_connection(self, host, is_secure):
+    def get_http_connection(self, host, port, is_secure):
         """
         Gets a connection from the pool for the named host.  Returns
         None if there is no connection that can be reused. It's the caller's
@@ -268,18 +268,18 @@ def get_http_connection(self, host, is_secure):
         """
         self.clean()
         with self.mutex:
-            key = (host, is_secure)
+            key = (host, port, is_secure)
             if key not in self.host_to_pool:
                 return None
             return self.host_to_pool[key].get()
 
-    def put_http_connection(self, host, is_secure, conn):
+    def put_http_connection(self, host, port, is_secure, conn):
         """
         Adds a connection to the pool of connections that can be
         reused for the named host.
         """
         with self.mutex:
-            key = (host, is_secure)
+            key = (host, port, is_secure)
             if key not in self.host_to_pool:
                 self.host_to_pool[key] = HostConnectionPool()
             self.host_to_pool[key].put(conn)
@@ -551,7 +551,7 @@ def __init__(self, host, aws_access_key_id=None,
             self.host_header = self.provider.host_header
 
         self._pool = ConnectionPool()
-        self._connection = (self.server_name(), self.is_secure)
+        self._connection = (self.host, self.port, self.is_secure)
         self._last_rs = None
         self._auth_handler = auth.get_auth_handler(
               host, config, self.provider, self._required_auth_capability())
@@ -680,12 +680,12 @@ def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
         self.no_proxy = os.environ.get('no_proxy', '') or os.environ.get('NO_PROXY', '')
         self.use_proxy = (self.proxy != None)
 
-    def get_http_connection(self, host, is_secure):
-        conn = self._pool.get_http_connection(host, is_secure)
+    def get_http_connection(self, host, port, is_secure):
+        conn = self._pool.get_http_connection(host, port, is_secure)
         if conn is not None:
             return conn
         else:
-            return self.new_http_connection(host, is_secure)
+            return self.new_http_connection(host, port, is_secure)
 
     def skip_proxy(self, host):
         if not self.no_proxy:
@@ -703,16 +703,29 @@ def skip_proxy(self, host):
 
         return False
 
-    def new_http_connection(self, host, is_secure):
-        if self.use_proxy and not is_secure and \
-                not self.skip_proxy(host):
-            host = '%s:%d' % (self.proxy, int(self.proxy_port))
+    def new_http_connection(self, host, port, is_secure):
         if host is None:
             host = self.server_name()
+
+        # Make sure the host is really just the host, not including
+        # the port number
+        host = host.split(':', 1)[0]
+
+        http_connection_kwargs = self.http_connection_kwargs.copy()
+
+        # Connection factories below expect a port keyword argument
+        http_connection_kwargs['port'] = port
+
+        # Override host with proxy settings if needed
+        if self.use_proxy and not is_secure and \
+                not self.skip_proxy(host):
+            host = self.proxy
+            http_connection_kwargs['port'] = int(self.proxy_port)
+
         if is_secure:
             boto.log.debug(
                     'establishing HTTPS connection: host=%s, kwargs=%s',
-                    host, self.http_connection_kwargs)
+                    host, http_connection_kwargs)
             if self.use_proxy and not self.skip_proxy(host):
                 connection = self.proxy_ssl(host, is_secure and 443 or 80)
             elif self.https_connection_factory:
@@ -720,35 +733,35 @@ def new_http_connection(self, host, is_secure):
             elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
                 connection = https_connection.CertValidatingHTTPSConnection(
                         host, ca_certs=self.ca_certificates_file,
-                        **self.http_connection_kwargs)
+                        **http_connection_kwargs)
             else:
                 connection = httplib.HTTPSConnection(host,
-                        **self.http_connection_kwargs)
+                        **http_connection_kwargs)
         else:
             boto.log.debug('establishing HTTP connection: kwargs=%s' %
-                    self.http_connection_kwargs)
+                    http_connection_kwargs)
             if self.https_connection_factory:
                 # even though the factory says https, this is too handy
                 # to not be able to allow overriding for http also.
                 connection = self.https_connection_factory(host,
-                    **self.http_connection_kwargs)
+                    **http_connection_kwargs)
             else:
                 connection = httplib.HTTPConnection(host,
-                    **self.http_connection_kwargs)
+                    **http_connection_kwargs)
         if self.debug > 1:
             connection.set_debuglevel(self.debug)
         # self.connection must be maintained for backwards-compatibility
         # however, it must be dynamically pulled from the connection pool
         # set a private variable which will enable that
         if host.split(':')[0] == self.host and is_secure == self.is_secure:
-            self._connection = (host, is_secure)
+            self._connection = (host, port, is_secure)
         # Set the response class of the http connection to use our custom
         # class.
         connection.response_class = HTTPResponse
         return connection
 
-    def put_http_connection(self, host, is_secure, connection):
-        self._pool.put_http_connection(host, is_secure, connection)
+    def put_http_connection(self, host, port, is_secure, connection):
+        self._pool.put_http_connection(host, port, is_secure, connection)
 
     def proxy_ssl(self, host=None, port=None):
         if host and port:
@@ -841,6 +854,7 @@ def _mexe(self, request, sender=None, override_num_retries=None,
         boto.log.debug('Data: %s' % request.body)
         boto.log.debug('Headers: %s' % request.headers)
         boto.log.debug('Host: %s' % request.host)
+        boto.log.debug('Port: %s' % request.port)
         boto.log.debug('Params: %s' % request.params)
         response = None
         body = None
@@ -850,7 +864,8 @@ def _mexe(self, request, sender=None, override_num_retries=None,
         else:
             num_retries = override_num_retries
         i = 0
-        connection = self.get_http_connection(request.host, self.is_secure)
+        connection = self.get_http_connection(request.host, request.port,
+                                              self.is_secure)
         while i <= num_retries:
             # Use binary exponential backoff to desynchronize client requests.
             next_sleep = random.random() * (2 ** i)
@@ -902,24 +917,29 @@ def _mexe(self, request, sender=None, override_num_retries=None,
                     if conn_header_value == 'close':
                         connection.close()
                     else:
-                        self.put_http_connection(request.host, self.is_secure,
-                                                 connection)
+                        self.put_http_connection(request.host, request.port,
+                                                 self.is_secure, connection)
                     return response
                 else:
                     scheme, request.host, request.path, \
                         params, query, fragment = urlparse.urlparse(location)
                     if query:
                         request.path += '?' + query
+                    # urlparse can return both host and port in netloc, so if
+                    # that's the case we need to split them up properly
+                    if ':' in request.host:
+                        request.host, request.port = request.host.split(':', 1)
                     msg = 'Redirecting: %s' % scheme + '://'
                     msg += request.host + request.path
                     boto.log.debug(msg)
                     connection = self.get_http_connection(request.host,
+                                                          request.port,
                                                           scheme == 'https')
                     response = None
                     continue
             except PleaseRetryException, e:
                 boto.log.debug('encountered a retry exception: %s' % e)
-                connection = self.new_http_connection(request.host,
+                connection = self.new_http_connection(request.host, request.port,
                                                       self.is_secure)
                 response = e.response
             except self.http_exceptions, e:
@@ -931,7 +951,7 @@ def _mexe(self, request, sender=None, override_num_retries=None,
                         raise e
                 boto.log.debug('encountered %s exception, reconnecting' % \
                                   e.__class__.__name__)
-                connection = self.new_http_connection(request.host,
+                connection = self.new_http_connection(request.host, request.port,
                                                       self.is_secure)
             time.sleep(next_sleep)
             i += 1
@@ -1021,7 +1041,7 @@ def get_utf8_value(self, value):
     def make_request(self, action, params=None, path='/', verb='GET'):
         http_request = self.build_base_http_request(verb, path, None,
                                                     params, {}, '',
-                                                    self.server_name())
+                                                    self.host)
         if action:
             http_request.params['Action'] = action
         if self.APIVersion:
diff --git a/boto/dynamodb2/layer1.py b/boto/dynamodb2/layer1.py
--- a/boto/dynamodb2/layer1.py
+++ b/boto/dynamodb2/layer1.py
@@ -1475,13 +1475,13 @@ def update_table(self, table_name, provisioned_throughput):
     def make_request(self, action, body):
         headers = {
             'X-Amz-Target': '%s.%s' % (self.TargetPrefix, action),
-            'Host': self.region.endpoint,
+            'Host': self.host,
             'Content-Type': 'application/x-amz-json-1.0',
             'Content-Length': str(len(body)),
         }
         http_request = self.build_base_http_request(
             method='POST', path='/', auth_path='/', params={},
-            headers=headers, data=body)
+            headers=headers, data=body, host=self.host)
         response = self._mexe(http_request, sender=None,
                               override_num_retries=self.NumberRetries,
                               retry_handler=self._retry_handler)


Test patch:
diff --git a/tests/unit/auth/test_sigv4.py b/tests/unit/auth/test_sigv4.py
--- a/tests/unit/auth/test_sigv4.py
+++ b/tests/unit/auth/test_sigv4.py
@@ -153,6 +153,25 @@ def test_credential_scope(self):
         region_name = credential_scope.split('/')[1]
         self.assertEqual(region_name, 'us-west-1')
 
+        # Test connections to custom locations, e.g. localhost:8080
+        auth = HmacAuthV4Handler('localhost', Mock(), self.provider,
+                                 service_name='iam')
+
+        request = HTTPRequest(
+            'POST', 'http', 'localhost', 8080,
+            '/', '/',
+            {'Action': 'ListAccountAliases', 'Version': '2010-05-08'},
+            {
+                'Content-Length': '44',
+                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                'X-Amz-Date': '20130808T013210Z'
+            },
+            'Action=ListAccountAliases&Version=2010-05-08')
+        credential_scope = auth.credential_scope(request)
+        timestamp, region, service, v = credential_scope.split('/')
+        self.assertEqual(region, 'localhost')
+        self.assertEqual(service, 'iam')
+
     def test_headers_to_sign(self):
         auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com',
                                  Mock(), self.provider)
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -19,6 +19,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 #
+from __future__ import with_statement
+
 import os
 import urlparse
 from tests.unit import unittest
@@ -82,11 +84,13 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                  api_version=None, security_token=None,
                  validate_certs=True):
         self.region = region
+        if host is None:
+            host = self.region.endpoint
         AWSQueryConnection.__init__(self, aws_access_key_id,
                                     aws_secret_access_key,
                                     is_secure, port, proxy, proxy_port,
                                     proxy_user, proxy_pass,
-                                    self.region.endpoint, debug,
+                                    host, debug,
                                     https_connection_factory, path,
                                     security_token,
                                     validate_certs=validate_certs)
@@ -311,6 +315,30 @@ def mock_put_conn(*args, **kwargs):
         # set or we have another problem.
         self.assertEqual(resp1.getheader('connection'), 'close')
 
+    def test_port_pooling(self):
+        conn = self.region.connect(aws_access_key_id='access_key',
+                                   aws_secret_access_key='secret',
+                                   port=8080)
+
+        # Pick a connection, then put it back
+        con1 = conn.get_http_connection(conn.host, conn.port, conn.is_secure)
+        conn.put_http_connection(conn.host, conn.port, conn.is_secure, con1)
+
+        # Pick another connection, which hopefully is the same yet again
+        con2 = conn.get_http_connection(conn.host, conn.port, conn.is_secure)
+        conn.put_http_connection(conn.host, conn.port, conn.is_secure, con2)
+
+        self.assertEqual(con1, con2)
+
+        # Change the port and make sure a new connection is made
+        conn.port = 8081
+
+        con3 = conn.get_http_connection(conn.host, conn.port, conn.is_secure)
+        conn.put_http_connection(conn.host, conn.port, conn.is_secure, con3)
+
+        self.assertNotEqual(con1, con3)
+
+
 class TestAWSQueryStatus(TestAWSQueryConnection):
 
     def test_get_status(self):