Repository: minio/minio-py (python)

Issue:
minio-py fails erroneously with minio.error.InvalidEndpointError: InvalidEndpointError: message: Amazon S3 hostname should be s3.amazonaws.com
I have minio installed to our kubernetes cluster on amazon. The hostname ends in amazonaws.com but is not S3. When I run the minio-py with the correct credentials, the code fails here:

https://github.com/minio/minio-py/blob/master/minio/helpers.py#L276-L279

The check for s3 ignores the fact that I may have an external IP exposed through amazon that is not s3, but rather minio - in this case hosted on our kubernetes cluster.

Do you need a pull request?

Pull request:
Do not fail input checks for endpoints with .amazonaws.com as prefix
Additionally also support extracting region from the endpoint, this
technique is an optimization avoids GetBucketLocation() request
and makes it more performant.

Fixes #570

Patch:
diff --git a/minio/api.py b/minio/api.py
index d08c5ca..5e76c9d 100644
--- a/minio/api.py
+++ b/minio/api.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Minio Python Library for Amazon S3 Compatible Cloud Storage, (C)
-# 2015, 2016 Minio, Inc.
+# 2015, 2016, 2017 Minio, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ minio.api
 
 This module implements the API.
 
-:copyright: (c) 2015, 2016 by Minio, Inc.
+:copyright: (c) 2015, 2016, 2017 by Minio, Inc.
 :license: Apache 2.0, see LICENSE for more details.
 
 """
@@ -67,6 +67,7 @@ from .helpers import (get_target_url, is_non_empty_string,
                       optimal_part_info,
                       is_valid_bucket_name, PartMetadata, read_full,
                       is_valid_bucket_notification_config,
+                      get_s3_region_from_endpoint,
                       mkdir_p, dump_http)
 from .helpers import (MAX_MULTIPART_OBJECT_SIZE,
                       MIN_PART_SIZE)
@@ -141,6 +142,11 @@ class Minio(object):
 
         # Parse url endpoints.
         url_components = urlsplit(endpoint_url)
+
+        # Extract region if possible from endpoint.
+        if not region:
+            region = get_s3_region_from_endpoint(endpoint)
+
         self._region = region
         self._region_map = dict()
         self._endpoint_url = url_components.geturl()
diff --git a/minio/helpers.py b/minio/helpers.py
index e1f7d7b..2e1b990 100644
--- a/minio/helpers.py
+++ b/minio/helpers.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Minio Python Library for Amazon S3 Compatible Cloud Storage, (C)
-# 2015, 2016 Minio, Inc.
+# 2015, 2016, 2017 Minio, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@ minio.helpers
 
 This module implements all helper functions.
 
-:copyright: (c) 2015 by Minio, Inc.
+:copyright: (c) 2015, 2016, 2017 by Minio, Inc.
 :license: Apache 2.0, see LICENSE for more details.
 
 """
@@ -50,6 +50,31 @@ _ALLOWED_HOSTNAME_REGEX = re.compile(
     '^((?!-)[A-Z\\d-]{1,63}(?<!-)\\.)*((?!-)[A-Z\\d-]{1,63}(?<!-))$',
     re.IGNORECASE)
 
+_EXTRACT_REGION_REGEX = re.compile('s3[.-]?(.+?).amazonaws.com')
+
+def get_s3_region_from_endpoint(endpoint):
+    """
+    Extracts and returns an AWS S3 region from an endpoint
+    of form `s3-ap-southeast-1.amazonaws.com`
+
+    :param endpoint: Endpoint region to be extracted.
+    """
+
+    # Extract region by regex search.
+    m = _EXTRACT_REGION_REGEX.search(endpoint)
+    if m:
+        # Regex matches, we have found a region.
+        region = m.group(1)
+        if region == 'external-1':
+            # Handle special scenario for us-east-1 URL.
+            return 'us-east-1'
+        if region.startswith('dualstack'):
+            # Handle special scenario for dualstack URL.
+            return region.split('.')[1]
+        return region
+
+    # No regex matches return None.
+    return None
 
 def dump_http(method, url, request_headers, response, output_stream):
     """
@@ -273,10 +298,6 @@ def is_valid_endpoint(endpoint):
         if not _ALLOWED_HOSTNAME_REGEX.match(hostname):
             raise InvalidEndpointError('Hostname does not meet URL standards.')
 
-        if hostname.endswith('.amazonaws.com') and \
-           (hostname != 's3.amazonaws.com'):
-            raise InvalidEndpointError('Amazon S3 hostname should be '
-                                       's3.amazonaws.com.')
     except AttributeError as error:
         raise TypeError(error)
 


Test patch:
diff --git a/tests/unit/minio_test.py b/tests/unit/minio_test.py
index bc4c1d5..0c329c3 100644
--- a/tests/unit/minio_test.py
+++ b/tests/unit/minio_test.py
@@ -22,7 +22,8 @@ from minio import Minio
 from minio import __version__ as minio_version
 from minio.api import _DEFAULT_USER_AGENT
 from minio.error import InvalidEndpointError, InvalidBucketError
-from minio.helpers import get_target_url, is_valid_bucket_name
+from minio.helpers import (get_target_url, is_valid_bucket_name,
+                           get_s3_region_from_endpoint)
 
 class ValidBucketName(TestCase):
     @raises(InvalidBucketError)
@@ -102,3 +103,28 @@ class UserAgentTests(TestCase):
     def test_set_app_info_requires_non_empty_version(self):
         client = Minio('localhost:9000')
         client.set_app_info('hello', '')
+
+class GetRegionTests(TestCase):
+    def test_region_none(self):
+        region = get_s3_region_from_endpoint('localhost')
+        eq_(region, None)
+
+    def test_region_us_west(self):
+        region = get_s3_region_from_endpoint('s3-us-west-1.amazonaws.com')
+        eq_(region, 'us-west-1')
+
+    def test_region_with_dot(self):
+        region = get_s3_region_from_endpoint('s3.us-west-1.amazonaws.com')
+        eq_(region, 'us-west-1')
+
+    def test_region_with_dualstack(self):
+        region = get_s3_region_from_endpoint('s3.dualstack.us-west-1.amazonaws.com')
+        eq_(region, 'us-west-1')
+
+    def test_region_us_east(self):
+        region = get_s3_region_from_endpoint('s3.amazonaws.com')
+        eq_(region, None)
+
+    @raises(TypeError)
+    def test_invalid_value(self):
+        region = get_s3_region_from_endpoint(None)