Repository: minio/minio-py (python)

Issue:
Allow presigned DELETE url's for minio-py
Original issue was filed against minio-java (Issue#605). This is the extension of it on the minio-py side.

Pull request description:
Presigned delete url
Fixes #577

Patch:
diff --git a/minio/api.py b/minio/api.py
index d08c5ca..cb7e207 100644
--- a/minio/api.py
+++ b/minio/api.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Minio Python Library for Amazon S3 Compatible Cloud Storage, (C)
-# 2015, 2016 Minio, Inc.
+# 2015, 2016, 2017 Minio, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ minio.api
 
 This module implements the API.
 
-:copyright: (c) 2015, 2016 by Minio, Inc.
+:copyright: (c) 2015, 2016, 2017 by Minio, Inc.
 :license: Apache 2.0, see LICENSE for more details.
 
 """
@@ -67,6 +67,7 @@ from .helpers import (get_target_url, is_non_empty_string,
                       optimal_part_info,
                       is_valid_bucket_name, PartMetadata, read_full,
                       is_valid_bucket_notification_config,
+                      get_s3_region_from_endpoint,
                       mkdir_p, dump_http)
 from .helpers import (MAX_MULTIPART_OBJECT_SIZE,
                       MIN_PART_SIZE)
@@ -96,7 +97,7 @@ _DEFAULT_USER_AGENT = 'Minio {0} {1}'.format(
 
 
 # Duration of 7 days in seconds
-_SEVEN_DAYS_SECONDS = 604800
+_MAX_EXPIRY_TIME = 604800 # 7 days in seconds
 
 # Number of parallel workers which upload parts
 _PARALLEL_UPLOADERS = 3
@@ -141,6 +142,11 @@ class Minio(object):
 
         # Parse url endpoints.
         url_components = urlsplit(endpoint_url)
+
+        # Extract region if possible from endpoint.
+        if not region:
+            region = get_s3_region_from_endpoint(endpoint)
+
         self._region = region
         self._region_map = dict()
         self._endpoint_url = url_components.geturl()
@@ -1204,6 +1210,54 @@ class Minio(object):
                                                upload.upload_id)
                 return
 
+    def presigned_url(self, method,
+                    bucket_name,
+                    object_name,
+                    expires=timedelta(days=7),
+                    response_headers=None):
+        """
+        Presigns a method on an object and provides a url
+
+        Example:
+            from datetime import timedelta
+
+            presignedURL = presigned_url('GET',
+                                         'bucket_name',
+                                         'object_name',
+                                         expires=timedelta(days=7))
+            print(presignedURL)
+
+        :param bucket_name: Bucket for the presigned url.
+        :param object_name: Object for which presigned url is generated.
+        :param expires: Optional expires argument to specify timedelta.
+                        Defaults to 7days.
+        :params response_headers: Optional response_headers argument to
+                                  specify response fields like date, size,
+                                  type of file, data about server, etc.
+        :return: Presigned put object url.
+        """
+        is_valid_bucket_name(bucket_name)
+        is_non_empty_string(object_name)
+
+        if expires.total_seconds() < 1 or \
+           expires.total_seconds() > _MAX_EXPIRY_TIME:
+            raise InvalidArgumentError('Expires param valid values'
+                                       ' are between 1 sec to'
+                                       ' {0} secs'.format(_MAX_EXPIRY_TIME))
+
+        region = self._get_bucket_region(bucket_name)
+        url = get_target_url(self._endpoint_url,
+                             bucket_name=bucket_name,
+                             object_name=object_name,
+                             bucket_region=region)
+
+        return presign_v4(method, url,
+                          self._access_key,
+                          self._secret_key,
+                          region=region,
+                          expires=int(expires.total_seconds()),
+                          response_headers=response_headers)
+
     def presigned_get_object(self, bucket_name, object_name,
                              expires=timedelta(days=7),
                              response_headers=None):
@@ -1225,16 +1279,11 @@ class Minio(object):
            Defaults to 7days.
         :return: Presigned url.
         """
-        if expires.total_seconds() < 1 or \
-           expires.total_seconds() > _SEVEN_DAYS_SECONDS:
-            raise InvalidArgumentError('Expires param valid values'
-                                       ' are between 1 secs to'
-                                       ' {0} secs'.format(_SEVEN_DAYS_SECONDS))
-
-        return self._presigned_get_partial_object(bucket_name,
-                                                  object_name,
-                                                  expires,
-                                                  response_headers=response_headers)
+        return self.presigned_url('GET',
+                                  bucket_name,
+                                  object_name,
+                                  expires,
+                                  response_headers=response_headers)
 
     def presigned_put_object(self, bucket_name, object_name,
                              expires=timedelta(days=7)):
@@ -1255,27 +1304,10 @@ class Minio(object):
            Defaults to 7days.
         :return: Presigned put object url.
         """
-        is_valid_bucket_name(bucket_name)
-        is_non_empty_string(object_name)
-
-        if expires.total_seconds() < 1 or \
-                        expires.total_seconds() > _SEVEN_DAYS_SECONDS:
-            raise InvalidArgumentError('Expires param valid values'
-                                       ' are between 1 secs to'
-                                       ' {0} secs'.format(_SEVEN_DAYS_SECONDS))
-
-        region = self._get_bucket_region(bucket_name)
-        url = get_target_url(self._endpoint_url,
-                             bucket_name=bucket_name,
-                             object_name=object_name,
-                             bucket_region=region)
-
-        presign_url = presign_v4('PUT', url,
-                                 self._access_key,
-                                 self._secret_key,
-                                 region=region,
-                                 expires=int(expires.total_seconds()))
-        return presign_url
+        return self.presigned_url('PUT',
+                                  bucket_name,
+                                  object_name,
+                                  expires)
 
     def presigned_post_policy(self, post_policy):
         """
@@ -1373,46 +1405,6 @@ class Minio(object):
 
         return response
 
-    def _presigned_get_partial_object(self, bucket_name, object_name,
-                                      expires=timedelta(days=7),
-                                      offset=0, length=0,
-                                      response_headers=None):
-        """
-        Presigns a get partial object request and provides a url,
-        this is a internal function not exposed.
-
-        :param bucket_name: Bucket for the presigned url.
-        :param object_name: Object for which presigned url is generated.
-        :param expires: optional expires argument to specify timedelta.
-           Defaults to 7days.
-        :param offset, length: optional defaults to '0, 0'.
-        :return: Presigned url.
-        """
-        is_valid_bucket_name(bucket_name)
-        is_non_empty_string(object_name)
-
-        region = self._get_bucket_region(bucket_name)
-        url = get_target_url(self._endpoint_url,
-                             bucket_name=bucket_name,
-                             object_name=object_name,
-                             bucket_region=region)
-
-        headers = {}
-        if offset != 0 or length != 0:
-            request_range = '{}-{}'.format(
-                offset, "" if length == 0 else offset + length - 1
-            )
-            headers['Range'] = 'bytes=' + request_range
-
-        presign_url = presign_v4('GET', url,
-                                 self._access_key,
-                                 self._secret_key,
-                                 region=region,
-                                 headers=headers,
-                                 response_headers=response_headers,
-                                 expires=int(expires.total_seconds()))
-        return presign_url
-
     def _do_put_object(self, bucket_name, object_name, part_data,
                        part_size, upload_id='', part_number=0,
                        metadata=None):
diff --git a/minio/helpers.py b/minio/helpers.py
index e1f7d7b..2e1b990 100644
--- a/minio/helpers.py
+++ b/minio/helpers.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Minio Python Library for Amazon S3 Compatible Cloud Storage, (C)
-# 2015, 2016 Minio, Inc.
+# 2015, 2016, 2017 Minio, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@ minio.helpers
 
 This module implements all helper functions.
 
-:copyright: (c) 2015 by Minio, Inc.
+:copyright: (c) 2015, 2016, 2017 by Minio, Inc.
 :license: Apache 2.0, see LICENSE for more details.
 
 """
@@ -50,6 +50,31 @@ _ALLOWED_HOSTNAME_REGEX = re.compile(
     '^((?!-)[A-Z\\d-]{1,63}(?<!-)\\.)*((?!-)[A-Z\\d-]{1,63}(?<!-))$',
     re.IGNORECASE)
 
+_EXTRACT_REGION_REGEX = re.compile('s3[.-]?(.+?).amazonaws.com')
+
+def get_s3_region_from_endpoint(endpoint):
+    """
+    Extracts and returns an AWS S3 region from an endpoint
+    of form `s3-ap-southeast-1.amazonaws.com`
+
+    :param endpoint: Endpoint region to be extracted.
+    """
+
+    # Extract region by regex search.
+    m = _EXTRACT_REGION_REGEX.search(endpoint)
+    if m:
+        # Regex matches, we have found a region.
+        region = m.group(1)
+        if region == 'external-1':
+            # Handle special scenario for us-east-1 URL.
+            return 'us-east-1'
+        if region.startswith('dualstack'):
+            # Handle special scenario for dualstack URL.
+            return region.split('.')[1]
+        return region
+
+    # No regex matches return None.
+    return None
 
 def dump_http(method, url, request_headers, response, output_stream):
     """
@@ -273,10 +298,6 @@ def is_valid_endpoint(endpoint):
         if not _ALLOWED_HOSTNAME_REGEX.match(hostname):
             raise InvalidEndpointError('Hostname does not meet URL standards.')
 
-        if hostname.endswith('.amazonaws.com') and \
-           (hostname != 's3.amazonaws.com'):
-            raise InvalidEndpointError('Amazon S3 hostname should be '
-                                       's3.amazonaws.com.')
     except AttributeError as error:
         raise TypeError(error)
 


Test patch:
diff --git a/tests/unit/minio_test.py b/tests/unit/minio_test.py
index bc4c1d5..0c329c3 100644
--- a/tests/unit/minio_test.py
+++ b/tests/unit/minio_test.py
@@ -22,7 +22,8 @@ from minio import Minio
 from minio import __version__ as minio_version
 from minio.api import _DEFAULT_USER_AGENT
 from minio.error import InvalidEndpointError, InvalidBucketError
-from minio.helpers import get_target_url, is_valid_bucket_name
+from minio.helpers import (get_target_url, is_valid_bucket_name,
+                           get_s3_region_from_endpoint)
 
 class ValidBucketName(TestCase):
     @raises(InvalidBucketError)
@@ -102,3 +103,28 @@ class UserAgentTests(TestCase):
     def test_set_app_info_requires_non_empty_version(self):
         client = Minio('localhost:9000')
         client.set_app_info('hello', '')
+
+class GetRegionTests(TestCase):
+    def test_region_none(self):
+        region = get_s3_region_from_endpoint('localhost')
+        eq_(region, None)
+
+    def test_region_us_west(self):
+        region = get_s3_region_from_endpoint('s3-us-west-1.amazonaws.com')
+        eq_(region, 'us-west-1')
+
+    def test_region_with_dot(self):
+        region = get_s3_region_from_endpoint('s3.us-west-1.amazonaws.com')
+        eq_(region, 'us-west-1')
+
+    def test_region_with_dualstack(self):
+        region = get_s3_region_from_endpoint('s3.dualstack.us-west-1.amazonaws.com')
+        eq_(region, 'us-west-1')
+
+    def test_region_us_east(self):
+        region = get_s3_region_from_endpoint('s3.amazonaws.com')
+        eq_(region, None)
+
+    @raises(TypeError)
+    def test_invalid_value(self):
+        region = get_s3_region_from_endpoint(None)