Repository: minio/minio-py
Pull request #1107 (2021-04-16 16:24:24)

list_objects(): add url encoding type support


Patch:
diff --git a/docs/API.md b/docs/API.md
index faf436f..c0ffe0a 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -182,21 +182,22 @@ client.remove_bucket("my-bucket")
 
 <a name="list_objects"></a>
 
-### list_objects(bucket_name, prefix=None, recursive=False, start_after=None, include_user_meta=False, include_version=False, use_api_v1=False)
+### list_objects(bucket_name, prefix=None, recursive=False, start_after=None, include_user_meta=False, include_version=False, use_api_v1=False, use_url_encoding_type=True)
 
 Lists object information of a bucket.
 
 __Parameters__
 
-| Param               | Type   | Description                                              |
-|:--------------------|:-------|:---------------------------------------------------------|
-| `bucket_name`       | _str_  | Name of the bucket.                                      |
-| `prefix`            | _str_  | Object name starts with prefix.                          |
-| `recursive`         | _bool_ | List recursively than directory structure emulation.     |
-| `start_after`       | _str_  | List objects after this key name.                        |
-| `include_user_meta` | _bool_ | MinIO specific flag to control to include user metadata. |
-| `include_version`   | _bool_ | Flag to control whether include object versions.         |
-| `use_api_v1`        | _bool_ | Flag to control to use ListObjectV1 S3 API or not.       |
+| Param                   | Type   | Description                                                  |
+|:------------------------|:-------|:-------------------------------------------------------------|
+| `bucket_name`           | _str_  | Name of the bucket.                                          |
+| `prefix`                | _str_  | Object name starts with prefix.                              |
+| `recursive`             | _bool_ | List recursively than directory structure emulation.         |
+| `start_after`           | _str_  | List objects after this key name.                            |
+| `include_user_meta`     | _bool_ | MinIO specific flag to control to include user metadata.     |
+| `include_version`       | _bool_ | Flag to control whether include object versions.             |
+| `use_api_v1`            | _bool_ | Flag to control to use ListObjectV1 S3 API or not.           |
+| `use_url_encoding_type` | _bool_ | Flag to control whether URL encoding type to be used or not. |
 
 __Return Value__
 
diff --git a/minio/api.py b/minio/api.py
index bc32683..d2202a6 100644
--- a/minio/api.py
+++ b/minio/api.py
@@ -1788,7 +1788,8 @@ class Minio:  # pylint: disable=too-many-public-methods
 
     def list_objects(self, bucket_name, prefix=None, recursive=False,
                      start_after=None, include_user_meta=False,
-                     include_version=False, use_api_v1=False):
+                     include_version=False, use_api_v1=False,
+                     use_url_encoding_type=True):
         """
         Lists object information of a bucket.
 
@@ -1801,6 +1802,8 @@ class Minio:  # pylint: disable=too-many-public-methods
         :param include_version: Flag to control whether include object
                                 versions.
         :param use_api_v1: Flag to control to use ListObjectV1 S3 API or not.
+        :param use_url_encoding_type: Flag to control whether URL encoding type
+                                      to be used or not.
         :return: Iterator of :class:`Object <Object>`.
 
         Example::
@@ -1843,6 +1846,7 @@ class Minio:  # pylint: disable=too-many-public-methods
             start_after=start_after,
             use_api_v1=use_api_v1,
             include_version=include_version,
+            encoding_type="url" if use_url_encoding_type else None,
         )
 
     def stat_object(self, bucket_name, object_name, ssec=None, version_id=None,
@@ -2753,7 +2757,7 @@ class Minio:  # pylint: disable=too-many-public-methods
             response = self._execute("GET", bucket_name, query_params=query)
 
             objects, is_truncated, start_after, version_id_marker = (
-                parse_list_objects(response, bucket_name)
+                parse_list_objects(response)
             )
 
             if not include_version:
diff --git a/minio/datatypes.py b/minio/datatypes.py
index e571235..e7a16d5 100644
--- a/minio/datatypes.py
+++ b/minio/datatypes.py
@@ -196,7 +196,8 @@ class Object:
         return self._content_type
 
     @classmethod
-    def fromxml(cls, element, bucket_name, is_delete_marker=False):
+    def fromxml(cls, element, bucket_name, is_delete_marker=False,
+                encoding_type=None):
         """Create new object with values from XML element."""
         tag = findtext(element, "LastModified")
         last_modified = None if tag is None else from_iso8601utc(tag)
@@ -219,9 +220,13 @@ class Object:
             key = child.tag.split("}")[1] if "}" in child.tag else child.tag
             metadata[key] = child.text
 
+        object_name = findtext(element, "Key", True)
+        if encoding_type == "url":
+            object_name = unquote(object_name)
+
         return cls(
             bucket_name,
-            findtext(element, "Key"),
+            object_name,
             last_modified=last_modified,
             etag=etag,
             size=size,
@@ -235,33 +240,51 @@ class Object:
         )
 
 
-def parse_list_objects(response, bucket_name):
+def parse_list_objects(response, bucket_name=None):
     """Parse ListObjects/ListObjectsV2/ListObjectVersions response."""
     element = ET.fromstring(response.data.decode())
+    bucket_name = findtext(element, "Name", True)
+    encoding_type = findtext(element, "EncodingType")
     elements = findall(element, "Contents")
-    objects = [Object.fromxml(tag, bucket_name) for tag in elements]
+    objects = [
+        Object.fromxml(tag, bucket_name, encoding_type=encoding_type)
+        for tag in elements
+    ]
     marker = objects[-1].object_name if objects else None
 
     elements = findall(element, "Version")
-    objects += [Object.fromxml(tag, bucket_name) for tag in elements]
+    objects += [
+        Object.fromxml(tag, bucket_name, encoding_type=encoding_type)
+        for tag in elements
+    ]
 
     elements = findall(element, "CommonPrefixes")
     objects += [
-        Object(bucket_name, findtext(tag, "Prefix"))
-        for tag in elements
+        Object(
+            bucket_name, unquote(findtext(tag, "Prefix", True))
+            if encoding_type == "url" else findtext(tag, "Prefix", True)
+        ) for tag in elements
     ]
 
     elements = findall(element, "DeleteMarker")
-    objects += [Object.fromxml(tag, bucket_name, True) for tag in elements]
+    objects += [
+        Object.fromxml(tag, bucket_name, is_delete_marker=True,
+                       encoding_type=encoding_type)
+        for tag in elements
+    ]
 
     is_truncated = (findtext(element, "IsTruncated") or "").lower() == "true"
     key_marker = findtext(element, "NextKeyMarker")
+    if key_marker and encoding_type == "url":
+        key_marker = unquote(key_marker)
     version_id_marker = findtext(element, "NextVersionIdMarker")
     continuation_token = findtext(element, "NextContinuationToken")
     if key_marker is not None:
         continuation_token = key_marker
     if continuation_token is None:
         continuation_token = findtext(element, "NextMarker")
+        if continuation_token and encoding_type == "url":
+            continuation_token = unquote(continuation_token)
     if continuation_token is None and is_truncated:
         continuation_token = marker
     return objects, is_truncated, continuation_token, version_id_marker
@@ -457,8 +480,13 @@ class ListPartsResult:
 class Upload:
     """ Upload information of a multipart upload."""
 
-    def __init__(self, element):
-        self._object_name = unquote(findtext(element, "Key", True))
+    def __init__(self, element, encoding_type=None):
+        self._encoding_type = encoding_type
+        self._object_name = findtext(element, "Key", True)
+        self._object_name = (
+            unquote(self._object_name) if self._encoding_type == "url"
+            else self._object_name
+        )
         self._upload_id = findtext(element, "UploadId")
         tag = find(element, "Initiator")
         self._initiator_id = (
@@ -525,14 +553,21 @@ class ListMultipartUploadsResult:
 
     def __init__(self, response):
         element = ET.fromstring(response.data.decode())
+        self._encoding_type = findtext(element, "EncodingType")
         self._bucket_name = findtext(element, "Bucket")
         self._key_marker = findtext(element, "KeyMarker")
         if self._key_marker:
-            self._key_marker = unquote(self._key_marker)
+            self._key_marker = (
+                unquote(self._key_marker) if self._encoding_type == "url"
+                else self._key_marker
+            )
         self._upload_id_marker = findtext(element, "UploadIdMarker")
         self._next_key_marker = findtext(element, "NextKeyMarker")
         if self._next_key_marker:
-            self._next_key_marker = unquote(self._next_key_marker)
+            self._next_key_marker = (
+                unquote(self._next_key_marker) if self._encoding_type == "url"
+                else self._next_key_marker
+            )
         self._next_upload_id_marker = findtext(element, "NextUploadIdMarker")
         self._max_uploads = findtext(element, "MaxUploads")
         if self._max_uploads:
@@ -542,7 +577,10 @@ class ListMultipartUploadsResult:
             self._is_truncated is not None and
             self._is_truncated.lower() == "true"
         )
-        self._uploads = [Upload(tag) for tag in findall(element, "Upload")]
+        self._uploads = [
+            Upload(tag, self._encoding_type)
+            for tag in findall(element, "Upload")
+        ]
 
     @property
     def bucket_name(self):
@@ -579,6 +617,11 @@ class ListMultipartUploadsResult:
         """Get is-truncated flag."""
         return self._is_truncated
 
+    @property
+    def encoding_type(self):
+        """Get encoding type."""
+        return self._encoding_type
+
     @property
     def uploads(self):
         """Get uploads."""


Test patch:
diff --git a/tests/unit/list_objects_test.py b/tests/unit/list_objects_test.py
index c66a942..f1c5bca 100644
--- a/tests/unit/list_objects_test.py
+++ b/tests/unit/list_objects_test.py
@@ -42,8 +42,8 @@ class ListObjectsTest(TestCase):
         mock_server.mock_add_request(
             MockResponse(
                 "GET",
-                "https://localhost:9000/bucket?delimiter=&list-type=2"
-                "&max-keys=1000&prefix=",
+                "https://localhost:9000/bucket?delimiter=&encoding-type=url"
+                "&list-type=2&max-keys=1000&prefix=",
                 {"User-Agent": _DEFAULT_USER_AGENT},
                 200,
                 content=mock_data.encode(),
@@ -86,8 +86,8 @@ class ListObjectsTest(TestCase):
         mock_server.mock_add_request(
             MockResponse(
                 "GET",
-                "https://localhost:9000/bucket?delimiter=%2F&list-type=2"
-                "&max-keys=1000&prefix=",
+                "https://localhost:9000/bucket?delimiter=%2F&encoding-type=url"
+                "&list-type=2&max-keys=1000&prefix=",
                 {"User-Agent": _DEFAULT_USER_AGENT},
                 200,
                 content=mock_data.encode(),
diff --git a/tests/unit/list_objects_v1_test.py b/tests/unit/list_objects_v1_test.py
index a9d5947..0ca2c21 100644
--- a/tests/unit/list_objects_v1_test.py
+++ b/tests/unit/list_objects_v1_test.py
@@ -42,8 +42,8 @@ class ListObjectsV1Test(TestCase):
         mock_server.mock_add_request(
             MockResponse(
                 "GET",
-                "https://localhost:9000/bucket"
-                "?delimiter=&max-keys=1000&prefix=",
+                "https://localhost:9000/bucket?delimiter=&encoding-type=url"
+                "&max-keys=1000&prefix=",
                 {"User-Agent": _DEFAULT_USER_AGENT},
                 200,
                 content=mock_data.encode(),
@@ -97,8 +97,8 @@ class ListObjectsV1Test(TestCase):
         mock_server.mock_add_request(
             MockResponse(
                 "GET",
-                "https://localhost:9000/bucket"
-                "?delimiter=%2F&max-keys=1000&prefix=",
+                "https://localhost:9000/bucket?delimiter=%2F&encoding-type=url"
+                "&max-keys=1000&prefix=",
                 {"User-Agent": _DEFAULT_USER_AGENT},
                 200,
                 content=mock_data.encode(),
@@ -112,8 +112,8 @@ class ListObjectsV1Test(TestCase):
             mock_server.mock_add_request(
                 MockResponse(
                     "GET",
-                    "https://localhost:9000/bucket"
-                    "?delimiter=%2F&max-keys=1000&prefix=",
+                    "https://localhost:9000/bucket?delimiter=%2F&encoding-type=url"
+                    "&max-keys=1000&prefix=",
                     {"User-Agent": _DEFAULT_USER_AGENT},
                     200,
                     content=b"",
@@ -194,8 +194,8 @@ class ListObjectsV1Test(TestCase):
         mock_server.mock_add_request(
             MockResponse(
                 "GET",
-                "https://localhost:9000/bucket"
-                "?delimiter=&max-keys=1000&prefix=",
+                "https://localhost:9000/bucket?delimiter=&encoding-type=url"
+                "&max-keys=1000&prefix=",
                 {"User-Agent": _DEFAULT_USER_AGENT},
                 200,
                 content=mock_data1.encode(),
@@ -210,8 +210,8 @@ class ListObjectsV1Test(TestCase):
             mock_server.mock_add_request(
                 MockResponse(
                     "GET",
-                    "https://localhost:9000/bucket"
-                    "?delimiter=&marker=marker&max-keys=1000&prefix=",
+                    "https://localhost:9000/bucket?delimiter=&encoding-type=url"
+                    "&marker=marker&max-keys=1000&prefix=",
                     {"User-Agent": _DEFAULT_USER_AGENT},
                     200,
                     content=mock_data2.encode(),