Repository: jschneier/django-storages

Issue:
Inefficient implementation of `listdir` in the S3 backend
The [current implementation](https://github.com/jschneier/django-storages/blob/f2880b16b57a36b241a54932255e1a852c0e0ac7/storages/backends/s3boto.py#L441) gets the lists of all objects in the bucket.

Not only is this inefficient, but it also breaks on large buckets because of a bug in boto: https://github.com/boto/boto/issues/3006

You should steal the [more efficient implementation](https://github.com/etianen/django-s3-storage/blob/e8c1cb2d2b65c8c7047c7851c65af31f091312c9/django_s3_storage/storage.py#L261) from django-s3-storage.



Discussion:

Resolving patch:
diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py
--- a/storages/backends/s3boto3.py
+++ b/storages/backends/s3boto3.py
@@ -520,25 +520,22 @@ def exists(self, name):
             return False
 
     def listdir(self, name):
-        name = self._normalize_name(self._clean_name(name))
-        # for the bucket.objects.filter and logic below name needs to end in /
-        # But for the root path "" we leave it as an empty string
-        if name and not name.endswith('/'):
-            name += '/'
+        path = self._normalize_name(self._clean_name(name))
+        # The path needs to end with a slash, but if the root is empty, leave
+        # it.
+        if path and not path.endswith('/'):
+            path += '/'
 
+        directories = []
         files = []
-        dirs = set()
-        base_parts = name.split("/")[:-1]
-        for item in self.bucket.objects.filter(Prefix=self._encode_name(name)):
-            parts = item.key.split("/")
-            parts = parts[len(base_parts):]
-            if len(parts) == 1:
-                # File
-                files.append(parts[0])
-            elif len(parts) > 1:
-                # Directory
-                dirs.add(parts[0])
-        return list(dirs), files
+        paginator = self.connection.meta.client.get_paginator('list_objects_v2')
+        pages = paginator.paginate(Bucket=self.bucket_name, Delimiter='/', Prefix=path)
+        for page in pages:
+            for entry in page.get('CommonPrefixes', ()):
+                directories.append(posixpath.relpath(entry['Prefix'], path))
+            for entry in page.get('Contents', ()):
+                files.append(posixpath.relpath(entry['Key'], path))
+        return directories, files
 
     def size(self, name):
         name = self._normalize_name(self._clean_name(name))


Test patch:
diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py
--- a/tests/test_s3boto3.py
+++ b/tests/test_s3boto3.py
@@ -399,50 +399,58 @@ def test_storage_delete(self):
         self.storage.bucket.Object.return_value.delete.assert_called_with()
 
     def test_storage_listdir_base(self):
-        file_names = ["some/path/1.txt", "2.txt", "other/path/3.txt", "4.txt"]
-
-        result = []
-        for p in file_names:
-            obj = mock.MagicMock()
-            obj.key = p
-            result.append(obj)
-        self.storage.bucket.objects.filter.return_value = iter(result)
-
-        dirs, files = self.storage.listdir("")
-        self.storage.bucket.objects.filter.assert_called_with(Prefix="")
-
-        self.assertEqual(len(dirs), 2)
-        for directory in ["some", "other"]:
-            self.assertTrue(directory in dirs,
-                            """ "%s" not in directory list "%s".""" % (
-                                directory, dirs))
-
-        self.assertEqual(len(files), 2)
-        for filename in ["2.txt", "4.txt"]:
-            self.assertTrue(filename in files,
-                            """ "%s" not in file list "%s".""" % (
-                                filename, files))
+        # Files:
+        #   some/path/1.txt
+        #   2.txt
+        #   other/path/3.txt
+        #   4.txt
+        pages = [
+            {
+                'CommonPrefixes': [
+                    {'Prefix': 'some'},
+                    {'Prefix': 'other'},
+                ],
+                'Contents': [
+                    {'Key': '2.txt'},
+                    {'Key': '4.txt'},
+                ],
+            },
+        ]
 
-    def test_storage_listdir_subdir(self):
-        file_names = ["some/path/1.txt", "some/2.txt"]
+        paginator = mock.MagicMock()
+        paginator.paginate.return_value = pages
+        self.storage._connections.connection.meta.client.get_paginator.return_value = paginator
+
+        dirs, files = self.storage.listdir('')
+        paginator.paginate.assert_called_with(Bucket=None, Delimiter='/', Prefix='')
 
-        result = []
-        for p in file_names:
-            obj = mock.MagicMock()
-            obj.key = p
-            result.append(obj)
-        self.storage.bucket.objects.filter.return_value = iter(result)
+        self.assertEqual(dirs, ['some', 'other'])
+        self.assertEqual(files, ['2.txt', '4.txt'])
+
+    def test_storage_listdir_subdir(self):
+        # Files:
+        #   some/path/1.txt
+        #   some/2.txt
+        pages = [
+            {
+                'CommonPrefixes': [
+                    {'Prefix': 'some/path'},
+                ],
+                'Contents': [
+                    {'Key': 'some/2.txt'},
+                ],
+            },
+        ]
 
-        dirs, files = self.storage.listdir("some/")
-        self.storage.bucket.objects.filter.assert_called_with(Prefix="some/")
+        paginator = mock.MagicMock()
+        paginator.paginate.return_value = pages
+        self.storage._connections.connection.meta.client.get_paginator.return_value = paginator
 
-        self.assertEqual(len(dirs), 1)
-        self.assertTrue('path' in dirs,
-                        """ "path" not in directory list "%s".""" % (dirs,))
+        dirs, files = self.storage.listdir('some/')
+        paginator.paginate.assert_called_with(Bucket=None, Delimiter='/', Prefix='some/')
 
-        self.assertEqual(len(files), 1)
-        self.assertTrue('2.txt' in files,
-                        """ "2.txt" not in files list "%s".""" % (files,))
+        self.assertEqual(dirs, ['path'])
+        self.assertEqual(files, ['2.txt'])
 
     def test_storage_size(self):
         obj = self.storage.bucket.Object.return_value