Repository: jschneier/django-storages

Issue:
S3BotoStorage fails when keys contain ":"
Trying to do this:
`storage.delete("2017-01-13T12:55:32.497821+00:00/somekey")`
Which fails like so:
```
  File "...", line 76, in ...
    storage.delete("2017-01-13T12:55:32.497821+00:00/somekey")
  File ".../local/lib/python2.7/site-packages/storages/backends/s3boto.py", line 438, in delete
    name = self._normalize_name(self._clean_name(name))
  File "/.../local/lib/python2.7/site-packages/storages/backends/s3boto.py", line 364, in _normalize_name
    name)
django.core.exceptions.SuspiciousOperation: Attempted access to "2017-01-13T12:55:32.497821+00:00/somekey" denied.
```

I have tracked this down to [storages.backends.s3boto.safe_join](https://github.com/jschneier/django-storages/blob/master/storages/backends/s3boto.py#L54)'s call to `urlparse.urljoin`. Having ":" confuses the join to skip the `/` at the beginning of `final_path` which then fails during the check that `final_path` is a sub path of `base`.

It's probably a small fix, but I unfortunately don't have enough knowledge of the urlparse lib to know how to do it. 

PS.
And yes, AWS supports having ":" part of the keys. Running `aws s3 rm s3://<bucket>/2017-01-13T12:55:32.497821+00:00/somekey` works fine.


Discussion:
Indeed. According to [Amazon's Docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html) the colon (":") should probably be url-encoded. That is a bug in the library, probably we should make `safe_join` significantly smarter and ensure it complies with the docs.
Out of curiosity how are you getting the file up to the server? If I use the normal Django fileuploader process then those characters are stripped out.
Also, renaming a file to have `:` as you do via the aws cli (which I believe uses boto3 anyway) urlencodes the characters.
This isn't so much of an issue in for the user uploads if it strips annoying characters then.

Our use case is for a backup script that uses datetimes for the folder names to be able to prune backups later solely based on the directory names. So we are mostly using django-storages as a convenient way to reuse the storage definitions we already use. We could of course just use boto3 directly, but it would be nice if it worked directly with django-storages.

So don't feel like it's too much of a priority to fix.
Ah, makes sense. What does the actual url that amazon shows you look like in S3? My hunch is it is already urlencoded which would make this a pretty easy fix.
Hi, sorry for late reply. Yes, the colons in the url are urlencoded.

So `2017-01-13T12:55:32.497821+00:00` turns into `2017-01-13T12%3A55%3A32.497821%2B00%3A00`

Resolving patch:
diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py
--- a/storages/backends/s3boto3.py
+++ b/storages/backends/s3boto3.py
@@ -22,7 +22,7 @@
     raise ImproperlyConfigured("Could not load Boto3's S3 bindings.\n"
                                "See https://github.com/boto/boto3")
 
-from storages.utils import setting
+from storages.utils import setting, safe_join
 
 boto3_version_info = tuple([int(i) for i in boto3_version.split('.')])
 
@@ -31,39 +31,6 @@
                                "higher.\nSee https://github.com/boto/boto3")
 
 
-def safe_join(base, *paths):
-    """
-    A version of django.utils._os.safe_join for S3 paths.
-
-    Joins one or more path components to the base path component
-    intelligently. Returns a normalized version of the final path.
-
-    The final path must be located inside of the base path component
-    (otherwise a ValueError is raised).
-
-    Paths outside the base path indicate a possible security
-    sensitive operation.
-    """
-    base_path = force_text(base)
-    base_path = base_path.rstrip('/')
-    paths = [force_text(p) for p in paths]
-
-    final_path = base_path
-    for path in paths:
-        final_path = urlparse.urljoin(final_path.rstrip('/') + "/", path)
-
-    # Ensure final_path starts with base_path and that the next character after
-    # the final path is '/' (or nothing, in which case final_path must be
-    # equal to base_path).
-    base_path_len = len(base_path)
-    if (not final_path.startswith(base_path) or
-            final_path[base_path_len:base_path_len + 1] not in ('', '/')):
-        raise ValueError('the joined path is located outside of the base path'
-                         ' component')
-
-    return final_path.lstrip('/')
-
-
 @deconstructible
 class S3Boto3StorageFile(File):
 
diff --git a/storages/utils.py b/storages/utils.py
--- a/storages/utils.py
+++ b/storages/utils.py
@@ -3,7 +3,6 @@
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.encoding import force_text
-from django.utils.six.moves.urllib import parse as urlparse
 
 
 def setting(name, default=None, strict=False):
@@ -63,16 +62,15 @@ def safe_join(base, *paths):
     base_path = base_path.rstrip('/')
     paths = [force_text(p) for p in paths]
 
-    final_path = base_path
-    for path in paths:
-        final_path = urlparse.urljoin(final_path.rstrip('/') + '/', path)
+    final_path = posixpath.normpath(posixpath.join(base_path + '/', *paths))
+    # posixpath.normpath() strips the trailing /. Add it back.
+    if paths[-1].endswith('/'):
+        final_path += '/'
 
     # Ensure final_path starts with base_path and that the next character after
-    # the final path is '/' (or nothing, in which case final_path must be
-    # equal to base_path).
+    # the final path is /.
     base_path_len = len(base_path)
-    if (not final_path.startswith(base_path) or
-            final_path[base_path_len:base_path_len + 1] not in ('', '/')):
+    if (not final_path.startswith(base_path) or final_path[base_path_len] != '/'):
         raise ValueError('the joined path is located outside of the base path'
                          ' component')
 


Test patch:
diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py
--- a/tests/test_s3boto3.py
+++ b/tests/test_s3boto3.py
@@ -18,11 +18,6 @@
 
 from storages.backends import s3boto3
 
-__all__ = (
-    'SafeJoinTest',
-    'S3Boto3StorageTests',
-)
-
 
 class S3Boto3TestCase(TestCase):
     def setUp(self):
@@ -30,43 +25,6 @@ def setUp(self):
         self.storage._connection = mock.MagicMock()
 
 
-class SafeJoinTest(TestCase):
-    def test_normal(self):
-        path = s3boto3.safe_join("", "path/to/somewhere", "other", "path/to/somewhere")
-        self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere")
-
-    def test_with_dot(self):
-        path = s3boto3.safe_join("", "path/./somewhere/../other", "..",
-                                 ".", "to/./somewhere")
-        self.assertEqual(path, "path/to/somewhere")
-
-    def test_base_url(self):
-        path = s3boto3.safe_join("base_url", "path/to/somewhere")
-        self.assertEqual(path, "base_url/path/to/somewhere")
-
-    def test_base_url_with_slash(self):
-        path = s3boto3.safe_join("base_url/", "path/to/somewhere")
-        self.assertEqual(path, "base_url/path/to/somewhere")
-
-    def test_suspicious_operation(self):
-        self.assertRaises(ValueError,
-                          s3boto3.safe_join, "base", "../../../../../../../etc/passwd")
-
-    def test_trailing_slash(self):
-        """
-        Test safe_join with paths that end with a trailing slash.
-        """
-        path = s3boto3.safe_join("base_url/", "path/to/somewhere/")
-        self.assertEqual(path, "base_url/path/to/somewhere/")
-
-    def test_trailing_slash_multi(self):
-        """
-        Test safe_join with multiple paths that end with a trailing slash.
-        """
-        path = s3boto3.safe_join("base_url/", "path/to/" "somewhere/")
-        self.assertEqual(path, "base_url/path/to/somewhere/")
-
-
 class S3Boto3StorageTests(S3Boto3TestCase):
 
     def test_clean_name(self):
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,3 +1,5 @@
+import datetime
+
 from django.test import TestCase
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
@@ -65,8 +67,10 @@ def test_base_url_with_slash(self):
         self.assertEqual(path, "base_url/path/to/somewhere")
 
     def test_suspicious_operation(self):
-        self.assertRaises(ValueError,
-                          utils.safe_join, "base", "../../../../../../../etc/passwd")
+        with self.assertRaises(ValueError):
+            utils.safe_join("base", "../../../../../../../etc/passwd")
+        with self.assertRaises(ValueError):
+            utils.safe_join("base", "/etc/passwd")
 
     def test_trailing_slash(self):
         """
@@ -79,5 +83,10 @@ def test_trailing_slash_multi(self):
         """
         Test safe_join with multiple paths that end with a trailing slash.
         """
-        path = utils.safe_join("base_url/", "path/to/" "somewhere/")
+        path = utils.safe_join("base_url/", "path/to/", "somewhere/")
         self.assertEqual(path, "base_url/path/to/somewhere/")
+
+    def test_datetime_isoformat(self):
+        dt = datetime.datetime(2017, 5, 19, 14, 45, 37, 123456)
+        path = utils.safe_join('base_url', dt.isoformat())
+        self.assertEqual(path, 'base_url/2017-05-19T14:45:37.123456')