Repository: aws/aws-cli

Issue:
S3 operations between buckets fails with unicode file names
I couldn't sync files that contained unicoded characters in their file names between two buckets. Eg.:

``` bash
copy failed: s3://source_bucket/Barça.jpeg to s3://target_bucket/Barça.jpeg
u'\xe7'
copy failed: s3://source_bucket/Sem-Título-5.jpg to s3://target_bucket/Sem-Título-5.jpg
u'\xed'
```

**Edit:** I should also say that I am running Python 2.7.5



Discussion:
Confirmed.  `local -> s3` and `s3 -> local` works, but `s3 -> s3` fails.  Traceback:

```
Traceback (most recent call last):
  File "aws-cli/awscli/customizations/s3/tasks.py", line 72, in __call__
    getattr(filename, filename.operation)()
  File "aws-cli/awscli/customizations/s3/fileinfo.py", line 304, in copy
    copy_source = quote(self.src)
  File "/usr/local/Cellar/python/2.7.3/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib.py", line 1268, in quote
    return ''.join(map(quoter, s))
KeyError: u'\u2713'
```

Should be a quick fix.  Thanks for the report.

The files containing non-ascii characters were correctly synced when Python3 was used.


Resolving patch:
diff --git a/awscli/customizations/s3/fileinfo.py b/awscli/customizations/s3/fileinfo.py
--- a/awscli/customizations/s3/fileinfo.py
+++ b/awscli/customizations/s3/fileinfo.py
@@ -301,7 +301,7 @@ def copy(self):
         """
         Copies a object in s3 to another location in s3.
         """
-        copy_source = quote(self.src)
+        copy_source = quote(self.src.encode('utf-8'), safe='/~')
         bucket, key = find_bucket_key(self.dest)
         params = {'endpoint': self.endpoint, 'bucket': bucket,
                   'copy_source': copy_source, 'key': key}


Test patch:
diff --git a/tests/integration/customizations/s3/test_s3handler.py b/tests/integration/customizations/s3/test_s3handler.py
--- a/tests/integration/customizations/s3/test_s3handler.py
+++ b/tests/integration/customizations/s3/test_s3handler.py
@@ -154,6 +154,34 @@ def test_multi_upload(self):
         self.assertIn(print_op, self.output.getvalue())
 
 
+class S3HandlerTestUnicodeMove(unittest.TestCase):
+    def setUp(self):
+        self.session = botocore.session.get_session(EnvironmentVariables)
+        params = {'region': 'us-east-1', 'acl': ['private']}
+        self.s3_handler = S3Handler(self.session, params)
+        self.bucket = make_s3_files(self.session, key1=u'\u2713')
+        self.bucket2 = create_bucket(self.session)
+        self.s3_files = [self.bucket + '/' + u'\u2713']
+        self.s3_files2 = [self.bucket2 + '/' + u'\u2713']
+
+    def tearDown(self):
+        s3_cleanup(self.bucket, self.session, key1=u'\u2713')
+        s3_cleanup(self.bucket2, self.session, key1=u'\u2713')
+
+    def test_move_unicode(self):
+        # Confirm there are no objects in the bucket.
+        self.assertEqual(len(list_contents(self.bucket2, self.session)), 0)
+        # Create file info objects to perform move.
+        tasks = []
+        for i in range(len(self.s3_files)):
+            tasks.append(FileInfo(src=self.s3_files[i], src_type='s3',
+                                  dest=self.s3_files2[i], dest_type='s3',
+                                  operation='move', size=0))
+        # Perform the move.
+        self.s3_handler.call(tasks)
+        self.assertEqual(len(list_contents(self.bucket2, self.session)), 1)
+
+
 class S3HandlerTestMove(unittest.TestCase):
     """
     This class tests the ability to move s3 objects.  The move
diff --git a/tests/unit/customizations/s3/__init__.py b/tests/unit/customizations/s3/__init__.py
--- a/tests/unit/customizations/s3/__init__.py
+++ b/tests/unit/customizations/s3/__init__.py
@@ -75,7 +75,7 @@ def clean_loc_files(files):
                 os.rmdir(filename)
 
 
-def make_s3_files(session):
+def make_s3_files(session, key1='text1.txt', key2='text2.txt'):
     """
     Creates a randomly generated bucket in s3 with the files text1.txt and
     another_directory/text2.txt inside.  The directory is manually created
@@ -91,14 +91,14 @@ def make_s3_files(session):
     string2 = "This is another test."
     http_response, response_data = operation.call(endpoint,
                                                   bucket=bucket,
-                                                  key='text1.txt',
+                                                  key=key1,
                                                   body=string1)
     http_response, response_data = operation.call(endpoint,
                                                   bucket=bucket,
                                                   key='another_directory/')
     http_response, r_data = operation.call(endpoint,
                                            bucket=bucket,
-                                           key='another_directory/text2.txt',
+                                           key='another_directory/%s' % key2,
                                            body=string2)
     return bucket
 
@@ -123,7 +123,7 @@ def create_bucket(session):
     return bucket_name
 
 
-def s3_cleanup(bucket, session):
+def s3_cleanup(bucket, session, key1='text1.txt', key2='text2.txt'):
     """
     Function to cleanup generated s3 bucket and files.
     """
@@ -133,13 +133,13 @@ def s3_cleanup(bucket, session):
     operation = service.get_operation('DeleteObject')
     http_response, r_data = operation.call(endpoint,
                                            bucket=bucket,
-                                           key='text1.txt')
+                                           key=key1)
     http_response, r_data = operation.call(endpoint,
                                            bucket=bucket,
                                            key='another_directory/')
     http_response, r_data = operation.call(endpoint,
                                            bucket=bucket,
-                                           key='another_directory/text2.txt')
+                                           key='another_directory/%s' % key2)
     operation = service.get_operation('DeleteBucket')
     http_response, r_data = operation.call(endpoint, bucket=bucket)
 
diff --git a/tests/unit/customizations/s3/fake_session.py b/tests/unit/customizations/s3/fake_session.py
--- a/tests/unit/customizations/s3/fake_session.py
+++ b/tests/unit/customizations/s3/fake_session.py
@@ -15,6 +15,7 @@
 import requests
 from six import StringIO
 
+from botocore.compat import unquote
 from mock import MagicMock
 
 from awscli.customizations.s3.filegenerator import find_bucket_key
@@ -211,6 +212,9 @@ def copy_object(self, kwargs):
         key = kwargs['key']
         copy_source = kwargs['copy_source']
         src_bucket, src_key = find_bucket_key(copy_source)
+        src_key = unquote(src_key)
+        if hasattr(src_key, 'decode'):
+            src_key = src_key.decode('utf-8')
         response_data = {}
         etag = ''
         if bucket in self.session.s3 or src_bucket in self.session.s3:
diff --git a/tests/unit/customizations/s3/test_s3handler.py b/tests/unit/customizations/s3/test_s3handler.py
--- a/tests/unit/customizations/s3/test_s3handler.py
+++ b/tests/unit/customizations/s3/test_s3handler.py
@@ -260,6 +260,16 @@ def tearDown(self):
         clean_loc_files(self.loc_files)
         s3_cleanup(self.bucket, self.session)
 
+    def test_move_unicode(self):
+        self.bucket2 = make_s3_files(self.session, key1=u'\u2713')
+        tasks = [FileInfo(src=self.bucket2 + '/' + u'\u2713',
+                          src_type='s3',
+                          dest=self.bucket + '/' + u'\u2713',
+                          dest_type='s3', operation='move',
+                          size=0)]
+        self.s3_handler.call(tasks)
+        self.assertEqual(len(list_contents(self.bucket, self.session)), 1)
+
     def test_move(self):
         # Create file info objects to perform move.
         files = [self.loc_files[0], self.loc_files[1]]