Repository: aws/aws-cli

Issue:
Sync to s3 hangs forever - 'ascii' codec can't encode character
Think I'm experiencing an issue similar to #2575 attempting to sync from my Mac's local filesystem to s3:

    UnicodeEncodeError: 'ascii' codec can't encode character u'\u0303' in position 11: ordinal not in range(128)
    2017-07-29 00:49:40,775 - Thread-1 - awscli.customizations.s3.results - DEBUG - Error processing result QueuedResult(transfer_type='upload'

    Traceback (most recent call last):
    File "/Users/jasonsturges/Library/Python/2.7/lib/python/site-packages/awscli/customizations/s3/results.py", line 604, in _process_result
        result_handler(result)
    File "/Users/jasonsturges/Library/Python/2.7/lib/python/site-packages/awscli/customizations/s3/results.py", line 241, in __call__
        result=result)
    File "/Users/jasonsturges/Library/Python/2.7/lib/python/site-packages/awscli/customizations/s3/results.py", line 267, in _record_queued_result
        self._get_ongoing_dict_key(result)] = total_transfer_size
    File "/Users/jasonsturges/Library/Python/2.7/lib/python/site-packages/awscli/customizations/s3/results.py", line 250, in _get_ongoing_dict_key
        str(el) for el in [result.transfer_type, result.src, result.dest])
    File "/Users/jasonsturges/Library/Python/2.7/lib/python/site-packages/awscli/customizations/s3/results.py", line 250, in <genexpr>
        str(el) for el in [result.transfer_type, result.src, result.dest])

Without the `--debug` option, it appears to hang forever at "0 files", repeatedly trying the same operations while pummeling the network.

Within the folder I'm attempting to sync, there's a folder with the 'ñ' character in the name.

Sync command is simply:

    aws s3 sync . s3://[bucket]/

Strange, as this folder exists in s3 - though I can't recall, it is possible that I initially uploaded it using the aws `cp` command instead of `sync` in the past.

Also strange is that adding `--dryrun` with sync works perfectly fine, correctly indicating a list of files to be uploaded.

Versions:

- aws-cli/1.11.94 Python/2.7.10 Darwin/15.6.0 botocore/1.5.57
- OS X El Capitan (10.11.6)




Discussion:

Resolving patch:
diff --git a/awscli/customizations/s3/results.py b/awscli/customizations/s3/results.py
--- a/awscli/customizations/s3/results.py
+++ b/awscli/customizations/s3/results.py
@@ -22,7 +22,7 @@
 from s3transfer.exceptions import FatalError
 from s3transfer.subscribers import BaseSubscriber
 
-from awscli.compat import queue
+from awscli.compat import queue, ensure_text_type
 from awscli.customizations.s3.utils import relative_path
 from awscli.customizations.s3.utils import human_readable_size
 from awscli.customizations.utils import uni_print
@@ -246,8 +246,11 @@ def _get_ongoing_dict_key(self, result):
                 'Any result using _get_ongoing_dict_key must subclass from '
                 'BaseResult. Provided result is of type: %s' % type(result)
             )
-        return ':'.join(
-            str(el) for el in [result.transfer_type, result.src, result.dest])
+        key_parts = []
+        for result_property in [result.transfer_type, result.src, result.dest]:
+            if result_property is not None:
+                key_parts.append(ensure_text_type(result_property))
+        return u':'.join(key_parts)
 
     def _pop_result_from_ongoing_dicts(self, result):
         ongoing_key = self._get_ongoing_dict_key(result)


Test patch:
diff --git a/tests/functional/s3/test_cp_command.py b/tests/functional/s3/test_cp_command.py
--- a/tests/functional/s3/test_cp_command.py
+++ b/tests/functional/s3/test_cp_command.py
@@ -316,7 +316,7 @@ def test_turn_off_glacier_warnings(self):
         self.assertEqual(len(self.operations_called), 1)
         self.assertEqual(self.operations_called[0][0].name, 'HeadObject')
         self.assertEqual('', stderr)
-    
+
     def test_cp_with_sse_flag(self):
         full_path = self.files.create_file('foo.txt', 'contents')
         cmdline = (
@@ -442,6 +442,23 @@ def test_cannot_use_recursive_with_stream(self):
             'Streaming currently is only compatible with non-recursive cp '
             'commands', stderr)
 
+    def test_upload_unicode_path(self):
+        self.parsed_responses = [
+            {'ContentLength': 10,
+             'LastModified': '00:00:00Z'},  # HeadObject
+            {'ETag': '"foo"'}  # PutObject
+        ]
+        command = u's3 cp s3://bucket/\u2603 s3://bucket/\u2713'
+        stdout, stderr, rc = self.run_cmd(command, expected_rc=0)
+
+        success_message = (
+            u'copy: s3://bucket/\u2603 to s3://bucket/\u2713'
+        )
+        self.assertIn(success_message, stdout)
+
+        progress_message = 'Completed 10 Bytes'
+        self.assertIn(progress_message, stdout)
+
 
 class TestStreamingCPCommand(BaseAWSCommandParamsTest):
     def test_streaming_upload(self):
diff --git a/tests/unit/customizations/s3/test_results.py b/tests/unit/customizations/s3/test_results.py
--- a/tests/unit/customizations/s3/test_results.py
+++ b/tests/unit/customizations/s3/test_results.py
@@ -787,6 +787,34 @@ def test_unknown_result_object(self):
         self.assertEqual(self.result_recorder.expected_files_transferred, 0)
         self.assertEqual(self.result_recorder.files_transferred, 0)
 
+    def test_result_with_unicode(self):
+        unicode_source = u'\u2713'
+        self.result_recorder(
+            QueuedResult(
+                transfer_type=self.transfer_type, src=unicode_source,
+                dest=self.dest, total_transfer_size=self.total_transfer_size
+            )
+        )
+        self.assertEqual(
+            self.result_recorder.expected_bytes_transferred,
+            self.total_transfer_size
+        )
+        self.assertEqual(self.result_recorder.expected_files_transferred, 1)
+
+    def test_result_with_encoded_unicode(self):
+        unicode_source = u'\u2713'.encode('utf-8')
+        self.result_recorder(
+            QueuedResult(
+                transfer_type=self.transfer_type, src=unicode_source,
+                dest=self.dest, total_transfer_size=self.total_transfer_size
+            )
+        )
+        self.assertEqual(
+            self.result_recorder.expected_bytes_transferred,
+            self.total_transfer_size
+        )
+        self.assertEqual(self.result_recorder.expected_files_transferred, 1)
+
 
 class BaseResultPrinterTest(unittest.TestCase):
     def setUp(self):