Repository: aws/aws-cli

Issue:
fix the UnicodeDecodeError when sort the local files
I figured out the [s3 sync break by character decoding failed](https://github.com/aws/aws-cli/issues/378) is caused by sorted function. It will happen when 

```
sorted(os.list(u"some unicode path name"))
```

in the sorted function, it will call `filename.decode(...)` and go into trouble when special character in the filename.

My case looks like this:

```
(PYENV)[root@jp1 awscli]# aws s3 sync /usr/local/apache/htdocs/IGDmanual/zh_TW s3://muzee.ixd.foo/data_002 --cache-control 3600 
Bad filename: krazykar##.html, Skip to upload
```

I have no idea how to decode it correctly, but to skip the bad filename looks good to me.

ps. other projects also got troubles in it:
1. https://bugs.pypy.org/issue1035
2. https://bugs.launchpad.net/bzr/+bug/715547



Discussion:

Resolving patch:
diff --git a/awscli/clidriver.py b/awscli/clidriver.py
--- a/awscli/clidriver.py
+++ b/awscli/clidriver.py
@@ -203,7 +203,8 @@ def _handle_top_level_args(self, args):
             self.session.set_debug_logger(logger_name='botocore')
             self.session.set_debug_logger(logger_name='awscli')
         else:
-            self.session.set_stream_logger(logger_name='awscli', log_level='ERROR')
+            self.session.set_stream_logger(logger_name='awscli',
+                                           log_level=logging.ERROR)
 
 
 class CLICommand(object):
diff --git a/awscli/customizations/s3/filegenerator.py b/awscli/customizations/s3/filegenerator.py
--- a/awscli/customizations/s3/filegenerator.py
+++ b/awscli/customizations/s3/filegenerator.py
@@ -11,7 +11,9 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 import os
+import sys
 
+from six import text_type
 from dateutil.parser import parse
 from dateutil.tz import tzlocal
 
@@ -19,6 +21,29 @@
 from awscli.customizations.s3.utils import find_bucket_key, get_file_stat
 
 
+# This class is provided primarily to provide a detailed error message.
+
+class FileDecodingError(Exception):
+    """Raised when there was an issue decoding the file."""
+
+    ADVICE = (
+        "Please check your locale settings.  The filename was decoded as: %s\n"
+        "On posix platforms, check the LC_CTYPE environment variable."
+            % (sys.getfilesystemencoding())
+    )
+
+    def __init__(self, directory, filename):
+        self.directory = directory
+        self.file_name = filename
+        self.error_message = (
+            'There was an error trying to decode the the file "%s" in '
+            'directory "%s". \n%s' % (self.file_name,
+                                      self.directory.encode('utf-8'),
+                                      self.ADVICE)
+        )
+        super(FileDecodingError, self).__init__(self.error_message)
+
+
 class FileGenerator(object):
     """
     This is a class the creates a generator to yield files based on information
@@ -89,6 +114,7 @@ def list_files(self, path, dir_op):
             # separator to any directories.  We can then sort the contents,
             # and ensure byte order.
             names = listdir(path)
+            self._check_paths_decoded(path, names)
             for i, name in enumerate(names):
                 file_path = join(path, name)
                 if isdir(file_path):
@@ -108,6 +134,17 @@ def list_files(self, path, dir_op):
                     size, last_update = get_file_stat(file_path)
                     yield file_path, size, last_update
 
+    def _check_paths_decoded(self, path, names):
+        # We can get a UnicodeDecodeError if we try to listdir(<unicode>) and
+        # can't decode the contents with sys.getfilesystemencoding().  In this
+        # case listdir() returns the bytestring, which means that
+        # join(<unicode>, <str>) could raise a UnicodeDecodeError.  When this
+        # happens we raise a FileDecodingError that provides more information
+        # into what's going on.
+        for name in names:
+            if not isinstance(name, text_type):
+                raise FileDecodingError(path, name)
+
     def list_objects(self, s3_path, dir_op):
         """
         This function yields the appropriate object or objects under a
diff --git a/awscli/customizations/s3/s3handler.py b/awscli/customizations/s3/s3handler.py
--- a/awscli/customizations/s3/s3handler.py
+++ b/awscli/customizations/s3/s3handler.py
@@ -80,8 +80,10 @@ def call(self, files):
             self.result_queue.join()
 
         except Exception as e:
-            LOGGER.debug('Exception caught during task execution: %s',
-                         str(e), exc_info=True)
+            LOGGER.error('Exception caught during task execution: %s',
+                          str(e))
+            # Log the traceback at DEBUG level.
+            LOGGER.debug(str(e), exc_info=True)
         except KeyboardInterrupt:
             self.interrupt.set()
             self.result_queue.put({'message': "Cleaning up. Please wait...",


Test patch:
diff --git a/tests/unit/customizations/s3/__init__.py b/tests/unit/customizations/s3/__init__.py
--- a/tests/unit/customizations/s3/__init__.py
+++ b/tests/unit/customizations/s3/__init__.py
@@ -14,6 +14,7 @@
 import random
 from tests import unittest
 
+import six
 from mock import patch
 
 
@@ -42,21 +43,22 @@ def make_loc_files():
     has the file text1.txt and the directory another_directory inside.  Inside
     of another_directory it creates the file text2.txt.
     """
-    directory1 = os.path.abspath('.') + os.sep + 'some_directory' + os.sep
+    directory1 = six.text_type(
+        os.path.abspath('.') + os.sep + 'some_directory' + os.sep)
     if not os.path.exists(directory1):
         os.mkdir(directory1)
 
     string1 = b"This is a test."
-    filename1 = directory1 + "text1.txt"
+    filename1 = directory1 + u"text1.txt"
     with open(filename1, 'wb') as file1:
         file1.write(string1)
 
-    directory2 = directory1 + 'another_directory' + os.sep
+    directory2 = directory1 + u'another_directory' + os.sep
     if not os.path.exists(directory2):
         os.mkdir(directory2)
 
     string2 = b"This is another test."
-    filename2 = directory2 + "text2.txt"
+    filename2 = directory2 + u"text2.txt"
     with open(filename2, 'wb') as file2:
         file2.write(string2)
 
diff --git a/tests/unit/customizations/s3/test_filegenerator.py b/tests/unit/customizations/s3/test_filegenerator.py
--- a/tests/unit/customizations/s3/test_filegenerator.py
+++ b/tests/unit/customizations/s3/test_filegenerator.py
@@ -11,13 +11,15 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 import os
-import unittest
+from tests import unittest
 import tempfile
 import shutil
 
 import six
+import mock
 
-from awscli.customizations.s3.filegenerator import FileGenerator
+from awscli.customizations.s3.filegenerator import FileGenerator, \
+    FileDecodingError
 from awscli.customizations.s3.fileinfo import FileInfo
 from awscli.customizations.s3.utils import get_file_stat
 import botocore.session
@@ -28,10 +30,10 @@
 
 class LocalFileGeneratorTest(unittest.TestCase):
     def setUp(self):
-        self.local_file = os.path.abspath('.') + os.sep + 'some_directory' \
-            + os.sep + 'text1.txt'
-        self.local_dir = os.path.abspath('.') + os.sep + 'some_directory' \
-            + os.sep
+        self.local_file = six.text_type(os.path.abspath('.') + os.sep + 'some_directory' \
+            + os.sep + 'text1.txt')
+        self.local_dir = six.text_type(os.path.abspath('.') + os.sep + 'some_directory' \
+            + os.sep)
         self.session = FakeSession()
         self.service = self.session.get_service('s3')
         self.endpoint = self.service.get_endpoint('us-east-1')
@@ -112,6 +114,15 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.directory)
 
+    @mock.patch('os.listdir')
+    def test_error_raised_on_decoding_error(self, listdir_mock):
+        # On Python3, sys.getdefaultencoding
+        file_generator = FileGenerator(None, None, None, None)
+        # utf-8 encoding for U+2713.
+        listdir_mock.return_value = [b'\xe2\x9c\x93']
+        with self.assertRaises(FileDecodingError):
+            list(file_generator.list_files(self.directory, dir_op=True))
+
     def test_list_files_is_in_sorted_order(self):
         p = os.path.join
         open(p(self.directory, 'test-123.txt'), 'w').close()
diff --git a/tests/unit/test_clidriver.py b/tests/unit/test_clidriver.py
--- a/tests/unit/test_clidriver.py
+++ b/tests/unit/test_clidriver.py
@@ -12,6 +12,7 @@
 # language governing permissions and limitations under the License.
 from tests import unittest
 from tests.unit import BaseAWSCommandParamsTest
+import logging
 
 import mock
 import six
@@ -171,7 +172,7 @@ def test_profile(self):
     def test_error_logger(self):
         driver = CLIDriver(session=self.session)
         driver.main('s3 list-objects --bucket foo --profile foo'.split())
-        expected = {'log_level': 'ERROR', 'logger_name': 'awscli'}
+        expected = {'log_level': logging.ERROR, 'logger_name': 'awscli'}
         self.assertEqual(driver.session.stream_logger_args[1], expected)