Repository: aws/aws-cli

Issue:
S3 transfer does not properly handle running out of disk space
If we run out of disk space (including inodes) we keep trying to sync.  We also don't throw any kind of exception when this happens.  I believe the root cause if from this code snippet:

```
def save_file(filename, response_data, last_update):
    body = response_data['Body']
    etag = response_data['ETag'][1:-1]
    d = os.path.dirname(filename)
    try:
        if not os.path.exists(d):
            os.makedirs(d)
    except Exception:      # <---- bad
        pass

```

We should be more selective in this case and only allow `errno.EEXIST` errors.



Discussion:

Resolving patch:
diff --git a/awscli/customizations/s3/fileinfo.py b/awscli/customizations/s3/fileinfo.py
--- a/awscli/customizations/s3/fileinfo.py
+++ b/awscli/customizations/s3/fileinfo.py
@@ -2,6 +2,7 @@
 import sys
 import time
 from functools import partial
+import errno
 import hashlib
 
 from dateutil.parser import parse
@@ -13,6 +14,10 @@
         guess_content_type, MD5Error
 
 
+class CreateDirectoryError(Exception):
+    pass
+
+
 def read_file(filename):
     """
     This reads the file into a form that can be sent to S3
@@ -34,8 +39,10 @@ def save_file(filename, response_data, last_update):
     try:
         if not os.path.exists(d):
             os.makedirs(d)
-    except Exception:
-        pass
+    except OSError as e:
+        if not e.errno == errno.EEXIST:
+            raise CreateDirectoryError(
+                "Could not create directory %s: %s" % (d, e))
     md5 = hashlib.md5()
     file_chunks = iter(partial(body.read, 1024 * 1024), b'')
     with open(filename, 'wb') as out_file:


Test patch:
diff --git a/tests/unit/customizations/s3/test_fileinfo.py b/tests/unit/customizations/s3/test_fileinfo.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/customizations/s3/test_fileinfo.py
@@ -0,0 +1,60 @@
+# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import os
+import tempfile
+import shutil
+from datetime import datetime
+from hashlib import md5
+
+import six
+import mock
+
+from tests import unittest
+from awscli.customizations.s3 import fileinfo
+
+
+class TestSaveFile(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.filename = os.path.join(self.tempdir, 'dir1', 'dir2', 'foo.txt')
+        etag = md5()
+        etag.update(b'foobar')
+        etag = etag.hexdigest()
+        self.response_data = {
+            'Body': six.BytesIO(b'foobar'),
+            'ETag': '"%s"' % etag,
+        }
+        self.last_update = datetime.now()
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test_save_file(self):
+        fileinfo.save_file(self.filename, self.response_data, self.last_update)
+        self.assertTrue(os.path.isfile(self.filename))
+
+    def test_save_file_dir_exists(self):
+        os.makedirs(os.path.dirname(self.filename))
+        # We should still be able to save the file.
+        fileinfo.save_file(self.filename, self.response_data, self.last_update)
+        self.assertTrue(os.path.isfile(self.filename))
+
+    @mock.patch('os.makedirs')
+    def test_makedir_other_exception(self, makedirs):
+        # If makedirs() raises any other kind of exception, we should
+        # propogate the exception.
+        makedirs.side_effect = RuntimeError()
+        with self.assertRaises(RuntimeError):
+            fileinfo.save_file(self.filename, self.response_data,
+                               self.last_update)
+        self.assertFalse(os.path.isfile(self.filename))