Repository: aws/aws-cli

Issue:
aws s3 sync fails if user does not own all files
I am finding that `aws s3 sync` will fail if the active user does not have ownership of files in the sync target, regardless of what permissions are set for group and other.

**Normal result**

```
$ aws s3 sync s3://example.com/ example/
download: s3://example.com/file.txt to example/file.txt
```

**Problematic result**

To reproduce the problem, create an empty file that is world-writable but owned by root. aws s3 sync will fail to sync this file, even though the effective permissions allow the user to write to the file.

```
$ mkdir example && touch example/file.txt
$ sudo chown root example/file.txt && sudo chmod 777 example/file.txt

$ aws s3 sync s3://example.com/ example/
download failed: s3://example.com/file.txt to example/file.txt
[Errno 1] Operation not permitted: '/example/file.txt'

$ ls -l example/file.txt
-rwxrwxrwx  1 root  wheel  54  7 May 17:55 example/file.txt
```

However, when running as the root user (`sudo aws s3 sync`) I cannot reproduce the problem. The file downloads as expected.

**Environment**

I am able to reproduce this issue on both machines I've tried it on:
- aws-cli/1.7.25 Python/2.7.6 Darwin/14.3.0 (Mac OS X 10.10)
- aws-cli/1.7.10 Python/2.7.6 Linux/3.13.0-40-generic (Ubuntu 14.04)



Discussion:
I can confirm. The issue is that when we download the file we need to set the last modified/access time to that of the S3 file in order to be able to round trip syncing given our sync strategy. It is the call that is modifying the last modified time that is failing. You can see it by running the following python script on the `file.txt` file that you made:

```
>>> import os
>>> os.utime('file.txt', (1, 1))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
OSError: [Errno 1] Operation not permitted: 'file.txt'
```

Will need to do research into if there is any way around this, other than running the command as the root.

I dug deeper on this one and noticed a few things, `os.utime` works when `None` is specified as the second argument as it sets the time attributes to be the current time and it fails when times are specified. Digging into the `utime` and `utimes` syscall source code we see the following comment

```
* If times==NULL, set access and modification to current time,
* must be owner or have write permission.
* Else, update from *times, must be owner or super user.
*/
long do_utimes(int dfd, const char __user *filename, struct timespec *times, int flags)
{
```

If you specify `*times` (the time tuple in `os.utime`) then you must be super user or owner (root) in this case. But if you just want to update to the current time then you only need to have write access to the file.

So this isn't a boto bug or python bug (could use some documentation though), this is the specified behaviour of the `utime` syscall. More info here:
http://lxr.free-electrons.com/source/fs/utimes.c#L132

Depending on what S3 is trying to do with `os.utime` we could potentially just use `os.utime(<PATH>, None)` to have it use the latest (in this edge case)


Resolving patch:
diff --git a/awscli/customizations/s3/executor.py b/awscli/customizations/s3/executor.py
--- a/awscli/customizations/s3/executor.py
+++ b/awscli/customizations/s3/executor.py
@@ -16,7 +16,7 @@
 import threading
 
 from awscli.customizations.s3.utils import uni_print, bytes_print, \
-    IORequest, IOCloseRequest, StablePriorityQueue
+    IORequest, IOCloseRequest, StablePriorityQueue, set_file_utime
 from awscli.customizations.s3.tasks import OrderableTask
 from awscli.compat import queue
 
@@ -181,8 +181,7 @@ def _handle_io_close_request(self, task):
             fileobj.close()
             del self.fd_descriptor_cache[task.filename]
         if task.desired_mtime is not None:
-            os.utime(task.filename, (task.desired_mtime,
-                                     task.desired_mtime))
+            set_file_utime(task.filename, task.desired_mtime)
 
     def _handle_stream_task(self, data):
         fileobj = sys.stdout
diff --git a/awscli/customizations/s3/fileinfo.py b/awscli/customizations/s3/fileinfo.py
--- a/awscli/customizations/s3/fileinfo.py
+++ b/awscli/customizations/s3/fileinfo.py
@@ -11,7 +11,7 @@
 
 from botocore.compat import quote
 from awscli.customizations.s3.utils import find_bucket_key, \
-    uni_print, guess_content_type, MD5Error, bytes_print
+    uni_print, guess_content_type, MD5Error, bytes_print, set_file_utime
 
 
 LOGGER = logging.getLogger(__name__)
@@ -68,7 +68,7 @@ def save_file(filename, response_data, last_update, is_stream=False):
     if not is_stream:
         last_update_tuple = last_update.timetuple()
         mod_timestamp = time.mktime(last_update_tuple)
-        os.utime(filename, (int(mod_timestamp), int(mod_timestamp)))
+        set_file_utime(filename, int(mod_timestamp))
     else:
         # Now write the output to stdout since the md5 is correct.
         bytes_print(payload)
diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py
--- a/awscli/customizations/s3/utils.py
+++ b/awscli/customizations/s3/utils.py
@@ -15,6 +15,7 @@
 import mimetypes
 import hashlib
 import math
+import errno
 import os
 import sys
 from collections import namedtuple, deque
@@ -353,6 +354,30 @@ def relative_path(filename, start=os.path.curdir):
         return os.path.abspath(filename)
 
 
+def set_file_utime(filename, desired_time):
+    """
+    Set the utime of a file, and if it fails, raise a more explicit error.
+
+    :param filename: the file to modify
+    :param desired_time: the epoch timestamp to set for atime and mtime.
+    :raises: SetFileUtimeError: if you do not have permission (errno 1)
+    :raises: OSError: for all errors other than errno 1
+    """
+    try:
+        os.utime(filename, (desired_time, desired_time))
+    except OSError as e:
+        # Only raise a more explicit exception when it is a permission issue.
+        if e.errno != errno.EPERM:
+            raise e
+        raise SetFileUtimeError(
+            ("The file was downloaded, but attempting to modify the "
+             "utime of the file failed. Is the file owned by another user?"))
+
+
+class SetFileUtimeError(Exception):
+    pass
+
+
 class ReadFileChunk(object):
     def __init__(self, filename, start_byte, size):
         self._filename = filename


Test patch:
diff --git a/tests/functional/s3/test_cp_command.py b/tests/functional/s3/test_cp_command.py
--- a/tests/functional/s3/test_cp_command.py
+++ b/tests/functional/s3/test_cp_command.py
@@ -175,3 +175,15 @@ def test_cp_succeeds_with_mimetype_errors(self):
         # Because of the decoding error the command should have succeeded
         # just that there was no content type added.
         self.assertNotIn('ContentType', self.last_kwargs)
+
+    def test_cp_fails_with_utime_errors_but_continues(self):
+        full_path = self.files.create_file('foo.txt', '')
+        cmdline = '%s s3://bucket/key.txt %s' % (self.prefix, full_path)
+        self.parsed_responses = [
+            {"ContentLength": "100", "LastModified": "00:00:00Z"},
+            {'ETag': '"foo-1"', 'Body': six.BytesIO(b'foo')}
+        ]
+        with mock.patch('os.utime') as mock_utime:
+            mock_utime.side_effect = OSError(1, '')
+            _, err, _ = self.run_cmd(cmdline, expected_rc=1)
+            self.assertIn('attempting to modify the utime', err)
diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py
--- a/tests/unit/customizations/s3/test_utils.py
+++ b/tests/unit/customizations/s3/test_utils.py
@@ -24,6 +24,7 @@
 from awscli.customizations.s3.utils import human_readable_size
 from awscli.customizations.s3.utils import human_readable_to_bytes
 from awscli.customizations.s3.utils import MAX_SINGLE_UPLOAD_SIZE
+from awscli.customizations.s3.utils import set_file_utime, SetFileUtimeError
 
 
 def test_human_readable_size():
@@ -382,5 +383,32 @@ def test_get_file_stat_error_message(self):
                     get_file_stat('myfilename.txt')
 
 
+class TestSetsFileUtime(unittest.TestCase):
+
+    def test_successfully_sets_utime(self):
+        now = datetime.datetime.now(tzlocal())
+        epoch_now = time.mktime(now.timetuple())
+        with temporary_file('w') as f:
+            set_file_utime(f.name, epoch_now)
+            _, update_time = get_file_stat(f.name)
+            self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
+
+    def test_throws_more_relevant_error_when_errno_1(self):
+        now = datetime.datetime.now(tzlocal())
+        epoch_now = time.mktime(now.timetuple())
+        with mock.patch('os.utime') as utime_mock:
+            utime_mock.side_effect = OSError(1, '')
+            with self.assertRaises(SetFileUtimeError):
+                set_file_utime('not_real_file', epoch_now)
+
+    def test_passes_through_other_os_errors(self):
+        now = datetime.datetime.now(tzlocal())
+        epoch_now = time.mktime(now.timetuple())
+        with mock.patch('os.utime') as utime_mock:
+            utime_mock.side_effect = OSError(2, '')
+            with self.assertRaises(OSError):
+                set_file_utime('not_real_file', epoch_now)
+
+
 if __name__ == "__main__":
     unittest.main()