Repository: getmoto/moto

Issue:
head-object for specific part returns ContentLength for the whole object
If an object is uploaded as a multipart, head-object with `PartNumber=1` set should return ContentLength for that part and not for the whole object. Here is a test that shows that with regular s3 ContentLength for a particular part is correct and with moto it is equal to the size of the whole object.

```
import uuid             
import boto3            
from moto import mock_s3


BUCKET = "dvc-test"                                                            
                                                                               
                                                                               
def _upload_multipart(s3, Bucket, Key):                                        
    mpu = s3.create_multipart_upload(Bucket=Bucket, Key=Key)                   
    mpu_id = mpu["UploadId"]                                                   
                                                                               
    parts = []                                                                 
    n_parts = 10                                                             
    for i in range(1, n_parts + 1):                                            
        # NOTE: Generation parts of variable size. Part size should be at      
        # least 5MB:                                                           
        # https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
        part_size = 5 * 1024 * 1024                                            
        body = str(i) * part_size                                              
        part = s3.upload_part(                                                 
            Bucket=Bucket,                                                     
            Key=Key,                                                           
            PartNumber=i,                                                      
            UploadId=mpu_id,                                                   
            Body=body,                                                         
            ContentLength=len(body),                                           
        )                                                                      
        parts.append({"PartNumber": i, "ETag": part["ETag"]})                  
                                                                               
    s3.complete_multipart_upload(                                              
        Bucket=Bucket,                                                         
        Key=Key,                                                               
        UploadId=mpu_id,                                                       
        MultipartUpload={"Parts": parts},                                      
    )                                                                          
                                                                               
    for i in range(1, n_parts + 1):                                            
        obj = s3.head_object(Bucket=Bucket, Key=Key, PartNumber=i)             
        assert obj["ContentLength"] == part_size                               
                                                                               
                                                                               
def _test():                                                                   
    session = boto3.session.Session(region_name="us-east-2")                   
    s3 = session.client("s3")                                                  
    Key = str(uuid.uuid4)                                                      
    _upload_multipart(s3, BUCKET, Key)                                         
                                                                               
                                                                               
def test_orig():                                                               
    _test()                                                                    
                                                                               
                                                                               
@mock_s3                                                                       
def test_moto():                                                               
    s3 = boto3.resource("s3")                                                  
    s3.create_bucket(Bucket=BUCKET)                                            
    _test()                                                                    
```

which, when ran with `pytest`, raises:

```
_________________________________________________ test_moto ________________________________________[14/1943]

    @mock_s3
    def test_moto():
        s3 = boto3.resource('s3')
        s3.create_bucket(Bucket=BUCKET)
>       _test()

test_moto.py:59:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
test_moto.py:48: in _test
    _upload_multipart(s3, BUCKET, Key)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

s3 = <botocore.client.S3 object at 0x7f94f4b21b38>, Bucket = 'dvc-test'
Key = '<function uuid4 at 0x7f94fe139268>'

    def _upload_multipart(s3, Bucket, Key):
        mpu = s3.create_multipart_upload(Bucket=Bucket, Key=Key)
        mpu_id = mpu["UploadId"]
    
        parts = []
        n_parts = 10
        for i in range(1, n_parts + 1):
            # NOTE: Generation parts of variable size. Part size should be at
            # least 5MB:
            # https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
            part_size = 5*1024*1024
            body = str(i)*part_size
            part = s3.upload_part(
                Bucket=Bucket,
                Key=Key,
                PartNumber=i,
                UploadId=mpu_id,
                Body=body,
                ContentLength=len(body),
    
            )
            parts.append({"PartNumber": i, "ETag": part["ETag"]})
    
        s3.complete_multipart_upload(
            Bucket=Bucket,
            Key=Key,
            UploadId=mpu_id,
            MultipartUpload={"Parts": parts},
        )
    
        for i in range(1, n_parts + 1):
            obj = s3.head_object(Bucket=Bucket, Key=Key, PartNumber=i)
>           assert obj["ContentLength"] == part_size
E           assert 57671680 == 5242880

test_moto.py:41: AssertionError
```

I've scanned through the moto's code, but wasn't able to find the place where this happens. If you could direct me where to look, I will be happy to submit a PR to fix this.

Additional info:

```
(3.7.0-dvc) ➜  dvc git:(fix-1410) ✗ python --version
Python 3.7.0
(3.7.0-dvc) ➜  dvc git:(fix-1410) ✗ pip freeze | grep boto
You are using pip version 10.0.1, however version 19.0.3 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
boto==2.49.0
boto3==1.9.115
botocore==1.12.117
(3.7.0-dvc) ➜  dvc git:(fix-1410) ✗ pip freeze | grep moto
You are using pip version 10.0.1, however version 19.0.3 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
moto==1.3.7
```

Thanks for creating such an amazing project! :slightly_smiling_face: 



Discussion:
Thanks for opening.

The code we would need to change is here: https://github.com/spulec/moto/blob/master/moto/s3/responses.py#L809-L828

It looks like we aren't doing anything with part numbers right now.

Resolving patch:
diff --git a/moto/s3/models.py b/moto/s3/models.py
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@@ -52,8 +52,17 @@ def version_id(self):
 
 class FakeKey(BaseModel):
 
-    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0,
-                 max_buffer_size=DEFAULT_KEY_BUFFER_SIZE):
+    def __init__(
+        self,
+        name,
+        value,
+        storage="STANDARD",
+        etag=None,
+        is_versioned=False,
+        version_id=0,
+        max_buffer_size=DEFAULT_KEY_BUFFER_SIZE,
+        multipart=None
+    ):
         self.name = name
         self.last_modified = datetime.datetime.utcnow()
         self.acl = get_canned_acl('private')
@@ -65,6 +74,7 @@ def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=Fals
         self._version_id = version_id
         self._is_versioned = is_versioned
         self._tagging = FakeTagging()
+        self.multipart = multipart
 
         self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size)
         self._max_buffer_size = max_buffer_size
@@ -782,7 +792,15 @@ def get_bucket_website_configuration(self, bucket_name):
         bucket = self.get_bucket(bucket_name)
         return bucket.website_configuration
 
-    def set_key(self, bucket_name, key_name, value, storage=None, etag=None):
+    def set_key(
+        self,
+        bucket_name,
+        key_name,
+        value,
+        storage=None,
+        etag=None,
+        multipart=None,
+    ):
         key_name = clean_key_name(key_name)
         if storage is not None and storage not in STORAGE_CLASS:
             raise InvalidStorageClass(storage=storage)
@@ -795,7 +813,9 @@ def set_key(self, bucket_name, key_name, value, storage=None, etag=None):
             storage=storage,
             etag=etag,
             is_versioned=bucket.is_versioned,
-            version_id=str(uuid.uuid4()) if bucket.is_versioned else None)
+            version_id=str(uuid.uuid4()) if bucket.is_versioned else None,
+            multipart=multipart,
+        )
 
         keys = [
             key for key in bucket.keys.getlist(key_name, [])
@@ -812,7 +832,7 @@ def append_to_key(self, bucket_name, key_name, value):
         key.append_to_value(value)
         return key
 
-    def get_key(self, bucket_name, key_name, version_id=None):
+    def get_key(self, bucket_name, key_name, version_id=None, part_number=None):
         key_name = clean_key_name(key_name)
         bucket = self.get_bucket(bucket_name)
         key = None
@@ -827,6 +847,9 @@ def get_key(self, bucket_name, key_name, version_id=None):
                         key = key_version
                         break
 
+            if part_number and key.multipart:
+                key = key.multipart.parts[part_number]
+
         if isinstance(key, FakeKey):
             return key
         else:
@@ -890,7 +913,12 @@ def complete_multipart(self, bucket_name, multipart_id, body):
             return
         del bucket.multiparts[multipart_id]
 
-        key = self.set_key(bucket_name, multipart.key_name, value, etag=etag)
+        key = self.set_key(
+            bucket_name,
+            multipart.key_name,
+            value, etag=etag,
+            multipart=multipart
+        )
         key.set_metadata(multipart.metadata)
         return key
 
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -809,13 +809,20 @@ def _key_response_put(self, request, body, bucket_name, query, key_name, headers
     def _key_response_head(self, bucket_name, query, key_name, headers):
         response_headers = {}
         version_id = query.get('versionId', [None])[0]
+        part_number = query.get('partNumber', [None])[0]
+        if part_number:
+            part_number = int(part_number)
 
         if_modified_since = headers.get('If-Modified-Since', None)
         if if_modified_since:
             if_modified_since = str_to_rfc_1123_datetime(if_modified_since)
 
         key = self.backend.get_key(
-            bucket_name, key_name, version_id=version_id)
+            bucket_name,
+            key_name,
+            version_id=version_id,
+            part_number=part_number
+        )
         if key:
             response_headers.update(key.metadata)
             response_headers.update(key.response_dict)


Test patch:
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -1671,6 +1671,42 @@ def test_boto3_multipart_etag():
     resp['ETag'].should.equal(EXPECTED_ETAG)
 
 
+@mock_s3
+@reduced_min_part_size
+def test_boto3_multipart_part_size():
+    s3 = boto3.client('s3', region_name='us-east-1')
+    s3.create_bucket(Bucket='mybucket')
+
+    mpu = s3.create_multipart_upload(Bucket='mybucket', Key='the-key')
+    mpu_id = mpu["UploadId"]
+
+    parts = []
+    n_parts = 10
+    for i in range(1, n_parts + 1):
+        part_size = 5 * 1024 * 1024
+        body = b'1' * part_size
+        part = s3.upload_part(
+            Bucket='mybucket',
+            Key='the-key',
+            PartNumber=i,
+            UploadId=mpu_id,
+            Body=body,
+            ContentLength=len(body),
+        )
+        parts.append({"PartNumber": i, "ETag": part["ETag"]})
+
+    s3.complete_multipart_upload(
+        Bucket='mybucket',
+        Key='the-key',
+        UploadId=mpu_id,
+        MultipartUpload={"Parts": parts},
+    )
+
+    for i in range(1, n_parts + 1):
+        obj = s3.head_object(Bucket='mybucket', Key='the-key', PartNumber=i)
+        assert obj["ContentLength"] == part_size
+
+
 @mock_s3
 def test_boto3_put_object_with_tagging():
     s3 = boto3.client('s3', region_name='us-east-1')