Repository: getmoto/moto

Issue:
moto.s3 cannot be used to test large (or many) file uploads
moto.s3 holds the data of all keys in memory, which can fill up quickly. It would be better if they were stored in temporary files (tempfile.SpooledTemporaryFile would probably be the best option).



Discussion:

Resolving patch:
diff --git a/moto/s3/models.py b/moto/s3/models.py
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@@ -8,6 +8,8 @@
 import codecs
 import random
 import string
+import tempfile
+import sys
 
 import six
 
@@ -21,6 +23,8 @@
 UPLOAD_ID_BYTES = 43
 UPLOAD_PART_MIN_SIZE = 5242880
 STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"]
+DEFAULT_KEY_BUFFER_SIZE = 16 * 1024 * 1024
+DEFAULT_TEXT_ENCODING = sys.getdefaultencoding()
 
 
 class FakeDeleteMarker(BaseModel):
@@ -42,9 +46,9 @@ def version_id(self):
 
 class FakeKey(BaseModel):
 
-    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0):
+    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0,
+                 max_buffer_size=DEFAULT_KEY_BUFFER_SIZE):
         self.name = name
-        self.value = value
         self.last_modified = datetime.datetime.utcnow()
         self.acl = get_canned_acl('private')
         self.website_redirect_location = None
@@ -56,10 +60,30 @@ def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=Fals
         self._is_versioned = is_versioned
         self._tagging = FakeTagging()
 
+        self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size)
+        self._max_buffer_size = max_buffer_size
+        self.value = value
+
     @property
     def version_id(self):
         return self._version_id
 
+    @property
+    def value(self):
+        self._value_buffer.seek(0)
+        return self._value_buffer.read()
+
+    @value.setter
+    def value(self, new_value):
+        self._value_buffer.seek(0)
+        self._value_buffer.truncate()
+
+        # Hack for working around moto's own unit tests; this probably won't
+        # actually get hit in normal use.
+        if isinstance(new_value, six.text_type):
+            new_value = new_value.encode(DEFAULT_TEXT_ENCODING)
+        self._value_buffer.write(new_value)
+
     def copy(self, new_name=None):
         r = copy.deepcopy(self)
         if new_name is not None:
@@ -83,7 +107,9 @@ def set_acl(self, acl):
         self.acl = acl
 
     def append_to_value(self, value):
-        self.value += value
+        self._value_buffer.seek(0, os.SEEK_END)
+        self._value_buffer.write(value)
+
         self.last_modified = datetime.datetime.utcnow()
         self._etag = None  # must recalculate etag
         if self._is_versioned:
@@ -101,11 +127,13 @@ def increment_version(self):
     def etag(self):
         if self._etag is None:
             value_md5 = hashlib.md5()
-            if isinstance(self.value, six.text_type):
-                value = self.value.encode("utf-8")
-            else:
-                value = self.value
-            value_md5.update(value)
+            self._value_buffer.seek(0)
+            while True:
+                block = self._value_buffer.read(DEFAULT_KEY_BUFFER_SIZE)
+                if not block:
+                    break
+                value_md5.update(block)
+
             self._etag = value_md5.hexdigest()
         return '"{0}"'.format(self._etag)
 
@@ -132,7 +160,7 @@ def response_dict(self):
         res = {
             'ETag': self.etag,
             'last-modified': self.last_modified_RFC1123,
-            'content-length': str(len(self.value)),
+            'content-length': str(self.size),
         }
         if self._storage_class != 'STANDARD':
             res['x-amz-storage-class'] = self._storage_class
@@ -150,7 +178,8 @@ def response_dict(self):
 
     @property
     def size(self):
-        return len(self.value)
+        self._value_buffer.seek(0, os.SEEK_END)
+        return self._value_buffer.tell()
 
     @property
     def storage_class(self):
@@ -161,6 +190,26 @@ def expiry_date(self):
         if self._expiry is not None:
             return self._expiry.strftime("%a, %d %b %Y %H:%M:%S GMT")
 
+    # Keys need to be pickleable due to some implementation details of boto3.
+    # Since file objects aren't pickleable, we need to override the default
+    # behavior. The following is adapted from the Python docs:
+    # https://docs.python.org/3/library/pickle.html#handling-stateful-objects
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state['value'] = self.value
+        del state['_value_buffer']
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__.update({
+            k: v for k, v in six.iteritems(state)
+            if k != 'value'
+        })
+
+        self._value_buffer = \
+            tempfile.SpooledTemporaryFile(max_size=self._max_buffer_size)
+        self.value = state['value']
+
 
 class FakeMultipart(BaseModel):
 


Test patch:
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -8,6 +8,7 @@
 from gzip import GzipFile
 from io import BytesIO
 import zlib
+import pickle
 
 import json
 import boto
@@ -65,6 +66,50 @@ def save(self):
         s3.put_object(Bucket='mybucket', Key=self.name, Body=self.value)
 
 
+@mock_s3
+def test_keys_are_pickleable():
+    """Keys must be pickleable due to boto3 implementation details."""
+    key = s3model.FakeKey('name', b'data!')
+    assert key.value == b'data!'
+
+    pickled = pickle.dumps(key)
+    loaded = pickle.loads(pickled)
+    assert loaded.value == key.value
+
+
+@mock_s3
+def test_append_to_value__basic():
+    key = s3model.FakeKey('name', b'data!')
+    assert key.value == b'data!'
+    assert key.size == 5
+
+    key.append_to_value(b' And even more data')
+    assert key.value == b'data! And even more data'
+    assert key.size == 24
+
+
+@mock_s3
+def test_append_to_value__nothing_added():
+    key = s3model.FakeKey('name', b'data!')
+    assert key.value == b'data!'
+    assert key.size == 5
+
+    key.append_to_value(b'')
+    assert key.value == b'data!'
+    assert key.size == 5
+
+
+@mock_s3
+def test_append_to_value__empty_key():
+    key = s3model.FakeKey('name', b'')
+    assert key.value == b''
+    assert key.size == 0
+
+    key.append_to_value(b'stuff')
+    assert key.value == b'stuff'
+    assert key.size == 5
+
+
 @mock_s3
 def test_my_model_save():
     # Create Bucket so that test can run