Repository: jschneier/django-storages

Issue:
'SpooledTemporaryFile' object has no attribute 'readinto'
Hi there,

after upgrading django-storages from v1.10.1 to v1.11 I've got the following error:

```
AttributeError: 'SpooledTemporaryFile' object has no attribute 'readinto'
  File "core/tasks.py", line 349, in issues_progress_report
    url = svc.upload(report)
  File "core/services/issues_progress.py", line 356, in upload
    name = self.storage.save(f'issues-progress/{org}/{file.name}', file)
  File "django/core/files/storage.py", line 52, in save
    return self._save(name, content)
  File "storages/backends/s3boto3.py", line 446, in _save
    obj.upload_fileobj(reader, ExtraArgs=params)
  File "boto3/s3/inject.py", line 619, in object_upload_fileobj
    return self.meta.client.upload_fileobj(
  File "boto3/s3/inject.py", line 539, in upload_fileobj
    return future.result()
  File "s3transfer/futures.py", line 106, in result
    return self._coordinator.result()
  File "s3transfer/futures.py", line 265, in result
    raise self._exception
  File "s3transfer/tasks.py", line 126, in __call__
    return self._execute_main(kwargs)
  File "s3transfer/tasks.py", line 150, in _execute_main
    return_value = self._main(**kwargs)
  File "s3transfer/upload.py", line 692, in _main
    client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args)
  File "botocore/client.py", line 357, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "botocore/client.py", line 652, in _make_api_call
    handler, event_response = self.meta.events.emit_until_response(
  File "botocore/hooks.py", line 360, in emit_until_response
    return self._emitter.emit_until_response(aliased_event_name, **kwargs)
  File "botocore/hooks.py", line 243, in emit_until_response
    responses = self._emit(event_name, kwargs, stop_on_response=True)
  File "botocore/hooks.py", line 211, in _emit
    response = handler(**kwargs)
  File "botocore/utils.py", line 2219, in conditionally_calculate_md5
    md5_digest = calculate_md5(body, **kwargs)
  File "botocore/utils.py", line 2196, in calculate_md5
    binary_md5 = _calculate_md5_from_file(body)
  File "botocore/utils.py", line 2208, in _calculate_md5_from_file
    for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
  File "botocore/utils.py", line 2208, in <lambda>
    for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
  File "s3transfer/utils.py", line 480, in read
    data = self._fileobj.read(amount_to_read)
  File "s3transfer/upload.py", line 86, in read
    return self._fileobj.read(amount)
  File "django/core/files/utils.py", line 17, in <lambda>
    readinto = property(lambda self: self.file.readinto)
```

I guess it may be caused by [this commit](https://github.com/jschneier/django-storages/pull/955/files). Reverting django-storages back to v1.10.1 fixed the issue. Other package versions:
- python==3.9.0
- Django==3.1.4
- boto3==1.16.38
- botocore==1.19.38

if I can provide anything else that may help - please let me know. Thanks!


Discussion:
Ugh, thanks. I need to invest in an integration test.

> On Dec 17, 2020, at 15:57, Denis Stebunov <notifications@github.com> wrote:
> 
> 
> Hi there,
> 
> after upgrading django-storages from v1.10.1 to v1.11 I've got the following error:
> 
> AttributeError: 'SpooledTemporaryFile' object has no attribute 'readinto'
>   File "core/tasks.py", line 349, in issues_progress_report
>     url = svc.upload(report)
>   File "core/services/issues_progress.py", line 356, in upload
>     name = self.storage.save(f'issues-progress/{org}/{file.name}', file)
>   File "django/core/files/storage.py", line 52, in save
>     return self._save(name, content)
>   File "storages/backends/s3boto3.py", line 446, in _save
>     obj.upload_fileobj(reader, ExtraArgs=params)
>   File "boto3/s3/inject.py", line 619, in object_upload_fileobj
>     return self.meta.client.upload_fileobj(
>   File "boto3/s3/inject.py", line 539, in upload_fileobj
>     return future.result()
>   File "s3transfer/futures.py", line 106, in result
>     return self._coordinator.result()
>   File "s3transfer/futures.py", line 265, in result
>     raise self._exception
>   File "s3transfer/tasks.py", line 126, in __call__
>     return self._execute_main(kwargs)
>   File "s3transfer/tasks.py", line 150, in _execute_main
>     return_value = self._main(**kwargs)
>   File "s3transfer/upload.py", line 692, in _main
>     client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args)
>   File "botocore/client.py", line 357, in _api_call
>     return self._make_api_call(operation_name, kwargs)
>   File "botocore/client.py", line 652, in _make_api_call
>     handler, event_response = self.meta.events.emit_until_response(
>   File "botocore/hooks.py", line 360, in emit_until_response
>     return self._emitter.emit_until_response(aliased_event_name, **kwargs)
>   File "botocore/hooks.py", line 243, in emit_until_response
>     responses = self._emit(event_name, kwargs, stop_on_response=True)
>   File "botocore/hooks.py", line 211, in _emit
>     response = handler(**kwargs)
>   File "botocore/utils.py", line 2219, in conditionally_calculate_md5
>     md5_digest = calculate_md5(body, **kwargs)
>   File "botocore/utils.py", line 2196, in calculate_md5
>     binary_md5 = _calculate_md5_from_file(body)
>   File "botocore/utils.py", line 2208, in _calculate_md5_from_file
>     for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
>   File "botocore/utils.py", line 2208, in <lambda>
>     for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
>   File "s3transfer/utils.py", line 480, in read
>     data = self._fileobj.read(amount_to_read)
>   File "s3transfer/upload.py", line 86, in read
>     return self._fileobj.read(amount)
>   File "django/core/files/utils.py", line 17, in <lambda>
>     readinto = property(lambda self: self.file.readinto)
> I guess it may be caused by this commit <https://github.com/jschneier/django-storages/pull/955/files>. Reverting django-storages back to v1.10.1 fixed the issue. Other package versions:
> 
> python==3.9.0
> Django==3.1.4
> boto3==1.16.38
> botocore==1.19.38
> if I can provide anything else that may help - please let me know. Thanks!
> 
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub <https://github.com/jschneier/django-storages/issues/967>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AAREDWAE2M6HEQOPRPPEUSLSVJWDBANCNFSM4VAHD7CA>.
> 


What is `file`? I'm having problems producing the same error message. I am on a different Python version but same boto3 & botocore.

Resolving patch:
diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py
--- a/storages/backends/s3boto3.py
+++ b/storages/backends/s3boto3.py
@@ -2,12 +2,14 @@
 import mimetypes
 import os
 import posixpath
+import tempfile
 import threading
 from datetime import datetime, timedelta
 from gzip import GzipFile
 from tempfile import SpooledTemporaryFile
 from urllib.parse import parse_qsl, urlsplit
 
+from django.contrib.staticfiles.storage import ManifestFilesMixin
 from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
 from django.core.files.base import File
 from django.utils.deconstruct import deconstructible
@@ -16,8 +18,8 @@
 
 from storages.base import BaseStorage
 from storages.utils import (
-    NonCloseableBufferedReader, check_location, get_available_overwrite_name,
-    lookup_env, safe_join, setting, to_bytes,
+    check_location, get_available_overwrite_name, lookup_env, safe_join,
+    setting, to_bytes,
 )
 
 try:
@@ -442,8 +444,7 @@ def _save(self, name, content):
 
         obj = self.bucket.Object(name)
         content.seek(0, os.SEEK_SET)
-        with NonCloseableBufferedReader(content) as reader:
-            obj.upload_fileobj(reader, ExtraArgs=params)
+        obj.upload_fileobj(content, ExtraArgs=params)
         return cleaned_name
 
     def delete(self, name):
@@ -582,3 +583,22 @@ def get_available_name(self, name, max_length=None):
         if self.file_overwrite:
             return get_available_overwrite_name(name, max_length)
         return super().get_available_name(name, max_length)
+
+
+class S3StaticStorage(S3Boto3Storage):
+    """Querystring auth must be disabled so that url() returns a consistent output."""
+    querystring_auth = False
+
+
+class S3ManifestStaticStorage(ManifestFilesMixin, S3StaticStorage):
+    """Copy the file before saving for compatibility with ManifestFilesMixin
+    which does not play nicely with boto3 automatically closing the file.
+
+    See: https://github.com/boto/s3transfer/issues/80#issuecomment-562356142
+    """
+
+    def _save(self, name, content):
+        content.seek(0)
+        with tempfile.SpooledTemporaryFile() as tmp:
+            tmp.write(content.read())
+            return super()._save(name, tmp)
diff --git a/storages/utils.py b/storages/utils.py
--- a/storages/utils.py
+++ b/storages/utils.py
@@ -1,4 +1,3 @@
-import io
 import os
 import posixpath
 
@@ -127,10 +126,3 @@ def get_available_overwrite_name(name, max_length):
             'allows sufficient "max_length".' % name
         )
     return os.path.join(dir_name, "{}{}".format(file_root, file_ext))
-
-
-# A buffered reader that does not actually close, workaround for
-# https://github.com/boto/s3transfer/issues/80#issuecomment-562356142
-class NonCloseableBufferedReader(io.BufferedReader):
-    def close(self):
-        self.flush()


Test patch:
diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py
--- a/tests/test_s3boto3.py
+++ b/tests/test_s3boto3.py
@@ -22,12 +22,6 @@ def setUp(self):
         self.storage._connections.connection = mock.MagicMock()
 
 
-def assert_called_upload(mock_uploadobj, content, extra):
-    assert mock_uploadobj.call_count == 1
-    assert mock_uploadobj.call_args[0][0].raw == content
-    assert mock_uploadobj.call_args[1] == extra
-
-
 class S3Boto3StorageTests(S3Boto3TestCase):
 
     def test_clean_name(self):
@@ -113,12 +107,12 @@ def test_storage_save(self):
         self.storage.bucket.Object.assert_called_once_with(name)
 
         obj = self.storage.bucket.Object.return_value
-        extra = {
-            'ExtraArgs': {
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
                 'ContentType': 'text/plain',
             }
-        }
-        assert_called_upload(obj.upload_fileobj, content, extra)
+        )
 
     def test_storage_save_with_default_acl(self):
         """
@@ -131,13 +125,13 @@ def test_storage_save_with_default_acl(self):
         self.storage.bucket.Object.assert_called_once_with(name)
 
         obj = self.storage.bucket.Object.return_value
-        extra = {
-            'ExtraArgs': {
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
                 'ContentType': 'text/plain',
                 'ACL': 'private',
             }
-        }
-        assert_called_upload(obj.upload_fileobj, content, extra)
+        )
 
     def test_storage_object_parameters_not_overwritten_by_default(self):
         """
@@ -151,13 +145,13 @@ def test_storage_object_parameters_not_overwritten_by_default(self):
         self.storage.bucket.Object.assert_called_once_with(name)
 
         obj = self.storage.bucket.Object.return_value
-        extra = {
-            'ExtraArgs': {
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
                 'ContentType': 'text/plain',
                 'ACL': 'private',
             }
-        }
-        assert_called_upload(obj.upload_fileobj, content, extra)
+        )
 
     def test_content_type(self):
         """
@@ -170,12 +164,12 @@ def test_content_type(self):
         self.storage.bucket.Object.assert_called_once_with(name)
 
         obj = self.storage.bucket.Object.return_value
-        extra = {
-            'ExtraArgs': {
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
                 'ContentType': 'image/jpeg',
             }
-        }
-        assert_called_upload(obj.upload_fileobj, content, extra)
+        )
 
     def test_storage_save_gzipped(self):
         """
@@ -185,13 +179,13 @@ def test_storage_save_gzipped(self):
         content = ContentFile("I am gzip'd")
         self.storage.save(name, content)
         obj = self.storage.bucket.Object.return_value
-        extra = {
-            'ExtraArgs': {
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
                 'ContentType': 'application/octet-stream',
                 'ContentEncoding': 'gzip',
             }
-        }
-        assert_called_upload(obj.upload_fileobj, content, extra)
+        )
 
     def test_storage_save_gzip(self):
         """