Repository: fossasia/open-event-server

Issue:
failure when saving images to aws bucket storage
**Describe the bug**

When setting up storage to aws, images do not get uploaded to bucket (they are lost after server reastart).

**To Reproduce**

Steps to reproduce the behavior:

1. Install on Heroku (manual install) from the  1.9.0 release
2. Go to admin setup, configure storage to a new empty AWS bucket
3. Login as a normal user, go to profile.
4. upload profile picture.
5.  restart the dyno on heroku.
6. login again as user - profile image is lost.

**Stacktrace**

Following is the traceback from heroku log.
It appears that the helper tries to read the bytestream of the image to memory as a string, and fails converting to unicode.
PR will follow...

```
2019-12-31T06:13:05.398139+00:00 app[web.1]: Exception on /v1/upload/image [POST]
2019-12-31T06:13:05.398157+00:00 app[web.1]: Traceback (most recent call last):
2019-12-31T06:13:05.398159+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
2019-12-31T06:13:05.398160+00:00 app[web.1]: response = self.full_dispatch_request()
2019-12-31T06:13:05.398162+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1951, in full_dispatch_request
2019-12-31T06:13:05.398163+00:00 app[web.1]: rv = self.handle_user_exception(e)
2019-12-31T06:13:05.398165+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2019-12-31T06:13:05.398166+00:00 app[web.1]: return cors_after_request(app.make_response(f(*args, **kwargs)))
2019-12-31T06:13:05.398168+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2019-12-31T06:13:05.398169+00:00 app[web.1]: return cors_after_request(app.make_response(f(*args, **kwargs)))
2019-12-31T06:13:05.398171+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1820, in handle_user_exception
2019-12-31T06:13:05.398172+00:00 app[web.1]: reraise(exc_type, exc_value, tb)
2019-12-31T06:13:05.398173+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
2019-12-31T06:13:05.398175+00:00 app[web.1]: raise value
2019-12-31T06:13:05.398176+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
2019-12-31T06:13:05.398178+00:00 app[web.1]: rv = self.dispatch_request()
2019-12-31T06:13:05.398179+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
2019-12-31T06:13:05.398181+00:00 app[web.1]: return self.view_functions[rule.endpoint](**req.view_args)
2019-12-31T06:13:05.398182+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_jwt_extended/view_decorators.py", line 108, in wrapper
2019-12-31T06:13:05.398184+00:00 app[web.1]: return fn(*args, **kwargs)
2019-12-31T06:13:05.398185+00:00 app[web.1]: File "/app/app/api/uploads.py", line 26, in upload_image
2019-12-31T06:13:05.398187+00:00 app[web.1]: UPLOAD_PATHS['temp']['image'].format(uuid=uuid.uuid4())
2019-12-31T06:13:05.398188+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 154, in upload
2019-12-31T06:13:05.398189+00:00 app[web.1]: return upload_to_aws(aws_bucket_name, aws_region, aws_key, aws_secret, uploaded_file, key, **kwargs)
2019-12-31T06:13:05.398191+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 227, in upload_to_aws
2019-12-31T06:13:05.398193+00:00 app[web.1]: file_data = file.read()
2019-12-31T06:13:05.398194+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 108, in read
2019-12-31T06:13:05.398195+00:00 app[web.1]: return self.file.read()
2019-12-31T06:13:05.398197+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/codecs.py", line 322, in decode
2019-12-31T06:13:05.398198+00:00 app[web.1]: (result, consumed) = self._buffer_decode(data, self.errors, final)
2019-12-31T06:13:05.398200+00:00 app[web.1]: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte
2019-12-31T06:13:05.398347+00:00 app[web.1]: Exception on /v1/upload/image [POST]
2019-12-31T06:13:05.398349+00:00 app[web.1]: Traceback (most recent call last):
2019-12-31T06:13:05.398351+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
2019-12-31T06:13:05.398353+00:00 app[web.1]: response = self.full_dispatch_request()
2019-12-31T06:13:05.398355+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1951, in full_dispatch_request
2019-12-31T06:13:05.398356+00:00 app[web.1]: rv = self.handle_user_exception(e)
2019-12-31T06:13:05.398358+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2019-12-31T06:13:05.398359+00:00 app[web.1]: return cors_after_request(app.make_response(f(*args, **kwargs)))
2019-12-31T06:13:05.398361+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2019-12-31T06:13:05.398362+00:00 app[web.1]: return cors_after_request(app.make_response(f(*args, **kwargs)))
2019-12-31T06:13:05.398364+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1820, in handle_user_exception
2019-12-31T06:13:05.398365+00:00 app[web.1]: reraise(exc_type, exc_value, tb)
2019-12-31T06:13:05.398367+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
2019-12-31T06:13:05.398368+00:00 app[web.1]: raise value
2019-12-31T06:13:05.398369+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
2019-12-31T06:13:05.398371+00:00 app[web.1]: rv = self.dispatch_request()
2019-12-31T06:13:05.398372+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
2019-12-31T06:13:05.398374+00:00 app[web.1]: return self.view_functions[rule.endpoint](**req.view_args)
2019-12-31T06:13:05.398375+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_jwt_extended/view_decorators.py", line 108, in wrapper
2019-12-31T06:13:05.398377+00:00 app[web.1]: return fn(*args, **kwargs)
2019-12-31T06:13:05.398378+00:00 app[web.1]: File "/app/app/api/uploads.py", line 26, in upload_image
2019-12-31T06:13:05.398379+00:00 app[web.1]: UPLOAD_PATHS['temp']['image'].format(uuid=uuid.uuid4())
2019-12-31T06:13:05.398381+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 154, in upload
2019-12-31T06:13:05.398382+00:00 app[web.1]: return upload_to_aws(aws_bucket_name, aws_region, aws_key, aws_secret, uploaded_file, key, **kwargs)
2019-12-31T06:13:05.398384+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 227, in upload_to_aws
2019-12-31T06:13:05.398385+00:00 app[web.1]: file_data = file.read()
2019-12-31T06:13:05.398386+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 108, in read
2019-12-31T06:13:05.398388+00:00 app[web.1]: return self.file.read()
2019-12-31T06:13:05.398389+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/codecs.py", line 322, in decode
2019-12-31T06:13:05.398391+00:00 app[web.1]: (result, consumed) = self._buffer_decode(data, self.errors, final)
2019-12-31T06:13:05.398392+00:00 app[web.1]: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte
2019-12-31T06:13:05.399330+00:00 app[web.1]: ERROR:app:Exception on /v1/upload/image [POST]
2019-12-31T06:13:05.399331+00:00 app[web.1]: Traceback (most recent call last):
2019-12-31T06:13:05.399333+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
2019-12-31T06:13:05.399334+00:00 app[web.1]: response = self.full_dispatch_request()
2019-12-31T06:13:05.399336+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1951, in full_dispatch_request
2019-12-31T06:13:05.399337+00:00 app[web.1]: rv = self.handle_user_exception(e)
2019-12-31T06:13:05.399339+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2019-12-31T06:13:05.399340+00:00 app[web.1]: return cors_after_request(app.make_response(f(*args, **kwargs)))
2019-12-31T06:13:05.399342+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_cors/extension.py", line 161, in wrapped_function
2019-12-31T06:13:05.399343+00:00 app[web.1]: return cors_after_request(app.make_response(f(*args, **kwargs)))
2019-12-31T06:13:05.399345+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1820, in handle_user_exception
2019-12-31T06:13:05.399346+00:00 app[web.1]: reraise(exc_type, exc_value, tb)
2019-12-31T06:13:05.399347+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
2019-12-31T06:13:05.399349+00:00 app[web.1]: raise value
2019-12-31T06:13:05.399350+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
2019-12-31T06:13:05.399352+00:00 app[web.1]: rv = self.dispatch_request()
2019-12-31T06:13:05.399353+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
2019-12-31T06:13:05.399355+00:00 app[web.1]: return self.view_functions[rule.endpoint](**req.view_args)
2019-12-31T06:13:05.399357+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/site-packages/flask_jwt_extended/view_decorators.py", line 108, in wrapper
2019-12-31T06:13:05.399358+00:00 app[web.1]: return fn(*args, **kwargs)
2019-12-31T06:13:05.399360+00:00 app[web.1]: File "/app/app/api/uploads.py", line 26, in upload_image
2019-12-31T06:13:05.399362+00:00 app[web.1]: UPLOAD_PATHS['temp']['image'].format(uuid=uuid.uuid4())
2019-12-31T06:13:05.399363+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 154, in upload
2019-12-31T06:13:05.399370+00:00 app[web.1]: return upload_to_aws(aws_bucket_name, aws_region, aws_key, aws_secret, uploaded_file, key, **kwargs)
2019-12-31T06:13:05.399371+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 227, in upload_to_aws
2019-12-31T06:13:05.399373+00:00 app[web.1]: file_data = file.read()
2019-12-31T06:13:05.399374+00:00 app[web.1]: File "/app/app/api/helpers/storage.py", line 108, in read
2019-12-31T06:13:05.399376+00:00 app[web.1]: return self.file.read()
2019-12-31T06:13:05.399377+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.7/codecs.py", line 322, in decode
2019-12-31T06:13:05.399379+00:00 app[web.1]: (result, consumed) = self._buffer_decode(data, self.errors, final)
2019-12-31T06:13:05.399380+00:00 app[web.1]: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte
```

**Additional details (please complete the following information):**
 - OS: Ubuntu (Heroku dyno)
 - Python Version 3.7
 - `HEAD` Commit hash f3bb95c0 (version 1.9.0)



Discussion:

Resolving patch:
diff --git a/app/api/helpers/storage.py b/app/api/helpers/storage.py
--- a/app/api/helpers/storage.py
+++ b/app/api/helpers/storage.py
@@ -99,7 +99,16 @@ class UploadedFile:
     def __init__(self, file_path, filename):
         self.file_path = file_path
         self.filename = filename
-        self.file = open(file_path)
+        self.file = open(file_path, 'rb')
+
+    def __len__(self):
+        position = self.file.tell()
+        try:
+            self.file.seek(0, os.SEEK_END)
+            last_position = self.file.tell()
+        finally:
+            self.file.seek(position)
+        return last_position
 
     def save(self, new_path):
         copyfile(self.file_path, new_path)
@@ -224,15 +233,15 @@ def upload_to_aws(bucket_name, aws_region, aws_key, aws_secret, file, key, acl='
         item.delete()
     # set object settings
 
-    file_data = file.read()
-    file_mime = magic.from_buffer(file_data, mime=True)
-    size = len(file_data)
-    sent = k.set_contents_from_string(
-        file_data,
+    file_mime = magic.from_file(file.file_path, mime=True)
+    size = len(file)
+    sent = k.set_contents_from_file(
+        file.file,
         headers={
             'Content-Disposition': 'attachment; filename=%s' % filename,
             'Content-Type': '%s' % file_mime
-        }
+        },
+        rewind=True
     )
     k.set_acl(acl)
     s3_url = 'https://%s.s3.amazonaws.com/' % bucket_name


Test patch:
diff --git a/tests/all/unit/api/helpers/test_storage.py b/tests/all/unit/api/helpers/test_storage.py
--- a/tests/all/unit/api/helpers/test_storage.py
+++ b/tests/all/unit/api/helpers/test_storage.py
@@ -1,8 +1,10 @@
 """Test file for storage functions."""
+import os
+from tempfile import TemporaryDirectory
 import unittest
 from unittest.mock import patch
 
-from app.api.helpers.storage import create_url, generate_hash
+from app.api.helpers.storage import create_url, generate_hash, UploadedFile
 
 
 class TestStorageHelperValidation(unittest.TestCase):
@@ -65,5 +67,25 @@ def patch_settings(settings):
             self.assertEqual(len(actual_output), 10)
 
 
+class TestUploadedFile(unittest.TestCase):
+    test_data = b'\xffhellothere'
+
+    def _uploaded_file(self, folder, filename='testfile.bin'):
+        path = os.path.join(folder, filename)
+        with open(path, 'wb') as f:
+            f.write(self.test_data)
+        return UploadedFile(path, filename)
+
+    def test_len(self):
+        with TemporaryDirectory() as folder:
+            file = self._uploaded_file(folder)
+            self.assertEqual(len(file), len(self.test_data))
+
+    def test_read(self):
+        with TemporaryDirectory() as folder:
+            file = self._uploaded_file(folder)
+            self.assertEqual(file.read(), self.test_data)
+
+
 if __name__ == '__main__':
     unittest.main()