Repository: ckan/ckan

Issue:
File uploads don't work on new Flask based API
For instance:

```
curl -X POST http://localhost:5000/api/action/resource_create \ 
-F "package_id=9222f234-2839-4864-8463-03dca09d6237" \ 
-F "name=Test upload API" \
-F "upload=@/home/adria/Downloads/valid.csv" \
-H "Authorization: API_KEY"
```

Works on 2.7 but not on master (the resource is created but no file is uploaded). The `data_dict` received by `resource_create` does not contain any `upload` key at all, so that is being lost in when [getting the request data](https://github.com/ckan/ckan/blob/master/ckan/views/api.py#L133).


Discussion:
Do you intend to work on this? Anything I can do to help?
I had a quick look at this. We need to fix this at three levels:

1. We need to include incoming files in the `data_dict` passed to actions:

  ```diff
diff --git a/ckan/views/api.py b/ckan/views/api.py
index b9ed2e6..d7d73f2 100644
--- a/ckan/views/api.py
+++ b/ckan/views/api.py
@@ -198,6 +198,9 @@ def _get_request_data(try_url_params=False):
         raise ValueError(u'Invalid request. Please use the POST method for '
                          'your request')
 
+    for field_name, file_ in request.files.iteritems():
+        request_data[field_name] = file_
+
     log.debug(u'Request data extracted: %r', request_data)
 
     return request_data


  ```

2. `Upload` and `ResourceUpload` classes need to also check against the [class](http://werkzeug.pocoo.org/docs/0.12/datastructures/#werkzeug.datastructures.FileStorage) used by Flask/Werkzeug

  ```diff
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
index f83817c..6de65c9 100644
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -7,6 +7,8 @@ import logging
 import magic
 import mimetypes
 
+from werkzeug.datastructures import FileStorage as flask_FileStorage
+
 import ckan.lib.munge as munge
 import ckan.logic as logic
 import ckan.plugins as plugins
@@ -206,7 +208,8 @@ class ResourceUpload(object):
         if config_mimetype_guess == 'file_ext':
             self.mimetype = mimetypes.guess_type(url)[0]
 
-        if isinstance(upload_field_storage, cgi.FieldStorage):
+        if isinstance(
+                upload_field_storage, (cgi.FieldStorage, flask_FileStorage)):
             self.filesize = 0  # bytes
 
             self.filename = upload_field_storage.filename


  ```

3. We need to refactor the same classes so any operation we did against the current `FieldStorage` object is supported against the new one, eg this is the first error I'm getting without changing anything:

  ```
  File "/home/adria/dev/pyenvs/ckan/src/ckan/ckan/lib/uploader.py", line 46, in get_resource_uploader
    upload = ResourceUpload(data_dict)
  File "/home/adria/dev/pyenvs/ckan/src/ckan/ckan/lib/uploader.py", line 220, in __init__
    self.upload_file = upload_field_storage.file
  File "/home/adria/dev/pyenvs/ckan/local/lib/python2.7/site-packages/werkzeug/datastructures.py", line 2723, in __getattr__
    return getattr(self.stream, name)
AttributeError: '_io.BytesIO' object has no attribute 'file'
  ```
  Not sure if we can handle each failing call individually or we need some kind of wrapper like the one @wardi mentioned.

4. And of course we need a test that uploads a file via the API to make sure this works going forward

@mattfullerton does this sound like something you could look into relatively soon? Otherwise @smotornyuk will work on it
here's what ckanapi.LocalCKAN does for file-like objects passed to the API: https://github.com/ckan/ckanapi/blob/master/ckanapi/localckan.py#L54-L69

There's a special case for streams. Some actions assume we can seek on uploaded files so in that case we need to create a temporary file.
@amercader I would have volunteered if it wasn't going to get looked at soon, as I have one CKAN where I'd like to stay on master and we do a lot of file uploads via API. Happy to let @smotornyuk work on it.

Resolving patch:
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -7,11 +7,14 @@
 import magic
 import mimetypes
 
+from werkzeug.datastructures import FileStorage as FlaskFileStorage
+
 import ckan.lib.munge as munge
 import ckan.logic as logic
 import ckan.plugins as plugins
 from ckan.common import config
 
+ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)
 log = logging.getLogger(__name__)
 
 _storage_path = None
@@ -19,6 +22,12 @@
 _max_image_size = None
 
 
+def _get_underlying_file(wrapper):
+    if isinstance(wrapper, FlaskFileStorage):
+        return wrapper.stream
+    return wrapper.file
+
+
 def get_uploader(upload_to, old_filename=None):
     '''Query IUploader plugins and return an uploader instance for general
     files.'''
@@ -130,13 +139,13 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field):
         if not self.storage_path:
             return
 
-        if isinstance(self.upload_field_storage, cgi.FieldStorage):
+        if isinstance(self.upload_field_storage, (ALLOWED_UPLOAD_TYPES)):
             self.filename = self.upload_field_storage.filename
             self.filename = str(datetime.datetime.utcnow()) + self.filename
             self.filename = munge.munge_filename_legacy(self.filename)
             self.filepath = os.path.join(self.storage_path, self.filename)
             data_dict[url_field] = self.filename
-            self.upload_file = self.upload_field_storage.file
+            self.upload_file = _get_underlying_file(self.upload_field_storage)
             self.tmp_filepath = self.filepath + '~'
         # keep the file if there has been no change
         elif self.old_filename and not self.old_filename.startswith('http'):
@@ -206,7 +215,7 @@ def __init__(self, resource):
         if config_mimetype_guess == 'file_ext':
             self.mimetype = mimetypes.guess_type(url)[0]
 
-        if isinstance(upload_field_storage, cgi.FieldStorage):
+        if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
             self.filesize = 0  # bytes
 
             self.filename = upload_field_storage.filename
@@ -214,8 +223,7 @@ def __init__(self, resource):
             resource['url'] = self.filename
             resource['url_type'] = 'upload'
             resource['last_modified'] = datetime.datetime.utcnow()
-            self.upload_file = upload_field_storage.file
-
+            self.upload_file = _get_underlying_file(upload_field_storage)
             self.upload_file.seek(0, os.SEEK_END)
             self.filesize = self.upload_file.tell()
             # go back to the beginning of the file buffer
diff --git a/ckan/views/api.py b/ckan/views/api.py
--- a/ckan/views/api.py
+++ b/ckan/views/api.py
@@ -197,7 +197,8 @@ def mixed(multi_dict):
     if request.method == u'PUT' and not request_data:
         raise ValueError(u'Invalid request. Please use the POST method for '
                          'your request')
-
+    for field_name, file_ in request.files.iteritems():
+        request_data[field_name] = file_
     log.debug(u'Request data extracted: %r', request_data)
 
     return request_data


Test patch:
diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py
--- a/ckan/tests/controllers/test_api.py
+++ b/ckan/tests/controllers/test_api.py
@@ -6,19 +6,67 @@
 '''
 import json
 import re
+import mock
+import __builtin__ as builtins
+from StringIO import StringIO
 
-from ckan.lib.helpers import url_for
 from nose.tools import assert_equal, assert_in, eq_
+from pyfakefs import fake_filesystem
 
+from ckan.lib.helpers import url_for
 import ckan.tests.helpers as helpers
 from ckan.tests import factories
-from ckan.lib import helpers as template_helpers
+from ckan.lib import helpers as template_helpers, uploader as ckan_uploader
 import ckan.plugins as p
 from ckan import model
 
+fs = fake_filesystem.FakeFilesystem()
+fake_os = fake_filesystem.FakeOsModule(fs)
+fake_open = fake_filesystem.FakeFileOpen(fs)
+real_open = open
+
+
+def mock_open_if_open_fails(*args, **kwargs):
+    try:
+        return real_open(*args, **kwargs)
+    except (OSError, IOError):
+        return fake_open(*args, **kwargs)
+
 
 class TestApiController(helpers.FunctionalTestBase):
 
+    @helpers.change_config('ckan.storage_path', '/doesnt_exist')
+    @mock.patch.object(builtins, 'open', side_effect=mock_open_if_open_fails)
+    @mock.patch.object(ckan_uploader, 'os', fake_os)
+    @mock.patch.object(ckan_uploader, '_storage_path', new='/doesnt_exist')
+    def test_resource_create_upload_file(self, _):
+        user = factories.User()
+        pkg = factories.Dataset(creator_user_id=user['id'])
+        # upload_content = StringIO()
+        # upload_content.write('test-content')
+
+        url = url_for(
+            controller='api',
+            action='action',
+            logic_function='resource_create', ver='/3')
+        env = {'REMOTE_USER': user['name'].encode('ascii')}
+        postparams = {
+            'name': 'test-flask-upload',
+            'package_id': pkg['id']
+        }
+        upload_content = 'test-content'
+        upload_info = ('upload', 'test-upload.txt', upload_content)
+        app = self._get_test_app()
+        resp = app.post(
+            url, params=postparams,
+            upload_files=[upload_info],
+            extra_environ=env
+            # content_type= 'application/json'
+        )
+        result = resp.json['result']
+        eq_('upload', result['url_type'])
+        eq_(len(upload_content), result['size'])
+
     def test_unicode_in_error_message_works_ok(self):
         # Use tag_delete to echo back some unicode
         app = self._get_test_app()