Repository: ckan/ckan

Issue:
Resource URL gets changed to ___
### CKAN Version if known (or site URL)
latest master

### The probem
When I create or edit a resource using the web form, whatever I set the resource.url to be, it gets saved as something like this: `http://myckansite.com/dataset/713c8c32-cbd8-46ce-90fb-86de1f0811d2/resource/59c0e90c-11e2-4d7f-af43-e098660745dc/download/___` (!)
It shouldn't change it. And the URL it sets it to gives a 404.


Discussion:
I bisected to find the cause: https://github.com/ckan/ckan/commit/ea4ceb404d826a48b8e66872640e6ffffe1ee0f7
```
Date:   Wed Aug 21 20:43:31 2019 +0100
    Upgrade werkzeug
```

Resolving patch:
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -214,7 +214,8 @@ def __init__(self, resource):
         if url and config_mimetype_guess == 'file_ext':
             self.mimetype = mimetypes.guess_type(url)[0]
 
-        if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
+        if bool(upload_field_storage) and \
+                isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
             self.filesize = 0  # bytes
 
             self.filename = upload_field_storage.filename


Test patch:
diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py
--- a/ckan/tests/controllers/test_package.py
+++ b/ckan/tests/controllers/test_package.py
@@ -121,22 +121,24 @@ def test_complete_package_with_one_resource(self, app):
 
     def test_complete_package_with_two_resources(self, app):
         env, response = _get_package_new_page(app)
-        form = response.forms["dataset-edit"]
-        form["name"] = u"complete-package-with-two-resources"
+        form = response.forms['dataset-edit']
+        form['name'] = u'complete-package-with-two-resources'
 
-        response = submit_and_follow(app, form, env, "save")
-        form = response.forms["resource-edit"]
-        form["url"] = u"http://example.com/resource0"
+        response = submit_and_follow(app, form, env, 'save')
+        form = response.forms['resource-edit']
+        form['url'] = u'http://example.com/resource0'
 
-        response = submit_and_follow(app, form, env, "save", "again")
-        form = response.forms["resource-edit"]
-        form["url"] = u"http://example.com/resource1"
+        response = submit_and_follow(app, form, env, 'save', 'again')
+        form = response.forms['resource-edit']
+        form['url'] = u'http://example.com/resource1'
 
-        submit_and_follow(app, form, env, "save", "go-metadata")
-        pkg = model.Package.by_name(u"complete-package-with-two-resources")
-        assert pkg.resources[0].url == u"http://example.com/resource0"
-        assert pkg.resources[1].url == u"http://example.com/resource1"
-        assert pkg.state == "active"
+        submit_and_follow(app, form, env, 'save', 'go-metadata')
+        pkg = model.Package.by_name(u'complete-package-with-two-resources')
+        assert pkg.resources[0].url == u'http://example.com/resource0'
+        assert pkg.resources[1].url == u'http://example.com/resource1'
+        assert pkg.state == 'active'
+
+    # resource upload is tested in TestExampleIUploaderPlugin
 
     def test_previous_button_works(self, app):
         env, response = _get_package_new_page(app)
diff --git a/ckan/tests/lib/test_uploader.py b/ckan/tests/lib/test_uploader.py
new file mode 100644
--- /dev/null
+++ b/ckan/tests/lib/test_uploader.py
@@ -0,0 +1,90 @@
+# encoding: utf-8
+import mock
+import tempfile
+import __builtin__ as builtins
+import flask
+import paste.fileapp
+import cStringIO
+
+from werkzeug.datastructures import FileStorage
+from pyfakefs import fake_filesystem
+from nose.tools import assert_equal as eq_
+
+from ckan.common import config
+import ckan.lib.uploader
+from ckan.lib.uploader import ResourceUpload
+from ckanext.example_iuploader.test.test_plugin import mock_open_if_open_fails
+
+
+fs = fake_filesystem.FakeFilesystem()
+fake_os = fake_filesystem.FakeOsModule(fs)
+
+
+class TestInitResourceUpload(object):
+    @mock.patch.object(ckan.lib.uploader, u'os', fake_os)
+    @mock.patch.object(builtins, u'open', side_effect=mock_open_if_open_fails)
+    @mock.patch.object(flask, u'send_file', side_effect=[b'DATA'])
+    @mock.patch.object(paste.fileapp, u'os', fake_os)
+    @mock.patch.object(config[u'pylons.h'], u'uploads_enabled',
+                       return_value=True)
+    @mock.patch.object(ckan.lib.uploader, u'_storage_path', new=u'/doesnt_exist')
+    def test_resource_without_upload_with_old_werkzeug(
+            self, mock_uploads_enabled, mock_open, send_file):
+        # this test data is based on real observation using a browser
+        # and werkzeug 0.14.1
+        res = {u'clear_upload': u'true',
+               u'format': u'CSV',
+               u'url': u'https://example.com/data.csv',
+               u'description': u'',
+               u'upload': u'',
+               u'package_id': u'dataset1',
+               u'id': u'8a3a874e-5ee1-4e43-bdaf-e2569cf72344',
+               u'name': u'data.csv'}
+        res_upload = ResourceUpload(res)
+
+        eq_(res_upload.filename, None)
+
+    @mock.patch.object(ckan.lib.uploader, u'os', fake_os)
+    @mock.patch.object(builtins, u'open', side_effect=mock_open_if_open_fails)
+    @mock.patch.object(flask, u'send_file', side_effect=[b'DATA'])
+    @mock.patch.object(paste.fileapp, u'os', fake_os)
+    @mock.patch.object(config[u'pylons.h'], u'uploads_enabled',
+                       return_value=True)
+    @mock.patch.object(ckan.lib.uploader, u'_storage_path', new=u'/doesnt_exist')
+    def test_resource_without_upload(
+            self, mock_uploads_enabled, mock_open, send_file):
+        # this test data is based on real observation using a browser
+        res = {u'clear_upload': u'true',
+               u'format': u'PNG',
+               u'url': u'https://example.com/data.csv',
+               u'description': u'',
+               u'upload': FileStorage(filename=u''),
+               u'package_id': u'dataset1',
+               u'id': u'8a3a874e-5ee1-4e43-bdaf-e2569cf72344',
+               u'name': u'data.csv'}
+        res_upload = ResourceUpload(res)
+
+        eq_(res_upload.filename, None)
+
+    @mock.patch.object(ckan.lib.uploader, u'os', fake_os)
+    @mock.patch.object(builtins, u'open', side_effect=mock_open_if_open_fails)
+    @mock.patch.object(flask, u'send_file', side_effect=[b'DATA'])
+    @mock.patch.object(paste.fileapp, u'os', fake_os)
+    @mock.patch.object(config[u'pylons.h'], u'uploads_enabled',
+                       return_value=True)
+    @mock.patch.object(ckan.lib.uploader, u'_storage_path', new=u'/doesnt_exist')
+    def test_resource_with_upload(
+            self, mock_uploads_enabled, mock_open, send_file):
+        # this test data is based on real observation using a browser
+        res = {u'clear_upload': u'',
+               u'format': u'PNG',
+               u'url': u'https://example.com/data.csv',
+               u'description': u'',
+               u'upload': FileStorage(filename=u'data.csv', content_type=u'CSV'),
+               u'package_id': u'dataset1',
+               u'id': u'8a3a874e-5ee1-4e43-bdaf-e2569cf72344',
+               u'name': u'data.csv'}
+        res_upload = ResourceUpload(res)
+
+        eq_(res_upload.filesize, 0)
+        eq_(res_upload.filename, u'data.csv')
diff --git a/ckan/tests/logic/action/test_create.py b/ckan/tests/logic/action/test_create.py
--- a/ckan/tests/logic/action/test_create.py
+++ b/ckan/tests/logic/action/test_create.py
@@ -8,6 +8,8 @@
 import mock
 import pytest
 
+from werkzeug.datastructures import FileStorage as FlaskFileStorage
+
 import ckan
 import ckan.logic as logic
 import ckan.model as model
@@ -26,9 +28,11 @@
 fake_open = fake_filesystem.FakeFileOpen(fs)
 
 
-class FakeFileStorage(cgi.FieldStorage):
-    def __init__(self, fp, filename):
-        self.file = fp
+class FakeFileStorage(FlaskFileStorage):
+    content_type = None
+
+    def __init__(self, stream, filename):
+        self.stream = stream
         self.filename = filename
         self.name = "upload"
 
@@ -535,7 +539,7 @@ def test_mimetype_by_upload_by_file(self, mock_open):
         NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41
         """
         )
-        test_resource = FakeFileStorage(test_file, "")
+        test_resource = FakeFileStorage(test_file, "test.csv")
 
         context = {}
         params = {
diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py
--- a/ckan/tests/logic/action/test_update.py
+++ b/ckan/tests/logic/action/test_update.py
@@ -6,6 +6,8 @@
 import mock
 import pytest
 
+from werkzeug.datastructures import FileStorage as FlaskFileStorage
+
 import ckan
 import ckan.lib.app_globals as app_globals
 import ckan.logic as logic
@@ -24,6 +26,15 @@
 fake_open = fake_filesystem.FakeFileOpen(fs)
 
 
+class FakeFileStorage(FlaskFileStorage):
+    content_type = None
+
+    def __init__(self, stream, filename):
+        self.stream = stream
+        self.filename = filename
+        self.name = "upload"
+
+
 def mock_open_if_open_fails(*args, **kwargs):
     try:
         return real_open(*args, **kwargs)
@@ -810,18 +821,6 @@ def test_calling_with_only_id_doesnt_update_anything(self):
 @pytest.mark.ckan_config("ckan.plugins", "image_view recline_view")
 @pytest.mark.usefixtures("clean_db", "with_plugins")
 class TestResourceUpdate(object):
-    import cgi
-
-    class FakeFileStorage(cgi.FieldStorage):
-        def __init__(self, fp, filename):
-            self.file = fp
-            self.filename = filename
-            self.name = "upload"
-
-        def setup(self):
-            import ckan.model as model
-
-            model.repo.rebuild_db()
 
     def test_url_only(self):
         dataset = factories.Dataset()
@@ -1074,7 +1073,7 @@ def test_mimetype_by_upload_by_file(self, mock_open):
         NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41
         """
         )
-        update_resource = TestResourceUpdate.FakeFileStorage(
+        update_resource = FakeFileStorage(
             update_file, "update_test"
         )
 
@@ -1124,7 +1123,7 @@ def test_mimetype_by_upload_by_filename(self, mock_open):
         }
         """
         )
-        test_resource = TestResourceUpdate.FakeFileStorage(
+        test_resource = FakeFileStorage(
             test_file, "test.json"
         )
         dataset = factories.Dataset()
@@ -1147,7 +1146,7 @@ def test_mimetype_by_upload_by_filename(self, mock_open):
         NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41
         """
         )
-        update_resource = TestResourceUpdate.FakeFileStorage(
+        update_resource = FakeFileStorage(
             update_file, "update_test.csv"
         )
 
@@ -1219,7 +1218,7 @@ def test_size_of_resource_by_upload(self, mock_open):
         }
         """
         )
-        test_resource = TestResourceUpdate.FakeFileStorage(
+        test_resource = FakeFileStorage(
             test_file, "test.json"
         )
         dataset = factories.Dataset()
@@ -1242,7 +1241,7 @@ def test_size_of_resource_by_upload(self, mock_open):
         NAZKO,1C08,1070,2016/01/05,20,31,,76,16,JAN-01,41
         """
         )
-        update_resource = TestResourceUpdate.FakeFileStorage(
+        update_resource = FakeFileStorage(
             update_file, "update_test.csv"
         )