Repository: ckan/ckan

Issue:
POST requests to API don't work with CSRF, tests bypass CSRF protection
**CKAN version**
2.10

**Describe the bug**
The API is completely unusable since https://github.com/ckan/ckan/pull/7058 was merged.

Any POST API call (and presumably PUT, DELETE etc) returns a `400 Bad Request: The CSRF token is missing` error, which in hindsight makes sense because we removed the `@csrf.exempt` decorator from the `api.action` endpoint. We'll need to see how we can implement:

* API call + Cookie authentication = CSRF enabled (to support current frontend API usage)
* API call + Header authentication = CSRF disabled (to support external API calls)

But more worryingly,I found out that the tests are happily ignoring all these issues,  by bypassing CSRF protection entirely.

https://github.com/ckan/ckan/blob/367fed2fa572e896c5eb703db9c663e097133e41/ckan/tests/helpers.py#L271-L272

This might made sense while we were implementing the feature, but I think we need to discuss what we want going forward.


Discussion:

Resolving patch:
diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py
--- a/ckan/config/middleware/flask_app.py
+++ b/ckan/config/middleware/flask_app.py
@@ -12,7 +12,7 @@
 from logging.handlers import SMTPHandler
 from typing import Any, Iterable, Optional, Union, cast
 
-from flask import Blueprint, send_from_directory
+from flask import Blueprint, send_from_directory, current_app
 from flask.ctx import _AppCtxGlobals
 from flask.sessions import SessionInterface
 from flask_multistatic import MultiStaticFlask
@@ -327,7 +327,10 @@ def load_user_from_request(request):  # type: ignore
         This callback function is called whenever a user could not be
         authenticated via the session cookie, so we fall back to the API token.
         """
+        g.login_via_auth_header = True
+
         user = _get_user_for_apitoken()
+
         return user
 
     # Update the main CKAN config object with the Flask specific keys
@@ -410,6 +413,16 @@ def ckan_before_request() -> Optional[Response]:
     # Sets g.user and g.userobj for extensions
     response = identify_user()
 
+    # Disable CSRF protection if user was logged in via the Authorization
+    # header
+    if g.get("login_via_auth_header"):
+        # Get the actual view function, as it might not match the endpoint,
+        # eg "organization.edit" -> "group.edit", or custom dataset types
+        endpoint = request.endpoint or ""
+        view = current_app.view_functions.get(endpoint)
+        dest = f"{view.__module__}.{view.__name__}"     # type: ignore
+        csrf.exempt(dest)
+
     # Set the csrf_field_name so we can use it in our templates
     g.csrf_field_name = config.get_value("WTF_CSRF_FIELD_NAME")
 


Test patch:
diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py
--- a/ckan/tests/controllers/test_api.py
+++ b/ckan/tests/controllers/test_api.py
@@ -380,3 +380,39 @@ def test_cookie_based_auth_disabled(app):
     res = app.get(url, environ_overrides=env)
 
     assert res.status_code == 200
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_header_based_auth_default(app):
+
+    sysadmin = factories.SysadminWithToken()
+    org = factories.Organization()
+    dataset = factories.Dataset(private=True, owner_org=org["id"])
+
+    url = url_for("api.action", ver=3, logic_function="package_show", id=dataset["id"])
+
+    env = {"Authorization": sysadmin["token"]}
+
+    res = app.get(url, environ_overrides=env)
+
+    assert res.status_code == 200
+
+
+@pytest.mark.usefixtures("clean_db")
+def test_header_based_auth_default_post(app):
+
+    sysadmin = factories.SysadminWithToken()
+    org = factories.Organization()
+    dataset = factories.Dataset(private=True, owner_org=org["id"])
+
+    url = url_for("api.action", ver=3, logic_function="package_patch")
+
+    env = {"Authorization": sysadmin["token"]}
+
+    data = {
+        "id": dataset["id"],
+        "notes": "updated",
+    }
+    res = app.post(url, environ_overrides=env, data=data)
+
+    assert res.status_code == 200
diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py
--- a/ckan/tests/controllers/test_group.py
+++ b/ckan/tests/controllers/test_group.py
@@ -350,19 +350,16 @@ def test_membership_list(self, app):
         assert user_roles["User One"] == "Admin"
         assert user_roles["User Two"] == "Member"
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_membership_add(self, current_user, app):
+    def test_membership_add(self, app):
         """Member can be added via add member page"""
         user = factories.User(fullname="My Owner")
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         factories.User(fullname="My Fullname", name="my-new-user")
         group = self._create_group(user["name"])
 
         url = url_for("group.member_new", id=group["name"])
         add_response = app.post(
             url,
+            environ_overrides={"REMOTE_USER": user["name"]},
             data={"save": "", "username": "my-new-user", "role": "member"},
         )
 
@@ -383,18 +380,15 @@ def test_membership_add(self, current_user, app):
         assert user_roles["My Owner"] == "Admin"
         assert user_roles["My Fullname"] == "Member"
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_membership_add_by_email(self, current_user, app, mail_server):
+    def test_membership_add_by_email(self, app, mail_server):
         user = factories.User(fullname="My Owner")
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         group = self._create_group(user["name"])
 
         url = url_for("group.member_new", id=group["name"])
         email = "invited_user@mailinator.com"
         app.post(
             url,
+            environ_overrides={"REMOTE_USER": user["name"]},
             data={"save": "", "email": email, "role": "member"},
             status=200
         )
@@ -403,20 +397,19 @@ def test_membership_add_by_email(self, current_user, app, mail_server):
         assert user.email == email
         assert group["id"] in user.get_group_ids(capacity="member")
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_membership_edit_page(self, current_user, app):
+    def test_membership_edit_page(self, app):
         """If `user` parameter provided, render edit page."""
         owner = factories.User(fullname="My Owner")
-        user_obj = model.User.get(owner["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         member = factories.User(fullname="My Fullname", name="my-user")
         group = self._create_group(owner["name"], users=[
             {'name': member['name'], 'capacity': 'admin'}
         ])
 
         url = url_for("group.member_new", id=group["name"], user=member['name'])
-        response = app.get(url)
+        response = app.get(
+            url,
+            environ_overrides={"REMOTE_USER": owner["name"]},
+        )
 
         page = BeautifulSoup(response.body)
         assert page.select_one('.page-heading').text.strip() == 'Edit Member'
@@ -425,19 +418,16 @@ def test_membership_edit_page(self, current_user, app):
         assert page.select_one('#username').get('value') == member['name']
 
     @pytest.mark.usefixtures("clean_db")
-    @mock.patch("flask_login.utils._get_user")
-    def test_admin_add(self, current_user, app):
+    def test_admin_add(self, app):
         """Admin can be added via add member page"""
         owner = factories.User(fullname="My Owner")
-        user_obj = model.User.get(owner["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         factories.User(fullname="My Fullname", name="my-admin-user")
         group = self._create_group(owner["name"])
 
         url = url_for("group.member_new", id=group["name"])
         add_response = app.post(
             url,
+            environ_overrides={"REMOTE_USER": owner["name"]},
             data={"save": "", "username": "my-admin-user", "role": "admin"},
         )
 
@@ -458,13 +448,9 @@ def test_admin_add(self, current_user, app):
         assert user_roles["My Owner"] == "Admin"
         assert user_roles["My Fullname"] == "Admin"
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_remove_member(self, current_user, app):
+    def test_remove_member(self,  app):
         """Member can be removed from group"""
         user = factories.User(fullname="My Owner")
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         user_two = factories.User(fullname="User Two")
 
         other_users = [{"name": user_two["id"], "capacity": "member"}]
@@ -475,7 +461,10 @@ def test_remove_member(self, current_user, app):
             "group.member_delete", user=user_two["id"], id=group["id"]
         )
 
-        remove_response = app.post(remove_url)
+        remove_response = app.post(
+            remove_url,
+            environ_overrides={"REMOTE_USER": user["name"]},
+        )
         assert helpers.body_contains(remove_response, "1 members")
 
         remove_response_html = BeautifulSoup(remove_response.body)
@@ -493,20 +482,21 @@ def test_remove_member(self, current_user, app):
         assert len(user_roles.keys()) == 1
         assert user_roles["My Owner"] == "Admin"
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_member_users_cannot_add_members(self, current_user, app):
+    def test_member_users_cannot_add_members(self, app):
         user = factories.User(fullname="My Owner")
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         group = factories.Group(
             users=[{"name": user["name"], "capacity": "member"}]
         )
 
-        app.get(url_for("group.member_new", id=group["id"]), status=403)
+        app.get(
+            url_for("group.member_new", id=group["id"]),
+            environ_overrides={"REMOTE_USER": user["name"]},
+            status=403
+        )
 
         app.post(
             url_for("group.member_new", id=group["id"]),
+            environ_overrides={"REMOTE_USER": user["name"]},
             data={
                 "id": "test",
                 "username": "test",
diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py
--- a/ckan/tests/controllers/test_package.py
+++ b/ckan/tests/controllers/test_package.py
@@ -33,13 +33,6 @@ def _get_location(res):
     return urlparse(location)._replace(scheme='', netloc='').geturl()
 
 
-def mock_current_user(current_user):
-    user = factories.User()
-    user_obj = model.User.get(user["name"])
-    # mock current_user
-    current_user.return_value = user_obj
-
-
 @pytest.mark.usefixtures("clean_db", "with_request_context")
 class TestPackageNew(object):
 
@@ -185,28 +178,38 @@ def test_complete_package_with_one_resource(self, app, user):
         assert pkg.resources[0].url == u"http://example.com/resource"
         assert pkg.state == "active"
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_complete_package_with_two_resources(self, current_user, app):
-        mock_current_user(current_user)
+    def test_complete_package_with_two_resources(self, app):
+
+        user = factories.User()
 
         url = url_for("dataset.new")
         name = factories.Dataset.stub().name
-        response = app.post(url, data={
-            "name": name,
-            "save": "",
-            "_ckan_phase": 1
-        }, follow_redirects=False)
+        response = app.post(
+            url,
+            data={
+                "name": name,
+                "save": "",
+                "_ckan_phase": 1
+            },
+            environ_overrides={"REMOTE_USER": user["name"]},
+            follow_redirects=False
+        )
         location = _get_location(response)
         app.post(location, data={
-            "id": "",
-            "url": "http://example.com/resource0",
-            "save": "again"
-        })
+                "id": "",
+                "url": "http://example.com/resource0",
+                "save": "again"
+            },
+
+            environ_overrides={"REMOTE_USER": user["name"]},
+        )
         app.post(location, data={
-            "id": "",
-            "url": "http://example.com/resource1",
-            "save": "go-metadata"
-        })
+                "id": "",
+                "url": "http://example.com/resource1",
+                "save": "go-metadata"
+            },
+            environ_overrides={"REMOTE_USER": user["name"]},
+        )
         pkg = model.Package.by_name(name)
         resources = sorted(pkg.resources, key=lambda r: r.url)
         assert resources[0].url == u"http://example.com/resource0"
@@ -232,23 +235,30 @@ def test_previous_button_works(self, app, user):
 
         assert '/dataset/edit/' in response.headers['location']
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_previous_button_populates_form(self, current_user, app):
-        mock_current_user(current_user)
+    def test_previous_button_populates_form(self, app):
+
+        user = factories.User()
 
         url = url_for("dataset.new")
         name = factories.Dataset.stub().name
-        response = app.post(url, data={
-            "name": name,
-            "save": "",
-            "_ckan_phase": 1
-        }, follow_redirects=False)
+        response = app.post(
+            url,
+            environ_overrides={"REMOTE_USER": user["name"]},
+            data={
+                "name": name,
+                "save": "",
+                "_ckan_phase": 1
+            },
+            follow_redirects=False
+        )
 
         location = _get_location(response)
         response = app.post(location, data={
             "id": "",
             "save": "go-dataset"
-        })
+            },
+            environ_overrides={"REMOTE_USER": user["name"]},
+        )
 
         assert 'name="title"' in response
         assert f'value="{name}"'
@@ -814,27 +824,27 @@ def test_logged_in_user_cannot_delete_owned_dataset(self, app, user):
         assert 403 == response.status_code
         assert helpers.body_contains(response, "Unauthorized to delete package")
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_confirm_cancel_delete(self, current_user, app):
+    def test_confirm_cancel_delete(self, app):
         """Test confirmation of deleting datasets
 
         When package_delete is made as a get request, it should return a
         'do you want to delete this dataset? confirmation page"""
         user = factories.User()
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         owner_org = factories.Organization(
             users=[{"name": user["name"], "capacity": "admin"}]
         )
         dataset = factories.Dataset(owner_org=owner_org["id"])
-        response = app.get(url_for("dataset.delete", id=dataset["name"]))
+        response = app.get(
+            url_for("dataset.delete", id=dataset["name"]),
+            environ_overrides={"REMOTE_USER": user["name"]},
+        )
         assert 200 == response.status_code
         message = "Are you sure you want to delete dataset - {name}?"
         assert helpers.body_contains(response, message.format(name=dataset["title"]))
 
         response = app.post(
             url_for("dataset.delete", id=dataset["name"]),
+            environ_overrides={"REMOTE_USER": user["name"]},
             data={"cancel": ""}
             )
         assert 200 == response.status_code
@@ -1085,12 +1095,8 @@ def test_resource_download_content_type(self, create_with_upload, app):
 @pytest.mark.ckan_config("ckan.plugins", "image_view")
 @pytest.mark.usefixtures("non_clean_db", "with_plugins", "with_request_context")
 class TestResourceView(object):
-    @mock.patch("flask_login.utils._get_user")
-    def test_resource_view_create(self, current_user, app):
+    def test_resource_view_create(self, app):
         user = factories.User()
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         owner_org = factories.Organization(
             users=[{"name": user["name"], "capacity": "admin"}]
         )
@@ -1104,15 +1110,15 @@ def test_resource_view_create(self, current_user, app):
             view_type="image_view",
         )
 
-        response = app.post(url, data={"title": "Test Image View"})
+        response = app.post(
+            url,
+            environ_overrides={"REMOTE_USER": user["name"]},
+            data={"title": "Test Image View"}
+        )
         assert helpers.body_contains(response, "Test Image View")
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_resource_view_edit(self, current_user, app):
+    def test_resource_view_edit(self, app):
         user = factories.User()
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         owner_org = factories.Organization(
             users=[{"name": user["name"], "capacity": "admin"}]
         )
@@ -1127,16 +1133,16 @@ def test_resource_view_edit(self, current_user, app):
             view_id=resource_view["id"],
         )
 
-        response = app.post(url, data={"title": "Updated RV Title"})
+        response = app.post(
+            url,
+            environ_overrides={"REMOTE_USER": user["name"]},
+            data={"title": "Updated RV Title"}
+        )
         assert helpers.body_contains(response, "Updated RV Title")
 
-    @mock.patch("flask_login.utils._get_user")
     @pytest.mark.ckan_config("ckan.views.default_views", "")
-    def test_resource_view_delete(self, current_user, app):
+    def test_resource_view_delete(self, app):
         user = factories.User()
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         owner_org = factories.Organization(
             users=[{"name": user["name"], "capacity": "admin"}]
         )
@@ -1151,7 +1157,11 @@ def test_resource_view_delete(self, current_user, app):
             view_id=resource_view["id"],
         )
 
-        response = app.post(url, data={"delete": "Delete"})
+        response = app.post(
+            url,
+            environ_overrides={"REMOTE_USER": user["name"]},
+            data={"delete": "Delete"}
+        )
         assert helpers.body_contains(response, "This resource has no views")
 
     def test_existent_resource_view_page_returns_ok_code(self, app):
@@ -1429,16 +1439,12 @@ def test_sysadmins_can_delete_any_resource(self, app, sysadmin):
         with pytest.raises(logic.NotFound):
             helpers.call_action("resource_show", id=resource["id"])
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_confirm_and_cancel_deleting_a_resource(self, current_user, app):
+    def test_confirm_and_cancel_deleting_a_resource(self, app):
         """Test confirmation of deleting resources
 
         When resource_delete is made as a get request, it should return a
         'do you want to delete this reource? confirmation page"""
         user = factories.User()
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         owner_org = factories.Organization(
             users=[{"name": user["name"], "capacity": "admin"}]
         )
@@ -1449,7 +1455,8 @@ def test_confirm_and_cancel_deleting_a_resource(self, current_user, app):
                 "{}_resource.delete".format(dataset["type"]),
                 id=dataset["name"],
                 resource_id=resource["id"],
-            )
+            ),
+            environ_overrides={"REMOTE_USER": user["name"]},
         )
         assert 200 == response.status_code
         message = "Are you sure you want to delete resource - {name}?"
@@ -1461,6 +1468,7 @@ def test_confirm_and_cancel_deleting_a_resource(self, current_user, app):
                 id=dataset["name"],
                 resource_id=resource["id"],
             ),
+            environ_overrides={"REMOTE_USER": user["name"]},
             data={"cancel": ""},
         )
         assert 200 == response.status_code
diff --git a/ckan/tests/controllers/test_user.py b/ckan/tests/controllers/test_user.py
--- a/ckan/tests/controllers/test_user.py
+++ b/ckan/tests/controllers/test_user.py
@@ -625,15 +625,12 @@ def test_request_reset_but_mailer_not_configured(
 
         assert "Error sending the email" in response
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_sysadmin_not_authorized(self, current_user, app):
+    def test_sysadmin_not_authorized(self, app):
         user = factories.User()
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
         app.post(
             url_for("user.sysadmin"),
             data={"username": user["name"], "status": "1"},
+            environ_overrides={"REMOTE_USER": user["name"]},
             status=403
         )
 
@@ -646,13 +643,9 @@ def test_sysadmin_invalid_user(self, app, sysadmin):
             status=404,
         )
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_sysadmin_promote_success(self, current_user, app):
+    def test_sysadmin_promote_success(self, app):
 
         sysadmin = factories.Sysadmin()
-        sysadmin_obj = model.User.get(sysadmin["name"])
-        # mock current_user
-        current_user.return_value = sysadmin_obj
         # create a normal user
         user = factories.User(fullname="Alice")
 
@@ -660,6 +653,7 @@ def test_sysadmin_promote_success(self, current_user, app):
         resp = app.post(
             url_for("user.sysadmin"),
             data={"username": user["name"], "status": "1"},
+            environ_overrides={"REMOTE_USER": sysadmin["name"]},
             status=200,
         )
         assert "Promoted Alice to sysadmin" in resp.body
@@ -668,12 +662,8 @@ def test_sysadmin_promote_success(self, current_user, app):
         userobj = model.User.get(user["id"])
         assert userobj.sysadmin
 
-    @mock.patch('flask_login.utils._get_user')
-    def test_sysadmin_revoke_success(self, current_user, app):
+    def test_sysadmin_revoke_success(self, app):
         sysadmin = factories.Sysadmin()
-        sysadmin_obj = model.User.get(sysadmin["name"])
-        # mock current_user
-        current_user.return_value = sysadmin_obj
         # create another sysadmin
         user = factories.Sysadmin(fullname="Bob")
 
@@ -681,6 +671,7 @@ def test_sysadmin_revoke_success(self, current_user, app):
         resp = app.post(
             url_for("user.sysadmin"),
             data={"username": user["name"], "status": "0"},
+            environ_overrides={"REMOTE_USER": sysadmin["name"]},
             status=200,
         )
         assert "Revoked sysadmin permission from Bob" in resp.body
diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py
--- a/ckan/tests/helpers.py
+++ b/ckan/tests/helpers.py
@@ -268,8 +268,6 @@ def test_dataset_search(self, app):
 
     """
     config["testing"] = True
-    # exempt the CKAN TESTS from csrf-validation.
-    config["WTF_CSRF_METHODS"] = []
     app = ckan.config.middleware.make_app(config)
     app = CKANTestApp(app)
 
diff --git a/ckanext/example_idatasetform/tests/test_example_idatasetform.py b/ckanext/example_idatasetform/tests/test_example_idatasetform.py
--- a/ckanext/example_idatasetform/tests/test_example_idatasetform.py
+++ b/ckanext/example_idatasetform/tests/test_example_idatasetform.py
@@ -175,19 +175,17 @@ def test_dataset_create_redirects(self, app, user):
             "fancy_type.read", id=name, _external=True
         )
 
-    @mock.patch("flask_login.utils._get_user")
-    def test_links_on_edit_pages(self, current_user, app):
-        user = factories.User()
-        user_obj = model.User.get(user["name"])
-        # mock current_user
-        current_user.return_value = user_obj
+    def test_links_on_edit_pages(self, app):
+        user = factories.Sysadmin()
 
         pkg = factories.Dataset(type="fancy_type", user=user)
         res = factories.Resource(package_id=pkg["id"], user=user)
-        page = bs4.BeautifulSoup(
-            app.get(
-                url_for("fancy_type.edit", id=pkg["name"])).body
+        response = app.get(
+            url_for("fancy_type.edit", id=pkg["name"]),
+            environ_overrides={"REMOTE_USER": user["name"]},
+            status=200,
         )
+        page = bs4.BeautifulSoup(response.body)
         page_header = page.find(class_="page-header")
         for action in ["edit", "resources", "read"]:
             assert page_header.find(
@@ -199,6 +197,7 @@ def test_links_on_edit_pages(self, current_user, app):
         )
         resp = app.post(
             url_for("fancy_type.edit", id=pkg["name"]),
+            environ_overrides={"REMOTE_USER": user["name"]},
             data={
                 "name": pkg["name"],
                 "save": "",
@@ -226,7 +225,8 @@ def test_links_on_edit_pages(self, current_user, app):
                     "fancy_type_resource.edit",
                     id=pkg["id"],
                     resource_id=res["id"],
-                )
+                ),
+                environ_overrides={"REMOTE_USER": user["name"]},
             ).body
         )
         page_header = page.find(class_="page-header")
@@ -257,6 +257,7 @@ def test_links_on_edit_pages(self, current_user, app):
                 id=pkg["name"],
                 resource_id=res["id"],
             ),
+            environ_overrides={"REMOTE_USER": user["name"]},
             data={"id": res["id"], "url": res["url"], "save": "go-metadata"},
             follow_redirects=False,
         )