Repository: ckan/ckan

Issue:
User clean up command
Some auth configurations can lead to the creation of spam user account. Default changes in #7208 will hopefully make more difficult for maintainers to end up in this situation but we can also provide a small command to cleanup spam user accounts. 

The logic should be:

* Filter any user accounts that
   1. Has a value for `image_url`
   2. The file referenced in `image_url` does not match one of the ones allowed by `ckan.uploads.user.mimetypes`
* Ask user to confirm (unless the `-f / --force` flag is passed) showing a list of the accounts that will be deleted (if admins want to keep one they can update the image_url value)
* Delete the user object and the linked uploaded file

Where should this command be available? 
1. Keep it on `ckan user clean` or something similar
2. Add a new `ckan clean ....` or `ckan cleanup ...` command that we can later expand to eg purge datasets / orgs, delete old activities etc

Any thoughts?



Discussion:

Resolving patch:
diff --git a/ckan/cli/clean.py b/ckan/cli/clean.py
new file mode 100644
--- /dev/null
+++ b/ckan/cli/clean.py
@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+import click
+import magic
+import os
+
+from typing import List
+
+from ckan import model
+from ckan import logic
+from ckan.common import config
+from ckan.lib.uploader import get_uploader
+from ckan.types import Context
+
+
+@click.group(short_help="Provide commands to clean entities from the database")
+@click.help_option("-h", "--help")
+def clean():
+    pass
+
+
+def _get_users_with_invalid_image(mimetypes: List[str]) -> List[model.User]:
+    """Returns a list of users containing images with mimetypes not supported.
+    """
+    users = model.User.all()
+    users_with_img = [u for u in users if u.image_url]
+    invalid = []
+    for user in users_with_img:
+        upload = get_uploader("user", old_filename=user.image_url)
+        filepath = upload.old_filepath  # type: ignore
+        if os.path.exists(filepath):
+            mimetype = magic.from_file(filepath, mime=True)
+            if mimetype not in mimetypes:
+                invalid.append(user)
+    return invalid
+
+
+@clean.command("users", short_help="Clean users containing invalid images.")
+@click.option(
+    "-f", "--force", is_flag=True, help="Do not ask for comfirmation."
+)
+def users(force: bool):
+    """Removes users with invalid images from the database.
+
+    Invalid images are the ones with mimetypes not defined in
+    `ckan.upload.user.mimetypes` configuration option.
+
+    This command will work only for CKAN's default Upload, other
+    extensions defining upload interfaces will need to implement its
+    own logic to retrieve and determine if an uploaded image contains
+    an invalid mimetype.
+
+    Example:
+
+      ckan clean users
+      ckan clean users --force
+
+    """
+    mimetypes = config.get_value("ckan.upload.user.mimetypes")
+    if not mimetypes:
+        click.echo("No mimetypes have been configured for user uploads.")
+        return
+
+    invalid = _get_users_with_invalid_image(mimetypes)
+
+    if not invalid:
+        click.echo("No users were found with invalid images.")
+        return
+
+    for user in invalid:
+        msg = "User {} has an invalid image: {}".format(
+            user.name, user.image_url
+        )
+        click.echo(msg)
+
+    if not force:
+        click.confirm("Permanently delete users and their images?", abort=True)
+
+    site_user = logic.get_action("get_site_user")({"ignore_auth": True}, {})
+    context: Context = {"user": site_user["name"]}
+
+    for user in invalid:
+        upload = get_uploader("user", old_filename=user.image_url)
+        file_path = upload.old_filepath  # type: ignore
+        try:
+            os.remove(file_path)
+        except Exception:
+            msg = "Cannot remove {}. User will not be deleted.".format(
+                file_path
+            )
+            click.echo(msg)
+        else:
+            logic.get_action("user_delete")(context, {"id": user.name})
+            click.secho("Deleted user: %s" % user.name, fg="green", bold=True)
diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py
--- a/ckan/cli/cli.py
+++ b/ckan/cli/cli.py
@@ -16,6 +16,7 @@
 from . import (
     asset,
     config,
+    clean,
     dataset,
     db, search_index, server,
     generate,
@@ -229,3 +230,4 @@ def ckan():
 ckan.add_command(user.user)
 ckan.add_command(views.views)
 ckan.add_command(shell.shell)
+ckan.add_command(clean.clean)
diff --git a/ckan/cli/user.py b/ckan/cli/user.py
--- a/ckan/cli/user.py
+++ b/ckan/cli/user.py
@@ -7,7 +7,6 @@
 import six
 import click
 
-
 import ckan.logic as logic
 import ckan.model as model
 from ckan.cli import error_shout


Test patch:
diff --git a/ckan/tests/cli/test_clean.py b/ckan/tests/cli/test_clean.py
new file mode 100644
--- /dev/null
+++ b/ckan/tests/cli/test_clean.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+import pytest
+
+from ckan.cli.cli import ckan
+from ckan.tests.helpers import call_action
+
+
+@pytest.mark.usefixtures("clean_db")
+class TestUserClean:
+    def test_output_if_there_are_not_invalid_users(self, cli):
+        result = cli.invoke(ckan, ["clean", "users"])
+        assert "No users were found with invalid images." in result.output
+
+    @pytest.mark.ckan_config("ckan.upload.user.mimetypes", "")
+    @pytest.mark.ckan_config("ckan.upload.user.types", "")
+    def test_confirm_dialog_if_no_force(
+        self, cli, monkeypatch, create_with_upload, faker, ckan_config
+    ):
+        fake_user = {
+            "name": "fake-user",
+            "email": "fake-user@example.com",
+            "password": "12345678",
+            "action": "user_create",
+            "upload_field_name": "image_upload",
+        }
+        fake_user = create_with_upload(
+            "<html><body>hello world</body></html>", "index.html", **fake_user
+        )
+
+        user = {
+            "name": "valid-user",
+            "email": "valid-user@example",
+            "password": "12345678",
+            "action": "user_create",
+            "upload_field_name": "image_upload",
+        }
+        user = create_with_upload(faker.image(), "image.png", **user)
+
+        monkeypatch.setitem(
+            ckan_config, "ckan.upload.user.mimetypes", "image/png"
+        )
+        result = cli.invoke(ckan, ["clean", "users"])
+
+        assert (
+            f"User {fake_user['name']} has an invalid image:"
+            f" {fake_user['image_url']}"
+            in result.output
+        )
+        assert (
+            f"User {user['name']} has an invalid image: {user['image_url']}"
+            not in result.output
+        )
+        assert "Permanently delete users and their images?" in result.output
+        users = call_action("user_list")
+        assert len(users) == 2
+
+    @pytest.mark.ckan_config("ckan.upload.user.mimetypes", "")
+    @pytest.mark.ckan_config("ckan.upload.user.types", "")
+    def test_correct_users_are_deleted(
+        self, cli, monkeypatch, create_with_upload, faker, ckan_config
+    ):
+        fake_user = {
+            "name": "fake-user",
+            "email": "fake-user@example.com",
+            "password": "12345678",
+            "action": "user_create",
+            "upload_field_name": "image_upload",
+        }
+        fake_user = create_with_upload(
+            "<html><body>hello world</body></html>", "index.html", **fake_user
+        )
+
+        user = {
+            "name": "valid-user",
+            "email": "valid-user@example",
+            "password": "12345678",
+            "action": "user_create",
+            "upload_field_name": "image_upload",
+        }
+        user = create_with_upload(faker.image(), "image.png", **user)
+
+        monkeypatch.setitem(
+            ckan_config, "ckan.upload.user.mimetypes", "image/png"
+        )
+        result = cli.invoke(ckan, ["clean", "users", "--force"])
+        users = call_action("user_list")
+        assert f"Deleted user: {fake_user['name']}" in result.output
+        assert len(users) == 1
+        assert users[0]["name"] == "valid-user"