Repository: ckan/ckan

Issue:
Activities dissappear from UI when internal processes are run by ignored users
**CKAN version**
2.0 onwards

**Describe the bug**
Activities shown on the organization or dashboard activity feed can suddenly disappear and show an empty list, eg:

![Screenshot_2020-10-28 Activity Stream - Harvesestion - Organisations - CKAN(1)](https://user-images.githubusercontent.com/200230/97434909-0052b400-1920-11eb-8bfd-5feae3d69e51.png)

 to

![Screenshot_2020-10-28 Activity Stream - Harvesestion - Organisations - CKAN(2)](https://user-images.githubusercontent.com/200230/97434925-05affe80-1920-11eb-958b-c3d02b35b4f2.png)

Apparently the user didn't change anything on the frontend, and if you check the database, the activities are there.

The problem is how we handle the [`ckan.hide-activity-from-users`](https://docs.ckan.org/en/latest/maintaining/configuration.html#ckan-hide-activity-from-users) configuration option. This allows you to define the users that you don't want to show activities from. It defaults to the internal site user as this is generally used to perform bulk internal processes like harvesting and you don't want to flood the activity stream with their operations.

The problem is that we first get the relevant activities from the database and *then* filter these activities in Python based on the users in the config option:

https://github.com/ckan/ckan/blob/ccc582d048b1e2d7ea602884c2d974ff363afce7/ckan/logic/action/get.py#L2665-L2671

The default limit is 31 so if we had 6 initial activities that were visible and at some point the site user goes and eg imports 50 datasets, we will get the most recent 31 activities by the site user from the database, and filter them all out so the none will be shown in the UI.

I could not find any reason for this filter to not be applied at the database query level, which is going to be more performant anyway.


**Steps to reproduce**
Steps to reproduce the behavior:

* Create an organization
* Perform some actions, eg update the org, create a dataset
* Run a harvester, or create more than 30 datasets using the site user API key (or the key of a user listed in `ckan.hide-activity-from-users`)

**Expected behavior**
Activities should not disappear



Discussion:

Resolving patch:
diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -53,23 +53,6 @@
 _text = sqlalchemy.text
 
 
-def _filter_activity_by_user(activity_list, users=[]):
-    '''
-    Return the given ``activity_list`` but with activities from the specified
-    users removed. The users parameters should be a list of ids.
-
-    A *new* filtered list is returned, the given ``activity_list`` itself is
-    not modified.
-    '''
-    if not len(users):
-        return activity_list
-    new_list = []
-    for activity in activity_list:
-        if activity.user_id not in users:
-            new_list.append(activity)
-    return new_list
-
-
 def _activity_stream_get_filtered_users():
     '''
     Get the list of users from the :ref:`ckan.hide_activity_from_users` config
@@ -2499,10 +2482,8 @@ def user_activity_list(context, data_dict):
     offset = data_dict.get('offset', 0)
     limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
-    _activity_objects = model.activity.user_activity_list(
+    activity_objects = model.activity.user_activity_list(
         user.id, limit=limit, offset=offset)
-    activity_objects = _filter_activity_by_user(
-        _activity_objects, _activity_stream_get_filtered_users())
 
     return model_dictize.activity_list_dictize(
         activity_objects, context,
@@ -2553,13 +2534,10 @@ def package_activity_list(context, data_dict):
     offset = int(data_dict.get('offset', 0))
     limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
-    _activity_objects = model.activity.package_activity_list(
-        package.id, limit=limit, offset=offset)
-    if not include_hidden_activity:
-        activity_objects = _filter_activity_by_user(
-            _activity_objects, _activity_stream_get_filtered_users())
-    else:
-        activity_objects = _activity_objects
+    activity_objects = model.activity.package_activity_list(
+        package.id, limit=limit, offset=offset,
+        include_hidden_activity=include_hidden_activity,
+    )
 
     return model_dictize.activity_list_dictize(
         activity_objects, context, include_data=data_dict['include_data'])
@@ -2608,13 +2586,10 @@ def group_activity_list(context, data_dict):
     group_show = logic.get_action('group_show')
     group_id = group_show(context, {'id': group_id})['id']
 
-    _activity_objects = model.activity.group_activity_list(
-        group_id, limit=limit, offset=offset)
-    if not include_hidden_activity:
-        activity_objects = _filter_activity_by_user(
-            _activity_objects, _activity_stream_get_filtered_users())
-    else:
-        activity_objects = _activity_objects
+    activity_objects = model.activity.group_activity_list(
+        group_id, limit=limit, offset=offset,
+        include_hidden_activity=include_hidden_activity,
+    )
 
     return model_dictize.activity_list_dictize(
         activity_objects, context,
@@ -2662,13 +2637,10 @@ def organization_activity_list(context, data_dict):
     org_show = logic.get_action('organization_show')
     org_id = org_show(context, {'id': org_id})['id']
 
-    _activity_objects = model.activity.organization_activity_list(
-        org_id, limit=limit, offset=offset)
-    if not include_hidden_activity:
-        activity_objects = _filter_activity_by_user(
-            _activity_objects, _activity_stream_get_filtered_users())
-    else:
-        activity_objects = _activity_objects
+    activity_objects = model.activity.organization_activity_list(
+        org_id, limit=limit, offset=offset,
+        include_hidden_activity=include_hidden_activity,
+    )
 
     return model_dictize.activity_list_dictize(
         activity_objects, context,
@@ -2698,12 +2670,9 @@ def recently_changed_packages_activity_list(context, data_dict):
     data_dict['include_data'] = False
     limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
-    _activity_objects = \
+    activity_objects = \
         model.activity.recently_changed_packages_activity_list(
             limit=limit, offset=offset)
-    activity_objects = _filter_activity_by_user(
-        _activity_objects,
-        _activity_stream_get_filtered_users())
 
     return model_dictize.activity_list_dictize(
         activity_objects, context,
@@ -3216,13 +3185,11 @@ def dashboard_activity_list(context, data_dict):
 
     # FIXME: Filter out activities whose subject or object the user is not
     # authorized to read.
-    _activity_tuple_objects = model.activity.dashboard_activity_list(
+    activity_objects = model.activity.dashboard_activity_list(
         user_id, limit=limit, offset=offset)
 
-    activity_tuple_list = _filter_activity_by_user(
-        _activity_tuple_objects, _activity_stream_get_filtered_users())
     activity_dicts = model_dictize.activity_list_dictize(
-        activity_tuple_list, context)
+        activity_objects, context)
 
     # Mark the new (not yet seen by user) activities.
     strptime = datetime.datetime.strptime
diff --git a/ckan/model/activity.py b/ckan/model/activity.py
--- a/ckan/model/activity.py
+++ b/ckan/model/activity.py
@@ -15,6 +15,7 @@
     text,
 )
 
+from ckan.common import config
 import ckan.model
 from ckan.model import meta
 from ckan.model import domain_object, types as _types
@@ -164,6 +165,9 @@ def user_activity_list(user_id, limit, offset):
 
     '''
     q = _user_activity_query(user_id, limit + offset)
+
+    q = _filter_activitites_from_users(q)
+
     return _activities_at_offset(q, limit, offset)
 
 
@@ -177,7 +181,8 @@ def _package_activity_query(package_id):
     return q
 
 
-def package_activity_list(package_id, limit, offset):
+def package_activity_list(
+        package_id, limit, offset, include_hidden_activity=False):
     '''Return the given dataset (package)'s public activity stream.
 
     Returns all activities about the given dataset, i.e. where the given
@@ -189,10 +194,14 @@ def package_activity_list(package_id, limit, offset):
 
     '''
     q = _package_activity_query(package_id)
+
+    if not include_hidden_activity:
+        q = _filter_activitites_from_users(q)
+
     return _activities_at_offset(q, limit, offset)
 
 
-def _group_activity_query(group_id):
+def _group_activity_query(group_id, include_hidden_activity=False):
     '''Return an SQLAlchemy query for all activities about group_id.
 
     Returns a query for all activities whose object is either the group itself
@@ -242,10 +251,13 @@ def _group_activity_query(group_id):
         )
     )
 
+    if not include_hidden_activity:
+        q = _filter_activitites_from_users(q)
+
     return q
 
 
-def _organization_activity_query(org_id):
+def _organization_activity_query(org_id, include_hidden_activity=False):
     '''Return an SQLAlchemy query for all activities about org_id.
 
     Returns a query for all activities whose object is either the org itself
@@ -278,11 +290,14 @@ def _organization_activity_query(org_id):
             model.Activity.object_id == org_id
         )
     )
+    if not include_hidden_activity:
+        q = _filter_activitites_from_users(q)
 
     return q
 
 
-def group_activity_list(group_id, limit, offset):
+def group_activity_list(group_id, limit, offset, include_hidden_activity=False):
+
     '''Return the given group's public activity stream.
 
     Returns activities where the given group or one of its datasets is the
@@ -293,11 +308,12 @@ def group_activity_list(group_id, limit, offset):
     etc.
 
     '''
-    q = _group_activity_query(group_id)
+    q = _group_activity_query(group_id, include_hidden_activity)
     return _activities_at_offset(q, limit, offset)
 
 
-def organization_activity_list(group_id, limit, offset):
+def organization_activity_list(
+        group_id, limit, offset, include_hidden_activity=False):
     '''Return the given org's public activity stream.
 
     Returns activities where the given org or one of its datasets is the
@@ -308,7 +324,7 @@ def organization_activity_list(group_id, limit, offset):
     etc.
 
     '''
-    q = _organization_activity_query(group_id)
+    q = _organization_activity_query(group_id, include_hidden_activity)
     return _activities_at_offset(q, limit, offset)
 
 
@@ -402,8 +418,12 @@ def dashboard_activity_list(user_id, limit, offset):
 
     '''
     q = _dashboard_activity_query(user_id, limit + offset)
+
+    q = _filter_activitites_from_users(q)
+
     return _activities_at_offset(q, limit, offset)
 
+
 def _changed_packages_activity_query():
     '''Return an SQLAlchemy query for all changed package activities.
 
@@ -425,4 +445,37 @@ def recently_changed_packages_activity_list(limit, offset):
 
     '''
     q = _changed_packages_activity_query()
+
+    q = _filter_activitites_from_users(q)
+
     return _activities_at_offset(q, limit, offset)
+
+
+def _filter_activitites_from_users(q):
+    '''
+    Adds a filter to an existing query object ot avoid activities from users
+    defined in :ref:`ckan.hide_activity_from_users` (defaults to the site user)
+    '''
+    users_to_avoid = _activity_stream_get_filtered_users()
+    if users_to_avoid:
+        q = q.filter(ckan.model.Activity.user_id.notin_(users_to_avoid))
+
+    return q
+
+
+def _activity_stream_get_filtered_users():
+    '''
+    Get the list of users from the :ref:`ckan.hide_activity_from_users` config
+    option and return a list of their ids. If the config is not specified,
+    returns the id of the site user.
+    '''
+    users = config.get('ckan.hide_activity_from_users')
+    if users:
+        users_list = users.split()
+    else:
+        from ckan.logic import get_action
+        context = {'ignore_auth': True}
+        site_user = get_action('get_site_user')(context)
+        users_list = [site_user.get('name')]
+
+    return ckan.model.User.user_ids_for_name_or_id(users_list)


Test patch:
diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py
--- a/ckan/tests/controllers/test_package.py
+++ b/ckan/tests/controllers/test_package.py
@@ -1864,7 +1864,8 @@ def test_redirect_when_given_id(self, app):
         assert response.headers['location'] == expected_url
 
     def test_redirect_also_with_activity_parameter(self, app):
-        dataset = factories.Dataset()
+        user = factories.User()
+        dataset = factories.Dataset(user=user)
         activity = activity_model.package_activity_list(
             dataset["id"], limit=1, offset=0
         )[0]
diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py
--- a/ckan/tests/logic/action/test_get.py
+++ b/ckan/tests/logic/action/test_get.py
@@ -3164,9 +3164,11 @@ def _create_bulk_package_activities(self, count):
         dataset = factories.Dataset()
         from ckan import model
 
+        user = factories.User()
+
         objs = [
             model.Activity(
-                user_id=None,
+                user_id=user['id'],
                 object_id=dataset["id"],
                 activity_type=None,
                 data=None,
@@ -3396,9 +3398,10 @@ def test_delete_org_by_updating_state(self):
         assert activities[0]["data"]["group"]["title"] == org["title"]
 
     def _create_bulk_user_activities(self, count):
-        user = factories.User()
         from ckan import model
 
+        user = factories.User()
+
         objs = [
             model.Activity(
                 user_id=user["id"],
@@ -3613,9 +3616,11 @@ def _create_bulk_group_activities(self, count):
         group = factories.Group()
         from ckan import model
 
+        user = factories.User()
+
         objs = [
             model.Activity(
-                user_id=None,
+                user_id=user['id'],
                 object_id=group["id"],
                 activity_type=None,
                 data=None,
@@ -3844,9 +3849,11 @@ def _create_bulk_org_activities(self, count):
         org = factories.Organization()
         from ckan import model
 
+        user = factories.User()
+
         objs = [
             model.Activity(
-                user_id=None,
+                user_id=user['id'],
                 object_id=org["id"],
                 activity_type=None,
                 data=None,
@@ -4013,9 +4020,11 @@ def test_delete_dataset(self):
     def _create_bulk_package_activities(self, count):
         from ckan import model
 
+        user = factories.User()
+
         objs = [
             model.Activity(
-                user_id=None,
+                user_id=user['id'],
                 object_id=None,
                 activity_type="new_package",
                 data=None,