Repository: StackStorm/st2

Issue:
Optimize deleting old action execution data
Using `st2-purge-executions` or the `st2garbagecollector` process, all `ActionExecution` objects are fully loaded into memory, resulting in extremely high memory usage when cleaning up execution data in cases where actions have a lot of data as input and running the actions very frequently.

In our case we're fetching network measurements from a number of devices on a 5 minute interval, which generates a lot of data by a large amount of actions.

The troublesome line is the following: [https://github.com/StackStorm/st2/blob/6df54d7f3703dad890b83b7feed4b43584b2861a/st2common/st2common/garbage_collection/executions.py#L82](url).
Here, all `ActionExecution` objects matching the filters are unnecessarily loaded in to memory; only the ids are needed at [https://github.com/StackStorm/st2/blob/6df54d7f3703dad890b83b7feed4b43584b2861a/st2common/st2common/garbage_collection/executions.py#L108](url). This causes the deletion process to be slow and memory intensive in our case.

Loading more `ActionExecution` data into memory than the available system memory leads to the garbage collection never deleting everything, because all data is fetched before starting to delete data.

The process would run smoother and safer if only (necessary) data i.e. the ids of the `ActionExecution`s were fetched, rather than all of the data.


Discussion:
Thanks for reporting this.

That's indeed a valid point and good catch - we should only load the fields we need.

The fix should be pretty easy and straight forward and I will look into it.

Resolving patch:
diff --git a/st2common/st2common/garbage_collection/executions.py b/st2common/st2common/garbage_collection/executions.py
--- a/st2common/st2common/garbage_collection/executions.py
+++ b/st2common/st2common/garbage_collection/executions.py
@@ -79,7 +79,9 @@ def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False)
     # 1. Delete ActionExecutionDB objects
     try:
         # Note: We call list() on the query set object because it's lazyily evaluated otherwise
-        to_delete_execution_dbs = list(ActionExecution.query(**exec_filters))
+        to_delete_execution_dbs = list(ActionExecution.query(only_fields=['id'],
+                                                             no_dereference=True,
+                                                             **exec_filters))
         deleted_count = ActionExecution.delete_by_query(**exec_filters)
     except InvalidQueryError as e:
         msg = ('Bad query (%s) used to delete execution instances: %s'
@@ -121,8 +123,12 @@ def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False)
     else:
         logger.info('Deleted %s execution output objects' % (deleted_count))
 
-    zombie_execution_instances = len(ActionExecution.query(**exec_filters))
-    zombie_liveaction_instances = len(LiveAction.query(**liveaction_filters))
+    zombie_execution_instances = len(ActionExecution.query(only_fields=['id'],
+                                                           no_dereference=True,
+                                                           **exec_filters))
+    zombie_liveaction_instances = len(LiveAction.query(only_fields=['id'],
+                                                      no_dereference=True,
+                                                       **liveaction_filters))
 
     if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0):
         logger.error('Zombie execution instances left: %d.', zombie_execution_instances)
diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py
--- a/st2common/st2common/models/db/__init__.py
+++ b/st2common/st2common/models/db/__init__.py
@@ -340,6 +340,8 @@ def query(self, *args, **filters):
         limit = filters.pop('limit', None)
         order_by = filters.pop('order_by', None)
         exclude_fields = filters.pop('exclude_fields', None)
+        only_fields = filters.pop('only_fields', None)
+        no_dereference = filters.pop('no_dereference', None)
 
         order_by = order_by or []
         exclude_fields = exclude_fields or []
@@ -357,6 +359,12 @@ def query(self, *args, **filters):
         if exclude_fields:
             result = result.exclude(*exclude_fields)
 
+        if only_fields:
+            result = result.only(*only_fields)
+
+        if no_dereference:
+            result = result.no_dereference()
+
         result = result.order_by(*order_by)
         result = result[offset:eop]
         log_query_and_profile_data_for_queryset(queryset=result)
diff --git a/st2common/st2common/models/utils/profiling.py b/st2common/st2common/models/utils/profiling.py
--- a/st2common/st2common/models/utils/profiling.py
+++ b/st2common/st2common/models/utils/profiling.py
@@ -77,6 +77,7 @@ def log_query_and_profile_data_for_queryset(queryset):
     limit = getattr(queryset, '_limit', None)
     collection = getattr(queryset, '_collection', None)
     collection_name = getattr(collection, 'name', None)
+    only_fields = getattr(queryset, 'only_fields', None)
 
     # Note: We need to clone the queryset when using explain because explain advances the cursor
     # internally which changes the function result
@@ -87,7 +88,8 @@ def log_query_and_profile_data_for_queryset(queryset):
         mongo_shell_query = construct_mongo_shell_query(mongo_query=mongo_query,
                                                         collection_name=collection_name,
                                                         ordering=ordering,
-                                                        limit=limit)
+                                                        limit=limit,
+                                                        only_fields=only_fields)
         extra = {'mongo_query': mongo_query, 'mongo_shell_query': mongo_shell_query}
         LOG.debug('MongoDB query: %s' % (mongo_shell_query), extra=extra)
         LOG.debug('MongoDB explain data: %s' % (explain_info))
@@ -95,7 +97,8 @@ def log_query_and_profile_data_for_queryset(queryset):
     return queryset
 
 
-def construct_mongo_shell_query(mongo_query, collection_name, ordering, limit):
+def construct_mongo_shell_query(mongo_query, collection_name, ordering, limit,
+                                only_fields=None):
     result = []
 
     # Select collection
@@ -109,6 +112,16 @@ def construct_mongo_shell_query(mongo_query, collection_name, ordering, limit):
         filter_predicate = ''
 
     part = 'find({filter_predicate})'.format(filter_predicate=filter_predicate)
+
+    # Include only fields (projection)
+    if only_fields:
+        projection_items = ['%s: 1' % (field) for field in only_fields]
+        projection = ', '.join(projection_items)
+        part = 'find({filter_predicate}, {{{projection}}})'.format(
+            filter_predicate=filter_predicate, projection=projection)
+    else:
+        part = 'find({filter_predicate})'.format(filter_predicate=filter_predicate)
+
     result.append(part)
 
     # Include ordering info (if any)


Test patch:
diff --git a/st2common/tests/unit/test_persistence.py b/st2common/tests/unit/test_persistence.py
--- a/st2common/tests/unit/test_persistence.py
+++ b/st2common/tests/unit/test_persistence.py
@@ -199,3 +199,61 @@ def test_escaped_field(self):
         self.assertIsNotNone(obj3)
         self.assertEqual(obj3.id, obj2.id)
         self.assertDictEqual(obj3.context, context)
+
+    def test_query_only_fields(self):
+        count = 5
+        ts = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
+        for i in range(count):
+            category = 'type1'
+            obj = FakeModelDB(name='test-%s' % (i), timestamp=ts, category=category)
+            self.access.add_or_update(obj)
+
+        model_dbs = FakeModel.query()
+        self.assertEqual(model_dbs[0].name, 'test-0')
+        self.assertEqual(model_dbs[0].timestamp, ts)
+        self.assertEqual(model_dbs[0].category, 'type1')
+
+        # only id
+        model_dbs = FakeModel.query(only_fields=['id'])
+        self.assertTrue(model_dbs[0].id)
+        self.assertEqual(model_dbs[0].name, None)
+        self.assertEqual(model_dbs[0].timestamp, None)
+        self.assertEqual(model_dbs[0].category, None)
+
+        # only name - note: id is always included
+        model_dbs = FakeModel.query(only_fields=['name'])
+        self.assertTrue(model_dbs[0].id)
+        self.assertEqual(model_dbs[0].name, 'test-0')
+        self.assertEqual(model_dbs[0].timestamp, None)
+        self.assertEqual(model_dbs[0].category, None)
+
+    def test_query_exclude_fields(self):
+        count = 5
+        ts = date_utils.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
+        for i in range(count):
+            category = 'type1'
+            obj = FakeModelDB(name='test-2-%s' % (i), timestamp=ts, category=category)
+            self.access.add_or_update(obj)
+
+        model_dbs = FakeModel.query()
+        self.assertEqual(model_dbs[0].name, 'test-2-0')
+        self.assertEqual(model_dbs[0].timestamp, ts)
+        self.assertEqual(model_dbs[0].category, 'type1')
+
+        model_dbs = FakeModel.query(exclude_fields=['name'])
+        self.assertTrue(model_dbs[0].id)
+        self.assertEqual(model_dbs[0].name, None)
+        self.assertEqual(model_dbs[0].timestamp, ts)
+        self.assertEqual(model_dbs[0].category, 'type1')
+
+        model_dbs = FakeModel.query(exclude_fields=['name', 'timestamp'])
+        self.assertTrue(model_dbs[0].id)
+        self.assertEqual(model_dbs[0].name, None)
+        self.assertEqual(model_dbs[0].timestamp, None)
+        self.assertEqual(model_dbs[0].category, 'type1')
+
+        model_dbs = FakeModel.query(exclude_fields=['name', 'timestamp', 'category'])
+        self.assertTrue(model_dbs[0].id)
+        self.assertEqual(model_dbs[0].name, None)
+        self.assertEqual(model_dbs[0].timestamp, None)
+        self.assertEqual(model_dbs[0].category, None)
diff --git a/st2common/tests/unit/test_purge_executions.py b/st2common/tests/unit/test_purge_executions.py
--- a/st2common/tests/unit/test_purge_executions.py
+++ b/st2common/tests/unit/test_purge_executions.py
@@ -16,6 +16,7 @@
 import copy
 from datetime import timedelta
 
+import mock
 import bson
 
 from st2common import log as logging
@@ -264,6 +265,27 @@ def test_purge_incomplete(self):
         stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
         self.assertEqual(len(stderr_dbs), 0)
 
+    @mock.patch('st2common.garbage_collection.executions.LiveAction')
+    @mock.patch('st2common.garbage_collection.executions.ActionExecution')
+    def test_purge_executions_whole_model_is_not_loaded_in_memory(self, mock_ActionExecution,
+                                                                  mock_LiveAction):
+        # Verify that whole execution objects are not loaded in memory and we just retrieve the
+        # id field
+        self.assertEqual(mock_ActionExecution.query.call_count, 0)
+        self.assertEqual(mock_LiveAction.query.call_count, 0)
+        now = date_utils.get_datetime_utc_now()
+        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)
+
+        self.assertEqual(mock_ActionExecution.query.call_count, 2)
+        self.assertEqual(mock_LiveAction.query.call_count, 1)
+
+        self.assertEqual(mock_ActionExecution.query.call_args_list[0][1]['only_fields'], ['id'])
+        self.assertTrue(mock_ActionExecution.query.call_args_list[0][1]['no_dereference'])
+        self.assertEqual(mock_ActionExecution.query.call_args_list[1][1]['only_fields'], ['id'])
+        self.assertTrue(mock_ActionExecution.query.call_args_list[1][1]['no_dereference'])
+        self.assertEqual(mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id'])
+        self.assertTrue(mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
+
     def _insert_mock_stdout_and_stderr_objects_for_execution(self, execution_id, count=5):
         execution_id = str(execution_id)