Repository: StackStorm/st2

Issue:
st2 execution tail CLI command should support nested (workflow) executions
Right now "st2 execution tail" CLI command only supports / works correctly with simple (non-workflow, non-nested) executions.

We should add support for more complex workflow executions to it.

This will require us to implement better filtering and tracking of child executions inside the CLI commands (same thing will need to be done on the WebUI side when adding support for streaming output there).


Discussion:
+1

We routinely write workflows with many layers of nesting (mainly because we reuse a lot of workflows).

Would be GREAT to tail the output instead of `while true; do st2 execution get xxx; sleep 5; done` (i've gotten really good at typing that)

Resolving patch:
diff --git a/st2client/st2client/commands/action.py b/st2client/st2client/commands/action.py
--- a/st2client/st2client/commands/action.py
+++ b/st2client/st2client/commands/action.py
@@ -448,7 +448,7 @@ def _get_execution_result(self, execution, action_exec_mgr, args, **kwargs):
         if args.tail:
             # Start tailing new execution
             print('Tailing execution "%s"' % (str(execution.id)))
-            execution_manager = self.manager
+            execution_manager = self.app.client.managers['LiveAction']
             stream_manager = self.app.client.managers['Stream']
             ActionExecutionTailCommand.tail_execution(execution=execution,
                                                       execution_manager=execution_manager,
@@ -1476,12 +1476,22 @@ def tail_execution(cls, execution_manager, stream_manager, execution, output_typ
                        include_metadata=False, **kwargs):
         execution_id = str(execution.id)
 
+        # Indicates if the execution we are tailing is a child execution in a workflow
+        context = cls.get_normalized_context_execution_task_event(execution.__dict__)
+        has_parent_attribute = bool(getattr(execution, 'parent', None))
+        has_parent_execution_id = bool(context['parent_execution_id'])
+
+        is_tailing_execution_child_execution = bool(has_parent_attribute or
+                                                    has_parent_execution_id)
+
         # Note: For non-workflow actions child_execution_id always matches parent_execution_id so
         # we don't need to do any other checks to determine if executions represents a workflow
-        # action
+        # action.
         parent_execution_id = execution_id  # noqa
 
-        # Execution has already finished
+        # Execution has already finished show existing output.
+        # NOTE: This doesn't recurse down into child executions if user is tailing a workflow
+        # execution
         if execution.status in LIVEACTION_COMPLETED_STATES:
             output = execution_manager.get_output(execution_id=execution_id,
                                                   output_type=output_type)
@@ -1489,15 +1499,30 @@ def tail_execution(cls, execution_manager, stream_manager, execution, output_typ
             print('Execution %s has completed (status=%s).' % (execution_id, execution.status))
             return
 
-        events = ['st2.execution__update', 'st2.execution.output__create']
+        # We keep track of all the workflow executions which could contain children.
+        # For simplicity, we simply keep track of all the execution ids which belong to a
+        # particular workflow.
+        workflow_execution_ids = set([parent_execution_id])
+
+        # Retrieve parent execution object so we can keep track of any existing children
+        # executions (only applies to already running executions).
+        filters = {
+            'params': {
+                'include_attributes': 'id,children'
+            }
+        }
+        execution = execution_manager.get_by_id(id=execution_id, **filters)
+
+        children_execution_ids = getattr(execution, 'children', [])
+        workflow_execution_ids.update(children_execution_ids)
 
+        events = ['st2.execution__update', 'st2.execution.output__create']
         for event in stream_manager.listen(events, **kwargs):
             status = event.get('status', None)
             is_execution_event = status is not None
 
-            # NOTE: Right now only a single level deep / nested workflows are supported
             if is_execution_event:
-                context = cls.get_normalized_context_execution_task_event(event=event)
+                context = cls.get_normalized_context_execution_task_event(event)
                 task_execution_id = context['execution_id']
                 task_name = context['task_name']
                 task_parent_execution_id = context['parent_execution_id']
@@ -1505,6 +1530,16 @@ def tail_execution(cls, execution_manager, stream_manager, execution, output_typ
                 # An execution is considered a child execution if it has parent execution id
                 is_child_execution = bool(task_parent_execution_id)
 
+                # Ignore executions which are not part of the execution we are tailing
+                if is_child_execution and not is_tailing_execution_child_execution:
+                    if task_parent_execution_id not in workflow_execution_ids:
+                        continue
+                else:
+                    if task_execution_id not in workflow_execution_ids:
+                        continue
+
+                workflow_execution_ids.add(task_execution_id)
+
                 if is_child_execution:
                     if status == LIVEACTION_STATUS_RUNNING:
                         print('Child execution (task=%s) %s has started.' % (task_name,
@@ -1515,12 +1550,19 @@ def tail_execution(cls, execution_manager, stream_manager, execution, output_typ
                         print('')
                         print('Child execution (task=%s) %s has finished (status=%s).' %
                               (task_name, task_execution_id, status))
-                        continue
+
+                        if is_tailing_execution_child_execution:
+                            # User is tailing a child execution inside a workflow, stop the command.
+                            break
+                        else:
+                            continue
                     else:
                         # We don't care about other child events so we simply skip then
                         continue
                 else:
-                    if status == LIVEACTION_STATUS_RUNNING:
+                    # NOTE: In some situations execution update event with "running" status is
+                    # dispatched twice so we ignore any duplicated events
+                    if status == LIVEACTION_STATUS_RUNNING and not event.get('children', []):
                         print('Execution %s has started.' % (execution_id))
                         print('')
                         continue
@@ -1533,6 +1575,11 @@ def tail_execution(cls, execution_manager, stream_manager, execution, output_typ
                         # We don't care about other execution events
                         continue
 
+            # Ignore events for executions which don't belong to the one we are tailing
+            event_execution_id = event['execution_id']
+            if event_execution_id not in workflow_execution_ids:
+                continue
+
             # Filter on output_type if provided
             event_output_type = event.get('output_type', None)
             if output_type != 'all' and output_type and (event_output_type != output_type):
@@ -1547,8 +1594,7 @@ def tail_execution(cls, execution_manager, stream_manager, execution, output_typ
     @classmethod
     def get_normalized_context_execution_task_event(cls, event):
         """
-        Return a dictionary with normalized context attributes for Action-Chain and Mistral
-        workflows.
+        Return a dictionary with normalized context attributes for execution event or object.
         """
         context = event.get('context', {})
 
@@ -1563,6 +1609,10 @@ def get_normalized_context_execution_task_event(cls, event):
             result['parent_execution_id'] = context.get('parent', {}).get('execution_id', None)
             result['execution_id'] = event['id']
             result['task_name'] = context.get('mistral', {}).get('task_name', 'unknown')
+        elif 'orquesta' in context:
+            result['parent_execution_id'] = context.get('parent', {}).get('execution_id', None)
+            result['execution_id'] = event['id']
+            result['task_name'] = context.get('orquesta', {}).get('task_name', 'unknown')
         else:
             # Action chain workflow
             result['parent_execution_id'] = context.get('parent', {}).get('execution_id', None)


Test patch:
diff --git a/st2client/tests/unit/test_execution_tail_command.py b/st2client/tests/unit/test_execution_tail_command.py
--- a/st2client/tests/unit/test_execution_tail_command.py
+++ b/st2client/tests/unit/test_execution_tail_command.py
@@ -197,7 +197,7 @@
 }
 
 MOCK_LIVEACTION_4_CHILD_1_OUTPUT_2 = {
-    'execution_id': 'mistral',
+    'execution_id': 'idmistralchild1',
     'timestamp': '1505732598',
     'output_type': 'stderr',
     'data': 'line mistral 5\n'
@@ -244,7 +244,7 @@
 }
 
 MOCK_LIVEACTION_4_CHILD_2_OUTPUT_1 = {
-    'execution_id': 'idchild2',
+    'execution_id': 'idmistralchild2',
     'timestamp': '1505732598',
     'output_type': 'stdout',
     'data': 'line mistral 100\n'
@@ -532,6 +532,23 @@ def test_tail_double_nested_mistral_workflow_execution(self, mock_stream_manager
             MOCK_LIVEACTION_4_CHILD_1_1_OUTPUT_1,
             MOCK_LIVEACTION_4_CHILD_1_1_OUTPUT_2,
 
+            # Another execution has started, this output should not be included
+            MOCK_LIVEACTION_3_RUNNING,
+
+            # Child task 1 started running
+            MOCK_LIVEACTION_3_CHILD_1_RUNNING,
+
+            # Output produced by the child task
+            MOCK_LIVEACTION_3_CHILD_1_OUTPUT_1,
+            MOCK_LIVEACTION_3_CHILD_1_OUTPUT_2,
+
+            # Child task 1 finished
+            MOCK_LIVEACTION_3_CHILD_1_SUCCEEDED,
+
+            # Parent workflow task finished
+            MOCK_LIVEACTION_3_SUCCEDED,
+            # End another execution
+
             # Child task 1 has finished
             MOCK_LIVEACTION_4_CHILD_1_1_SUCCEEDED,
 
@@ -585,5 +602,56 @@ def test_tail_double_nested_mistral_workflow_execution(self, mock_stream_manager
 
 Execution idfoo4 has completed (status=succeeded).
 """.lstrip()
+
+        self.assertEqual(stdout, expected_result)
+        self.assertEqual(stderr, '')
+
+    @mock.patch.object(
+        httpclient.HTTPClient, 'get',
+        mock.MagicMock(return_value=base.FakeResponse(json.dumps(MOCK_LIVEACTION_4_CHILD_2_RUNNING),
+                                                     200, 'OK')))
+    @mock.patch('st2client.client.StreamManager', autospec=True)
+    def test_tail_child_execution_directly(self, mock_stream_manager):
+        argv = ['execution', 'tail', 'idfoo4']
+
+        MOCK_EVENTS = [
+            # Child task 2 started running
+            MOCK_LIVEACTION_4_CHILD_2_RUNNING,
+
+            # Output produced by child task
+            MOCK_LIVEACTION_4_CHILD_2_OUTPUT_1,
+
+            # Other executions should not interfere
+            # Child task 1 started running
+            MOCK_LIVEACTION_3_CHILD_1_RUNNING,
+
+            # Child task 1 finished (sub workflow)
+            MOCK_LIVEACTION_4_CHILD_1_SUCCEEDED,
+
+            # Child task 2 finished
+            MOCK_LIVEACTION_4_CHILD_2_TIMED_OUT
+        ]
+
+        mock_cls = mock.Mock()
+        mock_cls.listen = mock.Mock()
+        mock_listen_generator = mock.Mock()
+        mock_listen_generator.return_value = MOCK_EVENTS
+        mock_cls.listen.side_effect = mock_listen_generator
+        mock_stream_manager.return_value = mock_cls
+
+        self.assertEqual(self.shell.run(argv), 0)
+        self.assertEqual(mock_listen_generator.call_count, 1)
+
+        stdout = self.stdout.getvalue()
+        stderr = self.stderr.getvalue()
+
+        expected_result = """
+Child execution (task=task_2) idmistralchild2 has started.
+
+line mistral 100
+
+Child execution (task=task_2) idmistralchild2 has finished (status=timeout).
+""".lstrip()
+
         self.assertEqual(stdout, expected_result)
         self.assertEqual(stderr, '')