Repository: avocado-framework/avocado

Issue:
NRunner plugin: support failfast feature
The failfast feature does not currently work with the nrunner plugin.  A job such as:

```python
#!/usr/bin/env python3

import sys
from avocado.core.job import Job

config = {'run.references': ['/bin/true', '/bin/false', '/bin/true', '/bin/true'],
          'run.test_runner': 'nrunner',
          'run.failfast': 'on'}

with Job(config) as j:
    sys.exit(j.run())
```

Should produce:

```
JOB ID     : dfa077af08bee71d82f4741b2ca5682ac94eef38
JOB LOG    : /home/cleber/avocado/job-results/job-2020-06-02T01.03-dfa077a/job.log
 (1/4) /bin/true: PASS (0.01 s)
 (2/4) /bin/false: FAIL: (0.06 s)
Interrupting job (failfast).
RESULTS    : PASS 1 | ERROR 0 | FAIL 1 | SKIP 2 | WARN 0 | INTERRUPT 0 | CANCEL 0
```



Discussion:
@clebergnu With the new architecture things are more distributed now and tasks can be running on a container or remote host.

Thinking about a `PodmanSpawner` or a `RemoteSpawner`, I can imagine both stopping the virtual machine or the container to avoid remaining tasks to be running after the "Interrupted" is received. Or we could have remote "agents/watchdogs". There are many ways to go here.

@clebergnu How do you imagine this possible implantation?
I think the main piece here is the scheduler. It will have a high-level view of each test execution and should know its status.

We should make the testers aware that running tests in parallel with the `failfast` parameter will not have the same "performance" when compared with today's implementation.

The implementation I can think of here is into the scheduler. As soon as one test fails, it is the scheduler's responsibility to signal to the running tests to cancel its execution in the best effort. Some tests may take some time to return its confirmation of cancelation because they were into high I/O, or any other reason, meaning the `failfast` will not be as fast as the current implementation, but a best effort to finish the job execution.
Part of #4802 

Resolving patch:
diff --git a/avocado/core/task/statemachine.py b/avocado/core/task/statemachine.py
--- a/avocado/core/task/statemachine.py
+++ b/avocado/core/task/statemachine.py
@@ -47,6 +47,52 @@ async def complete(self):
                            self._ready, self._started])
         return not pending
 
+    async def abort(self, status_reason=None):
+        """Abort all non-started tasks.
+
+        This method will move all non-started tasks to finished with a specific
+        reason.
+
+        :param status_reason: string reason. Optional.
+        """
+        await self.abort_queue('requested', status_reason)
+        await self.abort_queue('triaging', status_reason)
+        await self.abort_queue('ready', status_reason)
+
+    async def abort_queue(self, queue_name, status_reason=None):
+        """Abort all tasks inside a specific queue adding a status reason.
+
+        :param queue_name: a string with the queue name.
+        :param status_reason: string reason. Optional.
+        """
+        to_remove = []
+        async with self._lock:
+            queue = getattr(self, queue_name)
+            for _ in range(len(queue)):
+                if queue_name == 'requested':
+                    runtime_task = queue.popleft()
+                else:
+                    runtime_task = queue.pop(0)
+                to_remove.append(runtime_task)
+
+        for task in to_remove:
+            await self.finish_task(task, status_reason)
+
+    async def finish_task(self, runtime_task, status_reason=None):
+        """Include a task to the finished queue with a specific reason.
+
+        This method is assuming that you have removed (pop) the task from the
+        original queue.
+
+        :param runtime_task: A running task object.
+        :param status_reason: string reason. Optional.
+        """
+        async with self._lock:
+            if runtime_task not in self.finished:
+                if status_reason:
+                    runtime_task.status = status_reason
+                self.finished.append(runtime_task)
+
 
 class Worker:
 
@@ -84,7 +130,9 @@ async def check_finished_dependencies(runtime_task):
                 for finished_rt_task in self._state_machine.finished:
                     if (finished_rt_task.task is task and
                             finished_rt_task.status == 'FAILED ON TRIAGE'):
-                        await fail_on_triage(runtime_task)
+
+                        await self._state_machine.finish_task(runtime_task,
+                                                              "FAILED ON TRIAGE")
                         return
                 # from here, this dependency `task` ran, so, let's check
                 # the its latest data in the status repo
@@ -99,16 +147,12 @@ async def check_finished_dependencies(runtime_task):
                     return
                 # if this dependency task failed, skip the parent task
                 if latest_task_data['result'] not in ['pass']:
-                    await fail_on_triage(runtime_task)
+                    await self._state_machine.finish_task(runtime_task,
+                                                          "FAILED ON TRIAGE")
                     return
             # everything is fine!
             return True
 
-        async def fail_on_triage(runtime_task):
-            async with self._state_machine.lock:
-                self._state_machine.finished.append(runtime_task)
-                runtime_task.status = 'FAILED ON TRIAGE'
-
         try:
             async with self._state_machine.lock:
                 runtime_task = self._state_machine.triaging.pop(0)
@@ -121,7 +165,8 @@ async def fail_on_triage(runtime_task):
             requirements_ok = (
                     await self._spawner.check_task_requirements(runtime_task))
             if not requirements_ok:
-                await fail_on_triage(runtime_task)
+                await self._state_machine.finish_task(runtime_task,
+                                                      "FAILED ON TRIAGE")
                 return
 
         # handle task dependencies
@@ -177,8 +222,8 @@ async def start(self):
             async with self._state_machine.lock:
                 self._state_machine.started.append(runtime_task)
         else:
-            async with self._state_machine.lock:
-                self._state_machine.finished.append(runtime_task)
+            await self._state_machine.finish_task(runtime_task,
+                                                  "FAILED ON START")
 
     async def monitor(self):
         """Reads from started, moves into finished."""
@@ -199,9 +244,12 @@ async def monitor(self):
                 runtime_task.status = 'FINISHED'
             except asyncio.TimeoutError:
                 runtime_task.status = 'FINISHED W/ TIMEOUT'
+        failfast = runtime_task.task.runnable.config.get('run.failfast')
+        result_stats = self._state_machine._status_repo.result_stats
+        if failfast and 'fail' in result_stats:
+            await self._state_machine.abort("FAILFAST is enabled")
 
-        async with self._state_machine.lock:
-            self._state_machine.finished.append(runtime_task)
+        await self._state_machine.finish_task(runtime_task)
 
     async def run(self):
         """Pushes Tasks forward and makes them do something with their lives."""


Test patch:
diff --git a/selftests/functional/test_nrunner.py b/selftests/functional/test_nrunner.py
--- a/selftests/functional/test_nrunner.py
+++ b/selftests/functional/test_nrunner.py
@@ -24,6 +24,27 @@ def test_custom_exit_codes(self):
         with Job.from_config(job_config=config) as job:
             self.assertEqual(job.run(), 0)
 
+    @skipUnlessPathExists('/bin/false')
+    @skipUnlessPathExists('/bin/true')
+    def test_failfast(self):
+        status_server = "127.0.0.1:%u" % find_free_port()
+        config = {'run.references': ['/bin/true',
+                                     '/bin/false',
+                                     '/bin/true',
+                                     '/bin/true'],
+                  'run.test_runner': 'nrunner',
+                  'run.failfast': True,
+                  'nrunner.shuffle': False,
+                  'nrunner.status_server_listen': status_server,
+                  'nrunner.status_server_uri': status_server,
+                  'nrunner.max_parallel_tasks': 1}
+        with Job.from_config(job_config=config) as job:
+            self.assertEqual(job.run(), 1)
+            self.assertEqual(job.result.passed, 1)
+            self.assertEqual(job.result.errors, 0)
+            self.assertEqual(job.result.failed, 1)
+            self.assertEqual(job.result.skipped, 2)
+
 
 class RunnableRun(unittest.TestCase):