Repository: project-receptor/receptor-satellite

Issue:
Implement v2 playbook_run_finished
[response] run playbook finished
This message is sent for each host individually when a Playbook execution on a given host finishes.

```
{
    "type": "playbook_run_finished",
    // id of the remediation execution
    "playbook_run_id": "a30f1d7c-ba75-465b-a217-63f3f553836f",
    "host": "01.example.com",
    "status": "success" || "failure" || "canceled",
    "version": 2,
    "connection_code": 0 (success) || 1 (error),
    "execution_code": 0 (success) || number (return code from ansible) || null (in case connection code is 1)
}
```

[response] run playbook finished ad1
https://bugzilla.redhat.com/show_bug.cgi?id=1833039
RHCLOUD-5370

There are certain failure situation that we know may happen (e.g. "This host is not known by Satellite"). Currently there is no way for sat-receptor to indicate these other than adding the error message into the "console" text field.

We should introduce code/error field with well-defined values so that Remediations can understand the semantics of what the problem was without having to parse the text field.

Extend playbook_run_finished
We can add a second type of failure:
- Satellite can’t reach host
- Satellite connected, but the run failed (e.g. problem in ansible playbook)


Comments:

Patch:
diff --git a/receptor_satellite/response/messages.py b/receptor_satellite/response/messages.py
index 23ee860..b16de05 100644
--- a/receptor_satellite/response/messages.py
+++ b/receptor_satellite/response/messages.py
@@ -30,12 +30,21 @@ def playbook_run_cancel_ack(playbook_run_id, status):
     }
 
 
-def playbook_run_finished(host, playbook_run_id, result=RESULT_SUCCESS):
+def playbook_run_finished(
+    host,
+    playbook_run_id,
+    result=RESULT_SUCCESS,
+    connection_error=False,
+    execution_code=0,
+):
     return {
+        "version": 2,
         "type": "playbook_run_finished",
         "playbook_run_id": playbook_run_id,
         "host": host,
         "status": result,
+        "connection_code": 1 if connection_error else 0,
+        "execution_code": None if connection_error else execution_code,
     }
 
 
diff --git a/receptor_satellite/response/response_queue.py b/receptor_satellite/response/response_queue.py
index 4cf0f45..dc5dab0 100644
--- a/receptor_satellite/response/response_queue.py
+++ b/receptor_satellite/response/response_queue.py
@@ -15,9 +15,18 @@ class ResponseQueue:
         )
 
     def playbook_run_finished(
-        self, host, playbook_run_id, result=constants.RESULT_SUCCESS
+        self,
+        host,
+        playbook_run_id,
+        result=constants.RESULT_SUCCESS,
+        connection_error=False,
+        exit_code=0,
     ):
-        self.queue.put(messages.playbook_run_finished(host, playbook_run_id, result))
+        self.queue.put(
+            messages.playbook_run_finished(
+                host, playbook_run_id, result, connection_error, exit_code
+            )
+        )
 
     def playbook_run_completed(
         self, playbook_run_id, status, connection_error=None, infrastructure_error=None
diff --git a/receptor_satellite/worker.py b/receptor_satellite/worker.py
index e16b45b..5140766 100644
--- a/receptor_satellite/worker.py
+++ b/receptor_satellite/worker.py
@@ -9,7 +9,8 @@ import receptor_satellite.response.constants as constants
 from .run_monitor import run_monitor
 
 # EXCEPTION means failure between capsule and the target host
-EXIT_STATUS_RE = re.compile(r"Exit status: ([0-9]+|EXCEPTION)", re.MULTILINE)
+EXIT_STATUS_RE = re.compile(r"Exit status: (([0-9]+)|EXCEPTION)", re.MULTILINE)
+UNREACHABLE_RE = re.compile(r"unreachable=[1-9][0-9]*")
 
 
 def receptor_export(func):
@@ -93,6 +94,8 @@ class Host:
         self.sequence = 0
         self.since = None if run.config.text_update_full else 0.0
         self.result = None
+        self.last_recap_line = ""
+        self.host_recap_re = re.compile(f"^.*{name}.*ok=[0-9]+")
 
     def mark_as_failed(self, message):
         queue = self.run.queue
@@ -119,16 +122,32 @@ class Host:
                     self.name, self.run.playbook_run_id, last_output, self.sequence
                 )
                 self.sequence += 1
+
+            possible_recaps = list(
+                filter(
+                    lambda x: re.match(self.host_recap_re, x), last_output.split("\n")
+                )
+            )
+            if len(possible_recaps) > 0:
+                self.last_recap_line = possible_recaps.pop()
+
             if body["complete"]:
+                connection_error = re.search(UNREACHABLE_RE, self.last_recap_line)
                 result = constants.HOST_RESULT_FAILURE
                 matches = re.findall(EXIT_STATUS_RE, last_output)
+                exit_code = None
                 # This means the job was already running on the host
                 if matches:
-                    # If exitcode is 0
-                    if matches[0] == "0":
-                        result = constants.HOST_RESULT_SUCCESS
-                    elif self.run.cancelled:
-                        result = constants.HOST_RESULT_CANCEL
+                    code = matches[0][1]
+                    # If there was an exit code
+                    if code != "":
+                        exit_code = int(code)
+                        if exit_code == 0:
+                            result = constants.HOST_RESULT_SUCCESS
+                        elif self.run.cancelled:
+                            result = constants.HOST_RESULT_CANCEL
+                        else:
+                            result = constants.HOST_RESULT_FAILURE
                 elif self.run.cancelled:
                     result = constants.HOST_RESULT_CANCEL
                 else:
@@ -139,6 +158,8 @@ class Host:
                     constants.HOST_RESULT_FAILURE
                     if result == constants.HOST_RESULT_INFRA_FAILURE
                     else result,
+                    connection_error or result == constants.HOST_RESULT_INFRA_FAILURE,
+                    exit_code,
                 )
                 self.result = result
                 break
diff --git a/setup.cfg b/setup.cfg
index 849e449..3c1f6ae 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -21,6 +21,7 @@ ignore=E201,E203,E221,E225,E231,E241,E251,E261,E265,E303,W291,W391,W293,E731,F40
 exclude=.tox,venv,awx/lib/site-packages,awx/plugins/inventory,awx/ui,awx/api/urls.py,awx/main/migrations,awx/main/south_migrations,awx/main/tests/data,node_modules/,awx/projects/,tools/docker,awx/settings/local_*.py,installer/openshift/settings.py,build/,installer/
 per-file-ignores =
     tests/constants.py:E501
+    tests/test_run.py:E501
 
 [metadata]
 license_file=LICENSE.md


Test patch:
diff --git a/tests/test_response_queue.py b/tests/test_response_queue.py
index 170a8dc..d53ddef 100644
--- a/tests/test_response_queue.py
+++ b/tests/test_response_queue.py
@@ -13,7 +13,10 @@ PLAYBOOK_RUN_COMPLETED_TEST_CASES = [
     ),
     (
         ("some-uuid", constants.RESULT_FAILURE, None, None),
-        messages.playbook_run_completed("some-uuid", constants.RESULT_FAILURE,),
+        messages.playbook_run_completed(
+            "some-uuid",
+            constants.RESULT_FAILURE,
+        ),
     ),
     (
         ("some-uuid", constants.RESULT_CANCEL, None, None),
diff --git a/tests/test_run.py b/tests/test_run.py
index 329fcb1..cfa658b 100644
--- a/tests/test_run.py
+++ b/tests/test_run.py
@@ -125,7 +125,11 @@ def poll_with_retries_scenario(request, base_scenario):
 
 @pytest.mark.asyncio
 async def test_poll_with_retries(poll_with_retries_scenario):
-    (queue, host, param,) = poll_with_retries_scenario
+    (
+        queue,
+        host,
+        param,
+    ) = poll_with_retries_scenario
     satellite_api = host.run.satellite_api
     satellite_api.responses = [
         param.api_output for _x in range(len(param.api_requests))
@@ -211,7 +215,7 @@ POLLING_LOOP_TEST_CASES = [
         queue_messages=[
             messages.playbook_run_update("host1", "play_id", "Exit status: 123", 0),
             messages.playbook_run_finished(
-                "host1", "play_id", constants.RESULT_FAILURE
+                "host1", "play_id", constants.RESULT_FAILURE, False, 123
             ),
         ],
     ),
@@ -226,7 +230,9 @@ POLLING_LOOP_TEST_CASES = [
         api_requests=[("output", (None, 1, None))],
         queue_messages=[
             messages.playbook_run_update("host1", "play_id", "Exit status: 123", 0),
-            messages.playbook_run_finished("host1", "play_id", constants.RESULT_CANCEL),
+            messages.playbook_run_finished(
+                "host1", "play_id", constants.RESULT_CANCEL, False, 123
+            ),
         ],
     ),
 ]
@@ -243,7 +249,11 @@ def polling_loop_scenario(request, base_scenario):
 
 @pytest.mark.asyncio
 async def test_polling_loop(polling_loop_scenario):
-    (queue, host, param,) = polling_loop_scenario
+    (
+        queue,
+        host,
+        param,
+    ) = polling_loop_scenario
     satellite_api = host.run.satellite_api
     satellite_api.responses = [
         param.api_output for _x in range(len(param.api_requests))
@@ -331,7 +341,10 @@ START_TEST_CASES = [
             messages.playbook_run_finished(
                 "host1", "play_id", constants.RESULT_SUCCESS
             ),
-            messages.playbook_run_completed("play_id", constants.RESULT_SUCCESS,),
+            messages.playbook_run_completed(
+                "play_id",
+                constants.RESULT_SUCCESS,
+            ),
         ],
         FakeLogger()
         .info("Playbook run play_id running as job invocation 123")
@@ -369,7 +382,7 @@ START_TEST_CASES = [
                 0,
             ),
             messages.playbook_run_finished(
-                "host1", "play_id", constants.HOST_RESULT_FAILURE
+                "host1", "play_id", constants.HOST_RESULT_FAILURE, True
             ),
             messages.playbook_run_completed(
                 "play_id",
@@ -390,16 +403,53 @@ START_TEST_CASES = [
                 error=None,
             ),
             dict(
+                error=None,
                 body={
+                    "complete": True,
                     "output": [
                         {
-                            "output": "Error initializing command: Net::SSH::AuthenticationFailed - Authentication failed for user root@centos-katello-3.14-0.example.com\n"  # noqa: E501
+                            "output": "\u001b[0;34mUsing /etc/ansible/ansible.cfg as config file\u001b[0m\n",
+                            "output_type": "stdout",
+                            "timestamp": 1600350676.69755,
+                        },
+                        {
+                            "output": "\n",
+                            "output_type": "stdout",
+                            "timestamp": 1600350677.70155,
+                        },
+                        {
+                            "output": "\r\nPLAY [all] *********************************************************************\n",
+                            "output_type": "stdout",
+                            "timestamp": 1600350677.70175,
+                        },
+                        {
+                            "output": "\r\nTASK [Gathering Facts] *********************************************************\n",
+                            "output_type": "stdout",
+                            "timestamp": 1600350677.70195,
+                        },
+                        {
+                            "output": "\n",
+                            "output_type": "stdout",
+                            "timestamp": 1600350677.70212,
+                        },
+                        {
+                            "output": '\u001b[1;31mfatal: [host1]: UNREACHABLE! => {"changed": false, "msg": "Invalid/incorrect password: Permission denied, please try again.\\r\\nPermission denied, please try again.\\r\\nReceived disconnect from 10.110.156.47 port 22:2: Too many authentication failures\\r\\nDisconnected from 10.110.156.47 port 22", "unreachable": true}\u001b[0m\n',
+                            "output_type": "stdout",
+                            "timestamp": 1600350684.0395,
+                        },
+                        {
+                            "output": "PLAY RECAP *********************************************************************\n\u001b[0;31mhost1\u001b[0m                   : ok=0    changed=0    \u001b[1;31munreachable=1   \u001b[0m failed=0    skipped=0    rescued=0    ignored=0   ",
+                            "output_type": "stdout",
+                            "timestamp": 1600350687.1491,
+                        },
+                        {
+                            "output": "Exit status: 1",
+                            "output_type": "stdout",
+                            "timestamp": 1600350688.1491,
                         },
-                        {"output": "Exit status: EXCEPTION"},
                     ],
-                    "complete": True,
+                    "refresh": False,
                 },
-                error=None,
             ),
         ],
         [
@@ -411,13 +461,16 @@ START_TEST_CASES = [
             messages.playbook_run_update(
                 "host1",
                 "play_id",
-                "Error initializing command: Net::SSH::AuthenticationFailed - Authentication failed for user root@centos-katello-3.14-0.example.com\nExit status: EXCEPTION",  # noqa: E501
+                '\x1b[0;34mUsing /etc/ansible/ansible.cfg as config file\x1b[0m\n\n\r\nPLAY [all] *********************************************************************\n\r\nTASK [Gathering Facts] *********************************************************\n\n\x1b[1;31mfatal: [host1]: UNREACHABLE! => {"changed": false, "msg": "Invalid/incorrect password: Permission denied, please try again.\\r\\nPermission denied, please try again.\\r\\nReceived disconnect from 10.110.156.47 port 22:2: Too many authentication failures\\r\\nDisconnected from 10.110.156.47 port 22", "unreachable": true}\x1b[0m\nPLAY RECAP *********************************************************************\n\x1b[0;31mhost1\x1b[0m                   : ok=0    changed=0    \x1b[1;31munreachable=1   \x1b[0m failed=0    skipped=0    rescued=0    ignored=0   Exit status: 1',
                 0,
             ),
             messages.playbook_run_finished(
-                "host1", "play_id", constants.HOST_RESULT_FAILURE
+                "host1", "play_id", constants.HOST_RESULT_FAILURE, True
+            ),
+            messages.playbook_run_completed(
+                "play_id",
+                constants.RESULT_FAILURE,
             ),
-            messages.playbook_run_completed("play_id", constants.RESULT_FAILURE,),
         ],
         FakeLogger()
         .info("Playbook run play_id running as job invocation 123")