Repository: Sceptre/sceptre

Issue:
Events missing from launch output
### Events missing from launch output
At some point after updating to Sceptre 3 events which appear in the web console are not included in the command output. It seems like if the stack update lasts long enough events will start to appear as they used to. If the stack launch is short then the command simply exits without feedback. This includes launch failures and roll-backs.

### Your environment
* Sceptre, version 3.2.0
* Python 3.9.9
* Official Python 3.9.9 Docker image

### Steps to reproduce
```$ sceptre launch -y <stack>```

This has been consistent across several accounts and versions of Sceptre. I've not yet identified which version the problem appears in.

### Expected behaviour
After the `Updating Stack` message we expect to see a number of resource state-change events and end with the final state of the stack.

### Actual behaviour
The process completes successfully, but events which appear in the web console are not output at the terminal:

```
$ sceptre launch -y prod/alb-internal.yaml
[2022-12-11 15:37:23] - prod/alb-internal - Launching Stack
[2022-12-11 15:37:24] - prod/alb-internal - Stack is in the UPDATE_COMPLETE state
[2022-12-11 15:37:24] - prod/alb-internal - Updating Stack
$
```

Comments:

Patch:
diff --git a/sceptre/helpers.py b/sceptre/helpers.py
index d0bcc25..0ca370e 100644
--- a/sceptre/helpers.py
+++ b/sceptre/helpers.py
@@ -1,6 +1,10 @@
 # -*- coding: utf-8 -*-
 from contextlib import contextmanager
+from datetime import datetime
 from os import sep
+from typing import Optional
+
+import dateutil.parser
 
 from sceptre.exceptions import PathConversionError
 
@@ -117,3 +121,20 @@ def null_context():
     available in py3.6, so providing it here instead.
     """
     yield
+
+
+def extract_datetime_from_aws_response_headers(boto_response: dict) -> Optional[datetime]:
+    """Returns a datetime.datetime extracted from the response metadata in a
+    boto response or None if it's unable to find or parse one.
+    :param boto_response: A dictionary returned from a boto client call
+    :returns a datetime.datetime or None
+    """
+    if boto_response is None:
+        return None
+    try:
+        return dateutil.parser.parse(boto_response["ResponseMetadata"]["HTTPHeaders"]["date"])
+    except (KeyError, dateutil.parser.ParserError):
+        # We expect a KeyError if the date isn't present in the response. We
+        # expect a ParserError if it's not well-formed. Any other error we want
+        # to pass along.
+        return None
diff --git a/sceptre/plan/actions.py b/sceptre/plan/actions.py
index 1576e1d..147de60 100644
--- a/sceptre/plan/actions.py
+++ b/sceptre/plan/actions.py
@@ -14,21 +14,19 @@ import typing
 import urllib
 from datetime import datetime, timedelta
 from os import path
-from typing import Union, Optional, Tuple, Dict
+from typing import Dict, Optional, Tuple, Union
 
 import botocore
 from dateutil.tz import tzutc
 
 from sceptre.config.reader import ConfigReader
 from sceptre.connection_manager import ConnectionManager
-from sceptre.exceptions import (
-    CannotUpdateFailedStackError,
-    ProtectedStackError,
-    StackDoesNotExistError,
-    UnknownStackChangeSetStatusError,
-    UnknownStackStatusError
-)
-from sceptre.helpers import normalise_path
+from sceptre.exceptions import (CannotUpdateFailedStackError,
+                                ProtectedStackError, StackDoesNotExistError,
+                                UnknownStackChangeSetStatusError,
+                                UnknownStackStatusError)
+from sceptre.helpers import (extract_datetime_from_aws_response_headers,
+                             normalise_path)
 from sceptre.hooks import add_stack_hooks
 from sceptre.stack import Stack
 from sceptre.stack_status import StackChangeSetStatus, StackStatus
@@ -95,7 +93,7 @@ class StackActions(object):
                 "%s - Create stack response: %s", self.stack.name, response
             )
 
-            status = self._wait_for_completion()
+            status = self._wait_for_completion(boto_response=response)
         except botocore.exceptions.ClientError as exp:
             if exp.response["Error"]["Code"] == "AlreadyExistsException":
                 self.logger.info(
@@ -141,7 +139,7 @@ class StackActions(object):
                 command="update_stack",
                 kwargs=update_stack_kwargs
             )
-            status = self._wait_for_completion(self.stack.stack_timeout)
+            status = self._wait_for_completion(self.stack.stack_timeout, boto_response=response)
             self.logger.debug(
                 "%s - Update Stack response: %s", self.stack.name, response
             )
@@ -180,7 +178,7 @@ class StackActions(object):
         self.logger.debug(
             "%s - Cancel update Stack response: %s", self.stack.name, response
         )
-        return self._wait_for_completion()
+        return self._wait_for_completion(boto_response=response)
 
     @add_stack_hooks
     def launch(self) -> StackStatus:
@@ -251,14 +249,14 @@ class StackActions(object):
 
         delete_stack_kwargs = {"StackName": self.stack.external_name}
         delete_stack_kwargs.update(self._get_role_arn())
-        self.connection_manager.call(
+        response = self.connection_manager.call(
             service="cloudformation",
             command="delete_stack",
             kwargs=delete_stack_kwargs
         )
 
         try:
-            status = self._wait_for_completion()
+            status = self._wait_for_completion(boto_response=response)
         except StackDoesNotExistError:
             status = StackStatus.COMPLETE
         except botocore.exceptions.ClientError as error:
@@ -549,7 +547,7 @@ class StackActions(object):
         self.logger.debug(
             "%s - Executing Change Set '%s'", self.stack.name, change_set_name
         )
-        self.connection_manager.call(
+        response = self.connection_manager.call(
             service="cloudformation",
             command="execute_change_set",
             kwargs={
@@ -557,8 +555,7 @@ class StackActions(object):
                 "StackName": self.stack.external_name
             }
         )
-
-        status = self._wait_for_completion()
+        status = self._wait_for_completion(boto_response=response)
         return status
 
     def change_set_creation_failed_due_to_no_changes(self, reason: str) -> bool:
@@ -767,15 +764,15 @@ class StackActions(object):
                 "currently enabled".format(self.stack.name)
             )
 
-    def _wait_for_completion(self, timeout=0):
+    def _wait_for_completion(self, timeout=0, boto_response: Optional[dict] = None) -> StackStatus:
         """
         Waits for a Stack operation to finish. Prints CloudFormation events
         while it waits.
 
         :param timeout: Timeout before returning, in minutes.
+        :param boto_response: Response from the boto call which initiated the stack change.
 
         :returns: The final Stack status.
-        :rtype: sceptre.stack_status.StackStatus
         """
         timeout = 60 * timeout
 
@@ -784,13 +781,14 @@ class StackActions(object):
 
         status = StackStatus.IN_PROGRESS
 
-        self.most_recent_event_datetime = (
-            datetime.now(tzutc()) - timedelta(seconds=3)
-        )
+        most_recent_event_datetime = extract_datetime_from_aws_response_headers(
+            boto_response
+        ) or (datetime.now(tzutc()) - timedelta(seconds=3))
+
         elapsed = 0
         while status == StackStatus.IN_PROGRESS and not timed_out(elapsed):
             status = self._get_simplified_status(self._get_status())
-            self._log_new_events()
+            most_recent_event_datetime = self._log_new_events(most_recent_event_datetime)
             time.sleep(4)
             elapsed += 4
 
@@ -843,15 +841,18 @@ class StackActions(object):
                 "{0} is unknown".format(status)
             )
 
-    def _log_new_events(self):
+    def _log_new_events(self, after_datetime: datetime) -> datetime:
         """
         Log the latest Stack events while the Stack is being built.
+
+        :param after_datetime: Only events after this datetime will be logged.
+        :returns: The datetime of the last logged event or after_datetime if no events were logged.
         """
         events = self.describe_events()["StackEvents"]
         events.reverse()
         new_events = [
             event for event in events
-            if event["Timestamp"] > self.most_recent_event_datetime
+            if event["Timestamp"] > after_datetime
         ]
         for event in new_events:
             self.logger.info(" ".join([
@@ -861,7 +862,8 @@ class StackActions(object):
                 event["ResourceStatus"],
                 event.get("ResourceStatusReason", "")
             ]))
-            self.most_recent_event_datetime = event["Timestamp"]
+            after_datetime = event["Timestamp"]
+        return after_datetime
 
     def wait_for_cs_completion(self, change_set_name):
         """


Test patch:
diff --git a/tests/test_actions.py b/tests/test_actions.py
index 3b3b5d1..611a902 100644
--- a/tests/test_actions.py
+++ b/tests/test_actions.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import datetime
 import json
-from unittest.mock import patch, sentinel, Mock, call
+from unittest.mock import patch, sentinel, Mock, call, ANY
 
 import pytest
 from botocore.exceptions import ClientError
@@ -127,7 +127,7 @@ class TestStackActions(object):
                 "TimeoutInMinutes": sentinel.timeout
             }
         )
-        mock_wait_for_completion.assert_called_once_with()
+        mock_wait_for_completion.assert_called_once_with(boto_response=ANY)
 
     @patch("sceptre.plan.actions.StackActions._wait_for_completion")
     def test_create_sends_correct_request_no_notifications(
@@ -162,7 +162,7 @@ class TestStackActions(object):
                 "TimeoutInMinutes": sentinel.stack_timeout
             }
         )
-        mock_wait_for_completion.assert_called_once_with()
+        mock_wait_for_completion.assert_called_once_with(boto_response=ANY)
 
     @patch("sceptre.plan.actions.StackActions._wait_for_completion")
     def test_create_sends_correct_request_with_no_failure_no_timeout(
@@ -194,7 +194,8 @@ class TestStackActions(object):
                 ]
             }
         )
-        mock_wait_for_completion.assert_called_once_with()
+
+        mock_wait_for_completion.assert_called_once_with(boto_response=ANY)
 
     @patch("sceptre.plan.actions.StackActions._wait_for_completion")
     def test_create_stack_already_exists(
@@ -245,7 +246,8 @@ class TestStackActions(object):
             }
         )
         mock_wait_for_completion.assert_called_once_with(
-            sentinel.stack_timeout
+            sentinel.stack_timeout,
+            boto_response=ANY
         )
 
     @patch("sceptre.plan.actions.StackActions._wait_for_completion")
@@ -284,7 +286,7 @@ class TestStackActions(object):
         ]
         self.actions.connection_manager.call.assert_has_calls(calls)
         mock_wait_for_completion.assert_has_calls(
-            [call(sentinel.stack_timeout), call()]
+            [call(sentinel.stack_timeout, boto_response=ANY), call(boto_response=ANY)]
         )
 
     @patch("sceptre.plan.actions.StackActions._wait_for_completion")
@@ -319,7 +321,8 @@ class TestStackActions(object):
             }
         )
         mock_wait_for_completion.assert_called_once_with(
-            sentinel.stack_timeout
+            sentinel.stack_timeout,
+            boto_response=ANY
         )
 
     @patch("sceptre.plan.actions.StackActions._wait_for_completion")
@@ -352,7 +355,7 @@ class TestStackActions(object):
             command="cancel_update_stack",
             kwargs={"StackName": sentinel.external_name}
         )
-        mock_wait_for_completion.assert_called_once_with()
+        mock_wait_for_completion.assert_called_once_with(boto_response=ANY)
 
     @patch("sceptre.plan.actions.StackActions.create")
     @patch("sceptre.plan.actions.StackActions._get_status")
@@ -707,7 +710,7 @@ class TestStackActions(object):
                 "StackName": sentinel.external_name
             }
         )
-        mock_wait_for_completion.assert_called_once_with()
+        mock_wait_for_completion.assert_called_once_with(boto_response=ANY)
 
     def test_execute_change_set__change_set_is_failed_for_no_changes__returns_0(self):
         def fake_describe(service, command, kwargs):
@@ -994,7 +997,8 @@ class TestStackActions(object):
         mock_get_simplified_status.return_value = StackStatus.COMPLETE
 
         self.actions._wait_for_completion()
-        mock_log_new_events.assert_called_once_with()
+        mock_log_new_events.assert_called_once()
+        assert type(mock_log_new_events.mock_calls[0].args[0]) is datetime.datetime
 
     @pytest.mark.parametrize("test_input,expected", [
         ("ROLLBACK_COMPLETE", StackStatus.FAILED),
@@ -1017,7 +1021,7 @@ class TestStackActions(object):
         mock_describe_events.return_value = {
             "StackEvents": []
         }
-        self.actions._log_new_events()
+        self.actions._log_new_events(datetime.datetime.utcnow())
         self.actions.describe_events.assert_called_once_with()
 
     @patch("sceptre.plan.actions.StackActions.describe_events")
@@ -1044,10 +1048,7 @@ class TestStackActions(object):
                 }
             ]
         }
-        self.actions.most_recent_event_datetime = (
-            datetime.datetime(2016, 3, 15, 14, 0, 0, 0, tzinfo=tzutc())
-        )
-        self.actions._log_new_events()
+        self.actions._log_new_events(datetime.datetime(2016, 3, 15, 14, 0, 0, 0, tzinfo=tzutc()))
 
     @patch("sceptre.plan.actions.StackActions._get_cs_status")
     def test_wait_for_cs_completion_calls_get_cs_status(
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index fafdaec..5b8c2c6 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -3,11 +3,13 @@
 import pytest
 
 from os.path import join, sep
+from datetime import datetime, timezone, timedelta
 
 from sceptre.exceptions import PathConversionError
 from sceptre.helpers import get_external_stack_name
 from sceptre.helpers import normalise_path
 from sceptre.helpers import sceptreise_path
+from sceptre.helpers import extract_datetime_from_aws_response_headers
 
 
 class TestHelpers(object):
@@ -67,3 +69,34 @@ class TestHelpers(object):
             sceptreise_path(
                 'this\\path\\is\\invalid\\'
             )
+
+    def test_get_response_datetime__response_is_valid__returns_datetime(self):
+        resp = {
+            "ResponseMetadata": {
+                "HTTPHeaders": {"date": "Wed, 16 Oct 2019 07:28:00 GMT"}
+            }
+        }
+        assert extract_datetime_from_aws_response_headers(resp) == datetime(2019, 10, 16, 7, 28, tzinfo=timezone.utc)
+
+    def test_get_response_datetime__response_has_offset__returns_datetime(self):
+        resp = {
+            "ResponseMetadata": {
+                "HTTPHeaders": {"date": "Wed, 16 Oct 2019 07:28:00 +0400"}
+            }
+        }
+        offset = timezone(timedelta(hours=4))
+        assert extract_datetime_from_aws_response_headers(resp) == datetime(2019, 10, 16, 7, 28, tzinfo=offset)
+
+    def test_get_response_datetime__date_string_is_invalid__returns_none(self):
+        resp = {
+            "ResponseMetadata": {
+                "HTTPHeaders": {"date": "garbage"}
+            }
+        }
+        assert extract_datetime_from_aws_response_headers(resp) is None
+
+    def test_get_response_datetime__response_is_empty__returns_none(self):
+        assert extract_datetime_from_aws_response_headers({}) is None
+
+    def test_get_response_datetime__response_is_none__returns_none(self):
+        assert extract_datetime_from_aws_response_headers(None) is None