Repository: cloud-custodian/cloud-custodian

Issue:
ec2 onhour - instance changed state during policy execution, causing failure
What I found in CloudTrail, after reviewing the following custodian-run.log, is that the instance causing the failure was terminated at 12:18 GMT.  What I think I can tell from ec2.py is that custodian is making efforts to ensure the start action is only attempted on stopped instances, but in this case, the ClientError on the terminated instance resulted in the rest of the onhour instances remaining un-started.

2018-11-15 12:21:32,933 - custodian.policy - DEBUG - Running policy ec2-start-xonhours-est-parent resource: ec2 region:us-east-1 c7n:0.8.32.0
2018-11-15 12:21:35,127 - custodian.cache - DEBUG - Using cache file /root/.cache/cloud-custodian.cache
2018-11-15 12:21:35,127 - custodian.resources.ec2 - DEBUG - Using cached c7n.resources.ec2.EC2: 3347
2018-11-15 12:21:39,402 - custodian.filters - INFO - InstanceOnHour 253 of 409 instances
2018-11-15 12:21:39,423 - custodian.resources.ec2 - DEBUG - Filtered from 3347 to 253 ec2
2018-11-15 12:21:39,423 - custodian.policy - INFO - policy: ec2-start-xonhours-est-parent resource:ec2 region:us-east-1 count:253 time:6.49
2018-11-15 12:21:39,494 - custodian.actions - INFO - Start 253 of 253 instances
2018-11-15 12:21:41,093 - custodian.output - ERROR - Error while executing policy
Traceback (most recent call last):
  File "/src/c7n/policy.py", line 273, in run
    results = a.process(resources)
  File "/src/c7n/resources/ec2.py", line 844, in process
    fails = self.process_instance_set(client, batch, itype, izone)
  File "/src/c7n/resources/ec2.py", line 863, in process_instance_set
    retry(client.start_instances, InstanceIds=instance_ids)
  File "/src/c7n/utils.py", line 360, in _retry
    return func(*args, **kw)
  File "/usr/local/lib/python2.7/site-packages/botocore/client.py", line 320, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python2.7/site-packages/botocore/client.py", line 623, in _make_api_call
    raise error_class(parsed_response, operation_name)
ClientError: An error occurred (IncorrectInstanceState) when calling the StartInstances operation: The instance 'i-xxxxxx' is not in a state from which it can be started.


Discussion:
there's some logic in the start action for batching and starting instances in smaller blocks, and on a particular set of errors retrying, and on failure to continue and then. in this case i think we could extend that logic to at least raise on incorrect state errors, that would still continue but fail the batch and the policy.  Going to more fine grained at instance level would need error string parsing from the instance id.


Resolving patch:
diff --git a/c7n/resources/ec2.py b/c7n/resources/ec2.py
--- a/c7n/resources/ec2.py
+++ b/c7n/resources/ec2.py
@@ -44,6 +44,8 @@
 from c7n.utils import type_schema, filter_empty
 
 
+RE_ERROR_INSTANCE_ID = re.compile("'(?P<instance_id>i-.*?)'")
+
 filters = FilterRegistry('ec2.filters')
 actions = ActionRegistry('ec2.actions')
 
@@ -947,8 +949,7 @@ def process(self, instances):
         if failures:
             fail_count = sum(map(len, failures.values()))
             msg = "Could not start %d of %d instances %s" % (
-                fail_count, len(instances),
-                utils.dumps(failures))
+                fail_count, len(instances), utils.dumps(failures))
             self.log.warning(msg)
             raise RuntimeError(msg)
 
@@ -958,12 +959,29 @@ def process_instance_set(self, client, instances, itype, izone):
                      'Client.RequestLimitExceeded'),
         retry = utils.get_retry(retryable, max_attempts=5)
         instance_ids = [i['InstanceId'] for i in instances]
-        try:
-            retry(client.start_instances, InstanceIds=instance_ids)
-        except ClientError as e:
-            if e.response['Error']['Code'] in retryable:
-                return True
-            raise
+        while instance_ids:
+            try:
+                retry(client.start_instances, InstanceIds=instance_ids)
+                break
+            except ClientError as e:
+                if e.response['Error']['Code'] in retryable:
+                    # we maxed out on our retries
+                    return True
+                elif e.response['Error']['Code'] == 'IncorrectInstanceState':
+                    instance_ids.remove(extract_instance_id(e))
+                else:
+                    raise
+
+
+def extract_instance_id(state_error):
+    "Extract an instance id from an error"
+    instance_id = None
+    match = RE_ERROR_INSTANCE_ID.search(str(state_error))
+    if match:
+        instance_id = match.groupdict().get('instance_id')
+    if match is None or instance_id is None:
+        raise ValueError("Could not extract instance id from error: %s" % state_error)
+    return instance_id
 
 
 @actions.register('resize')
@@ -1102,17 +1120,12 @@ def process(self, instances):
         return instances
 
     def _run_instances_op(self, op, instance_ids):
-        while True:
+        while instance_ids:
             try:
                 return self.manager.retry(op, InstanceIds=instance_ids)
             except ClientError as e:
                 if e.response['Error']['Code'] == 'IncorrectInstanceState':
-                    msg = e.response['Error']['Message']
-                    e_instance_id = msg[msg.find("'") + 1:msg.rfind("'")]
-                    instance_ids.remove(e_instance_id)
-                    if not instance_ids:
-                        return
-                    continue
+                    instance_ids.remove(extract_instance_id(e))
                 raise
 
 


Test patch:
diff --git a/tests/test_ec2.py b/tests/test_ec2.py
--- a/tests/test_ec2.py
+++ b/tests/test_ec2.py
@@ -23,7 +23,7 @@
 from mock import mock
 from jsonschema.exceptions import ValidationError
 
-from c7n.exceptions import PolicyValidationError
+from c7n.exceptions import PolicyValidationError, ClientError
 from c7n.resources import ec2
 from c7n.resources.ec2 import actions, QueryFilter
 from c7n import tags, utils
@@ -844,6 +844,48 @@ def test_ec2_reboot(self):
 
 class TestStart(BaseTest):
 
+    def test_invalid_state_extract(self):
+        self.assertEqual(
+            ec2.extract_instance_id(
+                ("An error occurred (IncorrectInstanceState) when calling "
+                 "the StartInstances operation: The instance 'i-abc123' is "
+                 "not in a state from which it can be started.")),
+            'i-abc123')
+        self.assertRaises(
+            ValueError,
+            ec2.extract_instance_id,
+            ("An error occurred (IncorrectInstanceState) when calling "
+             "the StartInstances operation: The instance is "
+             "not in a state from which it can be started."))
+
+    def test_ec2_start_handle_invalid_state(self):
+        policy = self.load_policy({
+            "name": "ec2-test-start",
+            "resource": "ec2",
+            "filters": [],
+            "actions": [{"type": "start"}],
+        })
+
+        client = mock.MagicMock()
+        client.start_instances.side_effect = ClientError(
+            {'Error': {
+                'Code': 'IncorrectInstanceState',
+                'Message': "The instance 'i-08270b9cfb568a1c4' is not in a state from which it can be started" # NOQA
+            }}, 'StartInstances')
+
+        start_action = policy.resource_manager.actions[0]
+        self.assertEqual(
+            start_action.process_instance_set(
+                client, [{'InstanceId': 'i-08270b9cfb568a1c4'}], 'm5.xlarge', 'us-east-1a'),
+            None)
+
+        client2 = mock.MagicMock()
+        client2.start_instances.side_effect = ValueError
+        self.assertRaises(
+            ValueError,
+            start_action.process_instance_set,
+            client2, [{'InstanceId': 'i-08270b9cfb568a1c4'}], 'm5.xlarge', 'us-east-1a')
+
     def test_ec2_start(self):
         session_factory = self.replay_flight_data("test_ec2_start")
         policy = self.load_policy(