Repository: cloud-custodian/cloud-custodian

Issue:
aws.asg - not-encrypted filter removed error handling for missing AMI's
in this commit:

https://github.com/cloud-custodian/cloud-custodian/commit/45d059bccc77df01f79256816038731d2756b12f#diff-8d7e1840f31e642d80e0fe0b85ef0324L266

the error handling for missing AMI's was removed, in the case that AMI's are not found, this will result in the ASG being filtered in as being not-encrypted rather than raising (per the previous behavior)


Discussion:
@jtroberts83 was noting issues for invalid as well on image.
Are there any logs or trace backs for when this exhibits?
the error handling for missing amis i found was out dated wrt to current api behavior, ie. the api doesn't return errors/exceptions on those anymore the same way that it would on a snapshot. i tested fetching a thousand amis with a single api call (aka no pagination needed), and 5 that were bad ami ids, and it returned all the ones extant and omitted the ones not found. the snapshot not found case was explicitly handled, bad amis just don't get returned by the api. there's definitely an issue here though, but we need more data to correlate to problem and solution. 
it looks like there is some region specific behavior to the api response.


in us-east-1 it will just omit the bad image id from the output

```
$ aws ec2 describe-images --owner=self  --image-ids ami-123f000c8c1f9f654  --region us-east-1
{
    "Images": []
}

```

in other regions it will report as an invalid ami id.
```
$ aws ec2 describe-images --owner=self  --image-ids ami-123f000c8c1f9f654  --region us-east-2

An error occurred (InvalidAMIID.NotFound) when calling the DescribeImages operation: The image id '[ami-123f000c8c1f9f654]' does not exist
$ aws ec2 describe-images --owner=self  --image-ids ami-123f000c8c1f9f654  --region us-west-2

An error occurred (InvalidAMIID.NotFound) when calling the DescribeImages operation: The image id '[ami-123f000c8c1f9f654]' does not exist

```




Resolving patch:
diff --git a/c7n/resources/ami.py b/c7n/resources/ami.py
--- a/c7n/resources/ami.py
+++ b/c7n/resources/ami.py
@@ -24,7 +24,7 @@
 from c7n.filters import (
     AgeFilter, Filter, OPERATORS, CrossAccountAccessFilter)
 from c7n.manager import resources
-from c7n.query import QueryResourceManager
+from c7n.query import QueryResourceManager, DescribeSource
 from c7n.resolver import ValuesFrom
 from c7n.utils import local_session, type_schema, chunks
 
@@ -54,6 +54,46 @@ def resources(self, query=None):
             query['Owners'] = ['self']
         return super(AMI, self).resources(query=query)
 
+    def get_source(self, source_type):
+        if source_type == 'describe':
+            return DescribeImageSource(self)
+        return super(AMI, self).get_source(source_type)
+
+
+class DescribeImageSource(DescribeSource):
+
+    def get_resources(self, ids, cache=True):
+        while ids:
+            try:
+                return super(DescribeImageSource, self).get_resources(ids, cache)
+            except ClientError as e:
+                bad_ami_ids = ErrorHandler.extract_bad_ami(e)
+                if bad_ami_ids:
+                    for b in bad_ami_ids:
+                        ids.remove(b)
+                    continue
+                raise
+        return []
+
+
+class ErrorHandler(object):
+
+    @staticmethod
+    def extract_bad_ami(e):
+        """Handle various client side errors when describing images"""
+        msg = e.response['Error']['Message']
+        error = e.response['Error']['Code']
+        e_ami_ids = None
+        if error == 'InvalidAMIID.NotFound':
+            e_ami_ids = [
+                e_ami_id.strip() for e_ami_id
+                in msg[msg.find("'[") + 2:msg.rfind("]'")].split(',')]
+            log.warning("Image not found %s" % e_ami_ids)
+        elif error == 'InvalidAMIID.Malformed':
+            e_ami_ids = [msg[msg.find('"') + 1:msg.rfind('"')]]
+            log.warning("Image id malformed %s" % e_ami_ids)
+        return e_ami_ids
+
 
 @AMI.action_registry.register('deregister')
 class Deregister(BaseAction):
diff --git a/c7n/resources/asg.py b/c7n/resources/asg.py
--- a/c7n/resources/asg.py
+++ b/c7n/resources/asg.py
@@ -165,9 +165,14 @@ def get_image_map(self):
         # amis, it doesn't seem to have any upper bound on number of
         # ImageIds to pass (Tested with 1k+ ImageIds)
         #
+        # Explicitly use a describe source. Can't use a config source
+        # since it won't have state for third party ami, we auto
+        # propagate source normally. Can't use a cache either as their
+        # not in the account.
         return {i['ImageId']: i for i in
-                self.manager.get_resource_manager('ami').get_resources(
-                    list(self.get_image_ids()), cache=False)}
+                self.manager.get_resource_manager(
+                    'ami').get_source('describe').get_resources(
+                        list(self.get_image_ids()), cache=False)}
 
     def get_security_group_ids(self):
         # return set of security group ids for given asg


Test patch:
diff --git a/tests/data/placebo/test_ami_not_found_err/ec2.DescribeImages_1.json b/tests/data/placebo/test_ami_not_found_err/ec2.DescribeImages_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ami_not_found_err/ec2.DescribeImages_1.json
@@ -0,0 +1,46 @@
+{
+    "status_code": 200,
+    "data": {
+        "Images": [
+            {
+                "Architecture": "x86_64",
+                "CreationDate": "2019-03-06T14:16:59.000Z",
+                "ImageId": "ami-041151726c89bed87",
+                "ImageLocation": "644160558196/test2",
+                "ImageType": "machine",
+                "Public": false,
+                "OwnerId": "644160558196",
+                "State": "available",
+                "BlockDeviceMappings": [
+                    {
+                        "DeviceName": "/dev/sda1",
+                        "Ebs": {
+                            "DeleteOnTermination": true,
+                            "SnapshotId": "snap-0bf9e1af959fbdddf",
+                            "VolumeSize": 1,
+                            "VolumeType": "gp2",
+                            "Encrypted": false
+                        }
+                    }
+                ],
+                "Description": "",
+                "Hypervisor": "xen",
+                "Name": "test2",
+                "RootDeviceName": "/dev/sda1",
+                "RootDeviceType": "ebs",
+                "VirtualizationType": "paravirtual"
+            }
+        ],
+        "ResponseMetadata": {
+            "RequestId": "ca5b9f00-60f6-4d7c-a506-5433a108dbe3",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "content-type": "text/xml;charset=UTF-8",
+                "content-length": "1469",
+                "date": "Wed, 06 Mar 2019 15:40:32 GMT",
+                "server": "AmazonEC2"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_ami.py b/tests/test_ami.py
--- a/tests/test_ami.py
+++ b/tests/test_ami.py
@@ -16,6 +16,8 @@
 import jmespath
 
 from c7n.exceptions import ClientError
+from c7n.resources.ami import ErrorHandler
+from c7n.query import DescribeSource
 from .common import BaseTest
 
 
@@ -37,6 +39,76 @@ def test_query(self):
         resources = p.run()
         self.assertEqual(len(resources), 1)
 
+    def test_err_ami(self):
+        factory = self.replay_flight_data("test_ami_not_found_err")
+        ami_id = 'ami-123f000eee1f9f654'
+        good_ami_id = 'ami-041151726c89bed87'
+        error_response = {"Error": {
+            "Message": "The image id '[%s]' does not exist" % (ami_id),
+            "Code": "InvalidAMIID.NotFound"}}
+
+        responses = [ClientError(error_response, "DescribeSnapshots")]
+
+        def base_get_resources(self, ids, cache=True):
+            if responses:
+                raise responses.pop()
+            return factory().client('ec2').describe_images(ImageIds=ids).get('Images')
+
+        self.patch(DescribeSource, 'get_resources', base_get_resources)
+
+        p = self.load_policy(
+            {'name': 'bad-ami', 'resource': 'ami'},
+            session_factory=factory)
+        resources = p.resource_manager.get_resources([ami_id, good_ami_id])
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(resources[0]['ImageId'], good_ami_id)
+
+    def test_err_get_ami_invalid(self):
+        operation_name = "DescribeSnapshots"
+        error_response = {
+            "Error": {
+                "Message": 'Invalid id: "ami123f000eee1f9f654"',
+                "Code": "InvalidAMIID.Malformed",
+            }
+        }
+        e = ClientError(error_response, operation_name)
+        ami = ErrorHandler.extract_bad_ami(e)
+        self.assertEqual(ami, ["ami123f000eee1f9f654"])
+
+        operation_name = "DescribeSnapshots"
+        error_response = {
+            "Error": {
+                "Message": 'Invalid id: "ami-1234567890abcdef0"',
+                "Code": "InvalidAMIID.Malformed",
+            }
+        }
+        e = ClientError(error_response, operation_name)
+        ami = ErrorHandler.extract_bad_ami(e)
+        self.assertEqual(ami, ["ami-1234567890abcdef0"])
+
+    def test_err_get_ami_notfound(self):
+        operation_name = "DescribeSnapshots"
+        error_response = {
+            "Error": {
+                "Message": "The image id '[ami-ffffffff]' does not exist",
+                "Code": "InvalidAMIID.NotFound"
+            }
+        }
+        e = ClientError(error_response, operation_name)
+        snap = ErrorHandler.extract_bad_ami(e)
+        self.assertEqual(snap, ["ami-ffffffff"])
+
+        operation_name = "DescribeSnapshots"
+        error_response = {
+            "Error": {
+                "Message": "The image id '[ami-11111111, ami-ffffffff]' does not exist",
+                "Code": "InvalidAMIID.NotFound"
+            }
+        }
+        e = ClientError(error_response, operation_name)
+        snap = ErrorHandler.extract_bad_ami(e)
+        self.assertEqual(snap, ["ami-11111111", "ami-ffffffff"])
+
     def test_deregister_delete_snaps(self):
         factory = self.replay_flight_data('test_ami_deregister_delete_snap')
         p = self.load_policy({