Repository: cloud-custodian/cloud-custodian

Issue:
Removing AMIs
I've come across a CI setup that has been generating AMIs on every git push for years, but noone has thought to clean it up.

I created a basic policy which will mark unused AMIs (using the unused filter) and then later deregister the AMIs that are marked. This seems to work well.

However, deleting AMIs is usually a two step process where first you have to deregister the AMI, then you have to remove the snapshots that the AMI pointed to. In the AMI resource code I don't see anything that removes the snapshots. In the EBS code I don't see anything that tests if the volume specified for the snapshot is valid. I think that unless snapshots are deleted when the AMI is deregistered you'd have to compute list of snapshots that are not in AMIs or of an exist volume to work out a list of what to remove (but this doesn't count snapshots of ELBs, I don't use that feature, so not sure what their metadata looks like).






Discussion:
we generally just have a separate policy for snapshot cleanup, ie. clean up snapshots missing a set of tags and older than n days.
@kapilt could you show me an example of a snapshot cleanup policy? Reading through the source I don’t think there is sufficient logic to implement anything but age based policies, but maybe there are tricks I’m not a are of.
a lot of filters get added in generically..

running `custodian schema ebs-snapshot.filters` will show the available, the `value` filter supports alot of options, in this context as a tag based exclusion ie. - "tag:backup": absent along with an age filter and then delete. 
@kapilt If you have an example of how you are managing the AMI snapshots that is what I'm most interested in - even with the flexibility of the value I don't see how it could be done.

its a two sweep/policy pass for us, delete old amis, and then delete old snapshots.
@zfLQ2qx2  Are you still looking at a solution for this? I use the resources.json file produced by the first policy and pass it as input to the 2nd one. Cannot use this method with lambda as lambda doesn't support local storage well. Right now using this policy with crontab

```
policies:
- name: del-60-automated
  resource: ami
  description: |
    Delete AMI that are older than 60 days and have been created by lambda job[finds it via ML_INSTANCE tag]
  filters:
       - "tag:ML_INSTANCE": present
       - "tag:retained_ami": absent
       - type: value
         value_type: age
         key: CreationDate
         value: 60
         op: greater-than

  description: |
    Deregisters AMIs that are older than 60 days and created by daily backup lambda
  actions:
    - deregister

- name: del-60-automated-snap
  resource: ebs-snapshot
  description: |
    Delete snapshots from previous policy resource json file
  filters:
    - type: value
      key: "SnapshotId"
      op: in
      value_from:
          url: file:/home/ubuntu/custodian/del-60-automated/resources.json
          format: json
          expr: "[].BlockDeviceMappings[].Ebs.SnapshotId"

  description: |
    Deletes snapshots associated with the AMIs deregistered by the del-60-automated policy

  actions:
     - delete

```

Resolving patch:
diff --git a/c7n/resources/ami.py b/c7n/resources/ami.py
--- a/c7n/resources/ami.py
+++ b/c7n/resources/ami.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2017 Capital One Services, LLC
+# Copyright 2015-2019 Capital One Services, LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,23 +17,21 @@
 import logging
 
 from concurrent.futures import as_completed
+import jmespath
 
-from c7n.actions import ActionRegistry, BaseAction
+from c7n.actions import BaseAction
+from c7n.exceptions import ClientError
 from c7n.filters import (
-    FilterRegistry, AgeFilter, Filter, OPERATORS, CrossAccountAccessFilter)
+    AgeFilter, Filter, OPERATORS, CrossAccountAccessFilter)
 from c7n.manager import resources
 from c7n.query import QueryResourceManager
 from c7n.resolver import ValuesFrom
-from c7n.utils import local_session, type_schema, get_retry, chunks
+from c7n.utils import local_session, type_schema, chunks
 
 
 log = logging.getLogger('custodian.ami')
 
 
-filters = FilterRegistry('ami.filters')
-actions = ActionRegistry('ami.actions')
-
-
 @resources.register('ami')
 class AMI(QueryResourceManager):
 
@@ -50,9 +48,6 @@ class resource_type(object):
         dimension = None
         date = 'CreationDate'
 
-    filter_registry = filters
-    action_registry = actions
-
     def resources(self, query=None):
         query = query or {}
         if query.get('Owners') is None:
@@ -60,7 +55,7 @@ def resources(self, query=None):
         return super(AMI, self).resources(query=query)
 
 
-@actions.register('deregister')
+@AMI.action_registry.register('deregister')
 class Deregister(BaseAction):
     """Action to deregister AMI
 
@@ -81,23 +76,32 @@ class Deregister(BaseAction):
                   - deregister
     """
 
-    schema = type_schema('deregister')
+    schema = type_schema('deregister', **{'delete-snapshots': {'type': 'boolean'}})
     permissions = ('ec2:DeregisterImage',)
+    snap_expr = jmespath.compile('BlockDeviceMappings[].Ebs.SnapshotId')
 
     def process(self, images):
         client = local_session(self.manager.session_factory).client('ec2')
-        retry = get_retry((
-            'RequestLimitExceeded', 'Client.RequestLimitExceeded'))
-
         image_count = len(images)
         images = [i for i in images if self.manager.ctx.options.account_id == i['OwnerId']]
         if len(images) != image_count:
             self.log.info("Implicitly filtered %d non owned images", image_count - len(images))
+
         for i in images:
-            retry(client.deregister_image, ImageId=i['ImageId'])
+            self.manager.retry(client.deregister_image, ImageId=i['ImageId'])
+
+            if not self.data.get('delete-snapshots'):
+                continue
+            snap_ids = self.snap_expr.search(i) or ()
+            for s in snap_ids:
+                try:
+                    self.manager.retry(client.delete_snapshot, SnapshotId=s)
+                except ClientError as e:
+                    if e.error['Code'] == 'InvalidSnapshot.InUse':
+                        continue
 
 
-@actions.register('remove-launch-permissions')
+@AMI.action_registry.register('remove-launch-permissions')
 class RemoveLaunchPermissions(BaseAction):
     """Action to remove the ability to launch an instance from an AMI
 
@@ -133,13 +137,15 @@ def process_image(self, client, image):
             ImageId=image['ImageId'], Attribute="launchPermission")
 
 
-@actions.register('copy')
+@AMI.action_registry.register('copy')
 class Copy(BaseAction):
     """Action to copy AMIs with optional encryption
 
     This action can copy AMIs while optionally encrypting or decrypting
     the target AMI. It is advised to use in conjunction with a filter.
 
+    Note there is a max in flight of 5 per account/region.
+
     :example:
 
     .. code-block:: yaml
@@ -186,7 +192,7 @@ def process(self, images):
                 KmsKeyId=self.data.get('key-id', ''))
 
 
-@filters.register('image-age')
+@AMI.filter_registry.register('image-age')
 class ImageAgeFilter(AgeFilter):
     """Filters images based on the age (in days)
 
@@ -209,7 +215,7 @@ class ImageAgeFilter(AgeFilter):
         days={'type': 'number', 'minimum': 0})
 
 
-@filters.register('unused')
+@AMI.filter_registry.register('unused')
 class ImageUnusedFilter(Filter):
     """Filters images based on usage
 
@@ -263,7 +269,7 @@ def process(self, resources, event=None):
         return [r for r in resources if r['ImageId'] in images]
 
 
-@filters.register('cross-account')
+@AMI.filter_registry.register('cross-account')
 class AmiCrossAccountFilter(CrossAccountAccessFilter):
 
     schema = type_schema(


Test patch:
diff --git a/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DeleteSnapshot_1.json b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DeleteSnapshot_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DeleteSnapshot_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200,
+    "data": {
+        "ResponseMetadata": {
+            "RequestId": "96aab799-3b2d-43da-9cf5-d329b38f0b20",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "content-type": "text/xml;charset=UTF-8",
+                "content-length": "229",
+                "date": "Fri, 01 Mar 2019 16:00:28 GMT",
+                "server": "AmazonEC2"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DeregisterImage_1.json b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DeregisterImage_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DeregisterImage_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200,
+    "data": {
+        "ResponseMetadata": {
+            "RequestId": "3c7b7e74-d0fe-4061-94b2-79dbd5a0774f",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "content-type": "text/xml;charset=UTF-8",
+                "content-length": "231",
+                "date": "Fri, 01 Mar 2019 16:00:24 GMT",
+                "server": "AmazonEC2"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DescribeImages_1.json b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DescribeImages_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DescribeImages_1.json
@@ -0,0 +1,56 @@
+{
+    "status_code": 200,
+    "data": {
+        "Images": [
+            {
+                "Architecture": "x86_64",
+                "CreationDate": "2018-11-04T06:24:45.000Z",
+                "ImageId": "ami-0515ff4f8f9dbeb31",
+                "ImageLocation": "644160558196/Ubuntu1404",
+                "ImageType": "machine",
+                "Public": false,
+                "OwnerId": "644160558196",
+                "State": "available",
+                "BlockDeviceMappings": [
+                    {
+                        "DeviceName": "/dev/sda1",
+                        "Ebs": {
+                            "DeleteOnTermination": true,
+                            "SnapshotId": "snap-0d16416527e67c663",
+                            "VolumeSize": 8,
+                            "VolumeType": "standard",
+                            "Encrypted": true
+                        }
+                    },
+                    {
+                        "DeviceName": "/dev/sdb",
+                        "VirtualName": "ephemeral0"
+                    },
+                    {
+                        "DeviceName": "/dev/sdc",
+                        "VirtualName": "ephemeral1"
+                    }
+                ],
+                "Description": "[Copied ami-3073884d from us-east-1] ubuntu/images/hvm/ubuntu-trusty-14.04-amd64-server-20180308",
+                "EnaSupport": true,
+                "Hypervisor": "xen",
+                "Name": "Ubuntu1404",
+                "RootDeviceName": "/dev/sda1",
+                "RootDeviceType": "ebs",
+                "SriovNetSupport": "simple",
+                "VirtualizationType": "hvm"
+            }
+        ],
+        "ResponseMetadata": {
+            "RequestId": "f03372b5-48b8-4a14-a03a-e91e8c33ddf0",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "content-type": "text/xml;charset=UTF-8",
+                "content-length": "1998",
+                "date": "Fri, 01 Mar 2019 16:00:20 GMT",
+                "server": "AmazonEC2"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DescribeSnapshots_1.json b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DescribeSnapshots_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ami_deregister_delete_snap/ec2.DescribeSnapshots_1.json
@@ -0,0 +1,20 @@
+{
+    "status_code": 400,
+    "data": {
+        "Error": {
+            "Code": "InvalidSnapshot.NotFound",
+            "Message": "The snapshot 'snap-0d16416527e67c663' does not exist."
+        },
+        "ResponseMetadata": {
+            "RequestId": "4cacefe7-d230-47f6-b844-5fa55f353032",
+            "HTTPStatusCode": 400,
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked",
+                "date": "Fri, 01 Mar 2019 16:00:30 GMT",
+                "connection": "close",
+                "server": "AmazonEC2"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_ami.py b/tests/test_ami.py
--- a/tests/test_ami.py
+++ b/tests/test_ami.py
@@ -13,6 +13,9 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import jmespath
+
+from c7n.exceptions import ClientError
 from .common import BaseTest
 
 
@@ -34,6 +37,23 @@ def test_query(self):
         resources = p.run()
         self.assertEqual(len(resources), 1)
 
+    def test_deregister_delete_snaps(self):
+        factory = self.replay_flight_data('test_ami_deregister_delete_snap')
+        p = self.load_policy({
+            'name': 'deregister-snap',
+            'resource': 'ami',
+            'actions': [{
+                'type': 'deregister',
+                'delete-snapshots': True}]},
+            session_factory=factory, config={'account_id': '644160558196'})
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        client = factory().client('ec2')
+        snap_ids = jmespath.search(
+            'BlockDeviceMappings[].Ebs.SnapshotId', resources[0])
+        self.assertRaises(
+            ClientError, client.describe_snapshots, SnapshotIds=snap_ids, OwnerIds=['self'])
+
     def test_unused_ami_with_asg_launch_templates(self):
         factory = self.replay_flight_data('test_unused_ami_launch_template')
         p = self.load_policy(