Repository: cloud-custodian/cloud-custodian

Issue:
aws.ebs-snapshot - add ability to do server side queries on ebs snapshots
the desribe_snapshots call allows the usage of user defined filters, this can be useful when writing policies against accounts that have a large number of snapshots (order of magnitude 100k+). currently custodian implicitly filters in only user-created snapshots

https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_snapshots


Discussion:
re context #3217

Resolving patch:
diff --git a/c7n/resources/ebs.py b/c7n/resources/ebs.py
--- a/c7n/resources/ebs.py
+++ b/c7n/resources/ebs.py
@@ -22,6 +22,7 @@
 from botocore.exceptions import ClientError
 from concurrent.futures import as_completed
 from dateutil.parser import parse as parse_date
+import six
 
 from c7n.actions import ActionRegistry, BaseAction
 from c7n.exceptions import PolicyValidationError
@@ -79,12 +80,77 @@ class resource_type(object):
     action_registry = ActionRegistry('ebs-snapshot.actions')
 
     def resources(self, query=None):
+        qfilters = QueryParser.parse(self.data.get('query', []))
         query = query or {}
+        if qfilters:
+            query['Filters'] = qfilters
         if query.get('OwnerIds') is None:
             query['OwnerIds'] = ['self']
         return super(Snapshot, self).resources(query=query)
 
 
+class QueryParser(object):
+
+    QuerySchema = {
+        'description': six.string_types,
+        'owner-alias': ('amazon', 'amazon-marketplace', 'microsoft'),
+        'owner-id': six.string_types,
+        'progress': six.string_types,
+        'snapshot-id': six.string_types,
+        'start-time': six.string_types,
+        'status': ('pending', 'completed', 'error'),
+        'tag': six.string_types,
+        'tag-key': six.string_types,
+        'volume-id': six.string_types,
+        'volume-size': six.string_types,
+    }
+
+    @classmethod
+    def parse(cls, data):
+        filters = []
+
+        if not isinstance(data, (tuple, list)):
+            raise PolicyValidationError(
+                "EBS Query invalid format, must be array of dicts %s" % (
+                    data))
+        for d in data:
+            if not isinstance(d, dict):
+                raise PolicyValidationError(
+                    "EBS Query Filter Invalid %s" % data)
+            if "Name" not in d or "Values" not in d:
+                raise PolicyValidationError(
+                    "EBS Query Filter Invalid Missing Key, Values in %s" % data)
+            key = d['Name']
+            values = d['Values']
+
+            if key not in cls.QuerySchema and not key.startswith('tag:'):
+                raise PolicyValidationError(
+                    "EBS Query Filter Invalid Key:%s Valid: %s" % (
+                        key, ", ".join(cls.QuerySchema.keys())))
+
+            vtype = cls.QuerySchema.get(key)
+            if vtype is None and key.startswith('tag'):
+                vtype = six.string_types
+
+            if not isinstance(values, list):
+                raise PolicyValidationError(
+                    "EBS Query Filter Invalid Values, must be array %s" % (data,))
+
+            for v in values:
+                if isinstance(vtype, tuple) and vtype != six.string_types:
+                    if v not in vtype:
+                        raise PolicyValidationError(
+                            "EBS Query Filter Invalid Value: %s Valid: %s" % (
+                                v, ", ".join(vtype)))
+                elif not isinstance(v, vtype):
+                    raise PolicyValidationError(
+                        "EBS Query Filter Invalid Value Type %s" % (data,))
+
+            filters.append(d)
+
+        return filters
+
+
 @Snapshot.filter_registry.register('age')
 class SnapshotAge(AgeFilter):
     """EBS Snapshot Age Filter
diff --git a/c7n/schema.py b/c7n/schema.py
--- a/c7n/schema.py
+++ b/c7n/schema.py
@@ -208,10 +208,8 @@ def generate(resource_types=()):
                 # in json schema inheritance prevent us from doing this
                 # on a type specific basis http://goo.gl/8UyRvQ
                 'query': {
-                    'type': 'array', 'items': {
-                        'type': 'object',
-                        'minProperties': 1,
-                        'maxProperties': 1}}
+                    'type': 'array', 'items': {'type': 'object'}}
+
             },
         },
         'policy-mode': {


Test patch:
diff --git a/tests/test_ebs.py b/tests/test_ebs.py
--- a/tests/test_ebs.py
+++ b/tests/test_ebs.py
@@ -18,17 +18,49 @@
 
 from botocore.exceptions import ClientError
 
-from .common import BaseTest, TestConfig as Config
+
+from c7n.exceptions import PolicyValidationError
+from c7n.executor import MainThreadExecutor
 from c7n.resources.ebs import (
     CopyInstanceTags,
     EncryptInstanceVolumes,
     CopySnapshot,
     Delete,
+    QueryParser
 )
-from c7n.executor import MainThreadExecutor
 
+from .common import BaseTest, TestConfig as Config
+
+
+class SnapshotQueryParse(BaseTest):
+
+    def test_query(self):
+        qfilters = [
+            {'Name': 'tag:Name', 'Values': ['Snapshot1']},
+            {'Name': 'status', 'Values': ['completed']}]
+        self.assertEqual(qfilters, QueryParser.parse(qfilters))
+
+    def test_invalid_query(self):
+        self.assertRaises(
+            PolicyValidationError, QueryParser.parse, {})
+
+        self.assertRaises(
+            PolicyValidationError, QueryParser.parse, [None])
+
+        self.assertRaises(
+            PolicyValidationError, QueryParser.parse, [{'X': 1}])
 
-logging.basicConfig(level=logging.DEBUG)
+        self.assertRaises(
+            PolicyValidationError, QueryParser.parse, [
+                {'Name': 'status', 'Values': 'completed'}])
+
+        self.assertRaises(
+            PolicyValidationError, QueryParser.parse, [
+                {'Name': 'status', 'Values': ['Completed']}])
+
+        self.assertRaises(
+            PolicyValidationError, QueryParser.parse, [
+                {'Name': 'snapshot-id', 'Values': [1]}])
 
 
 class SnapshotAccessTest(BaseTest):
@@ -58,31 +90,31 @@ def test_snapshot_access(self):
 
 class SnapshotDetachTest(BaseTest):
 
-        def test_volume_detach(self):
-            factory = self.replay_flight_data('test_ebs_detach')
-            p = self.load_policy(
-                {
-                    'name': 'volume-detach',
-                    'resource': 'ebs',
-                    'filters': [{'VolumeId': 'vol-0850cf7c8e949c318'}],
-                    'actions': [
-                        {
-                            'type': 'detach'
-                        }
-                    ]
-                }, config=Config.empty(), session_factory=factory)
-            resources = p.run()
-            self.assertEqual(len(resources), 1)
-
-            client = factory(region="us-east-1").client('ec2')
-            volumelist = []
-            volumelist.append(resources[0]['VolumeId'])
-            response = client.describe_volumes(VolumeIds=volumelist)
-
-            for resp in response['Volumes']:
-                for attachment in resp['Attachments']:
-                    self.assertTrue(attachment['State'] == "detached" or
-                            attachment['State'] == "detaching")
+    def test_volume_detach(self):
+        factory = self.replay_flight_data('test_ebs_detach')
+        p = self.load_policy(
+            {
+                'name': 'volume-detach',
+                'resource': 'ebs',
+                'filters': [{'VolumeId': 'vol-0850cf7c8e949c318'}],
+                'actions': [
+                    {
+                        'type': 'detach'
+                    }
+                ]
+            }, config=Config.empty(), session_factory=factory)
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+
+        client = factory(region="us-east-1").client('ec2')
+        volumelist = []
+        volumelist.append(resources[0]['VolumeId'])
+        response = client.describe_volumes(VolumeIds=volumelist)
+
+        for resp in response['Volumes']:
+            for attachment in resp['Attachments']:
+                self.assertTrue(attachment['State'] == "detached" or
+                                attachment['State'] == "detaching")
 
 
 class SnapshotCopyTest(BaseTest):