Repository: cloud-custodian/cloud-custodian

Issue:
lambda policy should validate ids expressions pre deploy
Policy yaml below works to create the lambda function however when triggered the function fails with this error:
**ParseError: Unexpected token: CopyImage: Parse error at column 5, token "CopyImage" (QUOTED_IDENTIFIER), for expression:
"event"CopyImage""**

thanks
Will

```
policies:

  - name: ec2-ami-compliance
    resource: ami

    mode:
      type: cloudtrail
      events:
       - source: ec2.amazonaws.com
         event: RegisterImage
         ids: "eventRegisterImage"
       - source: ec2.amazonaws.com
         event: DeregisterImage
         ids: "eventDeregisterImage"
       - source: ec2.amazonaws.com
         event: CopyImage
         ids: event"CopyImage"
       - source: ec2.amazonaws.com
         event: ImportImage
         ids: "eventImportImage"
       - source: ec2.amazonaws.com
         event: ModifyImageAttribute
         ids: "eventModifyImageAttribute"
       - source: ec2.amazonaws.com
         event: ResetImageAttribute
         ids: "eventResetImageAttribute"

    filters:
       - type: value
         key: ImageId
         op: ni
         value:
          - ami-1234546

    actions:
      - type: notify
        to:
         - event-user
         - resource-creator
         - user@email.com
        template: policy-template
        transport:
          type: sqs
          region: us-east-1_
```


Discussion:
seems like a jmespath syntax error on an invalid ids key, we should pre-validate this before deploying. re invalid expression. 

```
ids: event"CopyImage"
```

note your policy ids selectors don't look valid either, here's an example of a working one for security group.

```
        - source: ec2.amazonaws.com
          event: AuthorizeSecurityGroupIngress
          ids: "requestParameters.groupId"
```

Thanks @kapilt 

Resolving patch:
diff --git a/c7n/commands.py b/c7n/commands.py
--- a/c7n/commands.py
+++ b/c7n/commands.py
@@ -176,7 +176,8 @@ def validate(options):
             null_config = Bag(dryrun=True, log_group=None, cache=None, assume_role="na")
             for p in data.get('policies', ()):
                 try:
-                    Policy(p, null_config, Bag())
+                    p = Policy(p, null_config, Bag())
+                    p.validate()
                 except Exception as e:
                     msg = "Policy: %s is invalid: %s" % (
                         p.get('name', 'unknown'), e)
diff --git a/c7n/filters/core.py b/c7n/filters/core.py
--- a/c7n/filters/core.py
+++ b/c7n/filters/core.py
@@ -126,7 +126,7 @@ def factory(self, data, manager=None):
                     self.plugin_type, data))
         filter_class = self.get(filter_type)
         if filter_class is not None:
-            return filter_class(data, manager).validate()
+            return filter_class(data, manager)
         else:
             raise FilterValidationError(
                 "%s Invalid filter type %s" % (
diff --git a/c7n/policy.py b/c7n/policy.py
--- a/c7n/policy.py
+++ b/c7n/policy.py
@@ -21,6 +21,7 @@
 
 import boto3
 from botocore.client import ClientError
+import jmespath
 
 from c7n.actions import EventAction
 from c7n.cwe import CloudWatchEvents
@@ -63,6 +64,9 @@ def load(options, path, format='yaml', validate=True, vars=None):
             raise Exception("Failed to validate on policy %s \n %s" % (errors[1], errors[0]))
 
     collection = PolicyCollection.from_data(data, options)
+    if validate:
+        # non schema validation of policies
+        [p.validate() for p in collection]
     return collection
 
 
@@ -193,6 +197,9 @@ def get_logs(self, start, end):
         """Retrieve logs for the policy"""
         raise NotImplementedError("subclass responsibility")
 
+    def validate(self):
+        """Validate configuration settings for execution mode."""
+
     def get_metrics(self, start, end, period):
         """Retrieve any associated metrics for the policy."""
         values = {}
@@ -450,6 +457,15 @@ def run(self, event, lambda_context):
 class CloudTrailMode(LambdaMode):
     """A lambda policy using cloudwatch events rules on cloudtrail api logs."""
 
+    def validate(self):
+        events = self.policy.data['mode'].get('events')
+        assert events, "cloud trail mode requires specifiying events to subscribe"
+        for e in events:
+            if isinstance(e, basestring):
+                assert e in CloudWatchEvents.trail_events, "event shortcut not defined"
+            if isinstance(e, dict):
+                jmespath.compile(e['ids'])
+
 
 class EC2InstanceState(LambdaMode):
     """a lambda policy that executes on ec2 instance state changes."""
@@ -573,6 +589,14 @@ def is_lambda(self):
             return False
         return True
 
+    def validate(self):
+        m = self.get_execution_mode()
+        m.validate()
+        for f in self.resource_manager.filters:
+            f.validate()
+        for a in self.resource_manager.actions:
+            a.validate()
+
     def push(self, event, lambda_ctx):
         mode = self.get_execution_mode()
         return mode.run(event, lambda_ctx)
@@ -605,13 +629,6 @@ def get_permissions(self):
             permissions.update(a.get_permissions())
         return permissions
 
-    def validate(self):
-        """validate settings, else raise validation error"""
-        for f in self.resource_manager.filters:
-            f.validate()
-        for a in self.resource_manager.actions:
-            a.validate()
-
     def __call__(self):
         """Run policy in default mode"""
         mode = self.get_execution_mode()


Test patch:
diff --git a/tests/common.py b/tests/common.py
--- a/tests/common.py
+++ b/tests/common.py
@@ -104,7 +104,9 @@ def load_policy(
             config['cache'] = os.path.join(temp_dir, 'c7n.cache')
             config['cache_period'] = 300
         conf = Config.empty(**config)
-        return policy.Policy(data, conf, session_factory)
+        p = policy.Policy(data, conf, session_factory)
+        p.validate()
+        return p
 
     def load_policy_set(self, data, config=None):
         filename = self.write_policy_file(data)
diff --git a/tests/test_elb.py b/tests/test_elb.py
--- a/tests/test_elb.py
+++ b/tests/test_elb.py
@@ -226,35 +226,37 @@ def test_ssl_matching(self):
             'test-elb-invalid-policy')
 
     def test_filter_validation_no_blacklist(self):
-        self.assertRaises(FilterValidationError,
-            self.load_policy, {
-                'name': 'test-ssl-ciphers',
-                'resource': 'elb',
-                'filters': [
-                    {'type': 'ssl-policy'}
-                ]},
-                session_factory=None, validate=False)
+        self.assertRaises(
+            FilterValidationError,
+            self.load_policy,
+            {'name': 'test-ssl-ciphers',
+             'resource': 'elb',
+             'filters': [
+                 {'type': 'ssl-policy'}
+             ]},
+            session_factory=None, validate=False)
 
     def test_filter_validation_blacklist_not_iterable(self):
-        self.assertRaises(FilterValidationError,
-            self.load_policy, {
-                'name': 'test-ssl-ciphers',
-                'resource': 'elb',
-                'filters': [
-                    {'type': 'ssl-policy', 'blacklist': 'single-value'}
-                ]},
-                session_factory=None, validate=False)
+        self.assertRaises(
+            FilterValidationError,
+            self.load_policy,
+            {'name': 'test-ssl-ciphers',
+             'resource': 'elb',
+             'filters': [
+                 {'type': 'ssl-policy', 'blacklist': 'single-value'}
+             ]},
+            session_factory=None, validate=False)
 
 
 class TestDefaultVpc(BaseTest):
 
     def test_elb_default_vpc(self):
         session_factory = self.replay_flight_data('test_elb_default_vpc')
-        p = self.load_policy(
-            {'name': 'elb-default-filters',
-             'resource': 'elb',
-             'filters': [
-                 {'type': 'default-vpc'}]},
+        p = self.load_policy({
+            'name': 'elb-default-filters',
+            'resource': 'elb',
+            'filters': [
+                {'type': 'default-vpc'}]},
             config={'region': 'us-west-2'},
             session_factory=session_factory)
 
diff --git a/tests/test_filters.py b/tests/test_filters.py
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -217,11 +217,11 @@ class TestRegexValue(unittest.TestCase):
     def test_regex_validate(self):
         self.assertRaises(
             base_filters.FilterValidationError,
-            filters.factory,
-            {'type': 'value',
-             'key': 'Color',
-             'value': '*green',
-             'op': 'regex'})
+            filters.factory({
+                'type': 'value',
+                'key': 'Color',
+                'value': '*green',
+                'op': 'regex'}).validate)
 
     def test_regex_match(self):
         f = filters.factory(
@@ -363,7 +363,7 @@ def test_resource_count_filter_validation(self):
             'value': 1,
         }
         self.assertRaises(
-            base_filters.FilterValidationError, filters.factory, f, {})
+            base_filters.FilterValidationError, filters.factory(f, {}).validate)
 
         # Bad `value`
         f = {
@@ -373,7 +373,7 @@ def test_resource_count_filter_validation(self):
             'value': 'foo',
         }
         self.assertRaises(
-            base_filters.FilterValidationError, filters.factory, f, {})
+            base_filters.FilterValidationError, filters.factory(f, {}).validate)
 
         # Missing `op`
         f = {
@@ -382,7 +382,7 @@ def test_resource_count_filter_validation(self):
             'value': 1,
         }
         self.assertRaises(
-            base_filters.FilterValidationError, filters.factory, f, {})
+            base_filters.FilterValidationError, filters.factory(f, {}).validate)
 
 
 class TestInstanceAge(BaseFilterTest):
@@ -486,8 +486,9 @@ def test_event_no_mode(self):
         f = {'type': 'event',
              'key': 'detail.state',
              'value': 'pending'}
+        f = filters.factory(f, b)
         self.assertRaises(
-            base_filters.FilterValidationError, filters.factory, f, b)
+            base_filters.FilterValidationError,  f.validate)
 
 
 class TestInstanceValue(BaseFilterTest):
@@ -542,21 +543,20 @@ def test_jmespath(self):
     def test_complex_validator(self):
         self.assertRaises(
             base_filters.FilterValidationError,
-            filters.factory,
-            {"key": "xyz",
-             "type": "value"})
+            filters.factory({
+                "key": "xyz", "type": "value"}).validate)
         self.assertRaises(
             base_filters.FilterValidationError,
-            filters.factory,
-            {"value": "xyz",
-             "type": "value"})
+            filters.factory({
+                "value": "xyz", "type": "value"}).validate)
+
         self.assertRaises(
             base_filters.FilterValidationError,
-            filters.factory,
-            {"key": "xyz",
-             "value": "xyz",
-             "op": "oo",
-             "type": "value"})
+            filters.factory({
+                "key": "xyz",
+                "value": "xyz",
+                "op": "oo",
+                "type": "value"}).validate)
 
     def test_complex_value_filter(self):
         self.assertFilter(
diff --git a/tests/test_vpc.py b/tests/test_vpc.py
--- a/tests/test_vpc.py
+++ b/tests/test_vpc.py
@@ -1049,6 +1049,6 @@ def test_egress_validation_error(self):
             {'name': 'sg-find2',
              'resource': 'security-group',
              'filters': [
-                {'type': 'egress',
-                 'InvalidKey': True},
-                {'GroupName': 'sg2'}]})
+                 {'type': 'egress',
+                  'InvalidKey': True},
+                 {'GroupName': 'sg2'}]})