Repository: cloud-custodian/cloud-custodian

Issue:
boolean operator optimization
now that we have skew models in the mix, we can do better optimization around filters nested in operators to do bulk operations instead of the individual invocation. its also a source of some potential ux bugs as some of our filters have to do bulk queries to optimize roundtrips and they currently aren't usable with nested operators.



Discussion:

Resolving patch:
diff --git a/c7n/filters/core.py b/c7n/filters/core.py
--- a/c7n/filters/core.py
+++ b/c7n/filters/core.py
@@ -137,10 +137,10 @@ class Filter(object):
 
     log = logging.getLogger('custodian.filters')
 
-    schema = {'type': 'object'}
-
     metrics = ()
 
+    schema = {'type': 'object'}
+
     def __init__(self, data, manager=None):
         self.data = data
         self.manager = manager
@@ -164,15 +164,29 @@ def __init__(self, data, registry, manager):
         super(Or, self).__init__(data)
         self.registry = registry
         self.filters = registry.parse(self.data.values()[0], manager)
+        self.manager = manager
 
-    # TODO support resource set processing with or (will need identity
-    # metadata per resource type), ala tags set_id or query metamodel branch
-    def __call__(self, i):
+    def process(self, resources, event=None):
+        if self.manager:
+            return self.process_set(resources, event)
+        return super(Or, self).process(resources, event)
+
+    def __call__(self, r):
+        """Fallback for older unit tests that don't utilize a query manager"""
         for f in self.filters:
-            if f(i):
+            if f(r):
                 return True
         return False
 
+    def process_set(self, resources, event):
+        resource_type = self.manager.query.resolve(self.manager.resource_type)
+        resource_map = {r[resource_type.id]: r for r in resources}
+        results = set()
+        for f in self.filters:
+            results = results.union([
+                r[resource_type.id] for r in f.process(resources, event)])
+        return [resource_map[r_id] for r_id in results]
+
 
 class And(Filter):
 
@@ -181,14 +195,15 @@ def __init__(self, data, registry, manager):
         self.registry = registry
         self.filters = registry.parse(self.data.values()[0], manager)
 
-    def __call__(self, i):
+    def process(self, resources, events=None):
         for f in self.filters:
-            if not f(i):
-                return False
-        return True
+            resources = f.process(resources, events)
+        return resources
 
 
 class Delay(Filter):
+    # Use of delay in a policy is highly suspect as its indicative
+    # of workaround races instead of deterministic behavior.
 
     schema = {'type': 'object', 'additionalProperties': False,
               'required': ['type', 'seconds'],
@@ -279,7 +294,7 @@ def match(self, i):
                 if t.get('Key') == tk:
                     r = t.get('Value')
                     break
-        elif not '.' in self.k and not '[' in self.k and not '(' in self.k:
+        elif '.' not in self.k and '[' not in self.k and '(' not in self.k:
             r = i.get(self.k)
         elif self.expr:
             r = self.expr.search(i)
@@ -310,9 +325,9 @@ def process_value_type(self, sentinel, value):
             return sentinel, value.strip().lower()
         elif self.vtype == 'integer':
             try:
-                v = int(value.strip())
+                value = int(value.strip())
             except ValueError:
-                v = 0
+                value = 0
         elif self.vtype == 'size':
             try:
                 return sentinel, len(value)
@@ -394,3 +409,4 @@ def process(self, resources, event=None):
         if self(event):
             return resources
         return []
+
diff --git a/c7n/policy.py b/c7n/policy.py
--- a/c7n/policy.py
+++ b/c7n/policy.py
@@ -452,8 +452,7 @@ def __call__(self):
     run = __call__
 
     def _write_file(self, rel_path, value):
-        with open(
-                os.path.join(self.ctx.log_dir, rel_path), 'w') as fh:
+        with open(os.path.join(self.ctx.log_dir, rel_path), 'w') as fh:
             fh.write(value)
 
     def get_resource_manager(self):
diff --git a/c7n/resources/kms.py b/c7n/resources/kms.py
--- a/c7n/resources/kms.py
+++ b/c7n/resources/kms.py
@@ -47,8 +47,7 @@ class Meta(object):
     resource_type = Meta
 
     def augment(self, resources):
-        resources = [r for r in resources if 'TargetKeyId' in r]
-        return super(KeyAlias, self).augment(resources)
+        return [r for r in resources if 'TargetKeyId' in r]
 
 
 @resources.register('kms-key')


Test patch:
diff --git a/tests/test_ec2.py b/tests/test_ec2.py
--- a/tests/test_ec2.py
+++ b/tests/test_ec2.py
@@ -289,6 +289,25 @@ def test_ec2_stop(self):
         resources = policy.run()
         self.assertEqual(len(resources), 1)
 
+class TestOr(BaseTest):
+
+    def test_ec2_or_condition(self):
+        session_factory = self.replay_flight_data(
+            'test_ec2_stop')
+        policy = self.load_policy({
+            'name': 'ec2-test-snapshot',
+            'resource': 'ec2',
+            'filters': [
+                {"or": [
+                    {"tag:Name": "CompileLambda"},
+                    {"tag:Name": "Spinnaker"}]}]
+        }, session_factory=session_factory)
+        resources = policy.run()
+        self.assertEqual(len(resources), 2)
+        self.assertEqual(
+            sorted([r['InstanceId'] for r in resources]),
+            [u'i-13413bd7', u'i-1aebf7c0'])
+
 
 class TestSnapshot(BaseTest):
 
diff --git a/tests/test_filters.py b/tests/test_filters.py
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -58,12 +58,13 @@ def test_or(self):
             'or': [
                 {'Architecture': 'x86_64'},
                 {'Architecture': 'armv8'}]})
+        results = [instance(Architecture='x86_64')]
         self.assertEqual(
-            f(instance(Architecture='x86_64')),
-            True)
+            f.process(results),
+            results)
         self.assertEqual(
-            f(instance(Architecture='amd64')),
-            False)
+            f.process([instance(Architecture='amd64')]),
+            [])
 
 
 class TestAndFilter(unittest.TestCase):
@@ -73,20 +74,21 @@ def test_and(self):
             'and': [
                 {'Architecture': 'x86_64'},
                 {'Color': 'green'}]})
+        results = [instance(Architecture='x86_64', Color='green')]
         self.assertEqual(
-            f(instance(
-                Architecture='x86_64',
-                Color='green')),
-            True)
+            f.process(results),
+            results)
         self.assertEqual(
-            f(instance(
-                Architecture='x86_64',
-                Color='blue')),
-            False)
+            f.process([
+                instance(
+                    Architecture='x86_64',
+                    Color='blue')]),
+            [])
         self.assertEqual(
-            f(instance(
-                Architecture='x86_64')),
-            False)
+            f.process([
+                instance(
+                    Architecture='x86_64')]),
+            [])
 
 
 class TestGlobValue(unittest.TestCase):
@@ -164,14 +166,13 @@ def i(d):
         self.assertFilter(fdata, i(three_months), False)
         self.assertFilter(fdata, i(two_months), False)
         self.assertFilter(fdata, i(one_month), True)
-        self.assertFilter(fdata, i(now), True)        
-        
+        self.assertFilter(fdata, i(now), True)
+
     def test_expiration(self):
 
         now = datetime.now(tz=tz.tzutc())
         three_months = now + timedelta(90)
         two_months = now + timedelta(60)
-        one_month = now + timedelta(30)
 
         def i(d):
             return instance(LaunchTime=d)
@@ -185,7 +186,7 @@ def i(d):
 
         self.assertFilter(fdata, i(three_months), False)
         self.assertFilter(fdata, i(two_months), True)
-        self.assertFilter(fdata, i(now), True)        
+        self.assertFilter(fdata, i(now), True)
 
 
 class TestInstanceAge(BaseFilterTest):
@@ -245,7 +246,6 @@ def i(d, action='stop'):
                  "Value": "not compliant: %s@%s" % (
                     action, d.strftime("%Y/%m/%d"))}])
 
-
         for ii, v in [
                 (i(yesterday), True),
                 (i(now), True),
@@ -336,7 +336,10 @@ def test_complex_validator(self):
 
     def test_complex_value_filter(self):
         self.assertFilter(
-            {"key": "length(BlockDeviceMappings[?Ebs.DeleteOnTermination == `true`].Ebs.DeleteOnTermination)",
+            {"key": (
+                "length(BlockDeviceMappings"
+                "[?Ebs.DeleteOnTermination == `true`]"
+                ".Ebs.DeleteOnTermination)"),
              "value": 0,
              "type": "value",
              "op": "gt"},