Repository: cloud-custodian/cloud-custodian

Issue:
u'Tags': KeyError when filtering by Tag on RDS instances
I have the following policy:
```
- name: rds-tag-compliance-subsystem
  mode:
    type: config-rule
    role: *****
  resource: rds
  description: |
    Checks for the absense of the 'subsystem' tag for RDS instances
  filters:
    - "tag:subsystem": absent
```

It results in the following error in the Lambda function:
```
u'Tags': KeyError
Traceback (most recent call last):
File "/var/task/custodian_policy.py", line 4, in run
return handler.dispatch_event(event, context)
File "/var/task/c7n/handler.py", line 109, in dispatch_event
p.push(event, context)
File "/var/task/c7n/policy.py", line 676, in push
return mode.run(event, lambda_ctx)
File "/var/task/c7n/policy.py", line 575, in run
resources = super(ConfigRuleMode, self).run(event, lambda_context)
File "/var/task/c7n/policy.py", line 436, in run
resources = self.resolve_resources(event)
File "/var/task/c7n/policy.py", line 558, in resolve_resources
return [source.load_resource(self.cfg_event['configurationItem'])]
File "/var/task/c7n/resources/rds.py", line 152, in load_resource
for t in json.loads(item['Tags'])]
KeyError: u'Tags'
```

When I look at the `rds.py` code, which has this:
```
    def load_resource(self, item):
        resource = super(ConfigRDS, self).load_resource(item)
        resource['Tags'] = [{u'Key': t['key'], u'Value': t['value']}
          for t in json.loads(item['Tags'])]
        return resource
```

And the event being processed:

```
{
   "configurationItemDiff":null,
   "configurationItem":{
      ...
      "configuration":{
        ...
      },
      "supplementaryConfiguration":{
         "Tags":[
            {
               "key":"workload-type",
               "value":"other"
            }
         ]
      },
      "tags":{
         "workload-type":"other"
      },
      ...
   },
   ...
}
```

then I think that it should either be looking for a lowercase 'tags' field or it should look for ["supplementaryConfiguration"]["Tags"]



Discussion:

Resolving patch:
diff --git a/c7n/resources/rds.py b/c7n/resources/rds.py
--- a/c7n/resources/rds.py
+++ b/c7n/resources/rds.py
@@ -45,7 +45,6 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import functools
-import json
 import itertools
 import logging
 import operator
@@ -149,7 +148,7 @@ class ConfigRDS(ConfigSource):
     def load_resource(self, item):
         resource = super(ConfigRDS, self).load_resource(item)
         resource['Tags'] = [{u'Key': t['key'], u'Value': t['value']}
-          for t in json.loads(item['Tags'])]
+          for t in item['supplementaryConfiguration']['Tags']]
         return resource
 
 
@@ -907,7 +906,7 @@ class ConfigRDSSnapshot(ConfigSource):
     def load_resource(self, item):
         resource = super(ConfigRDSSnapshot, self).load_resource(item)
         resource['Tags'] = [{u'Key': t['key'], u'Value': t['value']}
-          for t in json.loads(item['Tags'])]
+          for t in item['supplementaryConfiguration']['Tags']]
         # TODO: Load DBSnapshotAttributes into annotation
         return resource
 
diff --git a/c7n/resources/s3.py b/c7n/resources/s3.py
--- a/c7n/resources/s3.py
+++ b/c7n/resources/s3.py
@@ -324,7 +324,9 @@ def handle_BucketTaggingConfiguration(self, resource, item_value):
             {"Key": k, "Value": v} for k, v in item_value['tagSets'][0]['tags'].items()]
 
     def handle_BucketVersioningConfiguration(self, resource, item_value):
-        assert item_value['status'] in ('Enabled', 'Suspended')
+        # Config defaults versioning to 'Off' for a null value
+        if item_value['status'] not in ('Enabled', 'Suspended'):
+            return
         resource['Versioning'] = {'Status': item_value['status']}
         if item_value['isMfaDeleteEnabled']:
             resource['Versioning']['MFADelete'] = item_value[


Test patch:
diff --git a/tests/data/config/rds-from-rule.json b/tests/data/config/rds-from-rule.json
new file mode 100644
--- /dev/null
+++ b/tests/data/config/rds-from-rule.json
@@ -0,0 +1,2 @@
+{"invokingEvent": "{\"configurationItemDiff\":null,\"configurationItem\":{\"relatedEvents\":[],\"relationships\":[{\"resourceId\":\"default\",\"resourceName\":null,\"resourceType\":\"AWS::RDS::DBSubnetGroup\",\"name\":\"Is associated with DBSubnetGroup\"},{\"resourceId\":\"sg-837620f1\",\"resourceName\":null,\"resourceType\":\"AWS::EC2::SecurityGroup\",\"name\":\"Is associated with SecurityGroup\"}],\"configuration\":{\"dBInstanceIdentifier\":\"c7n-data-index\",\"dBInstanceClass\":\"db.t2.small\",\"engine\":\"postgres\",\"dBInstanceStatus\":\"available\",\"masterUsername\":\"c7nroot\",\"dBName\":\"c7ndata\",\"endpoint\":{\"address\":\"c7n-data-index.cz6wy8cci3uf.us-east-1.rds.amazonaws.com\",\"port\":5432,\"hostedZoneId\":\"Z2R2ITUGPM61AM\"},\"allocatedStorage\":32,\"instanceCreateTime\":\"2017-10-28T15:25:58.412Z\",\"preferredBackupWindow\":\"05:19-05:49\",\"backupRetentionPeriod\":7,\"dBSecurityGroups\":[],\"vpcSecurityGroups\":[{\"vpcSecurityGroupId\":\"sg-837620f1\",\"status\":\"active\"}],\"dBParameterGroups\":[{\"dBParameterGroupName\":\"default.postgres9.6\",\"parameterApplyStatus\":\"in-sync\"}],\"availabilityZone\":\"us-east-1e\",\"dBSubnetGroup\":{\"dBSubnetGroupName\":\"default\",\"dBSubnetGroupDescription\":\"default\",\"vpcId\":\"vpc-d2d616b5\",\"subnetGroupStatus\":\"Complete\",\"subnets\":[{\"subnetIdentifier\":\"subnet-3a334610\",\"subnetAvailabilityZone\":{\"name\":\"us-east-1d\"},\"subnetStatus\":\"Active\"},{\"subnetIdentifier\":\"subnet-efbcccb7\",\"subnetAvailabilityZone\":{\"name\":\"us-east-1b\"},\"subnetStatus\":\"Active\"},{\"subnetIdentifier\":\"subnet-e3b194de\",\"subnetAvailabilityZone\":{\"name\":\"us-east-1e\"},\"subnetStatus\":\"Active\"},{\"subnetIdentifier\":\"subnet-914763e7\",\"subnetAvailabilityZone\":{\"name\":\"us-east-1a\"},\"subnetStatus\":\"Active\"}],\"dBSubnetGroupArn\":null},\"preferredMaintenanceWindow\":\"sat:08:19-sat:08:49\",\"pendingModifiedValues\":{\"dBInstanceClass\":null,\"allocatedStorage\":null,\"masterUserPassword\":null,\"port\":null,\"backupRetentionPeriod\":null,\"multiAZ\":null,\"engineVersion\":null,\"licenseModel\":null,\"iops\":null,\"dBInstanceIdentifier\":null,\"storageType\":null,\"cACertificateIdentifier\":null,\"dBSubnetGroupName\":null},\"latestRestorableTime\":\"2017-10-30T16:22:39.000Z\",\"multiAZ\":false,\"engineVersion\":\"9.6.3\",\"autoMinorVersionUpgrade\":true,\"readReplicaSourceDBInstanceIdentifier\":null,\"readReplicaDBInstanceIdentifiers\":[],\"readReplicaDBClusterIdentifiers\":[],\"licenseModel\":\"postgresql-license\",\"iops\":null,\"optionGroupMemberships\":[{\"optionGroupName\":\"default:postgres-9-6\",\"status\":\"in-sync\"}],\"characterSetName\":null,\"secondaryAvailabilityZone\":null,\"publiclyAccessible\":true,\"statusInfos\":[],\"storageType\":\"gp2\",\"tdeCredentialArn\":null,\"dbInstancePort\":0,\"dBClusterIdentifier\":null,\"storageEncrypted\":false,\"kmsKeyId\":null,\"dbiResourceId\":\"db-KOCU7UR74VM3AC4R3H2UXX7PE4\",\"cACertificateIdentifier\":\"rds-ca-2015\",\"domainMemberships\":[],\"copyTagsToSnapshot\":true,\"monitoringInterval\":0,\"enhancedMonitoringResourceArn\":null,\"monitoringRoleArn\":null,\"promotionTier\":null,\"dBInstanceArn\":\"arn:aws:rds:us-east-1:644160558196:db:c7n-data-index\",\"timezone\":null,\"iAMDatabaseAuthenticationEnabled\":false},\"supplementaryConfiguration\":{\"Tags\":[{\"key\":\"workload-type\",\"value\":\"other\"}]},\"tags\":{\"workload-type\":\"other\"},\"configurationItemVersion\":\"1.2\",\"configurationItemCaptureTime\":\"2017-10-30T16:28:22.554Z\",\"configurationStateId\":1509380902554,\"awsAccountId\":\"644160558196\",\"configurationItemStatus\":\"OK\",\"resourceType\":\"AWS::RDS::DBInstance\",\"resourceId\":\"db-KOCU7UR74VM3AC4R3H2UXX7PE4\",\"resourceName\":\"c7n-data-index\",\"ARN\":\"arn:aws:rds:us-east-1:644160558196:db:c7n-data-index\",\"awsRegion\":\"us-east-1\",\"availabilityZone\":\"us-east-1e\",\"configurationStateMd5Hash\":\"3af295262801937f6948189cfa0d5874\",\"resourceCreationTime\":\"2017-10-28T15:25:58.412Z\"},\"notificationCreationTime\":\"2017-10-30T21:01:51.345Z\",\"messageType\":\"ConfigurationItemChangeNotification\",\"recordVersion\":\"1.2\"}"}
+
diff --git a/tests/data/config/s3-from-rule.json b/tests/data/config/s3-from-rule.json
new file mode 100644
--- /dev/null
+++ b/tests/data/config/s3-from-rule.json
@@ -0,0 +1 @@
+{"invokingEvent": "{\"configurationItemDiff\":null,\"configurationItem\":{\"relatedEvents\":[\"99f04786-16e3-4f91-98a7-b7d710fe1e3b\"],\"relationships\":[],\"configuration\":{\"name\":\"c7n-fire-logs\",\"owner\":{\"displayName\":\"mandeep.bal\",\"id\":\"e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15\"},\"creationDate\":\"2017-09-15T02:05:40.000Z\"},\"supplementaryConfiguration\":{\"AccessControlList\":\"{\\\"grantSet\\\":null,\\\"grantList\\\":[{\\\"grantee\\\":{\\\"id\\\":\\\"e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15\\\",\\\"displayName\\\":\\\"mandeep.bal\\\"},\\\"permission\\\":\\\"FullControl\\\"}],\\\"owner\\\":{\\\"displayName\\\":\\\"mandeep.bal\\\",\\\"id\\\":\\\"e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15\\\"},\\\"isRequesterCharged\\\":false}\",\"BucketAccelerateConfiguration\":{\"status\":null},\"BucketLoggingConfiguration\":{\"destinationBucketName\":null,\"logFilePrefix\":null},\"BucketNotificationConfiguration\":{\"configurations\":{}},\"BucketPolicy\":{\"policyText\":null},\"BucketVersioningConfiguration\":{\"status\":\"Off\",\"isMfaDeleteEnabled\":null},\"IsRequesterPaysEnabled\":false},\"tags\":{},\"configurationItemVersion\":\"1.2\",\"configurationItemCaptureTime\":\"2017-09-15T02:08:26.719Z\",\"configurationStateId\":1505441306719,\"awsAccountId\":\"644160558196\",\"configurationItemStatus\":\"ResourceDiscovered\",\"resourceType\":\"AWS::S3::Bucket\",\"resourceId\":\"c7n-fire-logs\",\"resourceName\":\"c7n-fire-logs\",\"ARN\":\"arn:aws:s3:::c7n-fire-logs\",\"awsRegion\":\"us-east-1\",\"availabilityZone\":\"Regional\",\"configurationStateMd5Hash\":\"6d3815aa37dbbd79e55abc72a11fb298\",\"resourceCreationTime\":\"2017-09-15T02:05:40.000Z\"},\"notificationCreationTime\":\"2017-10-30T08:54:48.029Z\",\"messageType\":\"ConfigurationItemChangeNotification\",\"recordVersion\":\"1.2\"}"}
diff --git a/tests/test_rds.py b/tests/test_rds.py
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -24,7 +24,7 @@
 
 from botocore.exceptions import ClientError
 import boto3
-from .common import BaseTest
+from .common import BaseTest, event_data
 
 from c7n.executor import MainThreadExecutor
 from c7n.filters import FilterValidationError
@@ -36,6 +36,16 @@
 
 class RDSTest(BaseTest):
 
+    def test_rds_config_event(self):
+        event = event_data('rds-from-rule.json', 'config')        
+        p = self.load_policy({'name': 'rds', 'resource': 'rds'})
+        source = p.resource_manager.get_source('config')
+        resource_config = json.loads(event['invokingEvent'])['configurationItem']
+        resource = source.load_resource(resource_config)
+        self.assertEqual(
+            resource['Tags'],
+            [{u'Key': u'workload-type', u'Value': u'other'}])
+        
     def test_rds_stop(self):
         session_factory = self.replay_flight_data('test_rds_stop')
         db_instance_id = 'rds-test-instance-1'
diff --git a/tests/test_s3.py b/tests/test_s3.py
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import datetime
 import functools
 import json
 import os
@@ -23,6 +24,7 @@
 from unittest import TestCase
 
 from botocore.exceptions import ClientError
+from dateutil.tz import tzutc
 
 from c7n.executor import MainThreadExecutor
 from c7n.resources import s3
@@ -664,6 +666,39 @@ def test_load_item_resource(self):
              })
 
 
+    def test_load_item_resource_config_event(self):
+        event = event_data('s3-from-rule.json', 'config')
+        p = self.load_policy({
+            'name': 's3cfg',
+            'resource': 's3'})
+        source = p.resource_manager.get_source('config')
+        self.maxDiff = None
+
+        resource_config = json.loads(event['invokingEvent'])['configurationItem']
+        resource = source.load_resource(resource_config)
+        self.assertEqual(
+            resource,
+            {u'Acl': {
+                u'Grants': [{
+                    u'Grantee': {
+                        u'ID': u'e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15',
+                        u'Type': u'CanonicalUser'},
+                        u'Permission': u'FULL_CONTROL'}],
+                u'Owner': {u'DisplayName': u'mandeep.bal',
+                           u'ID': u'e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15'}},
+             u'CreationDate': datetime.datetime(2017, 9, 15, 2, 5, 40, tzinfo=tzutc()),
+             u'Lifecycle': None,
+             u'Location': None,
+             u'Logging': None,
+             u'Name': u'c7n-fire-logs',
+             u'Notification': {},
+             u'Policy': None,
+             u'Replication': None,
+             u'Tags': [],
+             u'Versioning': None,
+             u'Website': None})
+
+
 class S3Test(BaseTest):
 
     def test_multipart_large_file(self):