Repository: cloud-custodian/cloud-custodian

Issue:
'source: config' errors for S3 resource
Policy snippet
```yml
  - name: blahblah
    resource: s3
    region: us-east-1
    source: config
    ...
```
Errors:
```
2017-10-22 11:25:17,078: policyrun:DEBUG Running policy:blahblah account:acct1 region:us-east-1
2017-10-22 11:25:44,412: custodian.resources.s3:ERROR Exception getting resources from config
 u'redirectsAllRequestsTo'
2017-10-22 11:25:44,414: custodian.output:ERROR Error while executing policy
Traceback (most recent call last):
 File "/usr/local/custodian/site-packages/c7n/policy.py", line 305, in run
  resources = self.policy.resource_manager.resources()
 File "/usr/local/custodian/site-packages/c7n/query.py", line 395, in resources
  resources = self.augment(self.source.resources(query))
 File "/usr/local/custodian/site-packages/c7n/query.py", line 305, in resources
  results.extend(f.result())
 File "/usr/local/custodian/site-packages/concurrent/futures/_base.py", line 422, in result
  return self.__get_result()
 File "/usr/local/custodian/site-packages/concurrent/futures/thread.py", line 62, in run
  result = self.fn(*self.args, **self.kwargs)
 File "/usr/local/custodian/site-packages/c7n/query.py", line 271, in get_resources
  results.append(self.load_resource(revisions[0]))
 File "/usr/local/custodian/site-packages/c7n/resources/s3.py", line 152, in load_resource
  method(resource, v)
 File "/usr/local/custodian/site-packages/c7n/resources/s3.py", line 343, in handle_BucketWebsiteConfiguration
  'HostName': item_value['redirectsAllRequestsTo']['hostName'],
KeyError: u'redirectsAllRequestsTo'
```
```
2017-10-22 11:14:17,252: policyrun:DEBUG Running policy:blahblah account:acct1 region:us-east-1
2017-10-22 11:14:40,089: custodian.resources.s3:ERROR Exception getting resources from config
 u'CloudFunctionConfiguration'
2017-10-22 11:14:40,092: custodian.output:ERROR Error while executing policy
Traceback (most recent call last):
 File "/usr/local/custodian/site-packages/c7n/policy.py", line 305, in run
  resources = self.policy.resource_manager.resources()
 File "/usr/local/custodian/site-packages/c7n/query.py", line 395, in resources
  resources = self.augment(self.source.resources(query))
 File "/usr/local/custodian/site-packages/c7n/query.py", line 305, in resources
  results.extend(f.result())
 File "/usr/local/custodian/site-packages/concurrent/futures/_base.py", line 422, in result
  return self.__get_result()
 File "/usr/local/custodian/site-packages/concurrent/futures/thread.py", line 62, in run
  result = self.fn(*self.args, **self.kwargs)
 File "/usr/local/custodian/site-packages/c7n/query.py", line 271, in get_resources
  results.append(self.load_resource(revisions[0]))
 File "/usr/local/custodian/site-packages/c7n/resources/s3.py", line 152, in load_resource
  method(resource, v)
 File "/usr/local/custodian/site-packages/c7n/resources/s3.py", line 287, in handle_BucketNotificationConfiguration
  d.setdefault(self.NotifyTypeMap[n['type']], []).append(ninfo)
KeyError: u'CloudFunctionConfiguration'
```
```
2017-10-22 10:59:24,309: policyrun:DEBUG Running policy:blahblah account:acct1 region:us-east-1
2017-10-22 10:59:28,550: custodian.resources.s3:ERROR Exception getting resources from config
 'NoneType' object is not subscriptable
2017-10-22 10:59:28,551: custodian.output:ERROR Error while executing policy
Traceback (most recent call last):
 File "/usr/local/custodian/site-packages/c7n/policy.py", line 305, in run
  resources = self.policy.resource_manager.resources()
 File "/usr/local/custodian/site-packages/c7n/query.py", line 395, in resources
  resources = self.augment(self.source.resources(query))
 File "/usr/local/custodian/site-packages/c7n/query.py", line 305, in resources
  results.extend(f.result())
 File "/usr/local/custodian/site-packages/concurrent/futures/_base.py", line 422, in result
  return self.__get_result()
 File "/usr/local/custodian/site-packages/concurrent/futures/thread.py", line 62, in run
  result = self.fn(*self.args, **self.kwargs)
 File "/usr/local/custodian/site-packages/c7n/query.py", line 271, in get_resources
  results.append(self.load_resource(revisions[0]))
 File "/usr/local/custodian/site-packages/c7n/resources/s3.py", line 152, in load_resource
  method(resource, v)
 File "/usr/local/custodian/site-packages/c7n/resources/s3.py", line 257, in handle_BucketLifecycleConfiguration
  rr['Filter'] = self.convertLifePredicate(r['filter']['predicate'])
 File "/usr/local/custodian/site-packages/c7n/resources/s3.py", line 262, in convertLifePredicate
  if p['type'] == 'LifecyclePrefixPredicate':
TypeError: 'NoneType' object is not subscriptable
```


Discussion:
we've got a few good unit tests here but it looks like their focused on full coverage semantics, its clear we need a few more to cover some of the delta cases. The s3 config representation requires the most transformation of all the config support resources.
i've been struggling to reproduce this or find an extant bucket that can trigger, lets chat to see if we can find a good test case.
spoke too soon, found one for for the first case.

Resolving patch:
diff --git a/c7n/resources/s3.py b/c7n/resources/s3.py
--- a/c7n/resources/s3.py
+++ b/c7n/resources/s3.py
@@ -257,7 +257,8 @@ def handle_BucketLifecycleConfiguration(self, resource, item_value):
             if 'filter' not in r or not r['filter']:
                 continue
 
-            rr['Filter'] = self.convertLifePredicate(r['filter']['predicate'])
+            if r['filter']['predicate']:
+                rr['Filter'] = self.convertLifePredicate(r['filter']['predicate'])
 
         resource['Lifecycle'] = {'Rules': rules}
 
@@ -281,6 +282,7 @@ def convertLifePredicate(self, p):
     NotifyTypeMap = {
         'QueueConfiguration': 'QueueConfigurations',
         'LambdaConfiguration': 'LambdaFunctionConfigurations',
+        'CloudFunctionConfiguration': 'LambdaFunctionConfigurations',
         'TopicConfiguration': 'TopicConfigurations'}
 
     def handle_BucketNotificationConfiguration(self, resource, item_value):
@@ -345,8 +347,8 @@ def handle_BucketWebsiteConfiguration(self, resource, item_value):
                 'Key': item_value['errorDocument']}
         if item_value['redirectAllRequestsTo']:
             website['RedirectAllRequestsTo'] = {
-                'HostName': item_value['redirectsAllRequestsTo']['hostName'],
-                'Protocol': item_value['redirectsAllRequestsTo']['protocol']}
+                'HostName': item_value['redirectAllRequestsTo']['hostName'],
+                'Protocol': item_value['redirectAllRequestsTo']['protocol']}
         for r in item_value['routingRules']:
             redirect = {}
             rule = {'Redirect': redirect}


Test patch:
diff --git a/tests/data/config/s3-lifecycle-null-predicate.json b/tests/data/config/s3-lifecycle-null-predicate.json
new file mode 100644
--- /dev/null
+++ b/tests/data/config/s3-lifecycle-null-predicate.json
@@ -0,0 +1,38 @@
+{
+    "version": "1.2",
+    "accountId": "644160558196",
+    "configurationItemCaptureTime": "2017-06-14T01:39:07.454000-04:00",
+    "configurationItemStatus": "ResourceDiscovered",
+    "configurationStateId": "1497418747454",
+    "configurationItemMD5Hash": "b15227ff0a9542b95573e2551213f207",
+    "arn": "arn:aws:s3:::storage-dev",
+    "resourceType": "AWS::S3::Bucket",
+    "resourceId": "storage-dev",
+    "resourceName": "storage-dev",
+    "awsRegion": "us-east-1",
+    "availabilityZone": "Regional",
+    "resourceCreationTime": "2017-02-16T16:17:05-05:00",
+    "tags": {
+	"App": "StorageDev",
+	"CMDBEnvironment": "dev",
+	"Environment": "dev",
+	"OwnerContact": "me@example.com",
+	"aws:cloudformation:logical-id": "UserDataBucket",
+	"aws:cloudformation:stack-id": "arn:aws:cloudformation:us-east-1:644160558196:stack/storage-dev/3cdd9720-f48d-11e6-a02a-503f23fb11fe",
+	"aws:cloudformation:stack-name": "storage-dev"
+    },
+    "relatedEvents": [],
+    "relationships": [],
+    "configuration": "{\"name\":\"storage-dev\",\"owner\":{\"displayName\":\"aws-example\",\"id\":\"c7bb704ac09\"},\"creationDate\":\"2017-02-16T21:17:05.000Z\"}",
+    "supplementaryConfiguration": {
+	"AccessControlList": "\"{\\\"grantSet\\\":null,\\\"grantList\\\":[{\\\"grantee\\\":{\\\"id\\\":\\\"c7bb704ac09\\\",\\\"displayName\\\":\\\"aws-example\\\"},\\\"permission\\\":\\\"FullControl\\\"}],\\\"owner\\\":{\\\"displayName\\\":\\\"aws-example\\\",\\\"id\\\":\\\"c7bb704ac09\\\"},\\\"isRequesterCharged\\\":false}\"",
+	"BucketAccelerateConfiguration": "{\"status\":null}",
+	"BucketLifecycleConfiguration": "{\"rules\":[{\"id\":\"RemoveAbortedUploads\",\"prefix\":null,\"status\":\"Enabled\",\"filter\":{\"predicate\":null},\"expirationInDays\":1,\"expiredObjectDeleteMarker\":false,\"noncurrentVersionExpirationInDays\":-1,\"expirationDate\":null,\"transitions\":null,\"noncurrentVersionTransitions\":null,\"abortIncompleteMultipartUpload\":{\"daysAfterInitiation\":1}},{\"id\":\"RemoveDeleteMarkers\",\"prefix\":null,\"status\":\"Enabled\",\"filter\":{\"predicate\":null},\"expirationInDays\":-1,\"expiredObjectDeleteMarker\":true,\"noncurrentVersionExpirationInDays\":10,\"expirationDate\":null,\"transitions\":null,\"noncurrentVersionTransitions\":null,\"abortIncompleteMultipartUpload\":null},{\"id\":\"RemoveDeleteTagged\",\"prefix\":null,\"status\":\"Enabled\",\"filter\":{\"predicate\":{\"type\":\"LifecycleTagPredicate\",\"tag\":{\"key\":\"ACTION\",\"value\":\"PURGE\"}}},\"expirationInDays\":1,\"expiredObjectDeleteMarker\":false,\"noncurrentVersionExpirationInDays\":-1,\"expirationDate\":null,\"transitions\":null,\"noncurrentVersionTransitions\":null,\"abortIncompleteMultipartUpload\":null}]}",
+	"BucketLoggingConfiguration": "{\"destinationBucketName\":null,\"logFilePrefix\":null}",
+	"BucketNotificationConfiguration": "{\"configurations\":{}}",
+	"BucketPolicy": "{\"policyText\":\"{\\\"Version\\\":\\\"2008-10-17\\\",\\\"Statement\\\":[{\\\"Sid\\\":\\\"DenyUnEncryptedObjectUploads\\\",\\\"Effect\\\":\\\"Deny\\\",\\\"Principal\\\":\\\"*\\\",\\\"Action\\\":\\\"s3:PutObject\\\",\\\"Resource\\\":\\\"arn:aws:s3:::storage-dev/*\\\",\\\"Condition\\\":{\\\"StringNotEquals\\\":{\\\"s3:x-amz-server-side-encryption\\\":\\\"AES256\\\"}}},{\\\"Sid\\\":\\\"RequiredEncryptedPutObject\\\",\\\"Effect\\\":\\\"Deny\\\",\\\"Principal\\\":\\\"*\\\",\\\"Action\\\":\\\"s3:PutObject\\\",\\\"Resource\\\":\\\"arn:aws:s3:::storage-dev/*\\\",\\\"Condition\\\":{\\\"StringNotEquals\\\":{\\\"s3:x-amz-server-side-encryption\\\":[\\\"AES256\\\",\\\"aws:kms\\\"]}}}]}\"}",
+	"BucketTaggingConfiguration": "{\"tagSets\":[{\"tags\":{\"aws:cloudformation:stack-name\":\"storage-dev\",\"App\":\"StorageDev\",\"aws:cloudformation:stack-id\":\"arn:aws:cloudformation:us-east-1:644160558196:stack/storage-dev/3cdd9720-f48d-11e6-a02a-503f23fb11fe\",\"OwnerContact\":\"me@example.com\",\"Environment\":\"dev\",\"aws:cloudformation:logical-id\":\"UserDataBucket\",\"CMDBEnvironment\":\"dev\"}}]}",
+	"BucketVersioningConfiguration": "{\"status\":\"Enabled\",\"isMfaDeleteEnabled\":null}",
+	"IsRequesterPaysEnabled": "false"
+    }
+}
diff --git a/tests/data/config/s3-website-redirect.json b/tests/data/config/s3-website-redirect.json
new file mode 100644
--- /dev/null
+++ b/tests/data/config/s3-website-redirect.json
@@ -0,0 +1,34 @@
+{
+    "version": "1.2",
+    "accountId": "644160558196",
+    "configurationItemCaptureTime": "2017-06-14T00:57:03.508000-04:00",
+    "configurationItemStatus": "ResourceDiscovered",
+    "configurationStateId": "1497416223508",
+    "configurationItemMD5Hash": "40d02f979d083aac58aa0ef1adf0e365",
+    "arn": "arn:aws:s3:::partnerships-lossmit-dev",
+    "resourceType": "AWS::S3::Bucket",
+    "resourceId": "analysis-dev",
+    "resourceName": "analysis-dev",
+    "awsRegion": "us-east-1",
+    "availabilityZone": "Regional",
+    "resourceCreationTime": "2017-01-31T07:59:21-05:00",
+    "tags": {
+	"App": "ANALYSIS",
+	"Environment": "QA",
+	"OwnerContact": "me@example.com"
+    },
+    "relatedEvents": [],
+    "relationships": [],
+    "configuration": "{\"name\":\"analysis-dev\",\"owner\":{\"displayName\":\"aws-example\",\"id\":\"b08aa2c849dbcbab5000111aaancc0002334123123123123123123\"},\"creationDate\":\"2017-01-31T12:59:21.000Z\"}",
+    "supplementaryConfiguration": {
+	"AccessControlList": "\"{\\\"grantSet\\\":null,\\\"grantList\\\":[{\\\"grantee\\\":{\\\"id\\\":\\\"b08aa2c849dbcbab5000111aaancc0002334123123123123123123\\\",\\\"displayName\\\":\\\"aws-example\\\"},\\\"permission\\\":\\\"FullControl\\\"}],\\\"owner\\\":{\\\"displayName\\\":\\\"aws-example\\\",\\\"id\\\":\\\"b08aa2c849dbcbab5000111aaancc0002334123123123123123123\\\"},\\\"isRequesterCharged\\\":false}\"",
+	"BucketAccelerateConfiguration": "{\"status\":null}",
+	"BucketLoggingConfiguration": "{\"destinationBucketName\":null,\"logFilePrefix\":null}",
+	"BucketNotificationConfiguration": "{\"configurations\":{\"5bbcd52e-b630-4b46-b458-99bc04537d78\":{\"type\":\"LambdaConfiguration\",\"functionARN\":\"arn:aws:lambda:us-east-1:644160558196:function:fNotification\",\"events\":[\"s3:ObjectCreated:*\"],\"objectPrefixes\":[],\"filter\":{\"s3KeyFilter\":{\"filterRules\":[{\"name\":\"Prefix\",\"value\":\"Source\"},{\"name\":\"Suffix\",\"value\":\"txt\"}]}}}}}",
+	"BucketPolicy": "{\"policyText\":\"{\\\"Version\\\":\\\"2012-10-17\\\",\\\"Statement\\\":[{\\\"Sid\\\":\\\"RequiredEncryptedPutObject\\\",\\\"Effect\\\":\\\"Deny\\\",\\\"Principal\\\":\\\"*\\\",\\\"Action\\\":\\\"s3:PutObject\\\",\\\"Resource\\\":\\\"arn:aws:s3:::analysis-dev/*\\\",\\\"Condition\\\":{\\\"StringNotEquals\\\":{\\\"s3:x-amz-server-side-encryption\\\":[\\\"AES256\\\",\\\"aws:kms\\\"]}}}]}\"}",
+	"BucketTaggingConfiguration": "{\"tagSets\":[{\"tags\":{\"OwnerContact\":\"me@example.com\",\"App\":\"ANALYSIS\",\"Environment\":\"QA\"}}]}",
+	"BucketVersioningConfiguration": "{\"status\":\"Off\",\"isMfaDeleteEnabled\":null}",
+	"BucketWebsiteConfiguration": "{\"indexDocumentSuffix\":null,\"errorDocument\":null,\"redirectAllRequestsTo\":{\"protocol\":\"https\",\"hostName\":\"www.google.com/\",\"replaceKeyPrefixWith\":null,\"replaceKeyWith\":null,\"httpRedirectCode\":null},\"routingRules\":[]}",
+	"IsRequesterPaysEnabled": "false"
+    }
+}
diff --git a/tests/test_s3.py b/tests/test_s3.py
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -651,6 +651,20 @@ def test_config_normalize_notification(self):
                   u'QueueArn': 'arn:aws:sqs:us-east-1:644160558196:test-queue',
                   u'Events': ['s3:ObjectCreated:*']}]})
 
+    def test_config_normalize_lifecycle_null_predicate(self):
+        event = event_data('s3-lifecycle-null-predicate.json', 'config')
+        p = self.load_policy({'name': 's3cfg', 'resource': 's3'})
+        source = p.resource_manager.get_source('config')
+        resource = source.load_resource(event)
+        rule = resource['Lifecycle']['Rules'][0]
+        self.assertEqual(
+            rule,
+            {'AbortIncompleteMultipartUpload': {'DaysAfterInitiation': 1},
+             'Expiration': {'Days': 1},
+             'ID': 'RemoveAbortedUploads',
+             'NoncurrentVersionExpiration': {'NoncurrentDays': -1},
+             'Status': 'Enabled'})
+
     def test_config_normalize_lifecycle_and_predicate(self):
         event = event_data('s3-lifecycle-and-predicate.json', 'config')
         p = self.load_policy({'name': 's3cfg', 'resource': 's3'})
@@ -719,6 +733,16 @@ def test_config_normalize_replication(self):
                         '/custodian-replicated-custodian-replicated'
                         '-west-s3-repl-role')}})
 
+    def test_config_normalize_website_redirect(self):
+        event = event_data('s3-website-redirect.json', 'config')
+        p = self.load_policy({'name': 's3cfg', 'resource': 's3'})
+        source = p.resource_manager.get_source('config')
+        self.maxDiff = None
+        resource = source.load_resource(event)
+        self.assertEqual(
+            resource['Website'],
+            {'RedirectAllRequestsTo': {'HostName': 'www.google.com/', 'Protocol': 'https'}})
+        
     def test_config_normalize_website(self):
         event = event_data('s3-website.json', 'config')
         p = self.load_policy({'name': 's3cfg', 'resource': 's3'})