Repository: cloud-custodian/cloud-custodian

Issue:
Access Key Groupings
If an IAM account has never been used the "access_keys.last_used_date" will be absent.  I want to disable access keys that have never been used AND the last rotated key is greater than 60 days.  Neither one of the access keys of the IAM account "read-s3" (see below) should violate this policy.  This seems to only happen when two access keys are active.

**Policy should match an access key with the following:**
- Key Active: YES
- Never Been Used: YES
- Access Key Rotated Greater Than 60 Days: YES


**The incorrect access keys that match from the following policy:**
**_"read-s3" Access Key 1:_**
- Key Active: YES
- Never Been Used: YES
- Access Key Rotated Greater Than 60 Days: NO

**_"read-s3" Access Key 2:_**
- Key Active: YES
- Never Been Used: NO
- Access Key Rotated Greater Than 60 Days: YES

I am wondering if it is grouping the access keys (similar to the old bug in the security groups) and pulling and evaluating dates/null values from multiple access keys.

**Policy:**
```
    name: iam_accounts-disable-never_used
    description: |
       Disable IAM accounts that have never been used in the last 60 days
    resource: iam-user
    filters:
      - type: credential
        key: access_keys.active
        value: true
      - type: credential
        key: access_keys.last_used_date
        value: absent
      - type: credential
        key: access_keys.last_rotated
        value_type: age
        value: 60
        op: gte
    actions:
      - type: remove-keys
        age: 60
        disable: true
```

**Resources File:**
```
[
  {
    "UserName": "read-s3",
    "c7n:credential-report": {
      "password_next_rotation": null,
      "password_enabled": false,
      "access_keys": [
        {
          "active": true,
          "last_rotated": "2016-12-23T15:55:44+00:00",
          "last_used_date": null,
          "last_used_service": null,
          "last_used_region": null
        },
        {
          "active": true,
          "last_rotated": "2016-09-26T18:39:07+00:00",
          "last_used_region": null,
          "last_used_date": "2017-01-09T23:21:00+00:00",
          "last_used_service": "s3"
        }
      ],
      "user_creation_time": "2016-09-26T18:39:07+00:00",
      "mfa_active": false,
      "arn": "arn:aws:iam::xxxxxxx:user/read-s3",
      "password_last_used": null,
      "password_last_changed": null,
      "user": "security-read-s3"
    },
    "CreateDate": "2016-09-26T18:39:07+00:00",
    "UserId": "xxxxx",
    "Path": "/",
    "Arn": "arn:aws:iam::xxxxxxxx:user/read-s3"
  }
]
```


Discussion:
I'm also having these issues.

Resolving patch:
diff --git a/c7n/filters/core.py b/c7n/filters/core.py
--- a/c7n/filters/core.py
+++ b/c7n/filters/core.py
@@ -161,6 +161,8 @@ class Filter(object):
     metrics = ()
     permissions = ()
     schema = {'type': 'object'}
+    # schema aliases get hoisted into a jsonschema definition
+    # location, and then referenced inline.
     schema_alias = None
 
     def __init__(self, data, manager=None):
@@ -179,6 +181,8 @@ def process(self, resources, event=None):
         return list(filter(self, resources))
 
     def get_block_operator(self):
+        """Determine the immediate parent boolean operator for a filter"""
+        # Top level operator is `and`
         block_stack = ['and']
         for f in self.manager.iter_filters(block_end=True):
             if f is None:
@@ -190,6 +194,33 @@ def get_block_operator(self):
                 break
         return block_stack[-1]
 
+    def merge_annotation(self, r, annotation_key, values):
+        block_op = self.get_block_operator()
+        if block_op in ('and', 'not'):
+            r[self.matched_annotation_key] = intersect_list(
+                values,
+                r.get(self.matched_annotation_key))
+
+        if not values and block_op != 'or':
+            return
+
+        r_matched = r.setdefault(self.matched_annotation_key, [])
+        for k in values:
+            if k not in r_matched:
+                r_matched.append(k)
+
+
+def intersect_list(a, b):
+    if b is None:
+        return a
+    elif a is None:
+        return b
+    res = []
+    for x in a:
+        if x in b:
+            res.append(x)
+    return res
+
 
 class BooleanGroupFilter(Filter):
 
diff --git a/c7n/resources/account.py b/c7n/resources/account.py
--- a/c7n/resources/account.py
+++ b/c7n/resources/account.py
@@ -85,7 +85,7 @@ def process(self, resources, event=None):
         results = []
         info = report.get('<root_account>')
         for r in resources:
-            if self.match(info):
+            if self.match(r, info):
                 r['c7n:credential-report'] = info
                 results.append(r)
         return results
diff --git a/c7n/resources/iam.py b/c7n/resources/iam.py
--- a/c7n/resources/iam.py
+++ b/c7n/resources/iam.py
@@ -13,8 +13,10 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from collections import OrderedDict
 import csv
 import datetime
+import functools
 import io
 from datetime import timedelta
 import itertools
@@ -22,11 +24,14 @@
 
 from concurrent.futures import as_completed
 from dateutil.tz import tzutc
+from dateutil.parser import parse as parse_date
+
 import six
 from botocore.exceptions import ClientError
-from collections import OrderedDict
+
 
 from c7n.actions import BaseAction
+from c7n.exceptions import PolicyValidationError
 from c7n.filters import ValueFilter, Filter, OPERATORS
 from c7n.filters.iamaccess import CrossAccountAccessFilter
 from c7n.manager import resources
@@ -770,6 +775,9 @@ class CredentialReport(Filter):
         ('cert_1_', 'certs'),
         ('cert_2_', 'certs'))
 
+    # for access keys only
+    matched_annotation_key = 'c7n:matched-keys'
+
     permissions = ('iam:GenerateCredentialReport',
                    'iam:GetCredentialReport')
 
@@ -846,7 +854,7 @@ def process(self, resources, event=None):
             self.matcher_config['key'] = self.data['key'].split('.', 1)[1]
         return []
 
-    def match(self, info):
+    def match(self, resource, info):
         if info is None:
             return False
         k = self.data.get('key')
@@ -855,12 +863,22 @@ def match(self, info):
             vf.annotate = False
             return vf(info)
 
+        # access key matching
         prefix, sk = k.split('.', 1)
         vf = ValueFilter(self.matcher_config)
         vf.annotate = False
+
+        # annotation merging with previous respecting block operators
+        k_matched = []
         for v in info.get(prefix, ()):
             if vf.match(v):
-                return True
+                k_matched.append(v)
+
+        for k in k_matched:
+            k['c7n:match-type'] = 'credential'
+
+        self.merge_annotation(resource, self.matched_annotation_key, k_matched)
+        return bool(k_matched)
 
 
 @User.filter_registry.register('credential')
@@ -874,7 +892,7 @@ def process(self, resources, event=None):
         results = []
         for r in resources:
             info = report.get(r['UserName'])
-            if self.match(info):
+            if self.match(r, info):
                 r['c7n:credential-report'] = info
                 results.append(r)
         return results
@@ -1019,26 +1037,37 @@ class UserAccessKey(ValueFilter):
 
     schema = type_schema('access-key', rinherit=ValueFilter.schema)
     permissions = ('iam:ListAccessKeys',)
+    annotation_key = 'c7n:AccessKeys'
+    matched_annotation_key = 'c7n:matched-keys'
+    annotate = False
 
-    def user_keys(self, user_set):
-        client = local_session(self.manager.session_factory).client('iam')
+    def get_user_keys(self, client, user_set):
         for u in user_set:
-            u['c7n:AccessKeys'] = client.list_access_keys(
+            u[self.annotation_key] = self.manager.retry(
+                client.list_access_keys,
                 UserName=u['UserName'])['AccessKeyMetadata']
 
     def process(self, resources, event=None):
-        user_set = chunks(resources, size=50)
+        client = local_session(self.manager.session_factory).client('iam')
         with self.executor_factory(max_workers=2) as w:
+            augment_set = [r for r in resources if self.annotation_key not in r]
             self.log.debug(
-                "Querying %d users' api keys" % len(resources))
-            list(w.map(self.user_keys, user_set))
+                "Querying %d users' api keys" % len(augment_set))
+            list(w.map(
+                functools.partial(self.get_user_keys, client),
+                chunks(augment_set, 50)))
 
         matched = []
         for r in resources:
-            for k in r['c7n:AccessKeys']:
+            k_matched = []
+            for k in r[self.annotation_key]:
                 if self.match(k):
-                    matched.append(r)
-                    break
+                    k_matched.append(k)
+            for k in k_matched:
+                k['c7n:matched-type'] = 'access'
+            self.merge_annotation(r, self.matched_annotation_key, k_matched)
+            if k_matched:
+                matched.append(r)
         return matched
 
 
@@ -1339,7 +1368,7 @@ class UserRemoveAccessKey(BaseAction):
 
         .. code-block:: yaml
 
-         - name: iam-mfa-active-keys-no-login
+         - name: iam-mfa-active-key-no-login
            resource: iam-user
            actions:
              - type: remove-keys
@@ -1350,15 +1379,29 @@ class UserRemoveAccessKey(BaseAction):
     """
 
     schema = type_schema(
-        'remove-keys', age={'type': 'number'}, disable={'type': 'boolean'})
+        'remove-keys',
+        matched={'type': 'boolean'},
+        age={'type': 'number'},
+        disable={'type': 'boolean'})
     permissions = ('iam:ListAccessKeys', 'iam:UpdateAccessKey',
                    'iam:DeleteAccessKey')
 
+    def validate(self):
+        if self.data.get('matched') and self.data.get('age'):
+            raise PolicyValidationError(
+                "policy:%s cant mix matched and age parameters")
+        ftypes = {f.type for f in self.manager.iter_filters()}
+        if 'credential' in ftypes and 'access-key' in ftypes:
+            raise PolicyValidationError(
+                "policy:%s cant mix credential and access-key filters w/ delete action")
+        return self
+
     def process(self, resources):
         client = local_session(self.manager.session_factory).client('iam')
 
         age = self.data.get('age')
         disable = self.data.get('disable')
+        matched = self.data.get('matched')
 
         if age:
             threshold_date = datetime.datetime.now(tz=tzutc()) - timedelta(age)
@@ -1367,7 +1410,15 @@ def process(self, resources):
             if 'c7n:AccessKeys' not in r:
                 r['c7n:AccessKeys'] = client.list_access_keys(
                     UserName=r['UserName'])['AccessKeyMetadata']
+
             keys = r['c7n:AccessKeys']
+            if matched:
+                m_keys = resolve_credential_keys(
+                    r.get(CredentialReport.matched_annotation_key),
+                    keys)
+                assert m_keys, "shouldn't have gotten this far without keys"
+                keys = m_keys
+
             for k in keys:
                 if age:
                     if not k['CreateDate'] < threshold_date:
@@ -1383,6 +1434,22 @@ def process(self, resources):
                         AccessKeyId=k['AccessKeyId'])
 
 
+def resolve_credential_keys(m_keys, keys):
+    res = []
+    for k in m_keys:
+        if k['c7n:match-type'] == 'credential':
+            c_date = parse_date(k['last_rotated'])
+            for ak in keys:
+                if c_date == ak['CreateDate']:
+                    ak = dict(ak)
+                    ak['c7n:match-type'] = 'access'
+                    if ak not in res:
+                        res.append(ak)
+        elif k not in res:
+            res.append(k)
+    return res
+
+
 #################
 #   IAM Groups  #
 #################


Test patch:
diff --git a/tests/data/placebo/test_iam_user_access_key_multi/iam.ListAccessKeys_1.json b/tests/data/placebo/test_iam_user_access_key_multi/iam.ListAccessKeys_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_iam_user_access_key_multi/iam.ListAccessKeys_1.json
@@ -0,0 +1,49 @@
+{
+    "status_code": 200,
+    "data": {
+        "AccessKeyMetadata": [
+            {
+                "UserName": "kapil",
+                "AccessKeyId": "AKIAIYJPJRH5H6NADJEA",
+                "Status": "Active",
+                "CreateDate": {
+                    "__class__": "datetime",
+                    "year": 2018,
+                    "month": 6,
+                    "day": 27,
+                    "hour": 22,
+                    "minute": 19,
+                    "second": 26,
+                    "microsecond": 0
+                }
+            },
+            {
+                "UserName": "kapil",
+                "AccessKeyId": "AKIAJYQQRLIX6ZMMRXQQ",
+                "Status": "Active",
+                "CreateDate": {
+                    "__class__": "datetime",
+                    "year": 2018,
+                    "month": 10,
+                    "day": 28,
+                    "hour": 14,
+                    "minute": 7,
+                    "second": 57,
+                    "microsecond": 0
+                }
+            }
+        ],
+        "IsTruncated": false,
+        "ResponseMetadata": {
+            "RequestId": "cf5c0469-f5e5-11e8-bb27-d33c775f2c45",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "cf5c0469-f5e5-11e8-bb27-d33c775f2c45",
+                "content-type": "text/xml",
+                "content-length": "801",
+                "date": "Sun, 02 Dec 2018 03:53:20 GMT"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
diff --git a/tests/data/placebo/test_iam_user_access_key_multi/iam.ListUsers_1.json b/tests/data/placebo/test_iam_user_access_key_multi/iam.ListUsers_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_iam_user_access_key_multi/iam.ListUsers_1.json
@@ -0,0 +1,46 @@
+{
+    "status_code": 200,
+    "data": {
+        "Users": [
+            {
+                "Path": "/",
+                "UserName": "kapil",
+                "UserId": "AIDAJEZOTH6YPO3DY45QW",
+                "Arn": "arn:aws:iam::644160558196:user/kapil",
+                "CreateDate": {
+                    "__class__": "datetime",
+                    "year": 2016,
+                    "month": 5,
+                    "day": 16,
+                    "hour": 19,
+                    "minute": 3,
+                    "second": 36,
+                    "microsecond": 0
+                },
+                "PasswordLastUsed": {
+                    "__class__": "datetime",
+                    "year": 2018,
+                    "month": 12,
+                    "day": 2,
+                    "hour": 3,
+                    "minute": 25,
+                    "second": 42,
+                    "microsecond": 0
+                }
+            }
+        ],
+        "IsTruncated": false,
+        "ResponseMetadata": {
+            "RequestId": "f4378d60-f5e4-11e8-8aa7-a1d0e8b456d5",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "f4378d60-f5e4-11e8-8aa7-a1d0e8b456d5",
+                "content-type": "text/xml",
+                "content-length": "5180",
+                "vary": "Accept-Encoding",
+                "date": "Sun, 02 Dec 2018 03:47:12 GMT"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
diff --git a/tests/data/placebo/test_iam_user_credential_multi_delete/iam.DeleteAccessKey_1.json b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.DeleteAccessKey_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.DeleteAccessKey_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200,
+    "data": {
+        "ResponseMetadata": {
+            "RequestId": "f0e6f8c7-f5e3-11e8-8700-7f377c0f7a24",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "f0e6f8c7-f5e3-11e8-8700-7f377c0f7a24",
+                "content-type": "text/xml",
+                "content-length": "210",
+                "date": "Sun, 02 Dec 2018 03:39:57 GMT"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_iam_user_credential_multi_delete/iam.GetCredentialReport_1.json b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.GetCredentialReport_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.GetCredentialReport_1.json
@@ -0,0 +1,29 @@
+{
+    "status_code": 200,
+    "data": {
+        "Content": "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\nkapil,arn:aws:iam::644160558196:user/kapil,2016-05-16T19:03:36+00:00,true,2018-12-01T14:56:42+00:00,2018-09-12T13:53:09+00:00,2018-12-11T13:53:09+00:00,false,true,2018-01-14T10:26:37+00:00,2018-11-21T06:54:00+00:00,us-east-1,ecr,true,2016-06-24T20:30:37+00:00,2018-12-01T22:25:00+00:00,us-east-1,sts,false,N/A,false,N/A\n",
+        "ReportFormat": "text/csv",
+        "GeneratedTime": {
+            "__class__": "datetime",
+            "year": 2018,
+            "month": 12,
+            "day": 2,
+            "hour": 1,
+            "minute": 18,
+            "second": 49,
+            "microsecond": 0
+        },
+        "ResponseMetadata": {
+            "RequestId": "f0c0abab-f5e3-11e8-acb5-7dfca7468def",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "f0c0abab-f5e3-11e8-acb5-7dfca7468def",
+                "content-type": "text/xml",
+                "content-length": "8569",
+                "vary": "Accept-Encoding",
+                "date": "Sun, 02 Dec 2018 03:39:57 GMT"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListAccessKeys_1.json b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListAccessKeys_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListAccessKeys_1.json
@@ -0,0 +1,49 @@
+{
+    "status_code": 200,
+    "data": {
+        "AccessKeyMetadata": [
+            {
+                "UserName": "kapil",
+                "AccessKeyId": "AKIAJEZJDQQX4TZNRMKQ",
+                "Status": "Active",
+                "CreateDate": {
+                    "__class__": "datetime",
+                    "year": 2018,
+                    "month": 12,
+                    "day": 2,
+                    "hour": 3,
+                    "minute": 34,
+                    "second": 13,
+                    "microsecond": 0
+                }
+            },
+            {
+                "UserName": "kapil",
+                "AccessKeyId": "AKIAIFX6HPKAM5F7JVAA",
+                "Status": "Active",
+                "CreateDate": {
+                    "__class__": "datetime",
+                    "year": 2016,
+                    "month": 6,
+                    "day": 24,
+                    "hour": 20,
+                    "minute": 30,
+                    "second": 37,
+                    "microsecond": 0
+                }
+            }
+        ],
+        "IsTruncated": false,
+        "ResponseMetadata": {
+            "RequestId": "f0df3074-f5e3-11e8-8700-7f377c0f7a24",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "f0df3074-f5e3-11e8-8700-7f377c0f7a24",
+                "content-type": "text/xml",
+                "content-length": "761",
+                "date": "Sun, 02 Dec 2018 03:39:57 GMT"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListAccessKeys_2.json b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListAccessKeys_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListAccessKeys_2.json
@@ -0,0 +1,34 @@
+{
+    "status_code": 200,
+    "data": {
+        "AccessKeyMetadata": [
+            {
+                "UserName": "kapil",
+                "AccessKeyId": "AKIAJEZJDQQX4TZNRMKQ",
+                "Status": "Active",
+                "CreateDate": {
+                    "__class__": "datetime",
+                    "year": 2018,
+                    "month": 12,
+                    "day": 2,
+                    "hour": 3,
+                    "minute": 34,
+                    "second": 13,
+                    "microsecond": 0
+                }
+            }
+        ],
+        "IsTruncated": false,
+        "ResponseMetadata": {
+            "RequestId": "f191de39-f5e3-11e8-860e-07ae61cefdc8",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "f191de39-f5e3-11e8-860e-07ae61cefdc8",
+                "content-type": "text/xml",
+                "content-length": "553",
+                "date": "Sun, 02 Dec 2018 03:39:58 GMT"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListUsers_1.json b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListUsers_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_iam_user_credential_multi_delete/iam.ListUsers_1.json
@@ -0,0 +1,46 @@
+{
+    "status_code": 200,
+    "data": {
+        "Users": [
+            {
+                "Path": "/",
+                "UserName": "kapil",
+                "UserId": "AIDAJEZOTH6YPO3DY45QW",
+                "Arn": "arn:aws:iam::644160558196:user/kapil",
+                "CreateDate": {
+                    "__class__": "datetime",
+                    "year": 2016,
+                    "month": 5,
+                    "day": 16,
+                    "hour": 19,
+                    "minute": 3,
+                    "second": 36,
+                    "microsecond": 0
+                },
+                "PasswordLastUsed": {
+                    "__class__": "datetime",
+                    "year": 2018,
+                    "month": 12,
+                    "day": 2,
+                    "hour": 3,
+                    "minute": 25,
+                    "second": 42,
+                    "microsecond": 0
+                }
+            }
+        ],
+        "IsTruncated": false,
+        "ResponseMetadata": {
+            "RequestId": "f0b4c496-f5e3-11e8-acb5-7dfca7468def",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "f0b4c496-f5e3-11e8-acb5-7dfca7468def",
+                "content-type": "text/xml",
+                "content-length": "5180",
+                "vary": "Accept-Encoding",
+                "date": "Sun, 02 Dec 2018 03:39:57 GMT"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
diff --git a/tests/data/placebo/test_iam_user_delete_options/iam.GenerateCredentialReport_1.json b/tests/data/placebo/test_iam_user_delete_options/iam.GenerateCredentialReport_1.json
deleted file mode 100644
--- a/tests/data/placebo/test_iam_user_delete_options/iam.GenerateCredentialReport_1.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-    "status_code": 200, 
-    "data": {
-        "State": "STARTED", 
-        "Description": "No report exists. Starting a new report generation task", 
-        "ResponseMetadata": {
-            "RetryAttempts": 0, 
-            "HTTPStatusCode": 200, 
-            "RequestId": "33c53360-652f-11e8-bcd0-e9452080145d", 
-            "HTTPHeaders": {
-                "x-amzn-requestid": "33c53360-652f-11e8-bcd0-e9452080145d", 
-                "date": "Fri, 01 Jun 2018 00:03:23 GMT", 
-                "content-length": "413", 
-                "content-type": "text/xml"
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/tests/data/placebo/test_iam_user_delete_options/iam.GetCredentialReport_1.json b/tests/data/placebo/test_iam_user_delete_options/iam.GetCredentialReport_1.json
--- a/tests/data/placebo/test_iam_user_delete_options/iam.GetCredentialReport_1.json
+++ b/tests/data/placebo/test_iam_user_delete_options/iam.GetCredentialReport_1.json
@@ -1,20 +1,29 @@
 {
-    "status_code": 410, 
+    "status_code": 200, 
     "data": {
+        "Content": "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\ntest_user,arn:aws:iam::644160558196:user/test_user,2018-05-30T19:46:19+00:00,true,no_information,2018-06-01T00:02:54+00:00,2018-08-30T00:02:54+00:00,false,true,2018-06-01T00:02:57+00:00,N/A,N/A,N/A,false,N/A,N/A,N/A,N/A,false,N/A,false,N/A", 
+        "GeneratedTime": {
+            "hour": 0, 
+            "__class__": "datetime", 
+            "month": 6, 
+            "second": 25, 
+            "microsecond": 0, 
+            "year": 2018, 
+            "day": 1, 
+            "minute": 3
+        }, 
         "ResponseMetadata": {
             "RetryAttempts": 0, 
-            "HTTPStatusCode": 410, 
-            "RequestId": "33bf18bf-652f-11e8-bcd0-e9452080145d", 
+            "HTTPStatusCode": 200, 
+            "RequestId": "39ccc738-652f-11e8-bcd0-e9452080145d", 
             "HTTPHeaders": {
-                "x-amzn-requestid": "33bf18bf-652f-11e8-bcd0-e9452080145d", 
-                "date": "Fri, 01 Jun 2018 00:03:23 GMT", 
-                "content-length": "224", 
-                "content-type": "text/xml"
+                "x-amzn-requestid": "39ccc738-652f-11e8-bcd0-e9452080145d", 
+                "vary": "Accept-Encoding", 
+                "content-length": "9257", 
+                "content-type": "text/xml", 
+                "date": "Fri, 01 Jun 2018 00:03:33 GMT"
             }
         }, 
-        "Error": {
-            "Code": "ReportNotPresent", 
-            "Type": "Sender"
-        }
+        "ReportFormat": "text/csv"
     }
-}
\ No newline at end of file
+}
diff --git a/tests/data/placebo/test_iam_user_delete_options/iam.GetCredentialReport_2.json b/tests/data/placebo/test_iam_user_delete_options/iam.GetCredentialReport_2.json
deleted file mode 100644
--- a/tests/data/placebo/test_iam_user_delete_options/iam.GetCredentialReport_2.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-    "status_code": 200, 
-    "data": {
-        "Content": "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\ntest_user,arn:aws:iam::644160558196:user/test_user,2018-05-30T19:46:19+00:00,true,no_information,2018-06-01T00:02:54+00:00,2018-08-30T00:02:54+00:00,false,true,2018-06-01T00:02:57+00:00,N/A,N/A,N/A,false,N/A,N/A,N/A,N/A,false,N/A,false,N/A", 
-        "GeneratedTime": {
-            "hour": 0, 
-            "__class__": "datetime", 
-            "month": 6, 
-            "second": 25, 
-            "microsecond": 0, 
-            "year": 2018, 
-            "day": 1, 
-            "minute": 3
-        }, 
-        "ResponseMetadata": {
-            "RetryAttempts": 0, 
-            "HTTPStatusCode": 200, 
-            "RequestId": "39ccc738-652f-11e8-bcd0-e9452080145d", 
-            "HTTPHeaders": {
-                "x-amzn-requestid": "39ccc738-652f-11e8-bcd0-e9452080145d", 
-                "vary": "Accept-Encoding", 
-                "content-length": "9257", 
-                "content-type": "text/xml", 
-                "date": "Fri, 01 Jun 2018 00:03:33 GMT"
-            }
-        }, 
-        "ReportFormat": "text/csv"
-    }
-}
diff --git a/tests/test_iam.py b/tests/test_iam.py
--- a/tests/test_iam.py
+++ b/tests/test_iam.py
@@ -17,6 +17,7 @@
 import datetime
 import os
 import tempfile
+import time
 
 from unittest import TestCase
 from .common import load_data, BaseTest, functional, TestConfig as Config
@@ -51,7 +52,7 @@
 
 class UserCredentialReportTest(BaseTest):
 
-    def test_credential_report_generatpoe(self):
+    def test_credential_report_generate(self):
         session_factory = self.replay_flight_data("test_iam_user_unused_keys")
         p = self.load_policy(
             {
@@ -76,6 +77,40 @@ def test_credential_report_generatpoe(self):
             ["Hazmat", "charmworld", "kaleb", "kapilt"],
         )
 
+    def test_credential_access_key_multifilter_delete(self):
+        factory = self.replay_flight_data('test_iam_user_credential_multi_delete')
+        p = self.load_policy({
+            'name': 'user-cred-multi',
+            'resource': 'iam-user',
+            'filters': [
+                {'UserName': 'kapil'},
+                {"type": "credential",
+                 "report_max_age": 1543724277,
+                 "key": "access_keys.last_used_date",
+                 "value": 30,
+                 'op': 'less-than',
+                 "value_type": "age"},
+                {"type": "credential",
+                 "report_max_age": 1543724277,
+                 "key": "access_keys.last_rotated",
+                 "value": 700,
+                 "op": "greater-than",
+                 'value_type': 'age'}],
+            'actions': [
+                {'type': 'remove-keys',
+                 'matched': True}]},
+            session_factory=factory)
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(len(resources[0]['c7n:matched-keys']), 1)
+        client = factory().client('iam')
+        if self.recording:
+            time.sleep(1)
+        keys = client.list_access_keys(UserName='kapil').get('AccessKeyMetadata')
+        self.assertEqual(len(keys), 1)
+        dt = parser.parse(resources[0]['c7n:matched-keys'][0]['last_rotated'])
+        self.assertNotEqual(keys[0]['CreateDate'], dt)
+
     def test_access_key_last_service(self):
         # Note we're reusing the old console users flight records
         session_factory = self.replay_flight_data("test_iam_user_console_old")
@@ -266,7 +301,24 @@ def test_iam_user_delete(self):
         users = client.list_users(PathPrefix="/test/").get("Users", [])
         self.assertEqual(users, [])
 
-    @functional
+    def test_iam_user_access_key_multi(self):
+        factory = self.replay_flight_data('test_iam_user_access_key_multi')
+        p = self.load_policy({
+            'name': 'user-del',
+            'resource': 'iam-user',
+            'filters': [
+                {'UserName': 'kapil'},
+                {'type': 'access-key', 'key': 'Status', 'value': 'Active'},
+                {'type': 'access-key', 'key': 'CreateDate',
+                 'value_type': 'age', 'value': 90, 'op': 'greater-than'}
+            ]},
+            session_factory=factory)
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(len(resources[0]['c7n:matched-keys']), 1)
+        self.assertEqual(
+            resources[0]['c7n:matched-keys'][0]['c7n:matched-type'], 'access')
+
     def test_iam_user_delete_some_access(self):
         factory = self.replay_flight_data("test_iam_user_delete_options")
         p = self.load_policy(
@@ -276,7 +328,8 @@ def test_iam_user_delete_some_access(self):
                 "filters": [
                     {"UserName": "test_user"},
                     {"type": "access-key", "key": "Status", "value": "Active"},
-                    {"type": "credential", "key": "password_enabled", "value": True}],
+                    {"type": "credential", "report_max_age": 1543724277,
+                     "key": "password_enabled", "value": True}],
                 "actions": [{
                     "type": "delete",
                     "options": ["console-access", "access-keys"]}],
diff --git a/tests/zpill.py b/tests/zpill.py
--- a/tests/zpill.py
+++ b/tests/zpill.py
@@ -83,6 +83,7 @@ def serialize(obj):
         result["__module__"] = obj.__module__
     except AttributeError:
         pass
+
     # Convert objects to dictionary representation based on type
     if isinstance(obj, datetime):
         result["year"] = obj.year
@@ -98,6 +99,9 @@ def serialize(obj):
         obj._raw_stream = StringIO(result["body"])
         obj._amount_read = 0
         return result
+    if isinstance(obj, bytes):
+        return obj.decode('utf8')
+
     # Raise a TypeError if the object isn't recognized
     raise TypeError("Type not serializable")