Repository: cloud-custodian/cloud-custodian

Issue:
mark-for-op and tag actions missing for resource=kinesis
The mark-for-op and tag actions are available for ec2, rds, and other resource types is not available for kinesis, but would desire them to be


Discussion:
I believe all tag related filters and actions will be much easier to add to resources once this PR is completed:
https://github.com/capitalone/cloud-custodian/pull/1315/files

Definitely agree that this needs to be done - thanks for adding it to the list!

@ocampocj Your pull request will also address the issue of tag action missing from cache-cluster as well it appears, right?
@ocampocj Your pull request also adds auto-tag-user to all resources, right?
@FireballDWF this PR will add tag filters and actions provided that `universal_taggable` gets added to a resource's resource_type class.
merged re universal taggable

Resolving patch:
diff --git a/c7n/resources/glacier.py b/c7n/resources/glacier.py
--- a/c7n/resources/glacier.py
+++ b/c7n/resources/glacier.py
@@ -32,6 +32,7 @@ class resource_type(object):
         enum_spec = ('list_vaults', 'VaultList', None)
         name = "VaultName"
         id = "VaultARN"
+        filter_name = None
         dimension = None
         universal_taggable = True
 
diff --git a/c7n/resources/kinesis.py b/c7n/resources/kinesis.py
--- a/c7n/resources/kinesis.py
+++ b/c7n/resources/kinesis.py
@@ -16,6 +16,7 @@
 from c7n.actions import Action
 from c7n.manager import resources
 from c7n.query import QueryResourceManager
+from c7n.tags import universal_augment
 from c7n.utils import local_session, type_schema
 
 
@@ -33,6 +34,11 @@ class resource_type(object):
         filter_type = None
         date = None
         dimension = 'StreamName'
+        universal_taggable = True
+
+    def augment(self, resources):
+        return universal_augment(
+            self, super(KinesisStream, self).augment(resources))
 
 
 @KinesisStream.action_registry.register('delete')
diff --git a/tools/dev/ratchet.py b/tools/dev/ratchet.py
--- a/tools/dev/ratchet.py
+++ b/tools/dev/ratchet.py
@@ -44,6 +44,8 @@ def process_testcase(self, case):
             result = nonsuccess.pop()
             self.failed.append(key)
             message = result.get('message')
+            if message is None:
+                return
             if message.startswith(INVALID_TYPE):
                 message = INVALID_TYPE
             self.failed_aggregates.setdefault(message, []).append(key)


Test patch:
diff --git a/tests/data/placebo/test_kinesis_stream_delete/kinesis.DeleteStream_1.json b/tests/data/placebo/test_kinesis_stream_delete/kinesis.DeleteStream_1.json
--- a/tests/data/placebo/test_kinesis_stream_delete/kinesis.DeleteStream_1.json
+++ b/tests/data/placebo/test_kinesis_stream_delete/kinesis.DeleteStream_1.json
@@ -4,13 +4,13 @@
         "ResponseMetadata": {
             "RetryAttempts": 0, 
             "HTTPStatusCode": 200, 
-            "RequestId": "fcbe810e-552c-77a0-ac0a-50ef44a5fdd2", 
+            "RequestId": "ca6f8aab-19b4-97f5-9866-87f9d10278df", 
             "HTTPHeaders": {
-                "x-amzn-requestid": "fcbe810e-552c-77a0-ac0a-50ef44a5fdd2", 
+                "x-amzn-requestid": "ca6f8aab-19b4-97f5-9866-87f9d10278df", 
                 "content-length": "0", 
-                "x-amz-id-2": "/oiU6wti79SNzpEX1eDEhMyvURu7VxPb0B3S4PfKg0SmWk1XWaHLZ/V+BAFkef+Io1eBtpd39gogZOtqMD1V87lwSer09/jsVGw7Jlt5RsE=", 
+                "x-amz-id-2": "n2aMNBQrgv9aiUCsOl71uGrxEQV47lfMyoY9mAfGwR1WZHTQDQPPFzG3u3HL2Nq5sNn+pAAbAqhFYRRk4PAIwp23HIc+xi8h", 
                 "server": "Apache-Coyote/1.1", 
-                "date": "Sun, 27 Nov 2016 12:20:48 GMT", 
+                "date": "Sat, 12 Aug 2017 14:05:05 GMT", 
                 "content-type": "application/x-amz-json-1.1"
             }
         }
diff --git a/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_1.json b/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_1.json
--- a/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_1.json
+++ b/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_1.json
@@ -4,13 +4,13 @@
         "ResponseMetadata": {
             "RetryAttempts": 0, 
             "HTTPStatusCode": 200, 
-            "RequestId": "ef4a2612-547f-f68d-bffe-f7f258bb0518", 
+            "RequestId": "da712176-7c8c-9817-8878-2c27b42cfde5", 
             "HTTPHeaders": {
-                "x-amzn-requestid": "ef4a2612-547f-f68d-bffe-f7f258bb0518", 
-                "content-length": "811", 
-                "x-amz-id-2": "Tx1Vr+Skd/K3foYUzMXjbU8nYhngFLInicbjEp2AyTxOlDeqW35CsB3s/o1Z/+SUHp0HmK93vyrdgycARTZXlcG3uTlLKNxEYvIOvIOhvDY=", 
+                "x-amzn-requestid": "da712176-7c8c-9817-8878-2c27b42cfde5", 
+                "content-length": "557", 
+                "x-amz-id-2": "06AHhU8LB0t7Fe5/FEpjt53U3dpYCqMT4zUYxxfz6SCR3EoNrRd6HVjAgFaIdWltUhBQHpNa5jZIs+lA7+JdYi9ocBVKnOVm", 
                 "server": "Apache-Coyote/1.1", 
-                "date": "Sun, 27 Nov 2016 12:20:48 GMT", 
+                "date": "Sat, 12 Aug 2017 14:05:04 GMT", 
                 "content-type": "application/x-amz-json-1.1"
             }
         }, 
@@ -21,22 +21,12 @@
             "Shards": [
                 {
                     "ShardId": "shardId-000000000000", 
-                    "HashKeyRange": {
-                        "EndingHashKey": "170141183460469231731687303715884105727", 
-                        "StartingHashKey": "0"
-                    }, 
-                    "SequenceNumberRange": {
-                        "StartingSequenceNumber": "49568027013051417042124392152163093257658725395469434882"
-                    }
-                }, 
-                {
-                    "ShardId": "shardId-000000000001", 
                     "HashKeyRange": {
                         "EndingHashKey": "340282366920938463463374607431768211455", 
-                        "StartingHashKey": "170141183460469231731687303715884105728"
+                        "StartingHashKey": "0"
                     }, 
                     "SequenceNumberRange": {
-                        "StartingSequenceNumber": "49568027013073717787322922775304628975931373756975415314"
+                        "StartingSequenceNumber": "49575983792626831963983842622784465898296159642543194114"
                     }
                 }
             ], 
@@ -46,6 +36,16 @@
                     "ShardLevelMetrics": []
                 }
             ], 
+            "StreamCreationTimestamp": {
+                "hour": 10, 
+                "__class__": "datetime", 
+                "month": 8, 
+                "second": 43, 
+                "microsecond": 0, 
+                "year": 2017, 
+                "day": 12, 
+                "minute": 0
+            }, 
             "StreamStatus": "ACTIVE"
         }
     }
diff --git a/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_2.json b/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_2.json
--- a/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_2.json
+++ b/tests/data/placebo/test_kinesis_stream_delete/kinesis.DescribeStream_2.json
@@ -4,27 +4,38 @@
         "ResponseMetadata": {
             "RetryAttempts": 0, 
             "HTTPStatusCode": 200, 
-            "RequestId": "ef4a2612-547f-f68d-bffe-f7f258bb0518", 
+            "RequestId": "fbb147a8-ddde-c6ba-a9b8-4afa2728c28b", 
             "HTTPHeaders": {
-                "x-amzn-requestid": "ef4a2612-547f-f68d-bffe-f7f258bb0518", 
-                "content-length": "811", 
-                "x-amz-id-2": "Tx1Vr+Skd/K3foYUzMXjbU8nYhngFLInicbjEp2AyTxOlDeqW35CsB3s/o1Z/+SUHp0HmK93vyrdgycARTZXlcG3uTlLKNxEYvIOvIOhvDY=", 
+                "x-amzn-requestid": "fbb147a8-ddde-c6ba-a9b8-4afa2728c28b", 
+                "content-length": "320", 
+                "x-amz-id-2": "emKVsHNM5ZUv2+r2G7gTVhMgES40lDzAZWZGiOIig096Kzql4n7NxcrR3Wa4au9+6WAfMlm5FHd7e2Q+xqVAPALIf7Ab8PgA", 
                 "server": "Apache-Coyote/1.1", 
-                "date": "Sun, 27 Nov 2016 12:20:48 GMT", 
+                "date": "Sat, 12 Aug 2017 14:05:06 GMT", 
                 "content-type": "application/x-amz-json-1.1"
             }
         }, 
         "StreamDescription": {
+            "HasMoreShards": false, 
             "RetentionPeriodHours": 24, 
             "StreamName": "sock-drawer", 
-            "Shards": [],
+            "Shards": [], 
             "StreamARN": "arn:aws:kinesis:us-east-1:644160558196:stream/sock-drawer", 
             "EnhancedMonitoring": [
                 {
                     "ShardLevelMetrics": []
                 }
             ], 
+            "StreamCreationTimestamp": {
+                "hour": 10, 
+                "__class__": "datetime", 
+                "month": 8, 
+                "second": 43, 
+                "microsecond": 0, 
+                "year": 2017, 
+                "day": 12, 
+                "minute": 0
+            }, 
             "StreamStatus": "DELETING"
         }
     }
-}
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_kinesis_stream_delete/kinesis.ListStreams_1.json b/tests/data/placebo/test_kinesis_stream_delete/kinesis.ListStreams_1.json
--- a/tests/data/placebo/test_kinesis_stream_delete/kinesis.ListStreams_1.json
+++ b/tests/data/placebo/test_kinesis_stream_delete/kinesis.ListStreams_1.json
@@ -8,13 +8,13 @@
         "ResponseMetadata": {
             "RetryAttempts": 0, 
             "HTTPStatusCode": 200, 
-            "RequestId": "cc2e3e7a-acc3-ff4a-9c9a-efa59ac1ed78", 
+            "RequestId": "cb6d2d76-ae4e-80ed-9964-20278305229d", 
             "HTTPHeaders": {
-                "x-amzn-requestid": "cc2e3e7a-acc3-ff4a-9c9a-efa59ac1ed78", 
+                "x-amzn-requestid": "cb6d2d76-ae4e-80ed-9964-20278305229d", 
                 "content-length": "54", 
-                "x-amz-id-2": "+yZmojZmBL542TX41W6ZHxROclXyptq645zQdKQdQrwLq+D0t/9NoQ5TZQfT8KESG4yw1E38k5cRCfuPZl/oNJA+4UwzTXQAHLc8GEeROfo=", 
+                "x-amz-id-2": "/uufCCQh1HZn4rcMzqDFr1B+rfqc/h1+aLrNTiHEeeHoNqX88c1HDT71FjFwapV5y/kAh8IhCfhCkfZWlsULIOz2tlJrgJX0", 
                 "server": "Apache-Coyote/1.1", 
-                "date": "Sun, 27 Nov 2016 12:20:47 GMT", 
+                "date": "Sat, 12 Aug 2017 14:05:04 GMT", 
                 "content-type": "application/x-amz-json-1.1"
             }
         }
diff --git a/tests/data/placebo/test_kinesis_stream_delete/tagging.GetResources_1.json b/tests/data/placebo/test_kinesis_stream_delete/tagging.GetResources_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_kinesis_stream_delete/tagging.GetResources_1.json
@@ -0,0 +1,18 @@
+{
+    "status_code": 200, 
+    "data": {
+        "PaginationToken": "", 
+        "ResourceTagMappingList": [], 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "3e252497-7f67-11e7-9836-a96970d9def5", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "3e252497-7f67-11e7-9836-a96970d9def5", 
+                "date": "Sat, 12 Aug 2017 14:05:05 GMT", 
+                "content-length": "50", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_kinesis_stream_query/kinesis.DescribeStream_1.json b/tests/data/placebo/test_kinesis_stream_query/kinesis.DescribeStream_1.json
--- a/tests/data/placebo/test_kinesis_stream_query/kinesis.DescribeStream_1.json
+++ b/tests/data/placebo/test_kinesis_stream_query/kinesis.DescribeStream_1.json
@@ -4,48 +4,71 @@
         "ResponseMetadata": {
             "RetryAttempts": 0, 
             "HTTPStatusCode": 200, 
-            "RequestId": "cb9bdd14-e00e-d3fc-9b2f-0c505aea6073", 
+            "RequestId": "c3aec702-b616-2110-91a7-cce52cff42a0", 
             "HTTPHeaders": {
-                "x-amzn-requestid": "cb9bdd14-e00e-d3fc-9b2f-0c505aea6073", 
-                "content-length": "811", 
-                "x-amz-id-2": "XHKqr2hANVUMsQcIl4hUw+GTNgwJJPGgrQD3UJm8/iTBKJq5X1L/YpqcPOGSfU5oNTSg5YyKJJb1Nl74Ny7VPiu4OQFobyWXj8dkokOj3ZI=", 
+                "x-amzn-requestid": "c3aec702-b616-2110-91a7-cce52cff42a0", 
+                "content-length": "1330", 
+                "x-amz-id-2": "YOHZ0WV8i85W1yCOkizbqwtEiRsGmzRyzqX6VkhdBY2PAraPUXVn9PU41lsD9OtQh3f3lAH97N0HIyoZbrPQWwPNHUQmM53CtHXxKDk1n+E=", 
                 "server": "Apache-Coyote/1.1", 
-                "date": "Sun, 27 Nov 2016 12:18:11 GMT", 
+                "date": "Sat, 12 Aug 2017 13:59:03 GMT", 
                 "content-type": "application/x-amz-json-1.1"
             }
         }, 
         "StreamDescription": {
             "HasMoreShards": false, 
             "RetentionPeriodHours": 24, 
-            "StreamName": "sock-drawer", 
+            "StreamName": "test-yes-encrypt", 
             "Shards": [
                 {
                     "ShardId": "shardId-000000000000", 
                     "HashKeyRange": {
-                        "EndingHashKey": "170141183460469231731687303715884105727", 
+                        "EndingHashKey": "340282366920938463463374607431768211455", 
                         "StartingHashKey": "0"
                     }, 
                     "SequenceNumberRange": {
-                        "StartingSequenceNumber": "49568027013051417042124392152163093257658725395469434882"
+                        "EndingSequenceNumber": "49575281968561928803165895440748552507745700754017484802", 
+                        "StartingSequenceNumber": "49575217660694676254955779315723496596016722889129590786"
                     }
                 }, 
                 {
                     "ShardId": "shardId-000000000001", 
+                    "HashKeyRange": {
+                        "EndingHashKey": "170141183460469231731687303715884105727", 
+                        "StartingHashKey": "0"
+                    }, 
+                    "ParentShardId": "shardId-000000000000", 
+                    "SequenceNumberRange": {
+                        "StartingSequenceNumber": "49575983704159775761412860620312271510700092505888653330"
+                    }
+                }, 
+                {
+                    "ShardId": "shardId-000000000002", 
                     "HashKeyRange": {
                         "EndingHashKey": "340282366920938463463374607431768211455", 
                         "StartingHashKey": "170141183460469231731687303715884105728"
                     }, 
+                    "ParentShardId": "shardId-000000000000", 
                     "SequenceNumberRange": {
-                        "StartingSequenceNumber": "49568027013073717787322922775304628975931373756975415314"
+                        "StartingSequenceNumber": "49575983704182076506611391243453807228972740867394633762"
                     }
                 }
             ], 
-            "StreamARN": "arn:aws:kinesis:us-east-1:644160558196:stream/sock-drawer", 
+            "StreamARN": "arn:aws:kinesis:us-east-1:644160558196:stream/test-yes-encrypt", 
             "EnhancedMonitoring": [
                 {
                     "ShardLevelMetrics": []
                 }
             ], 
+            "StreamCreationTimestamp": {
+                "hour": 13, 
+                "__class__": "datetime", 
+                "month": 7, 
+                "second": 44, 
+                "microsecond": 0, 
+                "year": 2017, 
+                "day": 18, 
+                "minute": 34
+            }, 
             "StreamStatus": "ACTIVE"
         }
     }
diff --git a/tests/data/placebo/test_kinesis_stream_query/kinesis.ListStreams_1.json b/tests/data/placebo/test_kinesis_stream_query/kinesis.ListStreams_1.json
--- a/tests/data/placebo/test_kinesis_stream_query/kinesis.ListStreams_1.json
+++ b/tests/data/placebo/test_kinesis_stream_query/kinesis.ListStreams_1.json
@@ -2,19 +2,19 @@
     "status_code": 200, 
     "data": {
         "StreamNames": [
-            "sock-drawer"
+            "test-yes-encrypt"
         ], 
         "HasMoreStreams": false, 
         "ResponseMetadata": {
             "RetryAttempts": 0, 
             "HTTPStatusCode": 200, 
-            "RequestId": "dca54172-34be-e0dd-8c11-90312e75b9ed", 
+            "RequestId": "cafc466b-7c45-5203-98f5-4d8c6d80cd71", 
             "HTTPHeaders": {
-                "x-amzn-requestid": "dca54172-34be-e0dd-8c11-90312e75b9ed", 
-                "content-length": "54", 
-                "x-amz-id-2": "PsDN7w1OLgeES9pjQv+5eBL3Fy8smykWzI6E1woeXdK87t4dc4u1i01v1Q8U4Ne1kDs09HVvMVAaUiulIx1KE7UzKh4maYr+2qmblC/QDT8=", 
+                "x-amzn-requestid": "cafc466b-7c45-5203-98f5-4d8c6d80cd71", 
+                "content-length": "59", 
+                "x-amz-id-2": "Qwp7FcZJ1ClDm5TyClkxPjG94HLWLCI+ug2n14JYNB9/l6ZWLCExXujLF27KJKz8OcmlLuXfmHD7NqB5I6sVt9O/LfOSmGo1KUtPMqIO1Hw=", 
                 "server": "Apache-Coyote/1.1", 
-                "date": "Sun, 27 Nov 2016 12:18:11 GMT", 
+                "date": "Sat, 12 Aug 2017 13:59:03 GMT", 
                 "content-type": "application/x-amz-json-1.1"
             }
         }
diff --git a/tests/data/placebo/test_kinesis_stream_query/tagging.GetResources_1.json b/tests/data/placebo/test_kinesis_stream_query/tagging.GetResources_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_kinesis_stream_query/tagging.GetResources_1.json
@@ -0,0 +1,28 @@
+{
+    "status_code": 200, 
+    "data": {
+        "PaginationToken": "", 
+        "ResourceTagMappingList": [
+            {
+                "ResourceARN": "arn:aws:kinesis:us-east-1:644160558196:stream/test-yes-encrypt", 
+                "Tags": [
+                    {
+                        "Value": "home", 
+                        "Key": "Origin"
+                    }
+                ]
+            }
+        ], 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "6684080e-7f66-11e7-832b-9b00bfea7ed6", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "6684080e-7f66-11e7-832b-9b00bfea7ed6", 
+                "date": "Sat, 12 Aug 2017 13:59:03 GMT", 
+                "content-length": "171", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_kinesis.py b/tests/test_kinesis.py
--- a/tests/test_kinesis.py
+++ b/tests/test_kinesis.py
@@ -24,11 +24,16 @@ def test_stream_query(self):
             'name': 'kstream',
             'resource': 'kinesis',
             'filters': [
-                {'type': 'value', 'value_type': 'size',
-                 'value': 2, 'key': 'Shards'}]
+                {'type': 'value',
+                 'value_type': 'size',
+                 'value': 3,
+                 'key': 'Shards'}]
             }, session_factory=factory)
         resources = p.run()
         self.assertEqual(len(resources), 1)
+        self.assertEqual(
+            resources[0]['Tags'],
+            [{'Key': 'Origin', 'Value': 'home'}])
         self.assertEqual(resources[0]['StreamStatus'], 'ACTIVE')
 
     def test_stream_delete(self):
diff --git a/tests/test_policy.py b/tests/test_policy.py
--- a/tests/test_policy.py
+++ b/tests/test_policy.py
@@ -87,6 +87,28 @@ def test_policy_manager_custom_permissions(self):
                  'ec2:DescribeTags',
                  'cloudwatch:GetMetricStatistics')))
 
+    def xtest_resource_filter_name(self):
+        # resources without a filter name won't play nice in
+        # lambda policies
+        missing = []
+        marker = object
+        for k, v in manager.resources.items():
+            if getattr(v.resource_type, 'filter_name', marker) is marker:
+                missing.append(k)
+        if missing:
+            self.fail("Missing filter name %s" % (', '.join(missing)))
+
+    def test_resource_augment_universal_mask(self):
+        # universal tag had a potential bad patterm of masking
+        # resource augmentation, scan resources to ensure
+        for k, v in manager.resources.items():
+            if not getattr(v.resource_type, 'universal_taggable', None):
+                continue
+            if v.augment.__name__ == 'universal_augment' and getattr(
+                    v.resource_type, 'detail_spec', None):
+                self.fail(
+                    "%s resource has universal augment masking resource augment" % k)
+
     def test_resource_permissions(self):
         self.capture_logging('c7n.cache')
         missing = []
diff --git a/tests/test_query.py b/tests/test_query.py
--- a/tests/test_query.py
+++ b/tests/test_query.py
@@ -63,12 +63,12 @@ def test_resources(self):
             session_factory=session_factory)
         resources = p.run()
         self.assertEqual(len(resources), 1)
-        
+
         output = self.capture_logging(
             name=p.resource_manager.log.name, level=logging.DEBUG)
         p.run()
         self.assertTrue("Using cached internet-gateway: 3", output.getvalue())
-        
+
     def test_get_resources(self):
         session_factory = self.replay_flight_data('test_query_manager_get')
         p = self.load_policy(
@@ -79,6 +79,3 @@ def test_get_resources(self):
         self.assertEqual(len(resources), 1)
         resources = p.resource_manager.get_resources(['igw-5bce113f'])
         self.assertEqual(resources, [])
-    
-             
-