Repository: cloud-custodian/cloud-custodian

Issue:
Add data pipeline resource
We need the ability to limit who can launch a Data Pipeline but no datapipeline resource is available.  At a minimum we would want to have tag, untag, mark-for-op, notify, and delete actions, and the and, marked-for-op, or, and value filters should be available.


Discussion:
@whit537 thanks
PR w/ verbose logging in https://github.com/whit537/cloud-custodian/pull/3.

Resolving patch:
diff --git a/c7n/resources/__init__.py b/c7n/resources/__init__.py
--- a/c7n/resources/__init__.py
+++ b/c7n/resources/__init__.py
@@ -35,6 +35,7 @@ def load_resources():
     import c7n.resources.directory
     import c7n.resources.directconnect
     import c7n.resources.dynamodb
+    import c7n.resources.datapipeline
     import c7n.resources.ebs
     import c7n.resources.ec2
     import c7n.resources.ecr
diff --git a/c7n/resources/datapipeline.py b/c7n/resources/datapipeline.py
new file mode 100644
--- /dev/null
+++ b/c7n/resources/datapipeline.py
@@ -0,0 +1,76 @@
+# Copyright 2016 Capital One Services, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Data Pipeline
+"""
+from botocore.exceptions import ClientError
+
+from c7n.manager import resources
+from c7n.query import QueryResourceManager
+from c7n.utils import chunks, local_session, get_retry
+
+
+@resources.register('datapipeline')
+class DataPipeline(QueryResourceManager):
+
+    retry = staticmethod(get_retry(('Throttled',)))
+
+    class resource_type(object):
+        service = 'datapipeline'
+        type = 'dataPipeline'
+        id = 'id'
+        name = 'name'
+        date = None
+        dimension = 'name'
+        enum_spec = ('list_pipelines', 'pipelineIdList', None)
+
+    def augment(self, resources):
+        filter(None, _datapipeline_info(
+            resources, self.session_factory, self.executor_factory,
+            self.retry))
+        return resources
+
+
+def _datapipeline_info(pipes, session_factory, executor_factory, retry):
+
+    def process_tags(pipe_set):
+        client = local_session(session_factory).client('datapipeline')
+        pipe_map = {pipe['id']: pipe for pipe in pipe_set}
+
+        while True:
+            try:
+                results = retry(
+                    client.describe_pipelines,
+                    pipelineIds=pipe_map.keys())
+                break
+            except ClientError as e:
+                if e.response['Error']['Code'] != 'PipelineNotFound':
+                    raise
+                msg = e.response['Error']['Message']
+                _, lb_name = msg.strip().rsplit(' ', 1)
+                pipe_map.pop(lb_name)
+                if not pipe_map:
+                    results = {'TagDescriptions': []}
+                    break
+                continue
+        for pipe_desc in results['pipelineDescriptionList']:
+            pipe = pipe_map[pipe_desc['pipelineId']]
+            pipe['tags'] = pipe_desc['tags']
+            for field in pipe_desc['fields']:
+                key = field['key']
+                if not key.startswith('@'):
+                    continue
+                pipe[key[1:]] = field['stringValue']
+
+    with executor_factory(max_workers=2) as w:
+        return list(w.map(process_tags, chunks(pipes, 20)))


Test patch:
diff --git a/tests/data/placebo/test_datapipeline_reporting/datapipeline.ActivatePipeline_1.json b/tests/data/placebo/test_datapipeline_reporting/datapipeline.ActivatePipeline_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_datapipeline_reporting/datapipeline.ActivatePipeline_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "74a8e0bc-07e1-11e7-90ed-1f399d793de2", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "74a8e0bc-07e1-11e7-90ed-1f399d793de2", 
+                "date": "Mon, 13 Mar 2017 11:37:35 GMT", 
+                "content-length": "2", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_datapipeline_reporting/datapipeline.AddTags_1.json b/tests/data/placebo/test_datapipeline_reporting/datapipeline.AddTags_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_datapipeline_reporting/datapipeline.AddTags_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "7488878e-07e1-11e7-91e2-bfd5af170b83", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "7488878e-07e1-11e7-91e2-bfd5af170b83", 
+                "date": "Mon, 13 Mar 2017 11:37:35 GMT", 
+                "content-length": "2", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_datapipeline_reporting/datapipeline.CreatePipeline_1.json b/tests/data/placebo/test_datapipeline_reporting/datapipeline.CreatePipeline_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_datapipeline_reporting/datapipeline.CreatePipeline_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "pipelineId": "df-0993359USAD6HT96D2W", 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "73948a7e-07e1-11e7-90ed-1f399d793de2", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "73948a7e-07e1-11e7-90ed-1f399d793de2", 
+                "date": "Mon, 13 Mar 2017 11:37:34 GMT", 
+                "content-length": "39", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_datapipeline_reporting/datapipeline.DeletePipeline_1.json b/tests/data/placebo/test_datapipeline_reporting/datapipeline.DeletePipeline_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_datapipeline_reporting/datapipeline.DeletePipeline_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "8fbec6a5-07e1-11e7-90ed-1f399d793de2", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "8fbec6a5-07e1-11e7-90ed-1f399d793de2", 
+                "date": "Mon, 13 Mar 2017 11:38:21 GMT", 
+                "content-length": "0", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_datapipeline_reporting/datapipeline.DescribePipelines_1.json b/tests/data/placebo/test_datapipeline_reporting/datapipeline.DescribePipelines_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_datapipeline_reporting/datapipeline.DescribePipelines_1.json
@@ -0,0 +1,82 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "75644193-07e1-11e7-91e2-bfd5af170b83", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "75644193-07e1-11e7-91e2-bfd5af170b83", 
+                "date": "Mon, 13 Mar 2017 11:37:37 GMT", 
+                "content-length": "849", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }, 
+        "pipelineDescriptionList": [
+            {
+                "fields": [
+                    {
+                        "stringValue": "2017-03-13T11:37:36", 
+                        "key": "@lastActivationTime"
+                    }, 
+                    {
+                        "stringValue": "[{\"key\":\"foo\",\"value\":\"bar\"}]", 
+                        "key": "*tags"
+                    }, 
+                    {
+                        "stringValue": "2017-03-13T11:37:34", 
+                        "key": "@creationTime"
+                    }, 
+                    {
+                        "stringValue": "PIPELINE", 
+                        "key": "@sphere"
+                    }, 
+                    {
+                        "stringValue": "AIDAIXI7ULG2SDYI3RBNM", 
+                        "key": "pipelineCreator"
+                    }, 
+                    {
+                        "stringValue": "1", 
+                        "key": "@version"
+                    }, 
+                    {
+                        "stringValue": "PipelinesFTW", 
+                        "key": "name"
+                    }, 
+                    {
+                        "stringValue": "df-0993359USAD6HT96D2W", 
+                        "key": "@id"
+                    }, 
+                    {
+                        "stringValue": "SCHEDULING", 
+                        "key": "@pipelineState"
+                    }, 
+                    {
+                        "stringValue": "644160558196", 
+                        "key": "@accountId"
+                    }, 
+                    {
+                        "stringValue": "PipelinesFTW", 
+                        "key": "uniqueId"
+                    }, 
+                    {
+                        "stringValue": "AIDAIXI7ULG2SDYI3RBNM", 
+                        "key": "@userId"
+                    }, 
+                    {
+                        "stringValue": "2017-03-13T11:37:36", 
+                        "key": "@firstActivationTime"
+                    }
+                ], 
+                "pipelineId": "df-0993359USAD6HT96D2W", 
+                "name": "PipelinesFTW", 
+                "tags": [
+                    {
+                        "value": "bar", 
+                        "key": "foo"
+                    }
+                ]
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_datapipeline_reporting/datapipeline.ListPipelines_1.json b/tests/data/placebo/test_datapipeline_reporting/datapipeline.ListPipelines_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_datapipeline_reporting/datapipeline.ListPipelines_1.json
@@ -0,0 +1,23 @@
+{
+    "status_code": 200, 
+    "data": {
+        "hasMoreResults": false, 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "751d2661-07e1-11e7-90ed-1f399d793de2", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "751d2661-07e1-11e7-90ed-1f399d793de2", 
+                "date": "Mon, 13 Mar 2017 11:37:36 GMT", 
+                "content-length": "97", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }, 
+        "pipelineIdList": [
+            {
+                "id": "df-0993359USAD6HT96D2W", 
+                "name": "PipelinesFTW"
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_datapipeline_reporting/datapipeline.PutPipelineDefinition_1.json b/tests/data/placebo/test_datapipeline_reporting/datapipeline.PutPipelineDefinition_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_datapipeline_reporting/datapipeline.PutPipelineDefinition_1.json
@@ -0,0 +1,32 @@
+{
+    "status_code": 200, 
+    "data": {
+        "validationErrors": [], 
+        "errored": false, 
+        "validationWarnings": [
+            {
+                "id": "SayHello", 
+                "warnings": [
+                    " It is recommended to set 'failureAndRerunMode'  to 'cascade' on Default object to enable cascade failure mode for Activity, Datanode, Precondition and Resource objects of a pipeline."
+                ]
+            }, 
+            {
+                "id": "Default", 
+                "warnings": [
+                    "'pipelineLogUri'is missing. It is recommended to set this value on Default object for better troubleshooting."
+                ]
+            }
+        ], 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "746b8998-07e1-11e7-90ed-1f399d793de2", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "746b8998-07e1-11e7-90ed-1f399d793de2", 
+                "date": "Mon, 13 Mar 2017 11:37:35 GMT", 
+                "content-length": "421", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_datapipeline.py b/tests/test_datapipeline.py
new file mode 100644
--- /dev/null
+++ b/tests/test_datapipeline.py
@@ -0,0 +1,73 @@
+# Copyright 2016 Capital One Services, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from common import BaseTest
+
+
+class DataPipelineTest(BaseTest):
+
+    def test_reporting(self):
+        factory = self.replay_flight_data('test_datapipeline_reporting')
+
+        session = factory()
+        client = session.client('datapipeline')
+        pipeline = client.create_pipeline(
+            name='PipelinesFTW', uniqueId='PipelinesFTW')
+        pipe_id = pipeline['pipelineId']
+        client.put_pipeline_definition(
+            pipelineId=pipe_id,
+            pipelineObjects=
+                [{"id": "Default", "name": "Default", "fields": [
+                    {"key": "workerGroup", "stringValue": "workerGroup"}]},
+                 {"id": "Schedule", "name": "Schedule", "fields": [
+                    {"key": "startDateTime",
+                     "stringValue": "2012-12-12T00:00:00"},
+                    {"key": "type", "stringValue": "Schedule"},
+                    {"key": "period", "stringValue": "1 hour"},
+                    {"key": "endDateTime",
+                     "stringValue": "2012-12-21T18:00:00"}]},
+                 {"id": "SayHello", "name": "SayHello", "fields": [
+                    {"key": "type", "stringValue": "ShellCommandActivity"},
+                    {"key": "command", "stringValue": "echo hello"},
+                    {"key": "parent", "refValue": "Default"},
+                    {"key": "schedule", "refValue": "Schedule"}]}
+                  ])
+        client.add_tags(
+            pipelineId=pipe_id,
+            tags=[{'key': 'foo', 'value': 'bar'}]
+        )
+        client.activate_pipeline(pipelineId=pipe_id)
+        self.addCleanup(client.delete_pipeline, pipelineId=pipe_id)
+
+        p = self.load_policy({
+            'name': 'datapipeline-report',
+            'resource': 'datapipeline',
+            'filters': [
+                {'name': 'PipelinesFTW'}],
+            },
+            config={'region': 'us-west-2'},
+            session_factory=factory)
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        resource = resources[0]
+        self.assertEqual(resource['name'], 'PipelinesFTW')
+        self.assertEqual(resource['tags'], [{'key': 'foo', 'value': 'bar'}])
+        self.assertEqual(resource['lastActivationTime'], '2017-03-13T11:37:36')
+        self.assertEqual(resource['creationTime'], '2017-03-13T11:37:34')
+        self.assertEqual(resource['sphere'], 'PIPELINE')
+        self.assertEqual(resource['version'], '1')
+        self.assertEqual(resource['id'], 'df-0993359USAD6HT96D2W')
+        self.assertEqual(resource['pipelineState'], 'SCHEDULING')
+        self.assertEqual(resource['accountId'], '644160558196')
+        self.assertEqual(resource['userId'], 'AIDAIXI7ULG2SDYI3RBNM')
+        self.assertEqual(resource['firstActivationTime'], '2017-03-13T11:37:36')