Repository: cloud-custodian/cloud-custodian

Issue:
aws - new generic resource action "invoke-stepfunction" enhancement
Enhancement which behaves just like action invoke-lambda but invokes (aka starts execution of) step function state machines for use cases which need sequencing and conditional logic which is more easily implemented in step functions.  

Doc hint: See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/stepfunctions.html#SFN.Client.start_execution


Discussion:
step functions have fairly limited size limits for inputs, invoke lambda we do as a bulk op, in this context the step function invoke per resource?
limit is 32k for per input https://docs.aws.amazon.com/step-functions/latest/dg/limits.html
Yes, invoke per resource would be expected.  The use case coming from the client is for s3 buckets, where the want to set some default policies if a there is no PutBucketPolicy call invoked within a certain time, yet don't want to overwrite any PutBuckeyPolicies which the original creator invokes shortly after creating the bucket.  Intermittently they are getting ErrorCode=OperationAborted from AWS with message " A conflicting conditional operation is currently in progress against this resource. Try again." with their set of custodian policies for S3 buckets.

Resolving patch:
diff --git a/c7n/resources/sfn.py b/c7n/resources/sfn.py
--- a/c7n/resources/sfn.py
+++ b/c7n/resources/sfn.py
@@ -13,9 +13,12 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+
+from c7n.actions import Action
 from c7n.manager import resources
 from c7n.query import QueryResourceManager
 from c7n.tags import Tag, RemoveTag
+from c7n.utils import type_schema, local_session, dumps, chunks
 
 
 @resources.register('step-machine')
@@ -35,6 +38,95 @@ class resource_type(object):
         filter_name = None
 
 
+class InvokeStepFunction(Action):
+    """Invoke step function on resources.
+
+    By default this will invoke a step function for each resource
+    providing both the `policy` and `resource` as input.
+
+    That behavior can be configured setting policy and bulk
+    boolean flags on the action.
+
+    If bulk action parameter is set to true, then the step
+    function will be invoked in bulk, with a set of resource arns
+    under the `resources` key.
+
+    The size of the batch can be configured via the batch-size
+    parameter. Note step function state (input, execution, etc)must
+    fit within 32k, we default to batch size 250.
+
+    :example:
+
+    .. code-block:: yaml
+
+       policies:
+         - name: invoke-step-function
+           resource: s3
+           filters:
+             - is-log-target
+             - "tag:IngestSetup": absent
+           actions:
+             - type: invoke-sfn
+               # This will cause the workflow to be invoked
+               # with many resources arns in a single execution.
+               # Note this is *not* the default.
+               bulk: true
+               batch-size: 10
+               state-machine: LogIngestSetup
+    """
+
+    schema = type_schema(
+        'invoke-sfn',
+        required=['state-machine'],
+        **{'state-machine': {'type': 'string'},
+           'batch-size': {'type': 'integer'},
+           'bulk': {'type': 'boolean'},
+           'policy': {'type': 'boolean'}})
+    schema_alias = True
+    permissions = ('stepfunctions:StartExecution',)
+
+    def process(self, resources):
+        client = local_session(
+            self.manager.session_factory).client('stepfunctions')
+        arn = self.data['state-machine']
+        if not arn.startswith('arn'):
+            arn = 'arn:aws:states:{}:{}:stateMachine:{}'.format(
+                self.manager.config.region, self.manager.config.account_id, arn)
+
+        params = {'stateMachineArn': arn}
+        pinput = {}
+
+        if self.data.get('policy', True):
+            pinput['policy'] = dict(self.manager.data)
+
+        resource_set = list(zip(self.manager.get_arns(resources), resources))
+        if self.data.get('bulk', False) is True:
+            return self.invoke_batch(client, params, pinput, resource_set)
+
+        for arn, r in resource_set:
+            pinput['resource'] = r
+            params['input'] = dumps(pinput)
+            r['c7n:execution-arn'] = self.manager.retry(
+                client.start_execution, **params).get('executionArn')
+
+    def invoke_batch(self, client, params, pinput, resource_set):
+        for batch_rset in chunks(resource_set, self.data.get('batch-size', 250)):
+            pinput['resources'] = [rarn for rarn, _ in batch_rset]
+            params['input'] = dumps(pinput)
+            exec_arn = self.manager.retry(
+                client.start_execution, **params).get('executionArn')
+            for _, r in resource_set:
+                r['c7n:execution-arn'] = exec_arn
+
+    @classmethod
+    def register(cls, registry, key):
+        for _, r in registry.items():
+            r.action_registry.register('invoke-sfn', cls)
+
+
+resources.subscribe(resources.EVENT_FINAL, InvokeStepFunction.register)
+
+
 @StepFunction.action_registry.register('tag')
 class TagStepFunction(Tag):
     """Action to create tag(s) on a step function


Test patch:
diff --git a/tests/data/placebo/test_invoke_sfn/states.DescribeExecution_1.json b/tests/data/placebo/test_invoke_sfn/states.DescribeExecution_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn/states.DescribeExecution_1.json
@@ -0,0 +1,41 @@
+{
+    "status_code": 200,
+    "data": {
+        "executionArn": "arn:aws:states:us-east-1:644160558196:execution:Helloworld:283ac279-1625-4a31-ae74-43e44d789bc1",
+        "stateMachineArn": "arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld",
+        "name": "283ac279-1625-4a31-ae74-43e44d789bc1",
+        "status": "SUCCEEDED",
+        "startDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 11,
+            "minute": 35,
+            "second": 10,
+            "microsecond": 366000
+        },
+        "stopDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 11,
+            "minute": 35,
+            "second": 10,
+            "microsecond": 480000
+        },
+        "input": "{\n\"policy\": {\n\"name\": \"test-invoke-sfn\",\n\"resource\": \"step-machine\",\n\"actions\": [\n{\n\"type\": \"invoke-sfn\",\n\"state-machine\": \"Helloworld\"\n}\n]\n},\n\"resource\": {\n\"stateMachineArn\": \"arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld\",\n\"name\": \"Helloworld\",\n\"creationDate\": \"2019-06-24T11:33:30.570000-04:00\",\n\"status\": \"ACTIVE\",\n\"definition\": \"{\\n  \\\"Comment\\\": \\\"A Hello World example of the Amazon States Language using a Pass state\\\",\\n  \\\"StartAt\\\": \\\"HelloWorld\\\",\\n  \\\"States\\\": {\\n    \\\"HelloWorld\\\": {\\n      \\\"Type\\\": \\\"Pass\\\",\\n      \\\"Result\\\": \\\"Hello World!\\\",\\n      \\\"End\\\": true\\n    }\\n  }\\n}\\n\",\n\"roleArn\": \"arn:aws:iam::644160558196:role/service-role/SfnDev\"\n}\n}",
+        "output": "\"Hello World!\"",
+        "ResponseMetadata": {
+            "RequestId": "a6dc752f-9695-11e9-ae07-7568ddb8b446",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "a6dc752f-9695-11e9-ae07-7568ddb8b446",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "1165"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_invoke_sfn/states.DescribeStateMachine_1.json b/tests/data/placebo/test_invoke_sfn/states.DescribeStateMachine_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn/states.DescribeStateMachine_1.json
@@ -0,0 +1,30 @@
+{
+    "status_code": 200,
+    "data": {
+        "stateMachineArn": "arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld",
+        "name": "Helloworld",
+        "status": "ACTIVE",
+        "definition": "{\n  \"Comment\": \"A Hello World example of the Amazon States Language using a Pass state\",\n  \"StartAt\": \"HelloWorld\",\n  \"States\": {\n    \"HelloWorld\": {\n      \"Type\": \"Pass\",\n      \"Result\": \"Hello World!\",\n      \"End\": true\n    }\n  }\n}\n",
+        "roleArn": "arn:aws:iam::644160558196:role/service-role/SfnDev",
+        "creationDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 11,
+            "minute": 33,
+            "second": 30,
+            "microsecond": 570000
+        },
+        "ResponseMetadata": {
+            "RequestId": "a6ad2886-9695-11e9-a55f-09f72fa884c5",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "a6ad2886-9695-11e9-a55f-09f72fa884c5",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "498"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_invoke_sfn/states.ListStateMachines_1.json b/tests/data/placebo/test_invoke_sfn/states.ListStateMachines_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn/states.ListStateMachines_1.json
@@ -0,0 +1,31 @@
+{
+    "status_code": 200,
+    "data": {
+        "stateMachines": [
+            {
+                "stateMachineArn": "arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld",
+                "name": "Helloworld",
+                "creationDate": {
+                    "__class__": "datetime",
+                    "year": 2019,
+                    "month": 6,
+                    "day": 24,
+                    "hour": 11,
+                    "minute": 33,
+                    "second": 30,
+                    "microsecond": 570000
+                }
+            }
+        ],
+        "ResponseMetadata": {
+            "RequestId": "a6a1ddb2-9695-11e9-8ce2-15d1ff097ae1",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "a6a1ddb2-9695-11e9-8ce2-15d1ff097ae1",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "154"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_invoke_sfn/states.StartExecution_1.json b/tests/data/placebo/test_invoke_sfn/states.StartExecution_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn/states.StartExecution_1.json
@@ -0,0 +1,26 @@
+{
+    "status_code": 200,
+    "data": {
+        "executionArn": "arn:aws:states:us-east-1:644160558196:execution:Helloworld:283ac279-1625-4a31-ae74-43e44d789bc1",
+        "startDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 11,
+            "minute": 35,
+            "second": 10,
+            "microsecond": 366000
+        },
+        "ResponseMetadata": {
+            "RequestId": "a6bbf4aa-9695-11e9-babd-4b277c0ed644",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "a6bbf4aa-9695-11e9-babd-4b277c0ed644",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "143"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_invoke_sfn_bulk/states.DescribeExecution_1.json b/tests/data/placebo/test_invoke_sfn_bulk/states.DescribeExecution_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn_bulk/states.DescribeExecution_1.json
@@ -0,0 +1,41 @@
+{
+    "status_code": 200,
+    "data": {
+        "executionArn": "arn:aws:states:us-east-1:644160558196:execution:Helloworld:4a863454-be66-4396-8633-2459380be901",
+        "stateMachineArn": "arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld",
+        "name": "4a863454-be66-4396-8633-2459380be901",
+        "status": "SUCCEEDED",
+        "startDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 12,
+            "minute": 38,
+            "second": 21,
+            "microsecond": 576000
+        },
+        "stopDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 12,
+            "minute": 38,
+            "second": 21,
+            "microsecond": 602000
+        },
+        "input": "{\n\"policy\": {\n\"name\": \"test-invoke-sfn-bulk\",\n\"resource\": \"step-machine\",\n\"actions\": [\n{\n\"type\": \"invoke-sfn\",\n\"bulk\": true,\n\"state-machine\": \"Helloworld\"\n}\n]\n},\n\"resources\": [\n\"arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld\"\n]\n}",
+        "output": "\"Hello World!\"",
+        "ResponseMetadata": {
+            "RequestId": "7a8e0ec3-969e-11e9-8761-3d11d2a66834",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "7a8e0ec3-969e-11e9-8761-3d11d2a66834",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "644"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_invoke_sfn_bulk/states.DescribeStateMachine_1.json b/tests/data/placebo/test_invoke_sfn_bulk/states.DescribeStateMachine_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn_bulk/states.DescribeStateMachine_1.json
@@ -0,0 +1,30 @@
+{
+    "status_code": 200,
+    "data": {
+        "stateMachineArn": "arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld",
+        "name": "Helloworld",
+        "status": "ACTIVE",
+        "definition": "{\n  \"Comment\": \"A Hello World example of the Amazon States Language using a Pass state\",\n  \"StartAt\": \"HelloWorld\",\n  \"States\": {\n    \"HelloWorld\": {\n      \"Type\": \"Pass\",\n      \"Result\": \"Hello World!\",\n      \"End\": true\n    }\n  }\n}\n",
+        "roleArn": "arn:aws:iam::644160558196:role/service-role/SfnDev",
+        "creationDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 11,
+            "minute": 33,
+            "second": 30,
+            "microsecond": 570000
+        },
+        "ResponseMetadata": {
+            "RequestId": "7a709bcd-969e-11e9-808f-3b46d0d8e481",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "7a709bcd-969e-11e9-808f-3b46d0d8e481",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "498"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_invoke_sfn_bulk/states.ListStateMachines_1.json b/tests/data/placebo/test_invoke_sfn_bulk/states.ListStateMachines_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn_bulk/states.ListStateMachines_1.json
@@ -0,0 +1,31 @@
+{
+    "status_code": 200,
+    "data": {
+        "stateMachines": [
+            {
+                "stateMachineArn": "arn:aws:states:us-east-1:644160558196:stateMachine:Helloworld",
+                "name": "Helloworld",
+                "creationDate": {
+                    "__class__": "datetime",
+                    "year": 2019,
+                    "month": 6,
+                    "day": 24,
+                    "hour": 11,
+                    "minute": 33,
+                    "second": 30,
+                    "microsecond": 570000
+                }
+            }
+        ],
+        "ResponseMetadata": {
+            "RequestId": "7a6577ad-969e-11e9-9dac-53c44a09848f",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "7a6577ad-969e-11e9-9dac-53c44a09848f",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "154"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_invoke_sfn_bulk/states.StartExecution_1.json b/tests/data/placebo/test_invoke_sfn_bulk/states.StartExecution_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_invoke_sfn_bulk/states.StartExecution_1.json
@@ -0,0 +1,26 @@
+{
+    "status_code": 200,
+    "data": {
+        "executionArn": "arn:aws:states:us-east-1:644160558196:execution:Helloworld:4a863454-be66-4396-8633-2459380be901",
+        "startDate": {
+            "__class__": "datetime",
+            "year": 2019,
+            "month": 6,
+            "day": 24,
+            "hour": 12,
+            "minute": 38,
+            "second": 21,
+            "microsecond": 576000
+        },
+        "ResponseMetadata": {
+            "RequestId": "7a7c0d6b-969e-11e9-a2dc-cb8fdcd59d30",
+            "HTTPStatusCode": 200,
+            "HTTPHeaders": {
+                "x-amzn-requestid": "7a7c0d6b-969e-11e9-a2dc-cb8fdcd59d30",
+                "content-type": "application/x-amz-json-1.0",
+                "content-length": "143"
+            },
+            "RetryAttempts": 0
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_sfn.py b/tests/test_sfn.py
--- a/tests/test_sfn.py
+++ b/tests/test_sfn.py
@@ -16,6 +16,52 @@
 
 
 class TestStepFunction(BaseTest):
+
+    def test_invoke_batch(self):
+        factory = self.replay_flight_data('test_invoke_sfn_bulk')
+        p = self.load_policy({
+            'name': 'test-invoke-sfn-bulk',
+            'resource': 'step-machine',
+            'actions': [{
+                'type': 'invoke-sfn',
+                'bulk': True,
+                'state-machine': 'Helloworld'}]},
+            session_factory=factory,
+            config={'account_id': '644160558196'})
+
+        resources = p.run()
+
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(resources[0]['name'], 'Helloworld')
+        self.assertTrue('c7n:execution-arn' in resources[0])
+        client = factory().client('stepfunctions')
+        self.assertEqual(
+            client.describe_execution(
+                executionArn=resources[0]['c7n:execution-arn'])['status'],
+            'SUCCEEDED')
+
+    def test_invoke_sfn(self):
+        factory = self.replay_flight_data('test_invoke_sfn')
+        p = self.load_policy({
+            'name': 'test-invoke-sfn',
+            'resource': 'step-machine',
+            'actions': [{
+                'type': 'invoke-sfn',
+                'state-machine': 'Helloworld'}]},
+            session_factory=factory,
+            config={'account_id': '644160558196'})
+
+        resources = p.run()
+
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(resources[0]['name'], 'Helloworld')
+        self.assertTrue('c7n:execution-arn' in resources[0])
+        client = factory().client('stepfunctions')
+        self.assertEqual(
+            client.describe_execution(
+                executionArn=resources[0]['c7n:execution-arn'])['status'],
+            'SUCCEEDED')
+
     def test_sfn_resource(self):
         session_factory = self.replay_flight_data('test_sfn_resource')
         p = self.load_policy(