Repository: awslabs/aws-ddk (ts)

Issue:
typescript: sync `AthenaSQLStage` to `main`
sync `AthenaSQLStage` to `main`


Pull request:
feat: syncing `AthenaSqlStage` to `main`
### Feature or Bugfix
-  syncing `AthenaSqlStage` to `main`

### Relates
- Closes #263

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.


Patch:
diff --git a/API.md b/API.md
index 3b852e7..b9c4af2 100644
--- a/API.md
+++ b/API.md
@@ -328,6 +328,8 @@ Any object.
 | <code><a href="#aws-ddk-core.AthenaSQLStage.property.alarmsEnabled">alarmsEnabled</a></code> | <code>boolean</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaSQLStage.property.cloudwatchAlarms">cloudwatchAlarms</a></code> | <code>aws-cdk-lib.aws_cloudwatch.Alarm[]</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaSQLStage.property.stateMachine">stateMachine</a></code> | <code>aws-cdk-lib.aws_stepfunctions.StateMachine</code> | *No description.* |
+| <code><a href="#aws-ddk-core.AthenaSQLStage.property.eventBridgeEventPath">eventBridgeEventPath</a></code> | <code>string</code> | *No description.* |
+| <code><a href="#aws-ddk-core.AthenaSQLStage.property.stateMachineInput">stateMachineInput</a></code> | <code>{[ key: string ]: any}</code> | *No description.* |
 
 ---
 
@@ -413,6 +415,26 @@ public readonly stateMachine: StateMachine;
 
 ---
 
+##### `eventBridgeEventPath`<sup>Optional</sup> <a name="eventBridgeEventPath" id="aws-ddk-core.AthenaSQLStage.property.eventBridgeEventPath"></a>
+
+```typescript
+public readonly eventBridgeEventPath: string;
+```
+
+- *Type:* string
+
+---
+
+##### `stateMachineInput`<sup>Optional</sup> <a name="stateMachineInput" id="aws-ddk-core.AthenaSQLStage.property.stateMachineInput"></a>
+
+```typescript
+public readonly stateMachineInput: {[ key: string ]: any};
+```
+
+- *Type:* {[ key: string ]: any}
+
+---
+
 
 ### BaseStack <a name="BaseStack" id="aws-ddk-core.BaseStack"></a>
 
@@ -5233,12 +5255,13 @@ const athenaToSQLStageProps: AthenaToSQLStageProps = { ... }
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.stateMachineFailedExecutionsAlarmThreshold">stateMachineFailedExecutionsAlarmThreshold</a></code> | <code>number</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.stateMachineInput">stateMachineInput</a></code> | <code>{[ key: string ]: any}</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.stateMachineName">stateMachineName</a></code> | <code>string</code> | *No description.* |
-| <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.queryString">queryString</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.catalogName">catalogName</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.databaseName">databaseName</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.encryptionKey">encryptionKey</a></code> | <code>aws-cdk-lib.aws_kms.Key</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.encryptionOption">encryptionOption</a></code> | <code>aws-cdk-lib.aws_stepfunctions_tasks.EncryptionOption</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.outputLocation">outputLocation</a></code> | <code>aws-cdk-lib.aws_s3.Location</code> | *No description.* |
+| <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.queryString">queryString</a></code> | <code>string</code> | *No description.* |
+| <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.queryStringPath">queryStringPath</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.AthenaToSQLStageProps.property.workGroup">workGroup</a></code> | <code>string</code> | *No description.* |
 
 ---
@@ -5323,16 +5346,6 @@ public readonly stateMachineName: string;
 
 ---
 
-##### `queryString`<sup>Required</sup> <a name="queryString" id="aws-ddk-core.AthenaToSQLStageProps.property.queryString"></a>
-
-```typescript
-public readonly queryString: string;
-```
-
-- *Type:* string
-
----
-
 ##### `catalogName`<sup>Optional</sup> <a name="catalogName" id="aws-ddk-core.AthenaToSQLStageProps.property.catalogName"></a>
 
 ```typescript
@@ -5383,6 +5396,26 @@ public readonly outputLocation: Location;
 
 ---
 
+##### `queryString`<sup>Optional</sup> <a name="queryString" id="aws-ddk-core.AthenaToSQLStageProps.property.queryString"></a>
+
+```typescript
+public readonly queryString: string;
+```
+
+- *Type:* string
+
+---
+
+##### `queryStringPath`<sup>Optional</sup> <a name="queryStringPath" id="aws-ddk-core.AthenaToSQLStageProps.property.queryStringPath"></a>
+
+```typescript
+public readonly queryStringPath: string;
+```
+
+- *Type:* string
+
+---
+
 ##### `workGroup`<sup>Optional</sup> <a name="workGroup" id="aws-ddk-core.AthenaToSQLStageProps.property.workGroup"></a>
 
 ```typescript
@@ -6530,11 +6563,15 @@ const glueTransformStageProps: GlueTransformStageProps = { ... }
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineFailedExecutionsAlarmThreshold">stateMachineFailedExecutionsAlarmThreshold</a></code> | <code>number</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineInput">stateMachineInput</a></code> | <code>{[ key: string ]: any}</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineName">stateMachineName</a></code> | <code>string</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.crawlerAllowFailure">crawlerAllowFailure</a></code> | <code>boolean</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.crawlerName">crawlerName</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.crawlerProps">crawlerProps</a></code> | <code>aws-cdk-lib.aws_glue.CfnCrawlerProps</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.jobName">jobName</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.jobProps">jobProps</a></code> | <code>@aws-cdk/aws-glue-alpha.JobProps</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.jobRunArgs">jobRunArgs</a></code> | <code>{[ key: string ]: any}</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryBackoffRate">stateMachineRetryBackoffRate</a></code> | <code>number</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryInterval">stateMachineRetryInterval</a></code> | <code>aws-cdk-lib.Duration</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryMaxAttempts">stateMachineRetryMaxAttempts</a></code> | <code>number</code> | *No description.* |
 
 ---
 
@@ -6618,6 +6655,16 @@ public readonly stateMachineName: string;
 
 ---
 
+##### `crawlerAllowFailure`<sup>Optional</sup> <a name="crawlerAllowFailure" id="aws-ddk-core.GlueTransformStageProps.property.crawlerAllowFailure"></a>
+
+```typescript
+public readonly crawlerAllowFailure: boolean;
+```
+
+- *Type:* boolean
+
+---
+
 ##### `crawlerName`<sup>Optional</sup> <a name="crawlerName" id="aws-ddk-core.GlueTransformStageProps.property.crawlerName"></a>
 
 ```typescript
@@ -6668,6 +6715,36 @@ public readonly jobRunArgs: {[ key: string ]: any};
 
 ---
 
+##### `stateMachineRetryBackoffRate`<sup>Optional</sup> <a name="stateMachineRetryBackoffRate" id="aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryBackoffRate"></a>
+
+```typescript
+public readonly stateMachineRetryBackoffRate: number;
+```
+
+- *Type:* number
+
+---
+
+##### `stateMachineRetryInterval`<sup>Optional</sup> <a name="stateMachineRetryInterval" id="aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryInterval"></a>
+
+```typescript
+public readonly stateMachineRetryInterval: Duration;
+```
+
+- *Type:* aws-cdk-lib.Duration
+
+---
+
+##### `stateMachineRetryMaxAttempts`<sup>Optional</sup> <a name="stateMachineRetryMaxAttempts" id="aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryMaxAttempts"></a>
+
+```typescript
+public readonly stateMachineRetryMaxAttempts: number;
+```
+
+- *Type:* number
+
+---
+
 ### PermissionsBoundaryProps <a name="PermissionsBoundaryProps" id="aws-ddk-core.PermissionsBoundaryProps"></a>
 
 #### Initializer <a name="Initializer" id="aws-ddk-core.PermissionsBoundaryProps.Initializer"></a>
diff --git a/src/stages/athena-sql.ts b/src/stages/athena-sql.ts
index e8e52a6..aa89a46 100644
--- a/src/stages/athena-sql.ts
+++ b/src/stages/athena-sql.ts
@@ -1,4 +1,5 @@
 import * as events from "aws-cdk-lib/aws-events";
+import * as eventsTargets from "aws-cdk-lib/aws-events-targets";
 import * as kms from "aws-cdk-lib/aws-kms";
 import * as s3 from "aws-cdk-lib/aws-s3";
 import * as sfn from "aws-cdk-lib/aws-stepfunctions";
@@ -7,7 +8,8 @@ import { Construct } from "constructs";
 import { StateMachineStage, StateMachineStageProps } from "../pipelines/stage";
 
 export interface AthenaToSQLStageProps extends StateMachineStageProps {
-  readonly queryString: string;
+  readonly queryString?: string;
+  readonly queryStringPath?: string;
   readonly workGroup?: string;
   readonly catalogName?: string;
   readonly databaseName?: string;
@@ -20,38 +22,51 @@ export class AthenaSQLStage extends StateMachineStage {
   readonly targets?: events.IRuleTarget[];
   readonly eventPattern?: events.EventPattern;
   readonly stateMachine: sfn.StateMachine;
+  readonly stateMachineInput?: { [key: string]: any };
+  readonly eventBridgeEventPath?: string;
 
   constructor(scope: Construct, id: string, props: AthenaToSQLStageProps) {
     super(scope, id, props);
 
-    const { queryString, workGroup, catalogName, databaseName, outputLocation } = props;
-
+    this.stateMachineInput = props.stateMachineInput;
     const encryptionOption = props.encryptionOption ?? tasks.EncryptionOption.S3_MANAGED;
     const encryptionKey = props.encryptionKey;
 
+    if (props.queryString && props.queryStringPath) {
+      throw TypeError("For this stage provide one of queryString or queryStringPath parameter, not both");
+    }
+
+    const queryStringInput = props.queryStringPath ? sfn.JsonPath.stringAt(props.queryStringPath) : props.queryString;
+    if (!queryStringInput) {
+      throw TypeError("For this stage one of queryString or queryStringPath parameter is required");
+    }
+
     const startQueryExec = new tasks.AthenaStartQueryExecution(this, "Start Query Exec", {
-      queryString: queryString,
+      queryString: queryStringInput,
       integrationPattern: sfn.IntegrationPattern.RUN_JOB,
       queryExecutionContext: {
-        catalogName: catalogName,
-        databaseName: databaseName,
+        catalogName: props.catalogName,
+        databaseName: props.databaseName,
       },
       resultConfiguration: {
         encryptionConfiguration: {
           encryptionOption: encryptionOption,
           encryptionKey: encryptionKey,
         },
-        outputLocation: outputLocation,
+        outputLocation: props.outputLocation,
       },
-      workGroup: workGroup,
+      workGroup: props.workGroup,
     });
 
     const definition = startQueryExec.next(new sfn.Succeed(this, "Success"));
 
-    ({
-      eventPattern: this.eventPattern,
-      targets: this.targets,
-      stateMachine: this.stateMachine,
-    } = this.createStateMachine(definition, props));
+    ({ eventPattern: this.eventPattern, stateMachine: this.stateMachine } = this.createStateMachine(definition, props));
+    this.targets = [
+      new eventsTargets.SfnStateMachine(this.stateMachine, {
+        input: props.queryString
+          ? events.RuleTargetInput.fromObject(this.stateMachineInput)
+          : events.RuleTargetInput.fromEventPath("$.detail"),
+      }),
+    ];
   }
 }
diff --git a/src/stages/glue-transform.ts b/src/stages/glue-transform.ts
index 3062ee0..a5d2580 100644
--- a/src/stages/glue-transform.ts
+++ b/src/stages/glue-transform.ts
@@ -13,6 +13,10 @@ export interface GlueTransformStageProps extends StateMachineStageProps {
   readonly jobRunArgs?: { [key: string]: any };
   readonly crawlerName?: string;
   readonly crawlerProps?: glue.CfnCrawlerProps;
+  readonly crawlerAllowFailure?: boolean;
+  readonly stateMachineRetryMaxAttempts?: number;
+  readonly stateMachineRetryBackoffRate?: number;
+  readonly stateMachineRetryInterval?: cdk.Duration;
 }
 
 export class GlueTransformStage extends StateMachineStage {
@@ -46,10 +50,18 @@ export class GlueTransformStage extends StateMachineStage {
 
     const successTask = new sfn.Succeed(this, "Success");
 
-    crawlObject.addCatch(successTask, {
+    crawlObject.addRetry({
       errors: ["Glue.CrawlerRunningException"],
+      maxAttempts: props.stateMachineRetryMaxAttempts ?? 3,
+      backoffRate: props.stateMachineRetryBackoffRate ?? 2,
+      interval: props.stateMachineRetryInterval ?? cdk.Duration.seconds(1),
     });
 
+    const crawlerAllowFailure = props.crawlerAllowFailure ?? true;
+    if (crawlerAllowFailure) {
+      crawlObject.addCatch(successTask, { errors: ["Glue.CrawlerRunningException"] });
+    }
+
     const definition = startJobRun.next(crawlObject.next(successTask));
 
     ({


Test patch:
diff --git a/test/athena-sql-stage.test.ts b/test/athena-sql-stage.test.ts
index c210167..92be40a 100644
--- a/test/athena-sql-stage.test.ts
+++ b/test/athena-sql-stage.test.ts
@@ -46,3 +46,35 @@ test("AthenaToSQL stage with additional properties", () => {
     StateMachineName: "dummy-statemachine",
   });
 });
+
+test("AthenaToSQL stage using 'queryStringPath'", () => {
+  const stack = new cdk.Stack();
+  new AthenaSQLStage(stack, "athena-sql", {
+    queryStringPath: "$.queryString",
+    workGroup: "primary",
+  });
+
+  const template = Template.fromStack(stack);
+  template.hasResourceProperties("AWS::StepFunctions::StateMachine", {
+    DefinitionString: {
+      "Fn::Join": ["", Match.arrayWith([Match.stringLikeRegexp("Parameters.*QueryString.*.queryString")])],
+    },
+  });
+});
+
+test("AthenaSQLStage missing query string property", () => {
+  const stack = new cdk.Stack();
+  expect(() => {
+    new AthenaSQLStage(stack, "Stage", {});
+  }).toThrowError("For this stage one of queryString or queryStringPath parameter is required");
+});
+
+test("AthenaSQLStage duplicate query string properties", () => {
+  const stack = new cdk.Stack();
+  expect(() => {
+    new AthenaSQLStage(stack, "Stage", {
+      queryString: "SELECT 1;",
+      queryStringPath: "$.queryString",
+    });
+  }).toThrowError("For this stage provide one of queryString or queryStringPath parameter, not both");
+});
diff --git a/test/glue-transform-stage.test.ts b/test/glue-transform-stage.test.ts
index 6a55010..bcd1768 100644
--- a/test/glue-transform-stage.test.ts
+++ b/test/glue-transform-stage.test.ts
@@ -90,3 +90,47 @@ test("GlueTranformStage must have 'crawlerName' or 'crawlerProps' set", () => {
     });
   }).toThrowError("'crawlerName' or 'crawlerProps' must be set to instantiate this stage");
 });
+
+test("GlueTransformStage retry settings", () => {
+  const stack = new cdk.Stack();
+
+  new GlueTransformStage(stack, "glue-transform", {
+    jobName: "myJob",
+    crawlerName: "myCrawler",
+    stateMachineRetryBackoffRate: 3,
+    stateMachineRetryInterval: cdk.Duration.seconds(5),
+    stateMachineRetryMaxAttempts: 2,
+  });
+
+  const template = Template.fromStack(stack);
+  template.hasResourceProperties("AWS::StepFunctions::StateMachine", {
+    DefinitionString: {
+      "Fn::Join": [
+        "",
+        Match.arrayWith([Match.stringLikeRegexp('Crawl Object.*IntervalSeconds":5.*MaxAttempts":2.*BackoffRate":3')]),
+      ],
+    },
+  });
+});
+
+test("GlueTransformStage crawler allo failure settings", () => {
+  const stack = new cdk.Stack();
+
+  new GlueTransformStage(stack, "glue-transform-disallow-failure", {
+    jobName: "myJob",
+    crawlerName: "myCrawler",
+    crawlerAllowFailure: false,
+  });
+
+  new GlueTransformStage(stack, "glue-transform-allow-failure", {
+    jobName: "myJob",
+    crawlerName: "myCrawler",
+  });
+
+  const template = Template.fromStack(stack);
+  template.hasResourceProperties("AWS::StepFunctions::StateMachine", {
+    DefinitionString: {
+      "Fn::Join": ["", Match.arrayWith([Match.stringLikeRegexp("Catch.*ErrorEquals.*Glue.CrawlerRunningException")])],
+    },
+  });
+});