Repository: awslabs/aws-ddk (language: python)

Issue:
typescript: update glue transform stage



Discussion / hints:

Fix patch:
diff --git a/API.md b/API.md
index 3b852e74..34cf1d71 100644
--- a/API.md
+++ b/API.md
@@ -6530,11 +6530,15 @@ const glueTransformStageProps: GlueTransformStageProps = { ... }
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineFailedExecutionsAlarmThreshold">stateMachineFailedExecutionsAlarmThreshold</a></code> | <code>number</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineInput">stateMachineInput</a></code> | <code>{[ key: string ]: any}</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineName">stateMachineName</a></code> | <code>string</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.crawlerAllowFailure">crawlerAllowFailure</a></code> | <code>boolean</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.crawlerName">crawlerName</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.crawlerProps">crawlerProps</a></code> | <code>aws-cdk-lib.aws_glue.CfnCrawlerProps</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.jobName">jobName</a></code> | <code>string</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.jobProps">jobProps</a></code> | <code>@aws-cdk/aws-glue-alpha.JobProps</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStageProps.property.jobRunArgs">jobRunArgs</a></code> | <code>{[ key: string ]: any}</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryBackoffRate">stateMachineRetryBackoffRate</a></code> | <code>number</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryInterval">stateMachineRetryInterval</a></code> | <code>aws-cdk-lib.Duration</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryMaxAttempts">stateMachineRetryMaxAttempts</a></code> | <code>number</code> | *No description.* |
 
 ---
 
@@ -6618,6 +6622,16 @@ public readonly stateMachineName: string;
 
 ---
 
+##### `crawlerAllowFailure`<sup>Optional</sup> <a name="crawlerAllowFailure" id="aws-ddk-core.GlueTransformStageProps.property.crawlerAllowFailure"></a>
+
+```typescript
+public readonly crawlerAllowFailure: boolean;
+```
+
+- *Type:* boolean
+
+---
+
 ##### `crawlerName`<sup>Optional</sup> <a name="crawlerName" id="aws-ddk-core.GlueTransformStageProps.property.crawlerName"></a>
 
 ```typescript
@@ -6668,6 +6682,36 @@ public readonly jobRunArgs: {[ key: string ]: any};
 
 ---
 
+##### `stateMachineRetryBackoffRate`<sup>Optional</sup> <a name="stateMachineRetryBackoffRate" id="aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryBackoffRate"></a>
+
+```typescript
+public readonly stateMachineRetryBackoffRate: number;
+```
+
+- *Type:* number
+
+---
+
+##### `stateMachineRetryInterval`<sup>Optional</sup> <a name="stateMachineRetryInterval" id="aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryInterval"></a>
+
+```typescript
+public readonly stateMachineRetryInterval: Duration;
+```
+
+- *Type:* aws-cdk-lib.Duration
+
+---
+
+##### `stateMachineRetryMaxAttempts`<sup>Optional</sup> <a name="stateMachineRetryMaxAttempts" id="aws-ddk-core.GlueTransformStageProps.property.stateMachineRetryMaxAttempts"></a>
+
+```typescript
+public readonly stateMachineRetryMaxAttempts: number;
+```
+
+- *Type:* number
+
+---
+
 ### PermissionsBoundaryProps <a name="PermissionsBoundaryProps" id="aws-ddk-core.PermissionsBoundaryProps"></a>
 
 #### Initializer <a name="Initializer" id="aws-ddk-core.PermissionsBoundaryProps.Initializer"></a>
diff --git a/src/stages/glue-transform.ts b/src/stages/glue-transform.ts
index 3062ee0f..a5d25800 100644
--- a/src/stages/glue-transform.ts
+++ b/src/stages/glue-transform.ts
@@ -13,6 +13,10 @@ export interface GlueTransformStageProps extends StateMachineStageProps {
   readonly jobRunArgs?: { [key: string]: any };
   readonly crawlerName?: string;
   readonly crawlerProps?: glue.CfnCrawlerProps;
+  readonly crawlerAllowFailure?: boolean;
+  readonly stateMachineRetryMaxAttempts?: number;
+  readonly stateMachineRetryBackoffRate?: number;
+  readonly stateMachineRetryInterval?: cdk.Duration;
 }
 
 export class GlueTransformStage extends StateMachineStage {
@@ -46,10 +50,18 @@ export class GlueTransformStage extends StateMachineStage {
 
     const successTask = new sfn.Succeed(this, "Success");
 
-    crawlObject.addCatch(successTask, {
+    crawlObject.addRetry({
       errors: ["Glue.CrawlerRunningException"],
+      maxAttempts: props.stateMachineRetryMaxAttempts ?? 3,
+      backoffRate: props.stateMachineRetryBackoffRate ?? 2,
+      interval: props.stateMachineRetryInterval ?? cdk.Duration.seconds(1),
     });
 
+    const crawlerAllowFailure = props.crawlerAllowFailure ?? true;
+    if (crawlerAllowFailure) {
+      crawlObject.addCatch(successTask, { errors: ["Glue.CrawlerRunningException"] });
+    }
+
     const definition = startJobRun.next(crawlObject.next(successTask));
 
     ({


Test patch:
diff --git a/test/glue-transform-stage.test.ts b/test/glue-transform-stage.test.ts
index 6a55010a..bcd17687 100644
--- a/test/glue-transform-stage.test.ts
+++ b/test/glue-transform-stage.test.ts
@@ -90,3 +90,47 @@ test("GlueTranformStage must have 'crawlerName' or 'crawlerProps' set", () => {
     });
   }).toThrowError("'crawlerName' or 'crawlerProps' must be set to instantiate this stage");
 });
+
+test("GlueTransformStage retry settings", () => {
+  const stack = new cdk.Stack();
+
+  new GlueTransformStage(stack, "glue-transform", {
+    jobName: "myJob",
+    crawlerName: "myCrawler",
+    stateMachineRetryBackoffRate: 3,
+    stateMachineRetryInterval: cdk.Duration.seconds(5),
+    stateMachineRetryMaxAttempts: 2,
+  });
+
+  const template = Template.fromStack(stack);
+  template.hasResourceProperties("AWS::StepFunctions::StateMachine", {
+    DefinitionString: {
+      "Fn::Join": [
+        "",
+        Match.arrayWith([Match.stringLikeRegexp('Crawl Object.*IntervalSeconds":5.*MaxAttempts":2.*BackoffRate":3')]),
+      ],
+    },
+  });
+});
+
+test("GlueTransformStage crawler allo failure settings", () => {
+  const stack = new cdk.Stack();
+
+  new GlueTransformStage(stack, "glue-transform-disallow-failure", {
+    jobName: "myJob",
+    crawlerName: "myCrawler",
+    crawlerAllowFailure: false,
+  });
+
+  new GlueTransformStage(stack, "glue-transform-allow-failure", {
+    jobName: "myJob",
+    crawlerName: "myCrawler",
+  });
+
+  const template = Template.fromStack(stack);
+  template.hasResourceProperties("AWS::StepFunctions::StateMachine", {
+    DefinitionString: {
+      "Fn::Join": ["", Match.arrayWith([Match.stringLikeRegexp("Catch.*ErrorEquals.*Glue.CrawlerRunningException")])],
+    },
+  });
+});