Repository: awslabs/aws-ddk (language: python)

Issue:
core: make glue crawler optional in glueTransform stage
A user should be able to use gluetransform stage to just trigger glue job and then optionally use crawler to catalog data based on their usecase. Currently having the glue crawler mandatory restricts the use of the stage


Discussion / hints:

Fix patch:
diff --git a/API.md b/API.md
index 96f8f178..bfc5a0a8 100644
--- a/API.md
+++ b/API.md
@@ -3901,8 +3901,10 @@ Any object.
 | <code><a href="#aws-ddk-core.GlueTransformStage.property.alarmsEnabled">alarmsEnabled</a></code> | <code>boolean</code> | Flag indicating whether the alarms are enabled for this stage. |
 | <code><a href="#aws-ddk-core.GlueTransformStage.property.cloudwatchAlarms">cloudwatchAlarms</a></code> | <code>aws-cdk-lib.aws_cloudwatch.Alarm[]</code> | List of CloudWatch Alarms linked to the stage. |
 | <code><a href="#aws-ddk-core.GlueTransformStage.property.stateMachine">stateMachine</a></code> | <code>aws-cdk-lib.aws_stepfunctions.StateMachine</code> | State machine. |
+| <code><a href="#aws-ddk-core.GlueTransformStage.property.definition">definition</a></code> | <code>aws-cdk-lib.aws_stepfunctions.IChainable</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStage.property.glueJob">glueJob</a></code> | <code>@aws-cdk/aws-glue-alpha.IJob</code> | *No description.* |
 | <code><a href="#aws-ddk-core.GlueTransformStage.property.crawler">crawler</a></code> | <code>aws-cdk-lib.aws_glue.CfnCrawler</code> | *No description.* |
+| <code><a href="#aws-ddk-core.GlueTransformStage.property.crawlerName">crawlerName</a></code> | <code>string</code> | *No description.* |
 
 ---
 
@@ -4007,6 +4009,16 @@ State machine.
 
 ---
 
+##### `definition`<sup>Required</sup> <a name="definition" id="aws-ddk-core.GlueTransformStage.property.definition"></a>
+
+```typescript
+public readonly definition: IChainable;
+```
+
+- *Type:* aws-cdk-lib.aws_stepfunctions.IChainable
+
+---
+
 ##### `glueJob`<sup>Required</sup> <a name="glueJob" id="aws-ddk-core.GlueTransformStage.property.glueJob"></a>
 
 ```typescript
@@ -4027,6 +4039,16 @@ public readonly crawler: CfnCrawler;
 
 ---
 
+##### `crawlerName`<sup>Optional</sup> <a name="crawlerName" id="aws-ddk-core.GlueTransformStage.property.crawlerName"></a>
+
+```typescript
+public readonly crawlerName: string;
+```
+
+- *Type:* string
+
+---
+
 
 ### MWAAEnvironment <a name="MWAAEnvironment" id="aws-ddk-core.MWAAEnvironment"></a>
 
diff --git a/src/stages/glue-transform.ts b/src/stages/glue-transform.ts
index 5347b019..d71e81fb 100644
--- a/src/stages/glue-transform.ts
+++ b/src/stages/glue-transform.ts
@@ -78,6 +78,8 @@ export class GlueTransformStage extends StateMachineStage {
   readonly stateMachine: sfn.StateMachine;
   readonly glueJob: glue_alpha.IJob;
   readonly crawler?: glue.CfnCrawler;
+  readonly crawlerName?: string;
+  readonly definition: sfn.IChainable;
 
   /**
    * Constructs `GlueTransformStage`.
@@ -91,8 +93,11 @@ export class GlueTransformStage extends StateMachineStage {
     this.glueJob = this.getGlueJob(scope, id, props);
     const jobRunArgs = props.jobRunArgs;
 
-    this.crawler = props.crawlerName ? undefined : this.getCrawler(props);
-    const crawlerName = this.crawler ? this.crawler.ref : props.crawlerName;
+    // Only setup crawler if one of 'crawlerName', 'crawlerProps' or 'crawlerRole' is provided
+    if (props.crawlerName || props.crawlerProps || props.crawlerRole) {
+      this.crawler = props.crawlerName ? undefined : this.getCrawler(props);
+      this.crawlerName = this.crawler ? this.crawler.ref : props.crawlerName;
+    }
 
     const startJobRun = new tasks.GlueStartJobRun(this, "Start Job Run", {
       glueJobName: this.glueJob.jobName,
@@ -102,36 +107,37 @@ export class GlueTransformStage extends StateMachineStage {
     });
 
     const stack = cdk.Stack.of(this);
-    const crawlObject = new tasks.CallAwsService(this, "Crawl Object", {
-      service: "glue",
-      action: "startCrawler",
-      parameters: {
-        Name: crawlerName,
-      },
-      iamResources: [`arn:${stack.partition}:glue:${stack.region}:${stack.account}:crawler/${crawlerName}`],
-    });
-
     const successTask = new sfn.Succeed(this, "Success");
 
-    crawlObject.addRetry({
-      errors: ["Glue.CrawlerRunningException"],
-      maxAttempts: props.stateMachineRetryMaxAttempts ?? 3,
-      backoffRate: props.stateMachineRetryBackoffRate ?? 2,
-      interval: props.stateMachineRetryInterval ?? cdk.Duration.seconds(1),
-    });
-
-    const crawlerAllowFailure = props.crawlerAllowFailure ?? true;
-    if (crawlerAllowFailure) {
-      crawlObject.addCatch(successTask, { errors: ["Glue.CrawlerRunningException"] });
+    if (this.crawlerName) {
+      const crawlObject = new tasks.CallAwsService(this, "Crawl Object", {
+        service: "glue",
+        action: "startCrawler",
+        parameters: {
+          Name: this.crawlerName,
+        },
+        iamResources: [`arn:${stack.partition}:glue:${stack.region}:${stack.account}:crawler/${this.crawlerName}`],
+      });
+      crawlObject.addRetry({
+        errors: ["Glue.CrawlerRunningException"],
+        maxAttempts: props.stateMachineRetryMaxAttempts ?? 3,
+        backoffRate: props.stateMachineRetryBackoffRate ?? 2,
+        interval: props.stateMachineRetryInterval ?? cdk.Duration.seconds(1),
+      });
+      const crawlerAllowFailure = props.crawlerAllowFailure ?? true;
+      if (crawlerAllowFailure) {
+        crawlObject.addCatch(successTask, { errors: ["Glue.CrawlerRunningException"] });
+      }
+      this.definition = startJobRun.next(crawlObject.next(successTask));
+    } else {
+      this.definition = startJobRun.next(successTask);
     }
 
-    const definition = startJobRun.next(crawlObject.next(successTask));
-
     ({
       eventPattern: this.eventPattern,
       targets: this.targets,
       stateMachine: this.stateMachine,
-    } = this.createStateMachine({ definition: definition, ...props }));
+    } = this.createStateMachine({ definition: this.definition, ...props }));
   }
 
   private getGlueJob(scope: Construct, id: string, props: GlueTransformStageProps): glue_alpha.IJob {


Test patch:
diff --git a/test/glue-transform-stage.test.ts b/test/glue-transform-stage.test.ts
index a92be247..62c3f3c0 100644
--- a/test/glue-transform-stage.test.ts
+++ b/test/glue-transform-stage.test.ts
@@ -107,15 +107,6 @@ test("GlueTranformStage must have 'jobName' or 'jobProps' set", () => {
   }).toThrowError("'jobName' or 'jobProps' must be set to instantiate this stage");
 });
 
-test("GlueTranformStage must set crawler role ", () => {
-  const stack = new cdk.Stack();
-  expect(() => {
-    new GlueTransformStage(stack, "Stage", {
-      jobName: "myJob",
-    });
-  }).toThrowError("Crawler Role must be set either by 'crawlerRole' or 'crawlerProps.role");
-});
-
 test("GlueTranformStage must set crawler targets", () => {
   const stack = new cdk.Stack();
   expect(() => {
@@ -148,7 +139,7 @@ test("GlueTransformStage retry settings", () => {
   });
 });
 
-test("GlueTransformStage crawler allo failure settings", () => {
+test("GlueTransformStage crawler allow failure settings", () => {
   const stack = new cdk.Stack();
 
   new GlueTransformStage(stack, "glue-transform-disallow-failure", {
@@ -169,3 +160,14 @@ test("GlueTransformStage crawler allo failure settings", () => {
     },
   });
 });
+
+test("GlueTransformStage no crawler", () => {
+  const stack = new cdk.Stack();
+
+  new GlueTransformStage(stack, "glue-transform-no-crawler", {
+    jobName: "myJob",
+  });
+
+  const template = Template.fromStack(stack);
+  template.resourceCountIs("AWS::Glue::Crawler", 0);
+});