Repository: awslabs/aws-ddk (language: python)

Issue:
DDK Integration Testing
Implement internal integration tests for DDK Stages.


Discussion / hints:

Fix patch:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ae209091..5148ed0a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -43,34 +43,44 @@ GitHub provides additional document on [forking a repository](https://help.githu
 ## Setting up a test environment
 A number of unit tests are available in the `test/` directory.
 
-#### Prerequisites
+### Prerequisites
 - Make sure `npm` is installed on your machine. (`npm` installation)[https://docs.npmjs.com/downloading-and-installing-node-js-and-npm]
 - Install projen. 
 ```
 npm install projen
 ```
 
-#### Unit Tests 
+### Unit Tests 
 ```
 npx projen test
 ```
 
 
-#### Integration Testing 
+### Integration Testing 
 The integration tests leverage the [`integ-runner` CLI ](https://github.com/aws/aws-cdk/tree/main/packages/%40aws-cdk/integ-runner)
 
-##### Prequisites
+#### Prequisites
 - AWS credentials must be configured in order to run integration tests.
 - Install the `integ-runner` cli.
 ```
 npm install @aws-cdk/integ-runner
 ```
 
-##### Usage 
+#### Usage 
 ```
 integ-runner --language typescript --update-on-failed --directory integ/
 ```
 
+#### Common Issues
+---
+##### **TSError: ⨯ Unable to compile TypeScript:integ.glue-transform.test.ts(3,8: error TS1259: Module '"path"' can only be default-imported using the 'esModuleInterop' flag**
+
+The following flag must be set in `compilerOptions[]` in the active `tsconfig.json` file. 
+```
+"esModuleInterop": true,
+```
+---
+
 ## Finding contributions to work on
 Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start.
 
diff --git a/integ/src/glue_script.py b/integ/src/glue_script.py
new file mode 100644
index 00000000..fc80254b
--- /dev/null
+++ b/integ/src/glue_script.py
@@ -0,0 +1,1 @@
+pass
\ No newline at end of file
diff --git a/src/stages/sqs-lambda.ts b/src/stages/sqs-lambda.ts
index 53d31b31..27744c3a 100644
--- a/src/stages/sqs-lambda.ts
+++ b/src/stages/sqs-lambda.ts
@@ -74,7 +74,9 @@ export class SqsToLambdaStage extends DataStage {
 
     const dlqEnabled = props.dlqEnabled ?? false;
     if (dlqEnabled == true) {
-      this.deadLetterQueue = new sqs.Queue(this, "Dead Letter Queue", {});
+      this.deadLetterQueue = new sqs.Queue(this, "Dead Letter Queue", {
+        fifo: props.sqsQueueProps?.fifo ? props.sqsQueueProps?.fifo : undefined,
+      });
     }
 
     if (props.sqsQueue) {
@@ -88,7 +90,7 @@ export class SqsToLambdaStage extends DataStage {
               maxReceiveCount: props.maxReceiveCount ?? 1,
             }
           : undefined,
-        fifo: props.sqsQueueProps?.fifo ?? false,
+        fifo: props.sqsQueueProps?.fifo ? props.sqsQueueProps?.fifo : undefined,
       });
     }
 


Test patch:
diff --git a/integ/integ.appflow-ingestion.test.ts b/integ/integ.appflow-ingestion.test.ts
new file mode 100644
index 00000000..8e6c8f00
--- /dev/null
+++ b/integ/integ.appflow-ingestion.test.ts
@@ -0,0 +1,51 @@
+import * as integration from "@aws-cdk/integ-tests-alpha";
+import * as cdk from "aws-cdk-lib";
+import { Duration } from "aws-cdk-lib";
+import * as appflow from "aws-cdk-lib/aws-appflow";
+import { Construct } from "constructs";
+import { RequireApproval } from "aws-cdk-lib/cloud-assembly-schema";
+
+import { AppFlowIngestionStage } from "../src";
+import { StateMachineStageProps } from "../src/pipelines/stage";
+
+interface AppFlowIngestionStageTestStackProps extends StateMachineStageProps {
+  readonly flowName?: string;
+  readonly flowExecutionStatusCheckPeriod?: Duration;
+  readonly destinationFlowConfig?: appflow.CfnFlow.DestinationFlowConfigProperty;
+  readonly sourceFlowConfig?: appflow.CfnFlow.SourceFlowConfigProperty;
+  readonly flowTasks?: appflow.CfnFlow.TaskProperty[];
+}
+
+class AppFlowIngestionStageTestStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: AppFlowIngestionStageTestStackProps) {
+    super(scope, id, props);
+
+    new AppFlowIngestionStage(this, "Stage", {
+      ...props,
+    });
+  }
+}
+
+const app = new cdk.App();
+new integration.IntegTest(app, "Glue Transform Stage Integration Tests", {
+    testCases: [
+      new AppFlowIngestionStageTestStack(app, "GlueTransformBasic", {
+        flowName: "dummy-appflow-flow",
+      }),
+    ],
+    diffAssets: true,
+    stackUpdateWorkflow: true,
+    cdkCommandOptions: {
+      deploy: {
+        args: {
+          requireApproval: RequireApproval.NEVER,
+          json: true,
+        },
+      },
+      destroy: {
+        args: {
+          force: true,
+        },
+      },
+    },
+});
diff --git a/integ/integ.athena-sql.test.ts b/integ/integ.athena-sql.test.ts
new file mode 100644
index 00000000..121340b9
--- /dev/null
+++ b/integ/integ.athena-sql.test.ts
@@ -0,0 +1,55 @@
+import * as integration from "@aws-cdk/integ-tests-alpha";
+import * as cdk from "aws-cdk-lib";
+import * as kms from "aws-cdk-lib/aws-kms";
+import * as s3 from "aws-cdk-lib/aws-s3";
+import * as tasks from "aws-cdk-lib/aws-stepfunctions-tasks";
+import { Construct } from "constructs";
+import { RequireApproval } from "aws-cdk-lib/cloud-assembly-schema";
+
+import { AthenaSQLStage } from "../src";
+import { StateMachineStageProps } from "../src/pipelines/stage";
+
+interface AthenaSqlStageTestStackProps extends StateMachineStageProps {
+  readonly queryString: string;
+  readonly workGroup?: string;
+  readonly catalogName?: string;
+  readonly databaseName?: string;
+  readonly outputLocation?: s3.Location;
+  readonly encryptionOption?: tasks.EncryptionOption;
+  readonly encryptionKey?: kms.Key;
+}
+
+class AthenaSqlStageTestStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: AthenaSqlStageTestStackProps) {
+    super(scope, id, props);
+
+    new AthenaSQLStage(this, "Stage", {
+      ...props,
+    });
+  }
+}
+
+const app = new cdk.App();
+new integration.IntegTest(app, "AthenaSQL Stage Integration Tests", {
+    testCases: [
+      new AthenaSqlStageTestStack(app, "GlueTransformBasic", {
+        queryString: "SELECT 1",
+        workGroup: "primary",
+      }),
+    ],
+    diffAssets: true,
+    stackUpdateWorkflow: true,
+    cdkCommandOptions: {
+      deploy: {
+        args: {
+          requireApproval: RequireApproval.NEVER,
+          json: true,
+        },
+      },
+      destroy: {
+        args: {
+          force: true,
+        },
+      },
+    },
+});
diff --git a/integ/integ.cicd-pipeline.test.ts b/integ/integ.cicd-pipeline.test.ts
new file mode 100644
index 00000000..b9e1dc63
--- /dev/null
+++ b/integ/integ.cicd-pipeline.test.ts
@@ -0,0 +1,65 @@
+import path from "path";
+import * as integration from "@aws-cdk/integ-tests-alpha";
+import * as cdk from "aws-cdk-lib";
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import * as s3 from "aws-cdk-lib/aws-s3";
+import * as glue_alpha from "@aws-cdk/aws-glue-alpha";
+import { Construct } from "constructs";
+import { RequireApproval } from "aws-cdk-lib/cloud-assembly-schema";
+
+import { CICDPipelineStack, DataPipeline, FirehoseToS3Stage, SqsToLambdaStage } from "../src";
+
+interface CICDPipelineTestStackProps extends cdk.StackProps {}
+
+class CICDPipelineTestStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: CICDPipelineTestStackProps) {
+    super(scope, id, props);
+    const devStage = new cdk.Stage(this, "dev", { env: { account: "228197580683" } });
+    const devStack = new cdk.Stack(devStage, "application-stack");
+
+    const bucket = new s3.Bucket(devStack, "Bucket");
+    const firehoseToS3Stage = new FirehoseToS3Stage(devStack, "Firehose To S3 Stage", { s3Bucket: bucket });
+
+    const sqsToLambdaStage = new SqsToLambdaStage(devStack, "SQS To Lambda Stage", {
+      lambdaFunctionProps: {
+        code: lambda.Code.fromInline("def lambda_handler(event, context): return 200"),
+        handler: "lambda_function.lambda_handler",
+        memorySize: 512,
+        runtime: lambda.Runtime.PYTHON_3_9,
+      },
+    });
+
+    const pipeline = new DataPipeline(devStack, "Pipeline", {});
+
+    pipeline.addStage({ stage: firehoseToS3Stage }).addStage({ stage: sqsToLambdaStage });
+
+    new CICDPipelineStack(this, "dummy-pipeline", { environmentId: "dev", pipelineName: "dummy-pipeline" })
+      .addSourceAction({ repositoryName: "dummy-repository" })
+      .addSynthAction()
+      .buildPipeline()
+      .addStage({ stageId: "dev", stage: devStage })
+      .synth();
+    }
+}
+
+const app = new cdk.App();
+new integration.IntegTest(app, "CICD Pipeline Stack Integration Tests", {
+    testCases: [
+      new CICDPipelineTestStack(app, "CICDPipeline", {}),
+    ],
+    diffAssets: true,
+    stackUpdateWorkflow: true,
+    cdkCommandOptions: {
+      deploy: {
+        args: {
+          requireApproval: RequireApproval.NEVER,
+          json: true,
+        },
+      },
+      destroy: {
+        args: {
+          force: true,
+        },
+      },
+    },
+});
diff --git a/integ/integ.data-pipelines.test.ts b/integ/integ.data-pipelines.test.ts
new file mode 100644
index 00000000..5429d98d
--- /dev/null
+++ b/integ/integ.data-pipelines.test.ts
@@ -0,0 +1,79 @@
+import path from "path";
+import * as integration from "@aws-cdk/integ-tests-alpha";
+import * as cdk from "aws-cdk-lib";
+import * as events from "aws-cdk-lib/aws-events";
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import * as s3 from "aws-cdk-lib/aws-s3";
+import * as glue_alpha from "@aws-cdk/aws-glue-alpha";
+import { Construct } from "constructs";
+import { RequireApproval } from "aws-cdk-lib/cloud-assembly-schema";
+
+import { DataPipeline, GlueTransformStage, FirehoseToS3Stage, SqsToLambdaStage } from "../src";
+
+interface DataPipelineTestStackProps extends cdk.StackProps {
+  readonly testWithScheduledEvent?: boolean;
+}
+
+class DataPipelineTestStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: DataPipelineTestStackProps) {
+    super(scope, id, props);
+    const bucket = new s3.Bucket(this, "Bucket");
+
+    const firehoseToS3Stage = new FirehoseToS3Stage(this, "Firehose To S3 Stage", { s3Bucket: bucket });
+
+    const sqsToLambdaStage = new SqsToLambdaStage(this, "SQS To Lambda Stage 2", {
+      lambdaFunctionProps: {
+        code: lambda.Code.fromInline("def lambda_handler(event, context): return 200"),
+        handler: "lambda_function.lambda_handler",
+        memorySize: 512,
+        runtime: lambda.Runtime.PYTHON_3_9,
+      },
+    });
+
+    const glueTransformStage = new GlueTransformStage(this, "Glue Transform Stage", {
+      jobProps: {
+        executable: glue_alpha.JobExecutable.pythonShell({
+          glueVersion: glue_alpha.GlueVersion.V1_0,
+          pythonVersion: glue_alpha.PythonVersion.THREE,
+          script: glue_alpha.Code.fromAsset(path.join(__dirname, "/src/glue_script.py")),
+        })
+      },
+      crawlerName: "dummy-crawler",
+    })
+
+    const pipeline = new DataPipeline(this, "Pipeline", {});
+
+    if (props.testWithScheduledEvent) {
+      pipeline.addNotifications().addStage({ stage: firehoseToS3Stage }).addStage({ stage: sqsToLambdaStage }).addStage({ stage: glueTransformStage, schedule: events.Schedule.rate(cdk.Duration.minutes(5))});
+    }
+    else {
+      pipeline.addNotifications().addStage({ stage: firehoseToS3Stage }).addStage({ stage: sqsToLambdaStage }).addStage({ stage: glueTransformStage });
+    }
+
+  }
+}
+
+const app = new cdk.App();
+new integration.IntegTest(app, "Sqs Lambda Stage Integration Tests", {
+    testCases: [
+      new DataPipelineTestStack(app, "DataPipelineBasic", {}),
+      new DataPipelineTestStack(app, "DataPipelineScheduleEvent", {
+        testWithScheduledEvent: true,
+      }),
+    ],
+    diffAssets: true,
+    stackUpdateWorkflow: true,
+    cdkCommandOptions: {
+      deploy: {
+        args: {
+          requireApproval: RequireApproval.NEVER,
+          json: true,
+        },
+      },
+      destroy: {
+        args: {
+          force: true,
+        },
+      },
+    },
+});
diff --git a/integ/integ.databrew-transform.test.ts b/integ/integ.databrew-transform.test.ts
new file mode 100644
index 00000000..a154f63a
--- /dev/null
+++ b/integ/integ.databrew-transform.test.ts
@@ -0,0 +1,52 @@
+import * as integration from "@aws-cdk/integ-tests-alpha";
+import * as cdk from "aws-cdk-lib";
+import * as databrew from "aws-cdk-lib/aws-databrew";
+import { Construct } from "constructs";
+import { RequireApproval } from "aws-cdk-lib/cloud-assembly-schema";
+
+import { DataBrewTransformStage } from "../src";
+import { StateMachineStageProps } from "../src/pipelines/stage";
+
+interface DataBrewTransformStageTestStackProps extends StateMachineStageProps {
+  readonly jobName?: string;
+  readonly jobRoleArn?: string;
+  readonly jobType?: string;
+  readonly datasetName?: string;
+  readonly recipe?: databrew.CfnJob.RecipeProperty;
+  readonly outputs?: databrew.CfnJob.OutputProperty[];
+  readonly createJob?: boolean;
+}
+
+class DataBrewTransformStageTestStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: DataBrewTransformStageTestStackProps) {
+    super(scope, id, props);
+
+    new DataBrewTransformStage(this, "Stage", {
+      ...props,
+    });
+  }
+}
+
+const app = new cdk.App();
+new integration.IntegTest(app, "DataBrew Transform Stage Integration Tests", {
+    testCases: [
+      new DataBrewTransformStageTestStack(app, "GlueTransformBasic", {
+        jobName: "dummy-job",
+      }),
+    ],
+    diffAssets: true,
+    stackUpdateWorkflow: true,
+    cdkCommandOptions: {
+      deploy: {
+        args: {
+          requireApproval: RequireApproval.NEVER,
+          json: true,
+        },
+      },
+      destroy: {
+        args: {
+          force: true,
+        },
+      },
+    },
+});
diff --git a/integ/integ.glue-transform.test.ts b/integ/integ.glue-transform.test.ts
new file mode 100644
index 00000000..cdb62686
--- /dev/null
+++ b/integ/integ.glue-transform.test.ts
@@ -0,0 +1,59 @@
+import * as integration from "@aws-cdk/integ-tests-alpha";
+import * as cdk from "aws-cdk-lib";
+import path from "path";
+import * as glue_alpha from "@aws-cdk/aws-glue-alpha";
+import * as glue from "aws-cdk-lib/aws-glue";
+import { Construct } from "constructs";
+import { RequireApproval } from "aws-cdk-lib/cloud-assembly-schema";
+
+import { GlueTransformStage } from "../src";
+import { StateMachineStageProps } from "../src/pipelines/stage";
+
+interface GlueTransformStageTestStackProps extends StateMachineStageProps {
+  readonly jobName?: string;
+  readonly jobProps?: glue_alpha.JobProps;
+  readonly jobRunArgs?: { [key: string]: any };
+  readonly crawlerName?: string;
+  readonly crawlerProps?: glue.CfnCrawlerProps;
+}
+
+class GlueTransformStageTestStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: GlueTransformStageTestStackProps) {
+    super(scope, id, props);
+
+    new GlueTransformStage(this, "Stage", {
+      ...props,
+    });
+  }
+}
+
+const app = new cdk.App();
+new integration.IntegTest(app, "Glue Tranform Stage Integration Tests", {
+    testCases: [
+      new GlueTransformStageTestStack(app, "GlueTransformBasic", {
+        jobProps: {
+          executable: glue_alpha.JobExecutable.pythonShell({
+            glueVersion: glue_alpha.GlueVersion.V1_0,
+            pythonVersion: glue_alpha.PythonVersion.THREE,
+            script: glue_alpha.Code.fromAsset(path.join(__dirname, "/src/glue_script.py")),
+          })
+        },
+        crawlerName: "dummy-crawler",
+      }),
+    ],
+    diffAssets: true,
+    stackUpdateWorkflow: true,
+    cdkCommandOptions: {
+      deploy: {
+        args: {
+          requireApproval: RequireApproval.NEVER,
+          json: true,
+        },
+      },
+      destroy: {
+        args: {
+          force: true,
+        },
+      },
+    },
+});
diff --git a/integ/integ.sqs-lambda.test.ts b/integ/integ.sqs-lambda.test.ts
new file mode 100644
index 00000000..432db057
--- /dev/null
+++ b/integ/integ.sqs-lambda.test.ts
@@ -0,0 +1,85 @@
+import * as integration from "@aws-cdk/integ-tests-alpha";
+import * as cdk from "aws-cdk-lib";
+import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
+import * as lambda from "aws-cdk-lib/aws-lambda";
+import * as sqs from "aws-cdk-lib/aws-sqs";
+import { Construct } from "constructs";
+import { RequireApproval } from "aws-cdk-lib/cloud-assembly-schema";
+
+import { SqsToLambdaStage } from "../src";
+
+interface SqsToLambdaStageFunctionProps extends lambda.FunctionProps {
+  readonly errorsAlarmThreshold?: number;
+  readonly errorsEvaluationPeriods?: number;
+  readonly errorsComparisonOperator?: cloudwatch.ComparisonOperator;
+}
+
+interface SqsToLambdaStageTestStackProps extends cdk.StackProps {
+  readonly lambdaFunction?: lambda.IFunction;
+  readonly lambdaFunctionProps?: SqsToLambdaStageFunctionProps;
+  readonly sqsQueue?: sqs.IQueue;
+  readonly sqsQueueProps?: sqs.QueueProps;
+  readonly batchSize?: number;
+  readonly maxBatchingWindow?: cdk.Duration;
+  readonly dlqEnabled?: boolean;
+  readonly maxReceiveCount?: number;
+  readonly messageGroupId?: string;
+}
+
+class SqsToLambdaStageTestStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: SqsToLambdaStageTestStackProps) {
+    super(scope, id, props);
+
+    new SqsToLambdaStage(this, "Stage", {
+      ...props,
+    });
+  }
+}
+
+const app = new cdk.App();
+new integration.IntegTest(app, "Sqs Lambda Stage Integration Tests", {
+    testCases: [
+      new SqsToLambdaStageTestStack(app, "SqsLambdaBasic", {
+        lambdaFunctionProps: {
+          code: lambda.Code.fromInline("def lambda_handler(event, context): return 200"),
+          handler: "lambda_function.lambda_handler",
+          memorySize: 512,
+          runtime: lambda.Runtime.PYTHON_3_9,
+        },
+      }),
+      new SqsToLambdaStageTestStack(app, "SqsLambdaFull", {
+        lambdaFunctionProps: {
+          code: lambda.Code.fromInline("def lambda_handler(event, context): return 200"),
+          handler: "lambda_function.lambda_handler",
+          memorySize: 512,
+          runtime: lambda.Runtime.PYTHON_3_9,
+          errorsAlarmThreshold: 10,
+          errorsEvaluationPeriods: 3,
+          errorsComparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+        },
+        batchSize: 5,
+        messageGroupId: "dummy-group",
+        maxReceiveCount: 2,
+        dlqEnabled: true,
+        sqsQueueProps: {
+          fifo: true,
+          visibilityTimeout: cdk.Duration.minutes(5),
+        },
+      }),
+    ],
+    diffAssets: true,
+    stackUpdateWorkflow: true,
+    cdkCommandOptions: {
+      deploy: {
+        args: {
+          requireApproval: RequireApproval.NEVER,
+          json: true,
+        },
+      },
+      destroy: {
+        args: {
+          force: true,
+        },
+      },
+    },
+});