Repository: ChristopheBougere/asl-validator
Pull request #116 (2023-01-05 03:23:21)

Isue115/inline and distributed maps
Added two test cases for the new map features.

The schema types could be better. 

massfords: @ChristopheBougere Sorry for the failing build. I think it's a commit message issue. I made some quick changes to the schemas to support the new types

I haven't read through all of the new Map features yet. The examples I've added pass, but the map schema type still needs some work to avoid accepting definitions with both new and old properties.
massfords: I'll have some time this weekend to clean up the PR and address the comments.


massfords: sorry, didn't mean to close the PR :(
massfords: @ChristopheBougere I've made a couple of changes to address the comments. 

Please do squash my commits in the merge. I didn't realize how out of date my branch was.

Patch:
diff --git a/src/checks/json-schema-errors.ts b/src/checks/json-schema-errors.ts
index de695ed..39c321a 100644
--- a/src/checks/json-schema-errors.ts
+++ b/src/checks/json-schema-errors.ts
@@ -4,6 +4,7 @@ import choice from '../schemas/choice.json';
 import fail from '../schemas/fail.json';
 import parallel from '../schemas/parallel.json';
 import pass from '../schemas/pass.json';
+import baseStateMachine from '../schemas/base-state-machine.json';
 import stateMachine from '../schemas/state-machine.json';
 import state from '../schemas/state.json';
 import succeed from '../schemas/succeed.json';
@@ -11,7 +12,7 @@ import task from '../schemas/task.json';
 import wait from '../schemas/wait.json';
 import map from '../schemas/map.json';
 import errors from '../schemas/errors.json';
-import {AslChecker, StateMachineErrorCode} from '../types';
+import {AslChecker, StateMachineError, StateMachineErrorCode} from '../types';
 import {registerAll} from "asl-path-validator";
 import {isArnFormatValid} from "./formats";
 
@@ -23,6 +24,7 @@ export const jsonSchemaErrors: AslChecker = (definition, options) => {
             fail,
             parallel,
             pass,
+            baseStateMachine,
             stateMachine,
             state,
             succeed,
@@ -95,5 +97,14 @@ export const jsonSchemaErrors: AslChecker = (definition, options) => {
                 instancePath: error.instancePath,
                 schemaPath: decodeURIComponent(error.schemaPath)
             }
-        }));
+        }))
+        // avoid returning a list of errors with duplicates
+        // this filter will only return items if they don't already appear in an earlier position in the array
+        .filter((v: StateMachineError, i, a) => {
+            return a.findIndex(v2 => {
+                // duplicates are based on the schemaPath and instancePath
+                return v2.schemaError.schemaPath === v.schemaError?.schemaPath &&
+                v2.schemaError.instancePath === v.schemaError?.instancePath
+            }) === i
+        });
 }
diff --git a/src/schemas/base-state-machine.json b/src/schemas/base-state-machine.json
new file mode 100644
index 0000000..890a6cc
--- /dev/null
+++ b/src/schemas/base-state-machine.json
@@ -0,0 +1,23 @@
+{
+  "$id": "http://asl-validator.cloud/base-state-machine.json#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "Comment": {
+      "type": "string"
+    },
+    "StartAt": {
+      "type": "string"
+    },
+    "States": {
+      "type": "object",
+      "patternProperties": {
+        "^.{1,80}$": {
+          "$ref": "state.json#"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "required": ["StartAt", "States"]
+}
diff --git a/src/schemas/map.json b/src/schemas/map.json
index b839e95..4667ac3 100644
--- a/src/schemas/map.json
+++ b/src/schemas/map.json
@@ -28,19 +28,90 @@
     "ItemsPath": {
       "$ref": "paths.json#/definitions/asl_ref_path"
     },
+    "Label": {
+      "type": "string"
+    },
     "MaxConcurrency": {
       "type": "number",
       "minimum": 0
     },
+    "ItemReader": {
+      "$comment": "Allow any object for now and update in a subsequent PR to support S3 buckets and other input sources",
+      "type": "object"
+    },
+    "ItemProcessor": {
+      "type": "object",
+      "allOf": [{ "$ref": "base-state-machine.json#" }],
+      "oneOf": [
+        {
+          "properties": {
+            "ProcessorConfig": {
+              "type": "object",
+              "properties": {
+                "Mode": {
+                  "type": "string",
+                  "enum": ["INLINE"]
+                }
+              }
+            }
+          }
+        },
+        {
+          "properties": {
+            "ProcessorConfig": {
+              "type": "object",
+              "properties": {
+                "Mode": {
+                  "type": "string",
+                  "enum": ["DISTRIBUTED"]
+                },
+                "ExecutionType": {
+                  "type": "string",
+                  "enum": ["EXPRESS", "STANDARD"]
+                }
+              }
+            }
+          }
+        }
+      ]
+    },
     "Iterator": {
-      "$ref": "state-machine.json#"
+      "$ref": "base-state-machine.json#"
     },
     "Parameters": {
       "$ref": "paths.json#/definitions/asl_payload_template"
     },
+    "ItemSelector": {
+      "$ref": "paths.json#/definitions/asl_payload_template"
+    },
+    "ItemBatcher":{
+      "type": "object",
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "MaxItemsPerBatch": {
+              "type": "number"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "properties": {
+            "MaxItemsPerBatchPath": {
+              "$ref": "paths.json#/definitions/asl_ref_path"
+            }
+          }
+        }
+      ]
+    },
     "ResultSelector": {
       "$ref": "paths.json#/definitions/asl_payload_template"
     },
+    "ResultWriter": {
+      "$comment": "Allow any object for now and update in a subsequent PR to support S3 buckets and other outputs",
+      "type": "object"
+    },
     "Retry": {
       "type": "array",
       "items": {
@@ -87,11 +158,32 @@
       }
     }
   },
-  "oneOf": [{
-    "required": ["Next"]
-  }, {
-    "required": ["End"]
-  }],
-  "required": ["Type", "Iterator"],
+  "oneOf": [
+    {
+      "required": [
+        "Next",
+        "ItemProcessor"
+      ]
+    },
+    {
+      "required": [
+        "Next",
+        "Iterator"
+      ]
+    },
+    {
+      "required": [
+        "End",
+        "Iterator"
+      ]
+    },
+    {
+      "required": [
+        "End",
+        "ItemProcessor"
+      ]
+    }
+  ],
+  "required": ["Type"],
   "additionalProperties": false
 }


Test patch:
diff --git a/src/__tests__/definitions/valid-map-distributed.asl.json b/src/__tests__/definitions/valid-map-distributed.asl.json
new file mode 100644
index 0000000..126952a
--- /dev/null
+++ b/src/__tests__/definitions/valid-map-distributed.asl.json
@@ -0,0 +1,48 @@
+{
+  "Comment": "Using Map state in Distributed mode",
+  "StartAt": "Map",
+  "States": {
+    "Map": {
+      "Type": "Map",
+      "ItemReader": {
+        "ReaderConfig": {
+          "InputType": "CSV",
+          "CSVHeaderLocation": "FIRST_ROW"
+        },
+        "Resource": "arn:aws:states:::s3:getObject",
+        "Parameters": {
+          "Bucket": "Database",
+          "Key": "csv-dataset/ratings.csv"
+        }
+      },
+      "ItemProcessor": {
+        "ProcessorConfig": {
+          "Mode": "DISTRIBUTED",
+          "ExecutionType": "EXPRESS"
+        },
+        "StartAt": "LambdaTask",
+        "States": {
+          "LambdaTask": {
+            "Type": "Task",
+            "Resource": "arn:aws:states:::lambda:invoke",
+            "OutputPath": "$.Payload",
+            "Parameters": {
+              "Payload.$": "$",
+              "FunctionName": "arn:aws:lambda:us-east-2:123456789012:function:processCSVData"
+            },
+            "End": true
+          }
+        }
+      },
+      "Label": "Map",
+      "End": true,
+      "ResultWriter": {
+        "Resource": "arn:aws:states:::s3:putObject",
+        "Parameters": {
+          "Bucket": "myOutputBucket",
+          "Prefix": "csvProcessJobs"
+        }
+      }
+    }
+  }
+}
diff --git a/src/__tests__/definitions/valid-map-inline.asl.json b/src/__tests__/definitions/valid-map-inline.asl.json
new file mode 100644
index 0000000..efa77ad
--- /dev/null
+++ b/src/__tests__/definitions/valid-map-inline.asl.json
@@ -0,0 +1,41 @@
+{
+  "Comment": "Using Map state in Inline mode",
+  "StartAt": "Pass",
+  "States": {
+    "Pass": {
+      "Type": "Pass",
+      "Next": "Map demo",
+      "Result": {
+        "foo": "bar",
+        "colors": [
+          "red",
+          "green",
+          "blue",
+          "yellow",
+          "white"
+        ]
+      }
+    },
+    "Map demo": {
+      "Type": "Map",
+      "ItemsPath": "$.colors",
+      "ItemProcessor": {
+        "ProcessorConfig": {
+          "Mode": "INLINE"
+        },
+        "StartAt": "Generate UUID",
+        "States": {
+          "Generate UUID": {
+            "Type": "Pass",
+            "End": true,
+            "Parameters": {
+              "uuid.$": "States.UUID()"
+            },
+            "OutputPath": "$.uuid"
+          }
+        }
+      },
+      "End": true
+    }
+  }
+}