Repository: alexcasalboni/aws-lambda-power-tuning
Pull request #86 (2020-05-18 15:08:58)

Add new test to cover count(payloads) > num case
Cover a new case encountered in #85 

TODO: 
- [x] Add more tests
- [x] Improve error message
- [x] Update documentation
- [x] Log payload in case of invocation error


Patch:
diff --git a/README-SAR.md b/README-SAR.md
index caf70b9..37d1d5a 100644
--- a/README-SAR.md
+++ b/README-SAR.md
@@ -74,6 +74,8 @@ If you don't configure any alias name, the state machine will only update the `$
 
 Weighted payloads can be used in scenarios where the payload structure and the corresponding performance/speed can vary a lot in production and you'd like to include multiple payloads in the tuning process.
 
+You may want to use weighted payloads also in case of functions with side effects that would be hard or impossible to test with the very same payload (for example, a function that deletes records from a database).
+
 You can use weighted payloads as follows in the execution input:
 
 ```json
@@ -93,6 +95,8 @@ For example, if `num=100` the first payload will be used 10 times, the second 30
 
 To simplify these calculations, you could use weights that sum up to 100.
 
+Note: the number of weighted payloads must always be smaller or equal than `num` (or `num >= count(payloads)`). For example, if you have 50 weighted payloads, you'll need to set at least `num: 50` so that each payload will be used at least once.
+
 ## State Machine Output
 
 The state machine will return the following output:
diff --git a/README.md b/README.md
index 7c91bcf..748c19c 100644
--- a/README.md
+++ b/README.md
@@ -181,6 +181,8 @@ If you don't configure any alias name, the state machine will only update the `$
 
 Weighted payloads can be used in scenarios where the payload structure and the corresponding performance/speed can vary a lot in production and you'd like to include multiple payloads in the tuning process.
 
+You may want to use weighted payloads also in case of functions with side effects that would be hard or impossible to test with the very same payload (for example, a function that deletes records from a database).
+
 You can use weighted payloads as follows in the execution input:
 
 ```json
@@ -200,6 +202,8 @@ For example, if `num=100` the first payload will be used 10 times, the second 30
 
 To simplify these calculations, you could use weights that sum up to 100.
 
+Note: the number of weighted payloads must always be smaller or equal than `num` (or `num >= count(payloads)`). For example, if you have 50 weighted payloads, you'll need to set at least `num: 50` so that each payload will be used at least once.
+
 
 ## State Machine Output
 
diff --git a/lambda/executor.js b/lambda/executor.js
index 4314a7e..4270d2e 100644
--- a/lambda/executor.js
+++ b/lambda/executor.js
@@ -70,6 +70,10 @@ const generatePayloads = (num, payloadInput) => {
             throw new Error('Invalid weighted payload structure');
         }
 
+        if (num < payloadInput.length) {
+            throw new Error(`You have ${payloadInput.length} payloads and only "num"=${num}. Please increase "num".`);
+        }
+
         // we use relative weights (not %), so here we compute the total weight
         const total = payloadInput.map(p => p.weight).reduce((a, b) => a + b, 0);
 
@@ -113,7 +117,7 @@ const runInParallel = async(num, lambdaARN, lambdaAlias, payloads) => {
         const data = await utils.invokeLambda(lambdaARN, lambdaAlias, payloads[i]);
         // invocation errors return 200 and contain FunctionError and Payload
         if (data.FunctionError) {
-            throw new Error('Invocation error: ' + data.Payload);
+            throw new Error(`Invocation error (running in parallel): ${data.Payload} with payload ${payloads[i]}`);
         }
         results.push(data);
     });
@@ -129,7 +133,7 @@ const runInSeries = async(num, lambdaARN, lambdaAlias, payloads) => {
         const data = await utils.invokeLambda(lambdaARN, lambdaAlias, payloads[i]);
         // invocation errors return 200 and contain FunctionError and Payload
         if (data.FunctionError) {
-            throw new Error('Invocation error: ' + data.Payload);
+            throw new Error(`Invocation error (running in series): ${data.Payload} with payload ${payloads[i]}`);
         }
         results.push(data);
     }


Test patch:
diff --git a/test/test-lambda.js b/test/test-lambda.js
index 624340b..cc5dd8e 100644
--- a/test/test-lambda.js
+++ b/test/test-lambda.js
@@ -470,6 +470,31 @@ describe('Lambda Functions', async() => {
             expect(counters.C).to.be(60);
         });
 
+        it('should explode if count(payloads) < num', async() => {
+            const weightedPayload = [
+                { payload: {test: 'A'}, weight: 5 },
+                { payload: {test: 'B'}, weight: 15 },
+                { payload: {test: 'C'}, weight: 30 },
+                { payload: {test: 'D'}, weight: 5 },
+                { payload: {test: 'E'}, weight: 15 },
+                { payload: {test: 'F'}, weight: 30 },
+                { payload: {test: 'G'}, weight: 30 },
+                { payload: {test: 'H'}, weight: 5 },
+                { payload: {test: 'I'}, weight: 15 },
+                { payload: {test: 'J'}, weight: 30 },
+            ];
+
+            expect(weightedPayload.length).to.be(10);
+
+            await invokeForFailure(handler, {
+                lambdaARN: 'arnOK',
+                value: '128',
+                num: 9, // num is too low here (# of payloads - 1)
+                payload: weightedPayload,
+            });
+
+        });
+
         it('should invoke the given cb, when done (not enough weight)', async() => {
             const weightedPayload = [
                 { payload: {test: 'A'}, weight: 10 },
@@ -542,7 +567,87 @@ describe('Lambda Functions', async() => {
                 value: '1024',
                 num: 10,
             });
+        });
+
+        it('should include payload in exception message if invocation fails (series)', async() => {
+            utils.invokeLambda = async() => {
+                return {
+                    FunctionError: 'Unhandled',
+                    Payload: '{"errorType": "MemoryError", "stackTrace": [["/var/task/lambda_function.py", 11, "lambda_handler", "blabla"], ["/var/task/lambda_function.py", 7, "blabla]]}',
+                };
+            };
+            const error = await invokeForFailure(handler, {
+                lambdaARN: 'arnOK',
+                value: '1024',
+                num: 10,
+                payload: 'SENTINEL',
+            });
+
+            expect(error.message).to.contain('SENTINEL');
+            expect(error.message).to.contain('in series');
+        });
+
+        it('should include payload in exception message if invocation fails (parallel)', async() => {
+            utils.invokeLambda = async() => {
+                return {
+                    FunctionError: 'Unhandled',
+                    Payload: '{"errorType": "MemoryError", "stackTrace": [["/var/task/lambda_function.py", 11, "lambda_handler", "blabla"], ["/var/task/lambda_function.py", 7, "blabla]]}',
+                };
+            };
+            const error = await invokeForFailure(handler, {
+                lambdaARN: 'arnOK',
+                value: '1024',
+                num: 10,
+                parallelInvocation: true,
+                payload: 'SENTINEL',
+            });
+
+            expect(error.message).to.contain('SENTINEL');
+            expect(error.message).to.contain('in parallel');
+        });
+
+
+        it('should include weighted payload in exception message if invocation fails (series)', async() => {
+            utils.invokeLambda = async() => {
+                return {
+                    FunctionError: 'Unhandled',
+                    Payload: '{"errorType": "MemoryError", "stackTrace": [["/var/task/lambda_function.py", 11, "lambda_handler", "blabla"], ["/var/task/lambda_function.py", 7, "blabla]]}',
+                };
+            };
+            const error = await invokeForFailure(handler, {
+                lambdaARN: 'arnOK',
+                value: '1024',
+                num: 10,
+                payload: [
+                    {payload: {key: 'SENTINEL1'}, weight: 1},
+                    {payload: {key: 'SENTINEL2'}, weight: 1},
+                ],
+            });
+
+            expect(error.message).to.contain('SENTINEL1');
+            expect(error.message).to.contain('in series');
+        });
+
+        it('should include weighted payload in exception message if invocation fails (parallel)', async() => {
+            utils.invokeLambda = async() => {
+                return {
+                    FunctionError: 'Unhandled',
+                    Payload: '{"errorType": "MemoryError", "stackTrace": [["/var/task/lambda_function.py", 11, "lambda_handler", "blabla"], ["/var/task/lambda_function.py", 7, "blabla]]}',
+                };
+            };
+            const error = await invokeForFailure(handler, {
+                lambdaARN: 'arnOK',
+                value: '1024',
+                num: 10,
+                parallelInvocation: true,
+                payload: [
+                    {payload: 'SENTINEL1', weight: 1},
+                    {payload: 'SENTINEL2', weight: 1},
+                ],
+            });
 
+            expect(error.message).to.contain('SENTINEL1');
+            expect(error.message).to.contain('in parallel');
         });
 
         it('should report an error if invocation fails (parallel)', async() => {