Repository: serverless/dashboard (language: python)

Issue:
feat: Add attributes and resource to req/res
## Description
We are hoping to add req/res data to the dev mode time line in serverless console but right now the req/res payloads do not contain enough information to surface/filter these in the dev mode console. This PR adds the attribute/resource data to the req/res payloads.


Discussion / hints:
@medikoo yeah I had run them on the last commit and I reran them just now to double check 👍

Fix patch:
diff --git a/node/packages/aws-lambda-otel-extension/external/otel-extension-external/helper.js b/node/packages/aws-lambda-otel-extension/external/otel-extension-external/helper.js
index 62ccb2446..bd9f67c06 100644
--- a/node/packages/aws-lambda-otel-extension/external/otel-extension-external/helper.js
+++ b/node/packages/aws-lambda-otel-extension/external/otel-extension-external/helper.js
@@ -1,5 +1,8 @@
 'use strict';
 
+const http = require('http');
+const https = require('https');
+
 const isObject = (value) => value && typeof value === 'object';
 
 const extensionVersion = (() => {
@@ -246,4 +249,8 @@ module.exports = {
   resourceAttributes,
   measureAttributes,
   stripResponseBlobData,
+  keepAliveAgents: {
+    http: new http.Agent({ keepAlive: true }),
+    https: new https.Agent({ keepAlive: true }),
+  },
 };
diff --git a/node/packages/aws-lambda-otel-extension/external/otel-extension-external/index.js b/node/packages/aws-lambda-otel-extension/external/otel-extension-external/index.js
index 37001de78..30b1020f7 100644
--- a/node/packages/aws-lambda-otel-extension/external/otel-extension-external/index.js
+++ b/node/packages/aws-lambda-otel-extension/external/otel-extension-external/index.js
@@ -20,7 +20,10 @@ module.exports = (async () => {
   const servers = new Set();
 
   const userSettings = require('./user-settings');
-  const { debugLog } = require('./helper');
+  const {
+    debugLog,
+    keepAliveAgents: { http: keepAliveAgent },
+  } = require('./helper');
   const reportOtelData = require('./report-otel-data');
   const {
     createMetricsPayload,
@@ -168,6 +171,7 @@ module.exports = (async () => {
       const request = http.request(
         `http://${process.env.AWS_LAMBDA_RUNTIME_API}/2020-08-15/logs`,
         {
+          agent: keepAliveAgent,
           method: 'PUT',
           headers: {
             'Content-Type': 'application/json',
@@ -212,6 +216,7 @@ module.exports = (async () => {
         const request = http.request(
           `${baseUrl}/event/next`,
           {
+            agent: keepAliveAgent,
             method: 'GET',
             headers: {
               'Content-Type': 'application/json',
@@ -286,16 +291,29 @@ module.exports = (async () => {
                       createLogPayload(data.record, currentRequestContext.logsQueue)
                     );
                   }
+                  if (currentRequestContext.responseEventPayload) {
+                    sendReport(
+                      'response',
+                      createResponsePayload(currentRequestContext.responseEventPayload, data.record)
+                    );
+                  }
                 }
                 break;
               case 'telemetryData':
                 lastTelemetryData = data;
                 if (data.record.responseEventPayload) {
-                  const { requestData } = getCurrentRequestContext();
-                  sendReport(
-                    'response',
-                    createResponsePayload(data.record.responseEventPayload, requestData)
-                  );
+                  const currentRequestContext = getCurrentRequestContext();
+                  if (!currentRequestContext.requestData) {
+                    currentRequestContext.responseEventPayload = data.record.responseEventPayload;
+                  } else {
+                    sendReport(
+                      'response',
+                      createResponsePayload(
+                        data.record.responseEventPayload,
+                        currentRequestContext.requestData
+                      )
+                    );
+                  }
                 }
                 sendReport('metrics', createMetricsPayload(data.requestId, data.record.function));
                 for (const tracePayload of createTracePayload(
@@ -321,6 +339,7 @@ module.exports = (async () => {
     const request = http.request(
       `${baseUrl}/register`,
       {
+        agent: keepAliveAgent,
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
diff --git a/node/packages/aws-lambda-otel-extension/external/otel-extension-external/report-otel-data.js b/node/packages/aws-lambda-otel-extension/external/otel-extension-external/report-otel-data.js
index a71adbd94..05e1c6288 100644
--- a/node/packages/aws-lambda-otel-extension/external/otel-extension-external/report-otel-data.js
+++ b/node/packages/aws-lambda-otel-extension/external/otel-extension-external/report-otel-data.js
@@ -2,7 +2,7 @@
 
 const createHttpRequest = require('http').request;
 const createHttpsRequest = require('https').request;
-const { debugLog } = require('./helper');
+const { debugLog, keepAliveAgents } = require('./helper');
 const userSettings = require('./user-settings');
 
 const isUrl = RegExp.prototype.test.bind(/^https?:\/\//);
@@ -39,32 +39,37 @@ const sendReport = async (data, { destination, outputType }) => {
   };
   await new Promise((resolve, reject) => {
     const httpRequestId = ++httpRequestIdTracker;
-    const createRequest = destination.startsWith('https') ? createHttpsRequest : createHttpRequest;
+    const isHttps = destination.startsWith('https');
+    const createRequest = isHttps ? createHttpsRequest : createHttpRequest;
     debugLog(`Request [${httpRequestId}]:`, destination);
     const requestStartTime = process.hrtime.bigint();
-    const request = createRequest(destination, { method: 'post', headers }, (response) => {
-      if (response.statusCode === 200) {
-        debugLog(
-          `Request [${httpRequestId}]: ok in: ${Math.round(
-            Number(process.hrtime.bigint() - requestStartTime) / 1000000
-          )}ms`
-        );
-        resolve();
-        return;
+    const request = createRequest(
+      destination,
+      { agent: isHttps ? keepAliveAgents.https : keepAliveAgents.http, method: 'post', headers },
+      (response) => {
+        if (response.statusCode === 200) {
+          debugLog(
+            `Request [${httpRequestId}]: ok in: ${Math.round(
+              Number(process.hrtime.bigint() - requestStartTime) / 1000000
+            )}ms`
+          );
+          resolve();
+          return;
+        }
+        let responseText = '';
+        response.on('data', (chunk) => {
+          responseText += String(chunk);
+        });
+        response.on('end', () => {
+          debugLog(
+            `Request [${httpRequestId}]: failed in: ${Math.round(
+              Number(process.hrtime.bigint() - requestStartTime) / 1000000
+            )}ms with: [${response.status}] ${responseText}`
+          );
+          resolve();
+        });
       }
-      let responseText = '';
-      response.on('data', (chunk) => {
-        responseText += String(chunk);
-      });
-      response.on('end', () => {
-        debugLog(
-          `Request [${httpRequestId}]: failed in: ${Math.round(
-            Number(process.hrtime.bigint() - requestStartTime) / 1000000
-          )}ms with: [${response.status}] ${responseText}`
-        );
-        resolve();
-      });
-    });
+    );
     request.on('error', reject);
     request.write(body);
     request.end();
diff --git a/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/index.js b/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/index.js
index 4efb57b00..5feb9b460 100644
--- a/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/index.js
+++ b/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/index.js
@@ -9,6 +9,7 @@ const debugLog = (...args) => {
 };
 debugLog('Internal extension: Init');
 
+const http = require('http');
 const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
 const { InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base');
 const { registerInstrumentations } = require('@opentelemetry/instrumentation');
@@ -32,7 +33,6 @@ const { RedisInstrumentation } = require('@opentelemetry/instrumentation-redis')
 const { FastifyInstrumentation } = require('@opentelemetry/instrumentation-fastify');
 const AwsLambdaInstrumentation = require('./aws-lambda-instrumentation');
 const { diag, DiagConsoleLogger } = require('@opentelemetry/api');
-const fetch = require('node-fetch');
 const SlsSpanProcessor = require('./span-processor');
 const { detectEventType } = require('./event-detection');
 const userSettings = require('./user-settings');
@@ -40,6 +40,8 @@ const userSettings = require('./user-settings');
 const OTEL_SERVER_PORT = 2772;
 const logLevel = getEnv().OTEL_LOG_LEVEL;
 diag.setLogger(new DiagConsoleLogger(), logLevel);
+const telemetryServerUrl = `http://localhost:${OTEL_SERVER_PORT}`;
+const keepAliveAgent = new http.Agent({ keepAlive: true });
 
 const tracerProvider = new NodeTracerProvider();
 const memoryExporter = new InMemorySpanExporter();
@@ -118,20 +120,47 @@ const requestHandler = async (span, { event, context }) => {
     };
   }
 
+  const requestBody = JSON.stringify(eventDataPayload);
+  debugLog('Internal extension: Send event data');
   if (process.env.TEST_DRY_LOG) {
-    process._rawDebug(
-      `${require('util').inspect(eventDataPayload, { depth: Infinity, colors: true })}\n`
-    );
+    process.stdout.write(`⚡ eventData: ${requestBody}\n`);
   } else {
-    debugLog('Internal extension: Send event data');
     // Send request data to external so that we can attach this data to logs
-    await fetch(`http://localhost:${OTEL_SERVER_PORT}`, {
-      method: 'post',
-      body: JSON.stringify(eventDataPayload),
-      headers: {
-        'Content-Type': 'application/json',
-      },
-    });
+    const requestStartTime = process.hrtime.bigint();
+    let requestSocket;
+    try {
+      await new Promise((resolve, reject) => {
+        const request = http.request(
+          telemetryServerUrl,
+          {
+            agent: keepAliveAgent,
+            method: 'post',
+            headers: {
+              'Content-Type': 'application/json',
+              'Content-Length': Buffer.byteLength(requestBody),
+            },
+          },
+          (response) => {
+            if (response.statusCode === 200) {
+              resolve();
+            } else {
+              reject(new Error(`Unexpected response status code: ${response.statusCode}`));
+            }
+          }
+        );
+        request.on('error', reject);
+        request.on('socket', (socket) => (requestSocket = socket));
+        request.write(requestBody);
+        request.end();
+      });
+    } finally {
+      if (requestSocket) requestSocket.unref();
+    }
+    debugLog(
+      `Internal extension request [eventData]: ok in: ${Math.round(
+        Number(process.hrtime.bigint() - requestStartTime) / 1000000
+      )}ms`
+    );
   }
 };
 
@@ -339,19 +368,46 @@ const responseHandler = async (span, { res, err }, isTimeout) => {
     };
   }
 
+  const requestBody = JSON.stringify(telemetryDataPayload);
+  debugLog('Internal extension: Send telemetry data');
   if (process.env.TEST_DRY_LOG) {
-    process._rawDebug(
-      `${require('util').inspect(telemetryDataPayload, { depth: Infinity, colors: true })}\n`
-    );
+    process.stdout.write(`⚡ telemetryData: ${requestBody}\n`);
   } else {
-    debugLog('Internal extension: Send telemetry data');
-    await fetch(`http://localhost:${OTEL_SERVER_PORT}`, {
-      method: 'post',
-      body: JSON.stringify(telemetryDataPayload),
-      headers: {
-        'Content-Type': 'application/json',
-      },
-    });
+    const requestStartTime = process.hrtime.bigint();
+    let requestSocket;
+    try {
+      await new Promise((resolve, reject) => {
+        const request = http.request(
+          telemetryServerUrl,
+          {
+            agent: keepAliveAgent,
+            method: 'post',
+            headers: {
+              'Content-Type': 'application/json',
+              'Content-Length': Buffer.byteLength(requestBody),
+            },
+          },
+          (response) => {
+            if (response.statusCode === 200) {
+              resolve();
+            } else {
+              reject(new Error(`Unexpected response status code: ${response.statusCode}`));
+            }
+          }
+        );
+        request.on('error', reject);
+        request.on('socket', (socket) => (requestSocket = socket));
+        request.write(requestBody);
+        request.end();
+      });
+    } finally {
+      if (requestSocket) requestSocket.unref();
+    }
+    debugLog(
+      `Internal extension request [telemetryData]: ok in: ${Math.round(
+        Number(process.hrtime.bigint() - requestStartTime) / 1000000
+      )}ms`
+    );
   }
 
   // Reset the exporter so we don't see duplicates
diff --git a/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/package.json b/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/package.json
index 849043842..a6401f3cf 100644
--- a/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/package.json
+++ b/node/packages/aws-lambda-otel-extension/internal/otel-extension-internal-node/package.json
@@ -24,7 +24,6 @@
     "@opentelemetry/sdk-trace-base": "^1.3.1",
     "@opentelemetry/sdk-trace-node": "^1.3.1",
     "lodash.clone": "^4.5.0",
-    "lodash.get": "^4.4.2",
-    "node-fetch": "^2.6.7"
+    "lodash.get": "^4.4.2"
   }
 }


Test patch:
diff --git a/node/packages/aws-lambda-otel-extension/test/benchmark/README.md b/node/packages/aws-lambda-otel-extension/test/benchmark/README.md
index dca23cec1..a44ef9cfa 100644
--- a/node/packages/aws-lambda-otel-extension/test/benchmark/README.md
+++ b/node/packages/aws-lambda-otel-extension/test/benchmark/README.md
@@ -43,18 +43,23 @@ Just external extension which processes lambda event cycle. As no internal exten
 
 Difference from `bare` is that external extension process is initialized by AWS, and its that process that monitors and queues lambda invocation event cycle.
 
-This additional process management introduces natural duration overhead which is best calculated by reading duration as reported by AWS for this variant and subtracting from it a `bare` variant duration _(current numbers show ca 100ms overhead for initialization, ca 60ms for first invocation , and ca 30ms for every following invocation)_
+This additional process management introduces natural duration overhead which is best calculated by reading duration as reported by AWS for this variant and subtracting from it a `bare` variant duration
 
 There's no time spent here on processing reports and propagating them to the ingestion server (as none are to be send from the internal extension). So internally measured invocation duration overhead of external extension is also reported as not existing
 
+### `internalOnly`
+
+Just internal extension which generates telemetry reports and logs them to the console (when external extension is loaded those would be send to the external extension)
+
+Difference from `bare` is that original lambda logic is additionally wrapped with internal extension log. Measured difference simply show what overhead is produced by gathering telemetry in context of the lambda handler process.
+
 ### `jsonLog`
 
-External & internal extensions are loaded. Internal extension operates fully, while in external extension from obtained reports just payloads are generated and logged into `stdout` (there's no conversion to Protobuf and not reporting to the remote ingestion server).
+External & internal extensions are loaded. Internal extension operates fully (generates telemetry and sends it to external extension), while in external extension just report payloads are generated and logged into `stdout` (there's no conversion to Protobuf and not reporting to the remote ingestion server).
 
 Difference from `externalOnly` is purely that internal extension is loaded and fully active and that it sends reports to the external extension.
-While external extension prepares the payloads and logs them to the console (in JSON form) overhead of that is negligible (internal overhead measurements are expected to show `0`).
 
-Comparing its duration overhead against `externalOnly` shows how much overhead do internal extension introduce.
+Comparing its duration overhead against `externalOnly` and subtracting overehead of `internalOnly` shows how much overhead is introduced by sending telemetry payloads to the external extension and processing it in its scope
 
 ### `protoLog`
 
@@ -156,7 +161,8 @@ While overhead durations as measured and reported by the extensions internally h
 With benchmark variants, as setup correctly. It's safe to assume that:
 
 - Comparing durations of `externalOnly` vs `bare` expose the overhead of the AWS Lambda event lifecycle handling in the external extension
-- Comparing durations of `jsonLog` vs `externalOnly` expose the overhead of the internal extension
+- Comparing durations of `internalOnly` vs `bare` expose the overhead of the internal extension telemetry generation
+- Comparing durations of `jsonLog` vs `externalOnly` and subtracting overhead as observed with `internalOnly` informs us of overhead introduced by sending telemetry payloads from internal to the external extension and processing it in its scope
 - Comparing durations of `protoLog` vs `jsonLog` expose the overhead of the Protobuf conversion that happens in external extension
 - Comparing durations of `protoConsole` vs `protoBuf` expose the overhead of the communication with ingestion server that happens in external extension
 
@@ -172,6 +178,8 @@ Run `npm install` in following folders:
 
 ### 2. Ensure AWS credentials
 
+_In same manner as for AWS CLI_
+
 ### 3. Configure environment variables
 
 - `AWS_REGION` - region against which benchmarks should be run
@@ -193,7 +201,8 @@ Generated benchmark results are output to the console in CSV format. To store th
 ./test/scripts/benchmark.js > benchmark.csv
 ```
 
-What use cases and benchmark variants are tested can be fine tuned with following CLI params:
+Benchmark run can additionally be configured with following CLI params
 
 - `--use-cases` Function use cases to test (e.g. `callback,express`)
 - `--benchmark-variants` Benchmark variants to test (e.g. `bare,externalOnly`)
+- `--memory-size` Memory size to provide to lambdas (by default it's 128MB).
diff --git a/node/packages/aws-lambda-otel-extension/test/benchmark/index.js b/node/packages/aws-lambda-otel-extension/test/benchmark/index.js
index cc12fd497..c5c9a0d86 100644
--- a/node/packages/aws-lambda-otel-extension/test/benchmark/index.js
+++ b/node/packages/aws-lambda-otel-extension/test/benchmark/index.js
@@ -44,11 +44,16 @@ const resolveIngestionData = async () => {
 };
 
 module.exports = async (options = {}) => {
+  const coreConfig = {};
+  await createCoreResources(coreConfig, { layerTypes: ['nodeAll', 'nodeInternal'] });
+
+  const memorySize = options.memorySize || 128;
   const allBenchmarkVariantsConfig = new Map([
     [
       'bare',
       {
         configuration: {
+          MemorySize: memorySize,
           Layers: [],
           Environment: { Variables: {} },
         },
@@ -58,6 +63,7 @@ module.exports = async (options = {}) => {
       'externalOnly',
       {
         configuration: {
+          MemorySize: memorySize,
           Environment: {
             Variables: {
               SLS_OTEL_USER_SETTINGS: JSON.stringify({ logs: { disabled: true } }),
@@ -67,10 +73,27 @@ module.exports = async (options = {}) => {
         },
       },
     ],
+    [
+      'internalOnly',
+      {
+        configuration: {
+          MemorySize: memorySize,
+          Layers: [coreConfig.layerInternalArn],
+          Environment: {
+            Variables: {
+              AWS_LAMBDA_EXEC_WRAPPER: '/opt/otel-extension-internal-node/exec-wrapper.sh',
+              DEBUG_SLS_OTEL_LAYER: '1',
+              TEST_DRY_LOG: '1',
+            },
+          },
+        },
+      },
+    ],
     [
       'jsonLog',
       {
         configuration: {
+          MemorySize: memorySize,
           Environment: {
             Variables: {
               AWS_LAMBDA_EXEC_WRAPPER: '/opt/otel-extension-internal-node/exec-wrapper.sh',
@@ -89,6 +112,7 @@ module.exports = async (options = {}) => {
       'protoLog',
       {
         configuration: {
+          MemorySize: memorySize,
           Environment: {
             Variables: {
               AWS_LAMBDA_EXEC_WRAPPER: '/opt/otel-extension-internal-node/exec-wrapper.sh',
@@ -107,6 +131,7 @@ module.exports = async (options = {}) => {
   if (token) {
     allBenchmarkVariantsConfig.set('protoConsole', {
       configuration: {
+        MemorySize: memorySize,
         Environment: {
           Variables: {
             AWS_LAMBDA_EXEC_WRAPPER: '/opt/otel-extension-internal-node/exec-wrapper.sh',
@@ -297,8 +322,6 @@ module.exports = async (options = {}) => {
 
   if (!useCasesConfig.size) throw new Error('No matching use case');
 
-  const coreConfig = {};
-  await createCoreResources(coreConfig);
   const testVariantsConfig = resolveTestVariantsConfig(useCasesConfig, { multiplyBy: 5 });
   for (const testConfig of testVariantsConfig) {
     testConfig.deferredResult = processFunction(testConfig, coreConfig).catch((error) => ({
diff --git a/node/packages/aws-lambda-otel-extension/test/lib/cleanup.js b/node/packages/aws-lambda-otel-extension/test/lib/cleanup.js
index 301da8d66..875961526 100644
--- a/node/packages/aws-lambda-otel-extension/test/lib/cleanup.js
+++ b/node/packages/aws-lambda-otel-extension/test/lib/cleanup.js
@@ -31,8 +31,8 @@ const deleteFunctions = async () => {
   );
 };
 
-const deleteLayers = async () => {
-  const layerVersions = (await awsRequest(Lambda, 'listLayerVersions', { LayerName: basename }))
+const deleteLayers = async (layerName) => {
+  const layerVersions = (await awsRequest(Lambda, 'listLayerVersions', { LayerName: layerName }))
     .LayerVersions;
   if (!layerVersions.length) return;
 
@@ -41,15 +41,19 @@ const deleteLayers = async () => {
     Array.from(Array(lastLayerVersion).keys()).map(async (index) => {
       const versionNumber = index + 1;
       await awsRequest(Lambda, 'deleteLayerVersion', {
-        LayerName: basename,
+        LayerName: layerName,
         VersionNumber: versionNumber,
       });
     })
   );
 
-  log.notice('Deleted all versions of layer %s', basename);
+  log.notice('Deleted all versions of layer %s', layerName);
 };
 
+const deleteDefaultLayers = () => deleteLayers(basename);
+const deleteInternalLayers = () => deleteLayers(`${basename}-internal`);
+const deleteAllLayers = () => Promise.all([deleteDefaultLayers(), deleteInternalLayers()]);
+
 const deleteRole = async () => {
   const policyArn = `arn:aws:iam::${
     (await awsRequest(STS, 'getCallerIdentity')).Account
@@ -86,5 +90,5 @@ const deleteRole = async () => {
 module.exports = async (options = {}) => {
   log.notice('Cleanup %s', basename);
   if (!options.skipFunctionsCleanup) await deleteFunctions();
-  await Promise.all([deleteLayers(), deleteRole()]);
+  await Promise.all([deleteAllLayers(), deleteRole()]);
 };
diff --git a/node/packages/aws-lambda-otel-extension/test/lib/create-core-resources.js b/node/packages/aws-lambda-otel-extension/test/lib/create-core-resources.js
index 558e58784..49c21cbe3 100644
--- a/node/packages/aws-lambda-otel-extension/test/lib/create-core-resources.js
+++ b/node/packages/aws-lambda-otel-extension/test/lib/create-core-resources.js
@@ -10,24 +10,55 @@ const buildLayer = require('../../scripts/lib/build');
 const awsRequest = require('../utils/aws-request');
 const basename = require('./basename');
 
-const layerFilename = path.resolve(__dirname, '../../dist/extension.zip');
-
-const createLayer = async (config) => {
-  if (!process.env.TEST_LAYER_FILENAME) {
+const createLayer = async (config, { layerName, filename, skipBuild, buildConfig }) => {
+  if (!skipBuild) {
     log.info('Building layer');
-    await buildLayer(layerFilename);
+    await buildLayer(filename, buildConfig);
   }
 
-  log.info('Publishing layer (%s) to AWS', process.env.TEST_LAYER_FILENAME || layerFilename);
+  log.info('Publishing layer (%s) to AWS', filename);
   await awsRequest(Lambda, 'publishLayerVersion', {
-    LayerName: basename,
-    Content: { ZipFile: await fsp.readFile(process.env.TEST_LAYER_FILENAME || layerFilename) },
+    LayerName: layerName,
+    Content: { ZipFile: await fsp.readFile(filename) },
   });
   log.info('Resolving layer ARN');
-  config.layerArn = (
-    await awsRequest(Lambda, 'listLayerVersions', { LayerName: basename })
+  const arn = (
+    await awsRequest(Lambda, 'listLayerVersions', { LayerName: layerName })
   ).LayerVersions.shift().LayerVersionArn;
-  log.info('Layer ready %s', config.layerArn);
+  log.info('Layer ready %s', arn);
+  return arn;
+};
+
+const createLayers = async (config, layerTypes) => {
+  return Promise.all(
+    Array.from(layerTypes, async (layerType) => {
+      switch (layerType) {
+        case 'nodeAll':
+          if (process.env.TEST_LAYER_FILENAME) {
+            config.layerArn = await createLayer(config, {
+              layerName: basename,
+              filename: process.env.TEST_LAYER_FILENAME,
+              skipBuild: true,
+            });
+            return;
+          }
+          config.layerArn = await createLayer(config, {
+            layerName: basename,
+            filename: path.resolve(__dirname, '../../dist/extension.zip'),
+          });
+          return;
+        case 'nodeInternal':
+          config.layerInternalArn = await createLayer(config, {
+            layerName: `${basename}-internal`,
+            filename: path.resolve(__dirname, '../../dist/extension.internal.zip'),
+            buildConfig: { mode: 1 },
+          });
+          return;
+        default:
+          throw new Error(`Unrecognized layer type: ${layerType}`);
+      }
+    })
+  );
 };
 
 const createRole = async (config) => {
@@ -91,9 +122,9 @@ const createRole = async (config) => {
   config.policyArn = policyArn;
 };
 
-module.exports = async (config) => {
+module.exports = async (config, options = {}) => {
   log.notice('Creating core resources %s', basename);
   config.accountId = (await awsRequest(STS, 'getCallerIdentity')).Account;
 
-  await Promise.all([createLayer(config), createRole(config)]);
+  await Promise.all([createLayers(config, options.layerTypes || ['nodeAll']), createRole(config)]);
 };
diff --git a/node/packages/aws-lambda-otel-extension/test/lib/process-function.js b/node/packages/aws-lambda-otel-extension/test/lib/process-function.js
index 4e985f50f..86d095899 100644
--- a/node/packages/aws-lambda-otel-extension/test/lib/process-function.js
+++ b/node/packages/aws-lambda-otel-extension/test/lib/process-function.js
@@ -27,7 +27,7 @@ const create = async (testConfig, coreConfig) => {
   try {
     await awsRequest(Lambda, 'createFunction', {
       Role: coreConfig.roleArn,
-      Runtime: 'nodejs14.x',
+      Runtime: 'nodejs16.x',
       Code: {
         ZipFile: await resolveDirZipBuffer(fixturesDirname),
       },
@@ -89,6 +89,8 @@ const invoke = async (testConfig) => {
   } catch (error) {
     if (error.message.includes('The role defined for the function cannot be assumed by Lambda')) {
       // Occassional race condition issue on AWS side, retry
+      log.error('Approached error, retying ivocation: %o', error);
+      await wait(100);
       return invoke(testConfig);
     }
     error.message = `"${testConfig.configuration.FunctionName}" invocation failed: ${error.message}`;
@@ -104,7 +106,7 @@ const invoke = async (testConfig) => {
   }
   if (result.FunctionError) {
     if (expectedOutcome.startsWith('error')) return duration;
-    throw new Error(`Invocation errored: ${result.FunctionError}`);
+    throw new Error(`Invocation of ${testConfig.name} errored: ${result.FunctionError}`);
   }
   return duration;
 };
@@ -142,25 +144,25 @@ const retrieveReports = async (testConfig) => {
   let processesData;
   do {
     const events = await retrieveAllEvents();
+
+    let startEventsCount = 0;
+    let reportEventsCount = 0;
+    for (const { message } of events) {
+      if (message.startsWith('START RequestId: ')) ++startEventsCount;
+      else if (message.startsWith('REPORT RequestId: ')) ++reportEventsCount;
+    }
+
     processesData = [];
     invocationsData = [];
-    if (!events.length) continue;
-    if (
-      !events[0].message.startsWith('START RequestId: ') &&
-      !events[0].message.startsWith('Extension: Ready for an event')
-    ) {
-      // Workaound AWS bug, where on some occasion only batch of later events is returned
-      log.error(
-        'Retry, due to unexpected CW events (for start time %d): %o',
-        testConfig.invokeStartTime,
-        events
-      );
+    if (startEventsCount < testConfig.invokeCount || reportEventsCount < testConfig.invokeCount) {
       continue;
     }
+
     log.debug('Events for %s: %o', testConfig.name, events);
     let currentInvocationData;
     let currentProcessData;
     let startedMessage;
+    let isExternalExtensionLoaded = false;
     const getCurrentInvocationData = () => {
       if (!currentInvocationData) {
         log.error(
@@ -174,6 +176,7 @@ const retrieveReports = async (testConfig) => {
     };
     for (const { message } of events) {
       if (message.startsWith('Extension overhead duration: external initialization')) {
+        isExternalExtensionLoaded = true;
         processesData.push(
           (currentProcessData = {
             extensionOverheadDurations: {
@@ -183,6 +186,10 @@ const retrieveReports = async (testConfig) => {
         );
         continue;
       }
+      if (!isExternalExtensionLoaded && message.startsWith('Internal extension: Init')) {
+        processesData.push((currentProcessData = { extensionOverheadDurations: {} }));
+        continue;
+      }
       if (message.startsWith('Extension overhead duration: internal initialization')) {
         currentProcessData.extensionOverheadDurations.internalInit = parseInt(
           message.slice(message.lastIndexOf(':') + 1),
@@ -258,7 +265,7 @@ const retrieveReports = async (testConfig) => {
         currentInvocationData = null;
       }
     }
-  } while (invocationsData.length < 2);
+  } while (invocationsData.length < testConfig.invokeCount);
 
   if (processesData.length !== 1 && handledOutcomes.has(testConfig.expectedOutcome)) {
     throw new Error(
@@ -286,7 +293,16 @@ module.exports = async (testConfig, coreConfig) => {
   do {
     await wait(3000);
     log.info('Invoke function #%d %s', counter, testConfig.name);
-    invokeDurations.push(await invoke(testConfig));
+    try {
+      invokeDurations.push(await invoke(testConfig));
+    } catch (error) {
+      if (error.message.includes('Lambda was unable to decrypt the environment variables')) {
+        // Rare error on AWS side, which we can recover from only by re-creating the lambda
+        log.error('Approached not-recoverable error, re-creating lambda: %o', error);
+        return module.exports(testConfig, coreConfig);
+      }
+      throw error;
+    }
   } while (++counter <= testConfig.invokeCount);
 
   log.info('Delete function %s', testConfig.name);
diff --git a/node/packages/aws-lambda-otel-extension/test/scripts/benchmark.js b/node/packages/aws-lambda-otel-extension/test/scripts/benchmark.js
index 01ed20fcd..0d37fac33 100755
--- a/node/packages/aws-lambda-otel-extension/test/scripts/benchmark.js
+++ b/node/packages/aws-lambda-otel-extension/test/scripts/benchmark.js
@@ -18,6 +18,7 @@ const resolveSet = (comaSeparatedValue) =>
 require('../benchmark')({
   benchmarkVariants: argv['benchmark-variants'] ? resolveSet(argv['benchmark-variants']) : null,
   useCases: argv['use-cases'] ? resolveSet(argv['use-cases']) : null,
+  memorySize: argv['memory-size'] ? Number(argv['memory-size']) : null,
 }).then((resultsMap) => {
   process.stdout.write(
     `${[