Repository: serverless/dashboard (language: python)

Issue:
Replace otel internal Lambda wrapper
It's to fix discovered issues as:
- Broken wrapping when esbuild bundling of ESM modules is involed
- Lack of support for ESM modules



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/integrate.yml b/.github/workflows/integrate.yml
index 34529944d..2736b81d3 100644
--- a/.github/workflows/integrate.yml
+++ b/.github/workflows/integrate.yml
@@ -47,6 +47,13 @@ jobs:
           path: packages/aws-lambda-otel-extension/opt/otel-extension/node_modules
           key: packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('packages/aws-lambda-otel-extension/opt/otel-extension/package.json') }}
           restore-keys: packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v16-${{ runner.os }}-${{ github.ref }}-
+      - name: Retrieve packages/aws-lambda-otel-extension/test/fixtures/lambdas from cache
+        id: cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules
+        uses: actions/cache@v2
+        with:
+          path: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node_modules
+          key: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json') }}
+          restore-keys: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v16-${{ runner.os }}-${{ github.ref }}-
       - name: Install Node.js and npm
         uses: actions/setup-node@v1
         with:
@@ -67,7 +74,11 @@ jobs:
         run: |
           cd packages/aws-lambda-otel-extension/opt/otel-extension
           npm update --no-save
-
+      - name: Install packages/aws-lambda-otel-extension/test/fixtures/lambdas dependencies
+        if: steps.cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules.outputs.cache-hit != 'true'
+        run: |
+          cd packages/aws-lambda-otel-extension/test/fixtures/lambdas
+          npm update --no-save
       - name: Unit tests
         run: |
           cd packages/aws-lambda-otel-extension
@@ -108,7 +119,13 @@ jobs:
           path: packages/aws-lambda-otel-extension/opt/otel-extension/node_modules
           key: packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('packages/aws-lambda-otel-extension/opt/otel-extension/package.json') }}
           restore-keys: packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v14-${{ runner.os }}-${{ github.ref }}-
-
+      - name: Retrieve packages/aws-lambda-otel-extension/test/fixtures/lambdas from cache
+        id: cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules
+        uses: actions/cache@v2
+        with:
+          path: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node_modules
+          key: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json') }}
+          restore-keys: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v14-${{ runner.os }}-${{ github.ref }}-
       - name: Install Node.js and npm
         uses: actions/setup-node@v1
         with:
@@ -129,6 +146,11 @@ jobs:
         run: |
           cd packages/aws-lambda-otel-extension/opt/otel-extension
           npm update --no-save
+      - name: Install packages/aws-lambda-otel-extension/test/fixtures/lambdas dependencies
+        if: steps.cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules.outputs.cache-hit != 'true'
+        run: |
+          cd packages/aws-lambda-otel-extension/test/fixtures/lambdas
+          npm update --no-save
 
       - name: Unit tests
         run: |
diff --git a/.github/workflows/publish-aws-lambda-otel-extension.yml b/.github/workflows/publish-aws-lambda-otel-extension.yml
index b88044140..dffe83e0a 100644
--- a/.github/workflows/publish-aws-lambda-otel-extension.yml
+++ b/.github/workflows/publish-aws-lambda-otel-extension.yml
@@ -42,6 +42,13 @@ jobs:
           path: packages/aws-lambda-otel-extension/opt/otel-extension/node_modules
           key: packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v16-${{ runner.os }}-refs/heads/main-${{ hashFiles('packages/aws-lambda-otel-extension/package.json') }}
           restore-keys: packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v16-${{ runner.os }}-refs/heads/main-
+      - name: Retrieve packages/aws-lambda-otel-extension/test/fixtures/lambdas from cache
+        id: cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules
+        uses: actions/cache@v2
+        with:
+          path: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node_modules
+          key: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v16-${{ runner.os }}-refs/heads/main-${{ hashFiles('packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json') }}
+          restore-keys: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v16-${{ runner.os }}-refs/heads/main-
       - name: Retrieve packages/aws-lambda-otel-extension-dist/node_modules from cache
         id: cacheAwsLambdaOtelExtensionDistNodeModules
         uses: actions/cache@v2
@@ -60,12 +67,26 @@ jobs:
           node-version: 16.x
           registry-url: https://registry.npmjs.org
 
+      - name: Install main project dependencies
+        if: steps.cacheNodeModules.outputs.cache-hit != 'true'
+        run: |
+          npm update --save-dev --no-save
       - name: Install packages/aws-lambda-otel-extension dependencies
         if: steps.cacheAwsLambdaOtelExtensionNodeModules.outputs.cache-hit != 'true'
         run: |
           cd packages/aws-lambda-otel-extension
           npm update --no-save
           npm update --save-dev --no-save
+      - name: Install packages/aws-lambda-otel-extension/opt/otel-extension dependencies
+        if: steps.cacheAwsLambdaOtelExtensionOptOtelExtensionNodeModules.outputs.cache-hit != 'true'
+        run: |
+          cd packages/aws-lambda-otel-extension/opt/otel-extension
+          npm update --no-save
+      - name: Install packages/aws-lambda-otel-extension/test/fixtures/lambdas dependencies
+        if: steps.cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules.outputs.cache-hit != 'true'
+        run: |
+          cd packages/aws-lambda-otel-extension/test/fixtures/lambdas
+          npm update --no-save
       - name: Install packages/aws-lambda-otel-extension-dist dependencies
         if: steps.cacheAwsLambdaOtelExtensionDistNodeModules.outputs.cache-hit != 'true'
         run: |
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index e8aa2d6cc..c65ead02f 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -65,15 +65,15 @@ jobs:
           restore-keys: |
             packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v16-${{ runner.os }}-${{ github.ref }}-
             packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v16-${{ runner.os }}-refs/heads/main-
-      - name: Retrieve packages/aws-lambda-otel-extension-dist/node_modules from cache
-        id: cacheAwsLambdaOtelExtensionDistNodeModules
+      - name: Retrieve packages/aws-lambda-otel-extension/test/fixtures/lambdas from cache
+        id: cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules
         uses: actions/cache@v2
         with:
-          path: packages/aws-lambda-otel-extension-dist/node_modules
-          key: packages/aws-lambda-otel-extension-dist/node-modules-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('packages/aws-lambda-otel-extension-dist/package.json') }}
+          path: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node_modules
+          key: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v16-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json') }}
           restore-keys: |
-            packages/aws-lambda-otel-extension-dist/node-modules-v16-${{ runner.os }}-${{ github.ref }}-
-            packages/aws-lambda-otel-extension-dist/node-modules-v16-${{ runner.os }}-refs/heads/main-
+            packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v16-${{ runner.os }}-${{ github.ref }}-
+            packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v16-${{ runner.os }}-refs/heads/main-
       - name: Install Node.js and npm
         uses: actions/setup-node@v1
         with:
@@ -94,11 +94,11 @@ jobs:
         run: |
           cd packages/aws-lambda-otel-extension/opt/otel-extension
           npm update --no-save
-      - name: Install packages/aws-lambda-otel-extension-dist dependencies
-        if: steps.cacheAwsLambdaOtelExtensionDistNodeModules.outputs.cache-hit != 'true'
+      - name: Install packages/aws-lambda-otel-extension/test/fixtures/lambdas dependencies
+        if: steps.cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules.outputs.cache-hit != 'true'
         run: |
-          cd packages/aws-lambda-otel-extension-dist
-          npm update --save-dev --no-save
+          cd packages/aws-lambda-otel-extension/test/fixtures/lambdas
+          npm update --no-save
 
       - name: Validate Prettier formatting
         run: npm run prettier-check:updated
@@ -152,7 +152,15 @@ jobs:
           restore-keys: |
             packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v14-${{ runner.os }}-${{ github.ref }}-
             packages/aws-lambda-otel-extension/opt/otel-extension/node-modules-v14-${{ runner.os }}-refs/heads/main-
-
+      - name: Retrieve packages/aws-lambda-otel-extension/test/fixtures/lambdas from cache
+        id: cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules
+        uses: actions/cache@v2
+        with:
+          path: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node_modules
+          key: packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v14-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json') }}
+          restore-keys: |
+            packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v14-${{ runner.os }}-${{ github.ref }}-
+            packages/aws-lambda-otel-extension/test/fixtures/lambdas/node-modules-v14-${{ runner.os }}-refs/heads/main-
       - name: Install Node.js and npm
         uses: actions/setup-node@v1
         with:
@@ -173,6 +181,11 @@ jobs:
         run: |
           cd packages/aws-lambda-otel-extension/opt/otel-extension
           npm update --no-save
+      - name: Install packages/aws-lambda-otel-extension/test/fixtures/lambdas dependencies
+        if: steps.cacheAwsLambdaOtelExtensionTestFixturesLambdasNodeModules.outputs.cache-hit != 'true'
+        run: |
+          cd packages/aws-lambda-otel-extension/test/fixtures/lambdas
+          npm update --no-save
 
       - name: Unit tests
         run: |
diff --git a/packages/aws-lambda-otel-extension/opt/otel-extension/internal/index.js b/packages/aws-lambda-otel-extension/opt/otel-extension/internal/index.js
index 19bab6b85..0747086d5 100644
--- a/packages/aws-lambda-otel-extension/opt/otel-extension/internal/index.js
+++ b/packages/aws-lambda-otel-extension/opt/otel-extension/internal/index.js
@@ -1,7 +1,5 @@
 'use strict';
 
-if (!require('./prepare-wrapper')()) return; // Bad handler, let error naturally surface
-
 const { gzipSync } = require('zlib');
 const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
 const { InMemorySpanExporter } = require('@opentelemetry/sdk-trace-base');
@@ -239,30 +237,34 @@ const responseHandler = async (span, { res, err }, isTimeout) => {
     )
   );
 
-  const logString = `⚡.${gzipSync(
-    JSON.stringify({
-      function: functionData,
-      traces: {
-        resourceSpans: [
-          {
-            resource: tracerProvider.resource.attributes,
-            instrumentationLibrarySpans: data,
-          },
-        ],
-      },
-    })
-  ).toString('base64')}`;
-
-  await fetch(`http://localhost:${OTEL_SERVER_PORT}`, {
-    method: 'post',
-    body: JSON.stringify({
-      recordType: 'telemetryData',
-      record: `${new Date().toISOString()}\t${executionId}\t${logString}`,
-    }),
-    headers: {
-      'Content-Type': 'application/json',
+  const telemetryDataPayload = {
+    function: functionData,
+    traces: {
+      resourceSpans: [
+        {
+          resource: tracerProvider.resource.attributes,
+          instrumentationLibrarySpans: data,
+        },
+      ],
     },
-  });
+  };
+  if (process.env.TEST_DRY_LOG) {
+    process._rawDebug(
+      `${require('util').inspect(telemetryDataPayload, { depth: Infinity, colors: true })}\n`
+    );
+  } else {
+    const logString = `⚡.${gzipSync(JSON.stringify(telemetryDataPayload)).toString('base64')}`;
+    await fetch(`http://localhost:${OTEL_SERVER_PORT}`, {
+      method: 'post',
+      body: JSON.stringify({
+        recordType: 'telemetryData',
+        record: `${new Date().toISOString()}\t${executionId}\t${logString}`,
+      }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+  }
 
   // Reset the exporter so we don't see duplicates
   memoryExporter.reset();
@@ -330,23 +332,30 @@ const instrumentations = [
           event.requestContext.timeEpoch;
       }
 
-      // Send request data to external so that we can attach this data to logs
-      await fetch(`http://localhost:${OTEL_SERVER_PORT}`, {
-        method: 'post',
-        body: JSON.stringify({
-          recordType: 'eventData',
-          record: {
-            eventData,
-            span: {
-              traceId: span.spanContext().traceId,
-              spanId: span.spanContext().spanId,
-            },
+      const eventDataPayload = {
+        recordType: 'eventData',
+        record: {
+          eventData,
+          span: {
+            traceId: span.spanContext().traceId,
+            spanId: span.spanContext().spanId,
           },
-        }),
-        headers: {
-          'Content-Type': 'application/json',
         },
-      });
+      };
+      if (process.env.TEST_DRY_LOG) {
+        process._rawDebug(
+          `${require('util').inspect(eventDataPayload, { depth: Infinity, colors: true })}\n`
+        );
+      } else {
+        // Send request data to external so that we can attach this data to logs
+        await fetch(`http://localhost:${OTEL_SERVER_PORT}`, {
+          method: 'post',
+          body: JSON.stringify(eventDataPayload),
+          headers: {
+            'Content-Type': 'application/json',
+          },
+        });
+      }
     },
     responseHook: async (span, { err, res }) => {
       clearTimeout(timeoutHandler);
@@ -397,3 +406,5 @@ async function initializeProvider() {
 }
 
 module.exports = initializeProvider();
+
+require('./prepare-wrapper')();
diff --git a/packages/aws-lambda-otel-extension/package.json b/packages/aws-lambda-otel-extension/package.json
index e1a4faaee..5a6e7170a 100644
--- a/packages/aws-lambda-otel-extension/package.json
+++ b/packages/aws-lambda-otel-extension/package.json
@@ -5,13 +5,14 @@
   "author": "Serverless, Inc.",
   "dependencies": {
     "adm-zip": "^0.5.9",
-    "child-process-ext": "^2.1.1",
     "esbuild": "^0.14.21",
     "essentials": "^1.2.0",
-    "fs2": "^0.3.9"
+    "fs2": "^0.3.9",
+    "semver": "^7.3.5"
   },
   "devDependencies": {
-    "deasync": "^0.1.24"
+    "deasync": "^0.1.24",
+    "type": "^2.6.0"
   },
   "scripts": {
     "build": "./scripts/build.js",
diff --git a/packages/aws-lambda-otel-extension/scripts/lib/build.js b/packages/aws-lambda-otel-extension/scripts/lib/build.js
index 2c1846e1f..98bf03736 100644
--- a/packages/aws-lambda-otel-extension/scripts/lib/build.js
+++ b/packages/aws-lambda-otel-extension/scripts/lib/build.js
@@ -2,15 +2,13 @@
 
 const path = require('path');
 const readdir = require('fs2/readdir');
-const rmdir = require('fs2/rmdir');
 const unlink = require('fs2/unlink');
 const AdmZip = require('adm-zip');
 const mkdir = require('fs2/mkdir');
-const spawn = require('child-process-ext/spawn');
+const ensureNpmDependencies = require('./ensure-npm-dependencies');
 
 const rootDir = path.resolve(__dirname, '../../');
 const optDir = path.resolve(rootDir, 'opt');
-const otelExtensionDir = path.resolve(optDir, 'otel-extension');
 
 module.exports = async (distFilename) => {
   const zip = new AdmZip();
@@ -18,13 +16,7 @@ module.exports = async (distFilename) => {
     unlink(distFilename, { loose: true }),
     mkdir(path.dirname(distFilename), { intermediate: true, silent: true }),
     (async () => {
-      await rmdir(path.resolve(otelExtensionDir, 'node_modules'), {
-        loose: true,
-        recursive: true,
-        force: true,
-      });
-      await spawn('npm', ['install'], { cwd: otelExtensionDir, stdio: 'inherit' });
-      await unlink(path.resolve(otelExtensionDir, 'package-lock.json'));
+      ensureNpmDependencies('opt/otel-extension');
       for (const relativeFilename of await readdir(optDir, {
         depth: Infinity,
         type: { file: true },
diff --git a/packages/aws-lambda-otel-extension/scripts/lib/ensure-npm-dependencies.js b/packages/aws-lambda-otel-extension/scripts/lib/ensure-npm-dependencies.js
new file mode 100644
index 000000000..123c16885
--- /dev/null
+++ b/packages/aws-lambda-otel-extension/scripts/lib/ensure-npm-dependencies.js
@@ -0,0 +1,37 @@
+'use strict';
+
+const path = require('path');
+const semver = require('semver');
+
+const rootPath = path.resolve(__dirname, '../../');
+
+module.exports = (dirname) => {
+  const pkgJson = require(path.resolve(rootPath, dirname, 'package'));
+  for (const [dependencyName, dependencyVersionRange] of Object.entries({
+    ...(pkgJson.devDependencies || {}),
+    ...(pkgJson.dependencies || {}),
+  })) {
+    const dependencyVersion = (() => {
+      try {
+        return require(path.resolve(
+          rootPath,
+          dirname,
+          'node_modules',
+          dependencyName,
+          'package.json'
+        )).version;
+      } catch {
+        throw new Error(
+          `Outdated dependencies state at "${dirname}" ("${dependencyName}" is not installed). ` +
+            'Please run "npm install"'
+        );
+      }
+    })();
+    if (!semver.satisfies(dependencyVersion, dependencyVersionRange)) {
+      throw new Error(
+        `Outdated dependencies state at "${dirname}" (unexpected version of "${dependencyName}" ). ` +
+          'Please run "npm install"'
+      );
+    }
+  }
+};


Test patch:
diff --git a/packages/aws-lambda-otel-extension/test/fixtures/lambdas/.eslintrc.js b/packages/aws-lambda-otel-extension/test/fixtures/lambdas/.eslintrc.js
new file mode 100644
index 000000000..f6e255b1a
--- /dev/null
+++ b/packages/aws-lambda-otel-extension/test/fixtures/lambdas/.eslintrc.js
@@ -0,0 +1,21 @@
+'use strict';
+
+const path = require('path');
+
+const projectDir = path.resolve(__dirname, '../../../../..');
+
+module.exports = {
+  extends: path.resolve(projectDir, '.eslintrc.js'),
+  rules: {
+    'import/no-extraneous-dependencies': [
+      'error',
+      {
+        packageDir: [
+          projectDir,
+          path.resolve(projectDir, 'packages/aws-lambda-otel-extension/test/fixtures/lambdas'),
+        ],
+      },
+    ],
+  },
+  ignorePatterns: ['esbuild-esm-callback-success.js'],
+};
diff --git a/packages/aws-lambda-otel-extension/test/fixtures/lambdas/express-app.js b/packages/aws-lambda-otel-extension/test/fixtures/lambdas/express-app.js
new file mode 100644
index 000000000..ca07a62da
--- /dev/null
+++ b/packages/aws-lambda-otel-extension/test/fixtures/lambdas/express-app.js
@@ -0,0 +1,15 @@
+'use strict';
+
+const serverless = require('serverless-http');
+const express = require('express');
+
+const app = express();
+app.use(express.json());
+
+app.get('/', (req, res) => {
+  return res.status(200).json({ message: 'Hello from root!' });
+});
+
+app.use((req, res) => res.status(404).json({ error: 'Not Found' }));
+
+module.exports.handler = serverless(app);
diff --git a/packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json b/packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json
new file mode 100644
index 000000000..0768364fd
--- /dev/null
+++ b/packages/aws-lambda-otel-extension/test/fixtures/lambdas/package.json
@@ -0,0 +1,6 @@
+{
+  "dependencies": {
+    "express": "^4.17.3",
+    "serverless-http": "^2.7.0"
+  }
+}
diff --git a/packages/aws-lambda-otel-extension/test/integration/index.test.js b/packages/aws-lambda-otel-extension/test/integration/index.test.js
index 9da37d4b6..5df4e9df4 100644
--- a/packages/aws-lambda-otel-extension/test/integration/index.test.js
+++ b/packages/aws-lambda-otel-extension/test/integration/index.test.js
@@ -13,6 +13,7 @@ const log = require('log').get('test');
 const buildLayer = require('../../scripts/lib/build');
 const resolveDirZipBuffer = require('../utils/resolve-dir-zip-buffer');
 const normalizeOtelAttributes = require('../utils/normalize-otel-attributes');
+const ensureNpmDependencies = require('../../scripts/lib/ensure-npm-dependencies');
 
 const nameTimeBase = new Date(2022, 1, 17).getTime();
 const layerFilename = path.resolve(__dirname, '../../dist/extension.zip');
@@ -20,7 +21,7 @@ const fixturesDirname = path.resolve(__dirname, '../fixtures/lambdas');
 
 describe('integration', function () {
   this.timeout(120000);
-  let name;
+  let basename;
   let s3;
   let lambda;
   let iam;
@@ -29,44 +30,17 @@ describe('integration', function () {
   let lambdasCodeZipBuffer;
   let roleArn;
 
-  const processFunction = async (handlerModuleName) => {
+  const basicHandlerModuleNames = [
+    'callback-success',
+    'esbuild-esm-callback-success',
+    'esm-callback-success/index',
+  ];
+
+  const processFunction = async (handlerModuleName, payload = { foo: 'bar' }) => {
     const functionBasename = handlerModuleName.includes(path.sep)
       ? path.dirname(handlerModuleName)
       : handlerModuleName;
-    const functionName = `${name}-${functionBasename}`;
-    const createFunction = async () => {
-      try {
-        await lambda.createFunction({
-          Code: {
-            ZipFile: lambdasCodeZipBuffer,
-          },
-          FunctionName: functionName,
-          Handler: `${handlerModuleName}.handler`,
-          Role: roleArn,
-          Runtime: 'nodejs14.x',
-          Environment: {
-            Variables: {
-              AWS_LAMBDA_EXEC_WRAPPER: '/opt/otel-extension/internal/exec-wrapper.sh',
-              SLS_OTEL_REPORT_TYPE: 'json',
-              SLS_OTEL_REPORT_S3_BUCKET: name,
-              OTEL_LOG_LEVEL: 'ALL',
-              DEBUG_SLS_OTEL_LAYER: '1',
-            },
-          },
-          Layers: [layerArn],
-        });
-      } catch (error) {
-        if (
-          error.message.includes('The role defined for the function cannot be assumed by Lambda') ||
-          error.message.includes('because the KMS key is invalid for CreateGrant')
-        ) {
-          // Occassional race condition issue on AWS side, retry
-          await createFunction();
-          return;
-        }
-        throw error;
-      }
-    };
+    const functionName = `${basename}-${functionBasename}`;
     const ensureIsActive = async () => {
       const {
         Configuration: { State: state },
@@ -76,16 +50,14 @@ describe('integration', function () {
     const invokeFunction = async () => {
       await lambda.invoke({
         FunctionName: functionName,
-        Payload: Buffer.from(JSON.stringify({ foo: 'bar' }), 'utf8'),
+        Payload: Buffer.from(JSON.stringify(payload), 'utf8'),
       });
     };
     const deleteFunction = async () => {
       await lambda.deleteFunction({ FunctionName: functionName });
     };
 
-    log.info('Create function %s', functionBasename);
-    await createFunction();
-    log.info('Wait for function to be active %s', functionBasename);
+    log.info('Ensure function is active %s', functionBasename);
     await ensureIsActive();
     log.info('Invoke function %s', functionBasename);
     await invokeFunction();
@@ -96,7 +68,7 @@ describe('integration', function () {
       await invokeFunction();
       await wait(1000);
       log.info('Retrieve list of generated S3 objects %s', functionBasename);
-      objects = ((await s3.listObjectsV2({ Bucket: name })).Contents || [])
+      objects = ((await s3.listObjectsV2({ Bucket: basename })).Contents || [])
         .map((object) => object.Key)
         .filter((key) => key.startsWith(`${functionName}/`));
     } while (!objects.length);
@@ -107,35 +79,40 @@ describe('integration', function () {
     return Promise.all(
       objects.map(async (objectKey) =>
         JSON.parse(
-          String(await streamToPromise((await s3.getObject({ Bucket: name, Key: objectKey })).Body))
+          String(
+            await streamToPromise((await s3.getObject({ Bucket: basename, Key: objectKey })).Body)
+          )
         )
       )
     );
   };
 
   before(async () => {
-    name = `test-otel-extension-${(Date.now() - nameTimeBase).toString(32)}`;
-    log.notice('Creating %s', name);
+    ensureNpmDependencies('test/fixtures/lambdas');
+    basename = `test-otel-extension-${(Date.now() - nameTimeBase).toString(32)}`;
+    log.notice('Creating %s', basename);
     s3 = new S3({ region: process.env.AWS_REGION });
     lambda = new Lambda({ region: process.env.AWS_REGION });
     iam = new IAM({ region: process.env.AWS_REGION });
 
     const createBucket = async () => {
-      log.info('Creating bucket %s', name);
-      await s3.createBucket({ Bucket: name });
-      log.info('Created bucket %s', name);
+      log.info('Creating bucket %s', basename);
+      await s3.createBucket({ Bucket: basename });
+      log.info('Created bucket %s', basename);
     };
     const createLayer = async () => {
-      log.info('Building layer');
-      await buildLayer(layerFilename);
+      if (!process.env.TEST_LAYER_FILENAME) {
+        log.info('Building layer');
+        await buildLayer(layerFilename);
+      }
 
-      log.info('Publishing layer to AWS');
+      log.info('Publishing layer (%s) to AWS', process.env.TEST_LAYER_FILENAME || layerFilename);
       await lambda.publishLayerVersion({
-        LayerName: name,
-        Content: { ZipFile: await fsp.readFile(layerFilename) },
+        LayerName: basename,
+        Content: { ZipFile: await fsp.readFile(process.env.TEST_LAYER_FILENAME || layerFilename) },
       });
       log.info('Resolving layer ARN');
-      layerArn = (await lambda.listLayerVersions({ LayerName: name })).LayerVersions.shift()
+      layerArn = (await lambda.listLayerVersions({ LayerName: basename })).LayerVersions.shift()
         .LayerVersionArn;
       log.info('Layer ready');
     };
@@ -150,7 +127,7 @@ describe('integration', function () {
         },
       ] = await Promise.all([
         iam.createRole({
-          RoleName: name,
+          RoleName: basename,
           AssumeRolePolicyDocument: JSON.stringify({
             Version: '2012-10-17',
             Statement: [
@@ -163,46 +140,89 @@ describe('integration', function () {
           }),
         }),
         iam.createPolicy({
-          PolicyName: name,
+          PolicyName: basename,
           PolicyDocument: JSON.stringify({
             Version: '2012-10-17',
             Statement: [
               {
                 Effect: 'Allow',
                 Action: ['logs:CreateLogStream', 'logs:CreateLogGroup'],
-                Resource: `arn:*:logs:*:*:log-group:/aws/lambda/${name}*:*`,
+                Resource: `arn:*:logs:*:*:log-group:/aws/lambda/${basename}*:*`,
               },
               {
                 Effect: 'Allow',
                 Action: ['logs:PutLogEvents'],
-                Resource: `arn:*:logs:*:*:log-group:/aws/lambda/${name}*:*:*`,
+                Resource: `arn:*:logs:*:*:log-group:/aws/lambda/${basename}*:*:*`,
               },
               {
                 Effect: 'Allow',
                 Action: ['s3:PutObject'],
-                Resource: `arn:aws:s3:::${name}/*`,
+                Resource: `arn:aws:s3:::${basename}/*`,
               },
             ],
           }),
         }),
       ]);
       log.info('Attaching IAM policy to role');
-      await iam.attachRolePolicy({ RoleName: name, PolicyArn: policyArn });
+      await iam.attachRolePolicy({ RoleName: basename, PolicyArn: policyArn });
       log.info('Attached IAM policy to role');
     };
+
+    const createFunctions = async () => {
+      await Promise.all(
+        [...basicHandlerModuleNames, 'express-app'].map(async function self(handlerModuleName) {
+          const functionBasename = handlerModuleName.includes(path.sep)
+            ? path.dirname(handlerModuleName)
+            : handlerModuleName;
+          const functionName = `${basename}-${functionBasename}`;
+          log.info('Create function %s', functionBasename);
+          try {
+            await lambda.createFunction({
+              Code: {
+                ZipFile: lambdasCodeZipBuffer,
+              },
+              FunctionName: functionName,
+              Handler: `${handlerModuleName}.handler`,
+              Role: roleArn,
+              Runtime: 'nodejs14.x',
+              Environment: {
+                Variables: {
+                  AWS_LAMBDA_EXEC_WRAPPER: '/opt/otel-extension/internal/exec-wrapper.sh',
+                  SLS_OTEL_REPORT_TYPE: 'json',
+                  SLS_OTEL_REPORT_S3_BUCKET: basename,
+                  OTEL_LOG_LEVEL: 'ALL',
+                  DEBUG_SLS_OTEL_LAYER: '1',
+                },
+              },
+              Layers: [layerArn],
+            });
+          } catch (error) {
+            if (
+              error.message.includes(
+                'The role defined for the function cannot be assumed by Lambda'
+              ) ||
+              error.message.includes('because the KMS key is invalid for CreateGrant')
+            ) {
+              // Occassional race condition issue on AWS side, retry
+              await self(handlerModuleName);
+              return;
+            }
+            throw error;
+          }
+        })
+      );
+    };
+
     [lambdasCodeZipBuffer] = await Promise.all([
       resolveDirZipBuffer(fixturesDirname),
       createBucket(),
       createLayer(),
       createRole(),
     ]);
+    await createFunctions();
   });
 
-  for (const handlerModuleName of [
-    'callback-success',
-    'esbuild-esm-callback-success',
-    'esm-callback-success/index',
-  ]) {
+  for (const handlerModuleName of basicHandlerModuleNames) {
     const functionBasename = handlerModuleName.includes(path.sep)
       ? path.dirname(handlerModuleName)
       : handlerModuleName;
@@ -218,31 +238,105 @@ describe('integration', function () {
         const resourceMetrics = normalizeOtelAttributes(
           metricsReport.resourceMetrics[0].resource.attributes
         );
-        expect(resourceMetrics['faas.name']).to.equal(`${name}-${functionBasename}`);
+        expect(resourceMetrics['faas.name']).to.equal(`${basename}-${functionBasename}`);
         const resourceSpans = normalizeOtelAttributes(
           tracesReport.resourceSpans[0].resource.attributes
         );
-        expect(resourceSpans['faas.name']).to.equal(`${name}-${functionBasename}`);
+        expect(resourceSpans['faas.name']).to.equal(`${basename}-${functionBasename}`);
       });
     });
   }
 
+  describe('express-app', () => {
+    let reports;
+    const handlerModuleName = 'express-app';
+    before(async () => {
+      reports = await processFunction(handlerModuleName, {
+        version: '2.0',
+        routeKey: '$default',
+        rawPath: '/',
+        rawQueryString: '',
+        headers: {
+          'accept':
+            'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+          'accept-encoding': 'gzip, deflate, br',
+          'accept-language': 'en-US,pl;q=0.7,en;q=0.3',
+          'content-length': '0',
+          'host': '1hqnqp4a70.execute-api.us-east-1.amazonaws.com',
+          'sec-fetch-dest': 'document',
+          'sec-fetch-mode': 'navigate',
+          'sec-fetch-site': 'none',
+          'sec-fetch-user': '?1',
+          'sec-gpc': '1',
+          'upgrade-insecure-requests': '1',
+          'user-agent':
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0',
+          'x-amzn-trace-id': 'Root=1-624605c4-7fcc8fe9188a3cb762dcd189',
+          'x-forwarded-for': '80.55.87.22',
+          'x-forwarded-port': '443',
+          'x-forwarded-proto': 'https',
+        },
+        requestContext: {
+          accountId: '992311060759',
+          apiId: '1hqnqp4a70',
+          domainName: '1hqnqp4a70.execute-api.us-east-1.amazonaws.com',
+          domainPrefix: '1hqnqp4a70',
+          http: {
+            method: 'GET',
+            path: '/',
+            protocol: 'HTTP/1.1',
+            sourceIp: '80.55.87.22',
+            userAgent:
+              'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0',
+          },
+          requestId: 'P3XWwjfgIAMEVFw=',
+          routeKey: '$default',
+          stage: '$default',
+          time: '31/Mar/2022:19:49:24 +0000',
+          timeEpoch: 1648756164620,
+        },
+        isBase64Encoded: false,
+      });
+      log.debug('resolved reports %o', reports);
+    });
+    it('test', () => {
+      const metricsReport = reports.find((reportArr) => !!('resourceMetrics' in reportArr[0]))[0];
+      const tracesReport = reports.find((reportArr) => !!('resourceSpans' in reportArr[0]))[0];
+      const resourceMetrics = normalizeOtelAttributes(
+        metricsReport.resourceMetrics[0].resource.attributes
+      );
+      expect(resourceMetrics['faas.name']).to.equal(`${basename}-${handlerModuleName}`);
+      const resourceSpans = normalizeOtelAttributes(
+        tracesReport.resourceSpans[0].resource.attributes
+      );
+      expect(resourceSpans['faas.name']).to.equal(`${basename}-${handlerModuleName}`);
+
+      const expressSpans = tracesReport.resourceSpans[0].instrumentationLibrarySpans.find(
+        ({ instrumentationLibrary: { name } }) => name === '@opentelemetry/instrumentation-express'
+      );
+
+      expect(expressSpans.spans.length).to.be.at.least(4);
+    });
+  });
+
   after(async () => {
     const deleteBucket = async () => {
-      const objects = ((await s3.listObjectsV2({ Bucket: name })).Contents || []).map((object) => ({
-        Key: object.Key,
-      }));
+      const objects = ((await s3.listObjectsV2({ Bucket: basename })).Contents || []).map(
+        (object) => ({
+          Key: object.Key,
+        })
+      );
       if (objects.length) {
-        await s3.deleteObjects({ Bucket: name, Delete: { Objects: objects } });
+        await s3.deleteObjects({ Bucket: basename, Delete: { Objects: objects } });
       }
-      await s3.deleteBucket({ Bucket: name });
+      await s3.deleteBucket({ Bucket: basename });
     };
     const deleteLayer = async () =>
-      lambda.deleteLayerVersion({ LayerName: name, VersionNumber: 1 });
+      lambda.deleteLayerVersion({ LayerName: basename, VersionNumber: 1 });
     const deleteRole = async () => {
-      await iam.detachRolePolicy({ RoleName: name, PolicyArn: policyArn });
+      await iam.detachRolePolicy({ RoleName: basename, PolicyArn: policyArn });
       return Promise.all([
-        iam.deleteRole({ RoleName: name }),
+        iam.deleteRole({ RoleName: basename }),
         iam.deletePolicy({ PolicyArn: policyArn }),
       ]);
     };
diff --git a/packages/aws-lambda-otel-extension/test/unit/external.test.js b/packages/aws-lambda-otel-extension/test/unit/external.test.js
index 34076a344..d0653610b 100644
--- a/packages/aws-lambda-otel-extension/test/unit/external.test.js
+++ b/packages/aws-lambda-otel-extension/test/unit/external.test.js
@@ -12,11 +12,13 @@ const normalizeOtelAttributes = require('../utils/normalize-otel-attributes');
 const { SAVE_FILE, SENT_FILE } = require('../../opt/otel-extension/external/helper');
 const { default: fetch } = require('node-fetch');
 const { OTEL_SERVER_PORT } = require('../../opt/otel-extension/lib/helper');
+const ensureNpmDependencies = require('../../scripts/lib/ensure-npm-dependencies');
 
 const port = 9001;
 
 describe('external', () => {
   before(async () => {
+    ensureNpmDependencies('opt/otel-extension');
     evilDns.add('sandbox', '127.0.0.1');
     process.env.AWS_LAMBDA_RUNTIME_API = `127.0.0.1:${port}`;
     process.env.SLS_OTEL_REPORT_TYPE = 'json';
diff --git a/packages/aws-lambda-otel-extension/test/unit/internal/event-detection.test.js b/packages/aws-lambda-otel-extension/test/unit/internal/event-detection.test.js
index dc3a62030..f40cb7155 100644
--- a/packages/aws-lambda-otel-extension/test/unit/internal/event-detection.test.js
+++ b/packages/aws-lambda-otel-extension/test/unit/internal/event-detection.test.js
@@ -2,9 +2,14 @@
 
 const { expect } = require('chai');
 
+const ensureNpmDependencies = require('../../../scripts/lib/ensure-npm-dependencies');
 const { detectEventType } = require('../../../opt/otel-extension/internal/event-detection');
 
 describe('detectEventType', () => {
+  before(() => {
+    ensureNpmDependencies('opt/otel-extension');
+  });
+
   it('should return alexa skill', () => {
     const event = {
       session: {
diff --git a/packages/aws-lambda-otel-extension/test/unit/internal/index.test.js b/packages/aws-lambda-otel-extension/test/unit/internal/index.test.js
index bf02adfe6..89de63f72 100644
--- a/packages/aws-lambda-otel-extension/test/unit/internal/index.test.js
+++ b/packages/aws-lambda-otel-extension/test/unit/internal/index.test.js
@@ -5,20 +5,20 @@ const http = require('http');
 const path = require('path');
 const { promisify } = require('util');
 const unzip = promisify(require('zlib').unzip);
+const isThenable = require('type/thenable/is');
 const log = require('log').get('test');
 const requireUncached = require('ncjsm/require-uncached');
-const overwriteStdoutWrite = require('process-utils/override-stdout-write');
 const { OTEL_SERVER_PORT } = require('../../../opt/otel-extension/lib/helper');
+const ensureNpmDependencies = require('../../../scripts/lib/ensure-npm-dependencies');
 
 const fixturesDirname = path.resolve(__dirname, '../../fixtures');
 
-const handleSuccess = async (handlerModuleName) => {
+const handleSuccess = async (handlerModuleName, payload = {}) => {
   process.env._HANDLER = `${handlerModuleName}.handler`;
   const functionName = handlerModuleName.includes(path.sep)
     ? path.dirname(handlerModuleName)
     : handlerModuleName;
   process.env.AWS_LAMBDA_FUNCTION_NAME = functionName;
-  let stdoutData = '';
 
   const logsQueue = [];
   let server;
@@ -30,15 +30,16 @@ const handleSuccess = async (handlerModuleName) => {
           body += data;
         });
         request.on('end', () => {
-          resolve(
-            (async () => {
-              response.writeHead(200, {});
-              response.end('OK');
-              server.close();
-              const data = JSON.parse(body);
-              logsQueue.push(data);
+          response.writeHead(200, {});
+          response.end('OK');
+          const data = JSON.parse(body);
+          logsQueue.push(data);
 
-              if (logsQueue.length > 1) {
+          if (logsQueue.length > 1) {
+            server.close();
+            resolve(
+              (async () => {
+                log.debug('logs: %o', logsQueue);
                 // Validate eventData record for log metadata
                 const logMetadata = logsQueue[0].record;
                 expect(logMetadata.eventData['123']['telemetry.sdk.language']).to.equal('nodejs');
@@ -53,14 +54,16 @@ const handleSuccess = async (handlerModuleName) => {
                 const report = JSON.parse(
                   String(await unzip(Buffer.from(reportCompressed, 'base64')))
                 );
-                log.debug('result report: %o', report);
+                log.debug('report: %o', report);
                 expect(report.function['telemetry.sdk.language']).to.equal('nodejs');
                 expect(report.function['telemetry.sdk.name']).to.equal('opentelemetry');
                 expect(report.function['faas.name']).to.equal(functionName);
                 expect(report.function.error).to.equal(false);
-              }
-            })()
-          );
+
+                return report;
+              })()
+            );
+          }
         });
       }
     });
@@ -69,27 +72,27 @@ const handleSuccess = async (handlerModuleName) => {
   server.listen(OTEL_SERVER_PORT);
 
   try {
-    await overwriteStdoutWrite(
-      (data) => (stdoutData += data),
-      async () =>
-        requireUncached(async () => {
-          await require('../../../opt/otel-extension/internal');
-          await new Promise((resolve) => {
-            require('../../../opt/otel-extension/internal/wrapper').handler(
-              {},
-              {
-                awsRequestId: '123',
-                functionName,
-                invokedFunctionArn: `arn:aws:lambda:us-east-1:123456789012:function:${functionName}`,
-                getRemainingTimeInMillis: () => 3000,
-              },
-              resolve
-            );
-          });
-        })
-    );
+    await requireUncached(async () => {
+      await require('../../../opt/otel-extension/internal');
+      await new Promise((resolve, reject) => {
+        const maybeThenable = require('../../../opt/otel-extension/internal/wrapper').handler(
+          payload,
+          {
+            awsRequestId: '123',
+            functionName,
+            invokedFunctionArn: `arn:aws:lambda:us-east-1:123456789012:function:${functionName}`,
+            getRemainingTimeInMillis: () => 3000,
+          },
+          (error, result) => {
+            if (error) reject(error);
+            else resolve(result);
+          }
+        );
+        if (isThenable(maybeThenable)) resolve(maybeThenable);
+      });
+    });
 
-    await deferredResultProcessing;
+    return await deferredResultProcessing;
   } finally {
     server.close();
   }
@@ -97,6 +100,8 @@ const handleSuccess = async (handlerModuleName) => {
 
 describe('internal', () => {
   before(() => {
+    ensureNpmDependencies('opt/otel-extension');
+    ensureNpmDependencies('test/fixtures/lambdas');
     process.env.AWS_LAMBDA_FUNCTION_VERSION = '$LATEST';
     process.env.AWS_REGION = 'us-east-1';
     process.env.LAMBDA_TASK_ROOT = path.resolve(fixturesDirname, 'lambdas');
@@ -116,4 +121,58 @@ describe('internal', () => {
   it('should handle esbuild ESM bundle result', async () =>
     handleSuccess('esbuild-esm-callback-success'));
   it('should handle ESM module', async () => handleSuccess('esm-callback-success/index'));
+  it('should handle "express" handler', async () => {
+    const report = await handleSuccess('express-app', {
+      version: '2.0',
+      routeKey: '$default',
+      rawPath: '/',
+      rawQueryString: '',
+      headers: {
+        'accept':
+          'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+        'accept-encoding': 'gzip, deflate, br',
+        'accept-language': 'en-US,pl;q=0.7,en;q=0.3',
+        'content-length': '0',
+        'host': '1hqnqp4a70.execute-api.us-east-1.amazonaws.com',
+        'sec-fetch-dest': 'document',
+        'sec-fetch-mode': 'navigate',
+        'sec-fetch-site': 'none',
+        'sec-fetch-user': '?1',
+        'sec-gpc': '1',
+        'upgrade-insecure-requests': '1',
+        'user-agent':
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0',
+        'x-amzn-trace-id': 'Root=1-624605c4-7fcc8fe9188a3cb762dcd189',
+        'x-forwarded-for': '80.55.87.22',
+        'x-forwarded-port': '443',
+        'x-forwarded-proto': 'https',
+      },
+      requestContext: {
+        accountId: '992311060759',
+        apiId: '1hqnqp4a70',
+        domainName: '1hqnqp4a70.execute-api.us-east-1.amazonaws.com',
+        domainPrefix: '1hqnqp4a70',
+        http: {
+          method: 'GET',
+          path: '/',
+          protocol: 'HTTP/1.1',
+          sourceIp: '80.55.87.22',
+          userAgent:
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0',
+        },
+        requestId: 'P3XWwjfgIAMEVFw=',
+        routeKey: '$default',
+        stage: '$default',
+        time: '31/Mar/2022:19:49:24 +0000',
+        timeEpoch: 1648756164620,
+      },
+      isBase64Encoded: false,
+    });
+
+    const expressSpans = report.traces.resourceSpans[0].instrumentationLibrarySpans.find(
+      ({ instrumentationLibrary: { name } }) => name === '@opentelemetry/instrumentation-express'
+    );
+
+    expect(expressSpans.spans.length).to.be.at.least(4);
+  });
 });