Repository: elastic/synthetics
Pull request #835 (2023-09-14 23:30:11)

feat: capture unhandled rejections across all pages
+ fix https://github.com/elastic/synthetics/issues/815
+ Available from Playwright 1.38.0 - PR updates the PW to the latest version and captures the unhandled rejections via the new `weberror` event from browser contest instead of relying on the page level `pageerror` event which does not provide the rejections across multiple pages.
+ Docs - https://playwright.dev/docs/api/class-weberror



Patch:
diff --git a/package-lock.json b/package-lock.json
index 5246aca..b263f26 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6610,9 +6610,9 @@
       }
     },
     "node_modules/istanbul-lib-instrument/node_modules/semver": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
       "dev": true,
       "bin": {
         "semver": "bin/semver.js"
@@ -8829,9 +8829,9 @@
       }
     },
     "node_modules/make-dir/node_modules/semver": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
       "dev": true,
       "bin": {
         "semver": "bin/semver.js"
@@ -15137,9 +15137,9 @@
       "dev": true
     },
     "node_modules/word-wrap": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
-      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
+      "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==",
       "dev": true,
       "engines": {
         "node": ">=0.10.0"
diff --git a/src/options.ts b/src/options.ts
index af5e002..83fe0dc 100644
--- a/src/options.ts
+++ b/src/options.ts
@@ -106,7 +106,7 @@ export async function normalizeOptions(
   );
   options.playwrightOptions = {
     ...playwrightOpts,
-    headless: cliArgs.headless ?? playwrightOpts?.headless,
+    headless: getHeadlessFlag(cliArgs.headless, playwrightOpts?.headless),
     chromiumSandbox: cliArgs.sandbox ?? playwrightOpts?.chromiumSandbox,
     ignoreHTTPSErrors:
       cliArgs.ignoreHttpsErrors ?? playwrightOpts?.ignoreHTTPSErrors,
@@ -140,6 +140,18 @@ export async function normalizeOptions(
   return options;
 }
 
+export function getHeadlessFlag(
+  cliHeadless: boolean,
+  configHeadless?: boolean
+) {
+  // if cliHeadless is false, then we don't care about configHeadless
+  if (!cliHeadless) {
+    return false;
+  }
+  // default is headless
+  return configHeadless ?? true;
+}
+
 export function validatePushOptions(opts: PushOptions) {
   if (opts.tags || opts.match) {
     throw error(`Aborted. Invalid CLI flags.
diff --git a/src/plugins/browser-console.ts b/src/plugins/browser-console.ts
index 0f63a19..9e48960 100644
--- a/src/plugins/browser-console.ts
+++ b/src/plugins/browser-console.ts
@@ -23,7 +23,7 @@
  *
  */
 
-import type { ConsoleMessage } from 'playwright-core';
+import type { ConsoleMessage, WebError } from 'playwright-core';
 import { BrowserMessage, Driver, StatusValue } from '../common_types';
 import { log } from '../core/logger';
 import { Step } from '../dsl';
@@ -58,11 +58,12 @@ export class BrowserConsole {
     }
   };
 
-  private pageErrorEventListener = (error: Error) => {
+  private webErrorListener = (webErr: WebError) => {
     if (!this._currentStep) {
       return;
     }
     const { name, index } = this._currentStep;
+    const error = webErr.error();
     this.messages.push({
       timestamp: getTimestamp(),
       text: error.message,
@@ -83,12 +84,12 @@ export class BrowserConsole {
   start() {
     log(`Plugins: started collecting console events`);
     this.driver.context.on('console', this.consoleEventListener);
-    this.driver.page.on('pageerror', this.pageErrorEventListener);
+    this.driver.context.on('weberror', this.webErrorListener);
   }
 
   stop() {
     this.driver.context.off('console', this.consoleEventListener);
-    this.driver.page.off('pageerror', this.pageErrorEventListener);
+    this.driver.context.off('weberror', this.webErrorListener);
     log(`Plugins: stopped collecting console events`);
     return this.messages;
   }


Test patch:
diff --git a/__tests__/options.test.ts b/__tests__/options.test.ts
index e0976d4..2536fdc 100644
--- a/__tests__/options.test.ts
+++ b/__tests__/options.test.ts
@@ -121,6 +121,20 @@ describe('options', () => {
   });
 
   it('cli arg headless override playwright headless arg', async () => {
+    const cliArgs: CliArgs = {
+      playwrightOptions: {
+        headless: true,
+      },
+      headless: false,
+    };
+    expect(await normalizeOptions(cliArgs)).toMatchObject({
+      playwrightOptions: {
+        headless: false,
+      },
+    });
+  });
+
+  it('cli arg headless default value is overridden by config file', async () => {
     const cliArgs: CliArgs = {
       playwrightOptions: {
         headless: false,
@@ -129,7 +143,7 @@ describe('options', () => {
     };
     expect(await normalizeOptions(cliArgs)).toMatchObject({
       playwrightOptions: {
-        headless: true,
+        headless: false,
       },
     });
   });
diff --git a/__tests__/plugins/browser-console.test.ts b/__tests__/plugins/browser-console.test.ts
index 3669ba8..a61002d 100644
--- a/__tests__/plugins/browser-console.test.ts
+++ b/__tests__/plugins/browser-console.test.ts
@@ -105,23 +105,37 @@ describe('BrowserConsole', () => {
     expect(referenceError?.step).toEqual(currentStep);
   });
 
-  it('should capture unhandled rejections', async () => {
+  it('capture unhandled rejections on all pages', async () => {
     const driver = await Gatherer.setupDriver({ wsEndpoint });
     const browserConsole = new BrowserConsole(driver);
     browserConsole.start();
     browserConsole._currentStep = currentStep;
     await driver.page.goto(server.TEST_PAGE);
-    await driver.page.setContent(
-      `<script>Promise.reject(new Error("Boom"))</script>`
-    );
-    await driver.page.waitForLoadState('networkidle');
+    await Promise.all([
+      driver.context.waitForEvent('weberror'),
+      driver.page.setContent(
+        `<script>Promise.reject(new Error("Boom"))</script>`
+      ),
+    ]);
+    // throw rejection inside new window
+    await Promise.all([
+      driver.page.evaluate(async () => {
+        const win = window.open();
+        Promise.reject('popup error');
+        (win as any).close();
+      }),
+      driver.context.waitForEvent('weberror'),
+      driver.page.waitForEvent('popup'),
+    ]);
     const messages = browserConsole.stop();
     await Gatherer.stop();
 
-    const unhandledError = messages.find(m => m.text.indexOf('Boom') >= 0);
-    expect(unhandledError?.type).toEqual('error');
-    expect(unhandledError?.step).toEqual(currentStep);
-    expect(unhandledError?.error?.stack).toContain('Error: Boom');
+    expect(messages.length).toEqual(2);
+    const [page1Err, page2Err] = messages;
+    expect(page1Err?.type).toEqual('error');
+    expect(page1Err?.step).toEqual(currentStep);
+    expect(page1Err?.error?.stack).toContain('Error: Boom');
+    expect(page2Err?.text).toEqual('popup error');
   });
 
   describe('Filtering', () => {