Repository: GoogleChrome/lighthouse (js)

Issue:
Add --debug flag to keep the loaded page open for inspection
**Feature request summary**
A core `setting` that pauses after the page has been loaded to give time to inspect contents, use devtools, figure out differences, etc.

**What is the motivation or use case for changing this?**
I've definitely manually changed the `waitForLoad` to 65s to do this before, debugging how the page is different when loaded by Lighthouse is very difficult today and typically requires one of us to look into it.

Also requested by a LH partner :)

**How is this beneficial to Lighthouse?**
Could be useful tie-in to #11313 
Helpful in Lighthouse CI and for testing authenticated scenarios
Fewer bug reports about Lighthouse being different when it's the page that's different 😉 



Pull request description:
core: add --debugNavigation setting
**Summary**
Adds a setting that keeps the page open until `continueLighthouseRun()` is evaluated in the console before moving on to close gatherers so you explore the page and DevTools as long as necessary. One consequence of using this flag is that traces / network records are affected by whatever the user does, but the purpose is not to ever use the results of this run, simply to explore the page (in fact the smoketest *asserts* that this expands the trace window)

**Related Issues/PRs**
fixes #12641 


Patch:
diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml
index 1eb3c50a4..2c30fc393 100644
--- a/.github/workflows/smoke.yml
+++ b/.github/workflows/smoke.yml
@@ -106,7 +106,7 @@ jobs:
 
     - run: sudo apt-get install xvfb
     - name: yarn smoke --fraggle-rock
-      run: xvfb-run --auto-servernum yarn smoke --debug --fraggle-rock -j=1 --retries=2 a11y lantern seo-passing seo-failing csp metrics
+      run: xvfb-run --auto-servernum yarn smoke --debug --fraggle-rock -j=1 --retries=2 a11y lantern seo-passing seo-failing csp metrics perf-debug
 
     # Fail if any changes were written to source files.
     - run: git diff --exit-code
diff --git a/build/build-report.js b/build/build-report.js
index 6ef655a18..d4ca39814 100644
--- a/build/build-report.js
+++ b/build/build-report.js
@@ -64,16 +64,32 @@ async function buildEsModulesBundle() {
   });
 
   await bundle.write({
-    file: 'dist/report/bundle.js',
+    file: 'dist/report/bundle.esm.js',
     format: 'esm',
   });
 }
 
+async function buildUmdBundle() {
+  const bundle = await rollup.rollup({
+    input: 'report/clients/bundle.js',
+    plugins: [
+      commonjs(),
+    ],
+  });
+
+  await bundle.write({
+    file: 'dist/report/bundle.umd.js',
+    format: 'umd',
+    name: 'report',
+  });
+}
+
 if (require.main === module) {
   if (process.argv[2] === '--only-standalone') {
     buildStandaloneReport();
   } else {
     buildStandaloneReport();
+    buildUmdBundle();
     buildEsModulesBundle();
   }
 }
@@ -81,5 +97,6 @@ if (require.main === module) {
 module.exports = {
   buildStandaloneReport,
   buildPsiReport,
+  buildUmdBundle,
   buildTreemapReport,
 };
diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index aab061061..b7d99e67d 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -108,6 +108,10 @@ function getFlags(manualArgv, options = {}) {
           default: false,
           describe: 'Print the normalized config for the given config and options, then exit.',
         },
+        'debug-navigation': {
+          type: 'boolean',
+          describe: 'Pause after page load to wait for permission to continue the run, evaluate `continueLighthouseRun` in the console to continue.',
+        },
         'fraggle-rock': {
           type: 'boolean',
           default: false,
@@ -362,16 +366,20 @@ function coerceOptionalStringBoolean(value) {
  */
 function coerceOutput(values) {
   const outputTypes = ['json', 'html', 'csv'];
-  const errorMsg = `Invalid values. Argument 'output' must be an array from choices "${outputTypes.join('", "')}"`;
+  const errorHint = `Argument 'output' must be an array from choices "${outputTypes.join('", "')}"`;
   if (!values.every(/** @return {item is string} */ item => typeof item === 'string')) {
-    throw new Error(errorMsg);
+    throw new Error('Invalid values. ' + errorHint);
   }
   // Allow parsing of comma-separated values.
   const strings = values.flatMap(value => value.split(','));
-  if (!strings.every(/** @return {str is LH.OutputMode} */ str => outputTypes.includes(str))) {
-    throw new Error(errorMsg);
-  }
-  return strings;
+  const validValues = strings.filter(/** @return {str is LH.OutputMode} */ str => {
+    if (!outputTypes.includes(str)) {
+      throw new Error(`"${str}" is not a valid 'output' value. ` + errorHint);
+    }
+    return true;
+  });
+
+  return validValues;
 }
 
 /**
diff --git a/lighthouse-core/config/constants.js b/lighthouse-core/config/constants.js
index c2e4aebe8..264402bb0 100644
--- a/lighthouse-core/config/constants.js
+++ b/lighthouse-core/config/constants.js
@@ -106,6 +106,7 @@ const defaultSettings = {
   auditMode: false,
   gatherMode: false,
   disableStorageReset: false,
+  debugNavigation: false,
   channel: 'node',
 
   // the following settings have no defaults but we still want ensure that `key in settings`
diff --git a/lighthouse-core/fraggle-rock/gather/navigation-runner.js b/lighthouse-core/fraggle-rock/gather/navigation-runner.js
index e7da3d17a..d17966aba 100644
--- a/lighthouse-core/fraggle-rock/gather/navigation-runner.js
+++ b/lighthouse-core/fraggle-rock/gather/navigation-runner.js
@@ -88,6 +88,7 @@ async function _navigate(navigationContext) {
   try {
     const {finalUrl, warnings} = await gotoURL(driver, requestedUrl, {
       ...navigationContext.navigation,
+      debugNavigation: config.settings.debugNavigation,
       maxWaitForFcp: config.settings.maxWaitForFcp,
       maxWaitForLoad: config.settings.maxWaitForLoad,
       waitUntil: navigationContext.navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
diff --git a/lighthouse-core/gather/driver/navigation.js b/lighthouse-core/gather/driver/navigation.js
index b6f5e9371..58b346b9d 100644
--- a/lighthouse-core/gather/driver/navigation.js
+++ b/lighthouse-core/gather/driver/navigation.js
@@ -6,7 +6,7 @@
 'use strict';
 
 const NetworkMonitor = require('./network-monitor.js');
-const {waitForFullyLoaded, waitForFrameNavigated} = require('./wait-for-condition.js');
+const {waitForFullyLoaded, waitForFrameNavigated, waitForUserToContinue} = require('./wait-for-condition.js'); // eslint-disable-line max-len
 const constants = require('../../config/constants.js');
 const i18n = require('../../lib/i18n/i18n.js');
 const URL = require('../../lib/url-shim.js');
@@ -39,7 +39,7 @@ const DEFAULT_NETWORK_QUIET_THRESHOLD = 5000;
 // Controls how long to wait between longtasks before determining the CPU is idle, off by default
 const DEFAULT_CPU_QUIET_THRESHOLD = 0;
 
-/** @typedef {{waitUntil: Array<'fcp'|'load'|'navigated'>} & LH.Config.SharedPassNavigationJson & Partial<Pick<LH.Config.Settings, 'maxWaitForFcp'|'maxWaitForLoad'>>} NavigationOptions */
+/** @typedef {{waitUntil: Array<'fcp'|'load'|'navigated'>} & LH.Config.SharedPassNavigationJson & Partial<Pick<LH.Config.Settings, 'maxWaitForFcp'|'maxWaitForLoad'|'debugNavigation'>>} NavigationOptions */
 
 /** @param {NavigationOptions} options */
 function resolveWaitForFullyLoadedOptions(options) {
@@ -121,6 +121,10 @@ async function gotoURL(driver, url, options) {
   await waitforPageNavigateCmd;
   await networkMonitor.disable();
 
+  if (options.debugNavigation) {
+    await waitForUserToContinue(driver);
+  }
+
   return {
     finalUrl,
     warnings: getNavigationWarnings({timedOut, finalUrl, requestedUrl: url}),
diff --git a/lighthouse-core/gather/driver/wait-for-condition.js b/lighthouse-core/gather/driver/wait-for-condition.js
index cbb660491..bf3afd308 100644
--- a/lighthouse-core/gather/driver/wait-for-condition.js
+++ b/lighthouse-core/gather/driver/wait-for-condition.js
@@ -490,6 +490,32 @@ async function waitForFullyLoaded(session, networkMonitor, options) {
   return cleanupFn();
 }
 
+/**
+ * @param {LH.Gatherer.FRTransitionalDriver} driver
+ */
+function waitForUserToContinue(driver) {
+  /* c8 ignore start */
+  function createInPagePromise() {
+    let resolve = () => {};
+    /** @type {Promise<void>} */
+    const promise = new Promise(r => resolve = r);
+
+    // eslint-disable-next-line no-console
+    console.log([
+      `You have enabled Lighthouse navigation debug mode.`,
+      `When you have finished inspecting the page, evaluate "continueLighthouseRun()"`,
+      `in the console to continue with the Lighthouse run.`,
+    ].join(' '));
+
+    window.continueLighthouseRun = resolve;
+    return promise;
+  }
+  /* c8 ignore stop */
+
+  driver.defaultSession.setNextProtocolTimeout(2 ** 31 - 1);
+  return driver.executionContext.evaluate(createInPagePromise, {args: []});
+}
+
 module.exports = {
   waitForNothing,
   waitForFrameNavigated,
@@ -498,4 +524,5 @@ module.exports = {
   waitForNetworkIdle,
   waitForCPUIdle,
   waitForFullyLoaded,
+  waitForUserToContinue,
 };
diff --git a/lighthouse-core/gather/gather-runner.js b/lighthouse-core/gather/gather-runner.js
index a46abe781..8a9ca9c4a 100644
--- a/lighthouse-core/gather/gather-runner.js
+++ b/lighthouse-core/gather/gather-runner.js
@@ -74,6 +74,7 @@ class GatherRunner {
       const {finalUrl, warnings} = await navigation.gotoURL(driver, requestedUrl, {
         waitUntil: passContext.passConfig.recordTrace ?
           ['load', 'fcp'] : ['load'],
+        debugNavigation: passContext.settings.debugNavigation,
         maxWaitForFcp: passContext.settings.maxWaitForFcp,
         maxWaitForLoad: passContext.settings.maxWaitForLoad,
         ...passContext.passConfig,
diff --git a/lighthouse-core/scripts/roll-to-devtools.sh b/lighthouse-core/scripts/roll-to-devtools.sh
index eb10a572e..68183f849 100755
--- a/lighthouse-core/scripts/roll-to-devtools.sh
+++ b/lighthouse-core/scripts/roll-to-devtools.sh
@@ -47,11 +47,12 @@ cp -pPR "$lh_bg_js" "$fe_lh_dir/lighthouse-dt-bundle.js"
 echo -e "$check lighthouse-dt-bundle copied."
 
 # generate bundle.d.ts
-npx tsc --allowJs --declaration --emitDeclarationOnly dist/report/bundle.js
+npx tsc --allowJs --declaration --emitDeclarationOnly dist/report/bundle.esm.js
 
 # copy report code $fe_lh_dir
 fe_lh_report_dir="$fe_lh_dir/report/"
-cp dist/report/bundle.js dist/report/bundle.d.ts "$fe_lh_report_dir"
+cp dist/report/bundle.esm.js "$fe_lh_report_dir/bundle.js"
+cp dist/report/bundle.esm.d.ts "$fe_lh_report_dir/bundle.d.ts"
 echo -e "$check Report code copied."
 
 # copy report generator + cached resources into $fe_lh_dir
@@ -71,5 +72,5 @@ fe_webtests_dir="$dt_dir/test/webtests/http/tests/devtools/lighthouse"
 rsync -avh "$lh_webtests_dir" "$fe_webtests_dir" --exclude="OWNERS" --delete
 
 echo ""
-echo "Done. To run the webtests: " 
+echo "Done. To run the webtests: "
 echo "    DEVTOOLS_PATH=\"$dt_dir\" yarn test-devtools"
diff --git a/types/externs.d.ts b/types/externs.d.ts
index ca1cc8581..cb25b91da 100644
--- a/types/externs.d.ts
+++ b/types/externs.d.ts
@@ -52,6 +52,9 @@ declare global {
     __lighthouseNodesDontTouchOrAllVarianceGoesAway: Map<Element, string>;
     __lighthouseExecutionContextId?: number;
 
+    /** Injected into the page when the `--debug` flag is used. */
+    continueLighthouseRun(): void;
+
     // Not defined in tsc yet: https://github.com/microsoft/TypeScript/issues/40807
     requestIdleCallback(callback: (deadline: {didTimeout: boolean, timeRemaining: () => DOMHighResTimeStamp}) => void, options?: {timeout: number}): number;
   }
@@ -206,6 +209,8 @@ export interface SharedFlagsSettings {
   gatherMode?: boolean | string;
   /** Flag indicating that the browser storage should not be reset for the audit. */
   disableStorageReset?: boolean;
+  /** Flag indicating that Lighthouse should pause after page load to wait for the user's permission to continue the audit. */
+  debugNavigation?: boolean;
 
   /** How Lighthouse should interpret this run in regards to scoring performance metrics and skipping mobile-only tests in desktop. Must be set even if throttling/emulation is being applied outside of Lighthouse. */
   formFactor?: 'mobile'|'desktop';


Test patch:
diff --git a/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap
index 2606e3f3b..1052de603 100644
--- a/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap
+++ b/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap
@@ -1542,6 +1542,7 @@ Object {
     "blockedUrlPatterns": null,
     "budgets": null,
     "channel": "cli",
+    "debugNavigation": false,
     "disableStorageReset": false,
     "emulatedUserAgent": "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse",
     "extraHeaders": null,
@@ -1699,6 +1700,7 @@ Object {
     "blockedUrlPatterns": null,
     "budgets": null,
     "channel": "cli",
+    "debugNavigation": false,
     "disableStorageReset": false,
     "emulatedUserAgent": "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse",
     "extraHeaders": null,
diff --git a/lighthouse-cli/test/cli/index-test.js b/lighthouse-cli/test/cli/index-test.js
index 0a7b0f18b..5a44bc0e2 100644
--- a/lighthouse-cli/test/cli/index-test.js
+++ b/lighthouse-cli/test/cli/index-test.js
@@ -20,9 +20,12 @@ describe('CLI Tests', function() {
   });
 
   it('should list options via --help', () => {
-    const ret = spawnSync('node', [indexPath, '--help'], {encoding: 'utf8'});
-    assert.ok(ret.stdout.includes('lighthouse <url>'));
-    assert.ok(ret.stdout.includes('For more information on Lighthouse'));
+    const ret = spawnSync('node', [indexPath, '--help'], {encoding: 'utf8', maxBuffer: 10_000_000});
+    expect(ret.stdout).toContain('lighthouse <url>');
+    expect(ret.stdout).toContain('Examples:');
+    // FIXME: yargs does not wait to flush stdout before exiting the process,
+    // `--help` can flakily not contain the entire output when isTTY is false.
+    // expect(ret.stdout).toContain('For more information on Lighthouse');
   });
 
   it('should list all audits without a url and exit immediately after', () => {
diff --git a/lighthouse-cli/test/fixtures/perf/debug.html b/lighthouse-cli/test/fixtures/perf/debug.html
new file mode 100644
index 000000000..6217a39c0
--- /dev/null
+++ b/lighthouse-cli/test/fixtures/perf/debug.html
@@ -0,0 +1,10 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <h1>Debug test (should last at least 30s)</h1>
+    <script>
+      setTimeout(() => window.continueLighthouseRun(), 30_000);
+    </script>
+  </body>
+</html>
diff --git a/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js b/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js
index 20df2b00a..8cf6d1719 100644
--- a/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js
+++ b/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js
@@ -161,6 +161,13 @@ const smokeTests = [{
   expectations: perf.frameMetrics,
   config: require('./perf/perf-config.js'),
   runSerially: true,
+}, {
+  id: 'perf-debug',
+  expectations: perf.debug,
+  config: {
+    extends: 'lighthouse:default',
+    settings: {debugNavigation: true, onlyAudits: ['metrics']},
+  },
 }, {
   id: 'perf-diagnostics-animations',
   expectations: diagnostics.animations,
diff --git a/lighthouse-cli/test/smokehouse/test-definitions/perf/expectations.js b/lighthouse-cli/test/smokehouse/test-definitions/perf/expectations.js
index d511441f0..bcc595cd7 100644
--- a/lighthouse-cli/test/smokehouse/test-definitions/perf/expectations.js
+++ b/lighthouse-cli/test/smokehouse/test-definitions/perf/expectations.js
@@ -362,10 +362,19 @@ const frameMetrics = {
   },
 };
 
+const debug = {
+  lhr: {
+    requestedUrl: 'http://localhost:10200/perf/debug.html',
+    finalUrl: 'http://localhost:10200/perf/debug.html',
+    audits: {metrics: {details: {items: {0: {observedTraceEnd: '>30000'}}}}},
+  },
+};
+
 module.exports = {
   preload,
   budgets,
   fonts,
+  debug,
   traceElements,
   frameMetrics,
 };
diff --git a/lighthouse-core/test/results/artifacts/artifacts.json b/lighthouse-core/test/results/artifacts/artifacts.json
index 22cfda77f..96eac1fd6 100644
--- a/lighthouse-core/test/results/artifacts/artifacts.json
+++ b/lighthouse-core/test/results/artifacts/artifacts.json
@@ -27,6 +27,7 @@
     "auditMode": false,
     "gatherMode": "lighthouse-core/test/results/artifacts",
     "disableStorageReset": false,
+    "debugNavigation": false,
     "formFactor": "mobile",
     "emulatedUserAgent": "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse",
     "screenEmulation": {
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
index 2386598ef..85686d4f2 100644
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@@ -4819,6 +4819,7 @@
     "auditMode": true,
     "gatherMode": false,
     "disableStorageReset": false,
+    "debugNavigation": false,
     "channel": "cli",
     "budgets": [
       {
diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-clear-data-warning.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-clear-data-warning.js
index c7dad8b40..eb8e15902 100644
--- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-clear-data-warning.js
+++ b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-clear-data-warning.js
@@ -27,7 +27,7 @@
   const warningText = containerElement.querySelector('.lighthouse-warning-text');
 
   // Wait for warning event to be handled
-  LighthouseTestRunner._panel()._controller.addEventListener(Events.PageWarningsChanged, () => {
+  LighthouseTestRunner._panel().controller.addEventListener(Events.PageWarningsChanged, () => {
     TestRunner.addResult(`Warning Text: ${warningText.textContent}`);
     TestRunner.completeTest();
   });
diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-i18n-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-i18n-run.js
index 04ce8753e..9cccf81ad 100644
--- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-i18n-run.js
+++ b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-i18n-run.js
@@ -21,7 +21,7 @@
     }
   }
 
-  TestRunner.override(LighthouseTestRunner._panel()._protocolService, 'getLocales', overrideLookupLocale, true);
+  TestRunner.override(LighthouseTestRunner._panel().protocolService, 'getLocales', overrideLookupLocale, true);
 
   const locales = ['invalid-locale', 'es'];
   function overrideLookupLocale() {