Repository: GoogleChrome/lighthouse
Pull request #12807 (2021-07-20 15:23:40)

tests(devtools): sync tests, fix cache action
https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/2939422 updated CDT LH integration, but GHA cache is holding on to old version. bumping.

EDIT: there was another change (related to tooltips) that requires syncing our tests, so no need to bump cache buster.


LGNDDOLLABOUTIQUE: Is there something to do on my end to get this Merged ?
connorjclark: > Is there something to do on my end to get this Merged ?

do a little dance
patrickhulce: sidenote: as anyone else on the LH team been able to repro the [protocol timeouts on byte/perf smokes](https://github.com/GoogleChrome/lighthouse/pull/12807/checks?check_run_id=3115532546#step:9:140)?

it still fairly frequently fails with a retry and I can't reproduce locally, in docker, or on my very slow windows testing box.



Patch:
diff --git a/.github/workflows/devtools.yml b/.github/workflows/devtools.yml
index dcce705d0..f18415141 100644
--- a/.github/workflows/devtools.yml
+++ b/.github/workflows/devtools.yml
@@ -33,7 +33,7 @@ jobs:
           ${{ env.BLINK_TOOLS_PATH }}
           ${{ github.workspace }}/lighthouse/.tmp/chromium-web-tests/content-shells
         # The number is how many times this hash key was manually updated to break the cache.
-        key: ${{ runner.os }}-2-${{ hashFiles('third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/*.js', 'lighthouse/.github/workflows/devtools.yml', 'lighthouse/lighthouse-core/test/chromium-web-tests/*', 'lighthouse/clients/devtools-entry.js', 'lighthouse/clients/devtools-report-assets.js', 'lighthouse/build/build-bundle.js', 'lighthouse/build/build-dt-report-resources.js') }}
+        key: ${{ runner.os }}-2-${{ hashFiles('lighthouse/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/*.js', 'lighthouse/.github/workflows/devtools.yml', 'lighthouse/lighthouse-core/test/chromium-web-tests/*', 'lighthouse/clients/devtools-entry.js', 'lighthouse/clients/devtools-report-assets.js', 'lighthouse/build/build-bundle.js', 'lighthouse/build/build-dt-report-resources.js') }}
 
     - name: Use Node.js 12.x
       uses: actions/setup-node@v1
diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index 49756c9fd..aab061061 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -108,6 +108,11 @@ function getFlags(manualArgv, options = {}) {
           default: false,
           describe: 'Print the normalized config for the given config and options, then exit.',
         },
+        'fraggle-rock': {
+          type: 'boolean',
+          default: false,
+          describe: '[EXPERIMENTAL] Use the new Fraggle Rock navigation runner to gather results.',
+        },
         'additional-trace-categories': {
           type: 'string',
           describe: 'Additional categories to capture with the trace (comma-delimited).',
diff --git a/lighthouse-cli/run.js b/lighthouse-cli/run.js
index a0898ec09..0c780e610 100644
--- a/lighthouse-cli/run.js
+++ b/lighthouse-cli/run.js
@@ -195,6 +195,25 @@ async function potentiallyKillChrome(launchedChrome) {
   });
 }
 
+/**
+ * @param {string} url
+ * @param {LH.CliFlags} flags
+ * @param {LH.Config.Json|undefined} config
+ * @param {ChromeLauncher.LaunchedChrome} launchedChrome
+ * @return {Promise<LH.RunnerResult|undefined>}
+ */
+async function runLighthouseWithFraggleRock(url, flags, config, launchedChrome) {
+  const fraggleRock = require('../lighthouse-core/fraggle-rock/api.js');
+  const puppeteer = require('puppeteer');
+  // @ts-expect-error - FIXME: lighthouse-logger is greedy and takes over `debug` for all packages
+  require('debug').enable('-puppeteer:*');
+  const browser = await puppeteer.connect({browserURL: `http://localhost:${launchedChrome.port}`});
+  const page = await browser.newPage();
+  flags.channel = 'fraggle-rock-cli';
+  const configContext = {configPath: flags.configPath, settingsOverrides: flags};
+  return fraggleRock.navigation({url, page, config, configContext});
+}
+
 /**
  * @param {string} url
  * @param {LH.CliFlags} flags
@@ -223,7 +242,9 @@ async function runLighthouse(url, flags, config) {
       flags.port = launchedChrome.port;
     }
 
-    const runnerResult = await lighthouse(url, flags, config);
+    const runnerResult = flags.fraggleRock && launchedChrome ?
+       await runLighthouseWithFraggleRock(url, flags, config, launchedChrome) :
+       await lighthouse(url, flags, config);
 
     // If in gatherMode only, there will be no runnerResult.
     if (runnerResult) {
diff --git a/lighthouse-core/fraggle-rock/config/config.js b/lighthouse-core/fraggle-rock/config/config.js
index db07e6a28..2d56c0cc2 100644
--- a/lighthouse-core/fraggle-rock/config/config.js
+++ b/lighthouse-core/fraggle-rock/config/config.js
@@ -163,7 +163,7 @@ function resolveNavigationsToDefns(navigations, artifactDefns) {
 
 /**
  * @param {LH.Config.Json|undefined} configJSON
- * @param {{gatherMode: LH.Gatherer.GatherMode, configPath?: string, settingsOverrides?: LH.SharedFlagsSettings}} context
+ * @param {Omit<LH.Config.FRContext, 'gatherMode'> & {gatherMode: LH.Gatherer.GatherMode}} context
  * @return {{config: LH.Config.FRConfig, warnings: string[]}}
  */
 function initializeConfig(configJSON, context) {
diff --git a/lighthouse-core/fraggle-rock/gather/navigation-runner.js b/lighthouse-core/fraggle-rock/gather/navigation-runner.js
index 9aa04b312..e7da3d17a 100644
--- a/lighthouse-core/fraggle-rock/gather/navigation-runner.js
+++ b/lighthouse-core/fraggle-rock/gather/navigation-runner.js
@@ -239,12 +239,12 @@ async function _cleanup({requestedUrl, driver, config}) {
 }
 
 /**
- * @param {{url: string, page: import('puppeteer').Page, config?: LH.Config.Json}} options
+ * @param {{url: string, page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
  * @return {Promise<LH.RunnerResult|undefined>}
  */
 async function navigation(options) {
-  const {url: requestedUrl, page} = options;
-  const {config} = initializeConfig(options.config, {gatherMode: 'navigation'});
+  const {url: requestedUrl, page, configContext = {}} = options;
+  const {config} = initializeConfig(options.config, {...configContext, gatherMode: 'navigation'});
   const computedCache = new Map();
 
   return Runner.run(
diff --git a/lighthouse-core/fraggle-rock/gather/snapshot-runner.js b/lighthouse-core/fraggle-rock/gather/snapshot-runner.js
index be0b7f139..35c3916ae 100644
--- a/lighthouse-core/fraggle-rock/gather/snapshot-runner.js
+++ b/lighthouse-core/fraggle-rock/gather/snapshot-runner.js
@@ -15,9 +15,10 @@ const {
 const {initializeConfig} = require('../config/config.js');
 const {getBaseArtifacts, finalizeArtifacts} = require('./base-artifacts.js');
 
-/** @param {{page: import('puppeteer').Page, config?: LH.Config.Json}} options */
+/** @param {{page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options */
 async function snapshot(options) {
-  const {config} = initializeConfig(options.config, {gatherMode: 'snapshot'});
+  const {configContext = {}} = options;
+  const {config} = initializeConfig(options.config, {...configContext, gatherMode: 'snapshot'});
   const driver = new Driver(options.page);
   await driver.connect();
 
diff --git a/lighthouse-core/fraggle-rock/gather/timespan-runner.js b/lighthouse-core/fraggle-rock/gather/timespan-runner.js
index 9fbb68276..2983aabf2 100644
--- a/lighthouse-core/fraggle-rock/gather/timespan-runner.js
+++ b/lighthouse-core/fraggle-rock/gather/timespan-runner.js
@@ -16,11 +16,12 @@ const {initializeConfig} = require('../config/config.js');
 const {getBaseArtifacts, finalizeArtifacts} = require('./base-artifacts.js');
 
 /**
- * @param {{page: import('puppeteer').Page, config?: LH.Config.Json}} options
+ * @param {{page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
  * @return {Promise<{endTimespan(): Promise<LH.RunnerResult|undefined>}>}
  */
 async function startTimespan(options) {
-  const {config} = initializeConfig(options.config, {gatherMode: 'timespan'});
+  const {configContext = {}} = options;
+  const {config} = initializeConfig(options.config, {...configContext, gatherMode: 'timespan'});
   const driver = new Driver(options.page);
   await driver.connect();
 
diff --git a/lighthouse-logger/index.js b/lighthouse-logger/index.js
index 7ae6ea5a5..f4f02c6be 100644
--- a/lighthouse-logger/index.js
+++ b/lighthouse-logger/index.js
@@ -61,6 +61,7 @@ class Log {
   }
 
   static loggerfn(title) {
+    title = `LH:${title}`;
     let log = loggersByTitle[title];
     if (!log) {
       log = debug(title);
@@ -82,16 +83,16 @@ class Log {
     level_ = level;
     switch (level) {
       case 'silent':
-        debug.enable('-*');
+        debug.enable('-LH:*');
         break;
       case 'verbose':
-        debug.enable('*');
+        debug.enable('LH:*');
         break;
       case 'error':
-        debug.enable('-*, *:error');
+        debug.enable('-LH:*, LH:*:error');
         break;
       default:
-        debug.enable('*, -*:verbose');
+        debug.enable('LH:*, -LH:*:verbose');
     }
   }
 
diff --git a/lighthouse-logger/package.json b/lighthouse-logger/package.json
index 41d4d0249..2456b62b7 100644
--- a/lighthouse-logger/package.json
+++ b/lighthouse-logger/package.json
@@ -1,6 +1,6 @@
 {
   "name": "lighthouse-logger",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "license": "Apache-2.0",
   "dependencies": {
     "debug": "^2.6.9",
diff --git a/types/config.d.ts b/types/config.d.ts
index d5038979c..98bcfa764 100644
--- a/types/config.d.ts
+++ b/types/config.d.ts
@@ -56,6 +56,16 @@ declare global {
         groups: Record<string, Group> | null;
       }
 
+      /**
+       * Additional information about the context in which a Fraggle Rock config should be interpreted.
+       * This information is typically set by the CLI or other channel integrations.
+       */
+      export interface FRContext {
+        gatherMode?: LH.Gatherer.GatherMode;
+        configPath?: string;
+        settingsOverrides?: LH.SharedFlagsSettings;
+      }
+
       interface SharedPassNavigationJson {
         /**
          * Controls the behavior when the navigation fails to complete (due to server error, no FCP, etc).
diff --git a/types/externs.d.ts b/types/externs.d.ts
index 86c13994f..bb195a163 100644
--- a/types/externs.d.ts
+++ b/types/externs.d.ts
@@ -257,6 +257,8 @@ declare global {
       quiet: boolean;
       /** A flag to print the normalized config for the given config and options, then exit. */
       printConfig: boolean;
+      /** Use the new Fraggle Rock navigation runner to gather CLI results. */
+      fraggleRock: boolean;
       /** Path to the file where precomputed lantern data should be read from. */
       precomputedLanternDataPath?: string;
       /** Path to the file where precomputed lantern data should be written to. */


Test patch:
diff --git a/lighthouse-cli/test/cli/bin-test.js b/lighthouse-cli/test/cli/bin-test.js
index 219c46083..4df8ac052 100644
--- a/lighthouse-cli/test/cli/bin-test.js
+++ b/lighthouse-cli/test/cli/bin-test.js
@@ -52,6 +52,7 @@ beforeEach(() => {
     view: false,
     verbose: false,
     quiet: false,
+    fraggleRock: false,
     port: 0,
     hostname: '',
     // Command modes
diff --git a/lighthouse-core/test/fraggle-rock/gather/navigation-runner-test.js b/lighthouse-core/test/fraggle-rock/gather/navigation-runner-test.js
index 4a4ef5f54..5cb204f34 100644
--- a/lighthouse-core/test/fraggle-rock/gather/navigation-runner-test.js
+++ b/lighthouse-core/test/fraggle-rock/gather/navigation-runner-test.js
@@ -7,15 +7,21 @@
 
 /* eslint-env jest */
 
-const {createMockDriver, mockDriverSubmodules} = require('./mock-driver.js');
+const {createMockDriver, mockDriverSubmodules, mockRunnerModule} = require('./mock-driver.js');
 const mocks = mockDriverSubmodules();
-const runner = require('../../../fraggle-rock/gather/navigation-runner.js');
 const {initializeConfig} = require('../../../fraggle-rock/config/config.js');
 const {defaultNavigationConfig} = require('../../../config/constants.js');
 const LighthouseError = require('../../../lib/lh-error.js');
 const DevtoolsLogGatherer = require('../../../gather/gatherers/devtools-log.js');
 const toDevtoolsLog = require('../../network-records-to-devtools-log.js');
 
+// Establish the mocks before we require our file under test.
+let mockRunnerRun = jest.fn();
+
+jest.mock('../../../runner.js', () => mockRunnerModule(() => mockRunnerRun));
+
+const runner = require('../../../fraggle-rock/gather/navigation-runner.js');
+
 /** @typedef {{meta: LH.Gatherer.GathererMeta<'Accessibility'>, getArtifact: jest.Mock<any, any>, startInstrumentation:jest.Mock<any, any>, stopInstrumentation: jest.Mock<any, any>, startSensitiveInstrumentation:jest.Mock<any, any>, stopSensitiveInstrumentation: jest.Mock<any, any>}} MockGatherer */
 
 describe('NavigationRunner', () => {
@@ -79,6 +85,7 @@ describe('NavigationRunner', () => {
 
   beforeEach(() => {
     requestedUrl = 'http://example.com';
+    mockRunnerRun = jest.fn();
     config = initializeConfig(undefined, {gatherMode: 'navigation'}).config;
     navigation = createNavigation().navigation;
     computedCache = new Map();
@@ -381,4 +388,27 @@ describe('NavigationRunner', () => {
       expect(mocks.storageMock.clearDataForOrigin).not.toHaveBeenCalled();
     });
   });
+
+  describe('navigation', () => {
+    it('should initialize config', async () => {
+      const settingsOverrides = {
+        formFactor: /** @type {'desktop'} */ ('desktop'),
+        maxWaitForLoad: 1234,
+        screenEmulation: {mobile: false},
+      };
+
+      const configContext = {settingsOverrides};
+      await runner.navigation({
+        url: 'http://example.com',
+        page: mockDriver._page.asPage(),
+        configContext,
+      });
+
+      expect(mockRunnerRun.mock.calls[0][1]).toMatchObject({
+        config: {
+          settings: settingsOverrides,
+        },
+      });
+    });
+  });
 });
diff --git a/lighthouse-core/test/fraggle-rock/gather/snapshot-runner-test.js b/lighthouse-core/test/fraggle-rock/gather/snapshot-runner-test.js
index 8c9f7afa0..aedba45df 100644
--- a/lighthouse-core/test/fraggle-rock/gather/snapshot-runner-test.js
+++ b/lighthouse-core/test/fraggle-rock/gather/snapshot-runner-test.js
@@ -89,6 +89,24 @@ describe('Snapshot Runner', () => {
     expect(gathererB.getArtifact).toHaveBeenCalled();
   });
 
+
+  it('should use configContext', async () => {
+    const settingsOverrides = {
+      formFactor: /** @type {'desktop'} */ ('desktop'),
+      maxWaitForLoad: 1234,
+      screenEmulation: {mobile: false},
+    };
+
+    const configContext = {settingsOverrides};
+    await snapshot({page, config, configContext});
+
+    expect(mockRunnerRun.mock.calls[0][1]).toMatchObject({
+      config: {
+        settings: settingsOverrides,
+      },
+    });
+  });
+
   it('should not invoke instrumentation methods', async () => {
     await snapshot({page, config});
     await mockRunnerRun.mock.calls[0][0]();
diff --git a/lighthouse-core/test/fraggle-rock/gather/timespan-runner-test.js b/lighthouse-core/test/fraggle-rock/gather/timespan-runner-test.js
index 8706dd94d..3728bf8d7 100644
--- a/lighthouse-core/test/fraggle-rock/gather/timespan-runner-test.js
+++ b/lighthouse-core/test/fraggle-rock/gather/timespan-runner-test.js
@@ -98,6 +98,24 @@ describe('Timespan Runner', () => {
     });
   });
 
+  it('should use configContext', async () => {
+    const settingsOverrides = {
+      formFactor: /** @type {'desktop'} */ ('desktop'),
+      maxWaitForLoad: 1234,
+      screenEmulation: {mobile: false},
+    };
+
+    const configContext = {settingsOverrides};
+    const timespan = await startTimespan({page, config, configContext});
+    await timespan.endTimespan();
+
+    expect(mockRunnerRun.mock.calls[0][1]).toMatchObject({
+      config: {
+        settings: settingsOverrides,
+      },
+    });
+  });
+
   it('should invoke stop instrumentation', async () => {
     const timespan = await startTimespan({page, config});
     await timespan.endTimespan();
diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-view-trace-run.js b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-view-trace-run.js
index 5264bc578..ab01fd449 100644
--- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-view-trace-run.js
+++ b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-view-trace-run.js
@@ -33,7 +33,7 @@
   });
   const viewTraceButton = LighthouseTestRunner.getResultsElement().querySelector('.lh-button--trace');
   TestRunner.addResult(`\nView Trace Button Text: "${viewTraceButton.textContent}"`);
-  TestRunner.addResult(`View Trace Button Title: "${UI.Tooltip.getContent(viewTraceButton)}"`);
+  TestRunner.addResult(`View Trace Button Title: "${viewTraceButton.title}"`);
   viewTraceButton.click();
   const viewShown = await waitForShowView;
   TestRunner.addResult(`\nShowing view: ${viewShown}`);