Repository: GoogleChrome/lighthouse
Pull request #12359 (2021-04-13 20:22:36)

tests: remove flaky Chrome launch from unit-cli
In the spirit of improving flaky tests, `unit-cli` has loooong had a flake based on a Chrome launch sometimes timing out. You know the one:

```
●  process.exit called with "1"

      84 | function printConnectionErrorAndExit() {
      85 |   console.error('Unable to connect to Chrome');
    > 86 |   return process.exit(_RUNTIME_ERROR_CODE);
         |                  ^
      87 | }
      88 | 
      89 | /** @return {never} */

      at printConnectionErrorAndExit (lighthouse-cli/run.js:86:18)
```

(for example: https://github.com/GoogleChrome/lighthouse/runs/2249363555?check_suite_focus=true)

The issue is that this is more or less a smoke test without any of the functionality we've built into smokehouse to handle dealing with launching and testing with Chrome in a CI environment.

Looking way back, the original point of the test (from #2602) was to make sure that Lighthouse can run when `lighthouse-cli` is accessed directly as a node module, since our smoke tests load Lighthouse via CLI. Very similar to the smoke tests, but we have no other test ensuring we don't break this particular use case. Since then, the assertions have grown to check a few other aspects of the run.

However the overlap between this and smokehouse's execution path is almost total (they both pass through `runLighthouse` in `run.js`), so it seems sufficient for this test to run lighthouse with `--auditMode` and the sample artifacts to make sure it's runnable, and trust smokehouse to verify `run.js` is exercising the gathering steps.

After this I believe there's no other Chrome launch in `unit-cli`, so it should be stable going forward.

connorjclark: want to also revert the `run: yarn unit-cli || yarn unit-cli --onlyFailures` bit?
brendankenny: > want to also revert the `run: yarn unit-cli || yarn unit-cli --onlyFailures` bit?

I like that confidence :) We can see how it goes

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 37c6d3602..f1ae3f5b8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -127,7 +127,7 @@ jobs:
 
     # For windows, just test the potentially platform-specific code.
     - name: yarn unit-cli
-      run: yarn unit-cli || yarn unit-cli --onlyFailures
+      run: yarn unit-cli
       if: matrix.os == 'windows-latest'
     - run: yarn diff:sample-json
       if: matrix.os == 'windows-latest'


Test patch:
diff --git a/lighthouse-cli/test/cli/run-test.js b/lighthouse-cli/test/cli/run-test.js
index 89e5acf4a..06c8aae5c 100644
--- a/lighthouse-cli/test/cli/run-test.js
+++ b/lighthouse-cli/test/cli/run-test.js
@@ -12,10 +12,11 @@ const fs = require('fs');
 
 const run = require('../../run.js');
 const parseChromeFlags = require('../../run.js').parseChromeFlags;
-const fastConfig = {
+/** @type {LH.Config.Json} */
+const testConfig = {
   'extends': 'lighthouse:default',
   'settings': {
-    'onlyAudits': ['viewport'],
+    'throttlingMethod': 'devtools',
   },
 };
 
@@ -28,7 +29,7 @@ jest.mock('lighthouse-plugin-simple', () => {
 const getFlags = require('../../cli-flags.js').getFlags;
 
 describe('CLI run', function() {
-  describe('LH round trip', () => {
+  describe('runLighthouse runs Lighthouse as a node module', () => {
     /** @type {LH.RunnerResult} */
     let passedResults;
     const filename = path.join(process.cwd(), 'run.ts.results.json');
@@ -36,14 +37,21 @@ describe('CLI run', function() {
     let fileResults;
 
     beforeAll(async () => {
-      const url = 'chrome://version';
-      const timeoutFlag = `--max-wait-for-load=${9000}`;
-      const pluginsFlag = '--plugins=lighthouse-plugin-simple';
+      const url = 'http://localhost:10200/dobetterweb/dbw_tester.html';
+      // eslint-disable-next-line max-len
+      const samplev2ArtifactsPath = __dirname + '/../../../lighthouse-core/test/results/artifacts/';
 
       // eslint-disable-next-line max-len
-      const flags = getFlags(`--output=json --output-path=${filename} ${pluginsFlag} ${timeoutFlag} ${url}`);
+      const flags = getFlags([
+        '--output=json',
+        `--output-path=${filename}`,
+        '--plugins=lighthouse-plugin-simple',
+        // Use sample artifacts to avoid gathering during a unit test.
+        `--audit-mode=${samplev2ArtifactsPath}`,
+        url,
+      ].join(' '));
 
-      const rawResult = await run.runLighthouse(url, flags, fastConfig);
+      const rawResult = await run.runLighthouse(url, flags, testConfig);
 
       if (!rawResult) {
         return assert.fail('no results');
@@ -60,7 +68,7 @@ describe('CLI run', function() {
 
     it('returns results that match the saved results', () => {
       const {lhr} = passedResults;
-      assert.equal(fileResults.audits.viewport.score, 0);
+      assert.equal(fileResults.audits.viewport.score, 1);
 
       // passed results match saved results
       assert.strictEqual(fileResults.fetchTime, lhr.fetchTime);