Repository: GoogleChrome/lighthouse
Pull request #6397 (2018-10-26 03:35:18)

new_audit: offline-start-url
Part of reorganization of the PWA category (#6395).

This splits the `start_url` offline check out of `webapp-install-banner.js` and into its own audit.

Based on top of #6396, so you may want to look at the diff between the two: https://github.com/GoogleChrome/lighthouse/compare/pwacat...start_url

brendankenny: rebased off master
brendankenny: > looks good to me outside i18n question! boy it is going to be so hard to remember all the various states of checking across LH versions

yeah, I don't envy anyone doing that :(

Patch:
diff --git a/lighthouse-core/audits/multi-check-audit.js b/lighthouse-core/audits/multi-check-audit.js
index fb8782799..35e62a59f 100644
--- a/lighthouse-core/audits/multi-check-audit.js
+++ b/lighthouse-core/audits/multi-check-audit.js
@@ -23,7 +23,7 @@ class MultiCheckAudit extends Audit {
   }
 
   /**
-   * @param {{failures: Array<string>, warnings?: Array<string>, manifestValues?: LH.Artifacts.ManifestValues}} result
+   * @param {{failures: Array<string>, manifestValues?: LH.Artifacts.ManifestValues}} result
    * @return {LH.Audit.Product}
    */
   static createAuditProduct(result) {
@@ -57,7 +57,6 @@ class MultiCheckAudit extends Audit {
     return {
       rawValue: true,
       details,
-      warnings: result.warnings,
     };
   }
 
@@ -66,7 +65,7 @@ class MultiCheckAudit extends Audit {
   /**
    * @param {LH.Artifacts} artifacts
    * @param {LH.Audit.Context} context
-   * @return {Promise<{failures: Array<string>, warnings?: Array<string>, manifestValues?: LH.Artifacts.ManifestValues}>}
+   * @return {Promise<{failures: Array<string>, manifestValues?: LH.Artifacts.ManifestValues}>}
    */
   static audit_(artifacts, context) {
     throw new Error('audit_ unimplemented');
diff --git a/lighthouse-core/audits/offline-start-url.js b/lighthouse-core/audits/offline-start-url.js
new file mode 100644
index 000000000..fb6f0b2c8
--- /dev/null
+++ b/lighthouse-core/audits/offline-start-url.js
@@ -0,0 +1,49 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Audit = require('./audit.js');
+
+class OfflineStartUrl extends Audit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return {
+      id: 'offline-start-url',
+      title: 'start_url responds with a 200 when offline',
+      failureTitle: 'start_url does not respond with a 200 when offline',
+      description: 'A service worker enables your web app to be reliable in unpredictable network conditions. [Learn more](https://developers.google.com/web/tools/lighthouse/audits/http-200-when-offline).',
+      requiredArtifacts: ['Manifest', 'StartUrl'],
+    };
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @return {LH.Audit.Product}
+   */
+  static audit(artifacts) {
+    // StartUrl gatherer will give explanations for failures, but need to take manifest parsing
+    // warnings from the manifest itself (e.g. invalid `start_url`, so fell back to document URL).
+    const warnings = [];
+    const manifest = artifacts.Manifest;
+    if (manifest && manifest.value && manifest.value.start_url.warning) {
+      const manifestWarning = manifest.value.start_url.warning;
+      warnings.push('We couldn\'t read the start_url from the manifest. As a result, the ' +
+          `start_url was assumed to be the document's URL. Error message: '${manifestWarning}'.`);
+    }
+
+    const hasOfflineStartUrl = artifacts.StartUrl.statusCode === 200;
+
+    return {
+      rawValue: hasOfflineStartUrl,
+      explanation: artifacts.StartUrl.explanation,
+      warnings,
+    };
+  }
+}
+
+module.exports = OfflineStartUrl;
diff --git a/lighthouse-core/audits/webapp-install-banner.js b/lighthouse-core/audits/webapp-install-banner.js
index 804238fb8..e9020a368 100644
--- a/lighthouse-core/audits/webapp-install-banner.js
+++ b/lighthouse-core/audits/webapp-install-banner.js
@@ -41,7 +41,7 @@ class WebappInstallBanner extends MultiCheckAudit {
       description: 'Browsers can proactively prompt users to add your app to their homescreen, ' +
           'which can lead to higher engagement. ' +
           '[Learn more](https://developers.google.com/web/tools/lighthouse/audits/install-prompt).',
-      requiredArtifacts: ['URL', 'ServiceWorker', 'Manifest', 'StartUrl'],
+      requiredArtifacts: ['URL', 'ServiceWorker', 'Manifest'],
     };
   }
 
@@ -94,55 +94,20 @@ class WebappInstallBanner extends MultiCheckAudit {
     return failures;
   }
 
-  /**
-   * @param {LH.Artifacts} artifacts
-   * @return {{failures: Array<string>, warnings: Array<string>}}
-   */
-  static assessOfflineStartUrl(artifacts) {
-    const failures = [];
-    const warnings = [];
-    const hasOfflineStartUrl = artifacts.StartUrl.statusCode === 200;
-
-    if (!hasOfflineStartUrl) {
-      failures.push('Service worker does not successfully serve the manifest\'s start_url');
-      if (artifacts.StartUrl.explanation) {
-        failures.push(artifacts.StartUrl.explanation);
-      }
-    }
-
-    if (artifacts.StartUrl.explanation) {
-      warnings.push(artifacts.StartUrl.explanation);
-    }
-
-    return {failures, warnings};
-  }
-
   /**
    * @param {LH.Artifacts} artifacts
    * @param {LH.Audit.Context} context
-   * @return {Promise<{failures: Array<string>, warnings: Array<string>, manifestValues: LH.Artifacts.ManifestValues}>}
+   * @return {Promise<{failures: Array<string>, manifestValues: LH.Artifacts.ManifestValues}>}
    */
   static async audit_(artifacts, context) {
-    /** @type {Array<string>} */
-    let offlineFailures = [];
-    /** @type {Array<string>} */
-    let offlineWarnings = [];
-
     const manifestValues = await ManifestValues.request(artifacts.Manifest, context);
     const manifestFailures = WebappInstallBanner.assessManifest(manifestValues);
     const swFailures = WebappInstallBanner.assessServiceWorker(artifacts);
-    if (!swFailures.length) {
-      const {failures, warnings} = WebappInstallBanner.assessOfflineStartUrl(artifacts);
-      offlineFailures = failures;
-      offlineWarnings = warnings;
-    }
 
     return {
-      warnings: offlineWarnings,
       failures: [
         ...manifestFailures,
         ...swFailures,
-        ...offlineFailures,
       ],
       manifestValues,
     };
diff --git a/lighthouse-core/config/default-config.js b/lighthouse-core/config/default-config.js
index d26faa2f7..8ddbf24b2 100644
--- a/lighthouse-core/config/default-config.js
+++ b/lighthouse-core/config/default-config.js
@@ -132,6 +132,7 @@ const defaultConfig = {
     'font-display',
     'network-requests',
     'metrics',
+    'offline-start-url',
     'manual/pwa-cross-browser',
     'manual/pwa-page-transitions',
     'manual/pwa-each-page-has-url',
@@ -334,10 +335,11 @@ const defaultConfig = {
         // Fast and Reliable
         {id: 'load-fast-enough-for-pwa', weight: 7, group: 'pwa-fast-reliable'},
         {id: 'works-offline', weight: 5, group: 'pwa-fast-reliable'},
+        {id: 'offline-start-url', weight: 1, group: 'pwa-fast-reliable'},
         // Installable
         {id: 'is-on-https', weight: 2, group: 'pwa-installable'},
         {id: 'service-worker', weight: 1, group: 'pwa-installable'},
-        {id: 'webapp-install-banner', weight: 3, group: 'pwa-installable'},
+        {id: 'webapp-install-banner', weight: 2, group: 'pwa-installable'},
         // Engaging
         {id: 'redirects-http', weight: 2, group: 'pwa-engaging'},
         {id: 'splash-screen', weight: 1, group: 'pwa-engaging'},
diff --git a/lighthouse-core/gather/gatherers/start-url.js b/lighthouse-core/gather/gatherers/start-url.js
index 9d7c59ec7..1d42543ac 100644
--- a/lighthouse-core/gather/gatherers/start-url.js
+++ b/lighthouse-core/gather/gatherers/start-url.js
@@ -7,7 +7,8 @@
 
 const Gatherer = require('./gatherer');
 const manifestParser = require('../../lib/manifest-parser');
-const Driver = require('../driver.js'); // eslint-disable-line no-unused-vars
+
+/** @typedef {import('../driver.js')} Driver */
 
 class StartUrl extends Gatherer {
   /**
@@ -29,13 +30,13 @@ class StartUrl extends Gatherer {
 
         return this._attemptManifestFetch(passContext.driver, startUrlInfo.startUrl);
       }).catch(() => {
-        return {statusCode: -1, explanation: 'Unable to fetch start URL via service worker'};
+        return {statusCode: -1, explanation: 'Unable to fetch start URL via service worker.'};
       });
   }
 
   /**
    * Read the parsed manifest and return failure reasons or the startUrl
-   * @param {?{value?: {start_url: {value?: string, warning?: string}}, warning?: string}} manifest
+   * @param {?{value?: {start_url: {value: string, warning?: string}}, warning?: string}} manifest
    * @return {{isReadFailure: true, reason: string}|{isReadFailure: false, startUrl: string}}
    */
   _readManifestStartUrl(manifest) {
@@ -43,19 +44,13 @@ class StartUrl extends Gatherer {
       const detailedMsg = manifest && manifest.warning;
 
       if (detailedMsg) {
-        return {isReadFailure: true, reason: `Error fetching web app manifest: ${detailedMsg}`};
+        return {isReadFailure: true, reason: `Error fetching web app manifest: ${detailedMsg}.`};
       } else {
-        return {isReadFailure: true, reason: `No usable web app manifest found on page`};
+        return {isReadFailure: true, reason: `No usable web app manifest found on page.`};
       }
     }
 
-    // Even if the start URL had an error, the browser will still supply a fallback URL.
-    // Therefore, we only set the warning here and continue with the fetch.
-    if (manifest.value.start_url.warning) {
-      return {isReadFailure: true, reason: manifest.value.start_url.warning};
-    }
-
-    // @ts-ignore - TODO(bckenny): should actually be testing value above, not warning
+    // Even if the start URL had a parser warning, the browser will still supply a fallback URL.
     return {isReadFailure: false, startUrl: manifest.value.start_url.value};
   }
 
@@ -70,7 +65,7 @@ class StartUrl extends Gatherer {
     // Wait up to 3s to get a matched network request from the fetch() to work
     const timeoutPromise = new Promise(resolve =>
       setTimeout(
-        () => resolve({statusCode: -1, explanation: 'Timed out waiting for fetched start_url'}),
+        () => resolve({statusCode: -1, explanation: 'Timed out waiting for fetched start_url.'}),
         3000
       )
     );
@@ -88,7 +83,7 @@ class StartUrl extends Gatherer {
         if (!response.fromServiceWorker) {
           return resolve({
             statusCode: -1,
-            explanation: 'Unable to fetch start URL via service worker',
+            explanation: 'Unable to fetch start URL via service worker.',
           });
         }
         // Successful SW-served fetch of the start_URL
diff --git a/lighthouse-core/lib/manifest-parser.js b/lighthouse-core/lib/manifest-parser.js
index 7468583b8..34262c1a7 100644
--- a/lighthouse-core/lib/manifest-parser.js
+++ b/lighthouse-core/lib/manifest-parser.js
@@ -115,6 +115,7 @@ function checkSameOrigin(url1, url2) {
  * @param {*} jsonInput
  * @param {string} manifestUrl
  * @param {string} documentUrl
+ * @return {{raw: any, value: string, warning?: string}}
  */
 function parseStartUrl(jsonInput, manifestUrl, documentUrl) {
   const raw = jsonInput.start_url;
@@ -127,10 +128,18 @@ function parseStartUrl(jsonInput, manifestUrl, documentUrl) {
       warning: 'ERROR: start_url string empty',
     };
   }
-  const parsedAsString = parseString(raw);
-  if (!parsedAsString.value) {
-    parsedAsString.value = documentUrl;
-    return parsedAsString;
+  if (raw === undefined) {
+    return {
+      raw,
+      value: documentUrl,
+    };
+  }
+  if (typeof raw !== 'string') {
+    return {
+      raw,
+      value: documentUrl,
+      warning: 'ERROR: expected a string.',
+    };
   }
 
   // 8.10(4) - construct URL with raw as input and manifestUrl as the base.
diff --git a/lighthouse-core/lib/proto-preprocessor.js b/lighthouse-core/lib/proto-preprocessor.js
index 721a488d1..454037f73 100644
--- a/lighthouse-core/lib/proto-preprocessor.js
+++ b/lighthouse-core/lib/proto-preprocessor.js
@@ -23,13 +23,14 @@ function processForProto(result) {
   const reportJson = JSON.parse(result);
 
   // Clean up the configSettings
+  // Note: This is not strictly required for conversion if protobuf parsing is set to
+  // 'ignore unknown fields' in the language of conversion.
   if (reportJson.configSettings) {
-    // make sure the 'output' field is an array
-    if (reportJson.configSettings.output) {
-      if (!Array.isArray(reportJson.configSettings.output)) {
-        reportJson.configSettings.output = [reportJson.configSettings.output];
-      }
-    }
+    // The settings that are in both proto and LHR
+    const {emulatedFormFactor, locale, onlyCategories} = reportJson.configSettings;
+
+    // @ts-ignore - intentionally only a subset of settings.
+    reportJson.configSettings = {emulatedFormFactor, locale, onlyCategories};
   }
 
   // Remove runtimeError if it is NO_ERROR
diff --git a/proto/lighthouse-result.proto b/proto/lighthouse-result.proto
index e4e0f4b1a..befb32303 100644
--- a/proto/lighthouse-result.proto
+++ b/proto/lighthouse-result.proto
@@ -111,64 +111,6 @@ message LighthouseResult {
 
   // Message containing the configuration settings for the LH run
   message ConfigSettings {
-    // The output types the audit made (json, html...)
-    repeated string output = 1;
-
-    // The maximum amount of time to wait for loading in ms
-    google.protobuf.DoubleValue max_wait_for_load = 2;
-
-    // Network throttling options
-    enum ThrottlingMethod {
-      // Unknown method.  Should not be used
-      UNKNOWN_THROTTLING_METHOD = 0;
-
-      // Use devtools to throttle the request
-      devtools = 1;
-
-      // Use no additional throttling (only throttling provided by system
-      // itself)
-      provided = 2;
-
-      // Simulate throttling with Lantern
-      simulate = 3;
-    }
-
-    // The throttling method used during this audit
-    ThrottlingMethod throttling_method = 3;
-
-    // Message containing the throttling settings used in an LH configuration
-    message Throttling {
-      // The round trip time in ms
-      google.protobuf.DoubleValue rtt_ms = 1;
-
-      // The throughput in kilobytes per second
-      google.protobuf.DoubleValue throughput_kbps = 2;
-
-      // The request latency in milliseconds
-      google.protobuf.DoubleValue request_latency_ms = 3;
-
-      // The download throughput in kilobytes per second
-      google.protobuf.DoubleValue download_throughput_kbps = 4;
-
-      // The upload throughput in kilobytes per second
-      google.protobuf.DoubleValue upload_throughput_kbps = 5;
-
-      // The amount of slowdown to apply to the cpu
-      google.protobuf.DoubleValue cpu_slowdown_multiplier = 6;
-    }
-
-    // The throttling settings used
-    Throttling throttling = 4;
-
-    // Flag indicating this audit was gather only or not
-    google.protobuf.BoolValue gather_mode = 5;
-
-    // Flag disabling clearing the browser cache before runs
-    google.protobuf.BoolValue disable_storage_reset = 6;
-
-    // Flag indicating if device emulation was enabled or not
-    google.protobuf.BoolValue disable_device_emulation = 7;
-
     // The possible form factors an audit can be run in
     enum EmulatedFormFactor {
       // Unknown form factor.  This should not be used
@@ -185,36 +127,14 @@ message LighthouseResult {
     }
 
     // The form factor used in the audit
-    EmulatedFormFactor emulated_form_factor = 8;
+    EmulatedFormFactor emulated_form_factor = 1;
 
     // The locale that was active during the audit
-    string locale = 9;
-
-    // List of URL patterns that were blocked during this audit
-    // nullable list of strings
-    google.protobuf.Value blocked_url_patterns = 10;
-
-    // Comma delimited list of trace categories to capture
-    // nullable string
-    google.protobuf.Value additional_trace_categories = 11;
-
-    // Map of extra HTTP Headers to pass in with the request
-    google.protobuf.Value extra_headers = 12;
-
-    // List of the audits that were preformed, empty if all were run
-    // nullable list of strings
-    google.protobuf.Value only_audits = 13;
+    string locale = 2;
 
     // List of the categories that were run, empty if all were run
     // nullable list of strings
-    google.protobuf.Value only_categories = 14;
-
-    // List of the audits that were skipped, empty if all were run
-    // nullable list of strings
-    google.protobuf.Value skip_audits = 15;
-
-    // Flag indicating whether this Lighthouse run was audit-only
-    google.protobuf.BoolValue audit_mode = 16;
+    google.protobuf.Value only_categories = 3;
   }
 
   // The settings that were used to run this audit
diff --git a/proto/sample_v2_round_trip.json b/proto/sample_v2_round_trip.json
index 3d9910611..0934f7b49 100644
--- a/proto/sample_v2_round_trip.json
+++ b/proto/sample_v2_round_trip.json
@@ -1630,6 +1630,14 @@
             "scoreDisplayMode": "not_applicable", 
             "title": "`<object>` elements have `[alt]` text"
         }, 
+        "offline-start-url": {
+            "description": "A service worker enables your web app to be reliable in unpredictable network conditions. [Learn more](https://developers.google.com/web/tools/lighthouse/audits/http-200-when-offline).", 
+            "id": "offline-start-url", 
+            "score": 0.0, 
+            "scoreDisplayMode": "binary", 
+            "title": "start_url does not respond with a 200 when offline", 
+            "warnings": []
+        }, 
         "offscreen-content-hidden": {
             "description": "Offscreen content is hidden with display: none or aria-hidden=true. [Learn more](https://developers.google.com/web/fundamentals/accessibility/how-to-review#try_it_with_a_screen_reader).", 
             "id": "offscreen-content-hidden", 
@@ -3041,6 +3049,11 @@
                     "id": "works-offline", 
                     "weight": 5.0
                 }, 
+                {
+                    "group": "pwa-fast-reliable", 
+                    "id": "offline-start-url", 
+                    "weight": 1.0
+                }, 
                 {
                     "group": "pwa-installable", 
                     "id": "is-on-https", 
@@ -3054,7 +3067,7 @@
                 {
                     "group": "pwa-installable", 
                     "id": "webapp-install-banner", 
-                    "weight": 3.0
+                    "weight": 2.0
                 }, 
                 {
                     "group": "pwa-engaging", 
@@ -3245,30 +3258,9 @@
         }
     }, 
     "configSettings": {
-        "additionalTraceCategories": null, 
-        "blockedUrlPatterns": null, 
-        "disableDeviceEmulation": false, 
-        "disableStorageReset": false, 
         "emulatedFormFactor": "mobile", 
-        "extraHeaders": null, 
-        "gatherMode": false, 
         "locale": "en-US", 
-        "maxWaitForLoad": 45000.0, 
-        "onlyAudits": null, 
-        "onlyCategories": null, 
-        "output": [
-            "json"
-        ], 
-        "skipAudits": null, 
-        "throttling": {
-            "cpuSlowdownMultiplier": 4.0, 
-            "downloadThroughputKbps": 1474.5600000000002, 
-            "requestLatencyMs": 562.5, 
-            "rttMs": 150.0, 
-            "throughputKbps": 1638.4, 
-            "uploadThroughputKbps": 675.0
-        }, 
-        "throttlingMethod": "devtools"
+        "onlyCategories": null
     }, 
     "environment": {
         "benchmarkIndex": 1000.0, 
@@ -3530,6 +3522,12 @@
                 "name": "lh:audit:metrics", 
                 "startTime": 0.0
             }, 
+            {
+                "duration": 100.0, 
+                "entryType": "measure", 
+                "name": "lh:audit:offline-start-url", 
+                "startTime": 0.0
+            }, 
             {
                 "duration": 100.0, 
                 "entryType": "measure", 


Test patch:
diff --git a/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap
index 2c64d0662..749933aa4 100644
--- a/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap
+++ b/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap
@@ -102,6 +102,9 @@ Object {
     Object {
       "path": "metrics",
     },
+    Object {
+      "path": "offline-start-url",
+    },
     Object {
       "path": "manual/pwa-cross-browser",
     },
@@ -820,6 +823,11 @@ Object {
           "id": "works-offline",
           "weight": 5,
         },
+        Object {
+          "group": "pwa-fast-reliable",
+          "id": "offline-start-url",
+          "weight": 1,
+        },
         Object {
           "group": "pwa-installable",
           "id": "is-on-https",
@@ -833,7 +841,7 @@ Object {
         Object {
           "group": "pwa-installable",
           "id": "webapp-install-banner",
-          "weight": 3,
+          "weight": 2,
         },
         Object {
           "group": "pwa-engaging",
diff --git a/lighthouse-cli/test/smokehouse/pwa-expectations.js b/lighthouse-cli/test/smokehouse/pwa-expectations.js
index e8042e406..dc61ca6b0 100644
--- a/lighthouse-cli/test/smokehouse/pwa-expectations.js
+++ b/lighthouse-cli/test/smokehouse/pwa-expectations.js
@@ -38,6 +38,9 @@ module.exports = [
       'works-offline': {
         score: 1,
       },
+      'offline-start-url': {
+        score: 1,
+      },
       'viewport': {
         score: 1,
       },
@@ -95,6 +98,9 @@ module.exports = [
       'works-offline': {
         score: 0,
       },
+      'offline-start-url': {
+        score: 1,
+      },
       'viewport': {
         score: 1,
       },
diff --git a/lighthouse-cli/test/smokehouse/pwa2-expectations.js b/lighthouse-cli/test/smokehouse/pwa2-expectations.js
index 36be890b3..28857e8bd 100644
--- a/lighthouse-cli/test/smokehouse/pwa2-expectations.js
+++ b/lighthouse-cli/test/smokehouse/pwa2-expectations.js
@@ -32,6 +32,9 @@ module.exports = [
       'works-offline': {
         score: 1,
       },
+      'offline-start-url': {
+        score: 1,
+      },
       'viewport': {
         score: 1,
       },
@@ -89,6 +92,9 @@ module.exports = [
       'works-offline': {
         score: 1,
       },
+      'offline-start-url': {
+        score: 1,
+      },
       'viewport': {
         score: 1,
       },
diff --git a/lighthouse-cli/test/smokehouse/pwa3-expectations.js b/lighthouse-cli/test/smokehouse/pwa3-expectations.js
index f28cdbc5f..6554e9fa3 100644
--- a/lighthouse-cli/test/smokehouse/pwa3-expectations.js
+++ b/lighthouse-cli/test/smokehouse/pwa3-expectations.js
@@ -30,6 +30,9 @@ module.exports = [
       'works-offline': {
         score: 1,
       },
+      'offline-start-url': {
+        score: 1,
+      },
       'viewport': {
         score: 1,
       },
diff --git a/lighthouse-core/test/audits/offline-start-url-test.js b/lighthouse-core/test/audits/offline-start-url-test.js
new file mode 100644
index 000000000..2d5c5597e
--- /dev/null
+++ b/lighthouse-core/test/audits/offline-start-url-test.js
@@ -0,0 +1,73 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const assert = require('assert');
+const OfflineStartUrlAudit = require('../../audits/offline-start-url.js');
+const manifestParser = require('../../lib/manifest-parser.js');
+
+/* eslint-env jest */
+
+function generateMockAuditContext() {
+  return {
+    computedCache: new Map(),
+  };
+}
+function getManifest(manifest) {
+  const documentUrl = 'https://example.com/';
+  const manifestUrl = `${documentUrl}manifest.json`;
+
+  return manifestParser(JSON.stringify(manifest), manifestUrl, documentUrl);
+}
+
+describe('Offline start_url audit', () => {
+  it('fails if start_url is not fetched', async () => {
+    const explanation = 'Unable to fetch start URL via service worker.';
+
+    const artifacts = {
+      StartUrl: {
+        statusCode: -1,
+        explanation,
+      },
+    };
+    const context = generateMockAuditContext();
+
+    const result = await OfflineStartUrlAudit.audit(artifacts, context);
+    assert.strictEqual(result.rawValue, false);
+    assert.strictEqual(result.explanation, explanation);
+    assert.strictEqual(result.warnings.length, 0);
+  });
+
+  it('passes if start_url is fetched', async () => {
+    const artifacts = {
+      StartUrl: {statusCode: 200},
+    };
+    const context = generateMockAuditContext();
+
+    const result = await OfflineStartUrlAudit.audit(artifacts, context);
+    assert.strictEqual(result.rawValue, true);
+    assert.strictEqual(result.explanation, undefined);
+    assert.strictEqual(result.warnings.length, 0);
+  });
+
+  it('warns if there was a manifest start_url parsing error', async () => {
+    const manifest = {
+      start_url: 'https://evil.com/',
+    };
+
+    const artifacts = {
+      Manifest: getManifest(manifest),
+      StartUrl: {statusCode: 200},
+    };
+    const context = generateMockAuditContext();
+
+    const result = await OfflineStartUrlAudit.audit(artifacts, context);
+    assert.strictEqual(result.rawValue, true);
+    assert.strictEqual(result.explanation, undefined);
+    assert.strictEqual(result.warnings.length, 1);
+    assert.ok(result.warnings[0].includes('start_url must be same-origin as document'));
+  });
+});
diff --git a/lighthouse-core/test/audits/webapp-install-banner-test.js b/lighthouse-core/test/audits/webapp-install-banner-test.js
index 1ed204e5e..9e9d4b2b6 100644
--- a/lighthouse-core/test/audits/webapp-install-banner-test.js
+++ b/lighthouse-core/test/audits/webapp-install-banner-test.js
@@ -25,7 +25,6 @@ function generateMockArtifacts(src = manifestSrc) {
         scriptURL: 'https://example.com/sw.js',
       }],
     },
-    StartUrl: {statusCode: 200},
     URL: {finalUrl: 'https://example.com'},
   }));
   return clonedArtifacts;
@@ -151,7 +150,6 @@ describe('PWA: webapp install banner audit', () => {
   it('fails if page had no SW', () => {
     const artifacts = generateMockArtifacts();
     artifacts.ServiceWorker.versions = [];
-    artifacts.StartUrl = {statusCode: -1};
     const context = generateMockAuditContext();
 
     return WebappInstallBannerAudit.audit(artifacts, context).then(result => {
@@ -161,28 +159,4 @@ describe('PWA: webapp install banner audit', () => {
       assert.strictEqual(failures.length, 1, failures);
     });
   });
-
-  it('fails if start_url is not cached', () => {
-    const artifacts = generateMockArtifacts();
-    artifacts.StartUrl = {statusCode: -1};
-    const context = generateMockAuditContext();
-
-    return WebappInstallBannerAudit.audit(artifacts, context).then(result => {
-      assert.strictEqual(result.rawValue, false);
-      assert.ok(result.explanation.includes('start_url'), result.explanation);
-      const failures = result.details.items[0].failures;
-      assert.strictEqual(failures.length, 1, failures);
-    });
-  });
-
-  it('includes warning from start_url', () => {
-    const artifacts = generateMockArtifacts();
-    artifacts.StartUrl = {statusCode: 200, explanation: 'Warning!'};
-    const context = generateMockAuditContext();
-
-    return WebappInstallBannerAudit.audit(artifacts, context).then(result => {
-      assert.strictEqual(result.rawValue, true);
-      assert.equal(result.warnings[0], 'Warning!');
-    });
-  });
 });
diff --git a/lighthouse-core/test/gather/gatherers/start-url-test.js b/lighthouse-core/test/gather/gatherers/start-url-test.js
index 6924a3033..df9b0fd37 100644
--- a/lighthouse-core/test/gather/gatherers/start-url-test.js
+++ b/lighthouse-core/test/gather/gatherers/start-url-test.js
@@ -82,7 +82,7 @@ describe('Start-url gatherer', () => {
       assert.ok(artifactWithQueryString.explanation, 'did not set debug string');
       assert.equal(artifactWithResponseNotFromSW.statusCode, -1);
       assert.equal(artifactWithResponseNotFromSW.explanation,
-          'Unable to fetch start URL via service worker');
+          'Unable to fetch start URL via service worker.');
     });
   });
 
@@ -120,7 +120,7 @@ describe('Start-url gatherer', () => {
     return startUrlGatherer.afterPass(options, tracingData)
       .then(artifact => {
         assert.equal(artifact.explanation,
-          `No usable web app manifest found on page`);
+          `No usable web app manifest found on page.`);
       });
   });
 
@@ -141,11 +141,11 @@ describe('Start-url gatherer', () => {
       .then(artifact => {
         assert.equal(artifact.explanation,
           `Error fetching web app manifest: ERROR: file isn't valid JSON: ` +
-          `SyntaxError: Unexpected token h in JSON at position 1`);
+          `SyntaxError: Unexpected token h in JSON at position 1.`);
       });
   });
 
-  it('returns a explanation when start_url cannot be found', () => {
+  it('times out when a start_url is too slow to respond', () => {
     const startUrlGatherer = new StartUrlGatherer();
     const options = {
       url: 'https://ifixit-pwa.appspot.com/',
@@ -154,20 +154,7 @@ describe('Start-url gatherer', () => {
 
     return startUrlGatherer.afterPass(options, tracingData)
       .then(artifact => {
-        assert.equal(artifact.explanation, 'ERROR: start_url string empty');
-      });
-  });
-
-  it('returns an error when origin is not the same', () => {
-    const startUrlGatherer = new StartUrlGatherer();
-    const options = {
-      url: 'https://ifixit-pwa.appspot.com/',
-      driver: wrapSendCommand(mockDriver, 'https://not-same-origin.com/'),
-    };
-
-    return startUrlGatherer.afterPass(options, tracingData)
-      .then(artifact => {
-        assert.equal(artifact.explanation, 'ERROR: start_url must be same-origin as document');
+        assert.equal(artifact.explanation, 'Timed out waiting for fetched start_url.');
       });
   });
 });
diff --git a/lighthouse-core/test/lib/manifest-parser-test.js b/lighthouse-core/test/lib/manifest-parser-test.js
index 5a0b37c0b..f07d83d66 100644
--- a/lighthouse-core/test/lib/manifest-parser-test.js
+++ b/lighthouse-core/test/lib/manifest-parser-test.js
@@ -190,7 +190,7 @@ describe('Manifest Parser', function() {
   describe('start_url parsing', () => {
     /* eslint-disable camelcase */
     // 8.10(3)
-    it('falls back to document URL and issues a warning for an invalid value', () => {
+    it('falls back to document URL and issues a warning for a non-string', () => {
       const manifestSrc = JSON.stringify({
         start_url: {},
       });
@@ -203,7 +203,7 @@ describe('Manifest Parser', function() {
       assert.equal(parsedUrl.value, docUrl);
     });
 
-    it('falls back to document URL and issues a warning for an invalid value', () => {
+    it('falls back to document URL and issues a warning for a non-string', () => {
       const manifestSrc = JSON.stringify({
         start_url: 6,
       });
diff --git a/lighthouse-core/test/lib/proto-preprocessor-test.js b/lighthouse-core/test/lib/proto-preprocessor-test.js
index cadb0a455..3a76964d6 100644
--- a/lighthouse-core/test/lib/proto-preprocessor-test.js
+++ b/lighthouse-core/test/lib/proto-preprocessor-test.js
@@ -9,17 +9,40 @@ const processForProto = require('../../lib/proto-preprocessor').processForProto;
 
 /* eslint-env jest */
 describe('processing for proto', () => {
-  it('cleans up configSettings', () => {
+  it('keeps only necessary configSettings', () => {
     const input = {
       'configSettings': {
-        'output': 'json',
+        'output': [
+          'json',
+        ],
+        'maxWaitForLoad': 45000,
+        'throttlingMethod': 'devtools',
+        'throttling': {
+          'rttMs': 150,
+          'throughputKbps': 1638.4,
+          'requestLatencyMs': 562.5,
+          'downloadThroughputKbps': 1474.5600000000002,
+          'uploadThroughputKbps': 675,
+          'cpuSlowdownMultiplier': 4,
+        },
+        'gatherMode': false,
+        'disableStorageReset': false,
+        'disableDeviceEmulation': false,
+        'emulatedFormFactor': 'mobile',
+        'locale': 'en-US',
+        'blockedUrlPatterns': null,
+        'additionalTraceCategories': null,
+        'extraHeaders': null,
+        'onlyAudits': null,
+        'onlyCategories': null,
+        'skipAudits': null,
       },
     };
     const expectation = {
       'configSettings': {
-        'output': [
-          'json',
-        ],
+        'emulatedFormFactor': 'mobile',
+        'locale': 'en-US',
+        'onlyCategories': null,
       },
     };
     const output = processForProto(JSON.stringify(input));
diff --git a/lighthouse-core/test/report/html/renderer/category-renderer-test.js b/lighthouse-core/test/report/html/renderer/category-renderer-test.js
index 0d4cb0751..27c1947bc 100644
--- a/lighthouse-core/test/report/html/renderer/category-renderer-test.js
+++ b/lighthouse-core/test/report/html/renderer/category-renderer-test.js
@@ -229,7 +229,7 @@ describe('CategoryRenderer', () => {
       const manualAudits = elem.querySelectorAll('.lh-audit-group--manual .lh-audit');
 
       assert.equal(passedAudits.length, 4);
-      assert.equal(failedAudits.length, 7);
+      assert.equal(failedAudits.length, 8);
       assert.equal(manualAudits.length, 3);
     });
 
@@ -242,7 +242,7 @@ describe('CategoryRenderer', () => {
       const failedAudits = elem.querySelectorAll('.lh-failed-audits .lh-audit');
 
       assert.equal(passedAudits.length, 0);
-      assert.equal(failedAudits.length, 11);
+      assert.equal(failedAudits.length, 12);
 
       assert.equal(elem.querySelector('.lh-passed-audits-summary'), null);
     });
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
index 7417f748c..424dc2862 100644
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@@ -1048,6 +1048,15 @@
         ]
       }
     },
+    "offline-start-url": {
+      "id": "offline-start-url",
+      "title": "start_url does not respond with a 200 when offline",
+      "description": "A service worker enables your web app to be reliable in unpredictable network conditions. [Learn more](https://developers.google.com/web/tools/lighthouse/audits/http-200-when-offline).",
+      "score": 0,
+      "scoreDisplayMode": "binary",
+      "rawValue": false,
+      "warnings": []
+    },
     "pwa-cross-browser": {
       "id": "pwa-cross-browser",
       "title": "Site works cross-browser",
@@ -2927,6 +2936,11 @@
           "weight": 5,
           "group": "pwa-fast-reliable"
         },
+        {
+          "id": "offline-start-url",
+          "weight": 1,
+          "group": "pwa-fast-reliable"
+        },
         {
           "id": "is-on-https",
           "weight": 2,
@@ -2939,7 +2953,7 @@
         },
         {
           "id": "webapp-install-banner",
-          "weight": 3,
+          "weight": 2,
           "group": "pwa-installable"
         },
         {
@@ -3652,6 +3666,12 @@
         "duration": 100,
         "entryType": "measure"
       },
+      {
+        "startTime": 0,
+        "name": "lh:audit:offline-start-url",
+        "duration": 100,
+        "entryType": "measure"
+      },
       {
         "startTime": 0,
         "name": "lh:audit:pwa-cross-browser",