Repository: GoogleChrome/lighthouse
Pull request #6959 (2019-01-09 01:35:54)

core: add Lighthouse plugins
Base implementation of #6070

Adds support for plugins that can add new audits (or reuse existing audits) and a new category to the report. I'll add some notes to #6070 about stuff left to do.

The implementation is really simple: the plugin JSON is found and parsed (in `config.js`) and turned into a `LH.Config.Json` (in `config-plugin.js`), then merged into the main config with the same merging mechanism used to extend configs. Since the plugin can only add audits and a category, the merging is append-only.

Sorry the PR is a bit long. It's mostly tests. The parsing of the plugin json is also written highly defensively since it's going to be accepting input from any old npm module.

~If it's distracting review-wise, the `docs/recipes/plugin/` can also be removed before we land it (and a better recipe added in a followup, I didn't spend long on this one), but I'll leave it in the PR for now so you can see an example of use.~

For an example of use (besides the test fixtures), see WIP recipe in #6965



brendankenny: I think the main decision I'm unsure about here is that the config property is under `LH.Config.Json.plugins` (plural), but we're also planning on supporting a `--plugin` (singular) CLI flag.

Normally CLI flags would have `--plugin` merged into `LH.Config.Json.settings.plugin`, but

- it seems like it doesn't make sense under `settings`, just like `extends` doesn't. It's a base part of the config, not part of the settings of the LH run.
- it makes sense as a plural in the config (since it's always an array), but singular on the command line (so you can do like `lighthouse --plugin memory https://example.com --view`)

so it seems ok to add this to the handful of special-cased flag/config mergings in `config.js` when we add support for the CLI flag.
brendankenny: pulled the recipe into #6965 because it was only half baked and added 80 lines to this PR

Patch:
diff --git a/lighthouse-core/audits/byte-efficiency/render-blocking-resources.js b/lighthouse-core/audits/byte-efficiency/render-blocking-resources.js
index 62d1bde7d..5e1e17e65 100644
--- a/lighthouse-core/audits/byte-efficiency/render-blocking-resources.js
+++ b/lighthouse-core/audits/byte-efficiency/render-blocking-resources.js
@@ -103,14 +103,12 @@ class RenderBlockingResources extends Audit {
     for (const resource of artifacts.TagsBlockingFirstPaint) {
       // Ignore any resources that finished after observed FCP (they're clearly not render-blocking)
       if (resource.endTime * 1000 > fcpTsInMs) continue;
-      // TODO(phulce): beacon these occurences to Sentry to improve FCP graph
+      // TODO: beacon to Sentry, https://github.com/GoogleChrome/lighthouse/issues/7041
       if (!nodesByUrl[resource.tag.url]) continue;
 
       const {node, nodeTiming} = nodesByUrl[resource.tag.url];
 
       // Mark this node and all its dependents as deferrable
-      // TODO(phulce): make this slightly more surgical
-      // i.e. the referenced font asset won't become inlined just because you inline the CSS
       node.traverse(node => deferredNodeIds.add(node.id));
 
       // "wastedMs" is the download time of the network request, responseReceived - requestSent
@@ -190,7 +188,6 @@ class RenderBlockingResources extends Audit {
   static async computeWastedCSSBytes(artifacts, context) {
     const wastedBytesByUrl = new Map();
     try {
-      // TODO(phulce): pull this out into computed artifact
       const results = await UnusedCSS.audit(artifacts, context);
       // @ts-ignore - TODO(bckenny): details types.
       for (const item of results.details.items) {
diff --git a/lighthouse-core/audits/content-width.js b/lighthouse-core/audits/content-width.js
index 7de0b62fb..3f4f06a6b 100644
--- a/lighthouse-core/audits/content-width.js
+++ b/lighthouse-core/audits/content-width.js
@@ -34,7 +34,6 @@ class ContentWidth extends Audit {
     const windowWidth = artifacts.ViewportDimensions.outerWidth;
     const widthsMatch = viewportWidth === windowWidth;
 
-    // TODO(phulce): refactor this `isMobile` boolean to be on context
     const isMobileHost = userAgent.includes('Android') || userAgent.includes('Mobile');
     const isMobile = context.settings.emulatedFormFactor === 'mobile' ||
       (context.settings.emulatedFormFactor !== 'desktop' && isMobileHost);
diff --git a/lighthouse-core/audits/metrics/first-contentful-paint-3g.js b/lighthouse-core/audits/metrics/first-contentful-paint-3g.js
new file mode 100644
index 000000000..a61adb2a8
--- /dev/null
+++ b/lighthouse-core/audits/metrics/first-contentful-paint-3g.js
@@ -0,0 +1,68 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Audit = require('../audit.js');
+const regular3G = require('../../config/constants.js').throttling.mobileRegluar3G;
+const i18n = require('../../lib/i18n/i18n.js');
+const FCP = require('./first-contentful-paint.js');
+const ComputedFcp = require('../../computed/metrics/first-contentful-paint.js');
+
+const i18nFilename = require.resolve('./first-contentful-paint.js');
+const str_ = i18n.createMessageInstanceIdFn(i18nFilename, FCP.UIStrings);
+
+class FirstContentfulPaint3G extends Audit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return {
+      id: 'first-contentful-paint-3g',
+      title: str_(FCP.UIStrings.title),
+      description: str_(FCP.UIStrings.description),
+      scoreDisplayMode: Audit.SCORING_MODES.NUMERIC,
+      requiredArtifacts: ['traces', 'devtoolsLogs'],
+    };
+  }
+
+  /**
+   * @return {LH.Audit.ScoreOptions}
+   */
+  static get defaultOptions() {
+    return {
+      // 75th and 95th percentiles HTTPArchive on Fast 3G -> multiply by 1.5 for RTT differential -> median and PODR
+      // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1
+      scorePODR: 3000,
+      scoreMedian: 6000,
+    };
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @param {LH.Audit.Context} context
+   * @return {Promise<LH.Audit.Product>}
+   */
+  static async audit(artifacts, context) {
+    const trace = artifacts.traces[Audit.DEFAULT_PASS];
+    const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS];
+    /** @type {LH.Config.Settings} */
+    const settings = {...context.settings, throttlingMethod: 'simulate', throttling: regular3G};
+    const metricComputationData = {trace, devtoolsLog, settings};
+    const metricResult = await ComputedFcp.request(metricComputationData, context);
+
+    return {
+      score: Audit.computeLogNormalScore(
+        metricResult.timing,
+        context.options.scorePODR,
+        context.options.scoreMedian
+      ),
+      rawValue: metricResult.timing,
+      displayValue: str_(i18n.UIStrings.seconds, {timeInMs: metricResult.timing}),
+    };
+  }
+}
+
+module.exports = FirstContentfulPaint3G;
diff --git a/lighthouse-core/config/config-plugin.js b/lighthouse-core/config/config-plugin.js
new file mode 100644
index 000000000..b5ec1000d
--- /dev/null
+++ b/lighthouse-core/config/config-plugin.js
@@ -0,0 +1,175 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @param {unknown} arr
+ * @return {arr is Array<Record<string, unknown>>}
+ */
+function isArrayOfUnknownObjects(arr) {
+  return Array.isArray(arr) && arr.every(isObjectOfUnknownProperties);
+}
+
+/**
+ * @param {unknown} val
+ * @return {val is Record<string, unknown>}
+ */
+function isObjectOfUnknownProperties(val) {
+  return typeof val === 'object' && val !== null && !Array.isArray(val);
+}
+
+/**
+ * Asserts that obj has no own properties, throwing a nice error message if it does.
+ * Plugin and object name are included for nicer logging.
+ * @param {Record<string, unknown>} obj
+ * @param {string} pluginName
+ * @param {string=} objectName
+ */
+function assertNoExcessProperties(obj, pluginName, objectName = '') {
+  if (objectName) {
+    objectName += ' ';
+  }
+
+  const invalidKeys = Object.keys(obj);
+  if (invalidKeys.length > 0) {
+    const keys = invalidKeys.join(', ');
+    throw new Error(`${pluginName} has unrecognized ${objectName}properties: [${keys}]`);
+  }
+}
+
+/**
+ * A set of methods for extracting and validating a Lighthouse plugin config.
+ */
+class ConfigPlugin {
+  /**
+   * Extract and validate the list of AuditDefns added by the plugin (or undefined
+   * if no additional audits are being added by the plugin).
+   * @param {unknown} auditsJson
+   * @param {string} pluginName
+   * @return {Array<{path: string}>|undefined}
+   */
+  static _parseAuditsList(auditsJson, pluginName) {
+    // Plugin audits aren't required (relying on LH default audits) so fall back to [].
+    if (auditsJson === undefined) {
+      return undefined;
+    } else if (!isArrayOfUnknownObjects(auditsJson)) {
+      throw new Error(`${pluginName} has an invalid audits array.`);
+    }
+
+    return auditsJson.map(auditDefnJson => {
+      const {path, ...invalidRest} = auditDefnJson;
+      assertNoExcessProperties(invalidRest, pluginName, 'audit');
+
+      if (typeof path !== 'string') {
+        throw new Error(`${pluginName} has a missing audit path.`);
+      }
+      return {
+        path,
+      };
+    });
+  }
+
+  /**
+   * Extract and validate the list of category AuditRefs added by the plugin.
+   * @param {unknown} auditRefsJson
+   * @param {string} pluginName
+   * @return {Array<LH.Config.AuditRefJson>}
+   */
+  static _parseAuditRefsList(auditRefsJson, pluginName) {
+    if (!isArrayOfUnknownObjects(auditRefsJson)) {
+      throw new Error(`${pluginName} has no valid auditsRefs.`);
+    }
+
+    return auditRefsJson.map(auditRefJson => {
+      const {id, weight, ...invalidRest} = auditRefJson;
+      assertNoExcessProperties(invalidRest, pluginName, 'auditRef');
+
+      if (typeof id !== 'string') {
+        throw new Error(`${pluginName} has an invalid auditRef id.`);
+      }
+      if (typeof weight !== 'number') {
+        throw new Error(`${pluginName} has an invalid auditRef weight.`);
+      }
+
+      return {
+        id,
+        weight,
+      };
+    });
+  }
+
+  /**
+   * Extract and validate the category added by the plugin.
+   * @param {unknown} categoryJson
+   * @param {string} pluginName
+   * @return {LH.Config.CategoryJson}
+   */
+  static _parseCategory(categoryJson, pluginName) {
+    if (!isObjectOfUnknownProperties(categoryJson)) {
+      throw new Error(`${pluginName} has no valid category.`);
+    }
+
+    const {
+      title,
+      description,
+      manualDescription,
+      auditRefs: auditRefsJson,
+      ...invalidRest
+    } = categoryJson;
+
+    assertNoExcessProperties(invalidRest, pluginName, 'category');
+
+    if (typeof title !== 'string') {
+      throw new Error(`${pluginName} has an invalid category tile.`);
+    }
+    if (typeof description !== 'string' && typeof description !== 'undefined') {
+      throw new Error(`${pluginName} has an invalid category description.`);
+    }
+    if (typeof manualDescription !== 'string' && typeof manualDescription !== 'undefined') {
+      throw new Error(`${pluginName} has an invalid category manualDescription.`);
+    }
+    const auditRefs = ConfigPlugin._parseAuditRefsList(auditRefsJson, pluginName);
+
+    return {
+      title,
+      auditRefs,
+      description: description,
+      manualDescription: manualDescription,
+    };
+  }
+
+  /**
+   * Extracts and validates a ConfigJson from the provided plugin input, throwing
+   * if it deviates from the expected object shape.
+   * @param {unknown} pluginJson
+   * @param {string} pluginName
+   * @return {LH.Config.Json}
+   */
+  static parsePlugin(pluginJson, pluginName) {
+    // Clone to prevent modifications or original and to deactivate any live properties.
+    pluginJson = JSON.parse(JSON.stringify(pluginJson));
+    if (!isObjectOfUnknownProperties(pluginJson)) {
+      throw new Error(`${pluginName} is not defined as an object.`);
+    }
+
+    const {
+      audits: pluginAuditsJson,
+      category: pluginCategoryJson,
+      ...invalidRest
+    } = pluginJson;
+
+    assertNoExcessProperties(invalidRest, pluginName);
+
+    return {
+      audits: ConfigPlugin._parseAuditsList(pluginAuditsJson, pluginName),
+      categories: {
+        [pluginName]: ConfigPlugin._parseCategory(pluginCategoryJson, pluginName),
+      },
+    };
+  }
+}
+
+module.exports = ConfigPlugin;
diff --git a/lighthouse-core/config/config.js b/lighthouse-core/config/config.js
index 54cf15322..bb297a43f 100644
--- a/lighthouse-core/config/config.js
+++ b/lighthouse-core/config/config.js
@@ -16,6 +16,7 @@ const log = require('lighthouse-logger');
 const path = require('path');
 const Audit = require('../audits/audit.js');
 const Runner = require('../runner.js');
+const ConfigPlugin = require('./config-plugin.js');
 
 /** @typedef {typeof import('../gather/gatherers/gatherer.js')} GathererConstructor */
 /** @typedef {InstanceType<GathererConstructor>} Gatherer */
@@ -162,6 +163,22 @@ function assertValidGatherer(gathererInstance, gathererName) {
   }
 }
 
+/**
+ * Throws if pluginName is invalid or (somehow) collides with a category in the
+ * configJSON being added to.
+ * @param {LH.Config.Json} configJSON
+ * @param {string} pluginName
+ */
+function assertValidPluginName(configJSON, pluginName) {
+  if (!pluginName.startsWith('lighthouse-plugin-')) {
+    throw new Error(`plugin name '${pluginName}' does not start with 'lighthouse-plugin-'`);
+  }
+
+  if (configJSON.categories && configJSON.categories[pluginName]) {
+    throw new Error(`plugin name '${pluginName}' not allowed because it is the id of a category already found in config`); // eslint-disable-line max-len
+  }
+}
+
 /**
  * Creates a settings object from potential flags object by dropping all the properties
  * that don't exist on Config.Settings.
@@ -184,7 +201,6 @@ function cleanFlagsForSettings(flags = {}) {
   return settings;
 }
 
-// TODO(phulce): disentangle this merge function
 /**
  * More widely typed than exposed merge() function, below.
  * @param {Object<string, any>|Array<any>|undefined|null} base
@@ -345,6 +361,9 @@ class Config {
     // The directory of the config path, if one was provided.
     const configDir = configPath ? path.dirname(configPath) : undefined;
 
+    // Validate and merge in plugins (if any).
+    configJSON = Config.mergePlugins(configJSON, configDir);
+
     const settings = Config.initSettings(configJSON.settings, flags);
 
     // Augment passes with necessary defaults and require gatherers.
@@ -438,6 +457,29 @@ class Config {
     return merge(baseJSON, extendJSON);
   }
 
+  /**
+   * @param {LH.Config.Json} configJSON
+   * @param {string=} configDir
+   * @return {LH.Config.Json}
+   */
+  static mergePlugins(configJSON, configDir) {
+    const pluginNames = configJSON.plugins;
+
+    if (pluginNames) {
+      for (const pluginName of pluginNames) {
+        assertValidPluginName(configJSON, pluginName);
+
+        const pluginPath = Config.resolveModule(pluginName, configDir, 'plugin');
+        const rawPluginJson = require(pluginPath);
+        const pluginJson = ConfigPlugin.parsePlugin(rawPluginJson, pluginName);
+
+        configJSON = Config.extendConfigJSON(configJSON, pluginJson);
+      }
+    }
+
+    return configJSON;
+  }
+
   /**
    * @param {LH.Config.Json['passes']} passes
    * @return {?Array<Required<LH.Config.PassJson>>}
diff --git a/lighthouse-core/config/constants.js b/lighthouse-core/config/constants.js
index 46d19987f..4aea3ef77 100644
--- a/lighthouse-core/config/constants.js
+++ b/lighthouse-core/config/constants.js
@@ -16,6 +16,8 @@ const DEVTOOLS_THROUGHPUT_ADJUSTMENT_FACTOR = 0.9;
 const throttling = {
   DEVTOOLS_RTT_ADJUSTMENT_FACTOR,
   DEVTOOLS_THROUGHPUT_ADJUSTMENT_FACTOR,
+  // These values align with WebPageTest's definition of "Fast 3G"
+  // But offer similar charateristics to roughly the 75th percentile of 4G connections.
   mobileSlow4G: {
     rttMs: 150,
     throughputKbps: 1.6 * 1024,
@@ -24,6 +26,17 @@ const throttling = {
     uploadThroughputKbps: 750 * DEVTOOLS_THROUGHPUT_ADJUSTMENT_FACTOR,
     cpuSlowdownMultiplier: 4,
   },
+  // These values partially align with WebPageTest's definition of "Regular 3G".
+  // These values are meant to roughly align with Chrome UX report's 3G definition which are based
+  // on HTTP RTT of 300-1400ms and downlink throughput of <700kbps.
+  mobileRegluar3G: {
+    rttMs: 300,
+    throughputKbps: 700,
+    requestLatencyMs: 300 * DEVTOOLS_RTT_ADJUSTMENT_FACTOR,
+    downloadThroughputKbps: 700 * DEVTOOLS_THROUGHPUT_ADJUSTMENT_FACTOR,
+    uploadThroughputKbps: 700 * DEVTOOLS_THROUGHPUT_ADJUSTMENT_FACTOR,
+    cpuSlowdownMultiplier: 4,
+  },
 };
 
 /** @type {LH.Config.Settings} */
diff --git a/lighthouse-core/config/lr-desktop-config.js b/lighthouse-core/config/lr-desktop-config.js
index 11fc64485..8e0aa8f28 100644
--- a/lighthouse-core/config/lr-desktop-config.js
+++ b/lighthouse-core/config/lr-desktop-config.js
@@ -18,7 +18,7 @@ const config = {
       throughputKbps: 10 * 1024,
       cpuSlowdownMultiplier: 1,
     },
-    // Skip the h2 audit so it doesn't lie to us. See #6539
+    // Skip the h2 audit so it doesn't lie to us. See https://github.com/GoogleChrome/lighthouse/issues/6539
     skipAudits: ['uses-http2'],
   },
   audits: [
diff --git a/lighthouse-core/config/lr-mobile-config.js b/lighthouse-core/config/lr-mobile-config.js
index 264663542..2c689ad1b 100644
--- a/lighthouse-core/config/lr-mobile-config.js
+++ b/lighthouse-core/config/lr-mobile-config.js
@@ -10,9 +10,20 @@ const config = {
   extends: 'lighthouse:default',
   settings: {
     maxWaitForLoad: 35 * 1000,
-    // Skip the h2 audit so it doesn't lie to us. See #6539
+    // Skip the h2 audit so it doesn't lie to us. See https://github.com/GoogleChrome/lighthouse/issues/6539
     skipAudits: ['uses-http2'],
   },
+  audits: [
+    'metrics/first-contentful-paint-3g',
+  ],
+  // @ts-ignore TODO(bckenny): type extended Config where e.g. category.title isn't required
+  categories: {
+    performance: {
+      auditRefs: [
+        {id: 'first-contentful-paint-3g', weight: 0},
+      ],
+    },
+  },
 };
 
 module.exports = config;
diff --git a/lighthouse-core/gather/driver.js b/lighthouse-core/gather/driver.js
index 60f59ee30..b2e26267f 100644
--- a/lighthouse-core/gather/driver.js
+++ b/lighthouse-core/gather/driver.js
@@ -1245,7 +1245,6 @@ class Driver {
    * @return {Promise<void>}
    */
   async beginEmulation(settings) {
-    // TODO(phulce): remove this flag on next breaking change
     if (!settings.disableDeviceEmulation) {
       if (settings.emulatedFormFactor === 'mobile') {
         await emulation.enableNexus5X(this);
diff --git a/lighthouse-core/gather/gatherers/dobetterweb/optimized-images.js b/lighthouse-core/gather/gatherers/dobetterweb/optimized-images.js
index 67484a2fd..6ca6a9a16 100644
--- a/lighthouse-core/gather/gatherers/dobetterweb/optimized-images.js
+++ b/lighthouse-core/gather/gatherers/dobetterweb/optimized-images.js
@@ -129,8 +129,6 @@ class OptimizedImages extends Gatherer {
    * @return {Promise<?{fromProtocol: boolean, originalSize: number, jpegSize: number, webpSize: number}>}
    */
   calculateImageStats(driver, networkRecord) {
-    // TODO(phulce): remove this dance of trying _getEncodedResponse with a fallback when Audits
-    // domain hits stable in Chrome 62
     return Promise.resolve(networkRecord.requestId).then(requestId => {
       if (this._getEncodedResponseUnsupported) return;
       return this._getEncodedResponse(driver, requestId, 'jpeg').then(jpegData => {
@@ -153,7 +151,7 @@ class OptimizedImages extends Gatherer {
     }).then(result => {
       if (result) return result;
 
-      // Take the slower fallback path if getEncodedResponse isn't available yet
+      // Take the slower fallback path if getEncodedResponse didn't work
       // CORS canvas tainting doesn't support cross-origin images, so skip them early
       if (!networkRecord.isSameOrigin && !networkRecord.isBase64DataUri) return null;
 
diff --git a/lighthouse-core/gather/gatherers/link-elements.js b/lighthouse-core/gather/gatherers/link-elements.js
index 4a74e3abe..9283265e0 100644
--- a/lighthouse-core/gather/gatherers/link-elements.js
+++ b/lighthouse-core/gather/gatherers/link-elements.js
@@ -16,7 +16,6 @@ class LinkElements extends Gatherer {
   async afterPass(passContext) {
     const driver = passContext.driver;
 
-    // TODO(phulce): merge this with the main resource header values too
     // We'll use evaluateAsync because the `node.getAttribute` method doesn't actually normalize
     // the values like access from JavaScript does.
     return driver.evaluateAsync(`(() => {
diff --git a/lighthouse-core/lib/dependency-graph/simulator/network-analyzer.js b/lighthouse-core/lib/dependency-graph/simulator/network-analyzer.js
index ef2187df5..7f2dd6b0d 100644
--- a/lighthouse-core/lib/dependency-graph/simulator/network-analyzer.js
+++ b/lighthouse-core/lib/dependency-graph/simulator/network-analyzer.js
@@ -289,8 +289,6 @@ class NetworkAnalyzer {
         );
       }
 
-      // TODO(phulce): compute the maximum number of parallel requests (N) and ensure we have at
-      // least N requests that required new connections
       const firstRecord = originRecords.reduce((a, b) => (a.startTime > b.startTime ? b : a));
       connectionWasReused.set(firstRecord.requestId, false);
     }
@@ -448,7 +446,6 @@ class NetworkAnalyzer {
    * @return {LH.Artifacts.NetworkRequest}
    */
   static findMainDocument(records) {
-    // TODO(phulce): handle more edge cases like client redirects, or plumb through finalUrl
     const documentRequests = records.filter(record => record.resourceType ===
         NetworkRequest.TYPES.Document);
     // The main document is the earliest document request, using position in networkRecords array to break ties.
diff --git a/lighthouse-core/lib/dependency-graph/simulator/simulator.js b/lighthouse-core/lib/dependency-graph/simulator/simulator.js
index fe46d21c2..9c6651520 100644
--- a/lighthouse-core/lib/dependency-graph/simulator/simulator.js
+++ b/lighthouse-core/lib/dependency-graph/simulator/simulator.js
@@ -106,7 +106,6 @@ class Simulator {
     this._cachedNodeListByStartTime = [];
     // NOTE: We don't actually need *all* of these sets, but the clarity that each node progresses
     // through the system is quite nice.
-    // TODO(phulce): consider refactoring this so that it's easier to follow
     for (const state of Object.values(NodeState)) {
       this._nodes[state] = new Set();
     }
diff --git a/lighthouse-core/lib/network-recorder.js b/lighthouse-core/lib/network-recorder.js
index 4de1df7ff..4264e34eb 100644
--- a/lighthouse-core/lib/network-recorder.js
+++ b/lighthouse-core/lib/network-recorder.js
@@ -221,7 +221,7 @@ class NetworkRecorder extends EventEmitter {
       return;
     }
 
-    // TODO(phulce): log these to sentry?
+    // TODO: beacon to Sentry, https://github.com/GoogleChrome/lighthouse/issues/7041
     if (!data.redirectResponse) {
       return;
     }
@@ -231,7 +231,6 @@ class NetworkRecorder extends EventEmitter {
     const modifiedData = {
       ...data,
       // Copy over the initiator as well to match DevTools behavior
-      // TODO(phulce): abandon this DT hack and update Lantern graph to handle it
       initiator: originalRequest.initiator,
       requestId: `${originalRequest.requestId}:redirect`,
     };
diff --git a/lighthouse-core/lib/network-request.js b/lighthouse-core/lib/network-request.js
index 883a851df..8c5fc69ec 100644
--- a/lighthouse-core/lib/network-request.js
+++ b/lighthouse-core/lib/network-request.js
@@ -51,7 +51,6 @@ const RESOURCE_TYPES = {
 module.exports = class NetworkRequest {
   constructor() {
     this.requestId = '';
-    // TODO(phulce): remove default DevTools connectionId
     this.connectionId = '0';
     this.connectionReused = false;
 
diff --git a/lighthouse-core/report/html/renderer/report-ui-features.js b/lighthouse-core/report/html/renderer/report-ui-features.js
index bfc1b627c..4ed23e7fc 100644
--- a/lighthouse-core/report/html/renderer/report-ui-features.js
+++ b/lighthouse-core/report/html/renderer/report-ui-features.js
@@ -30,7 +30,7 @@ class ReportUIFeatures {
    * @param {DOM} dom
    */
   constructor(dom) {
-    /** @type {LH.ReportResult} */
+    /** @type {LH.Result} */
     this.json; // eslint-disable-line no-unused-expressions
     /** @type {DOM} */
     this._dom = dom;
@@ -80,7 +80,7 @@ class ReportUIFeatures {
   /**
    * Adds export button, print, and other functionality to the report. The method
    * should be called whenever the report needs to be re-rendered.
-   * @param {LH.ReportResult} report
+   * @param {LH.Result} report
    */
   initFeatures(report) {
     if (this._dom.isDevTools()) return;
@@ -358,7 +358,7 @@ class ReportUIFeatures {
   /**
    * Opens a new tab to the online viewer and sends the local page's JSON results
    * to the online viewer using postMessage.
-   * @param {LH.ReportResult} reportJson
+   * @param {LH.Result} reportJson
    * @param {string} viewerPath
    * @protected
    */
diff --git a/lighthouse-core/runner.js b/lighthouse-core/runner.js
index cb9fb5f02..e08294dc1 100644
--- a/lighthouse-core/runner.js
+++ b/lighthouse-core/runner.js
@@ -227,7 +227,6 @@ class Runner {
       const normalizedGatherSettings = Object.assign({}, artifacts.settings, overrides);
       const normalizedAuditSettings = Object.assign({}, settings, overrides);
 
-      // TODO(phulce): allow change of throttling method to `simulate`
       if (!isDeepEqual(normalizedGatherSettings, normalizedAuditSettings)) {
         throw new Error('Cannot change settings between gathering and auditing');
       }
diff --git a/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh b/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh
index 99bbc750b..7f2908446 100644
--- a/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh
+++ b/lighthouse-core/scripts/assert-golden-lhr-unchanged.sh
@@ -28,7 +28,6 @@ trap teardown EXIT
 
 colorText "Generating a fresh LHR..." "$purple"
 set -x
-# TODO(phulce): add a lantern LHR-differ
 node "$lhroot_path/lighthouse-cli" -A="$lhroot_path/lighthouse-core/test/results/artifacts" --throttling-method=devtools --quiet --output=json --output-path="$freshLHRPath"
 set +x
 
diff --git a/lighthouse-viewer/app/src/github-api.js b/lighthouse-viewer/app/src/github-api.js
index ee6f4bec4..9a0cc319a 100644
--- a/lighthouse-viewer/app/src/github-api.js
+++ b/lighthouse-viewer/app/src/github-api.js
@@ -7,7 +7,7 @@
 
 /* global logger, FirebaseAuth, idbKeyval, getFilenamePrefix */
 
-/** @typedef {{etag: ?string, content: LH.ReportResult}} CachableGist */
+/** @typedef {{etag: ?string, content: LH.Result}} CachableGist */
 
 /**
  * Wrapper around the GitHub API for reading/writing gists.
@@ -24,7 +24,7 @@ class GithubApi {
 
   /**
    * Creates a gist under the users account.
-   * @param {LH.ReportResult} jsonFile The gist file body.
+   * @param {LH.Result} jsonFile The gist file body.
    * @return {Promise<string>} id of the created gist.
    */
   createGist(jsonFile) {
@@ -73,7 +73,7 @@ class GithubApi {
   /**
    * Fetches a Lighthouse report from a gist.
    * @param {string} id The id of a gist.
-   * @return {Promise<LH.ReportResult>}
+   * @return {Promise<LH.Result>}
    */
   getGistFileContentAsJson(id) {
     logger.log('Fetching report from GitHub...', false);
@@ -132,7 +132,7 @@ class GithubApi {
                 .then(resp => resp.json())
                 .then(content => ({etag, content}));
             }
-            const lhr = /** @type {LH.ReportResult} */ (JSON.parse(f.content));
+            const lhr = /** @type {LH.Result} */ (JSON.parse(f.content));
             return {etag, content: lhr};
           });
         });
diff --git a/lighthouse-viewer/app/src/lighthouse-report-viewer.js b/lighthouse-viewer/app/src/lighthouse-report-viewer.js
index f0464dfd5..e59551dfb 100644
--- a/lighthouse-viewer/app/src/lighthouse-report-viewer.js
+++ b/lighthouse-viewer/app/src/lighthouse-report-viewer.js
@@ -106,7 +106,7 @@ class LighthouseReportViewer {
 
   /**
    * Basic Lighthouse report JSON validation.
-   * @param {LH.ReportResult} reportJson
+   * @param {LH.Result} reportJson
    * @private
    */
   _validateReportJson(reportJson) {
@@ -132,10 +132,17 @@ class LighthouseReportViewer {
   }
 
   /**
-   * @param {LH.ReportResult} json
+   * @param {LH.Result} json
    * @private
    */
+  // TODO: Really, `json` should really have type `unknown` and
+  // we can have _validateReportJson verify that it's an LH.Result
   _replaceReportHtml(json) {
+    // Allow users to view the runnerResult
+    if ('lhr' in json) {
+      json = /** @type {LH.RunnerResult} */ (json).lhr;
+    }
+
     this._validateReportJson(json);
 
     // Redirect to old viewer if a v2 report. v3 and v4 both handled by v4 viewer.
@@ -202,7 +209,7 @@ class LighthouseReportViewer {
 
   /**
    * Stores v2.x report in IDB, then navigates to legacy viewer in current tab.
-   * @param {LH.ReportResult} reportJson
+   * @param {LH.Result} reportJson
    * @private
    */
   _loadInLegacyViewerVersion(reportJson) {
@@ -242,7 +249,7 @@ class LighthouseReportViewer {
 
   /**
    * Saves the current report by creating a gist on GitHub.
-   * @param {LH.ReportResult} reportJson
+   * @param {LH.Result} reportJson
    * @return {Promise<string|void>} id of the created gist.
    * @private
    */
diff --git a/lighthouse-viewer/app/src/viewer-ui-features.js b/lighthouse-viewer/app/src/viewer-ui-features.js
index a5406b72a..0cb8a27e1 100644
--- a/lighthouse-viewer/app/src/viewer-ui-features.js
+++ b/lighthouse-viewer/app/src/viewer-ui-features.js
@@ -14,7 +14,7 @@
 class ViewerUIFeatures extends ReportUIFeatures {
   /**
    * @param {DOM} dom
-   * @param {?function(LH.ReportResult): void} saveGistCallback
+   * @param {?function(LH.Result): void} saveGistCallback
    */
   constructor(dom, saveGistCallback) {
     super(dom);
@@ -23,7 +23,7 @@ class ViewerUIFeatures extends ReportUIFeatures {
   }
 
   /**
-   * @param {LH.ReportResult} report
+   * @param {LH.Result} report
    * @override
    */
   initFeatures(report) {
diff --git a/third-party/snyk/snapshot.json b/third-party/snyk/snapshot.json
index a886e508e..18e7cff82 100644
--- a/third-party/snyk/snapshot.json
+++ b/third-party/snyk/snapshot.json
@@ -27,8 +27,10 @@
       {"id":"npm:backbone:20110701","severity":"medium","semver":{"vulnerable":["<0.5.0"]}}
     ],
     "bootstrap":[
-      {"id":"npm:bootstrap:20180529","severity":"medium","semver":{"vulnerable":[">=4.0.0 <4.1.2"]}},
-      {"id":"npm:bootstrap:20160627","severity":"medium","semver":{"vulnerable":["<3.4.0 || >=4.0.0-alpha <4.0.0-beta.2"]}},
+      {"id":"SNYK-JS-BOOTSTRAP-72890","severity":"medium","semver":{"vulnerable":["<3.4.0"]}},
+      {"id":"SNYK-JS-BOOTSTRAP-72889","severity":"medium","semver":{"vulnerable":["<3.4.0"]}},
+      {"id":"npm:bootstrap:20180529","severity":"medium","semver":{"vulnerable":["<3.4.0",">=4.0.0 <4.1.2"]}},
+      {"id":"npm:bootstrap:20160627","severity":"medium","semver":{"vulnerable":["<3.4.0",">=4.0.0 <=4.0.0"]}},
       {"id":"npm:bootstrap:20120510","severity":"medium","semver":{"vulnerable":["<2.1.0"]}}
     ],
     "dojo":[
diff --git a/types/config.d.ts b/types/config.d.ts
index 3992fb3d2..08c35958c 100644
--- a/types/config.d.ts
+++ b/types/config.d.ts
@@ -20,6 +20,7 @@ declare global {
         audits?: Config.AuditJson[] | null;
         categories?: Record<string, CategoryJson> | null;
         groups?: Record<string, Config.GroupJson> | null;
+        plugins?: Array<string>,
       }
 
       export interface SettingsJson extends SharedFlagsSettings {
@@ -107,6 +108,14 @@ declare global {
       }
       export interface Group extends GroupJson {}
 
+      export interface Plugin {
+        /** Optionally provide more audits to run in addition to those specified by the base config. */
+        audits?: Array<{path: string}>;
+        /** Provide a category to display the plugin results in the report. */
+        category: LH.Config.Category;
+        // TODO(bckenny): groups
+      }
+
       export type MergeOptionsOfItems = <T extends {path?: string, options: Record<string, any>}>(items: T[]) => T[];
 
       export type Merge = {


Test patch:
diff --git a/lighthouse-cli/test/smokehouse/byte-config.js b/lighthouse-cli/test/smokehouse/byte-config.js
index 78c58b7fb..6718ab526 100644
--- a/lighthouse-cli/test/smokehouse/byte-config.js
+++ b/lighthouse-cli/test/smokehouse/byte-config.js
@@ -22,8 +22,6 @@ module.exports = {
       'unused-css-rules',
       'unused-javascript',
     ],
-
-    // TODO(phulce): re-write testers to work with faster lantern loading
     throttlingMethod: 'devtools',
   },
 };
diff --git a/lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js b/lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js
index 0822e727c..d4a379638 100644
--- a/lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js
+++ b/lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js
@@ -5,9 +5,6 @@
  */
 'use strict';
 
-// TODO(phulce): assert more score values once Lantern can tie more trace events back to images.
-// See https://github.com/GoogleChrome/lighthouse/issues/4600.
-
 /**
  * Expected Lighthouse audit values for byte efficiency tests
  */
diff --git a/lighthouse-core/test/audits/metrics/first-contentful-paint-3g-test.js b/lighthouse-core/test/audits/metrics/first-contentful-paint-3g-test.js
new file mode 100644
index 000000000..b569abe2f
--- /dev/null
+++ b/lighthouse-core/test/audits/metrics/first-contentful-paint-3g-test.js
@@ -0,0 +1,36 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const FCP3G = require('../../../audits/metrics/first-contentful-paint-3g.js');
+const options = FCP3G.defaultOptions;
+
+const pwaTrace = require('../../fixtures/traces/progressive-app-m60.json');
+const pwaDevtoolsLog = require('../../fixtures/traces/progressive-app-m60.devtools.log.json');
+
+/* eslint-env jest */
+
+describe('Performance: first-contentful-paint-3g audit', () => {
+  it('evaluates valid input correctly', async () => {
+    const artifacts = {
+      traces: {
+        [FCP3G.DEFAULT_PASS]: pwaTrace,
+      },
+      devtoolsLogs: {
+        [FCP3G.DEFAULT_PASS]: pwaDevtoolsLog,
+      },
+    };
+
+    const result = await FCP3G.audit(artifacts, {settings: {}, options, computedCache: new Map()});
+    // Use InlineSnapshot here so changes to Lantern coefficients can be easily updated en masse
+    expect({score: result.score, value: Math.round(result.rawValue)}).toMatchInlineSnapshot(`
+Object {
+  "score": 1,
+  "value": 1661,
+}
+`);
+  });
+});
diff --git a/lighthouse-core/test/config/__snapshots__/config-plugin-test.js.snap b/lighthouse-core/test/config/__snapshots__/config-plugin-test.js.snap
new file mode 100644
index 000000000..4318e5c17
--- /dev/null
+++ b/lighthouse-core/test/config/__snapshots__/config-plugin-test.js.snap
@@ -0,0 +1,28 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`ConfigPlugin accepts a well formed plugin 1`] = `
+Object {
+  "audits": Array [
+    Object {
+      "path": "not/a/path/audit.js",
+    },
+  ],
+  "categories": Object {
+    "lighthouse-plugin-nice-plugin": Object {
+      "auditRefs": Array [
+        Object {
+          "id": "nice-audit",
+          "weight": 1,
+        },
+        Object {
+          "id": "installable-manifest",
+          "weight": 220,
+        },
+      ],
+      "description": "A nice plugin for nice testing",
+      "manualDescription": undefined,
+      "title": "Nice Plugin",
+    },
+  },
+}
+`;
diff --git a/lighthouse-core/test/config/config-plugin-test.js b/lighthouse-core/test/config/config-plugin-test.js
new file mode 100644
index 000000000..9308235bb
--- /dev/null
+++ b/lighthouse-core/test/config/config-plugin-test.js
@@ -0,0 +1,256 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const assert = require('assert');
+const ConfigPlugin = require('../../config/config-plugin.js');
+
+/* eslint-env jest */
+
+/**
+ * @param {any} val
+ * @return {any}
+ */
+function deepClone(val) {
+  return JSON.parse(JSON.stringify(val));
+}
+
+const nicePluginName = 'lighthouse-plugin-nice-plugin';
+const nicePlugin = {
+  audits: [{path: 'not/a/path/audit.js'}],
+  category: {
+    title: 'Nice Plugin',
+    description: 'A nice plugin for nice testing',
+    auditRefs: [
+      {id: 'nice-audit', weight: 1},
+      {id: 'installable-manifest', weight: 220},
+    ],
+  },
+};
+
+describe('ConfigPlugin', () => {
+  it('accepts a well formed plugin', () => {
+    const pluginJson = ConfigPlugin.parsePlugin(nicePlugin, nicePluginName);
+    expect(pluginJson).toMatchSnapshot();
+  });
+
+  it('throws on something other than a plugin object', () => {
+    assert.throws(() => ConfigPlugin.parsePlugin(5, nicePluginName),
+      /^Error: lighthouse-plugin-nice-plugin is not defined as an object/);
+  });
+
+  it('throws on an array instead of a plugin object', () => {
+    assert.throws(() => ConfigPlugin.parsePlugin([], nicePluginName),
+      /^Error: lighthouse-plugin-nice-plugin is not defined as an object/);
+  });
+
+  it('throws if there are excess plugin properties', () => {
+    const pluginClone = deepClone(nicePlugin);
+    pluginClone.extraProperty = 'extra';
+
+    assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+      /^Error: lighthouse-plugin-nice-plugin has unrecognized properties: \[extraProperty\]$/);
+  });
+
+  it('deals only with the JSON roundtrip version of the passed-in object', () => {
+    const evilAudits = [{path: 'not/a/path/audit.js'}];
+    const evilCategory = {
+      title: 'Evil Plugin',
+      description: 'A plugin that\'s trying to undermine you.',
+      manualDescription: 'Still here.',
+      auditRefs: [
+        {id: 'evil-audit', weight: 0},
+      ],
+    };
+
+    const evilPlugin = {
+      // Getter should be flattened to just value.
+      get audits() {
+        return evilAudits;
+      },
+      // Excess property would normally throw, but live function is dropped by stringify/parse.
+      evilProperty() {
+        throw new Error('get out of here');
+      },
+      category: evilCategory,
+    };
+
+    const pluginJson = ConfigPlugin.parsePlugin(evilPlugin, 'lighthouse-plugin-evil');
+    assert.deepStrictEqual(pluginJson, {
+      audits: evilAudits,
+      categories: {
+        'lighthouse-plugin-evil': evilCategory,
+      },
+    });
+    assert.strictEqual(Object.getOwnPropertyDescriptor(pluginJson, 'audits').get, undefined);
+  });
+
+  describe('`audits` array', () => {
+    it('correctly passes through the contained audits', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.audits.push({path: 'second/audit.js'});
+      const pluginJson = ConfigPlugin.parsePlugin(pluginClone, nicePluginName);
+
+      assert.strictEqual(pluginJson.audits[0].path, 'not/a/path/audit.js');
+      assert.strictEqual(pluginJson.audits[1].path, 'second/audit.js');
+    });
+
+    it('accepts a plugin with no new audits added', () => {
+      const pluginClone = deepClone(nicePlugin);
+      delete pluginClone.audits;
+      const pluginJson = ConfigPlugin.parsePlugin(pluginClone, nicePluginName);
+
+      assert.strictEqual(pluginJson.audits, undefined);
+    });
+
+    it('throws if not an array', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.audits = 5;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has an invalid audits array/);
+    });
+
+    it('throws if it contains non-objects', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.audits[0] = 5;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has an invalid audits array/);
+    });
+
+    it('throws if it contains objects with excess properties', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.audits[0].extraProperty = 'extra';
+      pluginClone.audits[0].otherProperty = 'other';
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin.*audit.*extraProperty.*otherProperty/);
+    });
+
+    it('throws if it contains objects with a missing `path`', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.audits[0].path = undefined;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has a missing audit path/);
+    });
+  });
+
+  describe('`category`', () => {
+    it('correctly adds the category under the plugin\'s name', () => {
+      const pluginJson = ConfigPlugin.parsePlugin(nicePlugin, nicePluginName);
+      assert.ok(pluginJson.categories[nicePluginName]);
+    });
+
+    it('throws if category is missing', () => {
+      const pluginClone = deepClone(nicePlugin);
+      delete pluginClone.category;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has no valid category/);
+    });
+
+    it('throws if category contains excess properties', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.category.extraProperty = 'extra';
+      pluginClone.category.otherProperty = 'other';
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin.*category.*extraProperty.*otherProperty/);
+    });
+
+    it('throws if category has an invalid or missing title', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.category.title = 55;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has an invalid category tile/);
+
+      const pluginClone2 = deepClone(nicePlugin);
+      delete pluginClone2.category.title;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone2, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has an invalid category tile/);
+    });
+
+    it('accepts a category with no description', () => {
+      const pluginClone = deepClone(nicePlugin);
+      delete pluginClone.category.description;
+      const pluginJson = ConfigPlugin.parsePlugin(pluginClone, nicePluginName);
+      assert.ok(pluginJson);
+    });
+
+    it('throws if category has an invalid description', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.category.description = 55;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has an invalid category description/);
+    });
+
+    it('accepts a category with no manualDescription', () => {
+      const pluginClone = deepClone(nicePlugin);
+      delete pluginClone.category.description;
+      const pluginJson = ConfigPlugin.parsePlugin(pluginClone, nicePluginName);
+      assert.ok(pluginJson);
+    });
+
+    it('throws if category has an invalid manualDescription', () => {
+      const pluginClone = deepClone(nicePlugin);
+      pluginClone.category.manualDescription = 55;
+      assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+        /^Error: lighthouse-plugin-nice-plugin has an invalid category manualDescription/);
+    });
+
+    describe('`category.auditRefs`', () => {
+      it('correctly passes through the contained auditRefs', () => {
+        const pluginJson = ConfigPlugin.parsePlugin(nicePlugin, nicePluginName);
+
+        const auditRefs = pluginJson.categories[nicePluginName].auditRefs;
+        assert.deepStrictEqual(auditRefs[0], {id: 'nice-audit', weight: 1});
+        assert.deepStrictEqual(auditRefs[1], {id: 'installable-manifest', weight: 220});
+      });
+
+      it('throws if auditRefs is missing', () => {
+        const pluginClone = deepClone(nicePlugin);
+        delete pluginClone.category.auditRefs;
+        assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+          /^Error: lighthouse-plugin-nice-plugin has no valid auditsRefs/);
+      });
+
+      it('throws if it contains non-objects', () => {
+        const pluginClone = deepClone(nicePlugin);
+        pluginClone.category.auditRefs[0] = 5;
+        assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+          /^Error: lighthouse-plugin-nice-plugin has no valid auditsRefs/);
+      });
+
+      it('throws if it contains objects with excess properties', () => {
+        const pluginClone = deepClone(nicePlugin);
+        pluginClone.category.auditRefs[0].extraProperty = 'extra';
+        pluginClone.category.auditRefs[0].otherProperty = 'other';
+        assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+          /^Error: lighthouse-plugin-nice-plugin.*auditRef.*extraProperty.*otherProperty/);
+      });
+
+      it('throws if it contains objects with an invalid or missing `id`', () => {
+        const pluginClone = deepClone(nicePlugin);
+        pluginClone.category.auditRefs[1].id = 55;
+        assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+          /^Error: lighthouse-plugin-nice-plugin has an invalid auditRef id/);
+
+        const pluginClone2 = deepClone(nicePlugin);
+        delete pluginClone2.category.auditRefs[0].id;
+        assert.throws(() => ConfigPlugin.parsePlugin(pluginClone2, nicePluginName),
+          /^Error: lighthouse-plugin-nice-plugin has an invalid auditRef id/);
+      });
+
+      it('throws if it contains objects with an invalid or missing `weight`', () => {
+        const pluginClone = deepClone(nicePlugin);
+        pluginClone.category.auditRefs[0].weight = 'NotAWeight';
+        assert.throws(() => ConfigPlugin.parsePlugin(pluginClone, nicePluginName),
+          /^Error: lighthouse-plugin-nice-plugin has an invalid auditRef weight/);
+
+        const pluginClone2 = deepClone(nicePlugin);
+        delete pluginClone2.category.auditRefs[1].weight;
+        assert.throws(() => ConfigPlugin.parsePlugin(pluginClone2, nicePluginName),
+          /^Error: lighthouse-plugin-nice-plugin has an invalid auditRef weight/);
+      });
+    });
+  });
+});
diff --git a/lighthouse-core/test/config/config-test.js b/lighthouse-core/test/config/config-test.js
index 8ea5b0202..d507179f1 100644
--- a/lighthouse-core/test/config/config-test.js
+++ b/lighthouse-core/test/config/config-test.js
@@ -696,6 +696,72 @@ describe('Config', () => {
     });
   });
 
+  describe('mergePlugins', () => {
+    const configFixturePath = __dirname + '/../fixtures/config-plugins/';
+
+    it('should append audits', () => {
+      const configJson = {
+        audits: ['installable-manifest', 'metrics'],
+        plugins: ['lighthouse-plugin-simple'],
+      };
+      const config = new Config(configJson, {configPath: configFixturePath});
+      assert.deepStrictEqual(config.audits.map(a => a.path),
+        ['installable-manifest', 'metrics', 'redirects', 'user-timings']);
+    });
+
+    it('should append a category', () => {
+      const configJson = {
+        extends: 'lighthouse:default',
+        plugins: ['lighthouse-plugin-simple'],
+      };
+      const config = new Config(configJson, {configPath: configFixturePath});
+      const categoryNames = Object.keys(config.categories);
+      assert.ok(categoryNames.length > 1);
+      assert.strictEqual(categoryNames[categoryNames.length - 1], 'lighthouse-plugin-simple');
+      assert.strictEqual(config.categories['lighthouse-plugin-simple'].title, 'Simple');
+    });
+
+    it('should throw if the plugin is invalid', () => {
+      const configJson = {
+        extends: 'lighthouse:default',
+        plugins: ['lighthouse-plugin-no-category'],
+      };
+      // Required to have a `category`, so plugin is invalid.
+      assert.throws(() => new Config(configJson, {configPath: configFixturePath}),
+        /^Error: lighthouse-plugin-no-category has no valid category/);
+    });
+
+    it('should throw if the plugin is not found', () => {
+      const configJson = {
+        extends: 'lighthouse:default',
+        plugins: ['lighthouse-plugin-not-a-plugin'],
+      };
+      assert.throws(() => new Config(configJson, {configPath: configFixturePath}),
+        /^Error: Unable to locate plugin: lighthouse-plugin-not-a-plugin/);
+    });
+
+    it('should throw if the plugin name does not begin with "lighthouse-plugin-"', () => {
+      const configJson = {
+        extends: 'lighthouse:default',
+        plugins: ['just-let-me-be-a-plugin'],
+      };
+      assert.throws(() => new Config(configJson, {configPath: configFixturePath}),
+        /^Error: plugin name 'just-let-me-be-a-plugin' does not start with 'lighthouse-plugin-'/);
+    });
+
+    it('should throw if the plugin name would shadow a category id', () => {
+      const configJson = {
+        extends: 'lighthouse:default',
+        plugins: ['lighthouse-plugin-simple'],
+        categories: {
+          'lighthouse-plugin-simple': {auditRefs: [{id: 'missing-audit'}]},
+        },
+      };
+      assert.throws(() => new Config(configJson, {configPath: configFixturePath}),
+        /^Error: plugin name 'lighthouse-plugin-simple' not allowed because it is the id of a category/); // eslint-disable-line max-len
+    });
+  });
+
   describe('getCategories', () => {
     it('returns the IDs & names of the categories', () => {
       const categories = Config.getCategories(origConfig);
diff --git a/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-category/package.json b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-category/package.json
new file mode 100644
index 000000000..20f6b933e
--- /dev/null
+++ b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-category/package.json
@@ -0,0 +1,5 @@
+{
+  "name": "lighthouse-plugin-no-category",
+  "private": true,
+  "main": "./plugin-no-category.js"
+}
diff --git a/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-category/plugin-no-category.js b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-category/plugin-no-category.js
new file mode 100644
index 000000000..24e317ce6
--- /dev/null
+++ b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-no-category/plugin-no-category.js
@@ -0,0 +1,14 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/** @type {LH.Config.Plugin} */
+module.exports = {
+  audits: [
+    {path: 'redirects'},
+    {path: 'user-timings'},
+  ],
+};
diff --git a/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple/package.json b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple/package.json
new file mode 100644
index 000000000..ab7c79447
--- /dev/null
+++ b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple/package.json
@@ -0,0 +1,5 @@
+{
+  "name": "lighthouse-plugin-simple",
+  "private": true,
+  "main": "./plugin-simple.js"
+}
diff --git a/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple/plugin-simple.js b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple/plugin-simple.js
new file mode 100644
index 000000000..ebf2b818d
--- /dev/null
+++ b/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple/plugin-simple.js
@@ -0,0 +1,18 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/** @type {LH.Config.Plugin} */
+module.exports = {
+  audits: [
+    {path: 'redirects'},
+    {path: 'user-timings'},
+  ],
+  category: {
+    title: 'Simple',
+    auditRefs: [],
+  },
+};
diff --git a/lighthouse-core/test/report/html/renderer/report-ui-features-test.js b/lighthouse-core/test/report/html/renderer/report-ui-features-test.js
index 1b3566fca..b9d4b8d40 100644
--- a/lighthouse-core/test/report/html/renderer/report-ui-features-test.js
+++ b/lighthouse-core/test/report/html/renderer/report-ui-features-test.js
@@ -19,7 +19,7 @@ const CategoryRenderer = require('../../../../report/html/renderer/category-rend
 const CriticalRequestChainRenderer = require(
     '../../../../report/html/renderer/crc-details-renderer.js');
 const ReportRenderer = require('../../../../report/html/renderer/report-renderer.js');
-const sampleResultsOrig = require('../../../results/sample_v2.json');
+const sampleResults = require('../../../results/sample_v2.json');
 
 const TEMPLATE_FILE = fs.readFileSync(__dirname +
     '/../../../../report/html/templates.html', 'utf8');
@@ -29,7 +29,6 @@ const TEMPLATE_FILE_REPORT = fs.readFileSync(__dirname +
 describe('ReportUIFeatures', () => {
   let renderer;
   let reportUIFeatures;
-  let sampleResults;
 
   beforeAll(() => {
     global.URL = URL;
@@ -74,7 +73,6 @@ describe('ReportUIFeatures', () => {
     const detailsRenderer = new DetailsRenderer(dom);
     const categoryRenderer = new CategoryRenderer(dom, detailsRenderer);
     renderer = new ReportRenderer(dom, categoryRenderer);
-    sampleResults = Util.prepareReportResult(sampleResultsOrig);
     reportUIFeatures = new ReportUIFeatures(dom2);
   });
 
@@ -100,7 +98,7 @@ describe('ReportUIFeatures', () => {
       renderer.renderReport(sampleResults, container);
 
       assert.equal(reportUIFeatures.json, undefined);
-      reportUIFeatures.initFeatures(Util.prepareReportResult(sampleResults));
+      reportUIFeatures.initFeatures(sampleResults);
       assert.ok(reportUIFeatures.json);
     });
   });