Repository: GoogleChrome/lighthouse
Pull request #7284 (2019-02-20 18:40:24)

tests(smokehouse): refactor to enable Smokerider
A few refactors to enable running smoke tests against Lightrider. Namely:

1. Move all test definitions to their own module, with a way to uncompact the references to other modules (the expectations / config).
2. Move all the LHR comparison / reporting code to a module.
3. LR entry option `keepRawValues`, since many smoke tests assert on that.

See internal bug b/124882661.

brendankenny: is it possible to make the brainstorming doc public? It would be nice to see the motivations/constraints before settling on a new design
Hoten: It's easier to share it individually. I sent @patrickhulce a link. I can do the same for anyone else who requests it.
Hoten: @brendankenny any questions or concerns about these changes after reviewing how it's utilized in cl/234828413?

Patch:
diff --git a/clients/lightrider-entry.js b/clients/lightrider-entry.js
index d46dd36d6..98d1dbc30 100644
--- a/clients/lightrider-entry.js
+++ b/clients/lightrider-entry.js
@@ -24,16 +24,19 @@ const LR_PRESETS = {
  * @param {Connection} connection
  * @param {string} url
  * @param {LH.Flags} flags Lighthouse flags, including `output`
- * @param {{lrDevice?: 'desktop'|'mobile', categoryIDs?: Array<string>, logAssets: boolean}} lrOpts Options coming from Lightrider
+ * @param {{lrDevice?: 'desktop'|'mobile', categoryIDs?: Array<string>, logAssets: boolean, keepRawValues: boolean}} lrOpts Options coming from Lightrider
  * @return {Promise<string|Array<string>|void>}
  */
-async function runLighthouseInLR(connection, url, flags, {lrDevice, categoryIDs, logAssets}) {
+async function runLighthouseInLR(connection, url, flags, lrOpts) {
+  const {lrDevice, categoryIDs, logAssets, keepRawValues} = lrOpts;
+
   // Certain fixes need to kick in under LR, see https://github.com/GoogleChrome/lighthouse/issues/5839
   global.isLightRider = true;
 
   // disableStorageReset because it causes render server hang
   flags.disableStorageReset = true;
   flags.logLevel = flags.logLevel || 'info';
+
   const config = lrDevice === 'desktop' ? LR_PRESETS.desktop : LR_PRESETS.mobile;
   if (categoryIDs) {
     config.settings = config.settings || {};
@@ -50,7 +53,7 @@ async function runLighthouseInLR(connection, url, flags, {lrDevice, categoryIDs,
 
     // pre process the LHR for proto
     if (flags.output === 'json' && typeof results.report === 'string') {
-      return preprocessor.processForProto(results.report);
+      return preprocessor.processForProto(results.report, {keepRawValues});
     }
 
     return results.report;
diff --git a/lighthouse-core/audits/seo/font-size.js b/lighthouse-core/audits/seo/font-size.js
index d33826f2b..2dad98bc9 100644
--- a/lighthouse-core/audits/seo/font-size.js
+++ b/lighthouse-core/audits/seo/font-size.js
@@ -207,7 +207,7 @@ class FontSize extends Audit {
       title: str_(UIStrings.title),
       failureTitle: str_(UIStrings.failureTitle),
       description: str_(UIStrings.description),
-      requiredArtifacts: ['FontSize', 'URL', 'MetaElements'],
+      requiredArtifacts: ['FontSize', 'URL', 'MetaElements', 'TestedAsMobileDevice'],
     };
   }
 
@@ -217,6 +217,14 @@ class FontSize extends Audit {
    * @return {Promise<LH.Audit.Product>}
    */
   static async audit(artifacts, context) {
+    if (!artifacts.TestedAsMobileDevice) {
+      // Font size isn't important to desktop SEO
+      return {
+        rawValue: true,
+        notApplicable: true,
+      };
+    }
+
     const viewportMeta = await ComputedViewportMeta.request(artifacts, context);
     if (!viewportMeta.isMobileOptimized) {
       return {
diff --git a/lighthouse-core/audits/seo/tap-targets.js b/lighthouse-core/audits/seo/tap-targets.js
index 4b3505c0b..7d95dea73 100644
--- a/lighthouse-core/audits/seo/tap-targets.js
+++ b/lighthouse-core/audits/seo/tap-targets.js
@@ -264,7 +264,7 @@ class TapTargets extends Audit {
       title: str_(UIStrings.title),
       failureTitle: str_(UIStrings.failureTitle),
       description: str_(UIStrings.description),
-      requiredArtifacts: ['MetaElements', 'TapTargets'],
+      requiredArtifacts: ['MetaElements', 'TapTargets', 'TestedAsMobileDevice'],
     };
   }
 
@@ -274,6 +274,15 @@ class TapTargets extends Audit {
    * @return {Promise<LH.Audit.Product>}
    */
   static async audit(artifacts, context) {
+    if (!artifacts.TestedAsMobileDevice) {
+      // Tap target sizes aren't important for desktop SEO, so disable the audit there.
+      // On desktop people also tend to have more precise pointing devices than fingers.
+      return {
+        rawValue: true,
+        notApplicable: true,
+      };
+    }
+
     const viewportMeta = await ComputedViewportMeta.request(artifacts, context);
     if (!viewportMeta.isMobileOptimized) {
       return {
diff --git a/lighthouse-core/lib/proto-preprocessor.js b/lighthouse-core/lib/proto-preprocessor.js
index fa73cec2f..77b40feda 100644
--- a/lighthouse-core/lib/proto-preprocessor.js
+++ b/lighthouse-core/lib/proto-preprocessor.js
@@ -17,8 +17,11 @@ const fs = require('fs');
 
 /**
   * @param {string} result
+  * @param {{keepRawValues?: boolean}} opts
   */
-function processForProto(result) {
+function processForProto(result, opts = {}) {
+  const {keepRawValues = false} = opts;
+
   /** @type {LH.Result} */
   const reportJson = JSON.parse(result);
 
@@ -53,8 +56,8 @@ function processForProto(result) {
           audit.scoreDisplayMode = 'notApplicable';
         }
       }
-      // Drop raw values. #6199
-      if ('rawValue' in audit) {
+      // Drop raw values. https://github.com/GoogleChrome/lighthouse/issues/6199
+      if (!keepRawValues && 'rawValue' in audit) {
         delete audit.rawValue;
       }
       // Normalize displayValue to always be a string, not an array. #6200
diff --git a/types/smokehouse.d.ts b/types/smokehouse.d.ts
new file mode 100644
index 000000000..a3393f4f4
--- /dev/null
+++ b/types/smokehouse.d.ts
@@ -0,0 +1,36 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+ declare module Smokehouse {
+  export interface Difference {
+    path: string;
+    actual: any;
+    expected: any;
+  }
+
+  export interface Comparison {
+    category: string;
+    actual: any;
+    expected: any;
+    equal: boolean;
+    diff?: Difference | null;
+  }
+
+  export type ExpectedLHR = Pick<LH.Result, 'audits' | 'finalUrl' | 'requestedUrl'> & { errorCode?: string }
+
+  export interface LHRComparison {
+    audits: Comparison[];
+    errorCode: Comparison;
+    finalUrl: Comparison;
+  }
+
+  export interface TestDfn {
+    id: string;
+    expectations: string;
+    config: string;
+    batch: string;
+  }
+}


Test patch:
diff --git a/lighthouse-cli/test/smokehouse/run-smoke.js b/lighthouse-cli/test/smokehouse/run-smoke.js
index 5399eb77b..7653821eb 100644
--- a/lighthouse-cli/test/smokehouse/run-smoke.js
+++ b/lighthouse-cli/test/smokehouse/run-smoke.js
@@ -14,88 +14,7 @@ const log = require('lighthouse-logger');
 
 /** @param {string} str */
 const purpleify = str => `${log.purple}${str}${log.reset}`;
-const smokehouseDir = 'lighthouse-cli/test/smokehouse/';
-
-/**
- * @typedef {object} SmoketestDfn
- * @property {string} id
- * @property {string} expectations
- * @property {string} config
- * @property {string | undefined} batch
- */
-
-/** @type {Array<SmoketestDfn>} */
-const SMOKETESTS = [{
-  id: 'a11y',
-  config: smokehouseDir + 'a11y/a11y-config.js',
-  expectations: 'a11y/expectations.js',
-  batch: 'parallel-first',
-}, {
-  id: 'errors',
-  expectations: smokehouseDir + 'error-expectations.js',
-  config: smokehouseDir + 'error-config.js',
-  batch: 'errors',
-}, {
-  id: 'oopif',
-  expectations: smokehouseDir + 'oopif-expectations.js',
-  config: smokehouseDir + 'oopif-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'pwa',
-  expectations: smokehouseDir + 'pwa-expectations.js',
-  config: smokehouseDir + 'pwa-config.js',
-  batch: 'parallel-second',
-}, {
-  id: 'pwa2',
-  expectations: smokehouseDir + 'pwa2-expectations.js',
-  config: smokehouseDir + 'pwa-config.js',
-  batch: 'parallel-second',
-}, {
-  id: 'pwa3',
-  expectations: smokehouseDir + 'pwa3-expectations.js',
-  config: smokehouseDir + 'pwa-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'dbw',
-  expectations: 'dobetterweb/dbw-expectations.js',
-  config: smokehouseDir + 'dbw-config.js',
-  batch: 'parallel-second',
-}, {
-  id: 'redirects',
-  expectations: 'redirects/expectations.js',
-  config: smokehouseDir + 'redirects-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'seo',
-  expectations: 'seo/expectations.js',
-  config: smokehouseDir + 'seo-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'offline',
-  expectations: 'offline-local/offline-expectations.js',
-  config: smokehouseDir + 'offline-config.js',
-  batch: 'offline',
-}, {
-  id: 'byte',
-  expectations: 'byte-efficiency/expectations.js',
-  config: smokehouseDir + 'byte-config.js',
-  batch: 'perf-opportunity',
-}, {
-  id: 'perf',
-  expectations: 'perf/expectations.js',
-  config: 'lighthouse-core/config/perf-config.js',
-  batch: 'perf-metric',
-}, {
-  id: 'lantern',
-  expectations: 'perf/lantern-expectations.js',
-  config: smokehouseDir + 'lantern-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'metrics',
-  expectations: 'tricky-metrics/expectations.js',
-  config: 'lighthouse-core/config/perf-config.js',
-  batch: 'parallel-second',
-}];
+const SMOKETESTS = require('./smoke-test-dfns').SMOKE_TEST_DFNS;
 
 /**
  * Display smokehouse output from child process
@@ -116,7 +35,7 @@ function displaySmokehouseOutput(result) {
 /**
  * Run smokehouse in child processes for selected smoketests
  * Display output from each as soon as they finish, but resolve function when ALL are complete
- * @param {Array<SmoketestDfn>} smokes
+ * @param {Array<Smokehouse.TestDfn>} smokes
  * @return {Promise<Array<{id: string, error?: Error}>>}
  */
 async function runSmokehouse(smokes) {
@@ -149,7 +68,7 @@ async function runSmokehouse(smokes) {
 /**
  * Determine batches of smoketests to run, based on argv
  * @param {string[]} argv
- * @return {Map<string|undefined, Array<SmoketestDfn>>}
+ * @return {Map<string|undefined, Array<Smokehouse.TestDfn>>}
  */
 function getSmoketestBatches(argv) {
   let smokes = [];
diff --git a/lighthouse-cli/test/smokehouse/smoke-test-dfns.js b/lighthouse-cli/test/smokehouse/smoke-test-dfns.js
new file mode 100644
index 000000000..6ca5d920d
--- /dev/null
+++ b/lighthouse-cli/test/smokehouse/smoke-test-dfns.js
@@ -0,0 +1,133 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const path = require('path');
+const smokehouseDir = 'lighthouse-cli/test/smokehouse/';
+
+/** @type {Array<Smokehouse.TestDfn>} */
+const SMOKE_TEST_DFNS = [{
+  id: 'a11y',
+  config: smokehouseDir + 'a11y/a11y-config.js',
+  expectations: 'a11y/expectations.js',
+  batch: 'parallel-first',
+}, {
+  id: 'errors',
+  expectations: smokehouseDir + 'error-expectations.js',
+  config: smokehouseDir + 'error-config.js',
+  batch: 'errors',
+}, {
+  id: 'oopif',
+  expectations: smokehouseDir + 'oopif-expectations.js',
+  config: smokehouseDir + 'oopif-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'pwa',
+  expectations: smokehouseDir + 'pwa-expectations.js',
+  config: smokehouseDir + 'pwa-config.js',
+  batch: 'parallel-second',
+}, {
+  id: 'pwa2',
+  expectations: smokehouseDir + 'pwa2-expectations.js',
+  config: smokehouseDir + 'pwa-config.js',
+  batch: 'parallel-second',
+}, {
+  id: 'pwa3',
+  expectations: smokehouseDir + 'pwa3-expectations.js',
+  config: smokehouseDir + 'pwa-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'dbw',
+  expectations: 'dobetterweb/dbw-expectations.js',
+  config: smokehouseDir + 'dbw-config.js',
+  batch: 'parallel-second',
+}, {
+  id: 'redirects',
+  expectations: 'redirects/expectations.js',
+  config: smokehouseDir + 'redirects-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'seo',
+  expectations: 'seo/expectations.js',
+  config: smokehouseDir + 'seo-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'offline',
+  expectations: 'offline-local/offline-expectations.js',
+  config: smokehouseDir + 'offline-config.js',
+  batch: 'offline',
+}, {
+  id: 'byte',
+  expectations: 'byte-efficiency/expectations.js',
+  config: smokehouseDir + 'byte-config.js',
+  batch: 'perf-opportunity',
+}, {
+  id: 'perf',
+  expectations: 'perf/expectations.js',
+  config: 'lighthouse-core/config/perf-config.js',
+  batch: 'perf-metric',
+}, {
+  id: 'lantern',
+  expectations: 'perf/lantern-expectations.js',
+  config: smokehouseDir + 'lantern-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'metrics',
+  expectations: 'tricky-metrics/expectations.js',
+  config: 'lighthouse-core/config/perf-config.js',
+  batch: 'parallel-second',
+}];
+
+/**
+ * Attempt to resolve a path relative to the smokehouse folder.
+ * If this fails, attempts to locate the path
+ * relative to the project root.
+ * @param {string} payloadPath
+ * @return {string}
+ */
+function resolveLocalOrProjectRoot(payloadPath) {
+  let resolved;
+  try {
+    resolved = require.resolve(__dirname + '/' + payloadPath);
+  } catch (e) {
+    const cwdPath = path.resolve(__dirname + '/../../../', payloadPath);
+    resolved = require.resolve(cwdPath);
+  }
+
+  return resolved;
+}
+
+/**
+ * @param {string} configPath
+ * @return {LH.Config.Json}
+ */
+function loadConfig(configPath) {
+  return require(configPath);
+}
+
+/**
+ * @param {string} expectationsPath
+ * @return {Smokehouse.ExpectedLHR[]}
+ */
+function loadExpectations(expectationsPath) {
+  return require(expectationsPath);
+}
+
+function getSmokeTests() {
+  return SMOKE_TEST_DFNS.map(smokeTestDfn => {
+    return {
+      id: smokeTestDfn.id,
+      config: loadConfig(resolveLocalOrProjectRoot(smokeTestDfn.config)),
+      expectations: loadExpectations(resolveLocalOrProjectRoot(smokeTestDfn.expectations)),
+      batch: smokeTestDfn.batch,
+    };
+  });
+}
+
+module.exports = {
+  SMOKE_TEST_DFNS,
+  getSmokeTests,
+};
diff --git a/lighthouse-cli/test/smokehouse/smokehouse-report.js b/lighthouse-cli/test/smokehouse/smokehouse-report.js
new file mode 100644
index 000000000..a08349928
--- /dev/null
+++ b/lighthouse-cli/test/smokehouse/smokehouse-report.js
@@ -0,0 +1,233 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+/* eslint-disable no-console */
+'use strict';
+
+const log = require('lighthouse-logger');
+
+const VERBOSE = Boolean(process.env.LH_SMOKE_VERBOSE);
+const NUMBER_REGEXP = /(?:\d|\.)+/.source;
+const OPS_REGEXP = /<=?|>=?|\+\/-/.source;
+// An optional number, optional whitespace, an operator, optional whitespace, a number.
+const NUMERICAL_EXPECTATION_REGEXP =
+  new RegExp(`^(${NUMBER_REGEXP})?\\s*(${OPS_REGEXP})\\s*(${NUMBER_REGEXP})$`);
+
+/**
+ * Checks if the actual value matches the expectation. Does not recursively search. This supports
+ *    - Greater than/less than operators, e.g. "<100", ">90"
+ *    - Regular expressions
+ *    - Strict equality
+ *
+ * @param {*} actual
+ * @param {*} expected
+ * @return {boolean}
+ */
+function matchesExpectation(actual, expected) {
+  if (typeof actual === 'number' && NUMERICAL_EXPECTATION_REGEXP.test(expected)) {
+    const parts = expected.match(NUMERICAL_EXPECTATION_REGEXP);
+    const [, prefixNumber, operator, postfixNumber] = parts;
+    switch (operator) {
+      case '>':
+        return actual > postfixNumber;
+      case '>=':
+        return actual >= postfixNumber;
+      case '<':
+        return actual < postfixNumber;
+      case '<=':
+        return actual <= postfixNumber;
+      case '+/-':
+        return Math.abs(actual - prefixNumber) <= postfixNumber;
+      default:
+        throw new Error(`unexpected operator ${operator}`);
+    }
+  } else if (typeof actual === 'string' && expected instanceof RegExp && expected.test(actual)) {
+    return true;
+  } else {
+    // Strict equality check, plus NaN equivalence.
+    return Object.is(actual, expected);
+  }
+}
+
+/**
+ * Walk down expected result, comparing to actual result. If a difference is found,
+ * the path to the difference is returned, along with the expected primitive value
+ * and the value actually found at that location. If no difference is found, returns
+ * null.
+ *
+ * Only checks own enumerable properties, not object prototypes, and will loop
+ * until the stack is exhausted, so works best with simple objects (e.g. parsed JSON).
+ * @param {string} path
+ * @param {*} actual
+ * @param {*} expected
+ * @return {(Smokehouse.Difference|null)}
+ */
+function findDifference(path, actual, expected) {
+  if (matchesExpectation(actual, expected)) {
+    return null;
+  }
+
+  // If they aren't both an object we can't recurse further, so this is the difference.
+  if (actual === null || expected === null || typeof actual !== 'object' ||
+      typeof expected !== 'object' || expected instanceof RegExp) {
+    return {
+      path,
+      actual,
+      expected,
+    };
+  }
+
+  // We only care that all expected's own properties are on actual (and not the other way around).
+  for (const key of Object.keys(expected)) {
+    // Bracket numbers, but property names requiring quotes will still be unquoted.
+    const keyAccessor = /^\d+$/.test(key) ? `[${key}]` : `.${key}`;
+    const keyPath = path + keyAccessor;
+    const expectedValue = expected[key];
+
+    if (!(key in actual)) {
+      return {path: keyPath, actual: undefined, expected: expectedValue};
+    }
+
+    const actualValue = actual[key];
+    const subDifference = findDifference(keyPath, actualValue, expectedValue);
+
+    // Break on first difference found.
+    if (subDifference) {
+      return subDifference;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Collate results into comparisons of actual and expected scores on each audit.
+ * @param {Smokehouse.ExpectedLHR} actual
+ * @param {Smokehouse.ExpectedLHR} expected
+ * @return {Smokehouse.LHRComparison}
+ */
+function collateResults(actual, expected) {
+  const auditNames = Object.keys(expected.audits);
+  const collatedAudits = auditNames.map(auditName => {
+    const actualResult = actual.audits[auditName];
+    if (!actualResult) {
+      throw new Error(`Config did not trigger run of expected audit ${auditName}`);
+    }
+
+    const expectedResult = expected.audits[auditName];
+    const diff = findDifference(auditName, actualResult, expectedResult);
+
+    return {
+      category: auditName,
+      actual: actualResult,
+      expected: expectedResult,
+      equal: !diff,
+      diff,
+    };
+  });
+
+  return {
+    audits: collatedAudits,
+    errorCode: {
+      category: 'error code',
+      actual: actual.errorCode,
+      expected: expected.errorCode,
+      equal: actual.errorCode === expected.errorCode,
+    },
+    finalUrl: {
+      category: 'final url',
+      actual: actual.finalUrl,
+      expected: expected.finalUrl,
+      equal: actual.finalUrl === expected.finalUrl,
+    },
+  };
+}
+
+/**
+ * @param {unknown} obj
+ */
+function isPlainObject(obj) {
+  return Object.prototype.toString.call(obj) === '[object Object]';
+}
+
+/**
+ * Log the result of an assertion of actual and expected results.
+ * @param {Smokehouse.Comparison} assertion
+ */
+function reportAssertion(assertion) {
+  // @ts-ignore - this doesn't exist now but could one day, so try not to break the future
+  const _toJSON = RegExp.prototype.toJSON;
+  // @ts-ignore
+  // eslint-disable-next-line no-extend-native
+  RegExp.prototype.toJSON = RegExp.prototype.toString;
+
+  if (assertion.equal) {
+    if (isPlainObject(assertion.actual)) {
+      console.log(`  ${log.greenify(log.tick)} ${assertion.category}`);
+    } else {
+      console.log(`  ${log.greenify(log.tick)} ${assertion.category}: ` +
+          log.greenify(assertion.actual));
+    }
+  } else {
+    if (assertion.diff) {
+      const diff = assertion.diff;
+      const fullActual = JSON.stringify(assertion.actual, null, 2).replace(/\n/g, '\n      ');
+      const msg = `
+  ${log.redify(log.cross)} difference at ${log.bold}${diff.path}${log.reset}
+              expected: ${JSON.stringify(diff.expected)}
+                 found: ${JSON.stringify(diff.actual)}
+
+          found result:
+      ${log.redify(fullActual)}
+`;
+      console.log(msg);
+    } else {
+      console.log(`  ${log.redify(log.cross)} ${assertion.category}:
+              expected: ${JSON.stringify(assertion.expected)}
+                 found: ${JSON.stringify(assertion.actual)}
+`);
+    }
+  }
+
+  // @ts-ignore
+  // eslint-disable-next-line no-extend-native
+  RegExp.prototype.toJSON = _toJSON;
+}
+
+/**
+ * Log all the comparisons between actual and expected test results, then print
+ * summary. Returns count of passed and failed tests.
+ * @param {Smokehouse.LHRComparison} results
+ * @return {{passed: number, failed: number}}
+ */
+function report(results) {
+  let correctCount = 0;
+  let failedCount = 0;
+
+  [results.finalUrl, results.errorCode, ...results.audits].forEach(auditAssertion => {
+    if (auditAssertion.equal) {
+      correctCount++;
+    } else {
+      failedCount++;
+    }
+
+    if (!auditAssertion.equal || VERBOSE) {
+      reportAssertion(auditAssertion);
+    }
+  });
+
+  const plural = correctCount === 1 ? '' : 's';
+  const correctStr = `${correctCount} assertion${plural}`;
+  const colorFn = correctCount === 0 ? log.redify : log.greenify;
+  console.log(`  Correctly passed ${colorFn(correctStr)}\n`);
+
+  return {
+    passed: correctCount,
+    failed: failedCount,
+  };
+}
+
+module.exports.collateResults = collateResults;
+module.exports.report = report;
diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js
index be6534d35..eb5d74a61 100755
--- a/lighthouse-cli/test/smokehouse/smokehouse.js
+++ b/lighthouse-cli/test/smokehouse/smokehouse.js
@@ -6,22 +6,6 @@
  */
 'use strict';
 
-/**
- * @typedef {{path: string, actual: *, expected: *}} Difference
- */
-
-/**
- * @typedef {{category: string, actual: *, expected: *, equal: boolean, diff?: Difference | null}} Comparison
- */
-
-/**
- * @typedef {Pick<LH.Result, 'audits' | 'finalUrl' | 'requestedUrl'> & {errorCode?: string}} ExpectedLHR
- */
-
-/**
- * @typedef {{audits: Comparison[], errorCode: Comparison, finalUrl: Comparison}} LHRComparison
- */
-
 /* eslint-disable no-console */
 
 const fs = require('fs');
@@ -29,17 +13,12 @@ const path = require('path');
 const spawnSync = require('child_process').spawnSync;
 const yargs = require('yargs');
 const log = require('lighthouse-logger');
+const {collateResults, report} = require('./smokehouse-report');
 
 const PROTOCOL_TIMEOUT_EXIT_CODE = 67;
 const PAGE_HUNG_EXIT_CODE = 68;
 const INSECURE_DOCUMENT_REQUEST_EXIT_CODE = 69;
 const RETRIES = 3;
-const VERBOSE = Boolean(process.env.LH_SMOKE_VERBOSE);
-const NUMBER_REGEXP = /(?:\d|\.)+/.source;
-const OPS_REGEXP = /<=?|>=?|\+\/-/.source;
-// An optional number, optional whitespace, an operator, optional whitespace, a number.
-const NUMERICAL_EXPECTATION_REGEXP =
-  new RegExp(`^(${NUMBER_REGEXP})?\\s*(${OPS_REGEXP})\\s*(${NUMBER_REGEXP})$`);
 
 /**
  * Attempt to resolve a path locally. If this fails, attempts to locate the path
@@ -64,7 +43,7 @@ function resolveLocalOrCwd(payloadPath) {
  * @param {string} url
  * @param {string} configPath
  * @param {boolean=} isDebug
- * @return {ExpectedLHR}
+ * @return {Smokehouse.ExpectedLHR}
  */
 function runLighthouse(url, configPath, isDebug) {
   isDebug = isDebug || Boolean(process.env.LH_SMOKE_DEBUG);
@@ -140,220 +119,6 @@ function runLighthouse(url, configPath, isDebug) {
   return JSON.parse(lhr);
 }
 
-/**
- * Checks if the actual value matches the expectation. Does not recursively search. This supports
- *    - Greater than/less than operators, e.g. "<100", ">90"
- *    - Regular expressions
- *    - Strict equality
- *
- * @param {*} actual
- * @param {*} expected
- * @return {boolean}
- */
-function matchesExpectation(actual, expected) {
-  if (typeof actual === 'number' && NUMERICAL_EXPECTATION_REGEXP.test(expected)) {
-    const parts = expected.match(NUMERICAL_EXPECTATION_REGEXP);
-    const [, prefixNumber, operator, postfixNumber] = parts;
-    switch (operator) {
-      case '>':
-        return actual > postfixNumber;
-      case '>=':
-        return actual >= postfixNumber;
-      case '<':
-        return actual < postfixNumber;
-      case '<=':
-        return actual <= postfixNumber;
-      case '+/-':
-        return Math.abs(actual - prefixNumber) <= postfixNumber;
-      default:
-        throw new Error(`unexpected operator ${operator}`);
-    }
-  } else if (typeof actual === 'string' && expected instanceof RegExp && expected.test(actual)) {
-    return true;
-  } else {
-    // Strict equality check, plus NaN equivalence.
-    return Object.is(actual, expected);
-  }
-}
-
-/**
- * Walk down expected result, comparing to actual result. If a difference is found,
- * the path to the difference is returned, along with the expected primitive value
- * and the value actually found at that location. If no difference is found, returns
- * null.
- *
- * Only checks own enumerable properties, not object prototypes, and will loop
- * until the stack is exhausted, so works best with simple objects (e.g. parsed JSON).
- * @param {string} path
- * @param {*} actual
- * @param {*} expected
- * @return {(Difference|null)}
- */
-function findDifference(path, actual, expected) {
-  if (matchesExpectation(actual, expected)) {
-    return null;
-  }
-
-  // If they aren't both an object we can't recurse further, so this is the difference.
-  if (actual === null || expected === null || typeof actual !== 'object' ||
-      typeof expected !== 'object' || expected instanceof RegExp) {
-    return {
-      path,
-      actual,
-      expected,
-    };
-  }
-
-  // We only care that all expected's own properties are on actual (and not the other way around).
-  for (const key of Object.keys(expected)) {
-    // Bracket numbers, but property names requiring quotes will still be unquoted.
-    const keyAccessor = /^\d+$/.test(key) ? `[${key}]` : `.${key}`;
-    const keyPath = path + keyAccessor;
-    const expectedValue = expected[key];
-
-    if (!(key in actual)) {
-      return {path: keyPath, actual: undefined, expected: expectedValue};
-    }
-
-    const actualValue = actual[key];
-    const subDifference = findDifference(keyPath, actualValue, expectedValue);
-
-    // Break on first difference found.
-    if (subDifference) {
-      return subDifference;
-    }
-  }
-
-  return null;
-}
-
-/**
- * Collate results into comparisons of actual and expected scores on each audit.
- * @param {ExpectedLHR} actual
- * @param {ExpectedLHR} expected
- * @return {LHRComparison}
- */
-function collateResults(actual, expected) {
-  const auditNames = Object.keys(expected.audits);
-  const collatedAudits = auditNames.map(auditName => {
-    const actualResult = actual.audits[auditName];
-    if (!actualResult) {
-      throw new Error(`Config did not trigger run of expected audit ${auditName}`);
-    }
-
-    const expectedResult = expected.audits[auditName];
-    const diff = findDifference(auditName, actualResult, expectedResult);
-
-    return {
-      category: auditName,
-      actual: actualResult,
-      expected: expectedResult,
-      equal: !diff,
-      diff,
-    };
-  });
-
-  return {
-    audits: collatedAudits,
-    errorCode: {
-      category: 'error code',
-      actual: actual.errorCode,
-      expected: expected.errorCode,
-      equal: actual.errorCode === expected.errorCode,
-    },
-    finalUrl: {
-      category: 'final url',
-      actual: actual.finalUrl,
-      expected: expected.finalUrl,
-      equal: actual.finalUrl === expected.finalUrl,
-    },
-  };
-}
-
-/**
- * @param {unknown} obj
- */
-function isPlainObject(obj) {
-  return Object.prototype.toString.call(obj) === '[object Object]';
-}
-
-/**
- * Log the result of an assertion of actual and expected results.
- * @param {Comparison} assertion
- */
-function reportAssertion(assertion) {
-  // @ts-ignore - this doesn't exist now but could one day, so try not to break the future
-  const _toJSON = RegExp.prototype.toJSON;
-  // @ts-ignore
-  // eslint-disable-next-line no-extend-native
-  RegExp.prototype.toJSON = RegExp.prototype.toString;
-
-  if (assertion.equal) {
-    if (isPlainObject(assertion.actual)) {
-      console.log(`  ${log.greenify(log.tick)} ${assertion.category}`);
-    } else {
-      console.log(`  ${log.greenify(log.tick)} ${assertion.category}: ` +
-          log.greenify(assertion.actual));
-    }
-  } else {
-    if (assertion.diff) {
-      const diff = assertion.diff;
-      const fullActual = JSON.stringify(assertion.actual, null, 2).replace(/\n/g, '\n      ');
-      const msg = `
-  ${log.redify(log.cross)} difference at ${log.bold}${diff.path}${log.reset}
-              expected: ${JSON.stringify(diff.expected)}
-                 found: ${JSON.stringify(diff.actual)}
-
-          found result:
-      ${log.redify(fullActual)}
-`;
-      console.log(msg);
-    } else {
-      console.log(`  ${log.redify(log.cross)} ${assertion.category}:
-              expected: ${JSON.stringify(assertion.expected)}
-                 found: ${JSON.stringify(assertion.actual)}
-`);
-    }
-  }
-
-  // @ts-ignore
-  // eslint-disable-next-line no-extend-native
-  RegExp.prototype.toJSON = _toJSON;
-}
-
-/**
- * Log all the comparisons between actual and expected test results, then print
- * summary. Returns count of passed and failed tests.
- * @param {LHRComparison} results
- * @return {{passed: number, failed: number}}
- */
-function report(results) {
-  let correctCount = 0;
-  let failedCount = 0;
-
-  [results.finalUrl, results.errorCode, ...results.audits].forEach(auditAssertion => {
-    if (auditAssertion.equal) {
-      correctCount++;
-    } else {
-      failedCount++;
-    }
-
-    if (!auditAssertion.equal || VERBOSE) {
-      reportAssertion(auditAssertion);
-    }
-  });
-
-  const plural = correctCount === 1 ? '' : 's';
-  const correctStr = `${correctCount} assertion${plural}`;
-  const colorFn = correctCount === 0 ? log.redify : log.greenify;
-  console.log(`  Correctly passed ${colorFn(correctStr)}\n`);
-
-  return {
-    passed: correctCount,
-    failed: failedCount,
-  };
-}
-
 const cli = yargs
   .help('help')
   .describe({
@@ -366,7 +131,7 @@ const cli = yargs
   .argv;
 
 const configPath = resolveLocalOrCwd(cli['config-path']);
-/** @type {ExpectedLHR[]} */
+/** @type {Smokehouse.ExpectedLHR[]} */
 const expectations = require(resolveLocalOrCwd(cli['expectations-path']));
 
 // Loop sequentially over expectations, comparing against Lighthouse run, and
diff --git a/lighthouse-core/test/audits/seo/font-size-test.js b/lighthouse-core/test/audits/seo/font-size-test.js
index c075fe132..8c093e1d2 100644
--- a/lighthouse-core/test/audits/seo/font-size-test.js
+++ b/lighthouse-core/test/audits/seo/font-size-test.js
@@ -22,6 +22,7 @@ describe('SEO: Font size audit', () => {
       URL,
       MetaElements: [],
       FontSize: [],
+      TestedAsMobileDevice: true,
     };
 
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
@@ -45,6 +46,7 @@ describe('SEO: Font size audit', () => {
           {textLength: 31, fontSize: 11, node: {nodeId: 2, localName: 'p', attributes: []}},
         ],
       },
+      TestedAsMobileDevice: true,
     };
 
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
@@ -66,6 +68,7 @@ describe('SEO: Font size audit', () => {
           {textLength: 0, fontSize: 11, node: {nodeId: 1, localName: 'p', attributes: []}},
         ],
       },
+      TestedAsMobileDevice: true,
     };
 
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
@@ -86,6 +89,7 @@ describe('SEO: Font size audit', () => {
           {textLength: 22, fontSize: 11, node: {nodeId: 2, localName: 'p', attributes: []}},
         ],
       },
+      TestedAsMobileDevice: true,
     };
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
     assert.equal(auditResult.rawValue, true);
@@ -123,6 +127,7 @@ describe('SEO: Font size audit', () => {
           {textLength: 2, fontSize: 10, node: {nodeId: 3}, cssRule: style2},
         ],
       },
+      TestedAsMobileDevice: true,
     };
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
 
@@ -145,6 +150,7 @@ describe('SEO: Font size audit', () => {
           {textLength: 10, fontSize: 10, node: {nodeId: 1, localName: 'p', attributes: []}},
         ],
       },
+      TestedAsMobileDevice: true,
     };
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
     assert.equal(auditResult.rawValue, false);
@@ -167,11 +173,13 @@ describe('SEO: Font size audit', () => {
           {textLength: 50, fontSize: 10, node: {nodeId: 1, localName: 'p', attributes: []}},
         ],
       },
+      TestedAsMobileDevice: true,
     };
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
     assert.equal(auditResult.rawValue, false);
-    expect(auditResult.explanation)
-      .toBeDisplayString('100% of text is too small (based on 50% sample).');
+    expect(auditResult.explanation).toBeDisplayString(
+      '100% of text is too small (based on 50% sample).'
+    );
     expect(auditResult.displayValue).toBeDisplayString('0% legible text');
   });
 
@@ -189,6 +197,7 @@ describe('SEO: Font size audit', () => {
           {textLength: 22, fontSize: 11, node: {nodeId: 2, localName: 'p', attributes: []}},
         ],
       },
+      TestedAsMobileDevice: true,
     };
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
     expect(auditResult.displayValue).toBeDisplayString('89.78% legible text');
@@ -208,8 +217,21 @@ describe('SEO: Font size audit', () => {
           {textLength: 4, fontSize: 11, node: {nodeId: 2, localName: 'p', attributes: []}},
         ],
       },
+      TestedAsMobileDevice: true,
     };
     const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
     expect(auditResult.displayValue).toBeDisplayString('2.48% legible text');
   });
+
+  it('is not applicable on desktop', async () => {
+    const artifacts = {
+      URL,
+      MetaElements: [],
+      FontSize: {},
+      TestedAsMobileDevice: false,
+    };
+    const auditResult = await FontSizeAudit.audit(artifacts, getFakeContext());
+    expect(auditResult.rawValue).toBe(true);
+    expect(auditResult.notApplicable).toBe(true);
+  });
 });
diff --git a/lighthouse-core/test/audits/seo/tap-targets-test.js b/lighthouse-core/test/audits/seo/tap-targets-test.js
index 977143a50..3683d9b0a 100644
--- a/lighthouse-core/test/audits/seo/tap-targets-test.js
+++ b/lighthouse-core/test/audits/seo/tap-targets-test.js
@@ -12,13 +12,14 @@ const assert = require('assert');
 
 const getFakeContext = () => ({computedCache: new Map()});
 
-function auditTapTargets(tapTargets, metaElements = [{
+function auditTapTargets(tapTargets, {MetaElements = [{
   name: 'viewport',
   content: 'width=device-width',
-}]) {
+}], TestedAsMobileDevice = true} = {}) {
   const artifacts = {
     TapTargets: tapTargets,
-    MetaElements: metaElements,
+    MetaElements,
+    TestedAsMobileDevice,
   };
 
   return TapTargetsAudit.audit(artifacts, getFakeContext());
@@ -203,11 +204,19 @@ describe('SEO: Tap targets audit', () => {
   });
 
   it('fails if no meta viewport tag is provided', async () => {
-    const auditResult = await auditTapTargets([], []);
+    const auditResult = await auditTapTargets([], {MetaElements: []});
     assert.equal(auditResult.rawValue, false);
 
     expect(auditResult.explanation).toBeDisplayString(
       /* eslint-disable max-len */
       'Tap targets are too small because there\'s no viewport meta tag optimized for mobile screens');
   });
+
+  it('is not applicable on desktop', async () => {
+    const auditResult = await auditTapTargets(getBorderlineTapTargets({
+      overlapSecondClientRect: true,
+    }), {TestedAsMobileDevice: false});
+    assert.equal(auditResult.rawValue, true);
+    assert.equal(auditResult.notApplicable, true);
+  });
 });