Repository: GoogleChrome/lighthouse
Pull request #12367 (2021-04-15 19:30:35)

core(fr): filter out manual-only categories
**Summary**
Improves the filtering of categories in Fraggle Rock to remove categories that consist only manual audits. This eliminates the UX oddity that you'd run timespan mode and get a `?` score for accessibility category because all its *real* audits were snapshot audits.

**Related Issues/PRs**
ref #11313 



Patch:
diff --git a/docs/architecture.md b/docs/architecture.md
index a1c3d9678..b487f229d 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -7,10 +7,12 @@ _Some incomplete notes_
 ## Components & Terminology
 
 * **Driver** - Interfaces with [Chrome Debugging Protocol](https://developer.chrome.com/devtools/docs/debugger-protocol)  ([API viewer](https://chromedevtools.github.io/debugger-protocol-viewer/))
-* **Gatherers** - Uses Driver to collect information about the page. Minimal post-processing.
-  * **Artifacts** - output of a gatherer
-* **Audit** - Tests for a single feature/optimization/metric. Using the Artifacts as input, an audit evaluates a test and resolves to a numeric score. See [Understanding Results](./understanding-results.md) for details of the LHR (Lighthouse Result object).
-  * **Computed Artifacts** - Generated on-demand from artifacts, these add additional meaning, and are often shared amongst multiple audits.
+* **Gatherers** - Uses Driver to collect information about the page. Minimal post-processing.  Run Lighthouse with `--gather-mode` to see the 3 primary outputs from gathering:
+  1. `artifacts.json`: The output from all [gatherers](../lighthouse-core/gather/gatherers).
+  2. `defaultPass.trace.json`: Most performance characteristics come from here. You can view it in the DevTools Peformance panel.
+  3. `defaultPass.devtoolslog.json`: A log of all the [DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/) events. Primary signal about network requests and page state.
+* **Audit** - The [audits](../lighthouse-core/audits) are tests for a single feature/optimization/metric. Using the Artifacts as input, an audit evaluates a test and resolves to a numeric score. See [Understanding Results](./understanding-results.md) for details of the LHR (Lighthouse Result object).
+  * **Computed Artifacts** - [Generated](../lighthouse-core/computed) on-demand from artifacts, these add additional meaning, and are often shared amongst multiple audits.
 * **Report** - The report UI, created client-side from the LHR. See [HTML Report Generation Overview](../lighthouse-core/report/html/readme.md) for details.
 
 ### Audit/Report terminology
diff --git a/lighthouse-core/fraggle-rock/config/filters.js b/lighthouse-core/fraggle-rock/config/filters.js
index d675a19ab..ec04449ac 100644
--- a/lighthouse-core/fraggle-rock/config/filters.js
+++ b/lighthouse-core/fraggle-rock/config/filters.js
@@ -5,6 +5,8 @@
  */
 'use strict';
 
+const Audit = require('../../audits/audit.js');
+
 /**
  * Filters an array of artifacts down to the set that supports the specified gather mode.
  *
@@ -47,17 +49,32 @@ function filterAuditsByAvailableArtifacts(audits, availableArtifacts) {
 function filterCategoriesByAvailableAudits(categories, availableAudits) {
   if (!categories) return categories;
 
-  const availableAuditIds = new Set(availableAudits.map(audit => audit.implementation.meta.id));
+  const availableAuditIdToMeta = new Map(
+    availableAudits.map(audit => [audit.implementation.meta.id, audit.implementation.meta])
+  );
 
-  return Object.fromEntries(
-    Object.entries(categories).map(([categoryId, category]) => {
+  const categoryEntries = Object.entries(categories)
+    .map(([categoryId, category]) => {
       const filteredCategory = {
         ...category,
-        auditRefs: category.auditRefs.filter(ref => availableAuditIds.has(ref.id)),
+        auditRefs: category.auditRefs.filter(ref => availableAuditIdToMeta.has(ref.id)),
       };
+
+      const didFilter = filteredCategory.auditRefs.length < category.auditRefs.length;
+      const hasOnlyManualAudits = filteredCategory.auditRefs.every(ref => {
+        const meta = availableAuditIdToMeta.get(ref.id);
+        if (!meta) return false;
+        return meta.scoreDisplayMode === Audit.SCORING_MODES.MANUAL;
+      });
+
+      // If we filtered out audits and the only ones left are manual, remove them too.
+      if (didFilter && hasOnlyManualAudits) filteredCategory.auditRefs = [];
+
       return [categoryId, filteredCategory];
     })
-  );
+    .filter(entry => typeof entry[1] === 'object' && entry[1].auditRefs.length);
+
+  return Object.fromEntries(categoryEntries);
 }
 
 /**


Test patch:
diff --git a/lighthouse-core/test/fraggle-rock/config/filters-test.js b/lighthouse-core/test/fraggle-rock/config/filters-test.js
index 99479e9d2..131f6f137 100644
--- a/lighthouse-core/test/fraggle-rock/config/filters-test.js
+++ b/lighthouse-core/test/fraggle-rock/config/filters-test.js
@@ -47,6 +47,14 @@ describe('Fraggle Rock Config Filtering', () => {
       ...auditMeta,
     };
   }
+  class ManualAudit extends BaseAudit {
+    static meta = {
+      id: 'manual',
+      scoreDisplayMode: BaseAudit.SCORING_MODES.MANUAL,
+      requiredArtifacts: /** @type {any} */ ([]),
+      ...auditMeta,
+    };
+  }
   class TimespanAudit extends BaseAudit {
     static meta = {
       id: 'timespan',
@@ -62,7 +70,7 @@ describe('Fraggle Rock Config Filtering', () => {
     };
   }
 
-  const audits = [SnapshotAudit, TimespanAudit, NavigationAudit].map(audit => ({
+  const audits = [SnapshotAudit, TimespanAudit, NavigationAudit, ManualAudit].map(audit => ({
     implementation: audit,
     options: {},
   }));
@@ -76,6 +84,7 @@ describe('Fraggle Rock Config Filtering', () => {
       const partialArtifacts = [{id: 'Snapshot', gatherer: {instance: snapshotGatherer}}];
       expect(filters.filterAuditsByAvailableArtifacts(audits, partialArtifacts)).toEqual([
         {implementation: SnapshotAudit, options: {}},
+        {implementation: ManualAudit, options: {}},
       ]);
     });
 
@@ -89,6 +98,7 @@ describe('Fraggle Rock Config Filtering', () => {
     snapshot: {title: 'Snapshot', auditRefs: [{id: 'snapshot', weight: 0}]},
     timespan: {title: 'Timespan', auditRefs: [{id: 'timespan', weight: 0}]},
     navigation: {title: 'Navigation', auditRefs: [{id: 'navigation', weight: 0}]},
+    manual: {title: 'Manual', auditRefs: [{id: 'manual', weight: 0}]},
     mixed: {
       title: 'Mixed',
       auditRefs: [
@@ -106,12 +116,36 @@ describe('Fraggle Rock Config Filtering', () => {
 
     it('should filter entire categories', () => {
       const partialAudits = [{implementation: SnapshotAudit, options: {}}];
-      expect(filters.filterCategoriesByAvailableAudits(categories, partialAudits)).toMatchObject({
+      const filtered = filters.filterCategoriesByAvailableAudits(categories, partialAudits);
+      expect(filtered).not.toMatchObject({
+        timespan: {},
+        navigation: {},
+      });
+      expect(filtered).toMatchObject({
         snapshot: {},
         mixed: {},
       });
     });
 
+    it('should filter entire categories when all remaining audits are manual', () => {
+      const partialAudits = [
+        {implementation: SnapshotAudit, options: {}},
+        {implementation: ManualAudit, options: {}},
+      ];
+
+      const filteredCategories = filters.filterCategoriesByAvailableAudits({
+        snapshot: categories.snapshot,
+        timespanWithManual: {
+          title: 'Timespan + Manual',
+          auditRefs: [
+            {id: 'timespan', weight: 0},
+            {id: 'manual', weight: 0},
+          ],
+        },
+      }, partialAudits);
+      expect(filteredCategories).not.toHaveProperty('timespanWithManual');
+    });
+
     it('should filter audits within categories', () => {
       const partialAudits = [{implementation: SnapshotAudit, options: {}}];
       const filtered = filters.filterCategoriesByAvailableAudits(categories, partialAudits);
@@ -139,8 +173,12 @@ describe('Fraggle Rock Config Filtering', () => {
       };
       expect(filters.filterConfigByGatherMode(config, 'snapshot')).toMatchObject({
         artifacts: [{id: 'Snapshot'}],
-        audits: [{implementation: SnapshotAudit}],
-        categories: {snapshot: {}, mixed: {auditRefs: [{id: 'snapshot'}]}},
+        audits: [{implementation: SnapshotAudit}, {implementation: ManualAudit}],
+        categories: {
+          snapshot: {},
+          manual: {},
+          mixed: {auditRefs: [{id: 'snapshot'}]},
+        },
       });
     });
   });