Repository: GoogleChrome/lighthouse (js)

Issue:
SpeedIndex not accurate with non-standard Lantern settings
<!-- Before creating an issue please make sure you are using the latest version and have checked for duplicate issues. -->

<!-- Before creating an Accessibility issue please test that it is reproducible upstream with axe (https://www.deque.com/axe/) first and file the issue there if necessary. -->

#### Provide the steps to reproduce
1. Run LH on https://facebook.com or any URL really

#### What is the current behavior?
Speed Index is overinflated by ~70%
#### What is the expected behavior?
Speed Index is more inline with FCP/FMP

#### Environment Information
* Affected Channels: Primarily LR desktop, CLI
* Lighthouse version: *
* Node.js version: *
* Operating System: *

**Related**
#6836  #6969


Pull request:
core(speedindex): scale coefficients according to throttling
**Summary**
This PR makes it possible to scale the lantern coefficients based on the throttling settings. Previously the desktop LR preset was giving *too* pessimistic estimates on Speed Index because of the hardcoded `1.4x` multiplier on the observed speed index.

Key Example Fix (facebook.com)

**Before**
![image](https://user-images.githubusercontent.com/2301202/51143644-9897ab80-1814-11e9-9f78-796d8b667634.png)


**After**
![image](https://user-images.githubusercontent.com/2301202/51143591-7f8efa80-1814-11e9-80b2-dcd85e51ab4e.png)


**Related Issues/PRs**
fixes #6971 


Patch:
diff --git a/lighthouse-core/computed/metrics/lantern-metric.js b/lighthouse-core/computed/metrics/lantern-metric.js
index 9362d589b..5c6fd1c50 100644
--- a/lighthouse-core/computed/metrics/lantern-metric.js
+++ b/lighthouse-core/computed/metrics/lantern-metric.js
@@ -42,6 +42,19 @@ class LanternMetricArtifact {
     throw new Error('COEFFICIENTS unimplemented!');
   }
 
+  /**
+   * Returns the coefficients, scaled by the throttling settings if needed by the metric.
+   * Some lantern metrics (speed-index) use components in their estimate that are not
+   * from the simulator. In this case, we need to adjust the coefficients as the target throttling
+   * settings change.
+   *
+   * @param {number} rttMs
+   * @return {LH.Gatherer.Simulation.MetricCoefficients}
+   */
+  static getScaledCoefficients(rttMs) { // eslint-disable-line no-unused-vars
+    return this.COEFFICIENTS;
+  }
+
   /**
    * @param {Node} dependencyGraph
    * @param {LH.Artifacts.TraceOfTab} traceOfTab
@@ -107,13 +120,14 @@ class LanternMetricArtifact {
       Object.assign({}, extras, {optimistic: false})
     );
 
+    const coefficients = this.getScaledCoefficients(simulator.rtt);
     // Estimates under 1s don't really follow the normal curve fit, minimize the impact of the intercept
-    const interceptMultiplier = this.COEFFICIENTS.intercept > 0 ?
+    const interceptMultiplier = coefficients.intercept > 0 ?
       Math.min(1, optimisticEstimate.timeInMs / 1000) : 1;
     const timing =
-      this.COEFFICIENTS.intercept * interceptMultiplier +
-      this.COEFFICIENTS.optimistic * optimisticEstimate.timeInMs +
-      this.COEFFICIENTS.pessimistic * pessimisticEstimate.timeInMs;
+      coefficients.intercept * interceptMultiplier +
+      coefficients.optimistic * optimisticEstimate.timeInMs +
+      coefficients.pessimistic * pessimisticEstimate.timeInMs;
 
     return {
       timing,
diff --git a/lighthouse-core/computed/metrics/lantern-speed-index.js b/lighthouse-core/computed/metrics/lantern-speed-index.js
index 7b6c5d53d..5e38e168d 100644
--- a/lighthouse-core/computed/metrics/lantern-speed-index.js
+++ b/lighthouse-core/computed/metrics/lantern-speed-index.js
@@ -10,6 +10,7 @@ const LanternMetric = require('./lantern-metric.js');
 const BaseNode = require('../../lib/dependency-graph/base-node.js');
 const Speedline = require('../speedline.js');
 const LanternFirstContentfulPaint = require('./lantern-first-contentful-paint.js');
+const defaultThrottling = require('../../config/constants.js').throttling;
 
 /** @typedef {BaseNode.Node} Node */
 
@@ -28,6 +29,33 @@ class LanternSpeedIndex extends LanternMetric {
     };
   }
 
+  /**
+   * @param {number} rttMs
+   * @return {LH.Gatherer.Simulation.MetricCoefficients}
+   */
+  static getScaledCoefficients(rttMs) { // eslint-disable-line no-unused-vars
+    // We want to scale our default coefficients based on the speed of the connection.
+    // We will linearly interpolate coefficients for the passed-in rttMs based on two pre-determined points:
+    //   1. Baseline point of 30 ms RTT where Speed Index should be a ~50/50 blend of optimistic/pessimistic.
+    //      30 ms was based on a typical home WiFi connection's actual RTT.
+    //      Coefficients here follow from the fact that the optimistic estimate should be very close
+    //      to reality at this connection speed and the pessimistic estimate compensates for minor
+    //      connection speed differences.
+    //   2. Default throttled point of 150 ms RTT where the default coefficients have been determined to be most accurate.
+    //      Coefficients here were determined through thorough analysis and linear regression on the
+    //      lantern test data set. See lighthouse-core/scripts/test-lantern.sh for more detail.
+    // While the coefficients haven't been analyzed at the interpolated points, it's our current best effort.
+    const defaultCoefficients = this.COEFFICIENTS;
+    const defaultRttExcess = defaultThrottling.mobileSlow4G.rttMs - 30;
+    const multiplier = Math.max((rttMs - 30) / defaultRttExcess, 0);
+
+    return {
+      intercept: defaultCoefficients.intercept * multiplier,
+      optimistic: 0.5 + (defaultCoefficients.optimistic - 0.5) * multiplier,
+      pessimistic: 0.5 + (defaultCoefficients.pessimistic - 0.5) * multiplier,
+    };
+  }
+
   /**
    * @param {Node} dependencyGraph
    * @return {Node}
diff --git a/lighthouse-core/lib/dependency-graph/simulator/simulator.js b/lighthouse-core/lib/dependency-graph/simulator/simulator.js
index d3b92b5dd..fe46d21c2 100644
--- a/lighthouse-core/lib/dependency-graph/simulator/simulator.js
+++ b/lighthouse-core/lib/dependency-graph/simulator/simulator.js
@@ -75,6 +75,11 @@ class Simulator {
     this._connectionPool = /** @type {ConnectionPool} */ (null);
   }
 
+  /** @return {number} */
+  get rtt() {
+    return this._rtt;
+  }
+
   /**
    * @param {Node} graph
    */
diff --git a/lighthouse-core/lib/i18n/en-US.json b/lighthouse-core/lib/i18n/en-US.json
index 3183629ed..084d1deef 100644
--- a/lighthouse-core/lib/i18n/en-US.json
+++ b/lighthouse-core/lib/i18n/en-US.json
@@ -1079,6 +1079,10 @@
     "message": "Values are estimated and may vary.",
     "description": "Disclaimer shown to users below the metric values (First Contentful Paint, Time to Interactive, etc) to warn them that the numbers they see will likely change slightly the next time they run Lighthouse."
   },
+  "lighthouse-core/report/html/renderer/util.js | warningAuditsGroupTitle": {
+    "message": "Passed audits but with warnings",
+    "description": "Section heading shown above a list of passed audits that contain warnings. Audits under this section do not negatively impact the score, but Lighthouse has generated some potentially actionable suggestions that should be reviewed. This section is expanded by default and displays after the failing audits."
+  },
   "lighthouse-core/report/html/renderer/util.js | warningHeader": {
     "message": "Warnings: ",
     "description": "This label is shown above a bulleted list of warnings. It is shown directly below an audit that produced warnings. Warnings describe situations the user should be aware of, as Lighthouse was unable to complete all the work required on this audit. For example, The 'Unable to decode image (biglogo.jpg)' warning may show up below an image encoding audit."
diff --git a/lighthouse-core/report/html/renderer/category-renderer.js b/lighthouse-core/report/html/renderer/category-renderer.js
index 0fa8be23b..c20523bcf 100644
--- a/lighthouse-core/report/html/renderer/category-renderer.js
+++ b/lighthouse-core/report/html/renderer/category-renderer.js
@@ -22,7 +22,7 @@
 /** @typedef {import('./report-renderer.js')} ReportRenderer */
 /** @typedef {import('./details-renderer.js')} DetailsRenderer */
 /** @typedef {import('./util.js')} Util */
-/** @typedef {'failed'|'manual'|'passed'|'notApplicable'} TopLevelClumpId */
+/** @typedef {'failed'|'warning'|'manual'|'passed'|'notApplicable'} TopLevelClumpId */
 
 class CategoryRenderer {
   /**
@@ -45,6 +45,7 @@ class CategoryRenderer {
    */
   get _clumpTitles() {
     return {
+      warning: Util.UIStrings.warningAuditsGroupTitle,
       manual: Util.UIStrings.manualAuditsGroupTitle,
       passed: Util.UIStrings.passedAuditsGroupTitle,
       notApplicable: Util.UIStrings.notApplicableAuditsGroupTitle,
@@ -264,6 +265,10 @@ class CategoryRenderer {
     const clumpTmpl = this.dom.cloneTemplate('#tmpl-lh-clump', this.templateContext);
     const clumpElement = this.dom.find('.lh-clump', clumpTmpl);
 
+    if (clumpId === 'warning') {
+      clumpElement.setAttribute('open', '');
+    }
+
     const summaryInnerEl = this.dom.find('.lh-audit-group__summary', clumpElement);
     const chevronEl = summaryInnerEl.appendChild(this._createChevron());
     chevronEl.title = Util.UIStrings.auditGroupExpandTooltip;
@@ -333,6 +338,14 @@ class CategoryRenderer {
     return tmpl;
   }
 
+  /**
+   * @param {LH.ReportResult.AuditRef} audit
+   * @return {boolean}
+   */
+  _auditHasWarning(audit) {
+    return Boolean(audit.result.warnings && audit.result.warnings.length);
+  }
+
   /**
    * Returns the id of the top-level clump to put this audit in.
    * @param {LH.ReportResult.AuditRef} auditRef
@@ -345,15 +358,19 @@ class CategoryRenderer {
     }
 
     if (Util.showAsPassed(auditRef.result)) {
-      return 'passed';
+      if (this._auditHasWarning(auditRef)) {
+        return 'warning';
+      } else {
+        return 'passed';
+      }
     } else {
       return 'failed';
     }
   }
 
   /**
-   * Renders a set of top level sections (clumps), under a status of failed, manual,
-   * passed, or notApplicable. The result ends up something like:
+   * Renders a set of top level sections (clumps), under a status of failed, warning,
+   * manual, passed, or notApplicable. The result ends up something like:
    *
    * failed clump
    *   ├── audit 1 (w/o group)
@@ -382,6 +399,7 @@ class CategoryRenderer {
     /** @type {Map<TopLevelClumpId, Array<LH.ReportResult.AuditRef>>} */
     const clumps = new Map();
     clumps.set('failed', []);
+    clumps.set('warning', []);
     clumps.set('manual', []);
     clumps.set('passed', []);
     clumps.set('notApplicable', []);
diff --git a/lighthouse-core/report/html/renderer/util.js b/lighthouse-core/report/html/renderer/util.js
index 68c4f3d6c..6a752d517 100644
--- a/lighthouse-core/report/html/renderer/util.js
+++ b/lighthouse-core/report/html/renderer/util.js
@@ -477,6 +477,8 @@ Util.UIStrings = {
   warningHeader: 'Warnings: ',
   /** The tooltip text on an expandable chevron icon. Clicking the icon expands a section to reveal a list of audit results that was hidden by default. */
   auditGroupExpandTooltip: 'Show audits',
+  /** Section heading shown above a list of passed audits that contain warnings. Audits under this section do not negatively impact the score, but Lighthouse has generated some potentially actionable suggestions that should be reviewed. This section is expanded by default and displays after the failing audits. */
+  warningAuditsGroupTitle: 'Passed audits but with warnings',
   /** Section heading shown above a list of audits that are passing. 'Passed' here refers to a passing grade. This section is collapsed by default, as the user should be focusing on the failed audits instead. Users can click this heading to reveal the list. */
   passedAuditsGroupTitle: 'Passed audits',
   /** Section heading shown above a list of audits that do not apply to the page. For example, if an audit is 'Are images optimized?', but the page has no images on it, the audit will be marked as not applicable. This is neither passing or failing. This section is collapsed by default, as the user should be focusing on the failed audits instead. Users can click this heading to reveal the list. */
diff --git a/lighthouse-core/report/html/report-styles.css b/lighthouse-core/report/html/report-styles.css
index ea4a195e1..8ece38dc6 100644
--- a/lighthouse-core/report/html/report-styles.css
+++ b/lighthouse-core/report/html/report-styles.css
@@ -93,6 +93,7 @@
   --check-circle-icon-url: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48"><path d="M0 0h48v48H0z" fill="none"/><path d="M24 4C12.95 4 4 12.95 4 24c0 11.04 8.95 20 20 20 11.04 0 20-8.96 20-20 0-11.05-8.96-20-20-20zm-4 30L10 24l2.83-2.83L20 28.34l15.17-15.17L38 16 20 34z" fill="%235E6268"/></svg>');
   --check-icon-url: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48"><path d="M0 0h48v48H0z" fill="none"/><path d="M18 32.34L9.66 24l-2.83 2.83L18 38l24-24-2.83-2.83z"/></svg>');
 
+  --warning-icon-url: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48"><title>warn</title><path fill="%235E6268" d="M2 42h44L24 4 2 42zm24-6h-4v-4h4v4zm0-8h-4v-8h4v8z"/></svg>');
   --search-icon-url: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48"><path d="M31 28h-1.59l-.55-.55C30.82 25.18 32 22.23 32 19c0-7.18-5.82-13-13-13S6 11.82 6 19s5.82 13 13 13c3.23 0 6.18-1.18 8.45-3.13l.55.55V31l10 9.98L40.98 38 31 28zm-12 0a9 9 0 1 1 .001-18.001A9 9 0 0 1 19 28z" fill="%235E6268"/><path d="M0 0h48v48H0z" fill="none" /></svg>');
   --remove-circle-icon-url: url('data:image/svg+xml;utf8,<svg height="24" width="24" xmlns="http://www.w3.org/2000/svg"><path d="M0 0h24v24H0z" fill="none"/><path d="M7 11v2h10v-2H7zm5-9C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm0 18c-4.41 0-8-3.59-8-8s3.59-8 8-8 8 3.59 8 8-3.59 8-8 8z" fill="%235E6268"/></svg>');
 
@@ -596,6 +597,10 @@
   vertical-align: middle;
 }
 
+.lh-clump--warning > summary .lh-audit-group__header::before {
+  content: '';
+  background-image: var(--warning-icon-url);
+}
 .lh-clump--manual > summary .lh-audit-group__header::before {
   content: '';
   background-image: var(--search-icon-url);
diff --git a/proto/lighthouse-result.proto b/proto/lighthouse-result.proto
index 26d3115c4..df0658e7e 100644
--- a/proto/lighthouse-result.proto
+++ b/proto/lighthouse-result.proto
@@ -331,6 +331,9 @@ message I18n {
 
     // The title of the lab data performance category
     string lab_data_title = 16;
+
+    // The heading that is shown above a list of audits that have warnings
+    string warning_audits_group_title = 17;
   }
 
   // The message holding all formatted strings
diff --git a/proto/sample_v2_round_trip.json b/proto/sample_v2_round_trip.json
index 83b4decae..f3afc6962 100644
--- a/proto/sample_v2_round_trip.json
+++ b/proto/sample_v2_round_trip.json
@@ -3286,6 +3286,7 @@
             "scorescaleLabel": "Score scale:", 
             "toplevelWarningsMessage": "There were issues affecting this run of Lighthouse:", 
             "varianceDisclaimer": "Values are estimated and may vary.", 
+            "warningAuditsGroupTitle": "Passed audits but with warnings", 
             "warningHeader": "Warnings: "
         }
     }, 


Test patch:
diff --git a/lighthouse-core/test/computed/metrics/__snapshots__/lantern-speed-index-test.js.snap b/lighthouse-core/test/computed/metrics/__snapshots__/lantern-speed-index-test.js.snap
deleted file mode 100644
index 8854004ff..000000000
--- a/lighthouse-core/test/computed/metrics/__snapshots__/lantern-speed-index-test.js.snap
+++ /dev/null
@@ -1,9 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`Metrics: Lantern Speed Index should compute predicted value 1`] = `
-Object {
-  "optimistic": 605,
-  "pessimistic": 1631,
-  "timing": 1657,
-}
-`;
diff --git a/lighthouse-core/test/computed/metrics/lantern-speed-index-test.js b/lighthouse-core/test/computed/metrics/lantern-speed-index-test.js
index f5cc91656..792a954eb 100644
--- a/lighthouse-core/test/computed/metrics/lantern-speed-index-test.js
+++ b/lighthouse-core/test/computed/metrics/lantern-speed-index-test.js
@@ -5,6 +5,7 @@
  */
 'use strict';
 
+const defaultThrottling = require('../../../config/constants.js').throttling.mobileSlow4G;
 const LanternSpeedIndex = require('../../../computed/metrics/lantern-speed-index.js');
 
 const trace = require('../../fixtures/traces/progressive-app-m60.json');
@@ -13,7 +14,7 @@ const devtoolsLog = require('../../fixtures/traces/progressive-app-m60.devtools.
 /* eslint-env jest */
 describe('Metrics: Lantern Speed Index', () => {
   it('should compute predicted value', async () => {
-    const settings = {};
+    const settings = {throttlingMethod: 'simulate', throttling: defaultThrottling};
     const context = {settings, computedCache: new Map()};
     const result = await LanternSpeedIndex.request({trace, devtoolsLog, settings}, context);
 
@@ -21,6 +22,51 @@ describe('Metrics: Lantern Speed Index', () => {
       timing: Math.round(result.timing),
       optimistic: Math.round(result.optimisticEstimate.timeInMs),
       pessimistic: Math.round(result.pessimisticEstimate.timeInMs),
-    }).toMatchSnapshot();
+    }).toMatchInlineSnapshot(`
+Object {
+  "optimistic": 605,
+  "pessimistic": 1631,
+  "timing": 1657,
+}
+`);
+  });
+
+  it('should compute predicted value for different settings', async () => {
+    const settings = {throttlingMethod: 'simulate', throttling: {...defaultThrottling, rttMs: 300}};
+    const context = {settings, computedCache: new Map()};
+    const result = await LanternSpeedIndex.request({trace, devtoolsLog, settings}, context);
+
+    expect({
+      timing: Math.round(result.timing),
+      optimistic: Math.round(result.optimisticEstimate.timeInMs),
+      pessimistic: Math.round(result.pessimisticEstimate.timeInMs),
+    }).toMatchInlineSnapshot(`
+Object {
+  "optimistic": 605,
+  "pessimistic": 2381,
+  "timing": 2958,
+}
+`);
+  });
+
+  it('should not scale coefficients at default', async () => {
+    const result = LanternSpeedIndex.getScaledCoefficients(defaultThrottling.rttMs);
+    expect(result).toEqual(LanternSpeedIndex.COEFFICIENTS);
+  });
+
+  it('should scale coefficients back', async () => {
+    const result = LanternSpeedIndex.getScaledCoefficients(5);
+    expect(result).toEqual({intercept: -0, pessimistic: 0.5, optimistic: 0.5});
+  });
+
+  it('should scale coefficients forward', async () => {
+    const result = LanternSpeedIndex.getScaledCoefficients(300);
+    expect(result).toMatchInlineSnapshot(`
+Object {
+  "intercept": -562.5,
+  "optimistic": 2.525,
+  "pessimistic": 0.8375,
+}
+`);
   });
 });
diff --git a/lighthouse-core/test/report/html/renderer/category-renderer-test.js b/lighthouse-core/test/report/html/renderer/category-renderer-test.js
index bbb554754..57be61f56 100644
--- a/lighthouse-core/test/report/html/renderer/category-renderer-test.js
+++ b/lighthouse-core/test/report/html/renderer/category-renderer-test.js
@@ -215,16 +215,36 @@ describe('CategoryRenderer', () => {
       assert.ok(description.querySelector('a'), 'description contains converted markdown links');
     });
 
-    // TODO waiting for decision regarding this header
-    it.skip('renders the failed audits grouped by group', () => {
-      const categoryDOM = renderer.render(category, sampleResults.categoryGroups);
-      const failedAudits = category.auditRefs.filter(audit => {
-        return audit.result.score !== 1 && !audit.result.scoreDisplayMode === 'notApplicable';
+    it('renders the failed audits grouped by group', () => {
+      // Fail all the audits.
+      const categoryClone = JSON.parse(JSON.stringify(category));
+      const auditRefs = categoryClone.auditRefs;
+      auditRefs.forEach(ref => {
+        ref.result.score = 0;
+        ref.result.scoreDisplayMode = 'binary';
       });
-      const failedAuditTags = new Set(failedAudits.map(audit => audit.group));
+      const categoryDOM = renderer.render(categoryClone, sampleResults.categoryGroups);
+
+      // All the group names in the config.
+      const groupNames = Array.from(new Set(auditRefs.map(ref => ref.group))).filter(Boolean);
+      assert.ok(groupNames.length > 5, `not enough groups found in category for test`);
+
+      // All the group roots in the DOM.
+      const failedGroupElems = Array.from(
+          categoryDOM.querySelectorAll('.lh-clump--failed > .lh-audit-group'));
+
+      assert.strictEqual(failedGroupElems.length, groupNames.length);
 
-      const failedAuditGroups = categoryDOM.querySelectorAll('.lh-category > div.lh-audit-group');
-      assert.equal(failedAuditGroups.length, failedAuditTags.size);
+      for (const groupName of groupNames) {
+        const groupAuditRefs = auditRefs.filter(ref => ref.group === groupName);
+        assert.ok(groupAuditRefs.length > 0, `no auditRefs found with group '${groupName}'`);
+
+        const className = `lh-audit-group--${groupName}`;
+        const groupElem = failedGroupElems.find(el => el.classList.contains(className));
+        const groupAuditElems = groupElem.querySelectorAll('.lh-audit');
+
+        assert.strictEqual(groupAuditElems.length, groupAuditRefs.length);
+      }
     });
 
     it('renders the passed audits ungrouped', () => {
@@ -314,16 +334,24 @@ describe('CategoryRenderer', () => {
     });
   });
 
-  describe('clumping passed/failed/manual', () => {
+  describe('clumping passed/failed/warning/manual', () => {
     it('separates audits in the DOM', () => {
       const category = sampleResults.reportCategories.find(c => c.id === 'pwa');
-      const elem = renderer.render(category, sampleResults.categoryGroups);
+      const categoryClone = JSON.parse(JSON.stringify(category));
+      // Give the first two passing grades warnings
+      const passingRefs = categoryClone.auditRefs.filter(ref => ref.result.score === 1);
+      passingRefs[0].result.warnings = ['Some warning'];
+      passingRefs[1].result.warnings = ['Some warning'];
+
+      const elem = renderer.render(categoryClone, sampleResults.categoryGroups);
       const passedAudits = elem.querySelectorAll('.lh-clump--passed .lh-audit');
       const failedAudits = elem.querySelectorAll('.lh-clump--failed .lh-audit');
+      const warningAudits = elem.querySelectorAll('.lh-clump--warning .lh-audit');
       const manualAudits = elem.querySelectorAll('.lh-clump--manual .lh-audit');
 
-      assert.equal(passedAudits.length, 4);
+      assert.equal(passedAudits.length, 2);
       assert.equal(failedAudits.length, 8);
+      assert.equal(warningAudits.length, 2);
       assert.equal(manualAudits.length, 3);
     });
 
@@ -338,6 +366,59 @@ describe('CategoryRenderer', () => {
       assert.equal(passedAudits.length, 0);
       assert.equal(failedAudits.length, 12);
     });
+
+    it('expands warning audit group', () => {
+      const category = sampleResults.reportCategories.find(c => c.id === 'pwa');
+      const categoryClone = JSON.parse(JSON.stringify(category));
+      categoryClone.auditRefs[0].result.warnings = ['Some warning'];
+
+      const auditDOM = renderer.render(categoryClone, sampleResults.categoryGroups);
+      const warningClumpEl = auditDOM.querySelector('.lh-clump--warning');
+      const isExpanded = warningClumpEl.hasAttribute('open');
+      assert.ok(isExpanded, 'Warning audit group should be expanded by default');
+    });
+
+    it('only passing audits with warnings show in warnings section', () => {
+      const failingWarning = 'Failed and warned';
+      const passingWarning = 'A passing warning';
+      const category = {
+        id: 'test',
+        title: 'Test',
+        score: 0,
+        auditRefs: [{
+          id: 'failing',
+          result: {
+            id: 'failing',
+            title: 'Failing with warning',
+            description: '',
+            scoreDisplayMode: 'numeric',
+            score: 0,
+            warnings: [failingWarning],
+          },
+        }, {
+          id: 'passing',
+          result: {
+            id: 'passing',
+            title: 'Passing with warning',
+            description: '',
+            scoreDisplayMode: 'numeric',
+            score: 1,
+            warnings: [passingWarning],
+          },
+        }],
+      };
+      const categoryDOM = renderer.render(category);
+
+      const shouldBeFailed = categoryDOM.querySelectorAll('.lh-clump--failed .lh-audit');
+      assert.strictEqual(shouldBeFailed.length, 1);
+      assert.strictEqual(shouldBeFailed[0].id, 'failing');
+      assert.ok(shouldBeFailed[0].textContent.includes(failingWarning));
+
+      const shouldBeWarning = categoryDOM.querySelectorAll('.lh-clump--warning .lh-audit');
+      assert.strictEqual(shouldBeWarning.length, 1);
+      assert.strictEqual(shouldBeWarning[0].id, 'passing');
+      assert.ok(shouldBeWarning[0].textContent.includes(passingWarning));
+    });
   });
 
   it('can set a custom templateContext', () => {
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
index cf95dde58..5d5ec4adc 100644
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@@ -4343,6 +4343,7 @@
       "scorescaleLabel": "Score scale:",
       "toplevelWarningsMessage": "There were issues affecting this run of Lighthouse:",
       "varianceDisclaimer": "Values are estimated and may vary.",
+      "warningAuditsGroupTitle": "Passed audits but with warnings",
       "warningHeader": "Warnings: "
     },
     "icuMessagePaths": {