Grafana dashboard: Pressure Stall Information
Description: Simple dashboard for displaying Pressure Stall Information for all the instances in a single dashboard.

Make sure `--collector.pressure` is enabled.

Make sure the Prometheus data source in grafana has `Scrape interval` set to the double of greatest scrape interval! This will no longer be necessary once https://github.com/grafana/grafana/issues/21417 is released.

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "7.1.5"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": 12949,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1599323912503,
  "links": [],
  "panels": [
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "When CPU, memory or IO devices are contended, workloads experience latency spikes, throughput losses, and run the risk of OOM kills.\n\nWithout an accurate measure of such contention, users are forced to either play it safe and under-utilize their hardware resources, or roll the dice and frequently suffer the disruptions resulting from excessive overcommit.\n\nThe psi feature identifies and quantifies the disruptions caused by such resource crunches and the time impact it has on complex workloads or even entire systems.\n\nHaving an accurate measure of productivity losses caused by resource scarcity aids users in sizing workloads to hardware–or provisioning hardware according to workload demand.\n\nAs psi aggregates this information in realtime, systems can be managed dynamically using techniques such as load shedding, migrating jobs to other systems or data centers, or strategically pausing or killing low priority or restartable batch jobs.\n\nThis allows maximizing hardware utilization without sacrificing workload health or risking major disruptions such as OOM kills.",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "links": []
        },
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 5,
      "gridPos": {
        "h": 13,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "hiddenSeries": false,
      "id": 4,
      "interval": "",
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "hideEmpty": true,
        "hideZero": false,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "total": true,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [
        {
          "targetBlank": true,
          "title": "kernel.org documentation on Pressure Stall Information",
          "url": "https://www.kernel.org/doc/html/latest/accounting/psi.html"
        }
      ],
      "maxPerRow": 2,
      "nullPointMode": "null",
      "percentage": false,
      "pluginVersion": "7.1.5",
      "pointradius": 0.5,
      "points": false,
      "renderer": "flot",
      "repeat": "instance",
      "repeatDirection": "h",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "irate(node_pressure_cpu_waiting_seconds_total{instance=~\"${instance}\"}[$__interval])",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "CPU waiting",
          "refId": "A"
        },
        {
          "expr": "irate(node_pressure_io_stalled_seconds_total{instance=~\"${instance}\"}[$__interval])",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "I/O stalled",
          "refId": "B"
        },
        {
          "expr": "irate(node_pressure_io_waiting_seconds_total{instance=~\"${instance}\"}[$__interval])",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "I/O waiting",
          "refId": "C"
        },
        {
          "expr": "irate(node_pressure_memory_stalled_seconds_total{instance=~\"${instance}\"}[$__interval])",
          "interval": "",
          "legendFormat": "Memory stalled",
          "refId": "D"
        },
        {
          "expr": "irate(node_pressure_memory_waiting_seconds_total{instance=~\"${instance}\"}[$__interval])",
          "interval": "",
          "legendFormat": "Memory waiting",
          "refId": "E"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "$instance Pressure Stall Information",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:49",
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:50",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "schemaVersion": 26,
  "style": "dark",
  "tags": [
    "linux",
    "performance",
    "pressure"
  ],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_PROMETHEUS}",
        "definition": "label_values(node_uname_info, instance)",
        "hide": 0,
        "includeAll": true,
        "label": "instance",
        "multi": true,
        "name": "instance",
        "options": [],
        "query": "label_values(node_uname_info, instance)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ]
  },
  "timezone": "",
  "title": "Pressure Stall Information",
  "uid": "DNwy_0vMz",
  "version": 11,
  "description": "Simple dashboard for displaying Pressure Stall Information for all the instances in a single dashboard."
}