Grafana dashboard: K8s / Storage / Volumes / Cluster
Description: Dashboard of Kubernetes / OpenShift volume information at cluster level as exported by Prometheus connected to Kubernetes / OpenShift.

# Overview
Dashboard of Kubernetes / OpenShift volume information at cluster level as exported by Prometheus connected to Kubernetes / OpenShift.

# Expects
* `pv_collector_bound_pvc_count`
* `pv_collector_unbound_pvc_count`
* `kube_persistentvolumeclaim_info`
* `kubelet_volume_stats_used_bytes`
* `kubelet_volume_stats_available_bytes`
* `kubelet_volume_stats_capacity_bytes`

# Published Grafana Alerts
* `Infrastructure Namespace Volumes Full in Week Based on Daily Use Rate Alert`
* `User Namespace Volumes Full in Week Based on Daily Use Rate Alert`

# Documentation
* Set up Prometheus on OpenShift - [Red Hat OpenShiftr Container Platform 3.11: Configuring Clusters: Prometheus Cluster Monitoring](https://docs.openshift.com/container-platform/3.11/install_config/prometheus_cluster_monitoring.html)

# Tested With
* OpenShift Container Platform (OCP)
  * v3.11.154
* OpenShift Container Storage (OCS)
  * 3.11
* Grafana
  * v6.5.2
* Prometheus
  * v2.3.2

# Related Boards
* [K8s / Storage / Heketi](https://grafana.com/grafana/dashboards/11453)
* [K8s / Storage / Volumes / Cluster](https://grafana.com/grafana/dashboards/11454)
* [K8s / Storage / Volumes / Namespace](https://grafana.com/grafana/dashboards/11455)


{
  "__inputs": [
    {
      "name": "DS_OPENSHIFT_PROMETHEUS",
      "label": "openshift-prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "panel",
      "id": "alertlist",
      "name": "Alert List",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "6.5.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "singlestat",
      "name": "Singlestat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "description": "Dashboard of Kubernetes / OpenShift volume information at cluster level as exported by Prometheus connected to Kubernetes / OpenShift.",
  "editable": true,
  "gnetId": 11454,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1577045029184,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 43,
      "panels": [],
      "title": "Current Alerts",
      "type": "row"
    },
    {
      "cacheTimeout": null,
      "colorBackground": true,
      "colorPrefix": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 7,
        "x": 0,
        "y": 1
      },
      "id": 30,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false,
        "ymax": null,
        "ymin": null
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "(\n  count (\n    (kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"})\n    and\n    (predict_linear(kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"}[1d], 7 * 24 * 60 * 60) < 0)\n  )\n)\nor\nvector(0)",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "alerts",
          "refId": "A"
        }
      ],
      "thresholds": "1,1",
      "timeFrom": null,
      "timeShift": null,
      "title": "Infrastructure Namespace Volumes Full in Week Based on Daily Use Rate",
      "type": "singlestat",
      "valueFontSize": "200%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": true,
      "colorPrefix": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 7,
        "x": 7,
        "y": 1
      },
      "id": 34,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false,
        "ymax": null,
        "ymin": null
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "(\n  count (\n    (\n      (kubelet_volume_stats_available_bytes {namespace!~\"(openshift-.*|kube-.*|default|logging)\"})\n    )\n    and\n    (predict_linear(kubelet_volume_stats_available_bytes[1d], 7 * 24 * 60 * 60) < 0)\n  )\n)\nor\nvector(0)",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "alerts",
          "refId": "A"
        }
      ],
      "thresholds": "1,1",
      "timeFrom": null,
      "timeShift": null,
      "title": "User Namespace Volumes Full in Week Based on Daily Use Rate",
      "type": "singlestat",
      "valueFontSize": "200%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "dashboardFilter": "",
      "dashboardTags": [],
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "folderId": null,
      "gridPos": {
        "h": 6,
        "w": 10,
        "x": 14,
        "y": 1
      },
      "id": 29,
      "limit": 10,
      "nameFilter": "",
      "onlyAlertsOnDashboard": true,
      "options": {},
      "show": "current",
      "sortOrder": 1,
      "stateFilter": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Dashboard Alerts",
      "type": "alertlist"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": true,
      "colors": [
        "#299c46",
        "#FF9830",
        "#FF9830"
      ],
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "decimals": 0,
      "editable": false,
      "error": false,
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 7,
        "x": 0,
        "y": 4
      },
      "id": 12,
      "interval": null,
      "isNew": false,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "span": 0,
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "lineColor": "rgb(31, 120, 193)"
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "count (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes ) and (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) >= ($pvc_percent_used_warning_threshold / 100)) or vector (0)",
          "format": "time_series",
          "instant": true,
          "refId": "A"
        }
      ],
      "thresholds": "1,1",
      "title": "Running PVCs Above % Used Warning Threshold",
      "type": "singlestat",
      "valueFontSize": "200%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": true,
      "colors": [
        "#299c46",
        "#FF9830",
        "#FF9830"
      ],
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "decimals": 0,
      "editable": false,
      "error": false,
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 7,
        "x": 7,
        "y": 4
      },
      "id": 6,
      "interval": null,
      "isNew": false,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "span": 0,
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "lineColor": "rgb(31, 120, 193)"
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "(max (sum by (exported_namespace) (pv_collector_unbound_pvc_count))) or (vector(0))",
          "format": "table",
          "instant": true,
          "refId": "A"
        }
      ],
      "thresholds": "1,1",
      "title": "Unbound PVCs",
      "type": "singlestat",
      "valueFontSize": "200%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "collapsed": true,
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 32,
      "panels": [
        {
          "cacheTimeout": null,
          "columns": [
            {
              "text": "Avg",
              "value": "avg"
            }
          ],
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fontSize": "100%",
          "gridPos": {
            "h": 14,
            "w": 7,
            "x": 0,
            "y": 2
          },
          "id": 38,
          "links": [],
          "options": {},
          "pageSize": null,
          "showHeader": true,
          "sort": {
            "col": 0,
            "desc": true
          },
          "styles": [
            {
              "alias": "Available",
              "colorMode": "row",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "Value",
              "thresholds": [
                "0",
                "0"
              ],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Namespace",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "namespace",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PVC",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "persistentvolumeclaim",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "(\n  (\n    (kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"} ) \n    and\n    (predict_linear(kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"}[1w], 7 * 24 * 60 * 60) < 0)\n  )\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Infrastructure Namespace Volumes Full in Week Based on Daily Use Rate - Current",
          "transform": "table",
          "type": "table"
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "A",
                    "1h",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "24h",
            "frequency": "1h",
            "handler": 1,
            "name": "Infrastructure Namespace Volumes Full in Week Based on Daily Use Rate Alert",
            "noDataState": "ok",
            "notifications": []
          },
          "aliasColors": {},
          "bars": true,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "fill": 10,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 17,
            "x": 7,
            "y": 2
          },
          "hiddenSeries": false,
          "id": 45,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": false,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": true,
          "targets": [
            {
              "expr": "(\n  (\n    ((kubelet_volume_stats_used_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"} ) * 0) + 1\n    and\n    (predict_linear(kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"}[1d], 7 * 24 * 60 * 60) < 0)\n  )\n)",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0
            }
          ],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Infrastructure Namespace Volumes Full in Week Based on Daily Use Rate - Alert History",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "short",
              "label": "# of Alerts",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 17,
            "x": 7,
            "y": 9
          },
          "hiddenSeries": false,
          "id": 27,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(\n  (predict_linear(kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"}[1d], 7 * 24 * 60 * 60) < 0)\n)",
              "format": "time_series",
              "hide": false,
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Infrastructure Namespace Volumes Full in Week Based on Daily Use Rate - Predicted Available Space History",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "bytes",
              "label": "1 Week Predicted Available Space",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "cacheTimeout": null,
          "columns": [
            {
              "text": "Avg",
              "value": "avg"
            }
          ],
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fontSize": "100%",
          "gridPos": {
            "h": 13,
            "w": 7,
            "x": 0,
            "y": 16
          },
          "id": 39,
          "links": [],
          "options": {},
          "pageSize": null,
          "showHeader": true,
          "sort": {
            "col": 0,
            "desc": true
          },
          "styles": [
            {
              "alias": "Available",
              "colorMode": "row",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "Value",
              "thresholds": [
                "0",
                "0"
              ],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Namespace",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "namespace",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PVC",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "persistentvolumeclaim",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "(\n  (\n    (\n      (kubelet_volume_stats_used_bytes {namespace!~\"(openshift-.*|kube-.*|default|logging)\"})\n    )\n    and\n    (predict_linear(kubelet_volume_stats_available_bytes[1d], 7 * 24 * 60 * 60) < 0)\n  )\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "User Namespace Volumes Full in Week Based on Daily Use Rate - Current",
          "transform": "table",
          "type": "table"
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "A",
                    "1h",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "24h",
            "frequency": "1h",
            "handler": 1,
            "name": "User Namespace Volumes Full in Week Based on Daily Use Rate Alert",
            "noDataState": "ok",
            "notifications": []
          },
          "aliasColors": {},
          "bars": true,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "fill": 10,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 17,
            "x": 7,
            "y": 16
          },
          "hiddenSeries": false,
          "id": 40,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": false,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": true,
          "targets": [
            {
              "expr": "(\n  (\n    (\n      (\n        (kubelet_volume_stats_used_bytes {namespace!~\"(openshift-.*|kube-.*|default|logging)\"})\n      )\n      * 0 + 1\n    )\n    and\n    (predict_linear(kubelet_volume_stats_available_bytes[1d], 7 * 24 * 60 * 60) < 0)\n  )\n)",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0
            }
          ],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "User Namespace Volumes Full in Week Based on Daily Use Rate - Alert History",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "short",
              "label": "# of Alerts",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 17,
            "x": 7,
            "y": 22
          },
          "hiddenSeries": false,
          "id": 46,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(\n  (predict_linear(kubelet_volume_stats_available_bytes {namespace!~\"(openshift-.*|kube-.*|default|logging)\"}[1d], 7 * 24 * 60 * 60) < 0)\n)",
              "format": "time_series",
              "hide": false,
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "User Namespace Volumes Full in Week Based on Daily Use Rate - Predicted Available Space History",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "bytes",
              "label": "1 Week Predicted Available Space",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "columns": [],
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "editable": false,
          "error": false,
          "fontSize": "100%",
          "gridPos": {
            "h": 6,
            "w": 7,
            "x": 0,
            "y": 29
          },
          "id": 9,
          "isNew": false,
          "options": {},
          "pageSize": null,
          "scroll": false,
          "showHeader": true,
          "sort": {
            "col": 16,
            "desc": true
          },
          "span": 0,
          "styles": [
            {
              "alias": "Used",
              "colorMode": "row",
              "colors": [
                "#73BF69",
                "#FF9830",
                "#FF9830"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #A",
              "thresholds": [
                "0",
                "0"
              ],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Capacity",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #B",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Free",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #C",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "% Used",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #D",
              "thresholds": [
                "($pvc_percent_used_warning_threshold / 100)",
                ".9"
              ],
              "type": "number",
              "unit": "percentunit"
            },
            {
              "alias": "StorageClass",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "storageclass",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PV",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "volumename",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Namespace",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "namespace",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PVC",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "persistentvolumeclaim",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Volume Stats Exist?",
              "colorMode": "value",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #F",
              "thresholds": [
                "1",
                "1"
              ],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "(kube_persistentvolumeclaim_info) and ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) >= ($pvc_percent_used_warning_threshold / 100))",
              "format": "table",
              "instant": true,
              "refId": "E"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes)) and ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) >= ($pvc_percent_used_warning_threshold / 100))",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) and ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) >= ($pvc_percent_used_warning_threshold / 100))",
              "format": "table",
              "instant": true,
              "refId": "B"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_available_bytes )) and ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) >= ($pvc_percent_used_warning_threshold / 100))",
              "format": "table",
              "instant": true,
              "refId": "C"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) >= ($pvc_percent_used_warning_threshold / 100)",
              "format": "table",
              "instant": true,
              "refId": "D"
            }
          ],
          "title": "Running PVCs Above % Used Warning Threshold Stats - Current",
          "transform": "table",
          "type": "table"
        },
        {
          "aliasColors": {},
          "bars": true,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fill": 5,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 17,
            "x": 7,
            "y": 29
          },
          "hiddenSeries": false,
          "id": 41,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": false,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": true,
          "targets": [
            {
              "expr": "(\n  ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) * 0 +1)\n  and\n  (\n    (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes ))\n  ) >= ($pvc_percent_used_warning_threshold / 100)\n)",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Running PVCs Above % Used Warning Threshold - History",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "short",
              "label": "# of Alerts",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Alert Details",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "id": 20,
      "panels": [
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": false,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "decimals": 0,
          "editable": false,
          "error": false,
          "format": "none",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "gridPos": {
            "h": 4,
            "w": 2,
            "x": 0,
            "y": 3
          },
          "id": 4,
          "interval": null,
          "isNew": false,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "options": {},
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 0,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "lineColor": "rgb(31, 120, 193)"
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "(sum (pv_collector_bound_pvc_count)) or vector(0)",
              "format": "table",
              "instant": true,
              "refId": "A"
            }
          ],
          "thresholds": "",
          "title": "Bound PVCs",
          "type": "singlestat",
          "valueFontSize": "200%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "columns": [],
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "editable": false,
          "error": false,
          "fontSize": "100%",
          "gridPos": {
            "h": 25,
            "w": 24,
            "x": 0,
            "y": 7
          },
          "id": 10,
          "isNew": false,
          "links": [
            {
              "includeVars": false,
              "title": "OpenShift Container Storage (OCS) 3.11: Operations Guide: 10.1. Available Metrics for File Storage and Block Storage",
              "type": "",
              "url": "https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/3.11/html/operations_guide/enable_vol_metrics#file_vol_metrics"
            }
          ],
          "options": {},
          "pageSize": null,
          "scroll": false,
          "showHeader": true,
          "sort": {
            "col": 16,
            "desc": true
          },
          "span": 0,
          "styles": [
            {
              "alias": "Used",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #A",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Capacity",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #B",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Free",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #C",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "% Used",
              "colorMode": "cell",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #D",
              "thresholds": [
                ".8",
                ".9"
              ],
              "type": "number",
              "unit": "percentunit"
            },
            {
              "alias": "StorageClass",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "storageclass",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PV",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "volumename",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Namespace",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "namespace",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PVC",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "persistentvolumeclaim",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Volume Stats Exist?",
              "colorMode": "value",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #F",
              "thresholds": [
                "1",
                "1"
              ],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "kube_persistentvolumeclaim_info",
              "format": "table",
              "instant": true,
              "refId": "E"
            },
            {
              "expr": "(1-max by (persistentvolumeclaim, namespace) (kube_persistentvolumeclaim_info ) ) unless (max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes )) or ((max by (persistentvolumeclaim, namespace) (kube_persistentvolumeclaim_info ) ) and (max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes )))",
              "format": "table",
              "instant": true,
              "refId": "F"
            },
            {
              "expr": "max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "refId": "A"
            },
            {
              "expr": "max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )",
              "format": "table",
              "instant": true,
              "refId": "B"
            },
            {
              "expr": "max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_available_bytes )",
              "format": "table",
              "instant": true,
              "refId": "C"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes ))",
              "format": "table",
              "instant": true,
              "refId": "D"
            }
          ],
          "title": "PVC Stats",
          "transform": "table",
          "type": "table"
        }
      ],
      "title": "Stats",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 22,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "editable": false,
          "error": false,
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 4
          },
          "hiddenSeries": false,
          "id": 15,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes ))",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "All Running PVCs Used Bytes",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "editable": false,
          "error": false,
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 12
          },
          "hiddenSeries": false,
          "id": 14,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes )) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes )) * 100",
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Running PVCs % Used",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percent",
              "logBase": 1,
              "max": 100,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Use",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "id": 24,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 5
          },
          "hiddenSeries": false,
          "id": 17,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(kubelet_volume_stats_used_bytes [1h])",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Hourly Volume Use Rate",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": null,
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 11
          },
          "hiddenSeries": false,
          "id": 18,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(kubelet_volume_stats_used_bytes [1d])",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Daily Volume Use Rate",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": null,
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 17
          },
          "hiddenSeries": false,
          "hideTimeOverride": false,
          "id": 25,
          "interval": "",
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(kubelet_volume_stats_used_bytes [1w])",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Weekly Volume Use Rate",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": null,
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Use Rate",
      "type": "row"
    }
  ],
  "schemaVersion": 21,
  "style": "dark",
  "tags": [
    "openshift",
    "k8s",
    "storage"
  ],
  "templating": {
    "list": [
      {
        "allFormat": "",
        "allValue": "",
        "current": {
          "text": "openshift-prometheus",
          "value": "openshift-prometheus"
        },
        "datasource": null,
        "hide": 0,
        "includeAll": false,
        "label": "Datasource",
        "multi": false,
        "multiFormat": "",
        "name": "DS_OPENSHIFT_PROMETHEUS",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "datasource"
      },
      {
        "allFormat": "",
        "allValue": "",
        "current": {
          "selected": false,
          "text": "80",
          "value": "80"
        },
        "datasource": null,
        "hide": 0,
        "includeAll": false,
        "label": "PVC % Used Warning Threshold",
        "multi": false,
        "multiFormat": "",
        "name": "pvc_percent_used_warning_threshold",
        "options": [
          {
            "selected": false,
            "text": "80",
            "value": "80"
          }
        ],
        "query": "80",
        "refresh": false,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-2d",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": null
  },
  "timezone": "",
  "title": "K8s / Storage / Volumes / Cluster"
}