Grafana dashboard: K8s / Storage / Volumes / Namespace
Description: Dashboard of Kubernetes / OpenShift volume information at namespace level as exported by Prometheus connected to Kubernetes / OpenShift.

# Overview
Dashboard of Kubernetes / OpenShift volume information at namespace level as exported by Prometheus connected to Kubernetes / OpenShift.

# Expects
* `pv_collector_bound_pvc_count`
* `pv_collector_unbound_pvc_count`
* `kube_persistentvolumeclaim_info`
* `kubelet_volume_stats_used_bytes`
* `kubelet_volume_stats_available_bytes`
* `kubelet_volume_stats_capacity_bytes`

# Documentation
* Set up Prometheus on OpenShift - [Red Hat OpenShiftr Container Platform 3.11: Configuring Clusters: Prometheus Cluster Monitoring](https://docs.openshift.com/container-platform/3.11/install_config/prometheus_cluster_monitoring.html)

# Tested With
* OpenShift Container Platform (OCP)
  * v3.11.154
* OpenShift Container Storage (OCS)
  * 3.11
* Grafana
  * v6.5.2
* Prometheus
  * v2.3.2

# Related Boards
* [K8s / Storage / Heketi](https://grafana.com/grafana/dashboards/11453)
* [K8s / Storage / Volumes / Cluster](https://grafana.com/grafana/dashboards/11454)
* [K8s / Storage / Volumes / Namespace](https://grafana.com/grafana/dashboards/11455)

{
  "__inputs": [
    {
      "name": "DS_OPENSHIFT_PROMETHEUS",
      "label": "openshift-prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "6.5.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "singlestat",
      "name": "Singlestat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "description": "Dashboard of Kubernetes / OpenShift volume information at namespace level as exported by Prometheus connected to Kubernetes / OpenShift.",
  "editable": true,
  "gnetId": 11455,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1577048652488,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 72,
      "panels": [],
      "title": "Current Alerts",
      "type": "row"
    },
    {
      "cacheTimeout": null,
      "colorBackground": true,
      "colorPrefix": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 14,
        "x": 0,
        "y": 1
      },
      "id": 59,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false,
        "ymax": null,
        "ymin": null
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "(\n count (\n   (kubelet_volume_stats_available_bytes {namespace=\"$namespace\"})\n   and\n   (predict_linear(kubelet_volume_stats_available_bytes {namespace=\"$namespace\"}[1d], 7 * 24 * 60 * 60) < 0)\n )\n)\nor\nvector(0)",
          "instant": false,
          "intervalFactor": 1,
          "legendFormat": "alerts",
          "refId": "A"
        }
      ],
      "thresholds": "1,1",
      "timeFrom": null,
      "timeShift": null,
      "title": "Volumes Full in Week Based on Daily Use Rate",
      "type": "singlestat",
      "valueFontSize": "200%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": true,
      "colors": [
        "#299c46",
        "#FF9830",
        "#FF9830"
      ],
      "datasource": "$DS_OPENSHIFT_PROMETHEUS",
      "decimals": 0,
      "editable": false,
      "error": false,
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 7,
        "x": 0,
        "y": 4
      },
      "id": 10,
      "interval": null,
      "isNew": false,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "span": 0,
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "lineColor": "rgb(31, 120, 193)"
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "count (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"}) and (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"})) >= ($pvc_percent_used_warning_threshold / 100)) or vector (0)",
          "format": "time_series",
          "instant": true,
          "refId": "A"
        }
      ],
      "thresholds": "1,1",
      "title": "Running PVCs Above % Used Warning Threshold",
      "type": "singlestat",
      "valueFontSize": "200%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": true,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "decimals": 0,
      "editable": false,
      "error": false,
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 7,
        "x": 7,
        "y": 4
      },
      "id": 6,
      "interval": null,
      "isNew": false,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "span": 0,
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "lineColor": "rgb(31, 120, 193)"
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "(max (sum by (exported_namespace) (pv_collector_unbound_pvc_count {exported_namespace = \"$namespace\"}))) or (vector(0))",
          "format": "table",
          "instant": true,
          "refId": "A"
        }
      ],
      "thresholds": "1,1",
      "title": "Unbound PVCs",
      "type": "singlestat",
      "valueFontSize": "200%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "collapsed": true,
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 57,
      "panels": [
        {
          "cacheTimeout": null,
          "columns": [
            {
              "text": "Avg",
              "value": "avg"
            }
          ],
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fontSize": "100%",
          "gridPos": {
            "h": 14,
            "w": 8,
            "x": 0,
            "y": 8
          },
          "id": 63,
          "links": [],
          "options": {},
          "pageSize": null,
          "showHeader": true,
          "sort": {
            "col": 0,
            "desc": true
          },
          "styles": [
            {
              "alias": "Available",
              "colorMode": "row",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "Value",
              "thresholds": [
                "0",
                "0"
              ],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Namespace",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "namespace",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PVC",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "persistentvolumeclaim",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "(\n  (\n    (kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"} ) \n    and\n    (predict_linear(kubelet_volume_stats_available_bytes {namespace=~\"(openshift-.*|kube-.*|default|logging)\"}[1w], 7 * 24 * 60 * 60) < 0)\n  )\n)",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Volumes Full in Week Based on Daily Use Rate - Current",
          "transform": "table",
          "type": "table"
        },
        {
          "aliasColors": {},
          "bars": true,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fill": 10,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 16,
            "x": 8,
            "y": 8
          },
          "hiddenSeries": false,
          "id": 61,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": false,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": true,
          "targets": [
            {
              "expr": "(\n  (\n    ((kubelet_volume_stats_used_bytes {namespace=\"$namespace\"} ) * 0) + 1\n    and\n    (predict_linear(kubelet_volume_stats_available_bytes {namespace=\"$namespace\"}[1d], 7 * 24 * 60 * 60) < 0)\n  )\n)",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{persistentvolumeclaim}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Volumes Full in Week Based on Daily Use Rate - Alert History",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "short",
              "label": "# of Alerts",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 16,
            "x": 8,
            "y": 15
          },
          "hiddenSeries": false,
          "id": 78,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(\n  (predict_linear(kubelet_volume_stats_available_bytes {namespace=\"$namespace\"}[1d], 7 * 24 * 60 * 60) < 0)\n)",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{persistentvolumeclaim}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Volumes Full in Week Based on Daily Use Rate -  Predicted Available Space History",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "bytes",
              "label": "1 Week Predicted Available Space",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "columns": [],
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "editable": false,
          "error": false,
          "fontSize": "100%",
          "gridPos": {
            "h": 6,
            "w": 8,
            "x": 0,
            "y": 22
          },
          "id": 12,
          "isNew": false,
          "options": {},
          "pageSize": null,
          "scroll": false,
          "showHeader": true,
          "sort": {
            "col": 16,
            "desc": true
          },
          "span": 0,
          "styles": [
            {
              "alias": "Used",
              "colorMode": "row",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "#FF9830",
                "#FF9830"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #A",
              "thresholds": [
                "0",
                "0"
              ],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Capacity",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #B",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Free",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #C",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "% Used",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #D",
              "thresholds": [
                "($pvc_percent_used_warning_threshold / 100)",
                ".9"
              ],
              "type": "number",
              "unit": "percentunit"
            },
            {
              "alias": "StorageClass",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "storageclass",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PV",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "volumename",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Namespace",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "namespace",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PVC",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "persistentvolumeclaim",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Volume Stats Exist?",
              "colorMode": "value",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #F",
              "thresholds": [
                "1",
                "1"
              ],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "(\n  (kube_persistentvolumeclaim_info {namespace=\"$namespace\"})\n  and\n  (\n    (\n      (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"}))\n      /\n      (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"}))\n    ) >= ($pvc_percent_used_warning_threshold / 100)\n  )\n)",
              "format": "table",
              "instant": true,
              "refId": "E"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) and ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"})) >= ($pvc_percent_used_warning_threshold / 100))",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"})) and ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"})) >= ($pvc_percent_used_warning_threshold / 100))",
              "format": "table",
              "instant": true,
              "refId": "B"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_available_bytes {namespace=\"$namespace\"})) and ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"})) >= ($pvc_percent_used_warning_threshold / 100))",
              "format": "table",
              "instant": true,
              "refId": "C"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"})) >= ($pvc_percent_used_warning_threshold / 100)",
              "format": "table",
              "instant": true,
              "refId": "D"
            }
          ],
          "title": "Running PVCs Above % Used Warning Threshold Stats - Current",
          "transform": "table",
          "type": "table"
        },
        {
          "aliasColors": {},
          "bars": true,
          "cacheTimeout": null,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "fill": 5,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 16,
            "x": 8,
            "y": 22
          },
          "hiddenSeries": false,
          "id": 65,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": false,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": true,
          "targets": [
            {
              "expr": "(\n  ((max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) * 0 +1)\n  and\n  (\n    (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"}))\n  ) >= ($pvc_percent_used_warning_threshold / 100)\n)",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Running PVCs Above % Used Warning Threshold",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Alert Details",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "id": 27,
      "panels": [
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": false,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
          "decimals": 0,
          "editable": false,
          "error": false,
          "format": "none",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "gridPos": {
            "h": 4,
            "w": 2,
            "x": 0,
            "y": 3
          },
          "id": 4,
          "interval": null,
          "isNew": false,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "options": {},
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 0,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "lineColor": "rgb(31, 120, 193)"
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "(sum (pv_collector_bound_pvc_count {exported_namespace = \"$namespace\"})) or vector(0)",
              "format": "table",
              "instant": true,
              "refId": "A"
            }
          ],
          "thresholds": "",
          "title": "Bound PVCs",
          "type": "singlestat",
          "valueFontSize": "200%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "columns": [],
          "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
          "editable": false,
          "error": false,
          "fontSize": "100%",
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 7
          },
          "id": 8,
          "isNew": false,
          "links": [
            {
              "includeVars": false,
              "targetBlank": true,
              "title": "OpenShift Container Storage (OCS) 3.11: Operations Guide: 10.1. Available Metrics for File Storage and Block Storage",
              "type": "",
              "url": "https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/3.11/html/operations_guide/enable_vol_metrics#file_vol_metrics"
            }
          ],
          "options": {},
          "pageSize": null,
          "scroll": false,
          "showHeader": true,
          "sort": {
            "col": 15,
            "desc": true
          },
          "span": 0,
          "styles": [
            {
              "alias": "Used",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #A",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Capacity",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #B",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "Free",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Value #C",
              "thresholds": [],
              "type": "number",
              "unit": "bytes"
            },
            {
              "alias": "% Free",
              "colorMode": "cell",
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #D",
              "thresholds": [
                ".8",
                ".9"
              ],
              "type": "number",
              "unit": "percentunit"
            },
            {
              "alias": "StorageClass",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "storageclass",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PV",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "volumename",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Namespace",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "namespace",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "PVC",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "persistentvolumeclaim",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "Volume Stats Exist?",
              "colorMode": "value",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "Value #F",
              "thresholds": [
                "1",
                "1"
              ],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": " (kube_persistentvolumeclaim_info { namespace = \"$namespace\" })",
              "format": "table",
              "instant": true,
              "refId": "E"
            },
            {
              "expr": "(1-max by (persistentvolumeclaim, namespace) (kube_persistentvolumeclaim_info { namespace = \"$namespace\" }) ) unless (max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes { namespace = \"$namespace\" })) or ((max by (persistentvolumeclaim, namespace) (kube_persistentvolumeclaim_info { namespace = \"$namespace\" }) ) and (max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes { namespace = \"$namespace\" })))",
              "format": "table",
              "instant": true,
              "refId": "F"
            },
            {
              "expr": "max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes { namespace = \"$namespace\" })",
              "format": "table",
              "instant": true,
              "intervalFactor": 1,
              "refId": "A"
            },
            {
              "expr": "max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes { namespace = \"$namespace\" })",
              "format": "table",
              "instant": true,
              "refId": "B"
            },
            {
              "expr": "max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_available_bytes { namespace = \"$namespace\" })",
              "format": "table",
              "instant": true,
              "refId": "C"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes { namespace = \"$namespace\" })) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes { namespace = \"$namespace\" }))",
              "format": "table",
              "instant": true,
              "refId": "D"
            }
          ],
          "title": "PVC Stats",
          "transform": "table",
          "type": "table"
        }
      ],
      "title": "Stats",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 34,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "editable": false,
          "error": false,
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 4
          },
          "hiddenSeries": false,
          "id": 16,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"})) / (max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\"})) * 100",
              "intervalFactor": 1,
              "legendFormat": "{{persistentvolumeclaim}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Running PVCs % Used",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percent",
              "logBase": 1,
              "max": 100,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "editable": false,
          "error": false,
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 12
          },
          "hiddenSeries": false,
          "id": 19,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\"}))",
              "intervalFactor": 1,
              "legendFormat": "{{persistentvolumeclaim}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "All Running PVCs Used Bytes",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Use",
      "type": "row"
    },
    {
      "CustomPanel": {
        "collapsed": false,
        "datasource": "openshift-prometheus",
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 33
        },
        "id": 14,
        "panels": [],
        "title": "Running PVCs Used Bytes vs Capacity",
        "type": "row"
      },
      "collapsed": true,
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "editable": false,
      "error": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "id": 14,
      "isNew": false,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "decimals": 0,
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up.",
          "editable": false,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 0,
            "y": 5
          },
          "hiddenSeries": false,
          "id": 20,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "repeat": "persistentvolumeclaim",
          "repeatDirection": "h",
          "scopedVars": {
            "persistentvolumeclaim": {
              "selected": false,
              "text": "logging-es-1",
              "value": "logging-es-1"
            }
          },
          "seriesOverrides": [
            {
              "alias": "Capacity",
              "color": "#C4162A",
              "fill": 0
            }
          ],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "intervalFactor": 1,
              "legendFormat": "Used",
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "legendFormat": "Capacity",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "$persistentvolumeclaim",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "decimals": 0,
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up.",
          "editable": false,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 6,
            "y": 5
          },
          "hiddenSeries": false,
          "id": 73,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "repeatDirection": "h",
          "repeatIteration": 1577033978570,
          "repeatPanelId": 20,
          "scopedVars": {
            "persistentvolumeclaim": {
              "selected": false,
              "text": "logging-es-ops-0",
              "value": "logging-es-ops-0"
            }
          },
          "seriesOverrides": [
            {
              "alias": "Capacity",
              "color": "#C4162A",
              "fill": 0
            }
          ],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "intervalFactor": 1,
              "legendFormat": "Used",
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "legendFormat": "Capacity",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "$persistentvolumeclaim",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "decimals": 0,
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up.",
          "editable": false,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 12,
            "y": 5
          },
          "hiddenSeries": false,
          "id": 74,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "repeatDirection": "h",
          "repeatIteration": 1577033978570,
          "repeatPanelId": 20,
          "scopedVars": {
            "persistentvolumeclaim": {
              "selected": false,
              "text": "logging-es-0",
              "value": "logging-es-0"
            }
          },
          "seriesOverrides": [
            {
              "alias": "Capacity",
              "color": "#C4162A",
              "fill": 0
            }
          ],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "intervalFactor": 1,
              "legendFormat": "Used",
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "legendFormat": "Capacity",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "$persistentvolumeclaim",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "decimals": 0,
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up.",
          "editable": false,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 18,
            "y": 5
          },
          "hiddenSeries": false,
          "id": 75,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "repeatDirection": "h",
          "repeatIteration": 1577033978570,
          "repeatPanelId": 20,
          "scopedVars": {
            "persistentvolumeclaim": {
              "selected": false,
              "text": "logging-es-ops-1",
              "value": "logging-es-ops-1"
            }
          },
          "seriesOverrides": [
            {
              "alias": "Capacity",
              "color": "#C4162A",
              "fill": 0
            }
          ],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "intervalFactor": 1,
              "legendFormat": "Used",
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "legendFormat": "Capacity",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "$persistentvolumeclaim",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "decimals": 0,
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up.",
          "editable": false,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 0,
            "y": 12
          },
          "hiddenSeries": false,
          "id": 76,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "repeatDirection": "h",
          "repeatIteration": 1577033978570,
          "repeatPanelId": 20,
          "scopedVars": {
            "persistentvolumeclaim": {
              "selected": false,
              "text": "logging-es-2",
              "value": "logging-es-2"
            }
          },
          "seriesOverrides": [
            {
              "alias": "Capacity",
              "color": "#C4162A",
              "fill": 0
            }
          ],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "intervalFactor": 1,
              "legendFormat": "Used",
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "legendFormat": "Capacity",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "$persistentvolumeclaim",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "decimals": 0,
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up.",
          "editable": false,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 6,
            "y": 12
          },
          "hiddenSeries": false,
          "id": 77,
          "isNew": false,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "repeatDirection": "h",
          "repeatIteration": 1577033978570,
          "repeatPanelId": 20,
          "scopedVars": {
            "persistentvolumeclaim": {
              "selected": false,
              "text": "logging-es-ops-2",
              "value": "logging-es-ops-2"
            }
          },
          "seriesOverrides": [
            {
              "alias": "Capacity",
              "color": "#C4162A",
              "fill": 0
            }
          ],
          "spaceLength": 10,
          "span": 0,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_used_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "intervalFactor": 1,
              "legendFormat": "Used",
              "refId": "A"
            },
            {
              "expr": "(max by (persistentvolumeclaim,namespace) (kubelet_volume_stats_capacity_bytes {namespace=\"$namespace\", persistentvolumeclaim=\"$persistentvolumeclaim\"}))",
              "legendFormat": "Capacity",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "$persistentvolumeclaim",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "format": "",
            "logBase": 0,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "bytes",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "span": 0,
      "title": "Use vs Capacity",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_OPENSHIFT_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 11
      },
      "id": 41,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 6
          },
          "hiddenSeries": false,
          "id": 43,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(kubelet_volume_stats_used_bytes {namespace=\"$namespace\"}[1h])",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Hourly Volume Use Rate",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": null,
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 12
          },
          "hiddenSeries": false,
          "id": 49,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(kubelet_volume_stats_used_bytes {namespace=\"$namespace\"}[1d])",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Daily Volume Use Rate",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": null,
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "$DS_OPENSHIFT_PROMETHEUS",
          "description": "WARNING: Any PVCs that are not bound to a running pod will not show up in this state.",
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 18
          },
          "hiddenSeries": false,
          "id": 50,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "dataLinks": []
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(kubelet_volume_stats_used_bytes {namespace=\"$namespace\"}[1w])",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 1,
              "legendFormat": "{{namespace}} ({{persistentvolumeclaim}})",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Weekly Volume Use Rate",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": null,
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Use Rate",
      "type": "row"
    }
  ],
  "schemaVersion": 21,
  "style": "dark",
  "tags": [
    "openshift",
    "k8s",
    "storage"
  ],
  "templating": {
    "list": [
      {
        "allFormat": "",
        "allValue": "",
        "current": {
          "text": "openshift-prometheus",
          "value": "openshift-prometheus"
        },
        "datasource": null,
        "hide": 0,
        "includeAll": false,
        "label": "Datasource",
        "multi": false,
        "multiFormat": "",
        "name": "DS_OPENSHIFT_PROMETHEUS",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "datasource"
      },
      {
        "allFormat": "",
        "allValue": "",
        "current": {},
        "datasource": "$DS_OPENSHIFT_PROMETHEUS",
        "definition": "",
        "hide": 0,
        "includeAll": false,
        "label": "namespace",
        "multi": false,
        "multiFormat": "",
        "name": "namespace",
        "options": [],
        "query": "label_values(kube_namespace_created,namespace)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allFormat": "",
        "allValue": "",
        "current": {
          "text": "80",
          "value": "80"
        },
        "datasource": null,
        "hide": 0,
        "includeAll": false,
        "label": "PVC % Used Warning Threshold",
        "multi": false,
        "multiFormat": "",
        "name": "pvc_percent_used_warning_threshold",
        "options": [
          {
            "selected": true,
            "text": "80",
            "value": "80"
          }
        ],
        "query": "80",
        "refresh": false,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "textbox"
      },
      {
        "allFormat": "",
        "allValue": "",
        "current": {},
        "datasource": "$DS_OPENSHIFT_PROMETHEUS",
        "definition": "",
        "hide": 2,
        "includeAll": true,
        "label": "",
        "multi": false,
        "multiFormat": "",
        "name": "persistentvolumeclaim",
        "options": [],
        "query": "label_values(kubelet_volume_stats_used_bytes {namespace=\"$namespace\"}, persistentvolumeclaim)",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": null
  },
  "timezone": "",
  "title": "K8s / Storage / Volumes / Namespace"
}