Grafana dashboard: Ceph Stats
Description: Proxmox Ceph RBD Poolstatistics

Dashboard + Telegraf Config for getting Pool Metrics from a Ceph Manager.

1. Configure Ceph to expose Metrics:

```
ceph mgr module enable telegraf
ceph telegraf config-set address udp://:8094
ceph telegraf config-set interval 10
```
_Important Hint: Execute those 3 commands in a row or you might run into a nasty Ceph bug that might cause your manager processes crashing: https://tracker.ceph.com/issues/43551_

2. Install Telegraf on all Nodes that run a Ceph manager Process.
3. Copy the collector Config to every Host that runs a manager. Restart Telegraf on all nodes.
4. Import Dashboard JSON to Grafana, assign Datasource.

{
  "__inputs": [
    {
      "name": "DS_INFLUXDB",
      "label": "InfluxDB",
      "description": "",
      "type": "datasource",
      "pluginId": "influxdb",
      "pluginName": "InfluxDB"
    }
  ],
  "__requires": [
    {
      "type": "panel",
      "id": "blackmirror1-singlestat-math-panel",
      "name": "Singlestat Math",
      "version": "1.1.7"
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "6.7.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "influxdb",
      "name": "InfluxDB",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "natel-discrete-panel",
      "name": "Discrete",
      "version": "0.0.9"
    },
    {
      "type": "panel",
      "id": "singlestat",
      "name": "Singlestat",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "$$hashKey": "object:19",
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Proxmox Ceph RBD Poolstatistics",
  "editable": true,
  "gnetId": 12029,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1586292945348,
  "links": [],
  "panels": [
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "hiddenSeries": false,
      "id": 36,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "data_bytes",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "none"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "data_bytes"
            }
          ]
        },
        {
          "alias": "bytes_used",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "none"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "bytes_used"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Bytes used",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "bytes",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "backgroundColor": "rgba(128,128,128,0.1)",
      "colorMaps": [
        {
          "color": "#CCC",
          "text": "N/A"
        },
        {
          "color": "#73BF69",
          "text": "OK"
        },
        {
          "color": "#FF9830",
          "text": "WARN"
        },
        {
          "color": "#F2495C",
          "text": "ERR"
        }
      ],
      "crosshairColor": "#8F070C",
      "datasource": "${DS_INFLUXDB}",
      "display": "timeline",
      "expandFromQueryS": 0,
      "extendLastValue": true,
      "gridPos": {
        "h": 4,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "highlightOnMouseover": true,
      "id": 31,
      "legendSortBy": "-ms",
      "lineColor": "rgba(0,0,0,0.1)",
      "metricNameColor": "#000000",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        },
        {
          "from": "0",
          "text": "OK",
          "to": "0"
        },
        {
          "from": "1",
          "text": "WARN",
          "to": "1"
        },
        {
          "from": "2",
          "text": "ERR",
          "to": "2"
        }
      ],
      "rowHeight": 35,
      "showLegend": true,
      "showLegendNames": true,
      "showLegendPercent": true,
      "showLegendValues": true,
      "showTimeAxis": true,
      "targets": [
        {
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT \"value\" FROM \"ceph_cluster_stats\" WHERE (\"type_instance\" = 'health') AND $timeFilter-1h",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              }
            ]
          ],
          "tags": [
            {
              "key": "type_instance",
              "operator": "=",
              "value": "health"
            }
          ]
        }
      ],
      "textSize": 22,
      "textSizeTime": 12,
      "timeFrom": null,
      "timeOptions": [
        {
          "name": "Years",
          "value": "years"
        },
        {
          "name": "Months",
          "value": "months"
        },
        {
          "name": "Weeks",
          "value": "weeks"
        },
        {
          "name": "Days",
          "value": "days"
        },
        {
          "name": "Hours",
          "value": "hours"
        },
        {
          "name": "Minutes",
          "value": "minutes"
        },
        {
          "name": "Seconds",
          "value": "seconds"
        },
        {
          "name": "Milliseconds",
          "value": "milliseconds"
        }
      ],
      "timePrecision": {
        "name": "Minutes",
        "value": "minutes"
      },
      "timeShift": null,
      "timeTextColor": "#d8d9da",
      "title": "Health Status History",
      "type": "natel-discrete-panel",
      "units": "short",
      "useTimePrecision": false,
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueTextColor": "#000000",
      "writeAllValues": true,
      "writeLastValue": true,
      "writeMetricNames": false
    },
    {
      "cacheTimeout": null,
      "circleBackground": false,
      "colorBackground": false,
      "colorValue": true,
      "datasource": "${DS_INFLUXDB}",
      "defaultColor": "rgb(117, 117, 117)",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 0,
        "y": 12
      },
      "id": 20,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "$$hashKey": "object:85",
          "name": "value to text",
          "value": 1
        },
        {
          "$$hashKey": "object:86",
          "name": "range to text",
          "value": 2
        }
      ],
      "math": "",
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sortOrder": "asc",
      "sortOrderOptions": [
        {
          "text": "Ascending",
          "value": "asc"
        },
        {
          "text": "Descending",
          "value": "desc"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"value\") FROM \"ceph_cluster_stats\" WHERE (\"dc\" =~ /^$DC$/ AND \"rack\" =~ /^$Rack$/ AND \"fsid\" =~ /^$fsid$/ AND \"type_instance\" = 'health') AND time >= now() - 10m",
          "rawQuery": true,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "health"
            }
          ]
        }
      ],
      "thresholds": [
        {
          "$$hashKey": "object:117",
          "color": "#73BF69",
          "value": 0
        },
        {
          "$$hashKey": "object:118",
          "color": "#FF780A",
          "value": 1
        },
        {
          "$$hashKey": "object:119",
          "color": "#F2495C",
          "value": 2
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Ceph Status",
      "tooltip": {
        "show": true
      },
      "type": "blackmirror1-singlestat-math-panel",
      "valueFontSize": "50%",
      "valueMappingColorBackground": "#767171",
      "valueMaps": [
        {
          "$$hashKey": "object:88",
          "op": "=",
          "text": "No data",
          "value": "null"
        },
        {
          "$$hashKey": "object:89",
          "op": "=",
          "text": "HEALTH_OK",
          "value": "0"
        },
        {
          "$$hashKey": "object:90",
          "op": "=",
          "text": "HEALTH_WARN",
          "value": "1"
        },
        {
          "$$hashKey": "object:91",
          "op": "=",
          "text": "HEALTH_ERR",
          "value": "2"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "circleBackground": false,
      "colorBackground": false,
      "colorValue": false,
      "datasource": "${DS_INFLUXDB}",
      "defaultColor": "rgb(117, 117, 117)",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 3,
        "y": 12
      },
      "id": 21,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "$$hashKey": "object:538",
          "name": "value to text",
          "value": 1
        },
        {
          "$$hashKey": "object:539",
          "name": "range to text",
          "value": 2
        }
      ],
      "math": "",
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sortOrder": "asc",
      "sortOrderOptions": [
        {
          "text": "Ascending",
          "value": "asc"
        },
        {
          "text": "Descending",
          "value": "desc"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "num_osd_in"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "OSDs in",
      "tooltip": {
        "show": true
      },
      "type": "blackmirror1-singlestat-math-panel",
      "valueFontSize": "80%",
      "valueMappingColorBackground": "#767171",
      "valueMaps": [
        {
          "$$hashKey": "object:541",
          "op": "=",
          "text": "No data",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "circleBackground": false,
      "colorBackground": false,
      "colorValue": false,
      "datasource": "${DS_INFLUXDB}",
      "defaultColor": "rgb(117, 117, 117)",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 6,
        "y": 12
      },
      "id": 38,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "$$hashKey": "object:538",
          "name": "value to text",
          "value": 1
        },
        {
          "$$hashKey": "object:539",
          "name": "range to text",
          "value": 2
        }
      ],
      "math": "",
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sortOrder": "asc",
      "sortOrderOptions": [
        {
          "text": "Ascending",
          "value": "asc"
        },
        {
          "text": "Descending",
          "value": "desc"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "num_osd_up"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "OSDs up",
      "tooltip": {
        "show": true
      },
      "type": "blackmirror1-singlestat-math-panel",
      "valueFontSize": "80%",
      "valueMappingColorBackground": "#767171",
      "valueMaps": [
        {
          "$$hashKey": "object:541",
          "op": "=",
          "text": "No data",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_INFLUXDB}",
      "decimals": 2,
      "format": "bytes",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 9,
        "y": 12
      },
      "id": 4,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "pluginVersion": "6.4.4",
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false,
        "ymax": null,
        "ymin": null
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "bytes_total"
            }
          ]
        }
      ],
      "thresholds": "",
      "timeFrom": null,
      "timeShift": null,
      "title": "Bytes Total",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_INFLUXDB}",
      "decimals": 2,
      "format": "bytes",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 12,
        "y": 12
      },
      "id": 2,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "pluginVersion": "6.4.4",
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false,
        "ymax": null,
        "ymin": null
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "bytes_used"
            }
          ]
        }
      ],
      "thresholds": "",
      "timeFrom": null,
      "timeShift": null,
      "title": "In Use",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "fill": 1,
      "fillGradient": 2,
      "gridPos": {
        "h": 6,
        "w": 9,
        "x": 15,
        "y": 12
      },
      "hiddenSeries": false,
      "id": 26,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "$$hashKey": "object:478",
          "alias": "type_instance num_pgs_snaptrim_error",
          "color": "#E02F44"
        },
        {
          "$$hashKey": "object:479",
          "alias": "type_instance num_pgs_snaptrim_wait",
          "color": "#FADE2A"
        },
        {
          "$$hashKey": "object:480",
          "alias": "type_instance num_pgs_snaptrim",
          "color": "#96D98D"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "type_instance $tag_type_instance",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "type_instance"
              ],
              "type": "tag"
            },
            {
              "params": [
                "none"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "num_pgs_snaptrim"
            },
            {
              "condition": "OR",
              "key": "type_instance",
              "operator": "=",
              "value": "num_pgs_snaptrim_wait"
            },
            {
              "condition": "OR",
              "key": "type_instance",
              "operator": "=",
              "value": "num_pgs_snaptrim_error"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Snapttrim Stats",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:505",
          "format": "short",
          "label": "PGs",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:506",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "cacheTimeout": null,
      "circleBackground": false,
      "colorBackground": true,
      "colorValue": false,
      "datasource": "${DS_INFLUXDB}",
      "decimals": 2,
      "defaultColor": "rgb(117, 117, 117)",
      "format": "percent",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 0,
        "y": 15
      },
      "id": 5,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "math": "(100/total)*used",
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "pluginVersion": "6.4.4",
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sortOrder": "asc",
      "sortOrderOptions": [
        {
          "text": "Ascending",
          "value": "asc"
        },
        {
          "text": "Descending",
          "value": "desc"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "alias": "total",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "bytes_total"
            }
          ]
        },
        {
          "alias": "used",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "bytes_used"
            }
          ]
        }
      ],
      "thresholds": [
        {
          "color": "#73BF69",
          "value": 0
        },
        {
          "color": "#FF9830",
          "value": 50
        },
        {
          "color": "#E02F44",
          "value": 66
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Space used",
      "tooltip": {
        "show": true
      },
      "type": "blackmirror1-singlestat-math-panel",
      "valueFontSize": "80%",
      "valueMappingColorBackground": "#767171",
      "valueMaps": [
        {
          "op": "=",
          "text": "No data",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "circleBackground": false,
      "colorBackground": false,
      "colorValue": true,
      "datasource": "${DS_INFLUXDB}",
      "defaultColor": "rgb(117, 117, 117)",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 3,
        "y": 15
      },
      "id": 37,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "$$hashKey": "object:538",
          "name": "value to text",
          "value": 1
        },
        {
          "$$hashKey": "object:539",
          "name": "range to text",
          "value": 2
        }
      ],
      "math": "total-in",
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sortOrder": "asc",
      "sortOrderOptions": [
        {
          "text": "Ascending",
          "value": "asc"
        },
        {
          "text": "Descending",
          "value": "desc"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "alias": "in",
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"value\") FROM \"ceph_cluster_stats\" WHERE (\"dc\" =~ /^$DC$/ AND \"rack\" =~ /^$Rack$/ AND \"fsid\" =~ /^$fsid$/ AND \"type_instance\" = 'num_osd_in') AND $timeFilter",
          "rawQuery": false,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "num_osd_in"
            }
          ]
        },
        {
          "alias": "total",
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"value\") FROM \"ceph_cluster_stats\" WHERE (\"dc\" =~ /^$DC$/ AND \"rack\" =~ /^$Rack$/ AND \"fsid\" =~ /^$fsid$/ AND \"type_instance\" = 'num_osd_in') AND $timeFilter",
          "rawQuery": false,
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "num_osd"
            }
          ]
        }
      ],
      "thresholds": [
        {
          "$$hashKey": "object:1175",
          "color": "#73BF69",
          "value": 0
        },
        {
          "$$hashKey": "object:1502",
          "color": "#FF7383",
          "value": 1
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "OSDs out",
      "tooltip": {
        "show": true
      },
      "type": "blackmirror1-singlestat-math-panel",
      "valueFontSize": "80%",
      "valueMappingColorBackground": "#767171",
      "valueMaps": [
        {
          "$$hashKey": "object:541",
          "op": "=",
          "text": "No data",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "circleBackground": false,
      "colorBackground": false,
      "colorValue": true,
      "datasource": "${DS_INFLUXDB}",
      "defaultColor": "rgb(117, 117, 117)",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 6,
        "y": 15
      },
      "id": 39,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "$$hashKey": "object:538",
          "name": "value to text",
          "value": 1
        },
        {
          "$$hashKey": "object:539",
          "name": "range to text",
          "value": 2
        }
      ],
      "math": "total-in",
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sortOrder": "asc",
      "sortOrderOptions": [
        {
          "text": "Ascending",
          "value": "asc"
        },
        {
          "text": "Descending",
          "value": "desc"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "alias": "in",
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"value\") FROM \"ceph_cluster_stats\" WHERE (\"dc\" =~ /^$DC$/ AND \"rack\" =~ /^$Rack$/ AND \"fsid\" =~ /^$fsid$/ AND \"type_instance\" = 'num_osd_in') AND $timeFilter",
          "rawQuery": false,
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "num_osd_up"
            }
          ]
        },
        {
          "alias": "total",
          "groupBy": [],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "query": "SELECT last(\"value\") FROM \"ceph_cluster_stats\" WHERE (\"dc\" =~ /^$DC$/ AND \"rack\" =~ /^$Rack$/ AND \"fsid\" =~ /^$fsid$/ AND \"type_instance\" = 'num_osd_in') AND $timeFilter",
          "rawQuery": false,
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "last"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "num_osd"
            }
          ]
        }
      ],
      "thresholds": [
        {
          "$$hashKey": "object:1175",
          "color": "#73BF69",
          "value": 0
        },
        {
          "$$hashKey": "object:1502",
          "color": "#FF7383",
          "value": 1
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "OSDs down",
      "tooltip": {
        "show": true
      },
      "type": "blackmirror1-singlestat-math-panel",
      "valueFontSize": "80%",
      "valueMappingColorBackground": "#767171",
      "valueMaps": [
        {
          "$$hashKey": "object:541",
          "op": "=",
          "text": "No data",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_INFLUXDB}",
      "decimals": 2,
      "format": "bytes",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 9,
        "y": 15
      },
      "id": 3,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "pluginVersion": "6.4.4",
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false,
        "ymax": null,
        "ymin": null
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "bytes_avail"
            }
          ]
        }
      ],
      "thresholds": "",
      "timeFrom": null,
      "timeShift": null,
      "title": "Free",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_INFLUXDB}",
      "decimals": 2,
      "format": "bytes",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 12,
        "y": 15
      },
      "id": 40,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "$$hashKey": "object:2074",
          "name": "value to text",
          "value": 1
        },
        {
          "$$hashKey": "object:2075",
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "pluginVersion": "6.4.4",
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false,
        "ymax": null,
        "ymin": null
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_cluster_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "data_bytes"
            }
          ]
        }
      ],
      "thresholds": "",
      "timeFrom": null,
      "timeShift": null,
      "title": "In Use (net)",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "$$hashKey": "object:2077",
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "collapsed": false,
      "datasource": "${DS_INFLUXDB}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 18
      },
      "id": 9,
      "panels": [],
      "repeat": "pool",
      "title": "Pool $pool",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "fill": 1,
      "fillGradient": 2,
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 19
      },
      "hiddenSeries": false,
      "id": 7,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "used",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_pool_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": [
            {
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "pool_name",
              "operator": "=~",
              "value": "/^$pool$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "kb_used"
            }
          ]
        },
        {
          "alias": "max_avail",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_pool_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              },
              {
                "params": [
                  "/1024"
                ],
                "type": "math"
              }
            ]
          ],
          "tags": [
            {
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "pool_name",
              "operator": "=~",
              "value": "/^$pool$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "max_avail"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Usage on $pool",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "deckbytes",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {
        "wr": "dark-purple"
      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 19
      },
      "hiddenSeries": false,
      "id": 15,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "wr_bytes",
          "transform": "negative-Y"
        },
        {
          "alias": "rd_iops",
          "yaxis": 2
        },
        {
          "alias": "wr_iops",
          "transform": "negative-Y",
          "yaxis": 2
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "rd_bytes",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_pool_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              },
              {
                "params": [
                  "1s"
                ],
                "type": "non_negative_derivative"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "pool_name",
              "operator": "=~",
              "value": "/^$pool$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "rd_bytes"
            }
          ]
        },
        {
          "alias": "wr_bytes",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_pool_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              },
              {
                "params": [
                  "1s"
                ],
                "type": "non_negative_derivative"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "pool_name",
              "operator": "=~",
              "value": "/^$pool$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "wr_bytes"
            }
          ]
        },
        {
          "alias": "rd_iops",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_pool_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "C",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              },
              {
                "params": [
                  "1s"
                ],
                "type": "non_negative_derivative"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "pool_name",
              "operator": "=~",
              "value": "/^$pool$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "rd"
            }
          ]
        },
        {
          "alias": "wr_iops",
          "groupBy": [
            {
              "params": [
                "$__interval"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "ceph_pool_stats",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "D",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              },
              {
                "params": [
                  "1s"
                ],
                "type": "non_negative_derivative"
              }
            ]
          ],
          "tags": [
            {
              "key": "dc",
              "operator": "=~",
              "value": "/^$DC$/"
            },
            {
              "condition": "AND",
              "key": "rack",
              "operator": "=~",
              "value": "/^$Rack$/"
            },
            {
              "condition": "AND",
              "key": "fsid",
              "operator": "=~",
              "value": "/^$fsid$/"
            },
            {
              "condition": "AND",
              "key": "pool_name",
              "operator": "=~",
              "value": "/^$pool$/"
            },
            {
              "condition": "AND",
              "key": "type_instance",
              "operator": "=",
              "value": "wr"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Throughput $pool",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "Bps",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "iops",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "refresh": "30s",
  "schemaVersion": 22,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_INFLUXDB}",
        "definition": "SHOW TAG VALUES FROM \"ceph_cluster_stats\" WITH KEY = \"dc\" ",
        "hide": 0,
        "includeAll": false,
        "index": -1,
        "label": null,
        "multi": false,
        "name": "DC",
        "options": [],
        "query": "SHOW TAG VALUES FROM \"ceph_cluster_stats\" WITH KEY = \"dc\" ",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_INFLUXDB}",
        "definition": "SHOW TAG VALUES FROM \"ceph_cluster_stats\" WITH KEY = \"rack\" WHERE \"dc\" = '$DC'",
        "hide": 0,
        "includeAll": false,
        "index": -1,
        "label": null,
        "multi": false,
        "name": "Rack",
        "options": [],
        "query": "SHOW TAG VALUES FROM \"ceph_cluster_stats\" WITH KEY = \"rack\" WHERE \"dc\" = '$DC'",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_INFLUXDB}",
        "definition": "SHOW TAG VALUES FROM \"ceph_cluster_stats\" WITH KEY = \"fsid\" where \"dc\" = '$DC' and \"rack\" = '$Rack'",
        "hide": 0,
        "includeAll": false,
        "index": -1,
        "label": null,
        "multi": false,
        "name": "fsid",
        "options": [],
        "query": "SHOW TAG VALUES FROM \"ceph_cluster_stats\" WITH KEY = \"fsid\" where \"dc\" = '$DC' and \"rack\" = '$Rack'",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_INFLUXDB}",
        "definition": "SHOW TAG VALUES FROM \"ceph_pool_stats\" WITH KEY = \"pool_name\" where \"fsid\" = '$fsid'",
        "hide": 0,
        "includeAll": true,
        "index": -1,
        "label": null,
        "multi": true,
        "name": "pool",
        "options": [],
        "query": "SHOW TAG VALUES FROM \"ceph_pool_stats\" WITH KEY = \"pool_name\" where \"fsid\" = '$fsid'",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ]
  },
  "timezone": "",
  "title": "Ceph Stats",
  "uid": "PyVGEwyZz",
  "variables": {
    "list": []
  },
  "version": 13
}