Grafana dashboard: ClusterLabs HA Cluster details
Description: Shows the details of a HA cluster running Pacemaker/Corosync. Brought to you by SUSE.

This dashboard shows the details of a HA cluster running Pacemaker/Corosync.

It is built on top of [`ha_cluster_exporter`](https://github.com/ClusterLabs/ha_cluster_exporter), but it also requires Prometheus [`node_exporter`](https://github.com/prometheus/node_exporter) to be configured on the target nodes, and it also assumes that the target nodes in each cluster are grouped via the `job` label.

It is mainly developed by [SUSE](https://www.suse.com).

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "panel",
      "id": "bargauge",
      "name": "Bar gauge",
      "version": ""
    },
    {
      "type": "panel",
      "id": "gauge",
      "name": "Gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "7.0.3"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table-old",
      "name": "Table (old)",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Shows the details of a HA cluster running Pacemaker/Corosync. Brought to you by SUSE.",
  "editable": true,
  "gnetId": 12229,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1594622954666,
  "links": [],
  "panels": [
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {},
          "decimals": 0,
          "mappings": [],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "dark-red",
                "value": null
              },
              {
                "color": "#EAB839",
                "value": 0.5
              },
              {
                "color": "dark-green",
                "value": 1
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 0,
        "y": 0
      },
      "id": 92,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.0.3",
      "targets": [
        {
          "expr": "count(ha_cluster_pacemaker_nodes{instance=\"$dc_instance\", status=\"online\"} == 1) / count(count(ha_cluster_pacemaker_nodes{instance=\"$dc_instance\"}) by (node))",
          "instant": true,
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Online nodes",
      "type": "gauge"
    },
    {
      "cacheTimeout": null,
      "columns": [],
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fontSize": "100%",
      "gridPos": {
        "h": 5,
        "w": 7,
        "x": 3,
        "y": 0
      },
      "id": 98,
      "links": [],
      "pageSize": null,
      "pluginVersion": "6.3.5",
      "scroll": true,
      "showHeader": true,
      "sort": {
        "col": 2,
        "desc": false
      },
      "styles": [
        {
          "alias": "Name",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "node",
          "thresholds": [],
          "type": "number",
          "unit": "short"
        },
        {
          "alias": "Type",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "type",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Online",
          "align": "auto",
          "colorMode": "value",
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 0,
          "link": false,
          "mappingType": 1,
          "pattern": "Value #A",
          "sanitize": false,
          "thresholds": [
            "0",
            "1"
          ],
          "type": "string",
          "unit": "short",
          "valueMaps": [
            {
              "text": "Yes",
              "value": "1"
            },
            {
              "text": "No",
              "value": "0"
            }
          ]
        },
        {
          "alias": "Ip",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "ip",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Unclean",
          "align": "auto",
          "colorMode": "cell",
          "colors": [
            "rgba(50, 172, 45, 0.97)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(245, 54, 54, 0.9)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "Value #C",
          "thresholds": [
            "1",
            "1"
          ],
          "type": "string",
          "unit": "short",
          "valueMaps": [
            {
              "text": "Yes",
              "value": "1"
            },
            {
              "text": "No",
              "value": "0"
            }
          ]
        },
        {
          "alias": "",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "decimals": 2,
          "pattern": "/.*/",
          "thresholds": [],
          "type": "hidden",
          "unit": "short"
        }
      ],
      "targets": [
        {
          "expr": "sum(label_replace(label_replace(node_uname_info{job=\"$cluster\"}, 'node', '$1', 'nodename', '(.*)'), 'ip', '$1', 'instance','(.*):.*')) by (node,ip)",
          "format": "table",
          "hide": true,
          "instant": true,
          "refId": "B"
        },
        {
          "expr": "sum(ha_cluster_pacemaker_nodes{status=\"unclean\", instance=\"$dc_instance\"}) by (node)",
          "format": "table",
          "hide": false,
          "instant": true,
          "refId": "C"
        },
        {
          "expr": "sum(ha_cluster_pacemaker_nodes{status=\"online\", instance=\"$dc_instance\"}) by (node)",
          "format": "table",
          "hide": false,
          "instant": true,
          "legendFormat": "",
          "refId": "A"
        },
        {
          "expr": "sum(ha_cluster_pacemaker_nodes{instance=\"$dc_instance\"}) by (type)",
          "format": "table",
          "hide": false,
          "instant": true,
          "refId": "D"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Nodes",
      "transform": "table",
      "type": "table-old"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {},
          "decimals": 0,
          "mappings": [],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "dark-red",
                "value": null
              },
              {
                "color": "#EAB839",
                "value": 0.5
              },
              {
                "color": "dark-green",
                "value": 1
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 10,
        "y": 0
      },
      "id": 168,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.0.3",
      "targets": [
        {
          "expr": "count(ha_cluster_pacemaker_resources{instance=\"$dc_instance\", status=\"active\"} == 1) / count(count(ha_cluster_pacemaker_resources{instance=\"$dc_instance\"}) by (resource,node))",
          "instant": true,
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Active Resources",
      "type": "gauge"
    },
    {
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {},
          "mappings": [],
          "max": 2,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 7,
        "x": 13,
        "y": 0
      },
      "id": 140,
      "options": {
        "displayMode": "gradient",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true
      },
      "pluginVersion": "7.0.3",
      "targets": [
        {
          "expr": "ha_cluster_corosync_quorum_votes{instance=\"$dc_instance\",type=\"expected_votes\"}",
          "instant": true,
          "legendFormat": "Expected votes",
          "refId": "A"
        },
        {
          "expr": "ha_cluster_corosync_quorum_votes{instance=\"$dc_instance\",type=\"highest_expected\"}",
          "instant": true,
          "legendFormat": "Highest expected votes",
          "refId": "B"
        },
        {
          "expr": "ha_cluster_corosync_quorum_votes{instance=\"$dc_instance\", type=\"total_votes\"}",
          "instant": true,
          "legendFormat": "Total votes",
          "refId": "C"
        },
        {
          "expr": "ha_cluster_corosync_quorum_votes{instance=\"$dc_instance\", type=\"quorum\"}",
          "instant": true,
          "legendFormat": "Quorum",
          "refId": "D"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Quorum votes",
      "type": "bargauge"
    },
    {
      "cacheTimeout": null,
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [
            {
              "id": 1,
              "op": "=",
              "text": "OK",
              "type": 1,
              "value": "1"
            },
            {
              "id": 2,
              "op": "=",
              "text": "KO",
              "type": 1,
              "value": "0"
            }
          ],
          "nullValueMode": "connected",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "#F2495C",
                "value": null
              },
              {
                "color": "#F2495C",
                "value": 0
              },
              {
                "color": "#37872D",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 2,
        "x": 20,
        "y": 0
      },
      "id": 138,
      "interval": null,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "background",
        "fieldOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": true
        }
      },
      "pluginVersion": "7.0.3",
      "targets": [
        {
          "expr": "ha_cluster_corosync_quorate{instance=\"$dc_instance\"}",
          "instant": true,
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Cluster quorate",
      "type": "stat"
    },
    {
      "cacheTimeout": null,
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "nullValueMode": "connected",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "#299c46",
                "value": null
              },
              {
                "color": "#37872D",
                "value": 0
              },
              {
                "color": "#d44a3a",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 2,
        "x": 22,
        "y": 0
      },
      "id": 136,
      "interval": null,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "background",
        "fieldOptions": {
          "calcs": [
            "mean"
          ]
        },
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": true
        }
      },
      "pluginVersion": "7.0.3",
      "targets": [
        {
          "expr": "ha_cluster_corosync_ring_errors{instance=\"$dc_instance\"}",
          "instant": true,
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Faulty Corosync Rings",
      "type": "stat"
    },
    {
      "cacheTimeout": null,
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "nullValueMode": "connected",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "rgb(255, 255, 255)",
                "value": null
              }
            ]
          },
          "unit": "dateTimeFromNow"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 4,
        "x": 20,
        "y": 3
      },
      "id": 159,
      "interval": null,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "fieldOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": true
        }
      },
      "pluginVersion": "7.0.3",
      "targets": [
        {
          "expr": "ha_cluster_pacemaker_config_last_change{instance=\"$dc_instance\"} * 1000",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Last CIB change",
      "type": "stat"
    },
    {
      "aliasColors": {
        "dc": "blue",
        "expected_up": "dark-green",
        "maintenance": "yellow",
        "online": "green",
        "pending": "dark-yellow",
        "shutdown": "orange",
        "standby": "light-yellow",
        "standby_onfail": "semi-dark-red",
        "unclean": "dark-red"
      },
      "bars": false,
      "cacheTimeout": null,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fill": 4,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 10,
        "x": 0,
        "y": 5
      },
      "hiddenSeries": false,
      "id": 104,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pluginVersion": "6.3.5",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum(ha_cluster_pacemaker_nodes{instance=\"$dc_instance\"}) by (status)",
          "instant": false,
          "interval": "",
          "legendFormat": "{{status}}",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Node statuses over time",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": 0,
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "decimals": 0,
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {
        "blocked": "orange",
        "failed": "red",
        "failure_ignored": "yellow"
      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 14,
        "x": 10,
        "y": 5
      },
      "hiddenSeries": false,
      "id": 167,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum(ha_cluster_pacemaker_resources{instance=\"$dc_instance\"}) by (status)",
          "legendFormat": "{{status}}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Resource statuses over time",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": 0,
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "decimals": 0,
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "columns": [],
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fontSize": "100%",
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 11
      },
      "id": 108,
      "pageSize": null,
      "pluginVersion": "6.3.5",
      "repeat": null,
      "repeatDirection": "h",
      "scroll": true,
      "showHeader": true,
      "sort": {
        "col": 3,
        "desc": false
      },
      "styles": [
        {
          "alias": "Resource",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "resource",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Status",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "status",
          "thresholds": [],
          "type": "string",
          "unit": "short",
          "valueMaps": []
        },
        {
          "alias": "Migration threshold",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 0,
          "mappingType": 1,
          "pattern": "Value #B",
          "thresholds": [],
          "type": "number",
          "unit": "none"
        },
        {
          "alias": "Node",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "node",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Fail count",
          "align": "auto",
          "colorMode": "value",
          "colors": [
            "rgba(50, 172, 45, 0.97)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(245, 54, 54, 0.9)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 0,
          "mappingType": 1,
          "pattern": "Value #C",
          "thresholds": [
            "1",
            "100"
          ],
          "type": "number",
          "unit": "none"
        },
        {
          "alias": "Role",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "role",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Type",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "agent",
          "thresholds": [],
          "type": "number",
          "unit": "short"
        },
        {
          "alias": "",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "decimals": 2,
          "pattern": "/.*/",
          "thresholds": [],
          "type": "hidden",
          "unit": "short"
        }
      ],
      "targets": [
        {
          "expr": "sum(ha_cluster_pacemaker_resources{instance=\"$dc_instance\"} == 1) by(resource, node, status, role, agent)",
          "format": "table",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        },
        {
          "expr": "sum(ha_cluster_pacemaker_migration_threshold{instance=\"$dc_instance\"}) by(resource, node)",
          "format": "table",
          "hide": false,
          "instant": true,
          "refId": "B"
        },
        {
          "expr": "sum(ha_cluster_pacemaker_fail_count{instance=\"$dc_instance\"}) by(resource, node)",
          "format": "table",
          "hide": false,
          "instant": true,
          "refId": "C"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Resources",
      "transform": "table",
      "type": "table-old"
    },
    {
      "columns": [],
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fontSize": "100%",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 20
      },
      "id": 169,
      "pageSize": null,
      "repeatDirection": "v",
      "scroll": true,
      "showHeader": true,
      "sort": {
        "col": 5,
        "desc": false
      },
      "styles": [
        {
          "alias": "Name",
          "align": "auto",
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "pattern": "name",
          "preserveFormat": false,
          "sanitize": false,
          "type": "string"
        },
        {
          "alias": "Value",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 0,
          "link": false,
          "mappingType": 1,
          "pattern": "value",
          "thresholds": [
            "1",
            "1"
          ],
          "type": "string",
          "unit": "short",
          "valueMaps": []
        },
        {
          "alias": "Node",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "nodename",
          "thresholds": [],
          "type": "number",
          "unit": "short"
        },
        {
          "alias": "Node",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "node",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "decimals": 2,
          "pattern": "/.*/",
          "thresholds": [],
          "type": "hidden",
          "unit": "short"
        }
      ],
      "targets": [
        {
          "expr": "ha_cluster_pacemaker_node_attributes{instance=\"$dc_instance\"}",
          "format": "table",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Node Attributes",
      "transform": "table",
      "type": "table-old"
    },
    {
      "columns": [],
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fontSize": "100%",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 20
      },
      "id": 157,
      "pageSize": null,
      "repeat": "node",
      "repeatDirection": "v",
      "scroll": true,
      "showHeader": true,
      "sort": {
        "col": 4,
        "desc": false
      },
      "styles": [
        {
          "alias": "Service",
          "align": "auto",
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "pattern": "name",
          "preserveFormat": false,
          "sanitize": false,
          "type": "string"
        },
        {
          "alias": "Active",
          "align": "auto",
          "colorMode": "cell",
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 0,
          "link": false,
          "mappingType": 1,
          "pattern": "Value",
          "thresholds": [
            "1",
            "1"
          ],
          "type": "string",
          "unit": "short",
          "valueMaps": [
            {
              "text": "Yes",
              "value": "1"
            },
            {
              "text": "No",
              "value": "0"
            }
          ]
        },
        {
          "alias": "Node",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "nodename",
          "thresholds": [],
          "type": "number",
          "unit": "short"
        },
        {
          "alias": "",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "decimals": 2,
          "pattern": "/.*/",
          "thresholds": [],
          "type": "hidden",
          "unit": "short"
        }
      ],
      "targets": [
        {
          "expr": "node_systemd_unit_state{name=~\"((corosync|pacemaker|sbd|prometheus|hawk).*)\", state=\"active\"} + on(instance) group_left(nodename) 0 * node_uname_info{job=\"$cluster\"}",
          "format": "table",
          "instant": true,
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "systemd units",
      "transform": "table",
      "type": "table-old"
    },
    {
      "columns": [],
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fontSize": "100%",
      "gridPos": {
        "h": 4,
        "w": 19,
        "x": 0,
        "y": 28
      },
      "id": 165,
      "pageSize": null,
      "scroll": true,
      "showHeader": true,
      "sort": {
        "col": 0,
        "desc": true
      },
      "styles": [
        {
          "alias": "Score",
          "align": "auto",
          "colorMode": "cell",
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 0,
          "mappingType": 1,
          "pattern": "Value",
          "preserveFormat": true,
          "rangeMaps": [],
          "sanitize": false,
          "thresholds": [
            "0",
            ""
          ],
          "type": "number",
          "unit": "none",
          "valueMaps": []
        },
        {
          "alias": "Constraint ID",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "constraint",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Resource",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "resource",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Node",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "node",
          "thresholds": [],
          "type": "number",
          "unit": "short"
        },
        {
          "alias": "Role",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "role",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "decimals": 2,
          "mappingType": 1,
          "pattern": "/.*/",
          "thresholds": [],
          "type": "hidden",
          "unit": "short",
          "valueMaps": []
        }
      ],
      "targets": [
        {
          "expr": "clamp_min(clamp_max(ha_cluster_pacemaker_location_constraints{instance=\"$dc_instance\"}, 1000000), -1000000)",
          "format": "table",
          "hide": false,
          "instant": true,
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Resource location constraints",
      "transform": "table",
      "type": "table-old"
    },
    {
      "cacheTimeout": null,
      "columns": [],
      "datasource": "${DS_PROMETHEUS}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fontSize": "100%",
      "gridPos": {
        "h": 4,
        "w": 5,
        "x": 19,
        "y": 28
      },
      "id": 142,
      "links": [],
      "pageSize": null,
      "scroll": true,
      "showHeader": true,
      "sort": {
        "col": 0,
        "desc": true
      },
      "styles": [
        {
          "alias": "Device",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "device",
          "thresholds": [],
          "type": "string",
          "unit": "short"
        },
        {
          "alias": "Status",
          "align": "auto",
          "colorMode": "cell",
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "decimals": 2,
          "mappingType": 1,
          "pattern": "Value #B",
          "thresholds": [
            "1",
            "1"
          ],
          "type": "string",
          "unit": "short",
          "valueMaps": [
            {
              "text": "Healthy",
              "value": "1"
            },
            {
              "text": "Unealthy",
              "value": "0"
            }
          ]
        },
        {
          "alias": "",
          "align": "auto",
          "colorMode": null,
          "colors": [
            "rgba(245, 54, 54, 0.9)",
            "rgba(237, 129, 40, 0.89)",
            "rgba(50, 172, 45, 0.97)"
          ],
          "dateFormat": "YYYY-MM-DD HH:mm:ss",
          "decimals": 2,
          "mappingType": 1,
          "pattern": "/.*/",
          "thresholds": [],
          "type": "hidden",
          "unit": "short"
        }
      ],
      "targets": [
        {
          "expr": "count(ha_cluster_sbd_devices{instance=\"$dc_instance\"}) by (device,status)",
          "format": "table",
          "instant": true,
          "refId": "A"
        },
        {
          "expr": "count(ha_cluster_sbd_devices{instance=\"$dc_instance\",status=\"healthy\"}) by (device,status)",
          "format": "table",
          "instant": true,
          "refId": "B"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "SBD Devices",
      "transform": "table",
      "type": "table-old"
    }
  ],
  "refresh": false,
  "schemaVersion": 25,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "Prometheus",
          "value": "Prometheus"
        },
        "hide": 2,
        "includeAll": false,
        "label": "Data Source",
        "multi": false,
        "name": "DS_PROMETHEUS",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "$DS_PROMETHEUS",
        "definition": "label_values(job)",
        "hide": 0,
        "includeAll": false,
        "label": "Cluster",
        "multi": false,
        "name": "cluster",
        "options": [],
        "query": "label_values(job)",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "$DS_PROMETHEUS",
        "definition": "query_result(label_replace((ha_cluster_pacemaker_nodes{job='$cluster',status='dc'} == 1), 'ip', '$1', 'instance', '(.*):.*') + on(node, ip) group_left label_replace(label_replace(0 * node_uname_info{job='$cluster'}, 'node', '$1', 'nodename', '(.*)'), 'ip', '$1', 'instance','(.*):.*'))",
        "hide": 2,
        "includeAll": false,
        "label": null,
        "multi": false,
        "name": "dc_instance",
        "options": [],
        "query": "query_result(label_replace((ha_cluster_pacemaker_nodes{job='$cluster',status='dc'} == 1), 'ip', '$1', 'instance', '(.*):.*') + on(node, ip) group_left label_replace(label_replace(0 * node_uname_info{job='$cluster'}, 'node', '$1', 'nodename', '(.*)'), 'ip', '$1', 'instance','(.*):.*'))",
        "refresh": 2,
        "regex": ".*instance=\"(.+?)\".*",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "ClusterLabs HA Cluster details",
  "uid": "Q5YJpwtZk1",
  "version": 4
}