Grafana dashboard: Cluster overview
Description: Nomad cluster dashboard. Using built in nomad's exporter and loki. Example cluster is present there https://github.com/theztd/startup-infra-docker in 

# Example nomad cluster dashboard

Monitored example cluster with example promtail, loki and prometheus configuration is live there [https://github.com/theztd/startup-infra-docker](https://github.com/theztd/startup-infra-docker)

**I will be happy for the feedback**

{
  "__inputs": [
    {
      "name": "DS_GRAFANACLOUD-GRAFY-PROM",
      "label": "grafanacloud-grafy-prom",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    },
    {
      "name": "DS_GRAFANACLOUD-GRAFY-LOGS",
      "label": "grafanacloud-grafy-logs",
      "description": "",
      "type": "datasource",
      "pluginId": "loki",
      "pluginName": "Loki"
    }
  ],
  "__elements": {},
  "__requires": [
    {
      "type": "panel",
      "id": "bargauge",
      "name": "Bar gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "9.2.7-8da65d62"
    },
    {
      "type": "panel",
      "id": "logs",
      "name": "Logs",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "loki",
      "name": "Loki",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "piechart",
      "name": "Pie chart",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "datasource",
          "uid": "grafana"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "Nomad cluster dashboard. Using built in nomad's exporter and loki. Example cluster is present there https://github.com/theztd/startup-infra-docker in ",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "gnetId": 17549,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 207,
      "panels": [],
      "title": "Cluster Summary",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "green",
            "mode": "fixed"
          },
          "decimals": 0,
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 0,
        "y": 1
      },
      "id": 221,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_job_status_running{}",
          "instant": true,
          "interval": "",
          "legendFormat": "Running",
          "refId": "A"
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "description": "In the whole cluster",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            }
          },
          "mappings": [],
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 2,
        "y": 1
      },
      "id": 107,
      "options": {
        "displayLabels": [
          "percent"
        ],
        "legend": {
          "displayMode": "list",
          "placement": "right",
          "showLegend": true,
          "values": []
        },
        "pieType": "donut",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "9.0.6",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum(nomad_client_allocs_cpu_allocated) by (exported_job)",
          "interval": "",
          "legendFormat": "{{exported_job}}",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum(nomad_client_unallocated_cpu)",
          "hide": false,
          "legendFormat": "Free cluster CPU",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "Allocated CPU  ($datacenter)",
      "type": "piechart"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "inspect": false
          },
          "links": [
            {
              "title": "Set time range",
              "url": "https://mon.cinarra.com/d/4NYOvS-7z/nomad?orgId=1&var-env=$env&var-alloc_id=${__data.fields.alloc_id}﻿﻿&from=﻿﻿${__data.fields.from}&to=${__data.fields.to}"
            }
          ],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "from"
            },
            "properties": [
              {
                "id": "custom.hidden",
                "value": true
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "to"
            },
            "properties": [
              {
                "id": "custom.hidden",
                "value": true
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "alloc_id"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 287
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Kills"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 45
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Instance"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 131
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "task"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 92
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 13,
        "x": 8,
        "y": 1
      },
      "id": 215,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "9.2.7-8da65d62",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "nomad_client_allocs_oom_killed{}",
          "format": "table",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "oom",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "timestamp(nomad_client_allocs_oom_killed{}) * 1000 - 3600000",
          "format": "table",
          "hide": false,
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "from",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": true,
          "expr": "timestamp(nomad_client_allocs_oom_killed{}) * 1000 + 3600000",
          "format": "table",
          "hide": false,
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "to",
          "refId": "C"
        }
      ],
      "title": "OOM kills",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "alloc_id"
          }
        },
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "pattern": "alloc_id|exporter_job 1|Time 1|instance 1|namespace 1|task 1|task_group 1|Value .*"
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Value #B": true,
              "Value #C": true,
              "namespace 1": true,
              "task_group 1": true
            },
            "indexByName": {
              "Time 1": 0,
              "Value #A": 6,
              "Value #B": 7,
              "Value #C": 8,
              "alloc_id": 3,
              "instance 1": 1,
              "namespace 1": 4,
              "task 1": 2,
              "task_group 1": 5
            },
            "renameByName": {
              "Time 1": "Time",
              "Value #A": "Kills",
              "Value #B": "from",
              "Value #C": "to",
              "alloc_id": "",
              "instance 1": "Instance",
              "namespace 1": "namespace",
              "task 1": "task",
              "task_group 1": "group"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Time"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "color": "red",
                  "index": 1,
                  "text": "Unhealthy"
                },
                "1": {
                  "color": "green",
                  "index": 0,
                  "text": "Healthy"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 12,
        "w": 3,
        "x": 21,
        "y": 1
      },
      "id": 229,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "up{job=\"nomad_metrics\"}",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "title": "Node status",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "light-orange",
            "mode": "fixed"
          },
          "decimals": 0,
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 0,
        "y": 5
      },
      "id": 217,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "nomad_nomad_job_status_pending",
          "instant": true,
          "interval": "",
          "legendFormat": "Pending",
          "refId": "A"
        }
      ],
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "filterable": true,
            "inspect": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Run"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "color-background-solid"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "green",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.width",
                "value": 64
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Complete"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "color-background-solid"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "semi-dark-green",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.width",
                "value": 65
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Failed"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "color-background-solid"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "semi-dark-red",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.width",
                "value": 63
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Starting"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "color-background-solid"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "text",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.width",
                "value": 80
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Quequed"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "color-background-solid"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "transparent",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.width",
                "value": 80
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Lost"
            },
            "properties": [
              {
                "id": "custom.displayMode",
                "value": "color-background-solid"
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "#000000",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.width",
                "value": 62
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "job"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 264
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Node"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 187
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Job"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 351
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 13,
        "x": 8,
        "y": 5
      },
      "id": 231,
      "options": {
        "footer": {
          "fields": [
            "Value #F",
            "Value #A",
            "Value #B",
            "Value #D",
            "Value #E",
            "Value #C"
          ],
          "reducer": [
            "sum"
          ],
          "show": true
        },
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "9.2.7-8da65d62",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_job_summary_complete{}",
          "format": "table",
          "instant": true,
          "interval": "",
          "legendFormat": "complete",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_job_summary_failed{}",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "failed",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_job_summary_lost{}",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "lost",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_job_summary_starting{env=\"$env\"}",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "starting",
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_job_summary_queued{env=\"$env\"}",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "queued",
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_job_summary_running{env=\"$env\"}",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "running",
          "refId": "F"
        }
      ],
      "title": "Allocations",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "task_group"
          }
        },
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "names": [
                "task_group",
                "Value #A",
                "Value #B",
                "Value #C",
                "host 1"
              ],
              "pattern": "exported_job 1|namespace 1|task_group|Value .*"
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "namespace 1": true,
              "task_group": true
            },
            "indexByName": {
              "Value #A": 4,
              "Value #B": 5,
              "Value #C": 6,
              "exported_job 1": 0,
              "host 1": 1,
              "namespace 1": 3,
              "task_group": 2
            },
            "renameByName": {
              "Value #A": "Run",
              "Value #B": "Failed",
              "Value #C": "Lost",
              "Value #D": "Starting",
              "Value #E": "Quequed",
              "Value #F": "Running",
              "exported_job 1": "Job",
              "host 1": "Node",
              "namespace 1": "namespace",
              "task_group": "group",
              "task_group 1": "Group"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "description": "In whole cluster",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            }
          },
          "mappings": [],
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 2,
        "y": 7
      },
      "id": 86,
      "options": {
        "displayLabels": [
          "percent"
        ],
        "legend": {
          "displayMode": "list",
          "placement": "right",
          "showLegend": true,
          "values": []
        },
        "pieType": "donut",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "9.0.6",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum(nomad_client_allocs_memory_allocated) by (exported_job)",
          "interval": "",
          "legendFormat": "{{exported_job}}",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "sum(nomad_client_host_memory_free)",
          "hide": false,
          "legendFormat": "Free cluster MEM",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "Allocated MEM ($datacenter)",
      "type": "piechart"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "super-light-red",
            "mode": "fixed"
          },
          "decimals": 0,
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 2,
        "x": 0,
        "y": 9
      },
      "id": 219,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "nomad_nomad_job_status_dead",
          "instant": true,
          "interval": "",
          "legendFormat": "Dead",
          "refId": "A"
        }
      ],
      "type": "stat"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "000000001"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 13
      },
      "id": 2,
      "panels": [],
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "000000001"
          },
          "refId": "A"
        }
      ],
      "title": "Node Status",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 1,
          "displayName": "($host)",
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "light-red",
                "value": null
              },
              {
                "color": "light-orange",
                "value": 360
              },
              {
                "color": "light-green",
                "value": 86400
              }
            ]
          },
          "unit": "dtdurations"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 0,
        "y": 14
      },
      "id": 4,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "repeat": "host",
      "repeatDirection": "v",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "expr": "nomad_client_uptime{datacenter=~\"$datacenter\", host=~\"$host\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "title": "Uptime",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "displayName": "$host",
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 3,
        "y": 14
      },
      "id": 11,
      "links": [],
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "repeat": "host",
      "repeatDirection": "v",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "expr": "nomad_client_host_memory_free{datacenter=~\"$datacenter\", host=~\"$host\"} /nomad_client_host_memory_total{datacenter=~\"$datacenter\", host=~\"$host\"}",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 1,
          "legendFormat": "Total",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "Free memmory",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 0,
          "mappings": [],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "light-red",
                "value": null
              },
              {
                "color": "light-orange",
                "value": 15
              },
              {
                "color": "light-green",
                "value": 30
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 5,
        "x": 9,
        "y": 14
      },
      "id": 13,
      "links": [],
      "options": {
        "displayMode": "lcd",
        "minVizHeight": 0,
        "minVizWidth": 0,
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true
      },
      "pluginVersion": "9.2.7-8da65d62",
      "repeat": "host",
      "repeatDirection": "v",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "expr": "avg(nomad_client_host_disk_available{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk) / max(nomad_client_host_disk_size{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk) ",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{disk}}",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "expr": "avg(nomad_client_host_disk_available{datacenter=~\"$datacenter\", host=~\"$host\"}) by (disk)",
          "format": "time_series",
          "hide": true,
          "intervalFactor": 1,
          "legendFormat": "Available - {{disk}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Free Disk Space ($host)",
      "type": "bargauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 14,
        "y": 14
      },
      "id": 58,
      "links": [],
      "options": {
        "displayMode": "lcd",
        "minVizHeight": 2,
        "minVizWidth": 0,
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true
      },
      "pluginVersion": "9.2.7-8da65d62",
      "repeat": "host",
      "repeatDirection": "v",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "expr": "nomad_client_allocations_migrating{datacenter=~\"$datacenter\", host=~\"$host\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Migrating",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "expr": "nomad_client_allocations_blocked{datacenter=~\"$datacenter\", host=~\"$host\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Blocked",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "expr": "nomad_client_allocations_pending{datacenter=~\"$datacenter\", host=~\"$host\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Pending",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "expr": "nomad_client_allocations_running{datacenter=~\"$datacenter\", host=~\"$host\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Running",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "expr": "nomad_client_allocations_terminal{datacenter=~\"$datacenter\", host=~\"$host\"}",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 1,
          "legendFormat": "Terminal",
          "refId": "E"
        }
      ],
      "title": "Allocation status",
      "type": "bargauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "displayName": "$host",
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "light-red",
                "value": null
              },
              {
                "color": "#EAB839",
                "value": 10
              },
              {
                "color": "light-green",
                "value": 50
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 18,
        "y": 14
      },
      "id": 33,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "repeat": "host",
      "repeatDirection": "v",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "editorMode": "code",
          "expr": "nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_cpu{datacenter=~\"$datacenter\", host=~\"$host\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Unalocated CPU",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "displayName": "$host",
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "light-red",
                "value": null
              },
              {
                "color": "#EAB839",
                "value": 10
              },
              {
                "color": "light-green",
                "value": 50
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 21,
        "y": 14
      },
      "id": 40,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "9.2.7-8da65d62",
      "repeat": "host",
      "repeatDirection": "v",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "expr": "nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}/(nomad_client_unallocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"}+nomad_client_allocated_memory{datacenter=~\"$datacenter\", host=~\"$host\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "title": "Unalocated Mem",
      "type": "stat"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 239,
      "panels": [],
      "title": "Logs",
      "type": "row"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_GRAFANACLOUD-GRAFY-LOGS}"
      },
      "description": "",
      "gridPos": {
        "h": 13,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "id": 127,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_GRAFANACLOUD-GRAFY-LOGS}"
          },
          "editorMode": "code",
          "expr": "{instance=~\"$host\"}",
          "hide": false,
          "queryType": "range",
          "refId": "A"
        }
      ],
      "title": "Logs ($datacenter)",
      "type": "logs"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 34
      },
      "id": 247,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "description": "Time difference between when a message was first passed into Raft and when the resulting commit took place on the leader",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "line"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "yellow",
                    "value": 100
                  },
                  {
                    "color": "red",
                    "value": 500
                  }
                ]
              },
              "unit": "ms"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 37
          },
          "id": 223,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
              },
              "exemplar": true,
              "expr": "nomad_raft_commitTime{quantile=\"0.9\"}",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ instance }}",
              "refId": "A"
            }
          ],
          "title": "Raft Commit Time",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
          },
          "description": "Number of Raft transactions - general indicator of the write load on the servers.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisCenteredZero": false,
                "axisColorMode": "text",
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "stepBefore",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "decimals": 0,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 37
          },
          "id": 225,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "list",
              "placement": "bottom",
              "showLegend": true
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
              },
              "editorMode": "code",
              "exemplar": true,
              "expr": "increase(nomad_raft_apply{}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{ instance }}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Raft Apply Count",
          "type": "timeseries"
        }
      ],
      "title": "RAFT status",
      "type": "row"
    }
  ],
  "refresh": "1m",
  "schemaVersion": 37,
  "style": "dark",
  "tags": [
    "nomad",
    "host",
    "cluster"
  ],
  "templating": {
    "list": [
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
        },
        "definition": "",
        "hide": 0,
        "includeAll": false,
        "label": "DC",
        "multi": false,
        "name": "datacenter",
        "options": [],
        "query": {
          "query": "label_values(nomad_client_uptime, datacenter)",
          "refId": "de1-datacenter-Variable-Query"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_GRAFANACLOUD-GRAFY-PROM}"
        },
        "definition": "",
        "hide": 0,
        "includeAll": true,
        "label": "Host",
        "multi": true,
        "name": "host",
        "options": [],
        "query": {
          "query": "label_values(nomad_client_uptime{datacenter=~\"$datacenter\"}, host)",
          "refId": "de1-host-Variable-Query"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-15m",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Cluster overview",
  "uid": "masiNomadCluster",
  "version": 11,
  "weekStart": ""
}