Grafana dashboard: HashiCorp Nomad
Description: A dashboard for HashiCorp Nomad https://www.nomadproject.io/

HashiCorp Nomad dashboard

{
  "__inputs": [
    {
      "name": "DS_VICTORIAMETRICS",
      "label": "VictoriaMetrics",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": [],
  "__requires": [
    {
      "type": "panel",
      "id": "bargauge",
      "name": "Bar gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "8.3.6"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "status-history",
      "name": "Status history",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1644948294626,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "color": "red",
                  "index": 1,
                  "text": "Unhealthy"
                },
                "1": {
                  "color": "green",
                  "index": 0,
                  "text": "Healthy"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 0,
        "y": 0
      },
      "id": 14,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.6",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_VICTORIAMETRICS}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_autopilot_healthy{env=\"$env\"}",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{ instance }}",
          "refId": "A"
        }
      ],
      "type": "stat"
    },
    {
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 6,
        "y": 0
      },
      "id": 16,
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "pluginVersion": "8.3.6",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_VICTORIAMETRICS}"
          },
          "exemplar": false,
          "expr": "nomad_nomad_autopilot_failure_tolerance{env=\"$env\"}",
          "instant": true,
          "interval": "",
          "legendFormat": "fault tolerance",
          "refId": "A"
        }
      ],
      "title": "Fault tolerance",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "continuous-RdYlGr"
          },
          "decimals": 0,
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 12,
        "y": 0
      },
      "id": 18,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.6",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_VICTORIAMETRICS}"
          },
          "exemplar": true,
          "expr": "100 -\nsum(nomad_client_unallocated_cpu{env=\"$env\"}) by (env)\n/\nsum(nomad_client_allocated_cpu{env=\"$env\"}\n+ nomad_client_unallocated_cpu{env=\"$env\"}) by (env) * 100",
          "interval": "",
          "legendFormat": "CPU Capacity",
          "refId": "A"
        }
      ],
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "continuous-RdYlGr"
          },
          "decimals": 0,
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 16,
        "y": 0
      },
      "id": 19,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.6",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_VICTORIAMETRICS}"
          },
          "exemplar": true,
          "expr": "100 -\nsum(nomad_client_unallocated_memory{env=\"$env\"}) by (env)\n/\nsum(nomad_client_allocated_memory{env=\"$env\"}\n+ nomad_client_unallocated_memory{env=\"$env\"}) by (env) * 100",
          "interval": "",
          "legendFormat": "Memory Capacity",
          "refId": "A"
        }
      ],
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "continuous-RdYlGr"
          },
          "decimals": 0,
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 20,
        "y": 0
      },
      "id": 20,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.6",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_VICTORIAMETRICS}"
          },
          "exemplar": true,
          "expr": "100 -\nsum(nomad_client_unallocated_disk{env=\"$env\"}) by (env)\n/\nsum(nomad_client_allocated_disk{env=\"$env\"}\n+ nomad_client_unallocated_disk{env=\"$env\"}) by (env) * 100",
          "interval": "",
          "legendFormat": "Disk Capacity",
          "refId": "A"
        }
      ],
      "type": "stat"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 4
      },
      "id": 28,
      "panels": [
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "links": [
                {
                  "targetBlank": true,
                  "title": "System dashboard",
                  "url": "https://mon.cinarra.com/d/b4FAZ7Mmz/system?orgId=1&var-env=$env&var-instance=${__data.fields.Client}"
                }
              ],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Node ID"
                },
                "properties": [
                  {
                    "id": "custom.width",
                    "value": 130
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Node class"
                },
                "properties": [
                  {
                    "id": "custom.width",
                    "value": 90
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Eligibility"
                },
                "properties": [
                  {
                    "id": "custom.width",
                    "value": 100
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Status"
                },
                "properties": [
                  {
                    "id": "custom.width",
                    "value": 80
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Uptime"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "s"
                  },
                  {
                    "id": "custom.width",
                    "value": 100
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "CPU"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "percent"
                  },
                  {
                    "id": "color",
                    "value": {
                      "mode": "continuous-RdYlGr"
                    }
                  },
                  {
                    "id": "custom.displayMode",
                    "value": "color-background"
                  },
                  {
                    "id": "min",
                    "value": 0
                  },
                  {
                    "id": "max",
                    "value": 100
                  },
                  {
                    "id": "custom.width",
                    "value": 70
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Memory"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "percent"
                  },
                  {
                    "id": "color",
                    "value": {
                      "mode": "continuous-RdYlGr"
                    }
                  },
                  {
                    "id": "custom.displayMode",
                    "value": "color-background"
                  },
                  {
                    "id": "min",
                    "value": 0
                  },
                  {
                    "id": "max",
                    "value": 100
                  },
                  {
                    "id": "custom.width",
                    "value": 70
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Disk"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "percent"
                  },
                  {
                    "id": "color",
                    "value": {
                      "mode": "continuous-RdYlGr"
                    }
                  },
                  {
                    "id": "custom.displayMode",
                    "value": "color-background"
                  },
                  {
                    "id": "min",
                    "value": 0
                  },
                  {
                    "id": "max",
                    "value": 100
                  },
                  {
                    "id": "custom.width",
                    "value": 70
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Allocs"
                },
                "properties": [
                  {
                    "id": "custom.width",
                    "value": 80
                  },
                  {
                    "id": "custom.displayMode",
                    "value": "color-text"
                  },
                  {
                    "id": "color"
                  },
                  {
                    "id": "thresholds",
                    "value": {
                      "mode": "absolute",
                      "steps": [
                        {
                          "color": "red"
                        },
                        {
                          "color": "text",
                          "value": 1
                        }
                      ]
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 5
          },
          "id": 22,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_uptime{env=\"$env\"}",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "uptime",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocations_running{env=\"$env\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "allocations running",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "100 -\nnomad_client_unallocated_cpu{env=\"$env\"}\n/\n(nomad_client_allocated_cpu{env=\"$env\"}\n+ nomad_client_unallocated_cpu{env=\"$env\"}) * 100",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "cpu utilization",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "100 -\nnomad_client_unallocated_memory{env=\"$env\"}\n/\n(nomad_client_allocated_memory{env=\"$env\"}\n+ nomad_client_unallocated_memory{env=\"$env\"}) * 100",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "memory utilization",
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "100 -\nnomad_client_unallocated_disk{env=\"$env\"}\n/\n(nomad_client_allocated_disk{env=\"$env\"}\n+ nomad_client_unallocated_disk{env=\"$env\"}) * 100",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "disk utilization",
              "refId": "E"
            }
          ],
          "title": "Clients",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "instance"
              }
            },
            {
              "id": "filterFieldsByName",
              "options": {
                "include": {
                  "pattern": "instance|node_.* 1|Value .*"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Value #A": 5,
                  "Value #B": 6,
                  "instance": 0,
                  "node_class 1": 2,
                  "node_id 1": 1,
                  "node_scheduling_eligibility 1": 4,
                  "node_status 1": 3
                },
                "renameByName": {
                  "Value #A": "Uptime",
                  "Value #B": "Allocs",
                  "Value #C": "CPU",
                  "Value #D": "Memory",
                  "Value #E": "Disk",
                  "instance": "Client",
                  "node_class": "Node class",
                  "node_class 1": "Node class",
                  "node_id": "Node ID",
                  "node_id 1": "Node ID",
                  "node_scheduling_eligibility": "Elegibility",
                  "node_scheduling_eligibility 1": "Eligibility",
                  "node_status": "Status",
                  "node_status 1": "Status"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "description": "This panel should show no activity in a stable cluster. Some leadership transition events are expected, such as during cluster upgrades. Unexpected events may indicate an issue with cluster stability",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "fillOpacity": 70,
                "lineWidth": 2
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "text"
                  }
                ]
              }
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byRegexp",
                  "options": ".* - candidate"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "light-orange",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byRegexp",
                  "options": ".* - leader"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "red",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 13
          },
          "hideTimeOverride": false,
          "id": 8,
          "options": {
            "colWidth": 1,
            "legend": {
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "rowHeight": 0.75,
            "showValue": "auto",
            "tooltip": {
              "mode": "single"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "increase(nomad_raft_state_candidate{env=\"$env\"}[1m]) > 0",
              "format": "time_series",
              "hide": false,
              "instant": false,
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{ instance }} - candidate",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "increase(nomad_raft_state_leader{env=\"$env\"}[1m]) > 0",
              "hide": false,
              "instant": false,
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{ instance }} - leader",
              "refId": "B"
            }
          ],
          "title": "Election Events",
          "transformations": [],
          "type": "status-history"
        },
        {
          "description": "Measures the time since the leader was last able to contact the follower nodes when checking its leader lease",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "line"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "yellow",
                    "value": 200
                  },
                  {
                    "color": "red",
                    "value": 500
                  }
                ]
              },
              "unit": "ms"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 13
          },
          "id": 36,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_raft_leader_lastContact{env=\"$env\",quantile=\"0.9\"}",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ instance }}",
              "refId": "C"
            }
          ],
          "title": "Last Contacted",
          "type": "timeseries"
        }
      ],
      "title": "Cluster",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 5
      },
      "id": 84,
      "panels": [
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 6
          },
          "id": 82,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocations_blocked{env=\"$env\",instance=~\"$client\"}",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "blocked",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocations_migrating{env=\"$env\",instance=~\"$client\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "migrating",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocations_pending{env=\"$env\",instance=~\"$client\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "pending",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocations_running{env=\"$env\",instance=~\"$client\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "running",
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocations_start{env=\"$env\",instance=~\"$client\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "start",
              "refId": "E"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocations_terminal{env=\"$env\",instance=~\"$client\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "terminal",
              "refId": "F"
            }
          ],
          "title": "Allocations",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "instance"
              }
            },
            {
              "id": "filterFieldsByName",
              "options": {
                "include": {
                  "pattern": "instance|Value .*"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {},
                "renameByName": {
                  "Value #A": "Blocked",
                  "Value #B": "Migrating",
                  "Value #C": "Pending",
                  "Value #D": "Start",
                  "Value #F": "Terminal",
                  "instance": "Client"
                }
              }
            }
          ],
          "type": "table"
        }
      ],
      "title": "Clients",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 45,
      "panels": [
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "fixedColor": "green",
                "mode": "fixed"
              },
              "decimals": 0,
              "mappings": [],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "text"
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 4,
            "w": 4,
            "x": 0,
            "y": 7
          },
          "id": 48,
          "options": {
            "colorMode": "background",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "textMode": "auto"
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_status_running{env=\"$env\"}",
              "instant": true,
              "interval": "",
              "legendFormat": "Running",
              "refId": "A"
            }
          ],
          "type": "stat"
        },
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": true
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Running"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "color-background-solid"
                  },
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "green",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.width",
                    "value": 80
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Complete"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "color-background-solid"
                  },
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "semi-dark-green",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.width",
                    "value": 80
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Failed"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "color-background-solid"
                  },
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "semi-dark-red",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.width",
                    "value": 80
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Starting"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "color-background-solid"
                  },
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "text",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.width",
                    "value": 80
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Quequed"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "color-background-solid"
                  },
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "transparent",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.width",
                    "value": 80
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Lost"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "color-background-solid"
                  },
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#000000",
                      "mode": "fixed"
                    }
                  },
                  {
                    "id": "custom.width",
                    "value": 80
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 12,
            "w": 20,
            "x": 4,
            "y": 7
          },
          "id": 47,
          "options": {
            "footer": {
              "fields": [
                "Value #F",
                "Value #A",
                "Value #B",
                "Value #D",
                "Value #E",
                "Value #C"
              ],
              "reducer": [
                "sum"
              ],
              "show": true
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_summary_complete{env=\"$env\"}",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "complete",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_summary_failed{env=\"$env\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "failed",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_summary_lost{env=\"$env\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "lost",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_summary_starting{env=\"$env\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "starting",
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_summary_queued{env=\"$env\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "queued",
              "refId": "E"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_summary_running{env=\"$env\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "running",
              "refId": "F"
            }
          ],
          "title": "Allocations",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "task_group"
              }
            },
            {
              "id": "filterFieldsByName",
              "options": {
                "include": {
                  "pattern": "exported_job 1|namespace 1|task_group|Value .*"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Value #A": 4,
                  "Value #B": 5,
                  "Value #C": 8,
                  "Value #D": 6,
                  "Value #E": 7,
                  "Value #F": 3,
                  "exported_job 1": 0,
                  "namespace 1": 2,
                  "task_group": 1
                },
                "renameByName": {
                  "Value #A": "Complete",
                  "Value #B": "Failed",
                  "Value #C": "Lost",
                  "Value #D": "Starting",
                  "Value #E": "Quequed",
                  "Value #F": "Running",
                  "exported_job 1": "job",
                  "namespace 1": "namespace",
                  "task_group": "group",
                  "task_group 1": "Group"
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "fixedColor": "text",
                "mode": "fixed"
              },
              "decimals": 0,
              "mappings": [],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "text"
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 4,
            "w": 4,
            "x": 0,
            "y": 11
          },
          "id": 49,
          "options": {
            "colorMode": "background",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "textMode": "auto"
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_status_pending{env=\"$env\"}",
              "instant": true,
              "interval": "",
              "legendFormat": "Pending",
              "refId": "A"
            }
          ],
          "type": "stat"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "fixedColor": "transparent",
                "mode": "fixed"
              },
              "decimals": 0,
              "mappings": [],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "text"
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 4,
            "w": 4,
            "x": 0,
            "y": 15
          },
          "id": 50,
          "options": {
            "colorMode": "background",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "textMode": "auto"
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_job_status_dead{env=\"$env\"}",
              "instant": true,
              "interval": "",
              "legendFormat": "Dead",
              "refId": "A"
            }
          ],
          "type": "stat"
        }
      ],
      "title": "Jobs",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 52,
      "panels": [
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto",
                "filterable": true
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "CPU"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "rothz"
                  },
                  {
                    "id": "custom.width",
                    "value": 90
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Memory"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "bytes"
                  },
                  {
                    "id": "custom.width",
                    "value": 90
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "M. max"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "bytes"
                  },
                  {
                    "id": "custom.width",
                    "value": 90
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 8
          },
          "id": 54,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"} * 1000000",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "cpu",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocs_memory_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "memory",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_client_allocs_memory_max_usage{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"}",
              "format": "table",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "memory max",
              "refId": "C"
            }
          ],
          "title": "Allocations",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "alloc_id"
              }
            },
            {
              "id": "filterFieldsByName",
              "options": {
                "include": {
                  "pattern": "alloc_id|exported_job 1|namespace 1|instance 1|task_group 1|task 1|Value .*"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {},
                "indexByName": {
                  "Value #A": 6,
                  "Value #B": 7,
                  "Value #C": 8,
                  "alloc_id": 5,
                  "exported_job 1": 0,
                  "instance 1": 4,
                  "namespace 1": 3,
                  "task 1": 2,
                  "task_group 1": 1
                },
                "renameByName": {
                  "Value #A": "CPU",
                  "Value #B": "Memory",
                  "Value #C": "M. max",
                  "alloc_id": "alloc_id",
                  "exported_job 1": "job",
                  "instance 1": "client",
                  "namespace 1": "namespace",
                  "task 1": "task",
                  "task_group 1": "group"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "field": "job"
                  }
                ]
              }
            }
          ],
          "type": "table"
        },
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "decimals": 0,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 10,
            "w": 12,
            "x": 0,
            "y": 16
          },
          "id": 55,
          "options": {
            "legend": {
              "calcs": [
                "mean"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Mean",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_client_allocs_cpu_total_percent{env=\"$env\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"}",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ exported_job }} - {{ task_group }} - {{ task }} - {{ instance }} - {{ alloc_id }} Mhz",
              "refId": "A"
            }
          ],
          "title": "CPU %",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "decimals": 0,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "rothz"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 10,
            "w": 12,
            "x": 12,
            "y": 16
          },
          "id": 65,
          "options": {
            "legend": {
              "calcs": [
                "mean"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Mean",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_client_allocs_cpu_total_ticks{env=\"$env\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"} * 1000000",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ exported_job }} - {{ task_group }} - {{ task }} - {{ instance }} - {{ alloc_id }} Mhz",
              "refId": "A"
            }
          ],
          "title": "CPU Ticks",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "decimals": 0,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 10,
            "w": 12,
            "x": 0,
            "y": 26
          },
          "id": 56,
          "options": {
            "legend": {
              "calcs": [
                "mean"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Mean",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "(nomad_client_allocs_memory_usage{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"}\n- \nnomad_client_allocs_memory_cache{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"})\n/\nnomad_client_allocs_memory_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"} * 100",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ exported_job }} - {{ task_group }} - {{ task }} - {{ instance }} - {{ alloc_id }}",
              "refId": "A"
            }
          ],
          "title": "Memory %",
          "type": "timeseries"
        },
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "decimals": 0,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 10,
            "w": 12,
            "x": 12,
            "y": 26
          },
          "id": 66,
          "options": {
            "legend": {
              "calcs": [
                "mean"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Mean",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "(nomad_client_allocs_memory_usage{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"}\n- \nnomad_client_allocs_memory_cache{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\",alloc_id=~\"$alloc_id\"})",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ exported_job }} - {{ task_group }} - {{ task }} - {{ instance }} - {{ alloc_id }}",
              "refId": "A"
            }
          ],
          "title": "Memory Bytes",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "links": [
                {
                  "title": "Set time range",
                  "url": "https://mon.cinarra.com/d/4NYOvS-7z/nomad?orgId=1&var-env=$env&var-alloc_id=${__data.fields.alloc_id}﻿﻿&from=﻿﻿${__data.fields.from}&to=${__data.fields.to}"
                }
              ],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "from"
                },
                "properties": [
                  {
                    "id": "custom.hidden",
                    "value": true
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "to"
                },
                "properties": [
                  {
                    "id": "custom.hidden",
                    "value": true
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 36
          },
          "id": 86,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "frameIndex": 0,
            "showHeader": true,
            "sortBy": []
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_client_allocs_oom_killed{env=\"$env\"}",
              "format": "table",
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "oom",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "timestamp(nomad_client_allocs_oom_killed{env=\"$env\"}) * 1000 - 3600000",
              "format": "table",
              "hide": false,
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "from",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "timestamp(nomad_client_allocs_oom_killed{env=\"$env\"}) * 1000 + 3600000",
              "format": "table",
              "hide": false,
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "to",
              "refId": "C"
            }
          ],
          "title": "OOM kills",
          "transformations": [
            {
              "id": "seriesToColumns",
              "options": {
                "byField": "alloc_id"
              }
            },
            {
              "id": "filterFieldsByName",
              "options": {
                "include": {
                  "pattern": "alloc_id|exporter_job 1|Time 1|instance 1|namespace 1|task 1|task_group 1|Value .*"
                }
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Value #B": false,
                  "Value #C": false
                },
                "indexByName": {
                  "Time 1": 0,
                  "Value #A": 6,
                  "Value #B": 7,
                  "alloc_id": 1,
                  "instance 1": 2,
                  "namespace 1": 3,
                  "task 1": 4,
                  "task_group 1": 5
                },
                "renameByName": {
                  "Time 1": "Time",
                  "Value #A": "Kills",
                  "Value #B": "from",
                  "Value #C": "to",
                  "alloc_id": "",
                  "instance 1": "client",
                  "namespace 1": "namespace",
                  "task 1": "task",
                  "task_group 1": "group"
                }
              }
            },
            {
              "id": "sortBy",
              "options": {
                "fields": {},
                "sort": [
                  {
                    "field": "Time"
                  }
                ]
              }
            }
          ],
          "type": "table"
        }
      ],
      "title": "Allocations",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "id": 60,
      "panels": [
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds",
                "seriesBy": "last"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "axisSoftMax": 200,
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "area"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "red"
                  },
                  {
                    "color": "light-orange",
                    "value": 50
                  },
                  {
                    "color": "green",
                    "value": 80
                  },
                  {
                    "color": "super-light-orange",
                    "value": 100
                  },
                  {
                    "color": "light-orange",
                    "value": 120
                  },
                  {
                    "color": "red",
                    "value": 150
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 10,
            "w": 24,
            "x": 0,
            "y": 45
          },
          "id": 63,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Mean",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "avg(nomad_client_allocs_cpu_total_percent{env=\"$env\"}) by (exported_job,task_group,task) <= 80",
              "format": "time_series",
              "hide": false,
              "instant": false,
              "interval": "",
              "legendFormat": "{{ exported_job }} - {{ task_group }} - {{ task }}",
              "refId": "A"
            }
          ],
          "title": "CPU",
          "transformations": [],
          "type": "timeseries"
        },
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds",
                "seriesBy": "last"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "axisSoftMax": 200,
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "area"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "red"
                  },
                  {
                    "color": "light-orange",
                    "value": 50
                  },
                  {
                    "color": "green",
                    "value": 80
                  },
                  {
                    "color": "super-light-orange",
                    "value": 100
                  },
                  {
                    "color": "light-orange",
                    "value": 120
                  },
                  {
                    "color": "red",
                    "value": 150
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 10,
            "w": 24,
            "x": 0,
            "y": 55
          },
          "id": 67,
          "options": {
            "legend": {
              "calcs": [
                "min",
                "max",
                "mean"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Mean",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "avg(\n(nomad_client_allocs_memory_usage{env=\"$env\"}\n-\nnomad_client_allocs_memory_cache{env=\"$env\"})\n/\nnomad_client_allocs_memory_allocated{env=\"$env\"} * 100\n) by (exported_job,task_group,task)",
              "format": "time_series",
              "hide": false,
              "instant": false,
              "interval": "",
              "legendFormat": "{{ exported_job }} - {{ task_group }} - {{ task }}",
              "refId": "A"
            }
          ],
          "title": "Memory",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Utilization",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 26,
      "panels": [
        {
          "description": "Time to write log, mark in flight, and start replication to followers",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "line"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "text"
                  }
                ]
              },
              "unit": "ms"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Logs"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "short"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Time"
                },
                "properties": [
                  {
                    "id": "thresholds",
                    "value": {
                      "mode": "absolute",
                      "steps": [
                        {
                          "color": "green"
                        },
                        {
                          "color": "light-orange",
                          "value": 200
                        },
                        {
                          "color": "red",
                          "value": 500
                        }
                      ]
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 66
          },
          "id": 24,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_raft_leader_dispatchLog{env=\"$env\",quantile=\"0.9\"}",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Time",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "increase(nomad_raft_leader_dispatchNumLogs{env=\"$env\"}[$__rate_interval])",
              "hide": false,
              "interval": "",
              "legendFormat": "Logs",
              "refId": "A"
            }
          ],
          "title": "Raft Dispatch Log",
          "type": "timeseries"
        },
        {
          "description": "Time difference between when a message was first passed into Raft and when the resulting commit took place on the leader",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "line"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "yellow",
                    "value": 100
                  },
                  {
                    "color": "red",
                    "value": 500
                  }
                ]
              },
              "unit": "ms"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 66
          },
          "id": 32,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_raft_commitTime{env=\"$env\",quantile=\"0.9\"}",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ instance }}",
              "refId": "A"
            }
          ],
          "title": "Raft Commit Time",
          "type": "timeseries"
        },
        {
          "description": "Number of Raft transactions - general indicator of the write load on the servers.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "stepBefore",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "decimals": 0,
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 74
          },
          "id": 30,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "increase(nomad_raft_apply{env=\"$env\"}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{ instance }}",
              "refId": "A"
            }
          ],
          "title": "Raft Apply Count",
          "type": "timeseries"
        },
        {
          "description": "The time it takes for a Raft transaction to be replicated to a quorum of followers",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "line"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "yellow",
                    "value": 100
                  },
                  {
                    "color": "red",
                    "value": 500
                  }
                ]
              },
              "unit": "ms"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 74
          },
          "id": 34,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_raft_rpc_appendEntries{env=\"$env\",quantile=\"0.9\"}",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ instance }}",
              "refId": "A"
            }
          ],
          "title": "Raft Append Entries",
          "type": "timeseries"
        },
        {
          "description": "Current index applied to FSM",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "continuous-BlPu"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "text"
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 82
          },
          "id": 43,
          "options": {
            "displayMode": "lcd",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_raft_appliedIndex{env=\"$env\"}",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ instance }}",
              "refId": "A"
            }
          ],
          "title": "Raft Applied Index",
          "type": "bargauge"
        },
        {
          "description": "Time it takes for a server to apply Raft entries to the internal state machine.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 30,
                "gradientMode": "opacity",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "smooth",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": true,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "line"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "yellow",
                    "value": 100
                  },
                  {
                    "color": "red",
                    "value": 500
                  }
                ]
              },
              "unit": "ms"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 82
          },
          "id": 37,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "max",
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "pluginVersion": "8.1.0",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_raft_fsm_apply{env=\"$env\",quantile=\"0.9\"}",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ instance }}",
              "refId": "A"
            }
          ],
          "title": "Raft FSM apply",
          "type": "timeseries"
        }
      ],
      "title": "Raft",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "id": 71,
      "panels": [
        {
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "custom": {
                "align": "auto",
                "displayMode": "auto"
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "red"
                  },
                  {
                    "color": "#EAB839",
                    "value": 172800010
                  },
                  {
                    "color": "green",
                    "value": 345600000
                  }
                ]
              },
              "unit": "ms"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "TTL"
                },
                "properties": [
                  {
                    "id": "custom.displayMode",
                    "value": "color-background"
                  },
                  {
                    "id": "color",
                    "value": {
                      "mode": "thresholds"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 0,
            "y": 67
          },
          "id": 69,
          "options": {
            "footer": {
              "fields": "",
              "reducer": [
                "sum"
              ],
              "show": false
            },
            "showHeader": true
          },
          "pluginVersion": "8.3.1",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": false,
              "expr": "nomad_nomad_vault_token_ttl{env=\"$env\"}",
              "format": "table",
              "instant": true,
              "interval": "",
              "legendFormat": "ttl",
              "refId": "A"
            }
          ],
          "title": "Server token TTL",
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true,
                  "__name__": true,
                  "env": true,
                  "job": true
                },
                "indexByName": {},
                "renameByName": {
                  "Value #A": "TTL",
                  "instance": ""
                }
              }
            }
          ],
          "type": "table"
        },
        {
          "description": "Time elapsed to manipulate Vault tokens",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  }
                ]
              },
              "unit": "ns"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 67
          },
          "id": 73,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_nomad_vault_create_token{quantile=\"0.9\"}",
              "interval": "",
              "legendFormat": "create",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_nomad_vault_lookup_token{quantile=\"0.9\"}",
              "hide": false,
              "interval": "",
              "legendFormat": "lookup",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_VICTORIAMETRICS}"
              },
              "exemplar": true,
              "expr": "nomad_nomad_vault_revoke_token{quantile=\"0.9\"}",
              "hide": false,
              "interval": "",
              "legendFormat": "revoke",
              "refId": "C"
            }
          ],
          "title": "Tokens",
          "type": "timeseries"
        }
      ],
      "title": "Vault",
      "type": "row"
    }
  ],
  "refresh": false,
  "schemaVersion": 34,
  "style": "dark",
  "tags": [
    "HashiCorp"
  ],
  "templating": {
    "list": [
      {
        "current": {},
        "definition": "label_values(up{job=\"nomad\"},env)",
        "hide": 0,
        "includeAll": false,
        "multi": false,
        "name": "env",
        "options": [],
        "query": {
          "query": "label_values(up{job=\"nomad\"},env)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query",
        "datasource": "${DS_VICTORIAMETRICS}"
      },
      {
        "current": {},
        "definition": "label_values(nomad_client_uptime{env=\"$env\"}, instance)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "client",
        "options": [],
        "query": {
          "query": "label_values(nomad_client_uptime{env=\"$env\"}, instance)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query",
        "datasource": "${DS_VICTORIAMETRICS}"
      },
      {
        "current": {},
        "definition": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\"},exported_job)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "job",
        "options": [],
        "query": {
          "query": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\"},exported_job)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query",
        "datasource": "${DS_VICTORIAMETRICS}"
      },
      {
        "current": {},
        "definition": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\"},task_group)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "group",
        "options": [],
        "query": {
          "query": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\"},task_group)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query",
        "datasource": "${DS_VICTORIAMETRICS}"
      },
      {
        "current": {},
        "definition": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\"},task)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "task",
        "options": [],
        "query": {
          "query": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\"},task)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query",
        "datasource": "${DS_VICTORIAMETRICS}"
      },
      {
        "current": {},
        "definition": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\"},alloc_id)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "alloc_id",
        "options": [],
        "query": {
          "query": "label_values(nomad_client_allocs_cpu_allocated{env=\"$env\",instance=~\"$client\",exported_job=~\"$job\",task_group=~\"$group\",task=~\"$task\"},alloc_id)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query",
        "datasource": "${DS_VICTORIAMETRICS}"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "HashiCorp Nomad",
  "uid": "4NYOvS-7z",
  "version": 4,
  "weekStart": "",
  "gnetId": 15764,
  "description": "A dashboard for HashiCorp Nomad https://www.nomadproject.io/"
}