Grafana dashboard: Alerts
Description: Alerts from prometheus.

Dashboards for ALERTS metrics from prometheus.

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "4.6.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "singlestat",
      "name": "Singlestat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Alerts from prometheus.",
  "editable": false,
  "gnetId": 5450,
  "graphTooltip": 0,
  "hideControls": false,
  "id": null,
  "links": [
    {
      "asDropdown": true,
      "icon": "external link",
      "includeVars": true,
      "keepTime": true,
      "tags": [
        "OS"
      ],
      "targetBlank": true,
      "title": "OS",
      "type": "dashboards"
    },
    {
      "asDropdown": true,
      "icon": "external link",
      "includeVars": true,
      "keepTime": true,
      "tags": [
        "MySQL"
      ],
      "targetBlank": true,
      "title": "MySQL",
      "type": "dashboards"
    },
    {
      "asDropdown": true,
      "icon": "external link",
      "includeVars": true,
      "keepTime": true,
      "tags": [
        "MongoDB"
      ],
      "targetBlank": true,
      "title": "MongoDB",
      "type": "dashboards"
    },
    {
      "asDropdown": true,
      "icon": "external link",
      "includeVars": true,
      "keepTime": true,
      "tags": [
        "App"
      ],
      "targetBlank": true,
      "title": "App",
      "type": "dashboards"
    }
  ],
  "refresh": "1m",
  "rows": [
    {
      "collapse": false,
      "height": 71,
      "panels": [
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": true,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_PROMETHEUS}",
          "format": "short",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "id": 23,
          "interval": null,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 1,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": false,
            "lineColor": "rgb(31, 120, 193)",
            "show": false
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "count(ALERTS{environment=~\"$environment\", service=~\"$service\", job=~\"$job\", alertname=~\"$alertname\", alertstate=~\"$alertstate\",severity=\"alert\"} > 0) OR on() vector(0)",
              "format": "time_series",
              "instant": true,
              "intervalFactor": 2,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": "1,10",
          "title": "Alerts",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": true,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_PROMETHEUS}",
          "format": "short",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "id": 24,
          "interval": null,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 1,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": false,
            "lineColor": "rgb(31, 120, 193)",
            "show": false
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "count(ALERTS{environment=~\"$environment\", service=~\"$service\", job=~\"$job\", alertname=~\"$alertname\", alertstate=~\"$alertstate\",severity=\"warning\"} > 0) OR on() vector(0)",
              "format": "time_series",
              "instant": true,
              "intervalFactor": 2,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": "1,10",
          "title": "Warnings",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": true,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_PROMETHEUS}",
          "decimals": 0,
          "format": "short",
          "gauge": {
            "maxValue": 500,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "height": "100",
          "id": 25,
          "interval": null,
          "links": [
            {
              "dashUri": "db/cross-server-graphs",
              "dashboard": "Cross Server Graphs",
              "includeVars": true,
              "keepTime": true,
              "targetBlank": true,
              "title": "Cross Server Graphs",
              "type": "dashboard"
            }
          ],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 2,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": true,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "max (node_load1{environment=~\"$environment\", service=~\"$service\"})",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 2,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": "10,17",
          "title": "Max  Load Average 1m",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": true,
          "colors": [
            "#d44a3a",
            "rgba(237, 129, 40, 0.89)",
            "#299c46"
          ],
          "datasource": "${DS_PROMETHEUS}",
          "decimals": null,
          "format": "bytes",
          "gauge": {
            "maxValue": 500,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "height": "100",
          "id": 28,
          "interval": null,
          "links": [
            {
              "dashUri": "db/cross-server-graphs",
              "dashboard": "Cross Server Graphs",
              "includeVars": true,
              "keepTime": true,
              "targetBlank": true,
              "title": "Cross Server Graphs",
              "type": "dashboard"
            }
          ],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 2,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": true,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "min(node_memory_MemAvailable{environment=~\"$environment\", service=~\"$service\"})",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 2,
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": "100000000,700000000",
          "title": "Min Memory Available",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": true,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_PROMETHEUS}",
          "decimals": null,
          "format": "short",
          "gauge": {
            "maxValue": 500,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "height": "100",
          "id": 29,
          "interval": null,
          "links": [
            {
              "dashUri": "db/network-stats",
              "dashboard": "Network stats",
              "includeVars": true,
              "keepTime": true,
              "targetBlank": true,
              "title": "Network stats",
              "type": "dashboard"
            }
          ],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 2,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": true,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "max(node_netstat_Tcp_CurrEstab{environment=~\"$environment\", service=~\"$service\"})",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 2,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": "950,1100",
          "title": "Max Tcp Established",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": true,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_PROMETHEUS}",
          "decimals": null,
          "format": "µs",
          "gauge": {
            "maxValue": 500,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "height": "100",
          "id": 27,
          "interval": null,
          "links": [
            {
              "dashUri": "db/prometheus",
              "dashboard": "Prometheus",
              "includeVars": false,
              "targetBlank": true,
              "title": "Prometheus",
              "type": "dashboard"
            }
          ],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 2,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": true,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "sum(irate(http_request_duration_microseconds_sum{job=\"prometheus\"}[5m])) /  sum(irate(http_request_duration_microseconds_count{job=\"prometheus\"}[5m]))",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 2,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": "900000,5000000",
          "title": "Prometheus HTTP Request Duration",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": false,
          "colors": [
            "#d44a3a",
            "rgba(237, 129, 40, 0.89)",
            "#299c46"
          ],
          "datasource": "${DS_PROMETHEUS}",
          "format": "short",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "height": "100",
          "id": 26,
          "interval": null,
          "links": [
            {
              "dashUri": "db/prometheus",
              "dashboard": "Prometheus",
              "targetBlank": true,
              "title": "Prometheus",
              "type": "dashboard"
            }
          ],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "span": 2,
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": true,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "count(up)",
              "format": "time_series",
              "instant": false,
              "intervalFactor": 2,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": "",
          "title": "Prometheus Targets",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": false,
      "title": "KPI",
      "titleSize": "h6"
    },
    {
      "collapse": false,
      "height": 307,
      "panels": [
        {
          "columns": [],
          "datasource": "${DS_PROMETHEUS}",
          "description": "Таблица с последними алертами",
          "editable": true,
          "error": false,
          "fontSize": "100%",
          "id": 21,
          "links": [],
          "pageSize": null,
          "scroll": true,
          "showHeader": true,
          "sort": {
            "col": 1,
            "desc": true
          },
          "span": 12,
          "styles": [
            {
              "alias": "Date",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "link": true,
              "linkTargetBlank": false,
              "linkTooltip": "",
              "linkUrl": "                      ",
              "pattern": "Time",
              "sanitize": true,
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            },
            {
              "alias": "count",
              "colorMode": null,
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 0,
              "pattern": "Value",
              "thresholds": [
                ""
              ],
              "type": "number",
              "unit": "short"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "link": true,
              "linkTargetBlank": true,
              "linkTooltip": "System Overview for $__cell",
              "linkUrl": "http://prom.prod.lan/graph/dashboard/db/system-overview?var-host=$__cell",
              "pattern": "instance",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "link": true,
              "linkTargetBlank": true,
              "linkTooltip": "Show alert",
              "linkUrl": "http://prom.prod.lan/prometheus/graph?g0.range_input=1h&g0.expr=ALERTS%7Balertname%3D%22$__cell%22%2Cinstance%3D%22$__cell_3%22%7D&g0.tab=1",
              "pattern": "alertname",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "count(ALERTS{environment=~\"$environment\", service=~\"$service\", job=~\"$job\", alertname=~\"$alertname\", alertstate=~\"$alertstate\",severity=\"alert\"} == 1) by (instance,alertname,service,environment,job)",
              "format": "table",
              "instant": true,
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "service: {{service}} environment: {{environment}} instance: {{instance}}  alertname: {{alertname}}   service: {{service}}",
              "refId": "A",
              "step": 4
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Current alerts table",
          "transform": "table",
          "type": "table"
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": false,
      "title": "Current alerts table",
      "titleSize": "h6"
    },
    {
      "collapse": false,
      "height": 354,
      "panels": [
        {
          "alert": {
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.9
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "B",
                    "10s",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "max"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "frequency": "60s",
            "handler": 1,
            "message": "Current alerts list:",
            "name": "alerts",
            "noDataState": "ok",
            "notifications": []
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "decimals": null,
          "description": "График алертов: имя хоста (instance) имя алерта (alertname)",
          "editable": true,
          "error": false,
          "fill": 2,
          "grid": {},
          "height": "400px",
          "id": 19,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "span": 12,
          "stack": true,
          "steppedLine": true,
          "targets": [
            {
              "calculatedInterval": "10s",
              "datasourceErrors": {},
              "errors": {},
              "expr": "count(ALERTS{environment=~\"$environment\", service=~\"$service\", job=~\"$job\", alertname=~\"$alertname\", alertstate=~\"$alertstate\",severity=\"alert\"} == 1)  by (instance,alertname)",
              "format": "time_series",
              "interval": "$interval",
              "intervalFactor": 1,
              "legendFormat": "{{instance}}  {{alertname}}",
              "metric": "probe_http_status_code",
              "refId": "A",
              "step": 60,
              "target": ""
            },
            {
              "calculatedInterval": "10s",
              "datasourceErrors": {},
              "errors": {},
              "expr": "count(ALERTS{alertstate=\"firing\",severity=\"alert\", environment=~'pro.*'} == 1)  by (instance,alertname)",
              "format": "time_series",
              "hide": true,
              "interval": "15s",
              "intervalFactor": 1,
              "legendFormat": "{{instance}}: {{alertname}}",
              "metric": "probe_http_status_code",
              "refId": "B",
              "step": 60,
              "target": ""
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0.9
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Alerts Graph: instance alertname",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": false,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": 0,
              "show": true
            },
            {
              "format": "none",
              "logBase": 1,
              "max": null,
              "min": 0,
              "show": false
            }
          ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": false,
      "title": "Alerts Graph",
      "titleSize": "h6"
    },
    {
      "collapse": false,
      "height": 250,
      "panels": [
        {
          "columns": [],
          "datasource": "${DS_PROMETHEUS}",
          "description": "Таблица с последними предупреждениями",
          "editable": true,
          "error": false,
          "fontSize": "100%",
          "id": 22,
          "links": [],
          "pageSize": null,
          "scroll": true,
          "showHeader": true,
          "sort": {
            "col": 1,
            "desc": true
          },
          "span": 12,
          "styles": [
            {
              "alias": "Date",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "link": false,
              "pattern": "Time",
              "sanitize": true,
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            },
            {
              "alias": "count",
              "colorMode": null,
              "colors": [
                "rgba(50, 172, 45, 0.97)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(245, 54, 54, 0.9)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 0,
              "pattern": "Value",
              "thresholds": [
                ""
              ],
              "type": "number",
              "unit": "short"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "link": true,
              "linkTooltip": "System Overview for $__cell",
              "linkUrl": "http://prom.prod.lan/graph/dashboard/db/system-overview?var-host=$__cell",
              "pattern": "instance",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "link": true,
              "linkTargetBlank": true,
              "linkTooltip": "Show warining",
              "linkUrl": "http://prom.prod.lan/prometheus/graph?g0.range_input=1h&g0.expr=ALERTS%7Balertname%3D%22$__cell%22%2Cinstance%3D%22$__cell_3%22%7D&g0.tab=1",
              "pattern": "alertname",
              "thresholds": [],
              "type": "string",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "count(ALERTS{environment=~\"$environment\", service=~\"$service\", job=~\"$job\", alertname=~\"$alertname\", alertstate=~\"$alertstate\",severity=\"warning\"} == 1) by (instance,alertname,service,environment,job)",
              "format": "table",
              "instant": true,
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "service: {{service}} environment: {{environment}} instance: {{instance}}  alertname: {{alertname}}   service: {{service}}",
              "refId": "A",
              "step": 4
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Current warnings table",
          "transform": "table",
          "type": "table"
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": false,
      "title": "Current warnings table",
      "titleSize": "h6"
    },
    {
      "collapse": false,
      "height": 405,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "decimals": null,
          "editable": true,
          "error": false,
          "fill": 2,
          "grid": {},
          "height": "500px",
          "id": 20,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "span": 12,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "calculatedInterval": "10s",
              "datasourceErrors": {},
              "errors": {},
              "expr": "count(ALERTS{environment=~\"$environment\", service=~\"$service\", job=~\"$job\", alertname=~\"$alertname\", alertstate=~\"$alertstate\",severity=\"warning\"} == 1)  by (instance,alertname)",
              "format": "time_series",
              "interval": "$interval",
              "intervalFactor": 1,
              "legendFormat": "{{instance}} {{alertname}}",
              "metric": "probe_http_status_code",
              "refId": "A",
              "step": 60,
              "target": ""
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": false,
              "line": true,
              "op": "gt",
              "value": 1
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Warnings Graph: instance alertname",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": false,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 0,
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": 0,
              "show": true
            },
            {
              "format": "none",
              "logBase": 1,
              "max": null,
              "min": 0,
              "show": false
            }
          ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": false,
      "title": "Warnings Graph",
      "titleSize": "h6"
    }
  ],
  "schemaVersion": 14,
  "style": "dark",
  "tags": [
    "App",
    "Alerts"
  ],
  "templating": {
    "list": [
      {
        "allFormat": "glob",
        "auto": true,
        "auto_count": 50,
        "auto_min": "5s",
        "current": {
          "text": "15s",
          "value": "15s"
        },
        "datasource": "Prometheus",
        "hide": 0,
        "includeAll": false,
        "label": "Interval",
        "multi": false,
        "multiFormat": "glob",
        "name": "interval",
        "options": [
          {
            "selected": false,
            "text": "auto",
            "value": "$__auto_interval"
          },
          {
            "selected": true,
            "text": "15s",
            "value": "15s"
          },
          {
            "selected": false,
            "text": "1m",
            "value": "1m"
          },
          {
            "selected": false,
            "text": "5m",
            "value": "5m"
          },
          {
            "selected": false,
            "text": "1h",
            "value": "1h"
          },
          {
            "selected": false,
            "text": "6h",
            "value": "6h"
          },
          {
            "selected": false,
            "text": "1d",
            "value": "1d"
          }
        ],
        "query": "15s,1m,5m,1h,6h,1d",
        "refresh": 2,
        "type": "interval"
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_PROMETHEUS}",
        "hide": 0,
        "includeAll": true,
        "label": "service",
        "multi": true,
        "name": "service",
        "options": [],
        "query": "label_values(ALERTS, service)",
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "tagValuesQuery": null,
        "tags": [],
        "tagsQuery": null,
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_PROMETHEUS}",
        "hide": 0,
        "includeAll": true,
        "label": "job",
        "multi": true,
        "name": "job",
        "options": [],
        "query": "label_values(ALERTS, job)",
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "tagValuesQuery": null,
        "tags": [],
        "tagsQuery": null,
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_PROMETHEUS}",
        "hide": 0,
        "includeAll": true,
        "label": "alertname",
        "multi": true,
        "name": "alertname",
        "options": [],
        "query": "label_values(ALERTS, alertname)",
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "tagValuesQuery": null,
        "tags": [],
        "tagsQuery": null,
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "selected": true,
          "text": "firing",
          "value": "firing"
        },
        "hide": 0,
        "includeAll": true,
        "label": "alertstate",
        "multi": true,
        "name": "alertstate",
        "options": [
          {
            "selected": false,
            "text": "All",
            "value": "$__all"
          },
          {
            "selected": true,
            "text": "firing",
            "value": "firing"
          },
          {
            "selected": false,
            "text": "pending",
            "value": "pending"
          }
        ],
        "query": "firing, pending",
        "type": "custom"
      },
      {
        "allValue": null,
        "current": {
          "tags": [],
          "text": "test",
          "value": [
            "test"
          ]
        },
        "hide": 0,
        "includeAll": true,
        "label": "environment",
        "multi": true,
        "name": "environment",
        "options": [
          {
            "selected": false,
            "text": "All",
            "value": "$__all"
          },
          {
            "selected": false,
            "text": "prod",
            "value": "prod"
          },
          {
            "selected": false,
            "text": "prod1",
            "value": "prod1"
          },
          {
            "selected": false,
            "text": "prod2",
            "value": "prod2"
          },
          {
            "selected": true,
            "text": "test",
            "value": "test"
          },
          {
            "selected": false,
            "text": "test2",
            "value": "test2"
          },
          {
            "selected": false,
            "text": "dev",
            "value": "dev"
          }
        ],
        "query": "prod, prod1, prod2, test ,test2, dev",
        "type": "custom"
      }
    ]
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
    "collapse": false,
    "enable": true,
    "notice": false,
    "now": true,
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "status": "Stable",
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ],
    "type": "timepicker"
  },
  "timezone": "browser",
  "title": "Alerts",
  "version": 365
}