Grafana dashboard: Kubernetes Monitoring
Description: An Extensive Kubernetes Monitoring Dashboard


{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    },
    {
      "name": "DS_PROMETHEUS-NOCORTEX",
      "label": "Prometheus-nocortex",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": [],
  "__requires": [
    {
      "type": "panel",
      "id": "bargauge",
      "name": "Bar gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "8.4.4"
    },
    {
      "type": "panel",
      "id": "grafana-piechart-panel",
      "name": "Pie Chart (old)",
      "version": "1.6.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph (old)",
      "version": ""
    },
    {
      "type": "panel",
      "id": "piechart",
      "name": "Pie chart",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table-old",
      "name": "Table (old)",
      "version": ""
    },
    {
      "type": "panel",
      "id": "text",
      "name": "Text",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    },
    {
      "type": "panel",
      "id": "yesoreyeram-boomtable-panel",
      "name": "Boom Table",
      "version": "1.4.1"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      },
      {
        "datasource": "-- Grafana --",
        "enable": false,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "limit": 100,
        "name": "deploy",
        "showIn": 0,
        "tags": "",
        "target": {
          "tags": [],
          "type": "tags"
        },
        "type": "tags"
      },
      {
        "datasource": "-- Grafana --",
        "enable": false,
        "hide": false,
        "iconColor": "#8AB8FF",
        "limit": 100,
        "name": "Deployments",
        "showIn": 0,
        "tags": "",
        "target": {
          "tags": [
            "deploy"
          ],
          "type": "tags"
        },
        "type": "tags"
      },
      {
        "datasource": "-- Grafana --",
        "enable": false,
        "hide": false,
        "iconColor": "#CA95E5",
        "limit": 100,
        "name": "Rollouts",
        "showIn": 0,
        "tags": "",
        "target": {
          "tags": [
            "rollout"
          ],
          "type": "tags"
        },
        "type": "tags"
      }
    ]
  },
  "description": "An Extensive Kubernetes Monitoring Dashboard",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1661235626686,
  "links": [
    {
      "$$hashKey": "object:242",
      "asDropdown": true,
      "icon": "external link",
      "tags": [
        "Kubernetes"
      ],
      "targetBlank": true,
      "title": "Related Kubernetes Dashboards",
      "type": "dashboards"
    },
    {
      "$$hashKey": "object:243",
      "asDropdown": true,
      "icon": "external link",
      "tags": [
        "Database"
      ],
      "targetBlank": true,
      "title": "Databases Related Dashboards",
      "type": "dashboards"
    },
    {
      "$$hashKey": "object:244",
      "asDropdown": true,
      "icon": "external link",
      "tags": [
        "S3"
      ],
      "title": "AWS Related Dashboards",
      "type": "dashboards"
    },
    {
      "$$hashKey": "object:245",
      "asDropdown": true,
      "icon": "external link",
      "tags": [
        "SRE"
      ],
      "targetBlank": true,
      "title": "SRE Dashboards",
      "type": "dashboards"
    }
  ],
  "liveNow": false,
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Memory used by applications",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 2,
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 3,
        "x": 9,
        "y": 0
      },
      "id": 82,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "none",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.4.4",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "sum(container_memory_usage_bytes{id=\"/\", cluster=\"$cluster\"} * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"}) by (label_nodegroup)",
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Used",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "The sum of usable Memory amongst all nodes",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 2,
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 3,
        "x": 12,
        "y": 0
      },
      "id": 83,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "none",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.4.4",
      "targets": [
        {
          "expr": "sum(node:memory:labeled{cluster=\"$cluster\", label_nodegroup=\"$group\"}) by (label_nodegroup)",
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Total",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "The total amount of Memory requests amongst all applications",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "decimals": 2,
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 3,
        "x": 15,
        "y": 0
      },
      "id": 84,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "none",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.4.4",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\", cluster=\"$cluster\"} * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"}) by (label_nodegroup)",
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Request",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "How many pods are left to allocate",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "match": "null",
                "result": {
                  "text": "N/A"
                }
              },
              "type": "special"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "locale"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 3,
        "x": 18,
        "y": 0
      },
      "hideTimeOverride": false,
      "id": 108,
      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "none",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.4.4",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "sum(kube_node_status_allocatable{cluster=\"$cluster\", resource=\"pods\"} * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"}) - sum(kubelet_running_pods{cluster=\"$cluster\"} * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"})",
          "hide": false,
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Allocatable Pods Left",
      "transparent": true,
      "type": "stat"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 2
      },
      "id": 234,
      "panels": [],
      "title": "Extra Cluster Info",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.hidden",
                "value": true
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value"
            },
            "properties": [
              {
                "id": "custom.hidden",
                "value": true
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "port_protocol"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 105
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "port_number"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 100
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "port_name"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 93
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "endpoint"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 293
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 11,
        "w": 8,
        "x": 0,
        "y": 3
      },
      "id": 232,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "endpoint"
          }
        ]
      },
      "pluginVersion": "8.4.4",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "group by (endpoint,port_name,port_number,port_protocol) (kube_endpoint_ports{cluster=\"$cluster\"})",
          "format": "table",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Pods endpoints ports",
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto",
            "displayMode": "auto"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "__name__"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 188
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 11,
        "w": 16,
        "x": 8,
        "y": 3
      },
      "id": 250,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "removed_release"
          }
        ]
      },
      "pluginVersion": "8.4.4",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "apiserver_requested_deprecated_apis{cluster=\"$cluster\"}",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Deprecated Kubernetes Resources",
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {
            "keepLabels": [
              "group",
              "job",
              "removed_release",
              "resource",
              "version",
              "name"
            ],
            "mode": "columns"
          }
        },
        {
          "id": "merge",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Value": true,
              "job": true
            },
            "indexByName": {
              "Time": 6,
              "Value": 7,
              "group": 1,
              "job": 5,
              "namespace": 0,
              "removed_release": 4,
              "resource": 3,
              "version": 2
            },
            "renameByName": {}
          }
        },
        {
          "id": "groupBy",
          "options": {
            "fields": {
              "group": {
                "aggregations": [
                  "lastNotNull"
                ],
                "operation": "groupby"
              },
              "job": {
                "aggregations": [],
                "operation": "groupby"
              },
              "namespace": {
                "aggregations": [
                  "lastNotNull"
                ],
                "operation": "groupby"
              },
              "removed_release": {
                "aggregations": [],
                "operation": "groupby"
              },
              "resource": {
                "aggregations": [
                  "lastNotNull"
                ],
                "operation": "groupby"
              },
              "version": {
                "aggregations": [],
                "operation": "groupby"
              }
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 14
      },
      "id": 39,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "rgba(50, 172, 45, 0.97)"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 50
                  },
                  {
                    "color": "rgba(245, 54, 54, 0.9)",
                    "value": 80
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 9,
            "x": 0,
            "y": 12
          },
          "id": 47,
          "links": [],
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", cluster=\"$cluster\"}[1m])) / sum (kube_node_status_allocatable{cluster=\"$cluster\",resource=\"cpu\"}) * 100",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Cluster CPU Usage",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "rgba(50, 172, 45, 0.97)"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 50
                  },
                  {
                    "color": "rgba(245, 54, 54, 0.9)",
                    "value": 80
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 9,
            "x": 9,
            "y": 12
          },
          "id": 45,
          "links": [],
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum (container_memory_usage_bytes{id=\"/\", cluster=\"$cluster\"}) / sum (kube_node_status_allocatable{cluster=\"$cluster\",resource=\"memory\"}) * 100",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Cluster Memory Usage",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "rgba(50, 172, 45, 0.97)"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 65
                  },
                  {
                    "color": "rgba(245, 54, 54, 0.9)",
                    "value": 90
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 6,
            "x": 18,
            "y": 12
          },
          "id": 72,
          "links": [],
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_status_allocatable{resource=\"ephemeral_storage\", cluster=\"$cluster\"}) / sum(kube_node_status_capacity{resource=\"ephemeral_storage\", cluster=\"$cluster\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "",
              "metric": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Cluster Filesystem Usage",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The total amount of CPU requests amongst all applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 0,
            "y": 18
          },
          "id": 63,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\", cluster=\"$cluster\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Request",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "CPU used by applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 3,
            "y": 18
          },
          "id": 53,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\", cluster=\"$cluster\"}[5m]))",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Used",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The sum of usable CPU amongst all nodes",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 6,
            "y": 18
          },
          "id": 55,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum (machine_cpu_cores{cluster=\"$cluster\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Total",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The total amount of Memory requests amongst all applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 2,
            "x": 9,
            "y": 18
          },
          "id": 64,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\", cluster=\"$cluster\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Request",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Memory used by applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 2,
            "x": 11,
            "y": 18
          },
          "id": 49,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(container_memory_usage_bytes{id=\"/\", cluster=\"$cluster\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Used",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Actual memory used by applications w/o data that can be evicted under memory pressure",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 13,
            "y": 18
          },
          "id": 85,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(container_memory_working_set_bytes{id=\"/\", cluster=\"$cluster\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Used (w/o Disposable)",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The sum of usable Memory amongst all nodes",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 2,
            "x": 16,
            "y": 18
          },
          "id": 51,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", resource=\"memory\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Total",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Space used by applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 18,
            "y": 18
          },
          "id": 59,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_status_capacity{resource=\"ephemeral_storage\"}) - sum(kube_node_status_allocatable{resource=\"ephemeral_storage\", cluster=\"$cluster\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Used",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The sum of usable space amongst all nodes",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 21,
            "y": 18
          },
          "id": 61,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_status_capacity{resource=\"ephemeral_storage\"})",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Total",
          "transparent": true,
          "type": "stat"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The values represent the `amount of  cores` currently being used",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 6,
          "gridPos": {
            "h": 6,
            "w": 13,
            "x": 0,
            "y": 20
          },
          "hiddenSeries": false,
          "id": 70,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", cluster=\"$cluster\"}[1m]))",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Cores In Use",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "sum(node:cpu_cores:labeled{cluster=\"$cluster\"})",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Total Amount of Cores",
              "refId": "B",
              "step": 10
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Cluster CPU Usage",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The values represent the `amount of  memory` currently being used",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 6,
          "gridPos": {
            "h": 6,
            "w": 11,
            "x": 13,
            "y": 20
          },
          "hiddenSeries": false,
          "id": 71,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(container_memory_usage_bytes{id=\"/\", cluster=\"$cluster\"})",
              "hide": false,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Memory In Use",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "sum(node:memory:labeled{cluster=\"$cluster\"})",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Total Amount of Memory",
              "refId": "B",
              "step": 10
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Cluster Memory Usage",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 6,
          "gridPos": {
            "h": 6,
            "w": 13,
            "x": 0,
            "y": 26
          },
          "hiddenSeries": false,
          "id": 101,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(kubelet_running_pods{cluster=\"$cluster\"})",
              "hide": false,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Total Amount of pods",
              "refId": "B",
              "step": 10
            },
            {
              "expr": "sum(kube_node_status_allocatable_pods{cluster=\"$cluster\"})",
              "hide": false,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Allocatable pods",
              "refId": "C",
              "step": 10
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Pods: Allocatable VS Actual",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:278",
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:279",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 6,
          "gridPos": {
            "h": 6,
            "w": 11,
            "x": 13,
            "y": 26
          },
          "hiddenSeries": false,
          "id": 57,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_status_capacity{resource=\"ephemeral_storage\"}) - sum(kube_node_status_allocatable{resource=\"ephemeral_storage\", cluster=\"$cluster\"})",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Usage",
              "metric": "",
              "refId": "A",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_status_capacity{resource=\"ephemeral_storage\"})",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Capacity",
              "metric": "",
              "refId": "B",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_status_allocatable{resource=\"ephemeral_storage\", cluster=\"$cluster\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "Allocatable",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Cluster Filesystem Usage",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.96
                  ],
                  "type": "lt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "B",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "Total ready pods are below readiness threshold - there might be a large replacement event with lots of nodes.",
            "name": "Ready pods are less than 96%",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 6,
          "gridPos": {
            "h": 6,
            "w": 13,
            "x": 0,
            "y": 32
          },
          "hiddenSeries": false,
          "id": 189,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(kube_pod_container_status_ready{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-service-.*\"})",
              "hide": true,
              "interval": "",
              "legendFormat": "",
              "refId": "E"
            },
            {
              "expr": "sum(kube_pod_container_status_waiting{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-service-.*\"})",
              "hide": true,
              "interval": "",
              "legendFormat": "",
              "refId": "F"
            },
            {
              "expr": "sum(kube_pod_container_status_ready{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-service-.*\"})/(sum(kube_pod_container_status_running{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-service-.*\"})+sum(kube_pod_container_status_waiting{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-service-.*\"}))",
              "hide": false,
              "interval": "",
              "legendFormat": "Service",
              "refId": "D"
            },
            {
              "expr": "sum(kube_pod_container_status_ready{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-worker-.*\"})/(sum(kube_pod_container_status_running{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-worker-.*\"})+sum(kube_pod_container_status_waiting{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*-worker-.*\"}))",
              "hide": false,
              "interval": "",
              "legendFormat": "Worker",
              "refId": "A"
            },
            {
              "expr": "sum(kube_pod_container_status_ready{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*\"})/(sum(kube_pod_container_status_running{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*\"})+sum(kube_pod_container_status_waiting{cluster=\"eks-prod-main-1\",namespace=\"default\",pod=~\".*\"}))",
              "interval": "",
              "legendFormat": "All",
              "refId": "B"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "lt",
              "value": 0.96,
              "visible": true
            }
          ],
          "timeFrom": "1h",
          "timeRegions": [],
          "title": "Ready Pods Percentage",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:259",
              "format": "percentunit",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:260",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 11,
            "x": 13,
            "y": 32
          },
          "id": 240,
          "options": {
            "legend": {
              "calcs": [
                "last",
                "max"
              ],
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_DrainingInitiated=\"true\"}) OR sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_draining_initiated=\"true\"})",
              "interval": "",
              "legendFormat": "Total Draining nodes",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_DrainingInitiated=\"true\"}) by (label_nodegroup) OR sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_draining_initiated=\"true\"}) by (label_nodegroup)",
              "hide": false,
              "interval": "",
              "legendFormat": "{{label_nodegroup}}",
              "refId": "B"
            }
          ],
          "title": "Draining nodes",
          "type": "timeseries"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "decimals": 2,
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 6,
          "grid": {},
          "gridPos": {
            "h": 6,
            "w": 24,
            "x": 0,
            "y": 38
          },
          "height": "200px",
          "hiddenSeries": false,
          "id": 41,
          "isNew": true,
          "legend": {
            "alignAsTable": false,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": false,
            "sideWidth": 200,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(instance:node_network_receive_bytes:rate:sum{cluster=\"$cluster\"})",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Received",
              "metric": "network",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "- sum(instance:node_network_transmit_bytes:rate:sum{cluster=\"$cluster\"})",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Sent",
              "metric": "network",
              "refId": "B",
              "step": 10
            },
            {
              "expr": "",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": "3h",
          "timeRegions": [],
          "title": "Cluster Network I/O pressure",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 0,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "Bps",
              "logBase": 1,
              "show": true
            },
            {
              "format": "Bps",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "breakPoint": "25%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fontSize": "80%",
          "format": "short",
          "gridPos": {
            "h": 6,
            "w": 6,
            "x": 0,
            "y": 44
          },
          "hideTimeOverride": true,
          "id": 94,
          "legend": {
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "pluginVersion": "6.5.2",
          "strokeWidth": 1,
          "targets": [
            {
              "expr": "sum(kube_node_labels{cluster=\"$cluster\"}) by (label_beta_kubernetes_io_instance_type)",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}}",
              "refId": "A",
              "step": 10
            }
          ],
          "timeFrom": "5m",
          "title": "Instance Type Distribution",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "aliasColors": {},
          "breakPoint": "50%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fontSize": "80%",
          "format": "short",
          "gridPos": {
            "h": 6,
            "w": 6,
            "x": 6,
            "y": 44
          },
          "id": 105,
          "legend": {
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "strokeWidth": 1,
          "targets": [
            {
              "expr": "sum(kube_node_labels{cluster=\"$cluster\"}) by (label_nodegroup)",
              "format": "time_series",
              "hide": false,
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "{{label_nodegroup}}",
              "metric": "network",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Instance count",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 7,
          "fillGradient": 7,
          "grid": {},
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 44
          },
          "height": "200px",
          "hiddenSeries": false,
          "id": 104,
          "isNew": true,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "sideWidth": 200,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(kube_node_labels{cluster=\"$cluster\"}) by (label_nodegroup)",
              "interval": "5m",
              "intervalFactor": 2,
              "legendFormat": "{{label_nodegroup}}",
              "metric": "network",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "sum(kube_node_labels{cluster=\"$cluster\"})",
              "hide": true,
              "legendFormat": "Total",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Instance Count Over Time",
          "tooltip": {
            "msResolution": false,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "format": "Bps",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                }
              },
              "mappings": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 6,
            "x": 0,
            "y": 50
          },
          "id": 244,
          "options": {
            "displayLabels": [
              "name",
              "value",
              "percent"
            ],
            "legend": {
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "pieType": "pie",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum (kube_node_labels{cluster=\"$cluster\"}) - sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\"}) > 0",
              "instant": true,
              "interval": "",
              "legendFormat": "On Demand",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\"})",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "Spot",
              "refId": "B"
            }
          ],
          "title": "Spot/OnDemand Distribution $cluster",
          "transparent": true,
          "type": "piechart"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Experimental",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 18,
            "x": 6,
            "y": 50
          },
          "id": 242,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull",
                "max"
              ],
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum (kube_node_labels{cluster=\"$cluster\"}) - sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\"})",
              "interval": "",
              "legendFormat": "On Demand",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "Spot",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "C"
            }
          ],
          "title": "sum(kube_node_labels{cluster=\"$cluster\"})",
          "transparent": true,
          "type": "timeseries"
        }
      ],
      "title": "Cluster Overview",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 15
      },
      "id": 43,
      "panels": [
        {
          "description": "",
          "gridPos": {
            "h": 2,
            "w": 24,
            "x": 0,
            "y": 5
          },
          "id": 79,
          "options": {
            "content": "<div>\n  <center>\n    <h1>\n      Statistics for \n      <font color=#00ffff>\n        $group\n      </font>\n    </h1>\n  </center>\n</div>",
            "mode": "html"
          },
          "pluginVersion": "8.4.4",
          "transparent": true,
          "type": "text"
        },
        {
          "description": "",
          "gridPos": {
            "h": 2,
            "w": 9,
            "x": 0,
            "y": 7
          },
          "id": 81,
          "options": {
            "content": "<div>\n  <center>\n    <h3>\n      CPU Inventory\n    </h1>\n  </center>\n</div>",
            "mode": "html"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "refId": "A",
              "target": ""
            }
          ],
          "type": "text"
        },
        {
          "description": "",
          "gridPos": {
            "h": 2,
            "w": 9,
            "x": 9,
            "y": 7
          },
          "id": 80,
          "options": {
            "content": "<div>\n  <center>\n    <h3>\n      Memory Inventory\n    </h1>\n  </center>\n</div>",
            "mode": "html"
          },
          "pluginVersion": "8.4.4",
          "type": "text"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Current running pods count",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "locale"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 2,
            "x": 18,
            "y": 7
          },
          "hideTimeOverride": false,
          "id": 106,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(kubelet_running_pods{cluster=\"$cluster\"} * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"})",
              "hide": false,
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{$group}}-running-pods",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Running Pods",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Maximum allowed pod count",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "locale"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 20,
            "y": 7
          },
          "hideTimeOverride": false,
          "id": 107,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(kube_node_status_allocatable{cluster=\"$cluster\", resource=\"pods\"} * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"})",
              "hide": false,
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{$group}}-running-pods",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Max Allocatable Pods",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "CPU used by applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 0,
            "y": 9
          },
          "id": 76,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", cluster=\"$cluster\"}[5m]) * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"}) by (label_nodegroup)",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Used",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The sum of usable CPU amongst all nodes",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 3,
            "y": 9
          },
          "id": 77,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(node:cpu_cores:labeled{cluster=\"$cluster\", label_nodegroup=\"$group\"}) by (label_nodegroup)",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Total",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The total amount of CPU requests amongst all applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 6,
            "y": 9
          },
          "id": 78,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(sum(kube_pod_container_resource_requests{resource=\"cpu\", cluster=\"$cluster\"}) by (node) * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"}) by (label_nodegroup)",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Request",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Memory used by applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 12,
            "y": 9
          },
          "id": 216,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(node:memory:labeled{cluster=\"$cluster\", label_nodegroup=\"$group\"})",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Total",
          "transparent": true,
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Memory used by applications",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 2,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 3,
            "x": 15,
            "y": 9
          },
          "id": 217,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(node:container_resource_requests:sum:labeled{resource=\"memory\", cluster=\"$cluster\", label_nodegroup=\"$group\"})",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Request",
          "transparent": true,
          "type": "stat"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "CPU total/used/requested by applications",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 7,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 11
          },
          "hiddenSeries": false,
          "id": 99,
          "legend": {
            "avg": true,
            "current": false,
            "max": true,
            "min": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", cluster=\"$cluster\"}[5m]) * on(node) group_left(label_nodegroup) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup=\"$group\"})) by (label_nodegroup)",
              "hide": false,
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Usage",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "sum(node:container_resource_requests:sum:labeled{resource=\"cpu\", cluster=\"$cluster\", label_nodegroup=\"$group\"})",
              "legendFormat": "Requests",
              "refId": "B"
            },
            {
              "expr": "sum(node:cpu_cores:labeled{cluster=\"$cluster\", label_nodegroup=\"$group\"}) by (label_nodegroup)",
              "hide": false,
              "legendFormat": "Total",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "CPU Total/Used/Requested over time",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": "cores",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Memory total/used/requested by applications",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 7,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 11
          },
          "hiddenSeries": false,
          "id": 102,
          "legend": {
            "avg": true,
            "current": false,
            "max": true,
            "min": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": true,
              "expr": "sum(container_memory_usage_bytes{id=\"/\", cluster=\"$cluster\"} * on(node) group_left(label_nodegroup)  avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup=\"$group\"})) by (label_nodegroup)",
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Usage",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "sum(node:container_resource_requests:sum:labeled{resource=\"memory\", cluster=\"$cluster\", label_nodegroup=\"$group\"})",
              "legendFormat": "Requests",
              "refId": "B"
            },
            {
              "expr": "sum(node:memory:labeled{cluster=\"$cluster\", label_nodegroup=\"$group\"}) by (label_nodegroup)",
              "legendFormat": "Total",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Memory Total/Used/Requested over time",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "label": "cores",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "breakPoint": "25%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The values represent the `amount of  memory` currently being used",
          "fontSize": "80%",
          "format": "short",
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 0,
            "y": 19
          },
          "id": 95,
          "legend": {
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "pluginVersion": "6.5.2",
          "strokeWidth": 1,
          "targets": [
            {
              "expr": "sum(kube_node_labels{cluster=\"$cluster\", label_nodegroup=\"$group\"}) by (label_beta_kubernetes_io_instance_type)",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}}",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Instance Type Distribution",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The values represent the `amount of  memory` currently being used",
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 18,
            "x": 6,
            "y": 19
          },
          "hiddenSeries": false,
          "id": 215,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "sort": "max",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "maxDataPoints": 100,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": true,
              "expr": "count(node:memory:labeled{cluster=\"$cluster\", label_nodegroup=\"$group\"}) by (label_beta_kubernetes_io_instance_type)",
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}}",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "count(node:memory:labeled{cluster=\"$cluster\", label_nodegroup=\"$group\"})",
              "hide": false,
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Total",
              "refId": "B",
              "step": 10
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Instance Type Distribution Over time",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Experimental",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                }
              },
              "mappings": []
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 6,
            "x": 0,
            "y": 26
          },
          "id": 247,
          "options": {
            "displayLabels": [
              "name",
              "value",
              "percent"
            ],
            "legend": {
              "displayMode": "list",
              "placement": "bottom"
            },
            "pieType": "pie",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum (kube_node_labels{cluster=\"$cluster\",label_nodegroup=\"$group\"}) - sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\",label_nodegroup=\"$group\"}) OR sum (kube_node_labels{cluster=\"$cluster\",label_nodegroup=\"$group\"})",
              "instant": true,
              "interval": "",
              "legendFormat": "On Demand",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\",label_nodegroup=\"$group\"})",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "Spot",
              "refId": "B"
            }
          ],
          "title": "On-Demand/Spot Distribution",
          "transparent": true,
          "type": "piechart"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Experimental",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 18,
            "x": 6,
            "y": 26
          },
          "id": 246,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull",
                "max"
              ],
              "displayMode": "table",
              "placement": "right",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum (kube_node_labels{cluster=\"$cluster\",label_nodegroup=\"$group\"}) - sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\",label_nodegroup=\"$group\"}) OR sum (kube_node_labels{cluster=\"$cluster\",label_nodegroup=\"$group\"})",
              "interval": "",
              "legendFormat": "On Demand",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_spotinst_io_node_lifecycle=\"spot\",label_nodegroup=\"$group\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "Spot",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_nodegroup=\"$group\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "Total",
              "refId": "C"
            }
          ],
          "title": "On-Demand/Spot On $cluster-$group",
          "transparent": true,
          "type": "timeseries"
        },
        {
          "aliasColors": {
            "Allocatable Pods": "light-blue",
            "Running Pods": "light-green",
            "nodes-default-allocatable-pods": "blue",
            "nodes-default-running-pods": "light-green"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 7,
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 34
          },
          "hiddenSeries": false,
          "id": 100,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(kubelet_running_pods{cluster=\"$cluster\"} * on(node) group_left(label_nodegroup) kube_node_labels{label_nodegroup=\"$group\"})",
              "hide": false,
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Running Pods",
              "refId": "A",
              "step": 10
            },
            {
              "expr": "sum(kube_node_status_allocatable{cluster=\"$cluster\", resource=\"pods\"} * on(node) group_left(label_nodegroup)  avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup=\"$group\"}))",
              "hide": false,
              "interval": "",
              "legendFormat": "Allocatable Pods",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Allocatable Pods VS Running Pods",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "locale",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "activePatternIndex": 8,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "debug_mode": false,
          "defaultPattern": {
            "bgColors": "green|orange|red",
            "bgColors_overrides": "0->green|2->red|1->yellow",
            "clickable_cells_link": "",
            "col_name": "Default Value",
            "decimals": 2,
            "defaultBGColor": "",
            "defaultTextColor": "",
            "delimiter": ".",
            "displayTemplate": "_1_",
            "enable_bgColor": false,
            "enable_bgColor_overrides": false,
            "enable_clickable_cells": false,
            "enable_textColor": false,
            "enable_textColor_overrides": false,
            "enable_time_based_thresholds": false,
            "enable_transform": false,
            "enable_transform_overrides": false,
            "filter": {
              "value_above": "",
              "value_below": ""
            },
            "format": "none",
            "name": "Default Pattern",
            "null_color": "darkred",
            "null_textcolor": "black",
            "null_value": "No data",
            "pattern": "*",
            "row_col_wrapper": "_",
            "row_name": "_series_",
            "textColors": "red|orange|green",
            "textColors_overrides": "0->red|2->green|1->yellow",
            "thresholds": "70,90",
            "time_based_thresholds": [],
            "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
            "transform_values": "_value_|_value_|_value_",
            "transform_values_overrides": "0->down|1->up",
            "valueName": "avg"
          },
          "default_title_for_rows": "Node",
          "font_size": "1.35rem",
          "gridPos": {
            "h": 7,
            "w": 24,
            "x": 0,
            "y": 42
          },
          "hideTimeOverride": true,
          "hide_first_column": false,
          "hide_headers": false,
          "id": 32,
          "links": [],
          "patterns": [
            {
              "bgColors": "red|orange|green",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "",
              "delimiter": ".",
              "disabled": false,
              "displayTemplate": "_value_",
              "enable_bgColor": false,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": true,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": true,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "locale",
              "name": "Up",
              "null_color": "brown",
              "null_textcolor": "",
              "null_value": "No data",
              "pattern": ".*\\.Up$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "IndianRed|SandyBrown|MediumSpringGreen",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "0,1",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "No|Pending|Yes",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "MediumSpringGreen|SandyBrown|IndianRed",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_ _2_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "percent",
              "name": "CPU Usage",
              "null_color": "brown",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.CPU\\.Usage$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "50,80",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "MediumSpringGreen|SandyBrown|IndianRed",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_ _2_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "percent",
              "name": "Memory Usage",
              "null_color": "brown",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.Memory\\.Usage$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "MediumSpringGreen|SandyBrown|IndianRed",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_ _2_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "percentunit",
              "name": "CPU Request",
              "null_color": "brown",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.CPU\\.Request$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "50,80",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "MediumSpringGreen|SandyBrown|IndianRed",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_ _2_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "percentunit",
              "name": "Memory Request",
              "null_color": "brown",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.Memory\\.Request$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "MediumSpringGreen|SandyBrown|IndianRed",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "percent",
              "name": "IO Wait",
              "null_color": "brown",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.IOWait$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "15,20",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "avg"
            },
            {
              "bgColors": "MediumSpringGreen|SandyBrown|IndianRed",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_2_ _3_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Normalised Load Average 15m",
              "null_color": "brown",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.Normalised\\.Load\\.Average\\.15m$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "100,150",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "IndianRed|SandyBrown|MediumSpringGreen",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "Uptime",
              "decimals": "1",
              "defaultBGColor": "",
              "defaultTextColor": "black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "s",
              "name": "Uptime",
              "null_color": "darkred",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.Uptime$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "180,",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "IndianRed|SandyBrown|MediumSpringGreen",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "Pods Left",
              "decimals": "0",
              "defaultBGColor": "",
              "defaultTextColor": "Black",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "string",
              "name": "Pods Left",
              "null_color": "darkred",
              "null_textcolor": "black",
              "null_value": "No data",
              "pattern": ".*\\.Pods\\.Left$",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "black",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "min"
            }
          ],
          "repeatDirection": "h",
          "row_col_wrapper": "_",
          "sorting_props": {
            "col_index": 7,
            "direction": "desc"
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "label_replace(\n  up{job=\"prometheus-node-exporter\"},\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}",
              "format": "time_series",
              "hide": false,
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{label_kubernetes_io_hostname}}.Up",
              "refId": "E"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "label_replace(\n  (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[1m])) by (instance)) * 100,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}\n",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{label_kubernetes_io_hostname}}.CPU.Usage",
              "refId": "A",
              "target": ""
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "label_replace(\n  (1 - ((avg_over_time(node_memory_MemFree_bytes[1m]) + avg_over_time(node_memory_Cached_bytes[$interval]) + avg_over_time(node_memory_Buffers_bytes[$interval])) / avg_over_time(node_memory_MemTotal_bytes[$interval]))) * 100,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}",
              "format": "time_series",
              "hide": false,
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{label_kubernetes_io_hostname}}.Memory.Usage",
              "refId": "B",
              "target": ""
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(node:container_cpu_request:avg:labeled{label_nodegroup=\"$group\", cluster=\"$cluster\"}) by (label_kubernetes_io_hostname)/sum(node:cpu_cores:labeled{label_nodegroup=\"$group\", cluster=\"$cluster\"}) by (label_kubernetes_io_hostname) ",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_kubernetes_io_hostname}}.CPU.Request",
              "refId": "F"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(node:container_memory_request:avg:labeled{label_nodegroup=\"$group\", cluster=\"$cluster\"}) by (label_kubernetes_io_hostname)/sum(node:memory:labeled{label_nodegroup=\"$group\", cluster=\"$cluster\"}) by (label_kubernetes_io_hostname) ",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_kubernetes_io_hostname}}.Memory.Request",
              "refId": "G"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "label_replace(\n    node_load5, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}",
              "format": "time_series",
              "hide": false,
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{label_kubernetes_io_hostname}}.Normalised.Load.Average.15m",
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "label_replace(\n  avg(rate(node_cpu_seconds_total{mode=\"iowait\"}[1m])) by (instance) * 100,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}",
              "format": "time_series",
              "hide": false,
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{label_kubernetes_io_hostname}}.IOWait",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_status_allocatable{cluster=\"$cluster\", resource=\"pods\"} * on(node) group_left(label_nodegroup, label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\"}) by (label_kubernetes_io_hostname)\n-\nsum(kubelet_running_pods{cluster=\"$cluster\"} * on(node) group_left(label_nodegroup, label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\"}) by (label_kubernetes_io_hostname)",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_kubernetes_io_hostname}}.Pods.Left",
              "refId": "H"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "label_replace(\n  (time() - node_boot_time_seconds),\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_kubernetes_io_hostname}}.Uptime",
              "refId": "I"
            }
          ],
          "text_alignment_firstcolumn": "",
          "text_alignment_header": "center",
          "text_alignment_values": "center",
          "timeFrom": "1m",
          "title": "[$group] Node Group Statistics",
          "transparent": true,
          "type": "yesoreyeram-boomtable-panel"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`idle` - Amount of time the processor was idle\n\n`user` - Amount of time running user space processes\n\n`system` - Amount of time CPU spent running the kernel\n\n`softirq` - How much time the processor has spent servicing software interrupts\n\n`irq` - How much time the processor has spent servicing hardware interrupts\n\n`steal` - How long the virtual CPU has spent waiting for the hypervisor to service another virtual CPU\n\n`nice` - How much time the CPU spent running user space processes that have been niced\n\n`iowait` - Idle time while waiting for an I/O operation to complete",
          "fieldConfig": {
            "defaults": {
              "links": [],
              "unit": "none"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 8,
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 49
          },
          "hiddenSeries": false,
          "id": 238,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "max",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "label_replace(\n    node_load1, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ label_kubernetes_io_hostname}} - Load Average [1m]",
              "refId": "A",
              "target": ""
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "label_replace(\n    node_load5, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ label_kubernetes_io_hostname}} - Load Average [5m]",
              "refId": "B",
              "target": ""
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "label_replace(\n    node_load15, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\", label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{ label_kubernetes_io_hostname}} - Load Average [15m]",
              "refId": "C",
              "target": ""
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Load Averages on $node",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:41",
              "decimals": 1,
              "format": "none",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:42",
              "format": "short",
              "label": "",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "breakPoint": "25%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "",
          "fontSize": "80%",
          "format": "short",
          "gridPos": {
            "h": 8,
            "w": 9,
            "x": 0,
            "y": 58
          },
          "hideTimeOverride": true,
          "id": 213,
          "legend": {
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "pluginVersion": "6.5.2",
          "strokeWidth": 1,
          "targets": [
            {
              "expr": "sum(kube_node_labels{cluster=\"$cluster\",label_nodegroup=~\"$group\"}) by (label_failure_domain_beta_kubernetes_io_zone)",
              "instant": true,
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "{{label_beta_kubernetes_io_instance_type}}",
              "refId": "A",
              "step": 10
            }
          ],
          "timeFrom": "5m",
          "title": "Nodes per AZ",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "activePatternIndex": 4,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "defaultPattern": {
            "bgColors": "green|orange|red",
            "bgColors_overrides": "0->green|2->red|1->yellow",
            "clickable_cells_link": "",
            "col_name": "Value",
            "decimals": 2,
            "defaultBGColor": "transparent",
            "defaultTextColor": "",
            "delimiter": ".",
            "displayTemplate": "_value_",
            "enable_bgColor": false,
            "enable_bgColor_overrides": false,
            "enable_clickable_cells": false,
            "enable_textColor": false,
            "enable_textColor_overrides": false,
            "enable_time_based_thresholds": false,
            "enable_transform": false,
            "enable_transform_overrides": false,
            "filter": {
              "value_above": "",
              "value_below": ""
            },
            "format": "none",
            "name": "Default Pattern",
            "null_color": "darkred",
            "null_textcolor": "black",
            "null_value": "No data",
            "pattern": "*",
            "row_col_wrapper": "_",
            "row_name": "_series_",
            "textColors": "red|orange|green",
            "textColors_overrides": "0->red|2->green|1->yellow",
            "thresholds": "70,90",
            "time_based_thresholds": [],
            "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
            "transform_values": "_value_|_value_|_value_",
            "transform_values_overrides": "0->down|1->up",
            "valueName": "avg"
          },
          "default_title_for_rows": "Availability zone",
          "font_size": "18px",
          "gridPos": {
            "h": 8,
            "w": 15,
            "x": 9,
            "y": 58
          },
          "id": 212,
          "patterns": [
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Unknown",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Unknown",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "4,20",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Pending",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Pending",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "4,20",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "green",
              "defaultTextColor": "",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": false,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Succeeded",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Succeeded",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "green",
              "defaultTextColor": "",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": false,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Running",
              "null_color": "red",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Running",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "red",
              "defaultTextColor": "",
              "delimiter": ".",
              "displayTemplate": "_value_",
              "enable_bgColor": false,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Failed",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Failed",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            }
          ],
          "pluginVersion": "7.1.1",
          "row_col_wrapper": "_",
          "sorting_props": {
            "col_index": 2,
            "direction": "asc"
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(phase,label_failure_domain_beta_kubernetes_io_zone)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Failed\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{ label_failure_domain_beta_kubernetes_io_zone }}.{{phase}}",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(phase,label_failure_domain_beta_kubernetes_io_zone)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Pending\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{ label_failure_domain_beta_kubernetes_io_zone }}.{{ phase }}",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(phase,label_failure_domain_beta_kubernetes_io_zone)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Unknown\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{ label_failure_domain_beta_kubernetes_io_zone }}.{{ phase }}",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(phase,label_failure_domain_beta_kubernetes_io_zone)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Succeeded\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{ label_failure_domain_beta_kubernetes_io_zone }}.{{ phase }}",
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(phase,label_failure_domain_beta_kubernetes_io_zone)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Running\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{ label_failure_domain_beta_kubernetes_io_zone }}.{{ phase }}",
              "refId": "E"
            }
          ],
          "title": "Pods / AZ",
          "type": "yesoreyeram-boomtable-panel"
        },
        {
          "activePatternIndex": 4,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "defaultPattern": {
            "bgColors": "green|orange|red",
            "bgColors_overrides": "0->green|2->red|1->yellow",
            "clickable_cells_link": "",
            "col_name": "Value",
            "decimals": 2,
            "defaultBGColor": "transparent",
            "defaultTextColor": "",
            "delimiter": "!",
            "displayTemplate": "_value_",
            "enable_bgColor": false,
            "enable_bgColor_overrides": false,
            "enable_clickable_cells": false,
            "enable_textColor": false,
            "enable_textColor_overrides": false,
            "enable_time_based_thresholds": false,
            "enable_transform": false,
            "enable_transform_overrides": false,
            "filter": {
              "value_above": "",
              "value_below": ""
            },
            "format": "none",
            "name": "Default Pattern",
            "null_color": "darkred",
            "null_textcolor": "black",
            "null_value": "No data",
            "pattern": "*",
            "row_col_wrapper": "_",
            "row_name": "_series_",
            "textColors": "red|orange|green",
            "textColors_overrides": "0->red|2->green|1->yellow",
            "thresholds": "70,90",
            "time_based_thresholds": [],
            "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
            "transform_values": "_value_|_value_|_value_",
            "transform_values_overrides": "0->down|1->up",
            "valueName": "avg"
          },
          "default_title_for_rows": "Availability zone - Node name - Instance Type",
          "font_size": "18px",
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 66
          },
          "id": 210,
          "patterns": [
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "",
              "delimiter": "!",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Unknown",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Unknown",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "4,20",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "",
              "defaultTextColor": "",
              "delimiter": "!",
              "displayTemplate": "_value_",
              "enable_bgColor": true,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Pending",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Pending",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "4,20",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "green",
              "defaultTextColor": "",
              "delimiter": "!",
              "displayTemplate": "_value_",
              "enable_bgColor": false,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Succeeded",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Succeeded",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "green",
              "defaultTextColor": "",
              "delimiter": "!",
              "displayTemplate": "_value_",
              "enable_bgColor": false,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Running",
              "null_color": "red",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Running",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            },
            {
              "bgColors": "green|orange|red",
              "bgColors_overrides": "0->green|2->red|1->yellow",
              "clickable_cells_link": "",
              "col_name": "_1_",
              "decimals": 2,
              "defaultBGColor": "red",
              "defaultTextColor": "",
              "delimiter": "!",
              "displayTemplate": "_value_",
              "enable_bgColor": false,
              "enable_bgColor_overrides": false,
              "enable_clickable_cells": false,
              "enable_textColor": false,
              "enable_textColor_overrides": false,
              "enable_time_based_thresholds": false,
              "enable_transform": false,
              "enable_transform_overrides": false,
              "filter": {
                "value_above": "",
                "value_below": ""
              },
              "format": "none",
              "name": "Failed",
              "null_color": "green",
              "null_textcolor": "white",
              "null_value": "No data",
              "pattern": "Failed",
              "row_col_wrapper": "_",
              "row_name": "_0_",
              "textColors": "red|orange|green",
              "textColors_overrides": "0->red|2->green|1->yellow",
              "thresholds": "70,90",
              "time_based_thresholds": [],
              "tooltipTemplate": "Series : _series_ <br/>Row Name : _row_name_ <br/>Col Name : _col_name_ <br/>Value : _value_",
              "transform_values": "_value_|_value_|_value_",
              "transform_values_overrides": "0->down|1->up",
              "valueName": "current"
            }
          ],
          "pluginVersion": "7.1.1",
          "row_col_wrapper": "_",
          "sorting_props": {
            "col_index": -1,
            "direction": "desc"
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(label_kubernetes_io_hostname,phase,label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Failed\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_failure_domain_beta_kubernetes_io_zone}} - {{ label_kubernetes_io_hostname }} - {{label_beta_kubernetes_io_instance_type}} !{{phase}}",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(label_kubernetes_io_hostname,phase,label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Pending\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_failure_domain_beta_kubernetes_io_zone}} - {{ label_kubernetes_io_hostname }} - {{label_beta_kubernetes_io_instance_type}} !{{phase}}",
              "refId": "B"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(label_kubernetes_io_hostname,phase,label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Unknown\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_failure_domain_beta_kubernetes_io_zone}} - {{ label_kubernetes_io_hostname }} - {{label_beta_kubernetes_io_instance_type}} !{{phase}}",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(label_kubernetes_io_hostname,phase,label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Succeeded\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_failure_domain_beta_kubernetes_io_zone}} - {{ label_kubernetes_io_hostname }} - {{label_beta_kubernetes_io_instance_type}} !{{phase}}",
              "refId": "D"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum by(label_kubernetes_io_hostname,phase,label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type)((node:pods_by_phase{cluster=~\"$cluster\",phase=\"Running\"}) * on(label_kubernetes_io_hostname) group_left(label_failure_domain_beta_kubernetes_io_zone,label_beta_kubernetes_io_instance_type) (kube_node_labels{cluster=~\"$cluster\",label_nodegroup=~\"$group\"}))",
              "instant": true,
              "interval": "",
              "legendFormat": "{{label_failure_domain_beta_kubernetes_io_zone}} - {{ label_kubernetes_io_hostname }} - {{label_beta_kubernetes_io_instance_type}} !{{phase}}",
              "refId": "E"
            }
          ],
          "title": "Pods / AZ / Node",
          "type": "yesoreyeram-boomtable-panel"
        }
      ],
      "title": "Node Group Overview",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 16
      },
      "id": 4,
      "panels": [
        {
          "description": "",
          "gridPos": {
            "h": 2,
            "w": 24,
            "x": 0,
            "y": 66
          },
          "id": 34,
          "options": {
            "content": "<div>\n  <center>\n    <h1>\n      Statistics for \n      <font color=#00ffff>\n        $node\n      </font>\n    </h1>\n  </center>\n</div>",
            "mode": "html"
          },
          "pluginVersion": "8.4.4",
          "transparent": true,
          "type": "text"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 8,
            "x": 0,
            "y": 68
          },
          "id": 114,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "mean"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(node:cpu_cores:labeled{label_kubernetes_io_hostname=\"$node\"})",
              "refId": "A"
            }
          ],
          "title": "Core Count",
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 2,
            "w": 8,
            "x": 9,
            "y": 68
          },
          "id": 115,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(node:memory:labeled{label_kubernetes_io_hostname=\"$node\"})",
              "refId": "A"
            }
          ],
          "title": "Total Memory",
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "norm 15m",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "decimals": 1,
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "min": 0,
              "thresholds": {
                "mode": "percentage",
                "steps": [
                  {
                    "color": "#299c46"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 60
                  },
                  {
                    "color": "#d44a3a",
                    "value": 80
                  }
                ]
              },
              "unit": "percentunit"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 3,
            "x": 17,
            "y": 68
          },
          "hideTimeOverride": true,
          "id": 30,
          "links": [],
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "exemplar": true,
              "expr": "node:normalized_load15{label_kubernetes_io_hostname=\"$node\", cluster=\"$cluster\"}",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "target": ""
            }
          ],
          "timeFrom": "1m",
          "title": "Normalised Load Average",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Currently running pods on the node",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "#d44a3a"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 5
                  },
                  {
                    "color": "#299c46",
                    "value": 15
                  }
                ]
              },
              "unit": "locale"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 4,
            "w": 2,
            "x": 20,
            "y": 68
          },
          "hideTimeOverride": true,
          "id": 236,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kubelet_running_pods{cluster=\"$cluster\"} * on(node) group_left(label_nodegroup, label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_kubernetes_io_hostname=\"$node\"}) by (label_kubernetes_io_hostname)",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "target": ""
            }
          ],
          "timeFrom": "1m",
          "title": "Running pods",
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Maximum pods count possible to run on this host",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "#d44a3a"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 5
                  },
                  {
                    "color": "#299c46",
                    "value": 15
                  }
                ]
              },
              "unit": "locale"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 4,
            "w": 2,
            "x": 22,
            "y": 68
          },
          "hideTimeOverride": true,
          "id": 110,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "sum(kube_node_status_allocatable{cluster=\"$cluster\", resource=\"pods\"} * on(node) group_left(label_nodegroup, label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_kubernetes_io_hostname=\"$node\"}) by (label_kubernetes_io_hostname)",
              "format": "time_series",
              "instant": true,
              "intervalFactor": 1,
              "refId": "A",
              "target": ""
            }
          ],
          "timeFrom": "1m",
          "title": "Max Pods",
          "type": "stat"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "#299c46"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 50
                  },
                  {
                    "color": "#d44a3a",
                    "value": 80
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 4,
            "x": 0,
            "y": 70
          },
          "hideTimeOverride": true,
          "id": 28,
          "links": [],
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "(1-(rate(node:cpu_seconds_total_per_mode:avg:labeled{cluster=\"$cluster\",label_kubernetes_io_hostname=\"$node\", mode=\"idle\"}[$interval])))* 100",
              "format": "time_series",
              "instant": true,
              "intervalFactor": 1,
              "refId": "A",
              "target": ""
            }
          ],
          "timeFrom": "1m",
          "title": "CPU Usage",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [],
              "max": 1,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "#EAB839",
                    "value": 0.5
                  },
                  {
                    "color": "red",
                    "value": 0.8
                  }
                ]
              },
              "unit": "percentunit"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 4,
            "x": 4,
            "y": 70
          },
          "id": 97,
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "(sum(node:container_cpu_request:avg:labeled{label_kubernetes_io_hostname=\"$node\"})) /\nsum(node:cpu_cores:labeled{label_kubernetes_io_hostname=\"$node\"}) ",
              "refId": "A"
            }
          ],
          "title": "CPU Request",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "#299c46"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 50
                  },
                  {
                    "color": "#d44a3a",
                    "value": 80
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 4,
            "x": 9,
            "y": 70
          },
          "hideTimeOverride": true,
          "id": 29,
          "links": [],
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "avg_over_time(node:memory_used_percent:labeled{cluster=\"$cluster\", label_kubernetes_io_hostname=\"$node\"}[$interval])",
              "format": "time_series",
              "instant": true,
              "intervalFactor": 1,
              "refId": "A",
              "target": ""
            }
          ],
          "timeFrom": "1m",
          "title": "Memory Usage",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [],
              "max": 1,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "#EAB839",
                    "value": 0.5
                  },
                  {
                    "color": "red",
                    "value": 0.8
                  }
                ]
              },
              "unit": "percentunit"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 6,
            "w": 4,
            "x": 13,
            "y": 70
          },
          "id": 98,
          "options": {
            "displayMode": "lcd",
            "orientation": "vertical",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "showUnfilled": true,
            "text": {}
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "expr": "(sum(node:container_memory_request:avg:labeled{label_kubernetes_io_hostname=\"$node\"})) /\nsum(node:memory:labeled{label_kubernetes_io_hostname=\"$node\"}) ",
              "refId": "A"
            }
          ],
          "title": "Memory Request",
          "transparent": true,
          "type": "bargauge"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Available pod count left to schedule on this host",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "match": "null",
                    "result": {
                      "text": "N/A"
                    }
                  },
                  "type": "special"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "#d44a3a"
                  },
                  {
                    "color": "rgba(237, 129, 40, 0.89)",
                    "value": 5
                  },
                  {
                    "color": "#299c46",
                    "value": 15
                  }
                ]
              },
              "unit": "locale"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 4,
            "w": 2,
            "x": 22,
            "y": 72
          },
          "hideTimeOverride": true,
          "id": 235,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "none",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "text": {},
            "textMode": "auto"
          },
          "pluginVersion": "8.4.4",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": false,
              "expr": "sum(kube_node_status_allocatable{cluster=\"$cluster\", resource=\"pods\"} * on(node) group_left(label_nodegroup, label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_kubernetes_io_hostname=\"$node\"}) by (label_kubernetes_io_hostname)\n-\nsum(kubelet_running_pods{cluster=\"$cluster\"} * on(node) group_left(label_nodegroup, label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_kubernetes_io_hostname=\"$node\"}) by (label_kubernetes_io_hostname)",
              "format": "time_series",
              "instant": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A",
              "target": ""
            }
          ],
          "timeFrom": "1m",
          "title": "Pods Left",
          "type": "stat"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`idle` - Amount of time the processor was idle\n\n`user` - Amount of time running user space processes\n\n`system` - Amount of time CPU spent running the kernel\n\n`softirq` - How much time the processor has spent servicing software interrupts\n\n`irq` - How much time the processor has spent servicing hardware interrupts\n\n`steal` - How long the virtual CPU has spent waiting for the hypervisor to service another virtual CPU\n\n`nice` - How much time the CPU spent running user space processes that have been niced\n\n`iowait` - Idle time while waiting for an I/O operation to complete",
          "fieldConfig": {
            "defaults": {
              "links": [],
              "unit": "none"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 8,
          "gridPos": {
            "h": 8,
            "w": 11,
            "x": 0,
            "y": 76
          },
          "hiddenSeries": false,
          "id": 237,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "max",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "label_replace(\n    node_load1, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\", label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Load Average [1m]",
              "refId": "A",
              "target": ""
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "label_replace(\n    node_load5, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\", label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Load Average [5m]",
              "refId": "B",
              "target": ""
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "label_replace(\n    node_load15, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) * on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_nodegroup=\"$group\", label_cluster=\"$cluster\", label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Load Average [15m]",
              "refId": "C",
              "target": ""
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Load Averages on $node",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:41",
              "decimals": 1,
              "format": "none",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:42",
              "format": "short",
              "label": "",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`idle` - Amount of time the processor was idle\n\n`user` - Amount of time running user space processes\n\n`system` - Amount of time CPU spent running the kernel\n\n`softirq` - How much time the processor has spent servicing software interrupts\n\n`irq` - How much time the processor has spent servicing hardware interrupts\n\n`steal` - How long the virtual CPU has spent waiting for the hypervisor to service another virtual CPU\n\n`nice` - How much time the CPU spent running user space processes that have been niced\n\n`iowait` - Idle time while waiting for an I/O operation to complete",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 8,
          "gridPos": {
            "h": 8,
            "w": 11,
            "x": 11,
            "y": 76
          },
          "hiddenSeries": false,
          "id": 2,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(irate(node:cpu_seconds_total_per_mode:avg:labeled{cluster=\"$cluster\",label_kubernetes_io_hostname=\"$node\"}[$interval]))* 100",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{mode}}",
              "refId": "A",
              "target": ""
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "CPU Usage by [mode]",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 1,
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "label": "",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "columns": [],
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "pods which do not have Running or Succeeded status ",
          "fontSize": "100%",
          "gridPos": {
            "h": 7,
            "w": 11,
            "x": 0,
            "y": 84
          },
          "id": 93,
          "links": [],
          "showHeader": true,
          "sort": {
            "col": 0,
            "desc": false
          },
          "styles": [
            {
              "alias": "Time",
              "align": "auto",
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Time",
              "type": "hidden"
            },
            {
              "alias": "",
              "align": "auto",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "Value",
              "thresholds": [],
              "type": "hidden",
              "unit": "short"
            },
            {
              "alias": "Pod Name",
              "align": "auto",
              "colors": [
                "#C4162A",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "link": false,
              "mappingType": 1,
              "pattern": "pod",
              "thresholds": [
                ""
              ],
              "type": "number",
              "unit": "short"
            },
            {
              "alias": "Phase",
              "align": "auto",
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "decimals": 2,
              "mappingType": 1,
              "pattern": "phase",
              "thresholds": [],
              "type": "number",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "expr": "avg(node:pods_by_phase{cluster=\"$cluster\",phase!=\"Running\",phase!=\"Succeeded\", label_kubernetes_io_hostname=\"$node\"}) by (pod, phase) > 0",
              "format": "table",
              "instant": true,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "Malfunctioning Pods List",
          "transform": "table",
          "transparent": true,
          "type": "table-old"
        },
        {
          "aliasColors": {
            "Memory In Use": "blue"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "The values represent the `amount of  memory` currently being used",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 2,
          "fillGradient": 6,
          "gridPos": {
            "h": 7,
            "w": 9,
            "x": 11,
            "y": 84
          },
          "hiddenSeries": false,
          "id": 113,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(node:memory_used_percent:labeled{cluster=\"$cluster\",label_kubernetes_io_hostname=\"$node\"})",
              "interval": "10s",
              "intervalFactor": 1,
              "legendFormat": "Memory In Use",
              "refId": "A",
              "step": 10
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Memory Usage",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:762",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:763",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Memoey usage by node",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 8,
          "gridPos": {
            "h": 7,
            "w": 4,
            "x": 20,
            "y": 84
          },
          "hiddenSeries": false,
          "id": 18,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg_over_time(node:memory_used_percent:labeled{cluster=\"$cluster\", label_kubernetes_io_hostname=\"$node\"}[$interval])",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "In use",
              "refId": "A",
              "target": ""
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Pods: Allocatable VS Actual",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "logBase": 1,
              "max": "100",
              "min": "0",
              "show": true
            },
            {
              "format": "short",
              "label": "",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "breakPoint": "50%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "decimals": 1,
          "description": "Node CPU usage breakdown by the pods occupying it",
          "fontSize": "80%",
          "format": "ms",
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 91
          },
          "id": 36,
          "legend": {
            "header": "",
            "percentage": false,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "strokeWidth": 1,
          "targets": [
            {
              "expr": "label_replace(\n  avg(rate(container_cpu_usage_seconds_total{pod_name=~\".+\", container_name=\"\"}[5m])) by (instance, pod_name) * 1000,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)\n* on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_kubernetes_io_hostname=\"$node\", label_cluster=\"$cluster\"} ",
              "instant": true,
              "legendFormat": "{{pod_name}}",
              "refId": "A"
            }
          ],
          "title": "CPU Usage by Pod",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "aliasColors": {},
          "breakPoint": "50%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Memory usage breakdown by the pods occupying it",
          "fontSize": "80%",
          "format": "bytes",
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 91
          },
          "id": 37,
          "legend": {
            "header": "",
            "percentage": false,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "strokeWidth": 1,
          "targets": [
            {
              "expr": "avg(container_memory_working_set_bytes{pod_name!=\"\", container_name=\"\"}) by (node, pod_name)\n* on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_kubernetes_io_hostname=\"$node\", label_cluster=\"$cluster\"} ",
              "instant": true,
              "interval": "",
              "legendFormat": "{{pod_name}}",
              "refId": "A"
            },
            {
              "exemplar": true,
              "expr": "sum(avg(container_memory_working_set_bytes{pod_name!=\"\", container_name=\"\"}) by (node, pod_name)\n* on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_kubernetes_io_hostname=\"$node\", label_cluster=\"$cluster\"} )",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "TOTAL",
              "refId": "B"
            }
          ],
          "title": "Memory Usage by Pod",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "aliasColors": {
            "Failed": "#F2495C",
            "Pending": "#F2CC0C",
            "Running": "#73BF69",
            "Succeeded": "#5794F2",
            "Unknown": "#CA95E5"
          },
          "breakPoint": "50%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "pods running on node",
          "fontSize": "80%",
          "format": "short",
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 99
          },
          "id": 92,
          "legend": {
            "show": true,
            "sideWidth": 400,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "strokeWidth": 1,
          "targets": [
            {
              "expr": "sum(node:pods_by_phase{label_kubernetes_io_hostname=\"$node\", cluster=\"$cluster\"}) by (phase)",
              "instant": true,
              "legendFormat": "{{phase}}",
              "refId": "A"
            }
          ],
          "title": "Pods Distribution by Status",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "aliasColors": {},
          "breakPoint": "50%",
          "combine": {
            "label": "Others",
            "threshold": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Memory usage breakdown by the pods occupying it",
          "fontSize": "80%",
          "format": "bytes",
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 99
          },
          "id": 228,
          "legend": {
            "header": "",
            "percentage": false,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "values": true
          },
          "legendType": "Right side",
          "links": [],
          "maxDataPoints": 3,
          "nullPointMode": "connected",
          "pieType": "pie",
          "strokeWidth": 1,
          "targets": [
            {
              "exemplar": true,
              "expr": "avg(container_memory_usage_bytes{pod_name!=\"\", container_name=\"\"}) by (node, pod_name)\n* on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_kubernetes_io_hostname=\"$node\", label_cluster=\"$cluster\"} ",
              "instant": true,
              "interval": "",
              "legendFormat": "{{pod_name}}",
              "refId": "A"
            },
            {
              "exemplar": true,
              "expr": "sum(avg(container_memory_usage_bytes{pod_name!=\"\", container_name=\"\"}) by (node, pod_name)\n* on(node) group_left(label_kubernetes_io_hostname) kube_node_labels{label_kubernetes_io_hostname=\"$node\", label_cluster=\"$cluster\"} )",
              "hide": false,
              "instant": true,
              "interval": "",
              "legendFormat": "TOTAL",
              "refId": "B"
            }
          ],
          "title": "Memory Usage by Pod including buff/cache",
          "transparent": true,
          "type": "grafana-piechart-panel",
          "valueName": "current"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Bytes written/read by instance",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 8,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 107
          },
          "hiddenSeries": false,
          "id": 9,
          "interval": "",
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "0-node:disk_written_bytes_total:avg:labeled{cluster=\"$cluster\",label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 1,
              "legendFormat": "Bytes Written",
              "refId": "A",
              "target": ""
            },
            {
              "expr": "node:disk_read_bytes_total:avg:labeled{cluster=\"$cluster\",label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 1,
              "legendFormat": "Bytes Read",
              "refId": "B",
              "target": ""
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Disk Bytes Written/Read",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "label": "",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Incoming/Outgoing network (bytes) by instance",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 8,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 107
          },
          "hiddenSeries": false,
          "id": 8,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "node:network_received_bytes_total:labeled{label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 1,
              "legendFormat": "Network In",
              "refId": "A",
              "target": ""
            },
            {
              "expr": "0 - node:network_sent_bytes_total:labeled{label_kubernetes_io_hostname=\"$node\"}",
              "legendFormat": "Network Out",
              "refId": "D"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "In/Out Network",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "label": "",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Disk write/read IOPS by instance",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 8,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 115
          },
          "hiddenSeries": false,
          "id": 24,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "0 - node:disk_writes_completed_total:avg:labeled{cluster=\"$cluster\",label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Write IOPS",
              "refId": "A",
              "target": ""
            },
            {
              "expr": "node:disk_reads_completed_total:avg:labeled{cluster=\"$cluster\",label_kubernetes_io_hostname=\"$node\"}",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Read IOPS",
              "refId": "B",
              "target": ""
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Disk Write/Read IOPS",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "locale",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "label": "",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "Node Overview",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 17
      },
      "id": 66,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Values `< 100%` means lower usage than request\n\nValues `= 100%` means exactly the same usage as request\n\nValues `> 100%` means higher usage than request",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 6
          },
          "hiddenSeries": false,
          "id": 68,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.1.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg(rate(container:cpu_usage_seconds_total:avg:labeled{cluster=\"$cluster\"}[5m])) by (container_name)/avg(container:resource_requests:avg:labeled{cluster=\"$cluster\",resource=\"cpu\"})  by (container_name) > 1.1",
              "legendFormat": "{{container_name}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": "30m",
          "timeRegions": [],
          "title": "Container(s) Exceeding CPU Request",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percentunit",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Values `< 100%` means lower usage than request\n\nValues `= 100%` means exactly the same usage as request\n\nValues `> 100%` means higher usage than request",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 6
          },
          "hiddenSeries": false,
          "id": 69,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.1.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg(container:memory_working_set_bytes:avg:labeled{cluster=\"$cluster\"}) by (container_name) / avg(container:resource_requests:avg:labeled{cluster=\"$cluster\",resource=\"memory\"}) by (container_name) > 1.1",
              "instant": false,
              "interval": "",
              "legendFormat": "{{container_name}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": "30m",
          "timeRegions": [],
          "title": "Container(s) Exceeding Memory Request",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percentunit",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Values `< 100%` means lower usage than request\n\nValues `= 100%` means exactly the same usage as request\n\nValues `> 100%` means higher usage than request",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 14
          },
          "hiddenSeries": false,
          "id": 73,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "max",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.1.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg(rate(container:cpu_usage_seconds_total:avg:labeled{cluster=\"$cluster\"}[5m])) by (container_name)/ avg(container:resource_requests:avg:labeled{cluster=\"$cluster\",resource=\"cpu\"}) by (container_name) < 0.5",
              "legendFormat": "{{container_name}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": "30m",
          "timeRegions": [],
          "title": "Container(s) Over-Requested CPU",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:59",
              "format": "percentunit",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:60",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "Values `< 100%` means lower usage than request\n\nValues `= 100%` means exactly the same usage as request\n\nValues `> 100%` means higher usage than request",
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 14
          },
          "hiddenSeries": false,
          "id": 74,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "avg",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.1.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg(container:memory_working_set_bytes:avg:labeled{cluster=\"$cluster\"}) by (container_name) / avg(container:resource_requests:avg:labeled{cluster=\"$cluster\",resource=\"memory\"}) by (container_name) <= 0.5",
              "interval": "",
              "legendFormat": "{{container_name}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": "30m",
          "timeRegions": [],
          "title": "Container(s) Over-Requested Memory",
          "tooltip": {
            "shared": true,
            "sort": 1,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percentunit",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "Pods Request Overview",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 18
      },
      "id": 219,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "decbytes"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 15
          },
          "hiddenSeries": false,
          "id": 221,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": false,
            "show": true,
            "sort": "max",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": true,
              "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\"}[5m])) by (name)",
              "interval": "",
              "legendFormat": "{{name}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Incoming",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "decbytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "decbytes"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 15
          },
          "hiddenSeries": false,
          "id": 222,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": true,
            "min": false,
            "show": true,
            "sort": "max",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": true,
              "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\"}[5m])) by (name)",
              "interval": "",
              "legendFormat": "{{name}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Outgoing",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "decbytes",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "Pod Network Usage",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 127,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editable": true,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 8
          },
          "hiddenSeries": false,
          "id": 125,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "total",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (proto)",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{proto}}",
              "refId": "A",
              "step": 60
            },
            {
              "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "total",
              "refId": "B",
              "step": 60
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Requests (total)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "cumulative"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:565",
              "format": "pps",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "$$hashKey": "object:566",
              "format": "pps",
              "logBase": 1,
              "min": 0,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editable": true,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 8
          },
          "hiddenSeries": false,
          "id": 123,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "total",
              "yaxis": 2
            },
            {
              "alias": "other",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(coredns_dns_request_type_count_total{instance=~\"$instance\"}[5m])) by (type)",
              "intervalFactor": 2,
              "legendFormat": "{{type}}",
              "refId": "A",
              "step": 60
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Requests (by qtype)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "cumulative"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "pps",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "pps",
              "logBase": 1,
              "min": 0,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editable": true,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 12,
            "x": 0,
            "y": 15
          },
          "hiddenSeries": false,
          "id": 121,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "misses",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(coredns_cache_hits_total{instance=~\"$instance\"}[5m])) by (type)",
              "intervalFactor": 2,
              "legendFormat": "hits:{{type}}",
              "refId": "A",
              "step": 40
            },
            {
              "expr": "sum(rate(coredns_cache_misses_total{instance=~\"$instance\"}[5m])) by (type)",
              "intervalFactor": 2,
              "legendFormat": "misses",
              "refId": "B",
              "step": 40
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Cache (hitrate)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "cumulative"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "pps",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "pps",
              "logBase": 1,
              "min": 0,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editable": true,
          "error": false,
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 12,
            "x": 12,
            "y": 15
          },
          "hiddenSeries": false,
          "id": 119,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "connected",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(coredns_cache_size{instance=~\"$instance\"}) by (type)",
              "intervalFactor": 2,
              "legendFormat": "{{type}}",
              "refId": "A",
              "step": 40
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Cache (size)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "cumulative"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "logBase": 1,
              "min": 0,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "min": 0,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "usagePercentageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "usagePercentageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "coredns Memory percentage usage alert",
            "name": "Aggregated Memory Usage coredns alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 22
          },
          "hiddenSeries": false,
          "id": 129,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"coredns\"}) / sum(workload:container_resource_requests:aggr{container=\"coredns\", resource=\"memory\", cluster=~\"eks.*main.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "usagePercentageMain",
              "refId": "usagePercentageMain"
            },
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"coredns\"}) / sum(workload:container_resource_requests:aggr{container=\"coredns\", resource=\"memory\", cluster=~\"eks.*mgmt.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "usagePercentageMgmt",
              "refId": "usagePercentageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 100,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage percentage coredns",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:350",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:351",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    95
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    95
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "coredns CPU is too high",
            "name": "Aggregated CPU Usage coredns alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 22
          },
          "hiddenSeries": false,
          "id": 131,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"coredns\"}) /\nsum(workload:container_resource_requests:aggr{workload=\"coredns\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"coredns\"}) /\nsum(workload:container_resource_requests:aggr{workload=\"coredns\", cluster=~\"eks.*mgmt.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 95,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage coredns",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:447",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:448",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.3
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "F",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "too many coredns replicas unavailable",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 17,
            "x": 4,
            "y": 30
          },
          "hiddenSeries": false,
          "id": 133,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas{cluster=\"eks-prod-main-1\", deployment=\"coredns\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Current",
              "refId": "A",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", deployment=\"coredns\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "B",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"coredns\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Unavailable",
              "refId": "C",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_updated{cluster=\"eks-prod-main-1\", deployment=\"coredns\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Updated",
              "refId": "D",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"coredns\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "E",
              "step": 30
            },
            {
              "expr": "(avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"coredns\"}) + 0.1) / avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"coredns\"})",
              "interval": "",
              "legendFormat": "precent of unavailable",
              "refId": "F"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0.3,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "coredns",
      "type": "row"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 16,
      "panels": [],
      "title": "INFRASTRUCTURE ALERTS",
      "type": "row"
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                95
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "10m",
        "frequency": "5m",
        "handler": 1,
        "message": "Please check the alerting node(s)",
        "name": "CPU (Request/Capacity) by NodeGroup alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "CPU usage out of request by Node Group",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 21
      },
      "hiddenSeries": false,
      "id": 17,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on(node) group_left(label_nodegroup, cluster) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels)) by (label_nodegroup)  / sum(kube_node_status_capacity{resource=\"cpu\"} * on(node) group_left(label_nodegroup, cluster) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels)) by (label_nodegroup) * 100",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_nodegroup}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 95,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "CPU (Request/Capacity) by NodeGroup",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:357",
          "decimals": 0,
          "format": "percent",
          "logBase": 1,
          "max": "100",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:358",
          "format": "short",
          "label": "",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                92
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "10m",
        "frequency": "5m",
        "handler": 1,
        "message": "Please check the alerting node(s)",
        "name": "Memory (Request/Capacity) by NodeGroup alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Memoey usage out of request by Node Group",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 21
      },
      "hiddenSeries": false,
      "id": 5,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "sum(kube_pod_container_resource_requests{label_nodegroup!=\"nodes-bigid-scanner\",resource=\"memory\"} * on(node) group_left(label_nodegroup) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup!=\"nodes-bigid-scanner\"})) by (label_nodegroup)  / sum(kube_node_status_allocatable{label_nodegroup!=\"nodes-bigid-scanner\",resource=\"memory\"} * on(node) group_left(label_nodegroup) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup!=\"nodes-bigid-scanner\"})) by (label_nodegroup) * 100\n",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_nodegroup}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 92,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "Memory (Request/Capacity) by NodeGroup",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:284",
          "decimals": 0,
          "format": "percent",
          "logBase": 1,
          "max": "100",
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:285",
          "format": "short",
          "label": "",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                65
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "15m",
        "frequency": "5m",
        "handler": 1,
        "name": "High IO Wait alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          },
          {
            "uid": "srXoYKsWz"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Alerting on high IO Wait",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 87,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "label_replace(\n  avg(rate(node_cpu_seconds_total{mode=\"iowait\"}[5m])) by (instance, mode) * 100,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup!=\"nodes-bigdata\"})",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_kubernetes_io_hostname}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 65,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "High IO Wait",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:125",
          "decimals": 1,
          "format": "percent",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:126",
          "format": "short",
          "label": "",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                70
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "15m",
        "frequency": "5m",
        "handler": 1,
        "name": "High IO Wait BigData alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          },
          {
            "uid": "srXoYKsWz"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Alerting on high IO Wait",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 192,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "label_replace(\n  avg(rate(node_cpu_seconds_total{mode=\"iowait\"}[5m])) by (instance, mode) * 100,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup=\"nodes-bigdata\"})",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_kubernetes_io_hostname}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 70,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "High IO Wait BigData",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:125",
          "decimals": 1,
          "format": "percent",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:126",
          "format": "short",
          "label": "",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                90
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "5m",
        "frequency": "1m",
        "handler": 1,
        "name": "Memory Usage [BigData] alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000016"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 37
      },
      "hiddenSeries": false,
      "hideTimeOverride": false,
      "id": 89,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": true,
          "expr": "label_replace(\n  (1 - ((avg_over_time(node_memory_MemFree_bytes[5m]) + avg_over_time(node_memory_Cached_bytes[5m]) + avg_over_time(node_memory_Buffers_bytes[5m])) / avg_over_time(node_memory_MemTotal_bytes[5m]))) * 100,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup=\"nodes-bigdata\"})",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_kubernetes_io_hostname}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 90,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "Memory Usage [BigData]",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:427",
          "format": "percent",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:428",
          "format": "short",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                2
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "10m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "30m",
        "frequency": "10m",
        "handler": 1,
        "message": "Please check the alerting node and see which pod performs the most workload",
        "name": "Load Average alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS-NOCORTEX}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 37
      },
      "hiddenSeries": false,
      "id": 13,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS-NOCORTEX}"
          },
          "exemplar": true,
          "expr": "label_replace(\n    node_load15, \n    \"node\", \n    \"ip-$1-$2-$3-$4.ec2.internal\", \n    \"instance\", \n    \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n) / on(node) group_left(cluster) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_status_capacity{resource=\"cpu\"}) * on(node) group_left(label_kubernetes_io_hostname, label_nodegroup) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup!~\"nodes-bigid-scanner|nodes-jobs\"})",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_kubernetes_io_hostname}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 2,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "Load Average",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:208",
          "format": "locale",
          "label": "",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:209",
          "format": "short",
          "label": "",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                90
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "15m",
        "frequency": "5m",
        "handler": 1,
        "message": "Please check the alerting instance(s).",
        "name": "[EKS] Disk Usage is CRITICAL alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          },
          {
            "uid": "srXoYKsWz"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Disk Usage per node",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 45
      },
      "hiddenSeries": false,
      "id": 86,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "label_replace(100 - ((node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"rootfs\",job=\"prometheus-node-exporter\"} * 100) / node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"rootfs\",job=\"prometheus-node-exporter\"}),\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\",\n  \"instance\",\n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname, label_nodegroup) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels)",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_kubernetes_io_hostname}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 90,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "Disk Usage",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:194",
          "format": "percent",
          "label": "",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:195",
          "format": "short",
          "label": "",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                80
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "15m",
        "frequency": "5m",
        "handler": 1,
        "name": "CPU Usage [BigData] alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000016"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 45
      },
      "hiddenSeries": false,
      "hideTimeOverride": false,
      "id": 88,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "label_replace(\n  (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100,\n  \"node\",\n  \"ip-$1-$2-$3-$4.ec2.internal\", \n  \"instance\", \n  \"([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}).([0-9]{1,3}):[0-9]{2,5}\"\n)  * on(node) group_left(label_kubernetes_io_hostname) avg without(label_node_kubernetes_io_exclude_from_external_load_balancers, label_alpha_service_controller_kubernetes_io_exclude_balancer, label_spotinst_io_node_lifecycle, instance) (kube_node_labels{label_nodegroup=\"nodes-bigdata\"})",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_kubernetes_io_hostname}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 80,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "CPU Utilization [BigData]",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:497",
          "format": "percent",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:498",
          "format": "short",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                100
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "15m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "30m",
        "frequency": "10m",
        "handler": 1,
        "message": "Non running pods amount exceeded threshold. A few possible options are:\n- Pods cannot be scheduled\n- Some deployments are misconfigured and pods are not able to start\nAn easy way to figure out what's going wrong is to run \"kubectl get pods | grep -v Running\" - which shows a list of all pods that are not currently in a running state",
        "name": "Non running pods amount exceeded threshold",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          },
          {
            "uid": "srXoYKsWz"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Sum of non running pods by cluster",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 53
      },
      "hiddenSeries": false,
      "id": 112,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) by (cluster,namespace)",
          "legendFormat": "{{cluster}} / namespace: {{namespace}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 100,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "Non Running Pods",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:714",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:715",
          "format": "short",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                50
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "15m",
        "frequency": "5m",
        "handler": 1,
        "message": "Too many image pull errors!",
        "name": "Image pull backoff/err count alert",
        "noDataState": "ok",
        "notifications": [
          {
            "uid": "000000071"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Sum of pending pods by cluster",
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 53
      },
      "hiddenSeries": false,
      "id": 223,
      "interval": "1m",
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": false,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": true,
          "expr": "sum(pods_image_pull_issues_counter:aggr_by_reason{cluster=~\"eks-prod-main-1\"})",
          "instant": false,
          "interval": "",
          "legendFormat": "Error",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 50,
          "visible": true
        }
      ],
      "timeFrom": "90m",
      "timeRegions": [],
      "title": "Image pull backoff/err count",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:714",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:715",
          "format": "short",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                0.85
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "15m",
        "frequency": "5m",
        "handler": 1,
        "message": "The kubernetes volume is filling up. You can get volume details by running:\nkubectl get pvc <pvc name> -o yaml",
        "name": "Kubernetes volume usage is above 80%",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "a9-D1uwWz"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit"
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 61
      },
      "hiddenSeries": false,
      "id": 191,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "kubelet_volume_stats_used_bytes{persistentvolumeclaim!~\"vmstorage-db-vmstorage-victoria-metrics-cluster-main.*\"} / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim!~\"vmstorage-db-vmstorage-victoria-metrics-cluster-main.*\"}",
          "interval": "",
          "legendFormat": "cluster: {{cluster}}, namespace: {{namespace}}, pvc: {{persistentvolumeclaim}}",
          "refId": "A"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 0.85,
          "visible": true
        }
      ],
      "timeFrom": "1h",
      "timeRegions": [],
      "title": "Kubernetes pv usage",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:632",
          "format": "percentunit",
          "logBase": 1,
          "max": "1",
          "show": true
        },
        {
          "$$hashKey": "object:633",
          "format": "short",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                15
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "keep_state",
        "for": "10m",
        "frequency": "5m",
        "handler": 1,
        "name": "% of Unhealthy Pods per Node alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "000000071"
          },
          {
            "uid": "srXoYKsWz"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "links": []
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 61
      },
      "hiddenSeries": false,
      "hideTimeOverride": false,
      "id": 90,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": "avg",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "8.4.4",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum(kube_pod_status_phase{phase=~\"Failed|Unknown\", namespace!=\"kubernetes-dashboard\", cluster=\"$cluster\"} * on(pod) group_left(node) kube_pod_info{cluster=\"$cluster\", pod_ip != \"\"}) by (node)  / sum(kube_pod_status_phase{namespace!=\"kubernetes-dashboard\", cluster=\"$cluster\"} * on(pod) group_left(node) kube_pod_info{cluster=\"$cluster\", pod_ip != \"\"}) by (node)  * on(node) group_left(label_nodegroup, label_kubernetes_io_hostname) kube_node_labels * 100",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{label_kubernetes_io_hostname}}",
          "refId": "A",
          "target": ""
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 15,
          "visible": true
        }
      ],
      "timeFrom": "15m",
      "timeRegions": [],
      "title": "% of Unhealthy Pods per Node",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:638",
          "format": "percent",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:639",
          "format": "short",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                0.2
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          },
          {
            "evaluator": {
              "params": [
                0.2
              ],
              "type": "gt"
            },
            "operator": {
              "type": "or"
            },
            "query": {
              "params": [
                "B",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "2h",
        "frequency": "30m",
        "handler": 1,
        "message": "There are more than 20% OD instances on the node-group, this could mean either an issue with SPOT, the Market is highly Volatile or We don't have enough instance types on the group",
        "name": "On-Demand Count per Node Group alert",
        "noDataState": "no_data",
        "notifications": [
          {
            "uid": "000000071"
          }
        ]
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Experimental",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 69
      },
      "id": 248,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max"
          ],
          "displayMode": "table",
          "placement": "right",
          "sortBy": "Last *",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "(sum(kube_node_labels{label_nodegroup=~\"nodes-default|nodes-workers\"}) by (cluster,label_nodegroup) - sum(kube_node_labels{label_spotinst_io_node_lifecycle=\"spot\",label_nodegroup=~\"nodes-default|nodes-workers\"}) by (cluster,label_nodegroup)) / (sum(kube_node_labels{label_nodegroup=~\"nodes-default|nodes-workers\"}) by (cluster,label_nodegroup))",
          "hide": false,
          "interval": "",
          "legendFormat": "[{{cluster}}]-[{{label_nodegroup}}]",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "(sum (kube_node_labels{label_nodegroup=~\"nodes-v2\"}) by (cluster,label_nodegroup) - sum(kube_node_labels{label_spotinst_io_node_lifecycle=\"spot\",label_nodegroup=~\"nodes-v2\"}) by (cluster,label_nodegroup)) / sum (kube_node_labels{label_nodegroup=~\"nodes-v2\"})",
          "hide": false,
          "interval": "",
          "legendFormat": "[{{cluster}}]-[{{label_nodegroup}}]",
          "refId": "B"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "op": "gt",
          "value": 0.2,
          "visible": true
        }
      ],
      "title": "On-Demand Count per Node Group",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 76
      },
      "id": 135,
      "panels": [
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    70
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsage",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "kiam-agent CPU is too high",
            "name": "Aggregated CPU Usage kiam-agent alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 71
          },
          "hiddenSeries": false,
          "id": 137,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"kiam-agent\"}) / sum(workload:container_resource_requests:aggr{workload=\"kiam-agent\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsage",
              "refId": "percentageUsage"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 70,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage kiam-agent",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:264",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:265",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "kiam-server CPU is too high",
            "name": "Aggregated CPU Usage kiam-server alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 71
          },
          "hiddenSeries": false,
          "id": 139,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"kiam-server\"}) / sum(workload:container_resource_requests:aggr{workload=\"kiam-server\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 100,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage kiam-server",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:354",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:355",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    300
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "kiam-server memory percentage is too high",
            "name": "Aggregated Memory Usage kiam-server alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "out of 100%\n",
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 79
          },
          "hiddenSeries": false,
          "id": 141,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"kiam-server\"}) / sum(workload:container_resource_requests:aggr{cluster=~\"eks.*main.*\",resource=\"memory\",container=\"kiam\",workload=\"kiam-server\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 300,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage percentage kiam-server",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:69",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:70",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "kiam-agent memory is too high",
            "name": "Aggregated Memory Usage percentage kiam-agent alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 79
          },
          "hiddenSeries": false,
          "id": 143,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"kiam-agent\"}) /\nsum(workload:container_resource_requests:aggr{cluster=~\"eks.*main.*\",resource=\"memory\",container=\"kiam\",workload=\"kiam-agent\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 100,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage percentage kiam-agent",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:166",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:167",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.3
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "F",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "there are more than 30% of unavailable replicas of kiam-server",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 15,
            "x": 4,
            "y": 87
          },
          "hiddenSeries": false,
          "id": 145,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas{cluster=\"eks-prod-main-1\", deployment=\"kiam-server\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Current",
              "refId": "A",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", deployment=\"kiam-server\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "B",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"kiam-server\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Unavailable",
              "refId": "C",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_updated{cluster=\"eks-prod-main-1\", deployment=\"kiam-server\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Updated",
              "refId": "D",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"kiam-server\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "E",
              "step": 30
            },
            {
              "expr": "(avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"kiam-server\"}) + 0.1) / avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"kiam-server\"})",
              "interval": "",
              "legendFormat": "precent of unavailable",
              "refId": "F"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0.3,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "kiam ALERTS",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 77
      },
      "id": 147,
      "panels": [
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    300
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "usagePercentageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    300
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "usagePercentageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "spotinst k8s controller memory it too high",
            "name": "Aggregated Memory Usage spotinst-kubernetes-cluster-controller alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              },
              {
                "uid": "srXoYKsWz"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 72
          },
          "hiddenSeries": false,
          "id": 149,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"spotinst-kubernetes-cluster-controller\"}) / sum(workload:container_resource_requests:aggr{container=\"spotinst-kubernetes-cluster-controller\", resource=\"memory\", workload=\"spotinst-kubernetes-cluster-controller\", cluster=~\"eks.*main.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "usagePercentageMain",
              "refId": "usagePercentageMain"
            },
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"spotinst-kubernetes-cluster-controller\"}) / sum(workload:container_resource_requests:aggr{container=\"spotinst-kubernetes-cluster-controller\", resource=\"memory\", workload=\"spotinst-kubernetes-cluster-controller\", cluster=~\"eks.*mgmt.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "usagePercentageMgmt",
              "refId": "usagePercentageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 300,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage spotinst-kubernetes-cluster-controller",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:844",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:845",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    300
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsage",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "spotinst-kubernetes-cluster-controller CPU is too high",
            "name": "Aggregated CPU Usage spotinst-kubernetes-cluster-controller alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 72
          },
          "hiddenSeries": false,
          "id": 151,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"spotinst-kubernetes-cluster-controller\"}) / sum(workload:container_resource_requests:aggr{workload=\"spotinst-kubernetes-cluster-controller\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsage",
              "refId": "percentageUsage"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 300,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage spotinst-kubernetes-cluster-controller",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:938",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:939",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.3
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "F",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "spotinst-kubernetes-cluster-controller too manny unavailable deployments",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 15,
            "x": 4,
            "y": 80
          },
          "hiddenSeries": false,
          "id": 153,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas{cluster=\"eks-prod-main-1\", deployment=\"spotinst-kubernetes-cluster-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Current",
              "refId": "A",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", deployment=\"spotinst-kubernetes-cluster-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "B",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"spotinst-kubernetes-cluster-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Unavailable",
              "refId": "C",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_updated{cluster=\"eks-prod-main-1\", deployment=\"spotinst-kubernetes-cluster-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Updated",
              "refId": "D",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"spotinst-kubernetes-cluster-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "E",
              "step": 30
            },
            {
              "expr": "(avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"spotinst-kubernetes-cluster-controller\"})+0.1)/avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"spotinst-kubernetes-cluster-controller\"})",
              "interval": "",
              "legendFormat": "precent of unavailable",
              "refId": "F"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0.3,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "spotinst-kubernetes-cluster-controller ALERTS",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 78
      },
      "id": 155,
      "panels": [
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    200
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    200
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "kube-state-metric memory is too high",
            "name": "Aggregated Memory Usage percentage kube-state-metrics alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 73
          },
          "hiddenSeries": false,
          "id": 157,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"kube-state-metrics\"}) /\nsum(workload:container_resource_requests:aggr{container=\"kube-state-metrics\",cluster=~\"eks.*main.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"kube-state-metrics\"}) /\nsum(workload:container_resource_requests:aggr{container=\"kube-state-metrics\",cluster=~\"eks.*mgmt.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 200,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage percentage kube-state-metrics",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:822",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:823",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    70
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    70
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "kube-state-metrics CPU is too high",
            "name": "Aggregated CPU Usage kube-state-metric alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 73
          },
          "hiddenSeries": false,
          "id": 159,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"kube-state-metrics\"}) / sum(workload:container_resource_requests:aggr{workload=\"kube-state-metrics\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"kube-state-metrics\"}) / sum(workload:container_resource_requests:aggr{workload=\"kube-state-metrics\", cluster=~\"eks.*mgmt.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 70,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage kube-state-metric",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:732",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:733",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "C",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "There are too many unavailable deployments in kube-state-metrics",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 81
          },
          "hiddenSeries": false,
          "id": 161,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas{cluster=\"eks-prod-main-1\",namespace=\"monitoring\", deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Current",
              "refId": "A",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", deployment=\"kube-state-metrics\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "B",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", namespace=\"monitoring\",deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Unavailable",
              "refId": "C",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_updated{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Updated",
              "refId": "D",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"kube-state-metrics\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "E",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_hpa_spec_max_replicas{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\" })",
              "interval": "",
              "legendFormat": "HPA Pod Limit Max",
              "refId": "F"
            },
            {
              "expr": "avg without(instance, pod) (kube_statefulset_status_replicas{cluster=\"eks-prod-main-1\", statefulset=\"kube-state-metrics\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "G",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_statefulset_replicas{cluster=\"eks-prod-main-1\", statefulset=\"kube-state-metrics\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "H",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_daemonset_status_number_available{cluster=\"eks-prod-main-1\", daemonset=\"kube-state-metrics\"})",
              "interval": "",
              "legendFormat": "Available",
              "refId": "J"
            },
            {
              "expr": "avg without(instance, pod) (kube_daemonset_status_number_unavailable{cluster=\"eks-prod-main-1\", daemonset=\"kube-state-metrics\"})",
              "interval": "",
              "legendFormat": "Unavailable",
              "refId": "K"
            },
            {
              "expr": "avg without(instance, pod) (kube_daemonset_status_desired_number_scheduled{cluster=\"eks-prod-main-1\", daemonset=\"kube-state-metrics\"})",
              "interval": "",
              "legendFormat": "Desired",
              "refId": "M"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    50
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "M",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "sum"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "keep_state",
            "for": "15m",
            "frequency": "5m",
            "handler": 1,
            "message": "---TEST---\nAlerted pod(s) have been restarted a total of more then 50 times in the past 15 minutes\nPossible mitigation steps:\n1. Review service workload at https://grafana.fiverr-gw.com/d/rNVbvyAGz/kubernetes-workload?orgId=1\n    if resources usage are too close to the limit, consider raising the requests and limits accordingly. if they are already high, consider searching for memory leaks.\n2. Review pod logs",
            "name": "Pod Restarts",
            "noDataState": "keep_state",
            "notifications": []
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "sum of restarts by pod",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 81
          },
          "hiddenSeries": false,
          "id": 214,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "total",
            "sortDesc": true,
            "total": true,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "kube_pod_container_status_restarts_total:idelta_1m:aggr_by_container",
              "interval": "",
              "legendFormat": "{{container}}",
              "refId": "M"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 50,
              "visible": true
            }
          ],
          "timeFrom": "2h",
          "timeRegions": [],
          "title": "Pod Restarts",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 89
          },
          "hiddenSeries": false,
          "id": 225,
          "interval": "1m",
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "avg",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": true,
              "expr": "sum(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\",namespace=\"default\",container!=\"fiverr-inbox\"}[5m]) * 60 * 5 > 0) by (container)",
              "instant": false,
              "interval": "",
              "legendFormat": "{{container}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": "30m",
          "timeRegions": [],
          "title": "Pod Restart Rate",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.8
                  ],
                  "type": "lt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "B",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    0.8
                  ],
                  "type": "lt"
                },
                "operator": {
                  "type": "or"
                },
                "query": {
                  "params": [
                    "O",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "5m",
            "handler": 1,
            "message": "There is a 20% drop of available replicas comparing to 30Minutes ago\nThis could point to a production issue, possible causes:\n1. Spot Mark Volatility\n2. Database issue\n3. Code/Deploy issue\n4. Keda/Hpa scalers issue",
            "name": "Deployment/Roilout Availability Tracker alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              },
              {
                "uid": "srXoYKsWz"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": [],
              "unit": "percentunit"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 89
          },
          "hiddenSeries": false,
          "id": 230,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": false,
            "min": true,
            "rightSide": true,
            "show": true,
            "sort": "current",
            "sortDesc": false,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", namespace=\"default\"})) / sum(avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", namespace=\"default\"})) offset 30m",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available Deployment pods over comparing to 30m ago",
              "refId": "B",
              "step": 30
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "sum(avg without(instance, pod) (rollout_info_replicas_available{cluster=\"eks-prod-main-1\"})) / sum(avg without(instance, pod) (rollout_info_replicas_available{cluster=\"eks-prod-main-1\"})) offset 30m",
              "format": "time_series",
              "hide": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available Rollout pods over comparing to 30m ago",
              "refId": "O",
              "step": 30
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "lt",
              "value": 0.8,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Deployment/Roilout Availability Tracker",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:1231",
              "format": "percentunit",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:1232",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "kube-state-metric ALERTS",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 79
      },
      "id": 163,
      "panels": [
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    105
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    105
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "k8s consul agent's memory usage percentage is too high",
            "name": "Aggregated Memory Usage consul alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 162
          },
          "hiddenSeries": false,
          "id": 165,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"k8s-consul-agent\"}) / sum(workload:container_resource_requests:aggr{container=\"consul\", cluster=~\"eks.*main.*\", resource=\"memory\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"k8s-consul-agent\"}) / sum(workload:container_resource_requests:aggr{container=\"consul\", cluster=~\"eks.*mgmt.*\", resource=\"memory\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 105,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage consul",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:478",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:479",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    105
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    105
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "k8s consul agent's CPU is too high",
            "name": "Aggregated CPU Usage coredns alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 162
          },
          "hiddenSeries": false,
          "id": 167,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"k8s-consul-agent\"}) / sum(workload:container_resource_requests:aggr{workload=\"k8s-consul-agent\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"k8s-consul-agent\"}) / sum(workload:container_resource_requests:aggr{workload=\"k8s-consul-agent\", cluster=~\"eks.*mgmt.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 105,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage consul",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:572",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:573",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    10
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "A",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "too manny unavailable daemonsets of  k8s-consul-agent",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 15,
            "x": 4,
            "y": 170
          },
          "hiddenSeries": false,
          "id": 169,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_daemonset_status_number_ready{cluster=\"eks-prod-main-1\", daemonset=\"k8s-consul-agent\"})",
              "interval": "",
              "legendFormat": "Ready",
              "refId": "I"
            },
            {
              "expr": "avg without(instance, pod) (kube_daemonset_status_number_available{cluster=\"eks-prod-main-1\", daemonset=\"k8s-consul-agent\"})",
              "interval": "",
              "legendFormat": "Available",
              "refId": "J"
            },
            {
              "expr": "avg without(instance, pod) (kube_daemonset_status_number_unavailable{cluster=\"eks-prod-main-1\", daemonset=\"k8s-consul-agent\"})",
              "interval": "",
              "legendFormat": "Unavailable",
              "refId": "K"
            },
            {
              "expr": "avg without(instance, pod) (kube_daemonset_status_desired_number_scheduled{cluster=\"eks-prod-main-1\", daemonset=\"k8s-consul-agent\"})",
              "interval": "",
              "legendFormat": "Desired",
              "refId": "M"
            },
            {
              "expr": "((avg without(instance, pod) (kube_daemonset_status_number_unavailable{cluster=\"eks-prod-main-1\", daemonset=\"k8s-consul-agent\"}) + 1)/avg without(instance, pod) (kube_daemonset_status_number_available{cluster=\"eks-prod-main-1\", daemonset=\"k8s-consul-agent\"})) *100",
              "interval": "",
              "legendFormat": "precent of unavailable",
              "refId": "A"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 10,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "consul ALERTS",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 80
      },
      "id": 179,
      "panels": [
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    300
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    300
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "prometheus memory is too high",
            "name": "Aggregated Memory Usage percentage prometheus alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 178
          },
          "hiddenSeries": false,
          "id": 181,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"prometheus-k8s-operator\"}) / sum(workload:container_resource_requests:aggr{cluster=~\"eks.*main.*\",resource=\"memory\",workload=\"prometheus-k8s-operator\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"prometheus-k8s-operator\"}) / sum(workload:container_resource_requests:aggr{cluster=~\"eks.*mgmt.*\",resource=\"memory\",workload=\"prometheus-k8s-operator\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 300,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage percentage prometheus",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:822",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:823",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "15m",
            "frequency": "1m",
            "handler": 1,
            "message": "cpu usage of prometheus is too high",
            "name": "Aggregated CPU Usage coredns alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 178
          },
          "hiddenSeries": false,
          "id": 183,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"prometheus-k8s-operator\"}) / sum(workload:container_resource_requests:aggr{workload=\"prometheus-k8s-operator\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"prometheus-k8s-operator\"}) / sum(workload:container_resource_requests:aggr{workload=\"prometheus-k8s-operator\", cluster=~\"eks.*mgmt.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 100,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage prometheus",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:732",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:733",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.3
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "L",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "There are more than 30 precent of unavailable replicas of prometheus-k8s-operator\"",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 8,
            "w": 16,
            "x": 3,
            "y": 186
          },
          "hiddenSeries": false,
          "id": 185,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Current",
              "refId": "A",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "B",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Unavailable",
              "refId": "C",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_updated{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Updated",
              "refId": "D",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "E",
              "step": 30
            },
            {
              "expr": " (avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})) + 0.1 / avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"prometheus-k8s-operator\"})",
              "instant": true,
              "interval": "",
              "legendFormat": "precent of unavailable",
              "refId": "L"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0.3,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "prometheus k8s operator ALERTS",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 81
      },
      "id": 194,
      "panels": [
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    200
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "traefik-ingress-controller's memory usage percentage is too high",
            "name": "Aggregated Memory Usage traefik alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 195
          },
          "hiddenSeries": false,
          "id": 196,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"traefik-ingress-controller\"}) /\nsum(workload:container_resource_requests:aggr{container=\"traefik-ingress-controller\",cluster=~\"eks.*main.*\",resource=\"memory\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 200,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage traefik-ingress-controller",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:478",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:479",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "traefik-ingress-controller's CPU is too high",
            "name": "Aggregated CPU Usage traefik-ingress-controller",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 195
          },
          "hiddenSeries": false,
          "id": 198,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"traefik-ingress-controller\"}) / sum(workload:container_resource_requests:aggr{workload=\"traefik-ingress-controller\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 100,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage traefik-ingress-controller",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:572",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:573",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    3000
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "A",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "Traefik has too many open connections",
            "name": "Traefik open connections alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              },
              {
                "uid": "srXoYKsWz"
              }
            ]
          },
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 203
          },
          "id": 227,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "exemplar": true,
              "expr": "sum(traefik_entrypoint_open_connections{cluster=~\"eks.*main.*\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "Traefik active connections",
              "refId": "A"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "op": "gt",
              "value": 3000,
              "visible": true
            }
          ],
          "title": "Traefik open connections",
          "type": "timeseries"
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.3
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "F",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "traefik-ingress-controller too manny unavailable deployments",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              },
              {
                "uid": "srXoYKsWz"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 15,
            "x": 0,
            "y": 211
          },
          "hiddenSeries": false,
          "id": 202,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas{cluster=\"eks-prod-main-1\", deployment=\"traefik-ingress-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Current",
              "refId": "A",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", deployment=\"traefik-ingress-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "B",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"traefik-ingress-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Unavailable",
              "refId": "C",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_updated{cluster=\"eks-prod-main-1\", deployment=\"traefik-ingress-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Updated",
              "refId": "D",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"traefik-ingress-controller\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "E",
              "step": 30
            },
            {
              "expr": "(avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"traefik-ingress-controller\"})+0.1)/avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"traefik-ingress-controller\"})",
              "interval": "",
              "legendFormat": "precent of unavailable",
              "refId": "F"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0.3,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "traefik-ingress ALERTS",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 82
      },
      "id": 204,
      "panels": [
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    200
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    200
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "ingress-nginx's memory usage percentage is too high",
            "name": "Aggregated Memory Usage consul alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 219
          },
          "hiddenSeries": false,
          "id": 209,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"ingress-nginx\"}) /\nsum(workload:container_resource_requests:aggr{container=\"ingress-nginx\",cluster=~\"eks.*main.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_memory_working_set_bytes:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"ingress-nginx\"}) /\nsum(workload:container_resource_requests:aggr{workload=\"ingress-nginx\",cluster=~\"eks.*mgmt.*\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 200,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated Memory Usage ingress-nginx",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:478",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:479",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMain",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              },
              {
                "evaluator": {
                  "params": [
                    100
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "percentageUsageMgmt",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "ingress-nginx's CPU is too high, consider scaling the number of nodes and pods",
            "name": "Aggregated CPU Usage coredns alert",
            "noDataState": "no_data",
            "notifications": [
              {
                "uid": "000000071"
              }
            ]
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 219
          },
          "hiddenSeries": false,
          "id": 207,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*main.*\", workload=\"ingress-nginx\"}) / sum(workload:container_resource_requests:aggr{workload=\"ingress-nginx\", cluster=~\"eks.*main.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMain",
              "refId": "percentageUsageMain"
            },
            {
              "expr": "sum(workload:container_cpu_usage_seconds:rate5m:aggr{aggr=\"avg\", cluster=~\"eks.*mgmt.*\", workload=\"ingress-nginx\"}) / sum(workload:container_resource_requests:aggr{workload=\"ingress-nginx\", cluster=~\"eks.*mgmt.*\", resource=\"cpu\"}) * 100",
              "hide": false,
              "interval": "",
              "legendFormat": "percentageUsageMgmt",
              "refId": "percentageUsageMgmt"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 100,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Aggregated CPU Usage ingress-nginx",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:572",
              "format": "percent",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:573",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "alert": {
            "alertRuleTags": {},
            "conditions": [
              {
                "evaluator": {
                  "params": [
                    0.3
                  ],
                  "type": "gt"
                },
                "operator": {
                  "type": "and"
                },
                "query": {
                  "params": [
                    "F",
                    "5m",
                    "now"
                  ]
                },
                "reducer": {
                  "params": [],
                  "type": "avg"
                },
                "type": "query"
              }
            ],
            "executionErrorState": "alerting",
            "for": "5m",
            "frequency": "1m",
            "handler": 1,
            "message": "nginx-ingress too manny unavailable deployments",
            "name": "Replica Status alert",
            "noDataState": "no_data",
            "notifications": []
          },
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "description": "`Current` - The number of replicas that are currently running in the system\n\n`Available` - The number of replicas that are passing readiness probes\n\n`Unavailable` - The opposite of available\n\n`Desired` - The number of replicas you asked for in the deployment file\n\n`Updated` - The number of replicas that are up to date with the desired state",
          "editable": true,
          "error": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "grid": {},
          "gridPos": {
            "h": 7,
            "w": 15,
            "x": 5,
            "y": 227
          },
          "hiddenSeries": false,
          "id": 205,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "hideZero": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "8.4.4",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas{cluster=\"eks-prod-main-1\", deployment=\"ingress-nginx\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Current",
              "refId": "A",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_available{cluster=\"eks-prod-main-1\", deployment=\"ingress-nginx\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Available",
              "refId": "B",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"ingress-nginx\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Unavailable",
              "refId": "C",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_status_replicas_updated{cluster=\"eks-prod-main-1\", deployment=\"ingress-nginx\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Updated",
              "refId": "D",
              "step": 30
            },
            {
              "expr": "avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"ingress-nginx\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Desired",
              "refId": "E",
              "step": 30
            },
            {
              "expr": "(avg without(instance, pod) (kube_deployment_status_replicas_unavailable{cluster=\"eks-prod-main-1\", deployment=\"ingress-nginx\"})+0.1)/avg without(instance, pod) (kube_deployment_spec_replicas{cluster=\"eks-prod-main-1\", deployment=\"ingress-nginx\"})",
              "interval": "",
              "legendFormat": "precent of unavailable",
              "refId": "F"
            }
          ],
          "thresholds": [
            {
              "colorMode": "critical",
              "fill": true,
              "line": true,
              "op": "gt",
              "value": 0.3,
              "visible": true
            }
          ],
          "timeRegions": [],
          "title": "Replica Status",
          "tooltip": {
            "msResolution": true,
            "shared": true,
            "sort": 2,
            "value_type": "cumulative"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:294",
              "format": "locale",
              "label": "",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:295",
              "format": "short",
              "logBase": 1,
              "show": false
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "nginx-ingress ALERTS",
      "type": "row"
    }
  ],
  "refresh": false,
  "schemaVersion": 35,
  "style": "dark",
  "tags": [
    "SRE",
    "Devops",
    "Kubernetes",
    "Prometheus",
    "kubernetes"
  ],
  "templating": {
    "list": [
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(label_cluster)",
        "hide": 0,
        "includeAll": false,
        "label": "Cluster",
        "multi": false,
        "name": "cluster",
        "options": [],
        "query": {
          "query": "label_values(label_cluster)",
          "refId": "prometheus-cluster-Variable-Query"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values({label_cluster=\"$cluster\"}, label_nodegroup)",
        "hide": 0,
        "includeAll": false,
        "label": "Group",
        "multi": false,
        "name": "group",
        "options": [],
        "query": {
          "query": "label_values({label_cluster=\"$cluster\"}, label_nodegroup)",
          "refId": "prometheus-group-Variable-Query"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": ".*",
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values({label_nodegroup=\"$group\", label_cluster=\"$cluster\"}, label_kubernetes_io_hostname)",
        "hide": 0,
        "includeAll": false,
        "label": "Node",
        "multi": false,
        "name": "node",
        "options": [],
        "query": {
          "query": "label_values({label_nodegroup=\"$group\", label_cluster=\"$cluster\"}, label_kubernetes_io_hostname)",
          "refId": "prometheus-node-Variable-Query"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {
          "selected": false,
          "text": "5m",
          "value": "5m"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Interval",
        "multi": false,
        "name": "interval",
        "options": [
          {
            "selected": false,
            "text": "1m",
            "value": "1m"
          },
          {
            "selected": true,
            "text": "5m",
            "value": "5m"
          },
          {
            "selected": false,
            "text": "15m",
            "value": "15m"
          },
          {
            "selected": false,
            "text": "30m",
            "value": "30m"
          },
          {
            "selected": false,
            "text": "1h",
            "value": "1h"
          }
        ],
        "query": "1m,5m,15m,30m,1h",
        "queryValue": "",
        "skipUrlSync": false,
        "type": "custom"
      },
      {
        "allValue": ".*",
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "up{job=\"coredns\"}",
        "hide": 0,
        "includeAll": true,
        "label": "instance",
        "multi": false,
        "name": "instance",
        "options": [],
        "query": {
          "query": "up{job=\"coredns\"}",
          "refId": "prometheus-instance-Variable-Query"
        },
        "refresh": 1,
        "regex": ".*instance=\"(.*?)\".*",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-30m",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "15s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Kubernetes Monitoring",
  "uid": "vpCdrgLZz",
  "version": 528,
  "weekStart": "",
  "gnetId": 16759
}