Grafana dashboard: Cluster & Node - Health & Scaling
Description: Node distribution on a number of axis (AZ, lifecycle, etc) + various other scaling metrics

Uses Prometheus node metrics to visualize node distribution by region, AZ, instance type, ASG, Lifecycle (spot vs OD), etc.

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": [],
  "__requires": [
    {
      "type": "panel",
      "id": "bargauge",
      "name": "Bar gauge",
      "version": ""
    },
    {
      "type": "panel",
      "id": "gauge",
      "name": "Gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "8.4.2"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "Node distribution on a number of axis (AZ, lifecycle, etc) + various other scaling metrics",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "gnetId": 15159,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1657126263453,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 18,
      "title": "At a Glance",
      "type": "row"
    },
    {
      "description": "Use this chart to spot potential performance issues in a given AZ.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "#EAB839",
                "value": 90
              },
              {
                "color": "green",
                "value": 99
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 1
      },
      "id": 21,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "8.4.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "((sum by (cluster) (cluster_autoscaler_nodes_count{cluster=~\"$cluster\",state=\"ready\"})) / (sum by (cluster) (cluster_autoscaler_nodes_count)))*100",
          "hide": false,
          "interval": "",
          "legendFormat": "{{ cluster }}",
          "refId": "B"
        }
      ],
      "title": "% Healthy Nodes",
      "type": "gauge"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "Nodes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "id": 5,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.4.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "count by (cluster) (kubelet_running_pods{account=~\"$awsaccount\",cluster=~\"$cluster\"})",
          "interval": "",
          "legendFormat": "{{ cluster }}",
          "refId": "A"
        }
      ],
      "title": "Node Count (Total)",
      "type": "stat"
    },
    {
      "description": "We care about both extremes of too many young nodes (< 2 hours old, could indicate something wrong with nodes launching and getting healthy) and too many old nodes (old AMI's, cluster autoscaler should be finding cheaper instances).",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "orange",
                "value": null
              },
              {
                "color": "yellow",
                "value": 7200
              },
              {
                "color": "green",
                "value": 28800
              },
              {
                "color": "green",
                "value": 86400
              },
              {
                "color": "yellow",
                "value": 2592000
              },
              {
                "color": "red",
                "value": 7776030
              }
            ]
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 10
      },
      "id": 22,
      "options": {
        "displayMode": "gradient",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "text": {
          "titleSize": 11,
          "valueSize": 11
        }
      },
      "pluginVersion": "8.4.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "node_time_seconds - node_boot_time_seconds{cluster=~\"$cluster\"}",
          "format": "time_series",
          "hide": false,
          "instant": false,
          "interval": "",
          "legendFormat": "{{ instance }} ({{ cluster }})",
          "refId": "A"
        }
      ],
      "title": "Node Ages",
      "transformations": [],
      "type": "bargauge"
    },
    {
      "description": "(Only those over 10% to minimize noise)\nEach host has a set iptables conntrack table size limit based on instance type (size) because it requires RAM. If the number of entries in the table hit the limit, most connectivity for PODs in and out of the node will halt! We warn at 50%, and crit at 80% to ensure we act before the node breaks.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 40
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 10
      },
      "id": 10,
      "options": {
        "displayMode": "gradient",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "limit": 100,
          "values": true
        },
        "showUnfilled": true,
        "text": {
          "titleSize": 11,
          "valueSize": 11
        }
      },
      "pluginVersion": "8.4.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "((max by (kubernetes_node) (node_nf_conntrack_entries{job=\"kubernetes-service-endpoints\",cluster=~\"$cluster\",account=~\"$awsaccount\"})) / (max by (kubernetes_node) (node_nf_conntrack_entries_limit{job=\"kubernetes-service-endpoints\",cluster=~\"$cluster\",account=~\"$awsaccount\"})) * 100) >= 10",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "legendFormat": "{{ kubernetes_node }}",
          "refId": "A"
        }
      ],
      "title": "IPTables Conntrack Table % Usage Gauge",
      "transformations": [
        {
          "id": "reduce",
          "options": {
            "labelsToFields": false,
            "reducers": [
              "lastNotNull"
            ]
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "desc": true,
                "field": "Last *"
              }
            ]
          }
        }
      ],
      "type": "bargauge"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 18
      },
      "id": 12,
      "panels": [
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 25,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineStyle": {
                  "fill": "solid"
                },
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 19
          },
          "id": 2,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "count(kubelet_running_pods{account=~\"$awsaccount\",cluster=~\"$cluster\"}) by (cluster,topology_kubernetes_io_zone)",
              "interval": "",
              "legendFormat": "{{ topology_kubernetes_io_zone}} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Node Count by Availability Zone (AZ)",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 25,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineStyle": {
                  "fill": "solid"
                },
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 19
          },
          "id": 7,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "count by (cluster,node_type) (kubelet_running_pods{account=~\"$awsaccount\",cluster=~\"$cluster\"})",
              "interval": "",
              "legendFormat": "{{ node_type }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Node Count by ASG",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 25,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineStyle": {
                  "fill": "solid"
                },
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 27
          },
          "id": 4,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "count(kubelet_running_pods{account=~\"$awsaccount\",cluster=~\"$cluster\"}) by (cluster,topology_kubernetes_io_region)",
              "interval": "",
              "legendFormat": "{{ topology_kubernetes_io_region }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Node Count by Region",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 25,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineStyle": {
                  "fill": "solid"
                },
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 27
          },
          "id": 6,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "count(kubelet_running_pods{account=~\"$awsaccount\",cluster=~\"$cluster\"}) by (cluster,node_kubernetes_io_lifecycle)",
              "interval": "",
              "legendFormat": "{{ node_kubernetes_io_lifecycle }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Node Count by Lifecycle",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 25,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineStyle": {
                  "fill": "solid"
                },
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 35
          },
          "id": 3,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "count(kubelet_running_pods{account=~\"$awsaccount\",cluster=~\"$cluster\"}) by (cluster,node_kubernetes_io_instance_type)",
              "interval": "",
              "legendFormat": "{{ node_kubernetes_io_instance_type }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Node Count by Instance Type",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineStyle": {
                  "fill": "solid"
                },
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "decimals": 0,
              "mappings": [],
              "max": 100,
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "percent"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 35
          },
          "id": 24,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "((sum by (cluster) (cluster_autoscaler_nodes_count{cluster=~\"$cluster\",state=\"ready\"})) / (sum by (cluster) (cluster_autoscaler_nodes_count)))*100",
              "interval": "",
              "legendFormat": "{{ state }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "% Healthy Nodes (According to AWS cluster autoscaler)",
          "type": "timeseries"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 43
          },
          "id": 23,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "pluginVersion": "8.3.3",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "count by (cluster) ((node_time_seconds - node_boot_time_seconds{cluster=~\"$cluster\"}) < 7200)",
              "hide": false,
              "interval": "",
              "legendFormat": "{{ instance }} - {{ cluster }}",
              "refId": "A"
            }
          ],
          "title": "Count of Nodes < than 1 Hour Old",
          "transformations": [],
          "type": "timeseries"
        }
      ],
      "title": "Cluster Node Distribution & Scaling",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 14,
      "panels": [
        {
          "description": "Use this chart to spot potential performance issues.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 2,
                "pointSize": 1,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 21,
            "w": 12,
            "x": 0,
            "y": 20
          },
          "id": 20,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "min",
                "max",
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "pluginVersion": "8.3.3",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "avg by (cluster,kubernetes_node) (node_load5{cluster=~\"$cluster\"})",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ kubernetes_node }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "5 Minute Node Load Average by Instance",
          "type": "timeseries"
        },
        {
          "description": "Available memory is how many bytes of memory are available to use without swapping (vs Free memory being only completely unused / wasted memory, which is inefficient).",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 2,
                "pointSize": 1,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "decbytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 21,
            "w": 12,
            "x": 12,
            "y": 20
          },
          "id": 28,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "min",
                "max",
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "pluginVersion": "8.3.3",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "avg by (cluster,kubernetes_node) (node_memory_MemAvailable_bytes{cluster=~\"$cluster\"})",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ kubernetes_node }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Available Memory by Instance",
          "type": "timeseries"
        },
        {
          "description": "Use this chart to spot memory utilization issues on nodes.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 2,
                "pointSize": 1,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 21,
            "w": 12,
            "x": 0,
            "y": 41
          },
          "id": 25,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "min",
                "max",
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "pluginVersion": "8.3.3",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "avg by (cluster,kubernetes_node) (node_vmstat_oom_kill{cluster=~\"$cluster\"})",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ kubernetes_node }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "OOM Kills by Node",
          "type": "timeseries"
        },
        {
          "description": "Find any hot-spots of network activity.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 2,
                "pointSize": 1,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "bps"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 21,
            "w": 12,
            "x": 12,
            "y": 41
          },
          "id": 27,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "min",
                "max",
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "pluginVersion": "8.3.3",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "(avg by (cluster,kubernetes_node) (irate(node_network_transmit_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]))*8) + (avg by (cluster,kubernetes_node) (irate(node_network_receive_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]))*8)",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ kubernetes_node }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Combined Network Throughput (Megabits/sec, Send+Receive) by Instance",
          "type": "timeseries"
        },
        {
          "description": "Orphaned sockets indicate a client(s) behaving poorly (intentional or buggy) when closing TCP connections.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 2,
                "pointSize": 1,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 21,
            "w": 12,
            "x": 12,
            "y": 62
          },
          "id": 26,
          "options": {
            "legend": {
              "calcs": [
                "mean",
                "min",
                "max",
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "pluginVersion": "8.3.3",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "avg by (cluster,kubernetes_node) (node_sockstat_TCP_orphan{cluster=~\"$cluster\"})",
              "format": "time_series",
              "instant": false,
              "interval": "",
              "legendFormat": "{{ kubernetes_node }} ({{ cluster }})",
              "refId": "A"
            }
          ],
          "title": "Orphaned TCP Sockets by Instance",
          "type": "timeseries"
        },
        {
          "description": "(Only those over 10% to minimize noise)\nEach host has a set iptables conntrack table size limit based on instance type (size) because it requires RAM. If the number of entries in the table hit the limit, most connectivity for PODs in and out of the node will halt! We warn at 50%, and crit at 80% to ensure we act before the node breaks.",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 21,
            "w": 12,
            "x": 12,
            "y": 83
          },
          "id": 9,
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "bottom",
              "sortBy": "Last *",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "${DS_PROMETHEUS}"
              },
              "exemplar": true,
              "expr": "((max by (kubernetes_node) (node_nf_conntrack_entries{job=\"kubernetes-service-endpoints\",cluster=~\"$cluster\",account=~\"$awsaccount\"})) / (max by (kubernetes_node) (node_nf_conntrack_entries_limit{job=\"kubernetes-service-endpoints\",cluster=~\"$cluster\",account=~\"$awsaccount\"})) * 100)",
              "interval": "",
              "legendFormat": "{{ kubernetes_node }}",
              "refId": "A"
            }
          ],
          "title": "IPTables Conntrack Table % Usage",
          "type": "timeseries"
        }
      ],
      "title": "Cluster Node Health",
      "type": "row"
    }
  ],
  "refresh": false,
  "schemaVersion": 35,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "Prometheus",
          "value": "Prometheus"
        },
        "hide": 0,
        "includeAll": false,
        "label": "datasource",
        "multi": false,
        "name": "DS_PROMETHEUS",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "query_result(avg(kube_deployment_labels) by (account))",
        "hide": 0,
        "includeAll": false,
        "label": "AWS Account",
        "multi": true,
        "name": "awsaccount",
        "options": [],
        "query": {
          "query": "query_result(avg(kube_deployment_labels) by (account))",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "/.*(?:account)=\"([^\"]*).*/",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "query_result(avg(kube_deployment_labels{account=~\"$awsaccount\"}) by (cluster))",
        "hide": 0,
        "includeAll": false,
        "label": "Cluster",
        "multi": true,
        "name": "cluster",
        "options": [],
        "query": {
          "query": "query_result(avg(kube_deployment_labels{account=~\"$awsaccount\"}) by (cluster))",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "/.*(?:cluster)=\"([^\"]*).*/",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Cluster & Node - Health & Scaling",
  "uid": "zrP9cXD7k",
  "version": 5,
  "weekStart": ""
}