Grafana dashboard: kafka core overview
Description: kafka core metrics overview

kafka core metrics overview

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "5.4.3"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": "5.0.0"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "5.0.0"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "graphTooltip": 0,
  "id": 1,
  "iteration": 1653013165391,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 12,
      "panels": [],
      "title": "kafka 集群相关监控",
      "type": "row"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 2,
        "x": 0,
        "y": 1
      },
      "id": 15,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "count(kafka_server_KafkaServer_BrokerState{for=\"$cluster\"})",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "kafka 集群 Broker 数",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 8,
        "x": 2,
        "y": 1
      },
      "id": 19,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum(kafka_consumergroup_lag_new{for=\"$cluster\"})",
          "instant": true,
          "interval": "",
          "legendFormat": "集群总堆积数",
          "refId": "A"
        }
      ],
      "title": "kafka 总堆积数",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 2,
        "x": 10,
        "y": 1
      },
      "id": 28,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum(kafka_server_ReplicaManager_PartitionCount{for=\"$cluster\"})",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "partition 数量",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 2,
        "x": 12,
        "y": 1
      },
      "id": 33,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "count(kafka_topic_partition{for=\"$cluster\"})",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "kafka topics 数",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 0
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 2,
        "x": 14,
        "y": 1
      },
      "id": 29,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "max(kafka_controller_OfflinePartitionsCount{for=\"$cluster\"})",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "无 Leader 的 Partition 的数量",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 2,
        "x": 16,
        "y": 1
      },
      "id": 30,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum(kafka_controller_ActiveControllerCount{for=\"$cluster\"})",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "活跃的 Controller 的数量",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 0
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 18,
        "y": 1
      },
      "id": 31,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum(increase(kafka_server_ReplicaManager_IsrExpandsPerSec_count{for=\"$cluster\"})[2m])",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "ISR的扩大(expansion)速率",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 0
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 21,
        "y": 1
      },
      "id": 32,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum(increase(kafka_server_ReplicaManager_IsrShrinksPerSec_count{for=\"$cluster\"})[2m])",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "ISR的收缩(shrink)速率",
      "type": "stat"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 9,
        "w": 10,
        "x": 0,
        "y": 6
      },
      "id": 20,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum(irate(kafka_MessagesInPerSec{for=\"$cluster\",rate=\"Count\"}))",
          "interval": "",
          "legendFormat": "集群消息生产速率",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 集群消息生产速率",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 9,
        "w": 14,
        "x": 10,
        "y": 6
      },
      "id": 23,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_MessagesInPerSecPerTopic{for=\"$cluster\",rate=\"Count\"}))",
          "interval": "",
          "legendFormat": "{{topic}} -{{host}} 每秒写入topic数",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 每秒写入topic消息条数",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 12,
        "w": 12,
        "x": 0,
        "y": 15
      },
      "id": 17,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "min(kafka_consumergroup_delaytime{for=\"$cluster\"}) by (consumergroup,topic)",
          "interval": "",
          "legendFormat": "消费组:{{consumergroup}} - topic:{{topic}}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 发送未消费时间情况",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "s",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 12,
        "w": 12,
        "x": 12,
        "y": 15
      },
      "id": 18,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum(kafka_consumergroup_lag_new{for=\"$cluster\"}) by (consumergroup,topic)",
          "interval": "",
          "legendFormat": "消费组:{{consumergroup}} - topic:{{topic}}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 堆积情况",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 27
      },
      "id": 24,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_BytesInPerSec{for=\"$cluster\",rate=\"Count\"}))",
          "interval": "",
          "legendFormat": "{{host}} 每秒写入字节",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 集群每秒写入字节",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 27
      },
      "id": 25,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_BytesInPerSecPerTopic{for=\"$cluster\",rate=\"Count\"}))",
          "interval": "",
          "legendFormat": "{{topic}} -{{host}} 每秒写入topic字节",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 每秒写入topic字节",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 36
      },
      "id": 26,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_BytesOutPerSec{for=\"$cluster\",rate=\"Count\"}))",
          "interval": "",
          "legendFormat": "{{host}} 每秒读出字节",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 集群每秒读出字节",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "decbytes",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 36
      },
      "id": 45,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_BytesOutPerSecPerTopic{for=\"$cluster\",rate=\"Count\"}))",
          "interval": "",
          "legendFormat": "{{topic}} -{{host}} 每秒读出topic字节",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 每秒读出topic字节",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "decbytes",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 45
      },
      "id": 94,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fill": 1,
          "gridPos": {
            "h": 9,
            "w": 9,
            "x": 0,
            "y": 2
          },
          "id": 92,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_KafkaRequestHandlerPoolRequestHandlerAvgIdlePercent{rate=\"OneMinuteRate\",for=\"$cluster\"}",
              "interval": "",
              "legendFormat": "{{host}}:请求处理程序线程空闲情况",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "请求处理程序线程空闲情况",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:218",
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:219",
              "format": "arcsec",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fill": 1,
          "gridPos": {
            "h": 9,
            "w": 7,
            "x": 9,
            "y": 2
          },
          "id": 91,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum(kafka_RequestChannelResponseQueueSize{for=\"$cluster\"})",
              "interval": "",
              "legendFormat": "响应队列的大小",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "响应队列的大小",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:218",
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:219",
              "format": "arcsec",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fill": 1,
          "gridPos": {
            "h": 9,
            "w": 8,
            "x": 16,
            "y": 2
          },
          "id": 90,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum(kafka_RequestChannelRequestQueueSize{for=\"$cluster\"})",
              "interval": "",
              "legendFormat": "请求队列的大小",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "请求队列的大小",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:218",
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:219",
              "format": "arcsec",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "gridPos": {
            "h": 12,
            "w": 12,
            "x": 0,
            "y": 11
          },
          "id": 63,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_network_SocketServer_NetworkProcessorAvgIdlePercent{for=\"$cluster\"}",
              "hide": true,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_network_SocketServer_NetworkProcessorAvgIdlePercent{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "NetworkProcessorAvgIdlePercent",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:155",
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:156",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 11
          },
          "id": 51,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": true,
            "min": true,
            "rightSide": true,
            "show": false,
            "sort": "min",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum(irate(kafka_server_ReplicaManager_IsrShrinksPerSec_count{for=\"$cluster\"}[2m]))",
              "hide": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum(irate(kafka_server_ReplicaManager_IsrShrinksPerSec_count{for=\"$cluster\"}[2m]))",
              "interval": "",
              "legendFormat": "",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "ReplicaManager_IsrShrinksPerSec_count ISR每秒收缩次数",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:263",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:264",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 17
          },
          "id": 55,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": false,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum(irate(kafka_server_ReplicaManager_IsrExpandsPerSec_count{for=\"$cluster\"}[2m]))",
              "hide": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "sum(irate(kafka_server_ReplicaManager_IsrExpandsPerSec_count{for=\"$cluster\"}[2m]))",
              "instant": false,
              "interval": "",
              "legendFormat": "",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "ReplicaManager_IsrExpandsPerSec_count ISR每秒扩展次数",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:893",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:894",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 0,
            "y": 23
          },
          "id": 59,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": false,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_server_ReplicaManager_UnderReplicatedPartitions{for=\"$cluster\"}",
              "format": "time_series",
              "hide": true,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{host}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_server_ReplicaManager_UnderReplicatedPartitions{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Under Replicated Partitionis",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:975",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:976",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 23
          },
          "id": 57,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "irate(kafka_log_LogFlushStats_LogFlushRateAndTimeMs_count{for=\"$cluster\"}[2m])",
              "format": "time_series",
              "hide": true,
              "interval": "",
              "intervalFactor": 10,
              "legendFormat": "{{host}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "irate(kafka_log_LogFlushStats_LogFlushRateAndTimeMs_count{for=\"$cluster\"}[2m])",
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "LogFlushStats_LogFlushRateAndTimeMs_Count",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:5121",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:5122",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 28
          },
          "id": 53,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_log_LogFlushStats_LogFlushRateAndTimeMs{for=\"$cluster\"}",
              "format": "time_series",
              "hide": true,
              "interval": "",
              "intervalFactor": 10,
              "legendFormat": "{{host}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_log_LogFlushStats_LogFlushRateAndTimeMs{for=\"$cluster\"}",
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "LogFlushStats_LogFlushRateAndTimeMs(TP99)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:443",
              "format": "ms",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:444",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 28
          },
          "id": 61,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": false,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "irate(kafka_server_KafkaRequestHandlerPool_RequestHandlerAvgIdlePercent_count{for=\"$cluster\"}[2m])",
              "format": "time_series",
              "hide": true,
              "interval": "",
              "intervalFactor": 10,
              "legendFormat": "{{host}}",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "irate(kafka_server_KafkaRequestHandlerPool_RequestHandlerAvgIdlePercent_count{for=\"$cluster\"}[2m])",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "KafkaRequestHandlerPool_RequestHandlerAvgIdlePercent",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:1057",
              "format": "ns",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:1058",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "kafka 集群高阶监控",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 46
      },
      "id": 10,
      "panels": [
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "thresholds"
              },
              "mappings": [
                {
                  "options": {
                    "0": {
                      "index": 0,
                      "text": "UP"
                    },
                    "1": {
                      "index": 1,
                      "text": "DOWN"
                    }
                  },
                  "type": "value"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 1
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 5,
            "w": 2,
            "x": 0,
            "y": 2
          },
          "id": 3,
          "options": {
            "colorMode": "background",
            "graphMode": "area",
            "justifyMode": "auto",
            "orientation": "auto",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "textMode": "auto"
          },
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "sum(zk_up{job=\"ck-zk-olap\"}) != bool 3 or absent(sum(zk_up{job=\"ck-zk-olap\"}) != bool 3)",
              "instant": true,
              "interval": "",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "title": "zk 集群状态",
          "type": "stat"
        },
        {
          "fieldConfig": {
            "defaults": {
              "color": {
                "fixedColor": "rgb(31, 120, 193)",
                "mode": "continuous-GrYlRd"
              },
              "mappings": [
                {
                  "options": {
                    "0": {
                      "index": 1,
                      "text": "备"
                    },
                    "1": {
                      "index": 0,
                      "text": "主"
                    }
                  },
                  "type": "value"
                }
              ],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "super-light-green"
                  },
                  {
                    "color": "super-light-red",
                    "value": 1
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 5,
            "w": 4,
            "x": 2,
            "y": 2
          },
          "id": 8,
          "links": [],
          "maxDataPoints": 100,
          "options": {
            "colorMode": "background",
            "graphMode": "none",
            "justifyMode": "auto",
            "orientation": "horizontal",
            "reduceOptions": {
              "calcs": [
                "last"
              ],
              "fields": "",
              "values": false
            },
            "textMode": "value_and_name"
          },
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": false,
              "expr": "zk_server_leader{job=~\"ck-zk-olap\"}",
              "instant": true,
              "interval": "",
              "legendFormat": "{{zk_host}}",
              "refId": "A"
            }
          ],
          "title": "zk 节点类型",
          "type": "stat"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 6,
            "y": 2
          },
          "id": 5,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "max(zk_min_latency{job=~\"ck-zk-olap\"})",
              "interval": "",
              "legendFormat": "zk 处理最小时延",
              "refId": "A"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "max(zk_max_latency{job=~\"ck-zk-olap\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "zk 处理最大时延",
              "refId": "B"
            },
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "max(zk_avg_latency{job=~\"ck-zk-olap\"})",
              "hide": false,
              "interval": "",
              "legendFormat": "zk 处理平均延迟",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "zk 处理时延情况",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:218",
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:219",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 12,
            "y": 2
          },
          "id": 6,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "zk_approximate_data_size{job=~\"ck-zk-olap\"}",
              "interval": "",
              "legendFormat": "{{zk_host}} 的存储数据",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "zk 存储数据量",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:67",
              "format": "bytes",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:68",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 18,
            "y": 2
          },
          "id": 13,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "zk_watch_count{job=\"ck-zk-olap\"}",
              "interval": "",
              "legendFormat": "{{zk_host}} 节点 watch 数",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "zk 的 watch 数目",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": false,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:67",
              "format": "short",
              "logBase": 1,
              "show": true
            },
            {
              "$$hashKey": "object:68",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "zk 相关监控",
      "type": "row"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 47
      },
      "id": 35,
      "panels": [],
      "title": "kafka Topic、ConsumerGroup 监控",
      "type": "row"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 10,
        "w": 10,
        "x": 0,
        "y": 48
      },
      "id": 37,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "sum(kafka_consumergroup_lag_new{for=\"$cluster\",topic=\"$topic\",consumergroup=~\"$consumergroup\"})",
          "instant": true,
          "interval": "",
          "legendFormat": "{{$topic}}堆积数",
          "refId": "A"
        }
      ],
      "title": "topic 堆积数",
      "type": "stat"
    },
    {
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 10,
        "w": 2,
        "x": 10,
        "y": 48
      },
      "id": 40,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": false,
          "expr": "kafka_topic_partition{for=\"$cluster\",topic=\"$topic\"}",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "partition 数量",
      "type": "stat"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 10,
        "w": 12,
        "x": 12,
        "y": 48
      },
      "id": 42,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_MessagesInPerSecPerTopic{for=\"$cluster\",rate=\"Count\",topic=\"$topic\",host=~\"$host\"}))",
          "interval": "",
          "legendFormat": "{{topic}} -{{host}} 每秒写入topic数",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 每秒写入topic消息条数",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 11,
        "w": 12,
        "x": 0,
        "y": 58
      },
      "id": 44,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "min(kafka_consumergroup_delaytime{for=\"$cluster\",topic=\"$topic\",consumergroup=~\"$consumergroup\"}) by (consumergroup,topic)",
          "interval": "",
          "legendFormat": "消费组:{{consumergroup}} - topic:{{topic}}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka topic 发送未消费时间情况",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "s",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 11,
        "w": 12,
        "x": 12,
        "y": 58
      },
      "id": 39,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum(kafka_consumergroup_lag_new{for=\"$cluster\",topic=\"$topic\",consumergroup=~\"$consumergroup\"}) by (consumergroup,topic)",
          "instant": false,
          "interval": "",
          "legendFormat": "消费组:{{consumergroup}} - topic:{{topic}}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka  topic 堆积情况",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 69
      },
      "id": 47,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_BytesInPerSecPerTopic{for=\"$cluster\",rate=\"Count\",topic=\"$topic\",host=~\"$host\"}))",
          "interval": "",
          "legendFormat": "{{topic}} -{{host}} 每秒写入topic字节",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 每秒写入topic字节",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "decbytes",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 1,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 69
      },
      "id": 49,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "${DS_PROMETHEUS}",
          "exemplar": true,
          "expr": "sum without(instance)(rate(kafka_BytesOutPerSecPerTopic{for=\"$cluster\",rate=\"Count\",topic=\"$topic\",host=~\"$host\"}))",
          "interval": "",
          "legendFormat": "{{topic}} -{{host}} 每秒读出topic字节",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeRegions": [],
      "title": "kafka 每秒读出topic字节",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "mode": "time",
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:218",
          "format": "decbytes",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:219",
          "format": "arcsec",
          "logBase": 1,
          "show": true
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 77
      },
      "id": 65,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 4
          },
          "id": 89,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_Runtime_Uptime{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "kafka_Runtime_Uptime",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:401",
              "format": "ms",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:402",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 4
          },
          "id": 87,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_Threading_ThreadCount{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "kafka_Threading_ThreadCount",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:154",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:155",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 10
          },
          "id": 83,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_GarbageCollector_G1_Young_Generation_CollectionTime{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "GarbageCollector_G1_Young_Generation_CollectionTime",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:810",
              "format": "ns",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:811",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 10
          },
          "id": 85,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_GarbageCollector_G1_Old_Generation_CollectionCount{for=\"$cluster\"}",
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "GarbageCollector_G1_Old_Generation_CollectionCount",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:565",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:566",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 16
          },
          "id": 67,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_Memory_HeapMemoryUsage_used{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "Memory_HeapMemoryUsage_used",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:646",
              "format": "bytes",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:647",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 16
          },
          "id": 69,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_MemoryPool_G1_Survivor_Space_used{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "MemoryPool_G1_Survivor_Space_used",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:482",
              "format": "bytes",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:483",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 22
          },
          "id": 79,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "sort": "max",
            "sortDesc": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_MemoryPool_G1_Eden_Space_used{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "MemoryPool_G1_Eden_Space_used",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:728",
              "format": "bytes",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:729",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 22
          },
          "id": 81,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_GarbageCollector_G1_Young_Generation_CollectionCount{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "GarbageCollector_G1_Young_Generation_CollectionCount",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:564",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:565",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 28
          },
          "id": 71,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_MemoryPool_G1_Old_Gen_used{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "MemoryPool_G1_Old_Gen_used",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:400",
              "format": "bytes",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:401",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 28
          },
          "id": 73,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": true,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_jvm_GarbageCollector_G1_Old_Generation_CollectionTime{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "GarbageCollector_G1_Old_Generation_CollectionTime",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:318",
              "format": "ms",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:319",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 34
          },
          "id": 75,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_OperatingSystem_FreePhysicalMemorySize{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "kafka_OperatingSystem_FreePhysicalMemorySize",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:236",
              "format": "bytes",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:237",
              "format": "short",
              "logBase": 1,
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fieldConfig": {
            "defaults": {
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 34
          },
          "id": 77,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": true,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "datasource": "${DS_PROMETHEUS}",
              "exemplar": true,
              "expr": "kafka_OperatingSystem_OpenFileDescriptorCount{for=\"$cluster\"}",
              "instant": false,
              "interval": "",
              "legendFormat": "{{host}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeRegions": [],
          "title": "kafka_OperatingSystem_OpenFileDescriptorCount",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "mode": "time",
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:319",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            },
            {
              "$$hashKey": "object:320",
              "format": "short",
              "logBase": 1,
              "min": "0",
              "show": true
            }
          ],
          "yaxis": {
            "align": false
          }
        }
      ],
      "title": "kafka 集群 JVM 监控",
      "type": "row"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 33,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "kafka-test",
          "value": "kafka-test"
        },
        "definition": "label_values(kafka_server_KafkaServer_BrokerState, for)",
        "hide": 0,
        "includeAll": false,
        "label": "cluster",
        "multi": false,
        "name": "cluster",
        "options": [],
        "query": {
          "query": "label_values(kafka_server_KafkaServer_BrokerState, for)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {
          "selected": false,
          "text": "All",
          "value": "$__all"
        },
        "definition": "label_values(kafka_server_KafkaServer_BrokerState{for=~\"$cluster\"},host)",
        "hide": 0,
        "includeAll": true,
        "label": "host",
        "multi": false,
        "name": "host",
        "options": [],
        "query": {
          "query": "label_values(kafka_server_KafkaServer_BrokerState{for=~\"$cluster\"},host)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {
          "selected": false,
          "text": "timer",
          "value": "timer"
        },
        "definition": "label_values(kafka_topic_partition{for=~\"$cluster\"},topic)",
        "hide": 0,
        "includeAll": false,
        "label": "topic",
        "multi": false,
        "name": "topic",
        "options": [],
        "query": {
          "query": "label_values(kafka_topic_partition{for=~\"$cluster\"},topic)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {
          "selected": false,
          "text": "All",
          "value": "$__all"
        },
        "definition": "label_values(kafka_consumergroup_lag_new{for=~\"$cluster\"},consumergroup)",
        "hide": 0,
        "includeAll": true,
        "label": "consumergroup",
        "multi": false,
        "name": "consumergroup",
        "options": [],
        "query": {
          "query": "label_values(kafka_consumergroup_lag_new{for=~\"$cluster\"},consumergroup)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-5m",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "kafka core overview",
  "uid": "R6z113_7k",
  "version": 140,
  "weekStart": "",
  "gnetId": 16307,
  "description": "kafka core metrics overview"
}