Grafana dashboard: ClickHouse
Description: ClickHouse internal exporter metrics

# ClickHouse

ClickHouse overview dashboard for internal exporter. [You can contribute here](https://github.com/weastur/grafana-dashboards)

## Requirements (tested, but may work on older versions too)

1. Grafana - 7 and up
1. ClickHouse - 21.3.4.25-lts and up
1. Prometheus
1. ClickHouse internal exporter ([how to](https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server_configuration_parameters-prometheus))

## Features

1. Info for: Queries (plus Select/Insert details), IO, Replicas, Merges, Zookeeper, Cache, Parts, Distributed tables, Background task pools
1. Show both trends and peaks (configurable)
1. `max_over_time` + PromQL subqueries to get all peaks
1. `$__rate_interval` to correct rate calc between points
1. `Max data points` set to 200 for all graph panel
1. Table with overall info for selected instances
1. Annotation for restart event


{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "7.5.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      },
      {
        "datasource": "${DS_PROMETHEUS}",
        "enable": true,
        "expr": "resets(ClickHouseAsyncMetrics_Uptime{instance=~\"$instance\"}[$__rate_interval])",
        "hide": false,
        "iconColor": "rgba(255, 96, 96, 1)",
        "name": "Restarts",
        "showIn": 0,
        "step": "60s",
        "tagKeys": "instance",
        "titleFormat": "Restart",
        "useValueForTime": false
      }
    ]
  },
  "description": "ClickHouse internal exporter metrics",
  "editable": true,
  "gnetId": 14192,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1617958425076,
  "links": [
    {
      "asDropdown": false,
      "icon": "external link",
      "includeVars": false,
      "keepTime": false,
      "tags": [],
      "targetBlank": true,
      "title": "ClickHouse monitoring docs",
      "tooltip": "",
      "type": "link",
      "url": "https://clickhouse.tech/docs/en/operations/monitoring/"
    }
  ],
  "panels": [
    {
      "datasource": "${DS_PROMETHEUS}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": null,
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "uptime"
            },
            "properties": [
              {
                "id": "unit",
                "value": "s"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 14,
      "options": {
        "frameIndex": 1,
        "showHeader": true
      },
      "pluginVersion": "7.5.2",
      "targets": [
        {
          "exemplar": false,
          "expr": "ClickHouseMetrics_VersionInteger",
          "format": "table",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        },
        {
          "exemplar": false,
          "expr": "ClickHouseMetrics_Revision",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "B"
        },
        {
          "exemplar": false,
          "expr": "ClickHouseAsyncMetrics_NumberOfTables",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "C"
        },
        {
          "exemplar": false,
          "expr": "ClickHouseAsyncMetrics_Uptime",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "D"
        },
        {
          "exemplar": false,
          "expr": "ClickHouseAsyncMetrics_NumberOfDatabases",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "E"
        }
      ],
      "title": "Nodes",
      "transformations": [
        {
          "id": "seriesToColumns",
          "options": {
            "byField": "instance"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time 1": true,
              "Time 10": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "Value #A 2": true,
              "Value #B 2": true,
              "Value #C 2": true,
              "Value #D 2": true,
              "Value #E 2": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "job 1": true,
              "job 10": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "job 9": true
            },
            "indexByName": {
              "Time 1": 6,
              "Time 10": 37,
              "Time 2": 9,
              "Time 3": 13,
              "Time 4": 16,
              "Time 5": 20,
              "Time 6": 23,
              "Time 7": 27,
              "Time 8": 30,
              "Time 9": 34,
              "Value #A 1": 1,
              "Value #A 2": 12,
              "Value #B 1": 2,
              "Value #B 2": 19,
              "Value #C 1": 3,
              "Value #C 2": 26,
              "Value #D 1": 5,
              "Value #D 2": 33,
              "Value #E 1": 4,
              "Value #E 2": 40,
              "__name__ 1": 7,
              "__name__ 10": 38,
              "__name__ 2": 10,
              "__name__ 3": 14,
              "__name__ 4": 17,
              "__name__ 5": 21,
              "__name__ 6": 24,
              "__name__ 7": 28,
              "__name__ 8": 31,
              "__name__ 9": 35,
              "instance": 0,
              "job 1": 8,
              "job 10": 39,
              "job 2": 11,
              "job 3": 15,
              "job 4": 18,
              "job 5": 22,
              "job 6": 25,
              "job 7": 29,
              "job 8": 32,
              "job 9": 36
            },
            "renameByName": {
              "Value #A 1": "version",
              "Value #B 1": "revision",
              "Value #C 1": "tables",
              "Value #D 1": "uptime",
              "Value #E 1": "databases"
            }
          }
        }
      ],
      "transparent": true,
      "type": "table"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "Total amount of memory (bytes) allocated by the server.",
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "hiddenSeries": false,
      "id": 29,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "maxDataPoints": 200,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.2",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": false,
          "expr": "ClickHouseMetrics_MemoryTracking",
          "interval": "",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Memory",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:176",
          "format": "bytes",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:177",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "Number of connections to TCP server (clients with native interface), also included server-server distributed query connections. \nNumber of connections to HTTP server",
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "hiddenSeries": false,
      "id": 73,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "maxDataPoints": 200,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.2",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": false,
          "expr": "ClickHouseMetrics_TCPConnection",
          "interval": "",
          "legendFormat": "tcp - {{instance}}",
          "refId": "A"
        },
        {
          "exemplar": false,
          "expr": "ClickHouseMetrics_HTTPConnection",
          "hide": false,
          "interval": "",
          "legendFormat": "http - {{instance}}",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Connections",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:176",
          "format": "short",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:177",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "Number of times the number of query processing threads was lowered due to slow reads.",
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 12,
        "x": 0,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 72,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "maxDataPoints": 200,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.2",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": false,
          "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
          "interval": "",
          "legendFormat": "peaks - {{instance}}",
          "refId": "A"
        },
        {
          "exemplar": false,
          "expr": "rate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\"}[$__rate_interval]) * $trends",
          "hide": false,
          "interval": "",
          "legendFormat": "trend - {{instance}}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Read backoff",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:176",
          "format": "cps",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:177",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_PROMETHEUS}",
      "description": "Number of reads from a file that were slow. This indicate system overload. Thresholds are controlled by read_backoff_* settings.",
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 12,
        "x": 12,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 71,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "maxDataPoints": 200,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.2",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "exemplar": false,
          "expr": "max_over_time( (irate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
          "interval": "",
          "legendFormat": "peaks - {{instance}}",
          "refId": "A"
        },
        {
          "exemplar": false,
          "expr": "rate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\"}[$__rate_interval]) * $trends",
          "hide": false,
          "interval": "",
          "legendFormat": "trend - {{instance}}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Slow reads",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:176",
          "format": "cps",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:177",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 18
      },
      "id": 12,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 19
          },
          "hiddenSeries": false,
          "id": 2,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Queries",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Same as Queries, but only for SELECT queries.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 19
          },
          "hiddenSeries": false,
          "id": 3,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "SELECT queries",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Same as Queries, but only for INSERT queries.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 27
          },
          "hiddenSeries": false,
          "id": 4,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "INSERT queries",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Same as Failed queries, but only for SELECT queries.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 27
          },
          "hiddenSeries": false,
          "id": 6,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Failed SELECT queries",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of failed queries.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 35
          },
          "hiddenSeries": false,
          "id": 5,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Failed queries",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times when memory limit exceeded for query.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 35
          },
          "hiddenSeries": false,
          "id": 85,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Queries memory limit",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of queries that are stopped and waiting due to 'priority' setting.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 43
          },
          "hiddenSeries": false,
          "id": 86,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Query preemted",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Same as Failed queries, but only for INSERT queries.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 43
          },
          "hiddenSeries": false,
          "id": 7,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Failed INSERT queries",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Avg queries latencies",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 51
          },
          "hiddenSeries": false,
          "id": 8,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "increase(ClickHouseProfileEvents_QueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Query latencies (avg)",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "µs",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": 1
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Avg SELECT queries latencies",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 51
          },
          "hiddenSeries": false,
          "id": 9,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "increase(ClickHouseProfileEvents_SelectQueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "SELECT query latencies (avg)",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "µs",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Avg INSERT queries latencies",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 59
          },
          "hiddenSeries": false,
          "id": 10,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "increase(ClickHouseProfileEvents_InsertQueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "INSERT query latencies (avg)",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "µs",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Queries",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 64,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of rows INSERTed to all tables.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 30
          },
          "hiddenSeries": false,
          "id": 62,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Inserted rows",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of bytes (uncompressed; for columns as they stored in memory) INSERTed to all tables.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 30
          },
          "hiddenSeries": false,
          "id": 65,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Inserted bytes",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 38
          },
          "hiddenSeries": false,
          "id": 66,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Delayed inserts",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 38
          },
          "hiddenSeries": false,
          "id": 67,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Rejected inserts",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Total number of milliseconds spent while the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 46
          },
          "hiddenSeries": false,
          "id": 69,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "increase(ClickHouseProfileEvents_DelayedInsertsMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Delayed inserts blocks (avg)",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "µs",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": 1
          }
        }
      ],
      "title": "Insert",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 125,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of data parts selected to read from a MergeTree table.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 27
          },
          "hiddenSeries": false,
          "id": 126,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Parts",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 27
          },
          "hiddenSeries": false,
          "id": 127,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Ranges",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of marks (index granules) selected to read from a MergeTree table.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 35
          },
          "hiddenSeries": false,
          "id": 128,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Marks",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of rows SELECTed from all tables.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 35
          },
          "hiddenSeries": false,
          "id": 129,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Rows",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 43
          },
          "hiddenSeries": false,
          "id": 130,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Bytes",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Select",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "id": 17,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times the 'lseek' function was called.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 33
          },
          "hiddenSeries": false,
          "id": 20,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_Seek{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_Seek{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "lseek",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "ops",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of files opened",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 33
          },
          "hiddenSeries": false,
          "id": 19,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Open Files",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "ops",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of reads (read/pread) from a file descriptor. Does not include sockets.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 41
          },
          "hiddenSeries": false,
          "id": 22,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Read from FD",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "ops",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 41
          },
          "hiddenSeries": false,
          "id": 23,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Write to FD",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "ops",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times the read (read/pread) from a file descriptor have failed.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 49
          },
          "hiddenSeries": false,
          "id": 24,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Read from FD failed",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "ops",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times the read (read/pread) from a file descriptor have failed.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 49
          },
          "hiddenSeries": false,
          "id": 25,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Read from FD failed",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "ops",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of bytes read from file descriptors. If the file is compressed, this will show the compressed data size.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 57
          },
          "hiddenSeries": false,
          "id": 21,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Read bytes from FD",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 57
          },
          "hiddenSeries": false,
          "id": 26,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Write bytes to FD",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 65
          },
          "hiddenSeries": false,
          "id": 51,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Read compressed bytes",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 65
          },
          "hiddenSeries": false,
          "id": 53,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Read uncompressed bytes",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 73
          },
          "hiddenSeries": false,
          "id": 52,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Compressed blocks",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "IO",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 22
      },
      "id": 56,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 31
          },
          "hiddenSeries": false,
          "id": 74,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_ReadonlyReplica",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Readonly replica",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times a data part was failed to download from replica of a ReplicatedMergeTree table.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 31
          },
          "hiddenSeries": false,
          "id": 58,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Replicated part failed fetches",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times a data part was downloaded from replica of a ReplicatedMergeTree table.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 39
          },
          "hiddenSeries": false,
          "id": 57,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Replicated part fetches",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times we prefer to download already merged part from replica of ReplicatedMergeTree table instead of performing a merge ourself (usually we prefer doing a merge ourself to save network traffic). This happens when we have not all source parts to perform a merge or when the data part is old enough.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 39
          },
          "hiddenSeries": false,
          "id": 60,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Replicated part fetches or merges",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times data parts of ReplicatedMergeTree tables were successfully merged.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 47
          },
          "hiddenSeries": false,
          "id": 59,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Replicated part merges",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times a data part that we wanted doesn't exist on any replica (even on replicas that are offline right now). That data parts are definitely lost. This is normal due to asynchronous replication (if quorum inserts were not enabled), when the replica on which the data part was written was failed and when it became online after fail it doesn't contain that data part.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 47
          },
          "hiddenSeries": false,
          "id": 61,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Replicated part data loss",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Replicas",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 76,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of launched background merges.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 29
          },
          "hiddenSeries": false,
          "id": 77,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Merge",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Rows read for background merges. This is the number of rows before merge.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 29
          },
          "hiddenSeries": false,
          "id": 78,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Merged rows",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Uncompressed bytes (for columns as they stored in memory) that was read for background merges. This is the number before merge.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 37
          },
          "hiddenSeries": false,
          "id": 79,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Merge uncompressed bytes",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Avg merge duration",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 37
          },
          "hiddenSeries": false,
          "id": 80,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "increase(ClickHouseProfileEvents_MergesTimeMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Merge duration (avg)",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "µs",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": 1
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of rows INSERTed to MergeTree tables.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 45
          },
          "hiddenSeries": false,
          "id": 81,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "MergeTree rows",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of blocks INSERTed to MergeTree tables. Each block forms a data part of level zero.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 45
          },
          "hiddenSeries": false,
          "id": 82,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "MergeTree blocks",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Uncompressed bytes (for columns as they stored in memory) INSERTed to MergeTree tables.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 53
          },
          "hiddenSeries": false,
          "id": 83,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "MergeTree uncompressed bytes",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Bytes written to filesystem for data INSERTed to MergeTree tables.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 53
          },
          "hiddenSeries": false,
          "id": 84,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "MergeTree compressed bytes",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 61
          },
          "hiddenSeries": false,
          "id": 123,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_DiskSpaceReservedForMerge",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Reserved space",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "bytes",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Merge",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 24
      },
      "id": 92,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 26
          },
          "hiddenSeries": false,
          "id": 102,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Uncompressed Cache Hits",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 26
          },
          "hiddenSeries": false,
          "id": 103,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Uncompressed Cache Misses",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 34
          },
          "hiddenSeries": false,
          "id": 104,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Mark Cache Hits",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 34
          },
          "hiddenSeries": false,
          "id": 105,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Mark Cache Misses",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Cache",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 25
      },
      "id": 90,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "The part is generating now, it is not in data_parts list.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 27
          },
          "hiddenSeries": false,
          "id": 114,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsTemporary",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Temporary",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "The part is in data_parts, but not used for SELECTs.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 27
          },
          "hiddenSeries": false,
          "id": 115,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsPreCommitted",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Pre commited",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "The part is in data_parts, but not used for SELECTs.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 35
          },
          "hiddenSeries": false,
          "id": 116,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsCommitted",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Commited",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 35
          },
          "hiddenSeries": false,
          "id": 117,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsOutdated",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Outdated",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Not active data part with identity refcounter, it is deleting right now by a cleaner.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 43
          },
          "hiddenSeries": false,
          "id": 118,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsDeleting",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Deleting",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Part was moved to another disk and should be deleted in own destructor.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 43
          },
          "hiddenSeries": false,
          "id": 119,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsDeleteOnDestroy",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Delete on destroy",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Wide parts.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 51
          },
          "hiddenSeries": false,
          "id": 120,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsWide",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Wide",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Compact parts.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 51
          },
          "hiddenSeries": false,
          "id": 122,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsCompact",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Compact",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "In-memory parts.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 59
          },
          "hiddenSeries": false,
          "id": 121,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_PartsInMemory",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "In-memory",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Parts",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 26
      },
      "id": 94,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 25
          },
          "hiddenSeries": false,
          "id": 112,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_DistributedSend",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Send",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 25
          },
          "hiddenSeries": false,
          "id": 113,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_DistributedFilesToInsert",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Files to insert",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 33
          },
          "hiddenSeries": false,
          "id": 106,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Distributed delayed inserts",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times the INSERT of a block to a Distributed table was rejected with 'Too many bytes' exception due to high number of pending bytes.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 33
          },
          "hiddenSeries": false,
          "id": 107,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Distributed rejected inserts",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Total number of milliseconds spent while the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 41
          },
          "hiddenSeries": false,
          "id": 108,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null as zero",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "increase(ClickHouseProfileEvents_DistributedDelayedInsertsMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[$__rate_interval])",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Distributed delayed inserts blocks (avg)",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "µs",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": 1
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Total count when distributed connection fails with retry",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 41
          },
          "hiddenSeries": false,
          "id": 109,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Distributed connection fail try",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Total count when distributed connection fails after all retries finished",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 49
          },
          "hiddenSeries": false,
          "id": 110,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Distributed connection fail at all",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 49
          },
          "hiddenSeries": false,
          "id": 111,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Distributed sync insertation timeout",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Distributed",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 27
      },
      "id": 88,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of active tasks in BackgroundProcessingPool (merges, mutations, or replication queue bookkeeping)",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 28
          },
          "hiddenSeries": false,
          "id": 95,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_BackgroundPoolTask",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "BackgroundPool task",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of active tasks in BackgroundFetchesPool",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 28
          },
          "hiddenSeries": false,
          "id": 96,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_BackgroundFetchesPoolTask",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "BackgroundFetchesPool task",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of active tasks in BackgroundProcessingPool for moves",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 36
          },
          "hiddenSeries": false,
          "id": 97,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_BackgroundMovePoolTask",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "BackgroundMovePool task",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 36
          },
          "hiddenSeries": false,
          "id": 98,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_BackgroundSchedulePoolTask",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "BackgroundSchedulePool task",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 44
          },
          "hiddenSeries": false,
          "id": 99,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_BackgroundBufferFlushSchedulePoolTask",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "BackgroundBufferFlushSchedulePool task",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 44
          },
          "hiddenSeries": false,
          "id": 100,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_BackgroundDistributedSchedulePoolTask",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "BackgroundDistributedSchedulePool task",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of active tasks in BackgroundProcessingPool for message streaming",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 52
          },
          "hiddenSeries": false,
          "id": 101,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_BackgroundMessageBrokerSchedulePoolTask",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "BackgroundMessageBrokerSchedulePool task",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Background pool",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": "${DS_PROMETHEUS}",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 28
      },
      "id": 28,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 29
          },
          "hiddenSeries": false,
          "id": 30,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Init",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 29
          },
          "hiddenSeries": false,
          "id": 31,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Transactions",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 37
          },
          "hiddenSeries": false,
          "id": 32,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "List",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 37
          },
          "hiddenSeries": false,
          "id": 33,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Create",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 45
          },
          "hiddenSeries": false,
          "id": 34,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Remove",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 45
          },
          "hiddenSeries": false,
          "id": 35,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Exists",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 53
          },
          "hiddenSeries": false,
          "id": 36,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Get",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 53
          },
          "hiddenSeries": false,
          "id": 37,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Set",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 61
          },
          "hiddenSeries": false,
          "id": 38,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Multi",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 61
          },
          "hiddenSeries": false,
          "id": 39,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Check",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 69
          },
          "hiddenSeries": false,
          "id": 40,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Close",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 69
          },
          "hiddenSeries": false,
          "id": 41,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Watch response",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "reqps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 77
          },
          "hiddenSeries": false,
          "id": 42,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "User Exceptions",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 77
          },
          "hiddenSeries": false,
          "id": 43,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Hardware Exceptions",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 85
          },
          "hiddenSeries": false,
          "id": 46,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Bytes received",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 85
          },
          "hiddenSeries": false,
          "id": 45,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Bytes sent",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "binBps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 93
          },
          "hiddenSeries": false,
          "id": 48,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_ZooKeeperSession",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Sessions",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of ephemeral nodes hold in ZooKeeper.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 93
          },
          "hiddenSeries": false,
          "id": 47,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_EphemeralNode",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Ephemeral Node",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of watches (event subscriptions) in ZooKeeper.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 101
          },
          "hiddenSeries": false,
          "id": 50,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_ZooKeeperWatch",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Watches",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of requests to ZooKeeper in fly.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 101
          },
          "hiddenSeries": false,
          "id": 49,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "ClickHouseMetrics_ZooKeeperRequest",
              "interval": "",
              "legendFormat": "{{instance}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Requests",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_PROMETHEUS}",
          "description": "Number of times an error happened while trying to remove ephemeral node. This is not an issue, because our implementation of ZooKeeper library guarantee that the session will expire and the node will be removed.",
          "fieldConfig": {
            "defaults": {},
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 109
          },
          "hiddenSeries": false,
          "id": 70,
          "legend": {
            "alignAsTable": true,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "maxDataPoints": 200,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "percentage": false,
          "pluginVersion": "7.5.2",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "exemplar": false,
              "expr": "max_over_time( (irate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks",
              "interval": "",
              "legendFormat": "peaks - {{instance}}",
              "refId": "A"
            },
            {
              "exemplar": false,
              "expr": "rate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\"}[$__rate_interval]) * $trends",
              "hide": false,
              "interval": "",
              "legendFormat": "trend - {{instance}}",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Remove ephemeral node failed",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": true,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "$$hashKey": "object:176",
              "format": "cps",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "$$hashKey": "object:177",
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Zookeeper",
      "type": "row"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 27,
  "style": "dark",
  "tags": [
    "ClickHouse",
    "DB"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "Prometheus",
          "value": "Prometheus"
        },
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": "DataSource",
        "multi": false,
        "name": "datasource",
        "options": [],
        "query": "prometheus",
        "queryValue": "",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_PROMETHEUS}",
        "definition": "label_values(ClickHouseAsyncMetrics_Uptime, instance)",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": "Node",
        "multi": true,
        "name": "instance",
        "options": [],
        "query": {
          "query": "label_values(ClickHouseAsyncMetrics_Uptime, instance)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 5,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "selected": false,
          "text": "Yes",
          "value": "1"
        },
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": "Show trends",
        "multi": false,
        "name": "trends",
        "options": [
          {
            "selected": true,
            "text": "Yes",
            "value": "1"
          },
          {
            "selected": false,
            "text": "No",
            "value": "null"
          }
        ],
        "query": "Yes : 1, No : null",
        "queryValue": "",
        "skipUrlSync": false,
        "type": "custom"
      },
      {
        "allValue": null,
        "current": {
          "selected": false,
          "text": "No",
          "value": "null"
        },
        "description": "Be aware of the points limit per timeseries for Grafana. This option may not work for a time range > 24h",
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": "Show peaks",
        "multi": false,
        "name": "peaks",
        "options": [
          {
            "selected": false,
            "text": "Yes",
            "value": "1"
          },
          {
            "selected": true,
            "text": "No",
            "value": "null"
          }
        ],
        "query": "Yes : 1, No : null",
        "queryValue": "",
        "skipUrlSync": false,
        "type": "custom"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "ClickHouse",
  "uid": "thEkJB_Mz",
  "version": 40
}