Grafana dashboard: Consul Mesh Metrics
Description: Envoy stats

Rendering metrics issued by envoy-sidecar when you're using service mesh with consul connect.            
It provides statistics about listeners, clusters and grpc (if available)  
filterable by source_consul_datacenter / envoy_cluster / consul_destination_target.  

{
  "__inputs": [
    {
      "name": "DS_THANOS-MASTER",
      "label": "thanos-master",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": [],
  "__requires": [
    {
      "type": "panel",
      "id": "bargauge",
      "name": "Bar gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "8.3.2"
    },
    {
      "type": "panel",
      "id": "heatmap",
      "name": "Heatmap",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "text",
      "name": "Text",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "Envoy stats",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "iteration": 1647610663822,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 40,
      "panels": [],
      "title": "Infos",
      "type": "row"
    },
    {
      "gridPos": {
        "h": 5,
        "w": 2,
        "x": 0,
        "y": 1
      },
      "id": 57,
      "options": {
        "content": "<img src=https://cncf-branding.netlify.app/img/projects/envoy/stacked/color/envoy-stacked-color.png />",
        "mode": "html"
      },
      "pluginVersion": "8.3.2",
      "transparent": true,
      "type": "text"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "color": "dark-red",
                  "index": 1,
                  "text": "DOWN"
                },
                "1": {
                  "color": "dark-green",
                  "index": 0,
                  "text": "UP"
                }
              },
              "type": "value"
            }
          ],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 10,
        "x": 2,
        "y": 1
      },
      "id": 41,
      "interval": "1m",
      "options": {
        "displayMode": "basic",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true
      },
      "pluginVersion": "8.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_server_live{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} ",
          "format": "time_series",
          "instant": true,
          "interval": "1m",
          "legendFormat": "{{kubernetes_pod_name}}",
          "refId": "A"
        }
      ],
      "title": "server live",
      "transformations": [],
      "transparent": true,
      "type": "bargauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "color": "dark-red",
                  "index": 0
                }
              },
              "type": "value"
            }
          ],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "dark-green",
                "value": null
              }
            ]
          },
          "unit": "d"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "id": 42,
      "interval": "1m",
      "options": {
        "displayMode": "basic",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true
      },
      "pluginVersion": "8.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_server_days_until_first_cert_expiring{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} ",
          "format": "time_series",
          "instant": true,
          "interval": "1m",
          "legendFormat": "{{kubernetes_pod_name}}",
          "refId": "A"
        }
      ],
      "title": "days until first cert expiring",
      "transformations": [],
      "transparent": true,
      "type": "bargauge"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 36,
      "panels": [],
      "title": "Listener statistics",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "date"
            },
            "properties": [
              {
                "id": "unit",
                "value": "dateTimeAsLocal"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 9,
        "x": 0,
        "y": 7
      },
      "id": 38,
      "interval": "1m",
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": false
      },
      "pluginVersion": "8.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_listener_manager_lds_update_time{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"} ",
          "format": "table",
          "instant": true,
          "interval": "1m",
          "legendFormat": "{{kubernetes_pod_name}}",
          "refId": "A"
        }
      ],
      "title": "LDS update time",
      "transformations": [
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "pattern": "/kubernetes_pod_name|Value/"
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Value #A": "date",
              "kubernetes_pod_name": "k8s pod name"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "desc": true,
                "field": "date"
              }
            ]
          }
        }
      ],
      "transparent": true,
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Sum of all listeners ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 9,
        "y": 7
      },
      "id": 59,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "sum(envoy_listener_manager_total_listeners_active{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} ) by (consul_source_datacenter) ",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{local_cluster}}",
          "refId": "A"
        }
      ],
      "title": "Total listeners",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total listeners added (either via static config or LDS).\nTotal listeners modified (via LDS).\nTotal listeners removed (via LDS).\nTotal listeners stopped.\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 7
      },
      "id": 37,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "envoy_listener_manager_listener_added{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} != 0",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} : added",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "envoy_listener_manager_listener_modified{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"}  != 0",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} : modified",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "envoy_listener_manager_listener_removed{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"}  != 0",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} : removed",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "envoy_listener_manager_listener_stopped{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"}  != 0",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} : stopped",
          "refId": "D"
        }
      ],
      "title": "Listener manager",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total connections rejected due to configured overload actions\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 14
      },
      "id": 48,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "repeatDirection": "h",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "rate(envoy_listener_downstream_cx_overload_reject{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"}[$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} ",
          "refId": "A"
        }
      ],
      "title": "Connections rejected by overload",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total active connections\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "cps"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/.*destroy$/"
            },
            "properties": [
              {
                "id": "custom.drawStyle",
                "value": "line"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 14
      },
      "id": 53,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "rate(envoy_listener_downstream_cx_total{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"}[$__interval]) ",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} : all",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "rate(envoy_listener_downstream_cx_destroy{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"}[$__interval]) ",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} : destroy",
          "refId": "B"
        }
      ],
      "title": "Connections",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total connections that didn’t match any filter chain\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 14
      },
      "id": 52,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "repeatDirection": "h",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "rate(envoy_listener_no_filter_chain_match{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"}[$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} ",
          "refId": "A"
        }
      ],
      "title": "Connections no filter chain",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total connections rejected due to enforcement of global connection limit\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 21
      },
      "id": 49,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "repeatDirection": "h",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "rate(envoy_listener_downstream_global_cx_overflow{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"}[$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} ",
          "refId": "A"
        }
      ],
      "title": "Connections rejected by global limit",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Sockets currently undergoing listener filter processing\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 21
      },
      "id": 51,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "repeatDirection": "h",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "envoy_listener_downstream_pre_cx_active{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"}",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} ",
          "refId": "A"
        }
      ],
      "title": "Sockets filter processing",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Sockets that timed out during listener filter processing\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/listeners/stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 21
      },
      "id": 50,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "repeatDirection": "h",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "rate(envoy_listener_downstream_pre_cx_timeout{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"}[$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} : {{consul_source_service}} : {{envoy_http_conn_manager_prefix}} {{envoy_listener_address}} ",
          "refId": "A"
        }
      ],
      "title": "Sockets that timed out",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 28
      },
      "id": 12,
      "panels": [],
      "title": "Cluster statistics",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "date"
            },
            "properties": [
              {
                "id": "unit",
                "value": "dateTimeAsLocal"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 9,
        "x": 0,
        "y": 29
      },
      "id": 54,
      "interval": "1m",
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": false
      },
      "pluginVersion": "8.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_cluster_manager_cds_update_time{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\"} ",
          "format": "table",
          "instant": true,
          "interval": "1m",
          "legendFormat": "{{kubernetes_pod_name}}",
          "refId": "A"
        }
      ],
      "title": "CDS update time",
      "transformations": [
        {
          "id": "filterFieldsByName",
          "options": {
            "include": {
              "pattern": "/kubernetes_pod_name|Value/"
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Value #A": "date",
              "kubernetes_pod_name": "k8s pod name"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "desc": true,
                "field": "date"
              }
            ]
          }
        }
      ],
      "transparent": true,
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Sum of all clusters",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 9,
        "y": 29
      },
      "id": 60,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.3.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "sum(envoy_cluster_manager_active_clusters{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} ) by (consul_source_datacenter) ",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{local_cluster}}",
          "refId": "A"
        }
      ],
      "title": "Total clusters",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Current healthy/degraded/excluded members\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 29
      },
      "id": 32,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_cluster_membership_degraded{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\",consul_destination_target=~\"^$consul_destination_target$\"} != 0",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{envoy_cluster_name}} : degraded",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_cluster_membership_excluded{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\",consul_destination_target=~\"^$consul_destination_target$\"} != 0",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{envoy_cluster_name}} : excluded",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_cluster_membership_healthy{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\",consul_destination_target=~\"^$consul_destination_target$\"} != 0",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{envoy_cluster_name}} : healthy",
          "refId": "C"
        }
      ],
      "title": "Cluster membership status",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Number of currently ejected hosts\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats#outlier-detection-statistics",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 36
      },
      "id": 72,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_cluster_outlier_detection_ejections_active{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\"}",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} 🠒 {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Outlier detection ejected hosts",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Number of enforced consecutive 5xx ejections\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats#outlier-detection-statistics",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 36
      },
      "id": 73,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_outlier_detection_ejections_enforced_consecutive_5xx{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\"}[$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} 🠒 {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Outlier detection consecutive 5xx ejections",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Number of enforced consecutive  gateway failure ejections\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats#outlier-detection-statistics",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 36
      },
      "id": 74,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_outlier_detection_ejections_enforced_consecutive_gateway_failure{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\"}[$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} 🠒 {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Outlier detection consecutive gateway failure",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total active connections\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 44
      },
      "id": 28,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "envoy_cluster_upstream_cx_active{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\",consul_destination_target!=\"\",consul_destination_target=~\"^$consul_destination_target$\"}",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : rx",
          "refId": "A"
        }
      ],
      "title": "Connection active ",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total connections closed via HTTP/1.1 connection close header or HTTP/2 or HTTP/3 GOAWAY\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 44
      },
      "id": 65,
      "interval": "30s",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "sum",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "delta(envoy_cluster_upstream_cx_close_notify{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"}[1m])",
          "hide": false,
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}}",
          "refId": "A"
        }
      ],
      "title": "Connection closed",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total connection failures\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 53
      },
      "id": 61,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_connect_fail{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : failures",
          "refId": "A"
        }
      ],
      "title": "Connection failures",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total connection connect/idle timeouts\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 53
      },
      "id": 66,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_connect_timeout{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : connect timeout",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_idle_timeout{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : idle timeout",
          "refId": "B"
        }
      ],
      "title": "Connection timeouts",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total consecutive connection failures exceeding configured connection attempts\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 53
      },
      "id": 14,
      "interval": "30s",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "sum",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "delta(envoy_cluster_upstream_cx_connect_attempts_exceeded{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"}[1m])",
          "hide": false,
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}}",
          "refId": "A"
        }
      ],
      "title": "Connection consecutive failures",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total destroyed connections\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 60
      },
      "id": 67,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_destroy{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} ",
          "refId": "A"
        }
      ],
      "title": "Connection destroyed ",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total destroyed local connections\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 60
      },
      "id": 68,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_destroy_local{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : local",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_destroy_local_with_active_rq{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : local_with_active_rq",
          "refId": "B"
        }
      ],
      "title": "Connection destroyed local",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total destroyed remote connections\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 60
      },
      "id": 69,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_destroy_remote{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : remote",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_destroy_remote_with_active_rq{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : remote_with_active_rq",
          "refId": "B"
        }
      ],
      "title": "Connection destroyed remote",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total times connection not established due to no healthy hosts\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 67
      },
      "id": 70,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_none_healthy{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} ",
          "refId": "A"
        }
      ],
      "title": "Connection not established no healthy hosts ",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total connection protocol errors\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 67
      },
      "id": 71,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_cx_protocol_error{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"} [1m]) ",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} ",
          "refId": "A"
        }
      ],
      "title": "Connection protocol errors",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/outlier#arch-overview-outlier-detection",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "binBps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 74
      },
      "id": 55,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "rate(envoy_cluster_upstream_cx_tx_bytes_total{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\",consul_destination_target!=\"\",consul_destination_target=~\"^$consul_destination_target$\"} [$__interval:])",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : rx",
          "refId": "A"
        }
      ],
      "title": "Connection transmit bytes",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/outlier#arch-overview-outlier-detection",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "binBps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 74
      },
      "id": 64,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "rate(envoy_cluster_upstream_cx_rx_bytes_total{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"$local_cluster\",consul_destination_target!=\"\",consul_destination_target=~\"^$consul_destination_target$\"} [$__interval:])",
          "hide": false,
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : rx",
          "refId": "A"
        }
      ],
      "title": "Connection sent bytes",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "rate of Total requests\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 5,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 83
      },
      "id": 13,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_rq_total{consul_destination_target!=\"\",consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} [$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} -> {{consul_destination_target}}",
          "refId": "A"
        }
      ],
      "title": "Request total",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total active requests",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 83
      },
      "id": 10,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "sum(envoy_cluster_upstream_rq_active{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\"}) by (consul_source_datacenter,consul_source_service,kubernetes_pod_name,envoy_cluster_name)",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} 🠒 {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Request active ",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total requests that timed out waiting for a response\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 83
      },
      "id": 80,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_rq_timeout{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} [$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} -> {{consul_destination_target}}",
          "refId": "A"
        }
      ],
      "title": "Request timeouts",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total upstream requests completed\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 91
      },
      "id": 75,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_rq_completed{consul_destination_target!=\"\",consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} [$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} -> {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Request completed",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total requests cancelled before obtaining a connection pool connection\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 91
      },
      "id": 76,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_rq_cancelled{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} [$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} -> {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Request cancelled",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total requests closed due to max duration reached\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 91
      },
      "id": 77,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "increase(envoy_cluster_upstream_rq_max_duration_reached{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\"} [$__interval])",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} -> {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Request closed max duration reached",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total requests pending a connection pool connection\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 99
      },
      "id": 63,
      "interval": "30s",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "sum",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "delta(envoy_cluster_upstream_rq_pending_total{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"}[1m])",
          "hide": false,
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}}",
          "refId": "A"
        }
      ],
      "title": "Request pending",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total requests that were failed due to a connection pool connection failure or remote connection termination\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 99
      },
      "id": 78,
      "interval": "30s",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "sum",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "delta(envoy_cluster_upstream_rq_pending_failure_eject{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"}[1m])",
          "hide": false,
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}}",
          "refId": "A"
        }
      ],
      "title": "Request pending failure eject",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "Total requests that overflowed connection pool or requests (mainly for HTTP/2 and above) circuit breaking and were failed\nhttps://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 25,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 99
      },
      "id": 79,
      "interval": "30s",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "sum",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "delta(envoy_cluster_upstream_rq_pending_overflow{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"}[1m])",
          "hide": false,
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}}",
          "refId": "A"
        }
      ],
      "title": "Request pending overwflow",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 3,
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 107
      },
      "id": 47,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Last",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "rate(envoy_cluster_external_upstream_rq{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"}[$__interval]) != 0",
          "interval": "1m",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} 🠒 {{envoy_cluster_name}}",
          "refId": "A"
        }
      ],
      "title": "Ext upstream requests",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "cards": {},
      "color": {
        "cardColor": "#b4ff00",
        "colorScale": "sqrt",
        "colorScheme": "interpolateSpectral",
        "exponent": 0.5,
        "mode": "spectrum"
      },
      "dataFormat": "tsbuckets",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "https://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats",
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 107
      },
      "heatmap": {},
      "hideZeroBuckets": true,
      "highlightCards": true,
      "id": 43,
      "interval": "1m",
      "legend": {
        "show": true
      },
      "reverseYBuckets": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": true,
          "expr": "sum(rate(envoy_cluster_external_upstream_rq_time_bucket{consul_source_datacenter=~\"^$consul_source_datacenter$\",local_cluster=~\"^$local_cluster$\",consul_full_target!=\"\",consul_destination_target=~\"^$consul_destination_target$\",consul_destination_target!=\"\"}[$__interval])) by (le)",
          "format": "heatmap",
          "interval": "1m",
          "legendFormat": "{{le}}",
          "refId": "A"
        }
      ],
      "title": "Ext upstream req time : $consul_destination_target",
      "tooltip": {
        "show": true,
        "showHistogram": false
      },
      "transparent": true,
      "type": "heatmap",
      "xAxis": {
        "show": true
      },
      "yAxis": {
        "format": "dtdurationms",
        "logBase": 1,
        "show": true
      },
      "yBucketBound": "auto"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 114
      },
      "id": 16,
      "panels": [],
      "title": "gRPC statistics",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/grpc_stats_filter",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "cps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 115
      },
      "id": 18,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Max",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "rate(label_replace({consul_source_datacenter=~\"$consul_source_datacenter\",consul_full_target!=\"\",__name__=~\"envoy_cluster_grpc_.+_success\"},\"name_label\",\"$1\",\"__name__\", \"(.+)\") [$__interval:])",
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : {{envoy_grpc_bridge_method}}",
          "refId": "A"
        }
      ],
      "title": "Successful service/method calls",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/grpc_stats_filter",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "cps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 115
      },
      "id": 23,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Max",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "rate(label_replace({consul_source_datacenter=~\"$consul_source_datacenter\",consul_full_target!=\"\",__name__=~\"envoy_cluster_grpc_.+_failure\"},\"name_label\",\"$1\",\"__name__\", \"(.+)\") [$__interval:])",
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{consul_destination_target}} : {{envoy_grpc_bridge_method}}",
          "refId": "A"
        }
      ],
      "title": "Failed service/method calls",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/grpc_stats_filter",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "cps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 123
      },
      "id": 24,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Max",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "rate(label_replace({consul_source_datacenter=~\"$consul_source_datacenter\",consul_full_target!=\"\",__name__=~\"envoy_cluster_grpc_.+_request_message_count\"},\"name_label\",\"$1\",\"__name__\", \"(.+)\") [$__interval:])",
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{envoy_cluster_name}} : {{envoy_grpc_bridge_method}}",
          "refId": "A"
        }
      ],
      "title": "Request message count for service/method calls",
      "transparent": true,
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_THANOS-MASTER}"
      },
      "description": "https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/grpc_stats_filter",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "cps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 123
      },
      "id": 25,
      "interval": "1m",
      "options": {
        "legend": {
          "calcs": [
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Max",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_THANOS-MASTER}"
          },
          "exemplar": false,
          "expr": "rate(label_replace({consul_source_datacenter=~\"$consul_source_datacenter\",consul_full_target!=\"\",__name__=~\"envoy_cluster_grpc_.+_response_message_count\"},\"name_label\",\"$1\",\"__name__\", \"(.+)\") [$__interval:])",
          "interval": "",
          "legendFormat": "{{consul_source_datacenter}} : {{kubernetes_pod_name}} → {{envoy_cluster_name}} : {{envoy_grpc_bridge_method}}",
          "refId": "A"
        }
      ],
      "title": "Response message count for service/method calls",
      "transparent": true,
      "type": "timeseries"
    }
  ],
  "refresh": "5m",
  "schemaVersion": 33,
  "style": "dark",
  "tags": [
    "consul",
    "envoy"
  ],
  "templating": {
    "list": [
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_THANOS-MASTER}"
        },
        "definition": "label_values(envoy_cluster_version,consul_source_datacenter)",
        "hide": 0,
        "includeAll": false,
        "multi": false,
        "name": "consul_source_datacenter",
        "options": [],
        "query": {
          "query": "label_values(envoy_cluster_version,consul_source_datacenter)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_THANOS-MASTER}"
        },
        "definition": "label_values(envoy_server_live{consul_source_datacenter=\"$consul_source_datacenter\"},local_cluster)",
        "hide": 0,
        "includeAll": true,
        "multi": false,
        "name": "local_cluster",
        "options": [],
        "query": {
          "query": "label_values(envoy_server_live{consul_source_datacenter=\"$consul_source_datacenter\"},local_cluster)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_THANOS-MASTER}"
        },
        "definition": "label_values(envoy_cluster_upstream_rq_time_bucket{consul_source_datacenter=\"$consul_source_datacenter\",local_cluster=\"$local_cluster\"},consul_destination_target)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "consul_destination_target",
        "options": [],
        "query": {
          "query": "label_values(envoy_cluster_upstream_rq_time_bucket{consul_source_datacenter=\"$consul_source_datacenter\",local_cluster=\"$local_cluster\"},consul_destination_target)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Consul Mesh Metrics",
  "uid": "zUb5Sya7z",
  "version": 93,
  "weekStart": "",
  "gnetId": 15859
}