Grafana dashboard: Cluster Issues Dashboard
Description: Cluster Issues by groundcover


{
  "__inputs": [],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "8.1.8"
    },
    {
      "type": "datasource",
      "id": "postgres",
      "name": "PostgreSQL",
      "version": "1.0.0"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      },
      {
        "datasource": "${ds_prom}",
        "enable": true,
        "expr": "changes(groundcover_deployment_event_total_counter{namespace='$namespace'} > 0)",
        "iconColor": "semi-dark-green",
        "name": "Deployments",
        "rawQuery": "SELECT\n  extract(epoch from time_column) AS time,\n  text_column as text,\n  tags_column as tags\nFROM\n  metric_table\nWHERE\n  $__timeFilter(time_column)\n",
        "step": "10s",
        "tagKeys": "namespace,deployment_name,deployment_event_type"
      },
      {
        "datasource": "${ds_prom}",
        "enable": true,
        "expr": "changes(groundcover_container_state_error_counter{namespace='$namespace'})",
        "iconColor": "red",
        "name": "Container Errors",
        "rawQuery": "SELECT\n  extract(epoch from time_column) AS time,\n  text_column as text,\n  tags_column as tags\nFROM\n  metric_table\nWHERE\n  $__timeFilter(time_column)\n",
        "step": "10s",
        "tagKeys": "namespace,container_name,state,state_reason"
      }
    ]
  },
  "description": "Cluster Issues by groundcover",
  "editable": true,
  "gnetId": 15819,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1646071701959,
  "links": [],
  "panels": [
    {
      "datasource": "${ds_prom}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "links": [
            {
              "targetBlank": true,
              "title": "go to workload",
              "url": "d/lTP9tuo7Z/workload-overview-dashboard?var-namespace=${namespace}&var-workload=${__field.labels.workload_name}&var-ds_prom=${ds_prom}&var-ds_sql=${ds_sql}&var-ds_logs=${ds_logs}"
            }
          ],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "super-light-green",
                "value": null
              },
              {
                "color": "#d94354",
                "value": 0.1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 73,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.1.8",
      "targets": [
        {
          "datasource": "${ds_prom}",
          "exemplar": false,
          "expr": "topk(16, count by(workload_name) (\n    (\n    groundcover_http_error_counter{namespace='$namespace', role='server'}\n    or groundcover_sql_error_counter{namespace='$namespace', role='client'}\n    or groundcover_dns_error_counter{namespace='$namespace', role='client'}\n    )[$__range]\n   )\n   or \n   (group by(workload_name) (groundcover_container_info_total_counter{namespace='$namespace'}[$__range]) - 1)\n )",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "{{workload_name}}",
          "refId": "B"
        }
      ],
      "title": "Workload Status",
      "type": "stat"
    },
    {
      "datasource": "${ds_prom}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 5
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 5,
        "x": 0,
        "y": 7
      },
      "id": 77,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.1.8",
      "targets": [
        {
          "datasource": "${ds_prom}",
          "exemplar": true,
          "expr": "count(\n    (groundcover_http_error_counter{namespace='$namespace', role='server'}\n    or groundcover_sql_error_counter{namespace='$namespace', role='client'}\n    or groundcover_dns_error_counter{namespace='$namespace', role='client'})[$__range]\n)",
          "instant": false,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Number of Issues",
      "type": "stat"
    },
    {
      "datasource": "${ds_prom}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "semi-dark-red",
            "mode": "continuous-BlPu"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 5,
        "x": 5,
        "y": 7
      },
      "id": 79,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.1.8",
      "targets": [
        {
          "datasource": "${ds_prom}",
          "exemplar": true,
          "expr": "count(count(groundcover_container_info_total_counter{namespace='$namespace'}[$__range]) by (workload_name))",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Number of Workloads",
      "type": "stat"
    },
    {
      "datasource": "${ds_prom}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "semi-dark-red",
            "mode": "continuous-BlPu"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 5,
        "x": 10,
        "y": 7
      },
      "id": 78,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "auto"
      },
      "pluginVersion": "8.1.8",
      "targets": [
        {
          "datasource": "${ds_prom}",
          "exemplar": true,
          "expr": "count(count(groundcover_container_info_total_counter{namespace='$namespace'}[$__range]) by (pod_name))",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Number of Pods",
      "type": "stat"
    },
    {
      "datasource": "${ds_sql}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "left",
            "displayMode": "auto"
          },
          "mappings": [
            {
              "options": {
                "dns": {
                  "index": 2,
                  "text": "DNS"
                },
                "grpc": {
                  "index": 1,
                  "text": "gRPC"
                },
                "http": {
                  "index": 0,
                  "text": "HTTP"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Workload"
            },
            "properties": [
              {
                "id": "custom.width"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Type"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 96
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Status Code"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 115
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Resource"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 198
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Incidents"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 77
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Workload"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": true,
                    "title": "go to workload",
                    "url": "d/lTP9tuo7Z/workload-overview-dashboard?var-namespace=${namespace}&var-workload=${__data.fields.Workload}&var-ds_prom=${ds_prom}&var-ds_sql=${ds_sql}&var-ds_logs=${ds_logs}"
                  }
                ]
              },
              {
                "id": "color",
                "value": {
                  "fixedColor": "#4b72ad",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.displayMode",
                "value": "color-text"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 14,
        "w": 9,
        "x": 15,
        "y": 7
      },
      "id": 81,
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "8.1.8",
      "targets": [
        {
          "datasource": "${ds_sql}",
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "  select * from (\n    (select last(issue_id, time) as issueId, 'http' as issueType, last(response_status_code::text, time) as issueCode, last(role, time) as role, last(clustered_path, time) as resourceName, last(namespace, time) as namespace, last(workload_name, time) as workloadName, last(remote_service_name, time) as remoteServiceName, last(remote_namespace, time) as remoteNamespace, last(reason, time) as description, last(method, time) as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from http_issues \n    where protocol = 'http' and role = 'server' and $__timeFilter(\"time\") and namespace = '$namespace'\n    group by issue_id \n    order by lastSeen desc\n    limit 20)\n    union\n    (select last(issue_id, time) as issueId, 'grpc' as issueType, last(response_status_code::text, time) as issueCode, last(role, time) as role, last(clustered_path, time) as resourceName, last(namespace, time) as namespace, last(workload_name, time) as workloadName, last(remote_service_name, time) as remoteServiceName, last(remote_namespace, time) as remoteNamespace, last(reason, time) as description, last(method, time) as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from http_issues \n    where protocol = 'grpc' and role = 'server' and $__timeFilter(\"time\") and namespace = '$namespace'\n    group by issue_id \n    order by lastSeen desc\n    limit 20)\n    union\n    (select last(issue_id, time) as issueId, 'dns' as issueType, last(reason, time) as issueCode, last(role, time) as role, last(query_name, time) as resourceName, last(namespace, time) as namespace, last(workload_name, time) as workloadName, last(remote_service_name, time) as remoteServiceName, last(remote_namespace, time) as remoteNamespace, last(reason, time) as description, last(query_type, time) as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from dns_issues \n    where $__timeFilter(\"time\") and namespace = '$namespace' and role = 'client'\n    group by issue_id\n    order by lastSeen desc\n    limit 20)\n    union\n    (select last(issue_id, time) as issueId, 'container_state' as issueType, last(state::text, time) as issueCode, '' as role, last(container_name, time) as resourceName, last(namespace, time) as namespace, last(workload_name, time), '' as remoteServiceName , '' as remoteNamespace, last(exit_code::text, time) as description, 'Container Error' as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from container_state_issues \n    where $__timeFilter(\"time\") and namespace = '$namespace'\n    group by issue_id\n    order by lastSeen desc\n    limit 20)\n    union\n    (select last(issue_id, time) as issueId, 'sql' as issueType, last(response_status::text, time) as issueCode, last(role, time) as role, last(query_fingerprint, time) as resourceName, last(namespace, time) as namespace, last(workload_name, time) as workloadName, last(remote_service_name, time) as remoteServiceName, last(remote_namespace, time) as remoteNamespace, last(reason, time) as description, last(dialect, time) as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from sql_issues \n    where $__timeFilter(\"time\") and namespace = '$namespace' and role = 'client'\n    group by issue_id\n    order by lastSeen desc\n    limit 20)\n    union\n    (select last(issue_id, time) as issueId, 'kafka' as issueType, last(reason::text, time) as issueCode, last(role, time) as role, CONCAT(last(topic, time), ',', last(partition, time)) as resourceName, last(namespace, time) as namespace, last(workload_name, time) as workloadName, last(remote_service_name, time) as remoteServiceName, last(remote_namespace, time) as remoteNamespace, last(reason, time) as description, last(client_type, time) as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from kafka_issues \n    where $__timeFilter(\"time\") and namespace = '$namespace' and role = 'client'\n    group by issue_id\n    order by lastSeen desc\n    limit 20)\n    union\n    (select last(issue_id, time) as issueId, 'deployment_event' as issueType, last(reason::text, time) as issueCode, '' as role, last(deployment_name, time) as resourceName, last(namespace, time) as namespace, last(deployment_name, time) as workloadName, '' as remoteServiceName, '' as remoteNamespace, last(reason, time) as description, last(deployment_event_type, time) as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from deployment_event_issues \n    where $__timeFilter(\"time\") and namespace = '$namespace'\n    group by issue_id\n    order by lastSeen desc\n    limit 20)\n    union\n    (select last(issue_id, time) as issueId, 'container_info' as issueType, last(reason::text, time) as issueCode, '' as role, last(container_name, time) as resourceName, last(namespace, time) as namespace, last(workload_name, time) as workloadName, '' as remoteServiceName, '' as remoteNamespace, last(reason, time) as description, '' as subType, to_char(min(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as firstSeen, to_char(max(time), 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as lastSeen, count(*) as incidents from container_info_issues \n    where $__timeFilter(\"time\") and namespace = '$namespace'\n    group by issue_id\n    order by lastSeen desc\n    limit 20)) issues\n  order by lastSeen desc\n  limit 20",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "percentile"
                ],
                "type": "column"
              }
            ]
          ],
          "table": "conn_stats_samples",
          "timeColumn": "\"time\"",
          "timeColumnType": "timestamp",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "title": "Open Issues",
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "description": true,
              "firstseen": true,
              "issueid": true,
              "lastseen": true,
              "namespace": true,
              "remotenamespace": true,
              "remoteservicename": true,
              "role": true,
              "subtype": true
            },
            "indexByName": {
              "incidents": 6,
              "issuecode": 3,
              "issueid": 0,
              "issuetype": 2,
              "lastseen": 5,
              "resourcename": 4,
              "workloadname": 1
            },
            "renameByName": {
              "incidents": "Incidents",
              "issuecode": "Status Code",
              "issueid": "",
              "issuetype": "Type",
              "lastseen": "",
              "resourcename": "Resource",
              "workloadname": "Workload"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": "${ds_prom}",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 14,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "reqps"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "HTTP"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "semi-dark-red",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "DNS"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "super-light-orange",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 15,
        "x": 0,
        "y": 12
      },
      "id": 75,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "alias(sum(rate(groundcover_http_error_counter{namespace='$namespace'}[$__rate_interval])), 'HTTP')",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        },
        {
          "exemplar": true,
          "expr": "alias(sum(rate(groundcover_dns_error_counter{namespace='$namespace'}[$__rate_interval])), 'DNS')",
          "hide": false,
          "interval": "",
          "legendFormat": "",
          "refId": "B"
        },
        {
          "exemplar": true,
          "expr": "alias(sum(rate(groundcover_sql_error_counter{namespace='$namespace'}[$__rate_interval])), 'SQL')",
          "hide": false,
          "interval": "",
          "legendFormat": "",
          "refId": "C"
        }
      ],
      "title": "Issue Velocity",
      "type": "timeseries"
    }
  ],
  "refresh": false,
  "schemaVersion": 30,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "groundcover",
          "value": "groundcover"
        },
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": "cluster",
        "multi": false,
        "name": "ds_prom",
        "options": [],
        "query": "prometheus",
        "queryValue": "",
        "refresh": 1,
        "regex": "(.*GC.*|.*gc.*|.*groundcover.*|.*Groundcover.*|.*GroundCover.*)",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${ds_prom}",
        "definition": "label_values(groundcover_container_info_total_counter{}, namespace)",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": null,
        "multi": false,
        "name": "namespace",
        "options": [],
        "query": {
          "query": "label_values(groundcover_container_info_total_counter{}, namespace)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "^(?!.*gc.*$|.*groundcover.*$).*",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "current": {
          "selected": false,
          "text": "groundcover-timescaledb",
          "value": "groundcover-timescaledb"
        },
        "description": null,
        "error": null,
        "hide": 2,
        "includeAll": false,
        "label": "SQL Data Source",
        "multi": false,
        "name": "ds_sql",
        "options": [],
        "query": "postgres",
        "queryValue": "",
        "refresh": 1,
        "regex": ".*$ds_prom.*",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "current": {
          "selected": false,
          "text": "groundcover-loki",
          "value": "groundcover-loki"
        },
        "description": null,
        "error": null,
        "hide": 2,
        "includeAll": false,
        "label": "Logs Data Source",
        "multi": false,
        "name": "ds_logs",
        "options": [],
        "query": "loki",
        "queryValue": "",
        "refresh": 1,
        "regex": ".*$ds_prom.*",
        "skipUrlSync": false,
        "type": "datasource"
      }
    ]
  },
  "time": {
    "from": "now-30m",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Cluster Issues Dashboard",
  "uid": "lTP9tuo65",
  "version": 2
}