Grafana dashboard: Generic Service
Description: Simplifies monitoring services on Kubernetes. Shows container logs, availability, resource usage and HTTP and gRPC traffic.

# Generic Service Dashboard

[![GitHub](https://shields.io/badge/GitHub-source-green?logo=github)](https://github.com/nano-byte/grafana-dashboards/tree/master/generic-service)

This Grafana dashboard simplifies monitoring typical "80% case" services on Kubernetes. It shows:

- Logs
- Availability and Resources
- HTTP and gRPC traffic

## Logs

The dashboard assumes you are using [Loki](https://grafana.com/oss/loki/) to aggregate container log files.

In order for log level filtering to work, you'll need to ensure log entries carry the label `level`.  
Supported values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`

A typical [Promtail configuration](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#pipeline_stages) to achieve this for JSON-encoded log output could look like this:

```yaml
pipelineStages:
  - docker: {}
  - cri: {}
  - json:
      expressions:
        ts: ts
        msg: msg
        level: level
  - timestamp:
      source: ts
      format: RFC3339Nano
      action_on_failure: skip
  - labels:
      level:
  - output:
      source: msg
```

## Availability and Resources

The dashboard works well with services deployed using the [Generic Service Helm Chart](https://artifacthub.io/packages/helm/nano-byte/generic-service). You can also it with other services, as long as they use Kubernetes `Deployment`s and containers with matching names.

The dashboard shows the average uptime of your service, a timeline when some or all replicas where down and a timeline when pod restarts occurred.

It also shows the containers' memory and CPU usage, along with requests and limits if they are specified.

## Traffic

The dashboard assumes you are using [NGINX Ingress Controller](https://kubernetes.github.io/ingress-nginx/) or [Istio](https://istio.io/) to route HTTP traffic to your containers. In order for traffic to be visualized you need to use `Ingress`es or `VirtualService`s with names matching your `Deployment`s.

If you use gRPC and want more detailed information to show up on the dashboard you can provide the necessary metrics by using one of these libraries:

- [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus) for Go
- [Nexogen.Libraries.Metrics.Grpc](https://github.com/nexogen-international/Nexogen.Libraries.Metrics#grpc) for .NET

{
  "__inputs": [
    {
      "name": "DS_LOKI",
      "label": "Loki",
      "description": "",
      "type": "datasource",
      "pluginId": "loki",
      "pluginName": "Loki"
    },
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": {},
  "__requires": [
    {
      "type": "panel",
      "id": "gauge",
      "name": "Gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "9.3.1"
    },
    {
      "type": "panel",
      "id": "heatmap",
      "name": "Heatmap",
      "version": ""
    },
    {
      "type": "panel",
      "id": "logs",
      "name": "Logs",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "loki",
      "name": "Loki",
      "version": "1.0.0"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "state-timeline",
      "name": "State timeline",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "$$hashKey": "object:519",
        "builtIn": 1,
        "datasource": {
          "type": "datasource",
          "uid": "grafana"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      },
      {
        "datasource": {
          "type": "datasource",
          "uid": "grafana"
        },
        "enable": true,
        "expr": "",
        "hide": true,
        "iconColor": "purple",
        "name": "Deployments",
        "target": {
          "refId": "Anno",
          "tags": [
            "$namespace",
            "$service"
          ],
          "type": "tags"
        }
      }
    ]
  },
  "description": "Simplifies monitoring services on Kubernetes. Shows container logs, availability, resource usage and HTTP and gRPC traffic.",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "datasource": "${DS_LOKI}",
      "description": "Logs reported by the main container (without sidecars)",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 101,
      "title": "Logs",
      "type": "row"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "gridPos": {
        "h": 6,
        "w": 18,
        "x": 0,
        "y": 1
      },
      "id": 36,
      "options": {
        "dedupStrategy": "numbers",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": false
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "expr": "{namespace=\"$namespace\", container=\"$service\", level=~\"$loglevel\"}",
          "refId": "A"
        }
      ],
      "title": "Messages",
      "type": "logs"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "Rate of log messages",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "Messages / 10 minutes",
            "axisPlacement": "auto",
            "barAlignment": -1,
            "drawStyle": "bars",
            "fillOpacity": 100,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 0,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Value"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "Gray",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Verbose"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "Gray",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Debug"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "Blue",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Information"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "Green",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Warning"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "Yellow",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Error"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "Red",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Fatal"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "Purple",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 18,
        "y": 1
      },
      "id": 104,
      "interval": "10m",
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "9.2.3",
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "editorMode": "code",
          "expr": "sum(count_over_time({namespace=\"$namespace\", container=\"$service\", level=~\"$loglevel\"}[10m])) by (level)",
          "legendFormat": "{{level}}",
          "queryType": "range",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Rate",
      "type": "timeseries"
    },
    {
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 55,
      "title": "Availability",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Average uptime of the service in the selected time range",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "continuous-RdYlGr"
          },
          "decimals": 2,
          "mappings": [],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 99.95
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 2,
        "x": 0,
        "y": 8
      },
      "id": 53,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "text": {}
      },
      "pluginVersion": "9.3.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "avg_over_time(\n  min(ceil(\n    kube_deployment_status_replicas_available{namespace=\"$namespace\",deployment=\"$service\"} /\n    kube_deployment_status_replicas{namespace=\"$namespace\",deployment=\"$service\"}\n  ))\n[$__range:1m]) or\navg_over_time(\n  min(ceil(\n    kube_statefulset_status_replicas_ready{namespace=\"$namespace\",statefulset=\"$service\"} /\n    kube_statefulset_status_replicas{namespace=\"$namespace\",statefulset=\"$service\"}\n  ))\n[$__range:1m])",
          "refId": "A"
        }
      ],
      "title": "Uptime",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Timeline showing when how many of the desired replicas were healthy",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "fillOpacity": 100,
            "lineWidth": 3,
            "spanNulls": false
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "index": 1,
                  "text": "None"
                },
                "1": {
                  "index": 0,
                  "text": "All"
                }
              },
              "type": "value"
            },
            {
              "options": {
                "from": 0,
                "result": {
                  "index": 2,
                  "text": "Some"
                },
                "to": 1
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.1
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 10,
        "x": 2,
        "y": 8
      },
      "id": 47,
      "options": {
        "alignValue": "center",
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "mergeValues": true,
        "rowHeight": 0.9,
        "showValue": "auto",
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "max by(deployment) (\n  kube_deployment_status_replicas_available{namespace=\"$namespace\",deployment=\"$service\"} /\n  kube_deployment_status_replicas{namespace=\"$namespace\",deployment=\"$service\"}\n) or\nmax by(statefulset) (\n  kube_statefulset_status_replicas_ready{namespace=\"$namespace\",statefulset=\"$service\"} /\n  kube_statefulset_status_replicas{namespace=\"$namespace\",statefulset=\"$service\"}\n)",
          "legendFormat": " ",
          "refId": "A"
        }
      ],
      "title": "Replicas healthy",
      "type": "state-timeline"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Timeline showing when container restarts occurred",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "red",
            "mode": "fixed"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "points",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 8,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "id": 49,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "none",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "clamp_max(increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\",container=\"$service\"}[$__rate_interval]) > 0, 1)",
          "refId": "A"
        }
      ],
      "title": "Restarts",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Running versions (Docker image tags) over time.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "hidden",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 100,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 0,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byType",
              "options": "time"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "auto"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 13,
        "x": 0,
        "y": 11
      },
      "id": 81,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "none",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "max by(version) (label_replace(kube_pod_container_info{namespace=\"$namespace\",container=\"$service\"}, \"version\", \"$1\", \"image\", \".*:(.*)\"))",
          "interval": "",
          "legendFormat": "__auto",
          "refId": "A"
        }
      ],
      "title": "Versions",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 15
      },
      "id": 2,
      "panels": [],
      "title": "Resources",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Memory usage of the main container (without sidecars)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Request"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#5794F2",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "dash": [
                    10,
                    10
                  ],
                  "fill": "dash"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Limit"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#F2495C",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "dash": [
                    10,
                    10
                  ],
                  "fill": "dash"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 13,
        "x": 0,
        "y": 16
      },
      "id": 8,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "min",
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.0.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "max by(pod) (container_memory_working_set_bytes{namespace=\"$namespace\",container=\"$service\"}) != 0",
          "format": "time_series",
          "legendFormat": "{{pod}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "min(kube_pod_container_resource_requests{resource=\"memory\",namespace=\"$namespace\",container=\"$service\"})",
          "legendFormat": "Request",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "min(kube_pod_container_resource_limits{resource=\"memory\",namespace=\"$namespace\",container=\"$service\"})",
          "legendFormat": "Limit",
          "refId": "C"
        }
      ],
      "title": "Memory usage",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "CPU usage of the main container (without sidecars)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Request"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#5794F2",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "dash": [
                    10,
                    10
                  ],
                  "fill": "dash"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Limit"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#F2495C",
                  "mode": "fixed"
                }
              },
              {
                "id": "custom.fillOpacity",
                "value": 0
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "dash": [
                    10,
                    10
                  ],
                  "fill": "dash"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 11,
        "x": 13,
        "y": 16
      },
      "id": 10,
      "links": [],
      "options": {
        "legend": {
          "calcs": [
            "min",
            "max",
            "mean",
            "last"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.0.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "(sum by (pod) (increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\",container=\"$service\"}[$__rate_interval])) > bool 0)\n* min by(pod) (kube_pod_container_resource_limits{resource=\"cpu\",namespace=\"$namespace\",container=\"$service\"}) != 0\nor\nmax by(pod) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container=\"$service\"}[$__rate_interval])) != 0",
          "format": "time_series",
          "legendFormat": "{{pod}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "min(kube_pod_container_resource_requests{resource=\"cpu\",namespace=\"$namespace\",container=\"$service\"})",
          "legendFormat": "Request",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "min(kube_pod_container_resource_limits{resource=\"cpu\",namespace=\"$namespace\",container=\"$service\"})",
          "legendFormat": "Limit",
          "refId": "C"
        }
      ],
      "title": "CPU usage",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 93,
      "panels": [],
      "title": "HTTP",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Distribution of response times as measured by ingress controller",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "scaleDistribution": {
              "type": "linear"
            }
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 13,
        "x": 0,
        "y": 24
      },
      "id": 97,
      "maxDataPoints": 20,
      "options": {
        "calculate": false,
        "cellGap": 1,
        "color": {
          "exponent": 0.5,
          "fill": "green",
          "mode": "scheme",
          "reverse": false,
          "scale": "exponential",
          "scheme": "Greens",
          "steps": 64
        },
        "exemplars": {
          "color": "rgba(255,0,255,0.5)"
        },
        "filterValues": {
          "le": 1e-9
        },
        "legend": {
          "show": false,
          "showLegend": true
        },
        "rowsFrame": {
          "layout": "auto"
        },
        "tooltip": {
          "show": true,
          "yHistogram": false
        },
        "yAxis": {
          "axisPlacement": "left",
          "reverse": false,
          "unit": "ms"
        }
      },
      "pluginVersion": "9.3.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "sum by(le) (increase(nginx_ingress_controller_response_duration_seconds_bucket{exported_namespace=\"$namespace\",exported_service=\"$service\"}[$__rate_interval])) or\nsum by(le) (increase(istio_request_duration_milliseconds_bucket{destination_service_namespace=\"$namespace\",destination_service_name=\"$service\",reporter=\"source\"}[$__rate_interval]))",
          "format": "heatmap",
          "legendFormat": "{{le}}",
          "refId": "A"
        }
      ],
      "title": "Response times",
      "type": "heatmap"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Request count by HTTP status code as measured by ingress controller",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "Requests / second",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 11,
        "x": 13,
        "y": 24
      },
      "id": 94,
      "interval": "",
      "options": {
        "legend": {
          "calcs": [
            "mean"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Mean",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.0.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "sum by(method, status) (rate(nginx_ingress_controller_requests{exported_namespace=\"$namespace\",exported_service=\"$service\"}[$__rate_interval])) or\nsum by(response_code) (rate(istio_requests_total{destination_service_namespace=\"$namespace\",destination_service_name=\"$service\",reporter=\"source\"}[$__rate_interval]))",
          "legendFormat": "{{method}}: {{status}}",
          "refId": "A"
        }
      ],
      "title": "Status codes",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 30
      },
      "id": 24,
      "panels": [],
      "title": "gRPC",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Request count by gRPC method and status code",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "Requests / second",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 13,
        "x": 0,
        "y": 31
      },
      "id": 28,
      "interval": "",
      "options": {
        "legend": {
          "calcs": [
            "max"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true,
          "sortBy": "Max",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "8.0.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "label_replace(\n  sum by(grpc_service, grpc_method, grpc_code) (\n    rate(grpc_server_requests_total{namespace=\"$namespace\",release=\"$service\"}[$__rate_interval])\n  ),\n  \"grpc_service\", \"$1\", \"grpc_service\", \"${grpc_package}(.*)\"\n) != 0 or\nlabel_replace(\n  sum by(grpc_service, grpc_method, grpc_code) (\n    rate(grpc_server_handled_total{namespace=\"$namespace\",release=\"$service\"}[$__rate_interval])\n  ),\n  \"grpc_service\", \"$1\", \"grpc_service\", \"${grpc_package}(.*)\"\n) != 0",
          "legendFormat": "{{grpc_service}}/{{grpc_method}}: {{grpc_code}}",
          "refId": "A"
        }
      ],
      "title": "Methods + status codes",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Number of gRPC responses with status codes other than OK",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "Errors / 10 minutes",
            "axisPlacement": "auto",
            "barAlignment": -1,
            "drawStyle": "bars",
            "fillOpacity": 100,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 0,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 11,
        "x": 13,
        "y": 31
      },
      "id": 91,
      "interval": "10m",
      "options": {
        "legend": {
          "calcs": [
            "sum"
          ],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "pluginVersion": "9.2.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": false,
          "expr": "sum by(grpc_code) (increase(grpc_server_requests_total{namespace=\"$namespace\",release=\"$service\",grpc_code!=\"OK\"}[10m])) != 0 or\nsum by(grpc_code) (increase(grpc_server_handled_total{namespace=\"$namespace\",release=\"$service\",grpc_code!=\"OK\"}[10m])) != 0",
          "legendFormat": "__auto",
          "refId": "A"
        }
      ],
      "title": "Error count",
      "type": "timeseries"
    }
  ],
  "refresh": "10s",
  "schemaVersion": 37,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(kube_pod_info, namespace)",
        "description": "Kubernetes namespace to inspect",
        "hide": 0,
        "includeAll": false,
        "label": "Namespace",
        "multi": false,
        "name": "namespace",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_info, namespace)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(kube_pod_container_info{namespace=\"$namespace\"}, container)",
        "description": "Name of service (Deployment, Container, etc.) to inspect",
        "hide": 0,
        "includeAll": false,
        "label": "Service",
        "multi": false,
        "name": "service",
        "options": [],
        "query": {
          "query": "label_values(kube_pod_container_info{namespace=\"$namespace\"}, container)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "definition": "label_values(grpc_server_requests_total{namespace=\"$namespace\",release=\"$service\"}, grpc_service)",
        "description": "The gRPC package name with a trailing dot (used to trim output for better readability)",
        "hide": 2,
        "includeAll": false,
        "label": "gRPC Package",
        "multi": false,
        "name": "grpc_package",
        "options": [],
        "query": {
          "query": "label_values(grpc_server_requests_total{namespace=\"$namespace\",release=\"$service\"}, grpc_service)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "/([a-z0-9\\._]+\\.).*/",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": [
            "fatal",
            "error",
            "warn"
          ],
          "value": [
            "fatal",
            "error",
            "warn"
          ]
        },
        "datasource": {
          "type": "loki",
          "uid": "${DS_LOKI}"
        },
        "definition": "label_values({namespace=\"$namespace\", container=\"$service\"}, level)",
        "description": "Log level of messages to show",
        "hide": 0,
        "includeAll": true,
        "label": "Log level",
        "multi": true,
        "name": "loglevel",
        "options": [],
        "query": "label_values({namespace=\"$namespace\", container=\"$service\"}, level)",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "filters": [],
        "hide": 0,
        "name": "Filters",
        "skipUrlSync": false,
        "type": "adhoc"
      }
    ]
  },
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Generic Service",
  "uid": "qqsCbY5Zz",
  "version": 10,
  "weekStart": "",
  "gnetId": 14759
}