Grafana dashboard: Kamon 2.x - API dashboard
Description: API dashboard for apps instrumented with Kamon 2.x

This dashboard shows metrics for both the server and client side.

The dashboard brings the following variables:

* `PROMETHEUS_DS`: available [Prometheus datasources].
* `job`: Prometheus jobs with Kamon instrumentation. Multiple selections is enabled.
* `instance`: Prometheus instances belonging to the selected jobs.
* `interval`: interval time to use in the queries: 30s, 1m, 2m, 5m, 10m, 30m, 1h, 6h, 12h, 1d, 7d, 14d, 30d.

Jobs and instances are in [Prometheus terms].

### What's included:

The following sections:

* **API overview:** throughput and latency for server side by status.
* **Client metrics:** throughput and latency for client side. One row for each operation.

Almost all metrics have a description.

### What's required:

It requires the apps to be instrumented with [Kamon 2.x] and any module with instrumentation for server and client side, such as *kamon-akka-http* and *kamon-play*.

### How to install:

It requires an action at the first time is imported: specify a properly value for the hidden variable `app_filter` in order to be able to list the jobs with kamon instrumentation properly.
An example:

Suppose you have the apps *app-1*, *app-2* and *app-3* instrumented with *Kamon* and they are scrapped by *Prometheus* using jobs with the same names, so there have to be the jobs *app-1*, *app-2* and *app-3* on *Prometheus*. In this case, a good value for the custom variable *app_filter* would be `app-1.*|app-2.*|app-3.*`. Take in account that this value will be used to load the query variable *job* whose query expression is `label_values(up{job=~"$app_filter"}, job)`.

### Issues / questions

Feedback is always welcome!

For issues and questions please go to the repo [cspinetta/kamon-grafana-dashboards].


[Prometheus]: https://prometheus.io/
[Kamon 2.x]: https://github.com/kamon-io/Kamon
[cspinetta/kamon-grafana-dashboards]: https://github.com/cspinetta/kamon-grafana-dashboards
[Prometheus terms]: https://prometheus.io/docs/concepts/jobs_instances/#jobs-and-instances
[Prometheus datasources]: https://grafana.com/docs/grafana/latest/features/datasources/prometheus/

{
  "__inputs": [],
  "__requires": [
    {
      "id": "grafana",
      "name": "Grafana",
      "type": "grafana",
      "version": "7.0.0"
    },
    {
      "id": "prometheus",
      "name": "Prometheus",
      "type": "datasource",
      "version": "1.0.0"
    }
  ],
  "annotations": {
    "list": []
  },
  "description": "API dashboard for apps instrumented with Kamon 2.x",
  "editable": false,
  "gnetId": 12317,
  "graphTooltip": 0,
  "hideControls": false,
  "id": null,
  "links": [],
  "panels": [
    {
      "collapse": false,
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 2,
      "panels": [],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "API overview",
      "titleSize": "h6",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "The average requests rate in the last $interval across all selected instances stacked by status",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 24,
        "x": 0,
        "y": 1
      },
      "id": 3,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": false,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "2xx rpm",
          "color": "rgb(55, 135, 45)"
        },
        {
          "alias": "3xx rpm",
          "color": "rgb(31, 96, 196)"
        },
        {
          "alias": "4xx rpm",
          "color": "rgb(224, 180, 0)"
        },
        {
          "alias": "5xx rpm",
          "color": "rgb(196, 22, 42)"
        }
      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(increase(http_server_requests_total{instance=~\"$instance\", http_status_code=\"2xx\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "2xx rpm",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(increase(http_server_requests_total{instance=~\"$instance\", http_status_code=\"3xx\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "3xx rpm",
          "refId": "B"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(increase(http_server_requests_total{instance=~\"$instance\", http_status_code=\"4xx\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "4xx rpm",
          "refId": "C"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(increase(http_server_requests_total{instance=~\"$instance\", http_status_code=\"5xx\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "5xx rpm",
          "refId": "D"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Throughput (avg. $interval)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "rpm",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "rpm",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "Latency of requests with 2xx status response over $interval",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 4,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "p99.9",
          "color": "rgb(55, 135, 45)"
        },
        {
          "alias": "p99",
          "color": "rgb(86, 166, 75)"
        },
        {
          "alias": "p90",
          "color": "rgb(150, 217, 141)"
        },
        {
          "alias": "p50",
          "color": "rgb(200, 242, 194)"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.999, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"2.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99.9",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.99, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"2.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99",
          "refId": "B"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.90, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"2.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p90",
          "refId": "C"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.50, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"2.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p50",
          "refId": "D"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Latency: only 2xx status response",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "Given:\n  * Success requests: status 2xx\n  * Valid requests: status 2xx or status 5xx\n\nThe uptime could be estimated by 'success requests' / 'valid requests'.\n\nThis is just an approximation because it doesn't take in account a lot of corner cases. For a more accurate metric it should be measured from outside the app.\n",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 16,
        "x": 0,
        "y": 13
      },
      "id": 5,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": false,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": true,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "uptime",
          "color": "rgb(55, 135, 45)"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(rate(http_server_requests_total{instance=~\"$instance\", http_status_code=\"2xx\"}[$interval]))\n /\n sum(rate(http_server_requests_total{instance=~\"$instance\", http_status_code=~\"2xx|5xx\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "uptime",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Approximated Uptime: 'requests with 2xx status response' / 'requests with 2xx or 5xx status response'",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "percentunit",
          "label": null,
          "logBase": 1,
          "max": 1,
          "min": 0,
          "show": true
        },
        {
          "format": "percentunit",
          "label": null,
          "logBase": 1,
          "max": 1,
          "min": 0,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "Relative frequency distribution of response 2xx, 3xx, 4xx and 5xx over $interval",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 8,
        "x": 16,
        "y": 13
      },
      "id": 6,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": false,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": true,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "Failures (5xx)",
          "color": "rgb(196, 22, 42)"
        },
        {
          "alias": "Client errors (4xx)",
          "color": "rgb(224, 180, 0)"
        },
        {
          "alias": "Redirects (3xx)",
          "color": "rgb(31, 96, 196)"
        },
        {
          "alias": "Success (2xx)",
          "color": "rgb(55, 135, 45)"
        }
      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(rate(http_server_requests_total{instance=~\"$instance\", http_status_code=\"5xx\"}[$interval]))\n/\nsum(rate(http_server_requests_total{instance=~\"$instance\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "Failures (5xx)",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(rate(http_server_requests_total{instance=~\"$instance\", http_status_code=\"4xx\"}[$interval]))\n/\nsum(rate(http_server_requests_total{instance=~\"$instance\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "Client errors (4xx)",
          "refId": "B"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(rate(http_server_requests_total{instance=~\"$instance\", http_status_code=\"3xx\"}[$interval]))\n/\nsum(rate(http_server_requests_total{instance=~\"$instance\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "Redirects (3xx)",
          "refId": "C"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(rate(http_server_requests_total{instance=~\"$instance\", http_status_code=\"2xx\"}[$interval]))\n/\nsum(rate(http_server_requests_total{instance=~\"$instance\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "Success (2xx)",
          "refId": "D"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "2xx | 4xx | 3xx | 5xx % (avg. $interval)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "percentunit",
          "label": null,
          "logBase": 1,
          "max": 100,
          "min": 0,
          "show": true
        },
        {
          "format": "percentunit",
          "label": null,
          "logBase": 1,
          "max": 100,
          "min": 0,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "Latency of requests with 3xx status response over $interval",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 8,
        "x": 0,
        "y": 19
      },
      "id": 7,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "p99.9",
          "color": "rgb(31, 96, 196)"
        },
        {
          "alias": "p99",
          "color": "rgb(50, 116, 217)"
        },
        {
          "alias": "p90",
          "color": "rgb(138, 184, 255)"
        },
        {
          "alias": "p50",
          "color": "rgb(192, 216, 255)"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.999, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"3.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99.9",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.99, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"3.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99",
          "refId": "B"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.90, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"3.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p90",
          "refId": "C"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.50, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"3.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p50",
          "refId": "D"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Latency: only 3xx status response",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "Latency of requests with 4xx status response over $interval",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 8,
        "x": 8,
        "y": 19
      },
      "id": 8,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "p99.9",
          "color": "rgb(224, 180, 0)"
        },
        {
          "alias": "p99",
          "color": "rgb(242, 204, 12)"
        },
        {
          "alias": "p90",
          "color": "rgb(255, 238, 82)"
        },
        {
          "alias": "p50",
          "color": "rgb(255, 248, 153)"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.999, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"4.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99.9",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.99, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"4.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99",
          "refId": "B"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.90, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"4.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p90",
          "refId": "C"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.50, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"4.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p50",
          "refId": "D"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Latency: only 4xx status response",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "Latency of requests with 5xx status response over $interval",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 8,
        "x": 16,
        "y": 19
      },
      "id": 9,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "p99.9",
          "color": "rgb(196, 22, 42)"
        },
        {
          "alias": "p99",
          "color": "rgb(224, 47, 68)"
        },
        {
          "alias": "p90",
          "color": "rgb(255, 115, 131)"
        },
        {
          "alias": "p50",
          "color": "rgb(255, 166, 176)"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.999, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"5.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99.9",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.99, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"5.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99",
          "refId": "B"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.90, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"5.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p90",
          "refId": "C"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.50, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"server\", http_status_code=~\"5.*\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p50",
          "refId": "D"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Latency: only 5xx status response",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    },
    {
      "collapse": false,
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 25
      },
      "id": 10,
      "panels": [],
      "repeat": "client_operation",
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "Clients metrics | $client_operation",
      "titleSize": "h6",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "The average outgoing requests rate in the last $interval across all selected instances",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 24,
        "x": 0,
        "y": 26
      },
      "id": 11,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": false,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "/.*with no error/",
          "color": "rgb(55, 135, 45)"
        },
        {
          "alias": "/.*with error/",
          "color": "rgb(196, 22, 42)"
        }
      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(increase(span_processing_time_seconds_count{instance=~\"$instance\", error=\"false\", span_kind=\"client\", operation=\"$client_operation\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{operation}} with no error",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "sum(increase(span_processing_time_seconds_count{instance=~\"$instance\", error=\"true\", span_kind=\"client\", operation=\"$client_operation\"}[$interval]))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{operation}} with error",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Throughput of outgoing requests over $interval",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "rpm",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "rpm",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$PROMETHEUS_DS",
      "description": "Latency of outgoing requests in the last $interval across all selected instances",
      "fill": 1,
      "gridPos": {
        "h": 6,
        "w": 24,
        "x": 0,
        "y": 32
      },
      "id": 12,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": true,
        "show": true,
        "sort": "max",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "repeat": null,
      "seriesOverrides": [
        {
          "alias": "/p99.*with no error/",
          "color": "rgb(55, 135, 45)"
        },
        {
          "alias": "/p90.*with no error/",
          "color": "rgb(86, 166, 75)"
        },
        {
          "alias": "/p50.*with no error/",
          "color": "rgb(150, 217, 141)"
        },
        {
          "alias": "/p99.*with error/",
          "color": "rgb(196, 22, 42)"
        },
        {
          "alias": "/p90.*with error/",
          "color": "rgb(224, 47, 68)"
        },
        {
          "alias": "/p50.*with error/",
          "color": "rgb(255, 115, 131)"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.99, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"client\", error=\"false\", operation=\"$client_operation\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99 {{operation}} with no error",
          "refId": "A"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.99, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"client\", error=\"true\", operation=\"$client_operation\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p99 {{operation}} with error",
          "refId": "B"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.90, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"client\", error=\"false\", operation=\"$client_operation\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p90 {{operation}} with no error",
          "refId": "C"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.90, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"client\", error=\"true\", operation=\"$client_operation\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p90 {{operation}} with error",
          "refId": "D"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.50, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"client\", error=\"false\", operation=\"$client_operation\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p50 {{operation}} with no error",
          "refId": "E"
        },
        {
          "datasource": "$PROMETHEUS_DS",
          "expr": "histogram_quantile(0.50, sum(rate(span_processing_time_seconds_bucket{instance=~\"$instance\", span_kind=\"client\", error=\"true\", operation=\"$client_operation\"}[$interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "p50 {{operation}} with error",
          "refId": "F"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "Latency of outgoing requests over $interval",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    }
  ],
  "refresh": "1m",
  "rows": [],
  "schemaVersion": "24",
  "style": "dark",
  "tags": [
    "kamon",
    "prometheus",
    "web-server"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "text": "Prometheus",
          "value": "Prometheus"
        },
        "hide": 1,
        "label": null,
        "name": "PROMETHEUS_DS",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "type": "datasource"
      },
      {
        "allValue": null,
        "current": {
          "text": "my-app.*|another-app.*",
          "value": "my-app.*|another-app.*"
        },
        "hide": 2,
        "includeAll": false,
        "label": "App filter",
        "multi": false,
        "name": "app_filter",
        "options": [
          {
            "text": "my-app.*|another-app.*",
            "value": "my-app.*|another-app.*"
          }
        ],
        "query": "my-app.*|another-app.*",
        "refresh": 0,
        "type": "custom"
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "$PROMETHEUS_DS",
        "hide": 0,
        "includeAll": true,
        "label": "Job",
        "multi": true,
        "name": "job",
        "options": [],
        "query": "label_values(up{job=~\"$app_filter\"}, job)",
        "refresh": 1,
        "regex": "",
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "$PROMETHEUS_DS",
        "hide": 0,
        "includeAll": true,
        "label": "Instance",
        "multi": true,
        "name": "instance",
        "options": [],
        "query": "label_values(up{job=~\"$job\"},instance)",
        "refresh": 1,
        "regex": "",
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "auto": false,
        "auto_count": 0,
        "auto_min": "10s",
        "current": {
          "text": "1m",
          "value": "1m"
        },
        "hide": 0,
        "label": "Interval",
        "name": "interval",
        "query": "30s,1m,2m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
        "refresh": 2,
        "type": "interval"
      },
      {
        "allValue": null,
        "current": {
          "text": "all",
          "value": "$__all"
        },
        "datasource": "$PROMETHEUS_DS",
        "hide": 2,
        "includeAll": true,
        "label": "",
        "multi": false,
        "name": "client_operation",
        "options": [],
        "query": "label_values(span_processing_time_seconds_count{job=~\"$job\", span_kind=\"client\"},operation)",
        "refresh": 1,
        "regex": "",
        "sort": 1,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "browser",
  "title": "Kamon 2.x - API dashboard",
  "version": "0.0.1"
}