Grafana dashboard: Cadence frontend
Description: An initial dashboard for prometheus metrics from https://github.com/uber/cadence

An initial dashboard for prometheus metrics from https://github.com/uber/cadence

{
  "__inputs": [
    {
      "name": "DS_Data_Source",
      "label": "Data_Source",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "6.2.2"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": 10373,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1560902714919,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 36,
      "panels": [],
      "repeat": null,
      "title": "Overall",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 1
      },
      "id": 30,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(cadence_errors[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.42651392099317587,
          "legendFormat": "Errors - {{instance}}:{{operation}}:",
          "refId": "A",
          "target": "fetch service:cadence-frontend deployment:$deployment name:{cadence_errors,cadence_requests} | transformNull 0 | sum name | scaleToSeconds 1 | aliasByTags name"
        },
        {
          "expr": "rate(cadence_requests[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Requests - {{instance}}:{{operation}}:{{domain}}",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Request Vs Error",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "id": 33,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(cadence_errors_bad_request[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.6793219188372182,
          "legendFormat": "bad_requests -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "A",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_bad_request | transformNull 0 | sum | scaleToSeconds 1 | alias BadRequest"
        },
        {
          "expr": "rate(cadence_errors_service_busy[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.22947422064141287,
          "legendFormat": "service_busy -  {{instance}}:{{operation}} : {{domain}}",
          "refId": "B",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_service_busy | transformNull 0 | sum | scaleToSeconds 1 | alias ServiceBusy"
        },
        {
          "expr": "rate(cadence_errors_entity_not_exists[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.6426927299281162,
          "legendFormat": "entity_not_exists -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "C",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_entity_not_exists | transformNull 0 | sum | scaleToSeconds 1 | alias EntityNotExist"
        },
        {
          "expr": "rate(cadence_errors_execution_already_started[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.0706418565091802,
          "legendFormat": "execution_already_started  -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "D",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_execution_already_started | transformNull 0 | sum | scaleToSeconds 1 | alias ExecutionAlreadyStarted"
        },
        {
          "expr": "rate(cadence_errors_domain_already_exists[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.9224890388804401,
          "legendFormat": "domain_already_exists  -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "E",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_domain_already_exists | transformNull 0 | sum | scaleToSeconds 1 | alias DomainAlreadyExists"
        },
        {
          "expr": "rate(cadence_errors_cancellation_already_requested[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.12556073497696874,
          "legendFormat": "cancellation_already_requested  -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "F",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_cancellation_already_requested | transformNull 0 | sum | scaleToSeconds 1 | alias CancellationAlreadyRequested"
        },
        {
          "expr": "rate(cadence_errors_context_timeout[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "errors_context_timeout  -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "G",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_context_timeout | transformNull 0 | sum | scaleToSeconds 1 | alias ContextTimeout"
        },
        {
          "expr": "rate(cadence_errors_limit_exceeded[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "limit_exceeded  -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "H",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_errors_limit_exceeded | transformNull 0 | sum | scaleToSeconds 1 | alias LimitExceeded"
        },
        {
          "expr": "rate(cadence_errors_query_failed[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "query_failed -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "I"
        },
        {
          "expr": "rate(cadence_errors_shard_ownership_lost[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "shard_ownership_lost -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "J"
        },
        {
          "expr": "",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "K"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Error Breakdown",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "id": 34,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "histogram_quantile($quantiles, (sum by (le, operation) (rate(cadence_latency_bucket[2m]))))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{operation}}",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Frontend API Latencies",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "ms",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 15
      },
      "id": 40,
      "panels": [],
      "title": "Frontend Client",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 16
      },
      "id": 43,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(cadence_client_errors[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.42651392099317587,
          "legendFormat": "errors -  {{instance}}:{{operation}} : {{domain}}",
          "refId": "A",
          "target": "fetch service:cadence-frontend deployment:$deployment name:{cadence_client_errors,cadence_client_requests} | transformNull 0 | sum name | scaleToSeconds 1 | aliasByTags name"
        },
        {
          "expr": "rate(cadence_client_requests[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "requests -   {{instance}}:{{operation}} : {{domain}}",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Frontend Client Request Vs Error",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 16
      },
      "id": 46,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "histogram_quantile($quantiles, (sum by (le, operation) (rate(cadence_client_latency_bucket[2m]))))",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.22745123265098632,
          "legendFormat": "{{operation}}",
          "refId": "A",
          "target": "fetch service:cadence-frontend deployment:$deployment  name:cadence_client_latency timertype:$latency  | transformNull 0 | aliasByTags operation"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Frontend Client API Latencies",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "ms",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 37,
      "panels": [],
      "repeat": null,
      "title": "API",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 24
      },
      "id": 1,
      "isNew": true,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "cadence.errors",
          "yaxis": 1
        },
        {
          "alias": "cadence.requests",
          "yaxis": 1
        },
        {
          "alias": "availability",
          "yaxis": 2
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(cadence_requests{operation=\"StartWorkflowExecution\"}[2m])",
          "format": "time_series",
          "hide": false,
          "intervalFactor": 1,
          "key": 0.5190158647646326,
          "legendFormat": "requests -  {{instance}}:{{operation}} : {{domain}}",
          "refId": "A",
          "target": "fetch service:$service deployment:$deployment name:cadence_{errors_*,requests} operation:startworkflowexecution | transformNull 0 | sum name | scaleToSeconds 1 | aliasByTags name"
        },
        {
          "expr": " rate(cadence_errors{ooperation=\"StartWorkflowExecution\"}[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "errors -  {{instance}}:{{operation}} : {{domain}}",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "StartWorfklowExecution",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "percent",
          "logBase": 1,
          "max": 100,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 24
      },
      "id": 2,
      "isNew": true,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": true,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "p99",
          "yaxis": 2
        },
        {
          "alias": "upper",
          "yaxis": 2
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "histogram_quantile($quantiles, sum(rate(cadence_latency_bucket[2m])) by (le, operation))",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.05105517950309202,
          "legendFormat": "{{operation}}",
          "refId": "A",
          "target": "fetch service:$service deployment:$deployment timertype:{lower,median,p99,upper} operation:startworkflowexecution | transformNull 0 |  summarize 1min avg | avg timertype | aliasByTags timertype"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Operation Latencies",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "ms",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "ms",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 32
      },
      "id": 3,
      "isNew": true,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "cadence.errors",
          "yaxis": 1
        },
        {
          "alias": "availability",
          "yaxis": 2
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(cadence_requests{operation=\"PollForActivityTask\"}[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.054221963407744767,
          "legendFormat": "requests -  {{instance}}:{{operation}} : {{domain}}",
          "refId": "A",
          "target": "fetch service:$service deployment:$deployment name:cadence_{errors_*,requests} operation:pollforactivitytask | transformNull 0 | sum name | scaleToSeconds 1 | aliasByTags name"
        },
        {
          "expr": "rate(cadence_errors{operation=\"PollForActivityTask\"}[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.32470798710307447,
          "legendFormat": "errors -  {{instance}}:{{operation}} : {{domain}}",
          "refId": "B",
          "target": "fetch service:cadence-frontend deployment:$deployment name:cadence_requests operation:pollforactivitytask | sum | summarize 1m sum | diff (fetch service:cadence-frontend deployment:$deployment operation:pollforactivitytask name:cadence_errors | sum | summarize 1m sum) | asPercent (fetch service:cadence-frontend deployment:$deployment name:cadence_requests operation:pollforactivitytask | sum | summarize 1m sum) | transformNull 100 | alias availability"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "PollForActivityTask",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "percent",
          "logBase": 1,
          "max": 100,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 40
      },
      "id": 38,
      "panels": [],
      "repeat": null,
      "title": "Persistence",
      "type": "row"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 41
      },
      "id": 27,
      "isNew": true,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "rate(persistence_errors[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "key": 0.42191288349501055,
          "legendFormat": "Errors - {{instance}}:{{operation}}:",
          "refId": "A",
          "target": "fetch service:cadence-frontend deployment:$deployment name:persistence_{errors,requests}  | transformNull 0 | sum name | scaleToSeconds 1 | aliasByTags name"
        },
        {
          "expr": "rate(persistence_requests[2m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Requests - {{instance}}:{{operation}}:",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Requests Vs Errors",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_Data_Source}",
      "editable": true,
      "error": false,
      "fill": 1,
      "grid": {},
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 41
      },
      "id": 47,
      "isNew": true,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 2,
      "links": [],
      "nullPointMode": "null",
      "options": {},
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "histogram_quantile($quantiles, sum(rate(persistence_latency_bucket[2m])) by (le, operation))",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{operation}}",
          "refId": "C"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Operation Latencies p99",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "cumulative"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "ms",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "refresh": "",
  "schemaVersion": 18,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {
          "selected": true,
          "tags": [],
          "text": "0.99",
          "value": "0.99"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Quantiles",
        "multi": false,
        "name": "quantiles",
        "options": [
          {
            "selected": false,
            "text": "0.5",
            "value": "0.5"
          },
          {
            "selected": false,
            "text": "0.95",
            "value": "0.95"
          },
          {
            "selected": true,
            "text": "0.99",
            "value": "0.99"
          }
        ],
        "query": "0.5, 0.95, 0.99",
        "skipUrlSync": false,
        "type": "custom"
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {
    "now": true,
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "browser",
  "title": "Cadence frontend",
  "uid": "000058219",
  "version": 35,
  "description": "An initial dashboard for prometheus metrics from https://github.com/uber/cadence"
}