Grafana dashboard: Mimir / Slow queries

The Slow queries dashboard shows details about the slowest queries for a given time range and enables you to filter results by a specific tenant.

If you enable [Grafana Tempo](https://grafana.com/oss/tempo/) tracing, the dashboard displays a link to the trace of each query.

This dashboard requires [Grafana Loki](https://grafana.com/oss/loki/) to fetch detailed query statistics from logs.

Grafana Mimir is an open source software project that provides a scalable long-term storage for Prometheus. For more information, visit [Grafana Mimir](https://grafana.com/oss/mimir/).

{
  "__requires": [
    {
      "id": "grafana",
      "name": "Grafana",
      "type": "grafana",
      "version": "8.0.0"
    }
  ],
  "annotations": {
    "list": []
  },
  "editable": true,
  "gnetId": 16020,
  "graphTooltip": 0,
  "hideControls": false,
  "links": [
    {
      "asDropdown": true,
      "icon": "external link",
      "includeVars": true,
      "keepTime": true,
      "tags": [
        "mimir"
      ],
      "targetBlank": false,
      "title": "Mimir dashboards",
      "type": "dashboards"
    }
  ],
  "refresh": "",
  "rows": [
    {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
          "datasource": "${lokidatasource}",
          "fieldConfig": {
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Time range"
                },
                "properties": [
                  {
                    "id": "mappings",
                    "value": [
                      {
                        "from": "",
                        "id": 1,
                        "text": "Instant query",
                        "to": "",
                        "type": 1,
                        "value": "0"
                      }
                    ]
                  },
                  {
                    "id": "unit",
                    "value": "s"
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Step"
                },
                "properties": [
                  {
                    "id": "unit",
                    "value": "s"
                  }
                ]
              }
            ]
          },
          "id": 1,
          "span": 12,
          "targets": [
            {
              "expr": "{cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | response_time > ${min_duration}",
              "instant": false,
              "legendFormat": "",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Slow queries",
          "transformations": [
            {
              "id": "extractFields",
              "options": {
                "source": "labels"
              }
            },
            {
              "id": "calculateField",
              "options": {
                "alias": "Time range",
                "binary": {
                  "left": "param_end",
                  "operator": "-",
                  "reducer": "sum",
                  "right": "param_start"
                },
                "mode": "binary",
                "reduce": {
                  "reducer": "sum"
                },
                "replaceFields": false
              }
            },
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Line": true,
                  "Time": true,
                  "caller": true,
                  "cluster": true,
                  "container": true,
                  "host": true,
                  "id": true,
                  "job": true,
                  "labels": true,
                  "level": true,
                  "line": true,
                  "method": true,
                  "msg": true,
                  "name": true,
                  "namespace": true,
                  "param_end": true,
                  "param_start": true,
                  "param_time": true,
                  "path": true,
                  "pod": true,
                  "pod_template_hash": true,
                  "query_wall_time_seconds": true,
                  "stream": true,
                  "traceID": true,
                  "tsNs": true
                },
                "indexByName": {
                  "Time range": 3,
                  "param_query": 2,
                  "param_step": 4,
                  "response_time": 5,
                  "ts": 0,
                  "user": 1
                },
                "renameByName": {
                  "org_id": "Tenant ID",
                  "param_query": "Query",
                  "param_step": "Step",
                  "response_time": "Duration"
                }
              }
            }
          ],
          "type": "table"
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "",
      "titleSize": "h6"
    }
  ],
  "schemaVersion": 14,
  "style": "dark",
  "tags": [
    "mimir"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "text": "default",
          "value": "default"
        },
        "hide": 0,
        "label": "Data Source",
        "name": "datasource",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "type": "datasource"
      },
      {
        "allValue": ".*",
        "current": {
          "text": "prod",
          "value": "prod"
        },
        "datasource": "$datasource",
        "hide": 0,
        "includeAll": true,
        "label": "cluster",
        "multi": false,
        "name": "cluster",
        "options": [],
        "query": "label_values(cortex_build_info, cluster)",
        "refresh": 1,
        "regex": "",
        "sort": 2,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "text": "prod",
          "value": "prod"
        },
        "datasource": "$datasource",
        "hide": 0,
        "includeAll": false,
        "label": "namespace",
        "multi": false,
        "name": "namespace",
        "options": [],
        "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
        "refresh": 1,
        "regex": "",
        "sort": 2,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "hide": 0,
        "includeAll": false,
        "label": "Logs datasource",
        "multi": false,
        "name": "lokidatasource",
        "query": "loki",
        "type": "datasource"
      },
      {
        "current": {
          "selected": true,
          "text": "5s",
          "value": "5s"
        },
        "hide": 0,
        "label": "Min duration",
        "name": "min_duration",
        "options": [
          {
            "selected": true,
            "text": "5s",
            "value": "5s"
          }
        ],
        "query": "5s",
        "type": "textbox"
      },
      {
        "current": {
          "selected": true,
          "text": ".*",
          "value": ".*"
        },
        "hide": 0,
        "label": "Tenant ID",
        "name": "tenant_id",
        "options": [
          {
            "selected": true,
            "text": ".*",
            "value": ".*"
          }
        ],
        "query": ".*",
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "utc",
  "title": "Mimir / Slow queries",
  "uid": "6089e1ce1e678788f46312a0a1e647e6",
  "version": 0
}