Grafana dashboard: Traefik On K8s Via Loki
Description: Loki on k8s using JSON Traefik access logs.

This Dashboard expects a Traefik helm chart installed and running in a k8s cluster.
and a Loki datasource enabled to collect traefik access logs.

You Need to update Traefik deployment and allow access logs in a Json format.

# Edit Traefik deployment and enable json access logs.
$ kubectl -n traefik edit deployment.apps/traefik  

      containers:
      - args:
        - --log.level=INFO
        - --log.format=json
        - --accesslog.format=json
        - --accesslog=true


{
  "__inputs": [
    {
      "name": "DS_LOKI",
      "label": "Loki",
      "description": "A Loki datasource collects logs from k8s clusters",
      "type": "datasource",
      "pluginId": "loki",
      "pluginName": "Loki"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "7.3.6"
    },
    {
      "type": "panel",
      "id": "grafana-piechart-panel",
      "name": "Pie Chart",
      "version": "1.3.8"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "panel",
      "id": "logs",
      "name": "Logs",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "loki",
      "name": "Loki",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "text",
      "name": "Text",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Loki on k8s using JSON Traefik access logs.",
  "editable": true,
  "gnetId": 17501,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "panels": [
    {
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "decimals": 4,
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 2,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 32,
      "interval": "",
      "options": {
        "content": "<div class=\"left dashboard-header\">\n   <img src=\"https://marketplace-assets.digitalocean.com/logos/loki-logo.svg\" style=\"height:29px;\"/>\n  <span>Traefik Dashboard</span>\n</div>",
        "mode": "html"
      },
      "pluginVersion": "7.3.6",
      "targets": [
        {
          "expr": "loki_build_info",
          "format": "table",
          "instant": false,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "",
      "transparent": true,
      "type": "text"
    },
    {
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 0,
        "y": 2
      },
      "id": 4,
      "interval": "1m",
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "pluginVersion": "7.3.6",
      "targets": [
        {
          "expr": "sum(count_over_time({app=~\"traefik\"} |= \"RequestProtocol\" [$__interval]))",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "timeFrom": "1h",
      "timeShift": null,
      "title": "Total requests  ",
      "transformations": [],
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "light-blue",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 9,
        "x": 6,
        "y": 2
      },
      "id": 5,
      "interval": "30s",
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "7.3.6",
      "targets": [
        {
          "expr": "sum by (OriginStatus) (count_over_time({app=~\"traefik\"}|= \"RequestProtocol\" | json |  __error__=\"\" [$__interval]))",
          "legendFormat": "HTTP Status: {{OriginStatus}}",
          "refId": "A"
        }
      ],
      "timeFrom": "3h",
      "timeShift": null,
      "title": "Requests per status code",
      "transformations": [],
      "transparent": true,
      "type": "stat"
    },
    {
      "cacheTimeout": null,
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 15,
        "y": 2
      },
      "id": 19,
      "interval": "5m",
      "links": [],
      "maxDataPoints": 1,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "pluginVersion": "7.3.6",
      "targets": [
        {
          "expr": " sum(rate({app=~\"traefik\"} |~ \"RequestProtocol\" | json | OriginStatus >= 400 |__error__=\"\"[$__interval])) / (sum(rate({app=~\"traefik\"} |~ \"RequestProtocol\" | json | __error__=\"\"[$__interval])) / 100)",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "timeFrom": "1h",
      "timeShift": null,
      "title": "% of  4/5xx ",
      "transparent": true,
      "type": "stat"
    },
    {
      "aliasColors": {},
      "bars": false,
      "cacheTimeout": null,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 4,
        "w": 5,
        "x": 19,
        "y": 2
      },
      "hiddenSeries": false,
      "id": 36,
      "interval": "1m",
      "legend": {
        "avg": false,
        "current": false,
        "hideEmpty": true,
        "hideZero": true,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.3.6",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": " sum by (OriginStatus,ServiceName) (count_over_time({app=~\"traefik\"} |~ \"RequestProtocol\" | json | OriginStatus >= 400 |__error__=\"\"[$__interval]))",
          "legendFormat": " {{ServiceName}} / {{OriginStatus}} ",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeFrom": "3h",
      "timeRegions": [],
      "timeShift": null,
      "title": " 4/5xx  Services",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": false,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:405",
          "decimals": 0,
          "format": "none",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": "0",
          "show": true
        },
        {
          "$$hashKey": "object:406",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Requests"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 100
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 0,
        "y": 6
      },
      "id": 22,
      "interval": "5m",
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "pluginVersion": "7.3.6",
      "targets": [
        {
          "expr": "count(sum by (ClientHost) (count_over_time({app=~\"traefik\"}|= \"RequestProtocol\" | json |  __error__=\"\" [$__interval])))",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "timeFrom": "5m",
      "timeShift": null,
      "title": "Users right now",
      "transformations": [],
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              }
            ]
          },
          "unit": "decbytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "IP Address"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 410
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 4,
        "w": 8,
        "x": 15,
        "y": 6
      },
      "id": 8,
      "interval": "1m",
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "pluginVersion": "7.3.6",
      "targets": [
        {
          "expr": "sum_over_time({app=~\"traefik\"}|= \"RequestProtocol\" | json | OriginStatus=200 | unwrap DownstreamContentSize |  __error__=\"\" [$__interval])",
          "legendFormat": "Bytes sent",
          "refId": "A"
        }
      ],
      "timeFrom": "5m",
      "timeShift": null,
      "title": "Total Bytes Sent",
      "transformations": [
        {
          "id": "reduce",
          "options": {
            "reducers": [
              "sum"
            ]
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Total": "Bytes Sent"
            }
          }
        }
      ],
      "transparent": true,
      "type": "stat"
    },
    {
      "aliasColors": {},
      "breakPoint": "50%",
      "cacheTimeout": null,
      "combine": {
        "label": "Others",
        "threshold": 0
      },
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Requests"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 100
              }
            ]
          }
        ]
      },
      "fontSize": "80%",
      "format": "short",
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 10
      },
      "id": 33,
      "interval": "5m",
      "legend": {
        "percentage": true,
        "show": true,
        "values": true
      },
      "legendType": "Right side",
      "links": [],
      "maxDataPoints": 3,
      "nullPointMode": "connected",
      "pieType": "pie",
      "pluginVersion": "7.3.4",
      "strokeWidth": 1,
      "targets": [
        {
          "expr": "sum by (RouterName) (count_over_time({app=~\"traefik\"}|= \"RequestProtocol\" | json |  __error__=\"\" [$__interval]))",
          "legendFormat": "{{RouterName}}",
          "refId": "A"
        }
      ],
      "timeFrom": "1h",
      "timeShift": null,
      "title": "Requests Route",
      "transformations": [],
      "transparent": true,
      "type": "grafana-piechart-panel",
      "valueName": "current"
    },
    {
      "datasource": "${DS_LOKI}",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 16,
        "x": 8,
        "y": 10
      },
      "id": 11,
      "interval": "",
      "options": {
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": false
      },
      "targets": [
        {
          "expr": "{app=~\"traefik\"} |= \"RequestProtocol\"| json  | line_format \"Status:{{.OriginStatus}} Client From {{.ClientAddr}} {{.RequestMethod}} {{.RequestAddr}}{{.RequestPath}} Route To {{.ServiceAddr}}\"",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "timeFrom": "5m",
      "timeShift": null,
      "title": "Logs",
      "transparent": true,
      "type": "logs"
    },
    {
      "aliasColors": {
        "95th percentile": "blue",
        "max latency": "super-light-blue"
      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "ns"
        },
        "overrides": []
      },
      "fill": 5,
      "fillGradient": 9,
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 0,
        "y": 18
      },
      "hiddenSeries": false,
      "id": 16,
      "interval": "30s",
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": true,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.3.6",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "$$hashKey": "object:232",
          "alias": "max latency",
          "dashes": true,
          "fillGradient": 3
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "quantile_over_time(0.95,{app=~\"traefik\"} |= \"RequestProtocol\"| json | unwrap Duration |  __error__=\"\"  [$__interval]) by (ServiceName)",
          "hide": false,
          "legendFormat": " {{ ServiceName }}",
          "refId": "C"
        }
      ],
      "thresholds": [
        {
          "$$hashKey": "object:168",
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 0.3,
          "yaxis": "left"
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "95th percentile of Request Time",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:178",
          "format": "ns",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:179",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {
        "95th percentile": "blue",
        "max latency": "super-light-blue"
      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "ns"
        },
        "overrides": []
      },
      "fill": 5,
      "fillGradient": 9,
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 8,
        "y": 18
      },
      "hiddenSeries": false,
      "id": 34,
      "interval": "30s",
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.3.6",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "$$hashKey": "object:232",
          "alias": "max latency",
          "dashes": true,
          "fillGradient": 3
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "max by (ServiceName) (max_over_time({app=~\"traefik\"} |= \"RequestProtocol\" |json | unwrap Duration |  __error__=\"\"  [$__interval]))",
          "hide": false,
          "legendFormat": "{{ ServiceName}}",
          "refId": "D"
        }
      ],
      "thresholds": [
        {
          "$$hashKey": "object:168",
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 0.3,
          "yaxis": "left"
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Max Age of Request Time",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:178",
          "format": "ns",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:179",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {
        "95th percentile": "blue",
        "max latency": "super-light-blue"
      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_LOKI}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "decbytes"
        },
        "overrides": []
      },
      "fill": 5,
      "fillGradient": 9,
      "gridPos": {
        "h": 9,
        "w": 8,
        "x": 16,
        "y": 18
      },
      "hiddenSeries": false,
      "id": 35,
      "interval": "30s",
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.3.6",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "$$hashKey": "object:232",
          "alias": "max latency",
          "dashes": true,
          "fillGradient": 3
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "sum by (ServiceName) (sum_over_time({app=~\"traefik\"} |= \"RequestProtocol\" |json | unwrap RequestContentSize |  __error__=\"\"  [$__interval]))",
          "hide": false,
          "legendFormat": "{{ ServiceName}}",
          "refId": "D"
        }
      ],
      "thresholds": [
        {
          "$$hashKey": "object:168",
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 0.3,
          "yaxis": "left"
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Requests Size",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "transparent": true,
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:178",
          "format": "decbytes",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:179",
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "refresh": false,
  "schemaVersion": 26,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ]
  },
  "timezone": "",
  "title": "Traefik On K8s Via Loki",
  "uid": "fJarCeaGk",
  "version": 1
}