Grafana dashboard: Syslog
Description: Telegraf / InfluxDB / Grafana as syslog receiver

# Telegraf / InfluxDB / Grafana as syslog receiver

UPDATE 03/2021: Fixed the hostname variable to filter for different syslog sources

UPDATE 03/2021: Message query now uses a text box variable instead of ad-hoc query, which stopped working with Grafana 7.x

UPDATE 09/2020: Rebuilt the dashboard to take advantage of the new table panel possibilities with Grafana 7.x, e.g. dedicated severity colors.

The Syslog dashboard shows a statistics graph panel at the top, based on the timeframe chosen. Plus a table view of all messages within this timeframe, including the usual columns like message time, appname, host, severity and message text. You can zoom into specific timeframes (syslog peaks or massive errors) by just marking the area by mouse in the graph panel. The table view adjusts accordingly.

It also provides some extra filters based on appname, hostname, severity and message text. 

## How to store classic UDP/514 syslog in InfluxDB via rsyslog

Because Telegraf only accepts TCP syslog messages in a certain format (RFC5424), the rsyslog daemon is used to receive classic RFC3164 Syslog messages via UDP port 514 and pipe them to the local Telegraf instance. So the first step is to adapt the rsyslog.conf to our needs:

    $WorkDirectory /var/spool/rsyslog  
    $ActionQueueType LinkedList  
    $ActionQueueFileName srvrfwd  
    $ActionResumeRetryCount -1  
    $ActionQueueSaveOnShutdown on  
    $ModLoad imudp #loads the udp module  
    $UDPServerAddress localhost 
    $UDPServerRun 514 
    *.* @@(o)127.0.0.1:6514;RSYSLOG_SyslogProtocol23Format

If you would like to filter between local and external syslog messages, there is the possibility of very sophisticated rulesets, for example:

    :hostname, contains, "grafanapi"
    *.notice @@(o)127.0.0.1:6514;RSYSLOG_SyslogProtocol23Format
    :hostname, !contains, "grafanapi"
    *.* @@(o)127.0.0.1:6514;RSYSLOG_SyslogProtocol23Format

Local logging messages are only forwarded when the severity level is equal or greater then notice. External Syslog messages (hostname != grafanapi) will be forwarded to Telegraf regardless of the severity level.

Visit my blog post over at [NWMichl Blog](https://nwmichl.net/2020/03/15/telegraf-influxdb-grafana-as-syslog-receiver/ "https://nwmichl.net/2020/03/15/telegraf-influxdb-grafana-as-syslog-receiver/") for full documentation and a syslog-ng alternative.

{
  "__inputs": [
    {
      "name": "DS_INFLUXDB",
      "label": "InfluxDB",
      "description": "",
      "type": "datasource",
      "pluginId": "influxdb",
      "pluginName": "InfluxDB"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "7.4.0"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "influxdb",
      "name": "InfluxDB",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Telegraf / InfluxDB / Grafana as syslog receiver",
  "editable": true,
  "gnetId": 12433,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1616092347593,
  "links": [],
  "panels": [
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_INFLUXDB}",
      "decimals": 0,
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "hiddenSeries": false,
      "id": 10,
      "interval": "",
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "sort": null,
        "sortDesc": null,
        "total": true,
        "values": true
      },
      "lines": false,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "connected",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "Info",
          "color": "rgb(80, 80, 80)",
          "stack": "A"
        },
        {
          "alias": "Notice",
          "color": "rgb(182, 182, 182)",
          "stack": "A"
        },
        {
          "alias": "Warning",
          "color": "#E0B400",
          "stack": "A"
        },
        {
          "alias": "Error",
          "color": "#FF780A",
          "stack": "A"
        },
        {
          "alias": "Critical",
          "color": "#E02F44",
          "stack": "A"
        },
        {
          "alias": "Alert",
          "color": "#8F3BB8",
          "stack": "A"
        },
        {
          "alias": "Emergency",
          "color": "#8F3BB8",
          "stack": "A"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "alias": "Info",
          "groupBy": [
            {
              "params": [
                "1m"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": [
            {
              "key": "severity",
              "operator": "=",
              "value": "info"
            },
            {
              "condition": "AND",
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        },
        {
          "alias": "Notice",
          "groupBy": [
            {
              "params": [
                "1m"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "B",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": [
            {
              "key": "severity",
              "operator": "=",
              "value": "notice"
            },
            {
              "condition": "AND",
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        },
        {
          "alias": "Warning",
          "groupBy": [
            {
              "params": [
                "1m"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "D",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": [
            {
              "key": "severity",
              "operator": "=",
              "value": "warning"
            },
            {
              "condition": "AND",
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        },
        {
          "alias": "Error",
          "groupBy": [
            {
              "params": [
                "1m"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "C",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": [
            {
              "key": "severity",
              "operator": "=",
              "value": "err"
            },
            {
              "condition": "AND",
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        },
        {
          "alias": "Critical",
          "groupBy": [
            {
              "params": [
                "1m"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "E",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": [
            {
              "key": "severity",
              "operator": "=",
              "value": "crit"
            },
            {
              "condition": "AND",
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        },
        {
          "alias": "Alert",
          "groupBy": [
            {
              "params": [
                "1m"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "F",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": [
            {
              "key": "severity",
              "operator": "=",
              "value": "alert"
            },
            {
              "condition": "AND",
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        },
        {
          "alias": "Emergency",
          "groupBy": [
            {
              "params": [
                "1m"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "G",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "count"
              }
            ]
          ],
          "tags": [
            {
              "key": "severity",
              "operator": "=",
              "value": "emerg"
            },
            {
              "condition": "AND",
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "syslog count",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": 0,
          "format": "none",
          "label": "Messages / min",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "datasource": "${DS_INFLUXDB}",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "left",
            "displayMode": "auto",
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "dark-purple",
                "value": null
              },
              {
                "color": "dark-red",
                "value": 2
              },
              {
                "color": "dark-orange",
                "value": 3
              },
              {
                "color": "dark-yellow",
                "value": 4
              },
              {
                "color": "rgb(150, 150, 150)",
                "value": 5
              },
              {
                "color": "rgb(51, 51, 51)",
                "value": 6
              },
              {
                "color": "rgb(5, 5, 5)",
                "value": 7
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "severity_code"
            },
            "properties": [
              {
                "id": "mappings",
                "value": [
                  {
                    "from": "",
                    "id": 0,
                    "text": "Debug",
                    "to": "",
                    "type": 1,
                    "value": "7"
                  },
                  {
                    "from": "",
                    "id": 1,
                    "text": "Info",
                    "to": "",
                    "type": 1,
                    "value": "6"
                  },
                  {
                    "from": "",
                    "id": 2,
                    "text": "Notice",
                    "to": "",
                    "type": 1,
                    "value": "5"
                  },
                  {
                    "from": "",
                    "id": 3,
                    "text": "Warning",
                    "to": "",
                    "type": 1,
                    "value": "4"
                  },
                  {
                    "from": "",
                    "id": 4,
                    "text": "Error",
                    "to": "",
                    "type": 1,
                    "value": "3"
                  },
                  {
                    "from": "",
                    "id": 5,
                    "text": "Critical",
                    "to": "",
                    "type": 1,
                    "value": "2"
                  },
                  {
                    "from": "",
                    "id": 6,
                    "text": "Alert",
                    "to": "",
                    "type": 1,
                    "value": "1"
                  },
                  {
                    "from": "",
                    "id": 7,
                    "text": "Emergency",
                    "to": "",
                    "type": 1,
                    "value": "0"
                  }
                ]
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              },
              {
                "id": "custom.width",
                "value": 119
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Time"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 163
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "hostname"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 194
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "appname"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 264
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 24,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 12,
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "desc": true,
            "displayName": "Time"
          }
        ]
      },
      "pluginVersion": "7.4.0",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "hostname"
              ],
              "type": "tag"
            },
            {
              "params": [
                "appname"
              ],
              "type": "tag"
            }
          ],
          "measurement": "syslog",
          "orderByTime": "ASC",
          "policy": "default",
          "refId": "A",
          "resultFormat": "table",
          "select": [
            [
              {
                "params": [
                  "severity_code"
                ],
                "type": "field"
              }
            ],
            [
              {
                "params": [
                  "message"
                ],
                "type": "field"
              }
            ]
          ],
          "tags": [
            {
              "key": "hostname",
              "operator": "=~",
              "value": "/^$hostname$/"
            },
            {
              "condition": "AND",
              "key": "appname",
              "operator": "=~",
              "value": "/^$appname$/"
            },
            {
              "condition": "AND",
              "key": "severity",
              "operator": "=~",
              "value": "/^$severity$/"
            },
            {
              "condition": "AND",
              "key": "message",
              "operator": "=~",
              "value": "/$Query/"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Syslog Messages",
      "type": "table"
    }
  ],
  "refresh": "5s",
  "schemaVersion": 27,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": "",
        "current": {},
        "datasource": "${DS_INFLUXDB}",
        "definition": "SHOW TAG VALUES FROM syslog WITH KEY=appname",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": "Appname",
        "multi": true,
        "name": "appname",
        "options": [],
        "query": "SHOW TAG VALUES FROM syslog WITH KEY=appname",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": "",
        "current": {},
        "datasource": "${DS_INFLUXDB}",
        "definition": "SHOW TAG VALUES FROM syslog WITH KEY=hostname",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": "Hostname",
        "multi": true,
        "name": "hostname",
        "options": [],
        "query": "SHOW TAG VALUES FROM syslog WITH KEY=hostname",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": "",
        "current": {},
        "datasource": "${DS_INFLUXDB}",
        "definition": "SHOW TAG VALUES FROM syslog WITH KEY=severity",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": "Severity",
        "multi": true,
        "name": "severity",
        "options": [],
        "query": "SHOW TAG VALUES FROM syslog WITH KEY=severity",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {
          "selected": false,
          "text": "",
          "value": ""
        },
        "description": "Querystring",
        "error": null,
        "hide": 0,
        "label": "MessageQuery",
        "name": "Query",
        "options": [
          {
            "selected": true,
            "text": "",
            "value": ""
          }
        ],
        "query": "",
        "skipUrlSync": false,
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      " 10s",
      " 30s",
      " 1m",
      " 5m"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Syslog",
  "uid": "TIYeHqRgZ",
  "version": 7
}