Grafana dashboard: Hydrolix - HydroLogs
Description: Hydrolix Logs dashboard, allows you to filter and drill down into any services running and get the last logs information

This dashboard helps users visualise Hydrolix logs collected in the K8s deployment.

{
  "__inputs": [
    {
      "name": "DS_GCP_- PROD TEST",
      "label": "Hydrolix",
      "description": "Clickhouse source for Hydrolix",
      "type": "datasource",
      "pluginId": "grafana-clickhouse-datasource",
      "pluginName": "ClickHouse"
    }
  ],
  "__elements": {},
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "9.2.1"
    },
    {
      "type": "datasource",
      "id": "grafana-clickhouse-datasource",
      "name": "ClickHouse",
      "version": "2.0.1"
    },
    {
      "type": "panel",
      "id": "logs",
      "name": "Logs",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "gnetId": 17165,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "datasource": {
        "type": "grafana-clickhouse-datasource",
        "uid": "${DS_GCP_- PROD TEST}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "log_level info"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "green",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "log_level warning"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "orange",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "log_level error"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "dark-red",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "log_level trace"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "log_level 1"
            },
            "properties": [
              {
                "id": "custom.hideFrom",
                "value": {
                  "legend": true,
                  "tooltip": true,
                  "viz": true
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 6,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 4,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "builderOptions": {
            "database": "hydro",
            "fields": [
              "level"
            ],
            "filters": [],
            "groupBy": [
              "level"
            ],
            "metrics": [
              {
                "aggregation": "count",
                "field": "level"
              }
            ],
            "mode": "trend",
            "orderBy": [],
            "table": "logs",
            "timeField": "timestamp",
            "timeFieldType": "DateTime64(3)"
          },
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "${DS_GCP_- PROD TEST}"
          },
          "format": 0,
          "meta": {
            "builderOptions": {
              "database": "hydro",
              "fields": [
                "level"
              ],
              "filters": [],
              "groupBy": [
                "level"
              ],
              "metrics": [
                {
                  "aggregation": "count",
                  "field": "level"
                }
              ],
              "mode": "trend",
              "orderBy": [],
              "table": "logs",
              "timeField": "timestamp",
              "timeFieldType": "DateTime64(3)"
            }
          },
          "queryType": "sql",
          "rawSql": "SELECT $__timeInterval(timestamp) as time, level, count(level) as log_level\nFROM hydro.logs\nWHERE $__timeFilter_ms(timestamp)\nAND $__conditionalAll(app IN (${app}), $app)\nAND $__conditionalAll(level IN (${level}), $level)\nAND $__conditionalAll(container IN (${container}), $container)\nAND $__conditionalAll(pool IN (${pool}), $pool)\nAND $__conditionalAll(component IN (${component}), $component)\nAND (\n  if('${logs:text}' = '', true, message LIKE '%${logs}%') \n  OR if('${logs:text}' = '', true, error LIKE '%${logs}%') \n  OR if('${logs:text}' = '', true, exception LIKE '%${logs}%')\n  OR if('${logs:text}' = '', true, query LIKE '%${logs}%')\n  )\nGROUP BY level, time\nORDER BY time ASC\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
          "refId": "A"
        }
      ],
      "title": "Log Level",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "grafana-clickhouse-datasource",
        "uid": "${DS_GCP_- PROD TEST}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 13,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 5,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "sum"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": true,
          "sortBy": "Total",
          "sortDesc": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "targets": [
        {
          "builderOptions": {
            "database": "hydro",
            "fields": [
              "level"
            ],
            "filters": [],
            "groupBy": [
              "level"
            ],
            "metrics": [
              {
                "aggregation": "count",
                "field": "level"
              }
            ],
            "mode": "trend",
            "orderBy": [],
            "table": "logs",
            "timeField": "timestamp",
            "timeFieldType": "DateTime64(3)"
          },
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "${DS_GCP_- PROD TEST}"
          },
          "format": 0,
          "meta": {
            "builderOptions": {
              "database": "hydro",
              "fields": [
                "level"
              ],
              "filters": [],
              "groupBy": [
                "level"
              ],
              "metrics": [
                {
                  "aggregation": "count",
                  "field": "level"
                }
              ],
              "mode": "trend",
              "orderBy": [],
              "table": "logs",
              "timeField": "timestamp",
              "timeFieldType": "DateTime64(3)"
            }
          },
          "queryType": "sql",
          "rawSql": "SELECT $__timeInterval(timestamp) as time, app, count(app) as service\nFROM hydro.logs\nWHERE $__timeFilter_ms(timestamp)\nAND $__conditionalAll(app IN (${app}), $app)\nAND $__conditionalAll(level IN (${level}), $level)\nAND $__conditionalAll(container IN (${container}), $container)\nAND $__conditionalAll(pool IN (${pool}), $pool)\nAND $__conditionalAll(component IN (${component}), $component)\nAND (\n  if('${logs:text}' = '', true, message LIKE '%${logs}%') \n  OR if('${logs:text}' = '', true, error LIKE '%${logs}%') \n  OR if('${logs:text}' = '', true, exception LIKE '%${logs}%')\n  OR if('${logs:text}' = '', true, query LIKE '%${logs}%')\n  )\nGROUP BY app, time\nORDER BY time ASC\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
          "refId": "A"
        }
      ],
      "title": "Log count per app",
      "type": "timeseries"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 7,
      "panels": [
        {
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "${DS_GCP_- PROD TEST}"
          },
          "gridPos": {
            "h": 17,
            "w": 24,
            "x": 0,
            "y": 20
          },
          "id": 2,
          "options": {
            "dedupStrategy": "none",
            "enableLogDetails": true,
            "prettifyLogMessage": true,
            "showCommonLabels": false,
            "showLabels": false,
            "showTime": false,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "targets": [
            {
              "datasource": {
                "type": "grafana-clickhouse-datasource",
                "uid": "${DS_GCP_- PROD TEST}"
              },
              "format": 1,
              "meta": {
                "builderOptions": {
                  "fields": [],
                  "limit": 100,
                  "mode": "list"
                }
              },
              "queryType": "sql",
              "rawSql": "SELECT * EXCEPT (catchall, \"kubernetes.pod_annotations\", \"kubernetes.pod_labels\", \"kubernetes.container_name\", \"kubernetes.namespace_labels\", files, \"kubernetes.pod_ips\"), toString(kubernetes.pod_annotations) as \"kubernetes.pod_annotations\", toString(kubernetes.pod_labels) as \"kubernetes.pod_labels\", toString(catchall) as \"catchall\", toString(kubernetes.container_name) as \"kubernetes.container_name\", toString(kubernetes.namespace_labels) as \"kubernetes.namespace_labels\", toString(kubernetes.pod_ips) as \"kubernetes.pod_ips\", toString(files) as files\nFROM hydro.logs\nWHERE $__timeFilter_ms(timestamp) \nAND $__conditionalAll(app IN (${app}), $app)\nAND $__conditionalAll(level IN (${level}), $level)\nAND $__conditionalAll(container IN (${container}), $container)\nAND $__conditionalAll(pool IN (${pool}), $pool)\nAND $__conditionalAll(component IN (${component}), $component)\nAND (\n  if('${logs:text}' = '', true, message LIKE '%${logs}%') \n  OR if('${logs:text}' = '', true, error LIKE '%${logs}%') \n  OR if('${logs:text}' = '', true, exception LIKE '%${logs}%')\n  OR if('${logs:text}' = '', true, query LIKE '%${logs}%')\n  )\nORDER BY timestamp DESC LIMIT 100\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
              "refId": "A"
            }
          ],
          "title": "Hydro Logs",
          "type": "logs"
        }
      ],
      "title": "Raw Logs",
      "type": "row"
    }
  ],
  "refresh": false,
  "schemaVersion": 37,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": "",
        "current": {},
        "datasource": {
          "type": "grafana-clickhouse-datasource",
          "uid": "${DS_GCP_- PROD TEST}"
        },
        "definition": "SELECT app FROM hydro.logs WHERE $__timeFilter_ms(timestamp) GROUP BY app LIMIT 50 SETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "hide": 0,
        "includeAll": true,
        "label": "Service",
        "multi": true,
        "name": "app",
        "options": [],
        "query": "SELECT app FROM hydro.logs WHERE $__timeFilter_ms(timestamp) GROUP BY app LIMIT 50 SETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {},
        "datasource": {
          "type": "grafana-clickhouse-datasource",
          "uid": "${DS_GCP_- PROD TEST}"
        },
        "definition": "SELECT component\nFROM hydro.logs \nWHERE $__timeFilter_ms(timestamp) \nAND $__conditionalAll(app IN (${app}), $app)\nGROUP BY component\nLIMIT 100\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "component",
        "options": [],
        "query": "SELECT component\nFROM hydro.logs \nWHERE $__timeFilter_ms(timestamp) \nAND $__conditionalAll(app IN (${app}), $app)\nGROUP BY component\nLIMIT 100\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {},
        "datasource": {
          "type": "grafana-clickhouse-datasource",
          "uid": "${DS_GCP_- PROD TEST}"
        },
        "definition": "SELECT container\nFROM hydro.logs \nWHERE $__timeFilter_ms(timestamp) \nAND $__conditionalAll(app IN (${app}), $app)\nGROUP BY container\nLIMIT 100\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "container",
        "options": [],
        "query": "SELECT container\nFROM hydro.logs \nWHERE $__timeFilter_ms(timestamp) \nAND $__conditionalAll(app IN (${app}), $app)\nGROUP BY container\nLIMIT 100\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {},
        "datasource": {
          "type": "grafana-clickhouse-datasource",
          "uid": "${DS_GCP_- PROD TEST}"
        },
        "definition": "SELECT arrayJoin( ['info','warning','error','debug','trace'])",
        "hide": 0,
        "includeAll": true,
        "label": "Log Level",
        "multi": true,
        "name": "level",
        "options": [],
        "query": "SELECT arrayJoin( ['info','warning','error','debug','trace'])",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      },
      {
        "current": {
          "selected": false,
          "text": "",
          "value": ""
        },
        "description": "Search for *text* in the logs data",
        "hide": 0,
        "label": "Search Logs",
        "name": "logs",
        "options": [
          {
            "selected": true,
            "text": "",
            "value": ""
          }
        ],
        "query": "",
        "skipUrlSync": false,
        "type": "textbox"
      },
      {
        "current": {},
        "datasource": {
          "type": "grafana-clickhouse-datasource",
          "uid": "${DS_GCP_- PROD TEST}"
        },
        "definition": "SELECT pool\nFROM hydro.logs \nWHERE $__timeFilter_ms(timestamp) \nAND $__conditionalAll(app IN (${app}), $app)\nGROUP BY pool\nLIMIT 100\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "name": "pool",
        "options": [],
        "query": "SELECT pool\nFROM hydro.logs \nWHERE $__timeFilter_ms(timestamp) \nAND $__conditionalAll(app IN (${app}), $app)\nGROUP BY pool\nLIMIT 100\nSETTINGS hdx_query_max_timerange_sec=604800, hdx_query_max_execution_time=60,hdx_query_admin_comment='${__user.login}'",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Hydrolix - HydroLogs",
  "uid": "d14EBDV4z",
  "version": 7,
  "weekStart": "",
  "description": "Hydrolix Logs dashboard, allows you to filter and drill down into any services running and get the last logs information"
}