Repository: NetApp/harvest
Pull request #1583 (2022-12-02 07:43:48)

fix: change display metrics map value to metric name


Patch:
diff --git a/cmd/collectors/ems/ems.go b/cmd/collectors/ems/ems.go
index 439b6af3..9ca5bd54 100644
--- a/cmd/collectors/ems/ems.go
+++ b/cmd/collectors/ems/ems.go
@@ -473,13 +473,6 @@ func (e *Ems) PollData() (map[string]*matrix.Matrix, error) {
 		instanceCount += len(v.GetInstances())
 	}
 
-	e.Logger.Info().
-		Int("instances", instanceCount).
-		Uint64("dataPoints", count).
-		Str("apiTime", apiD.String()).
-		Str("parseTime", parseD.String()).
-		Msg("Collected")
-
 	_ = e.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds())
 	_ = e.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds())
 	_ = e.Metadata.LazySetValueUint64("metrics", "data", count)
diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go
index 3a501c59..60e9500d 100644
--- a/cmd/tools/grafana/grafana.go
+++ b/cmd/tools/grafana/grafana.go
@@ -946,19 +946,8 @@ grafana export --addr my.grafana.server:3000 --serverfolder server_folder --dire
 }
 
 func init() {
-	Cmd.AddCommand(importCmd, exportCmd)
-
-	Cmd.PersistentFlags().StringVar(&opts.config, "config", "./harvest.yml", "harvest config file path")
-	Cmd.PersistentFlags().StringVarP(&opts.addr, "addr", "a", "http://127.0.0.1:3000", "Address of Grafana server (IP, FQDN or hostname)")
-	Cmd.PersistentFlags().StringVarP(&opts.token, "token", "t", "", "API token issued by Grafana server for authentication")
-	Cmd.PersistentFlags().StringVarP(&opts.prefix, "prefix", "p", "", "Use global metric prefix in queries")
-	Cmd.PersistentFlags().StringVarP(&opts.datasource, "datasource", "s", grafanaDataSource, "Grafana datasource for the dashboards")
-	Cmd.PersistentFlags().BoolVarP(&opts.variable, "variable", "v", false, "Use datasource as variable, overrides: --datasource")
-	Cmd.PersistentFlags().BoolVarP(&opts.useHTTPS, "https", "S", false, "Use HTTPS")
-	Cmd.PersistentFlags().BoolVarP(&opts.overwrite, "overwrite", "o", false, "Overwrite existing dashboard with same title")
-	Cmd.PersistentFlags().BoolVarP(&opts.useInsecureTLS, "insecure", "k", false, "Allow insecure server connections when using SSL")
-	Cmd.PersistentFlags().StringVarP(&opts.serverfolder.name, "serverfolder", "f", "", "Grafana folder name for dashboards")
-	Cmd.PersistentFlags().StringVarP(&opts.dir, "directory", "d", "", "When importing, import dashboards from this local directory.\nWhen exporting, local directory to write dashboards to")
+	Cmd.AddCommand(importCmd, exportCmd, metricsCmd)
+	addFlags(importCmd, exportCmd)
 
 	importCmd.PersistentFlags().StringSliceVar(&opts.labels, "labels", nil,
 		"For each label, create a variable and add as chained query to other variables")
@@ -966,6 +955,26 @@ func init() {
 		"Modify the dashboards to add multi-select dropdowns for each variable")
 	_ = importCmd.PersistentFlags().MarkHidden("multi")
 
-	_ = Cmd.MarkPersistentFlagRequired("serverfolder")
-	_ = Cmd.MarkPersistentFlagRequired("directory")
+	metricsCmd.PersistentFlags().StringVarP(&opts.dir, "directory", "d",
+		"", "local directory that contains dashboards (searched recursively).")
+
+}
+
+func addFlags(commands ...*cobra.Command) {
+	for _, command := range commands {
+		cmd := command
+		cmd.PersistentFlags().StringVar(&opts.config, "config", "./harvest.yml", "harvest config file path")
+		cmd.PersistentFlags().StringVarP(&opts.addr, "addr", "a", "http://127.0.0.1:3000", "Address of Grafana server (IP, FQDN or hostname)")
+		cmd.PersistentFlags().StringVarP(&opts.token, "token", "t", "", "API token issued by Grafana server for authentication")
+		cmd.PersistentFlags().StringVarP(&opts.prefix, "prefix", "p", "", "Use global metric prefix in queries")
+		cmd.PersistentFlags().StringVarP(&opts.datasource, "datasource", "s", grafanaDataSource, "Grafana datasource for the dashboards")
+		cmd.PersistentFlags().BoolVarP(&opts.variable, "variable", "v", false, "Use datasource as variable, overrides: --datasource")
+		cmd.PersistentFlags().BoolVarP(&opts.useHTTPS, "https", "S", false, "Use HTTPS")
+		cmd.PersistentFlags().BoolVarP(&opts.overwrite, "overwrite", "o", false, "Overwrite existing dashboard with same title")
+		cmd.PersistentFlags().BoolVarP(&opts.useInsecureTLS, "insecure", "k", false, "Allow insecure server connections when using SSL")
+		cmd.PersistentFlags().StringVarP(&opts.serverfolder.name, "serverfolder", "f", "", "Grafana folder name for dashboards")
+		cmd.PersistentFlags().StringVarP(&opts.dir, "directory", "d", "", "When importing, import dashboards from this local directory.\nWhen exporting, local directory to write dashboards to")
+		_ = cmd.MarkPersistentFlagRequired("serverfolder")
+		_ = cmd.MarkPersistentFlagRequired("directory")
+	}
 }
diff --git a/cmd/tools/grafana/metrics.go b/cmd/tools/grafana/metrics.go
new file mode 100644
index 00000000..d6a37155
--- /dev/null
+++ b/cmd/tools/grafana/metrics.go
@@ -0,0 +1,179 @@
+package grafana
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/tidwall/gjson"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+var varRe = regexp.MustCompile(`\$(\w+)`)
+var metricRe = regexp.MustCompile(`(\w+)\{`)
+
+var metricsCmd = &cobra.Command{
+	Use:   "metrics",
+	Short: "print dashboard metrics",
+	Run:   doMetrics,
+}
+
+func doMetrics(_ *cobra.Command, _ []string) {
+	adjustOptions()
+	validateImport()
+	visitDashboards(opts.dir, func(path string, data []byte) {
+		visitExpressionsAndQueries(path, data)
+	})
+}
+
+type exprP struct {
+	path string
+	expr string
+	vars []string
+}
+
+func visitExpressionsAndQueries(path string, data []byte) {
+	// collect all expressions
+	expressions := make([]exprP, 0)
+	gjson.GetBytes(data, "panels").ForEach(func(key, value gjson.Result) bool {
+		doTarget("", key, value, func(path string, expr string, format string) {
+			expressions = append(expressions, newExpr(path, expr))
+		})
+		value.Get("panels").ForEach(func(key2, value2 gjson.Result) bool {
+			pathPrefix := fmt.Sprintf("panels[%d].", key.Int())
+			doTarget(pathPrefix, key2, value2, func(path string, expr string, format string) {
+				expressions = append(expressions, newExpr(path, expr))
+			})
+			return true
+		})
+		return true
+	})
+
+	metricsSeen := make(map[string]struct{})
+	for _, expr := range expressions {
+		allMatches := metricRe.FindAllStringSubmatch(expr.expr, -1)
+		for _, match := range allMatches {
+			m := match[1]
+			if len(m) == 0 {
+				continue
+			}
+			metricsSeen[m] = struct{}{}
+		}
+	}
+
+	// collect all variables
+	variables := allVariables(data)
+	for _, v := range variables {
+		allMatches := metricRe.FindAllStringSubmatch(v.query, -1)
+		for _, match := range allMatches {
+			m := match[1]
+			if len(m) == 0 {
+				continue
+			}
+			metricsSeen[m] = struct{}{}
+		}
+	}
+
+	fmt.Printf("%s\n", shortPath(path))
+	metrics := setToList(metricsSeen)
+	for _, metric := range metrics {
+		fmt.Printf("- %s\n", metric)
+	}
+	fmt.Println()
+}
+
+func setToList(seen map[string]struct{}) []string {
+	list := make([]string, 0)
+	for k := range seen {
+		list = append(list, k)
+	}
+	sort.Strings(list)
+	return list
+}
+
+type variable struct {
+	name  string
+	kind  string
+	query string
+	path  string
+}
+
+func allVariables(data []byte) []variable {
+	variables := make([]variable, 0)
+	gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool {
+		// The datasource variable can be ignored
+		if value.Get("type").String() == "datasource" {
+			return true
+		}
+
+		variables = append(variables, variable{
+			name:  value.Get("name").String(),
+			kind:  value.Get("type").String(),
+			query: value.Get("query.query").String(),
+			path:  key.String(),
+		})
+		return true
+	})
+	return variables
+}
+
+func newExpr(path string, expr string) exprP {
+	allMatches := varRe.FindAllStringSubmatch(expr, -1)
+	vars := make([]string, 0, len(allMatches))
+	for _, match := range allMatches {
+		vars = append(vars, match[1])
+	}
+	return exprP{
+		path: path,
+		expr: expr,
+		vars: vars,
+	}
+}
+
+func doTarget(pathPrefix string, key gjson.Result, value gjson.Result,
+	exprFunc func(path string, expr string, format string)) {
+	kind := value.Get("type").String()
+	if kind == "row" {
+		return
+	}
+	path := fmt.Sprintf("%spanels[%d]", pathPrefix, key.Int())
+	targetsSlice := value.Get("targets").Array()
+	for i, targetN := range targetsSlice {
+		expr := targetN.Get("expr").String()
+		pathWithTarget := path + ".targets[" + strconv.Itoa(i) + "]"
+		exprFunc(pathWithTarget, expr, kind)
+	}
+}
+
+func visitDashboards(dir string, eachDash func(path string, data []byte)) {
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if strings.Contains(path, "influxdb") {
+			return nil
+		}
+		if err != nil {
+			log.Fatal("failed to read directory:", err)
+		}
+		ext := filepath.Ext(path)
+		if ext != ".json" {
+			return nil
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			log.Fatalf("failed to read dashboards path=%s err=%v", path, err)
+		}
+		eachDash(path, data)
+		return nil
+	})
+	if err != nil {
+		log.Fatal("failed to read dashboards:", err)
+	}
+}
+
+func shortPath(dashPath string) string {
+	splits := strings.Split(dashPath, string(filepath.Separator))
+	return strings.Join(splits[len(splits)-2:], string(filepath.Separator))
+}
diff --git a/conf/restperf/9.12.0/nfsv3_node.yaml b/conf/restperf/9.12.0/nfsv3_node.yaml
index 843eb91d..1d5c010d 100644
--- a/conf/restperf/9.12.0/nfsv3_node.yaml
+++ b/conf/restperf/9.12.0/nfsv3_node.yaml
@@ -9,7 +9,7 @@ global_labels:
 counters:
   - ^^id
   - ^name                          => node
-  - ops
+  - ops                            => total_ops  # "nfs_ops" already used in system_node.yaml
   - read_ops
   - write_ops
   - throughput
diff --git a/grafana/dashboards/cmode/cdot.json b/grafana/dashboards/cmode/cdot.json
index bb83f9e4..42086e6f 100644
--- a/grafana/dashboards/cmode/cdot.json
+++ b/grafana/dashboards/cmode/cdot.json
@@ -1649,7 +1649,8 @@
             "legend": {
               "calcs": [
                 "mean",
-                "lastNotNull"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -1670,7 +1671,7 @@
               "refId": "A"
             }
           ],
-          "title": "Top Average Throughput by Volumes ",
+          "title": "Top Average Throughput by Volumes",
           "transformations": [],
           "type": "timeseries"
         },
diff --git a/grafana/dashboards/cmode/cluster.json b/grafana/dashboards/cmode/cluster.json
index 0ac7362f..3cfea907 100644
--- a/grafana/dashboards/cmode/cluster.json
+++ b/grafana/dashboards/cmode/cluster.json
@@ -2889,9 +2889,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -2980,9 +2980,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/lun.json b/grafana/dashboards/cmode/lun.json
index 19fecf0d..407b4195 100644
--- a/grafana/dashboards/cmode/lun.json
+++ b/grafana/dashboards/cmode/lun.json
@@ -3299,7 +3299,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -3387,7 +3389,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -3850,9 +3854,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -3944,9 +3948,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4038,9 +4042,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4145,9 +4149,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4239,9 +4243,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4375,9 +4379,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/node.json b/grafana/dashboards/cmode/node.json
index c32f520c..fa871f3b 100644
--- a/grafana/dashboards/cmode/node.json
+++ b/grafana/dashboards/cmode/node.json
@@ -2796,9 +2796,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -2891,9 +2891,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -2987,9 +2987,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/power.json b/grafana/dashboards/cmode/power.json
index 00f58710..982604e7 100644
--- a/grafana/dashboards/cmode/power.json
+++ b/grafana/dashboards/cmode/power.json
@@ -562,9 +562,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -669,9 +669,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -760,9 +760,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -850,9 +850,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -941,9 +941,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/grafana/dashboards/cmode/qtree.json b/grafana/dashboards/cmode/qtree.json
index 1b1f42f0..cf7130f0 100644
--- a/grafana/dashboards/cmode/qtree.json
+++ b/grafana/dashboards/cmode/qtree.json
@@ -519,8 +519,9 @@
             "legend": {
               "calcs": [
                 "min",
-                "max",
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -605,8 +606,9 @@
             "legend": {
               "calcs": [
                 "min",
-                "max",
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/shelf.json b/grafana/dashboards/cmode/shelf.json
index a604e0a9..501e3b9a 100644
--- a/grafana/dashboards/cmode/shelf.json
+++ b/grafana/dashboards/cmode/shelf.json
@@ -1440,9 +1440,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1531,9 +1531,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/grafana/dashboards/cmode/snapmirror.json b/grafana/dashboards/cmode/snapmirror.json
index 0571b4db..0f4dc4d5 100644
--- a/grafana/dashboards/cmode/snapmirror.json
+++ b/grafana/dashboards/cmode/snapmirror.json
@@ -1094,9 +1094,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1191,9 +1192,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1298,9 +1300,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1404,9 +1407,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1508,9 +1512,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1605,9 +1610,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1711,9 +1717,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1817,9 +1824,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1939,9 +1947,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -2035,9 +2044,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/grafana/dashboards/cmode/svm.json b/grafana/dashboards/cmode/svm.json
index afc3fb75..abe6f6c0 100644
--- a/grafana/dashboards/cmode/svm.json
+++ b/grafana/dashboards/cmode/svm.json
@@ -6269,7 +6269,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "mean"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/volume.json b/grafana/dashboards/cmode/volume.json
index 768d1745..c9fe8002 100644
--- a/grafana/dashboards/cmode/volume.json
+++ b/grafana/dashboards/cmode/volume.json
@@ -436,7 +436,8 @@
         "legend": {
           "calcs": [
             "mean",
-            "lastNotNull"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -526,7 +527,8 @@
         "legend": {
           "calcs": [
             "mean",
-            "lastNotNull"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go
index 60eb9d80..52ff07ab 100644
--- a/pkg/matrix/matrix.go
+++ b/pkg/matrix/matrix.go
@@ -26,7 +26,7 @@ type Matrix struct {
 	globalLabels   *dict.Dict
 	instances      map[string]*Instance
 	metrics        map[string]Metric // ONTAP metric name => metric (in templates, this is left side)
-	displayMetrics map[string]Metric // display name of metric to => metric (in templates, this is right side)
+	displayMetrics map[string]string // display name of metric to => metric name (in templates, this is right side)
 	exportOptions  *node.Node
 	exportable     bool
 }
@@ -36,7 +36,7 @@ func New(uuid, object string, identifier string) *Matrix {
 	me.globalLabels = dict.New()
 	me.instances = make(map[string]*Instance, 0)
 	me.metrics = make(map[string]Metric, 0)
-	me.displayMetrics = make(map[string]Metric, 0)
+	me.displayMetrics = make(map[string]string, 0)
 	me.exportable = true
 	return &me
 }
@@ -82,7 +82,7 @@ func (m *Matrix) Clone(withData, withMetrics, withInstances bool) *Matrix {
 		for key, metric := range m.GetMetrics() {
 			c := metric.Clone(withData)
 			clone.metrics[key] = c
-			clone.displayMetrics[c.GetName()] = c
+			clone.displayMetrics[c.GetName()] = key
 		}
 	}
 
@@ -98,8 +98,8 @@ func (m *Matrix) Reset() {
 }
 
 func (m *Matrix) DisplayMetric(name string) Metric {
-	if metric, has := m.displayMetrics[name]; has {
-		return metric
+	if metricKey, has := m.displayMetrics[name]; has {
+		return m.GetMetric(metricKey)
 	}
 	return nil
 }
@@ -166,7 +166,7 @@ func (m *Matrix) addMetric(key string, metric Metric) error {
 	// Histograms and arrays don't support display metrics yet, last write wins
 	metric.Reset(len(m.instances))
 	m.metrics[key] = metric
-	m.displayMetrics[metric.GetName()] = metric
+	m.displayMetrics[metric.GetName()] = key
 	return nil
 }
 
@@ -180,7 +180,7 @@ func (m *Matrix) RemoveExceptMetric(key string) {
 		return
 	}
 	m.metrics = make(map[string]Metric)
-	m.displayMetrics = make(map[string]Metric)
+	m.displayMetrics = make(map[string]string)
 	_ = m.addMetric(key, prev)
 }
 


Test patch:
diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go
index 8226a6b9..aba22cb5 100644
--- a/cmd/tools/grafana/dashboard_test.go
+++ b/cmd/tools/grafana/dashboard_test.go
@@ -3,9 +3,6 @@ package grafana
 import (
 	"fmt"
 	"github.com/tidwall/gjson"
-	"log"
-	"os"
-	"path/filepath"
 	"regexp"
 	"sort"
 	"strconv"
@@ -33,27 +30,6 @@ func checkDashboardForDatasource(t *testing.T, path string, data []byte) {
 	})
 }
 
-func visitDashboards(dir string, eachDash func(path string, data []byte)) {
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			log.Fatal("failed to read directory:", err)
-		}
-		ext := filepath.Ext(path)
-		if ext != ".json" {
-			return nil
-		}
-		data, err := os.ReadFile(path)
-		if err != nil {
-			log.Fatalf("failed to read dashboards path=%s err=%v", path, err)
-		}
-		eachDash(path, data)
-		return nil
-	})
-	if err != nil {
-		log.Fatal("failed to read dashboards:", err)
-	}
-}
-
 func TestUnitsAndExprMatch(t *testing.T) {
 	mt := newMetricsTable()
 	dir := "../../../grafana/dashboards/cmode"
@@ -281,11 +257,6 @@ func doPanel(pathPrefix string, key gjson.Result, value gjson.Result, mt *metric
 	return true
 }
 
-func shortPath(dashPath string) string {
-	splits := strings.Split(dashPath, string(filepath.Separator))
-	return strings.Join(splits[len(splits)-2:], string(filepath.Separator))
-}
-
 func unitForExpr(e expression, overrides []override, defaultUnit string,
 	valueToName map[string]string, numExpressions int) string {
 
@@ -475,12 +446,6 @@ func TestTopKRange(t *testing.T) {
 	})
 }
 
-type exprP struct {
-	path string
-	expr string
-	vars []string
-}
-
 func checkTopKRange(t *testing.T, path string, data []byte) {
 	// collect all expressions
 	expressions := make([]exprP, 0)
@@ -531,62 +496,6 @@ func checkTopKRange(t *testing.T, path string, data []byte) {
 	}
 }
 
-func doTarget(pathPrefix string, key gjson.Result, value gjson.Result,
-	exprFunc func(path string, expr string, format string)) {
-	kind := value.Get("type").String()
-	if kind == "row" {
-		return
-	}
-	path := fmt.Sprintf("%spanels[%d]", pathPrefix, key.Int())
-	targetsSlice := value.Get("targets").Array()
-	for i, targetN := range targetsSlice {
-		expr := targetN.Get("expr").String()
-		pathWithTarget := path + ".targets[" + strconv.Itoa(i) + "]"
-		exprFunc(pathWithTarget, expr, kind)
-	}
-}
-
-var varRe = regexp.MustCompile(`\$(\w+)`)
-
-func newExpr(path string, expr string) exprP {
-	allMatches := varRe.FindAllStringSubmatch(expr, -1)
-	vars := make([]string, 0, len(allMatches))
-	for _, match := range allMatches {
-		vars = append(vars, match[1])
-	}
-	return exprP{
-		path: path,
-		expr: expr,
-		vars: vars,
-	}
-}
-
-type variable struct {
-	name  string
-	kind  string
-	query string
-	path  string
-}
-
-func allVariables(data []byte) []variable {
-	variables := make([]variable, 0)
-	gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool {
-		// The datasource variable can be ignored
-		if value.Get("type").String() == "datasource" {
-			return true
-		}
-
-		variables = append(variables, variable{
-			name:  value.Get("name").String(),
-			kind:  value.Get("type").String(),
-			query: value.Get("query.query").String(),
-			path:  key.String(),
-		})
-		return true
-	})
-	return variables
-}
-
 func TestOnlyHighlightsExpanded(t *testing.T) {
 	dir := "../../../grafana/dashboards/cmode"
 
@@ -642,3 +551,71 @@ func visitAllPanels(data []byte, handle func(path string, key gjson.Result, valu
 		return true
 	})
 }
+
+func TestLegends(t *testing.T) {
+	dir := "../../../grafana/dashboards/cmode"
+	visitDashboards(dir, func(path string, data []byte) {
+		checkLegends(t, path, data)
+	})
+}
+
+func checkLegends(t *testing.T, path string, data []byte) {
+	// collect all legends
+	dashPath := shortPath(path)
+
+	visitAllPanels(data, func(path string, key, value gjson.Result) {
+		doLegends(t, value, dashPath)
+	})
+}
+
+func doLegends(t *testing.T, value gjson.Result, dashPath string) {
+	wantDisplayMode := "table"
+	wantPlacement := "bottom"
+
+	kind := value.Get("type").String()
+	if kind == "row" || kind == "piechart" {
+		return
+	}
+	optionsData := value.Get("options")
+	if legendData := optionsData.Get("legend"); legendData.Exists() {
+		legendDisplayMode := legendData.Get("displayMode").String()
+		legendPlacementData := legendData.Get("placement").String()
+		title := value.Get("title").String()
+		calcsData := legendData.Get("calcs").Array()
+		var calcsSlice []string
+		for _, result := range calcsData {
+			calcsSlice = append(calcsSlice, result.String())
+		}
+		checkLegendCalculations(t, calcsSlice, dashPath, title)
+
+		// Skip hidden legends
+		if legendDisplayMode == "hidden" {
+			return
+		}
+		if legendDisplayMode != wantDisplayMode {
+			t.Errorf(`dashboard=%s, panel="%s", display mode want=%s got=%s val %v`, dashPath, title, wantDisplayMode, legendDisplayMode, legendData)
+		}
+
+		if legendPlacementData != wantPlacement {
+			t.Errorf(`dashboard=%s, panel="%s", legend placement want=%s got=%s val %v`, dashPath, title, wantPlacement, legendPlacementData, legendData)
+		}
+	}
+}
+
+func checkLegendCalculations(t *testing.T, gotLegendCalculations []string, dashPath, title string) {
+	wantLegendNoMin := strings.Join([]string{"mean", "lastNotNull", "max"}, ",")
+	wantLegendWithMin := "min," + wantLegendNoMin
+	got := strings.Join(gotLegendCalculations, ",")
+	if strings.Contains(got, "sum") {
+		return
+	}
+	if strings.Contains(got, "min") {
+		if got != wantLegendWithMin {
+			t.Errorf(`dashboard=%s, panel="%s", got=[%s] want=[%s]`, dashPath, title, got, wantLegendWithMin)
+		}
+	} else {
+		if got != wantLegendNoMin {
+			t.Errorf(`dashboard=%s, panel="%s", got=[%s] want=[%s]`, dashPath, title, got, wantLegendNoMin)
+		}
+	}
+}