Repository: NetApp/harvest
Pull request #1576 (2022-12-01 11:58:18)

feat: added dashboard tests for legends details
This PR covers these:
- calcs: should be minimum of `{mean, lastNotNull, max}`, exception: when `sum` exist, other calculation may not be exist
- displayMode: should be `table`, exception: `hidden`
- placement: `bottom`


Case1 for calculation missing:
```
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Destination Relationships per Node, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Destination - Break Operations, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Destination - Resync Operations, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Destination - Update Operations, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Source Relationships per Node, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Source - Break Operations, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Source - Resync Operations, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Source - Update Operations, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Source Relationships per SVM, calculation section(s) mean not found
    dashboard_test.go:704: dashboard=cmode/snapmirror.json, panel=Destination Relationships per SVM, calculation section(s) mean not found

```

Case2 and 3 for DisplayMode and Placement:
```
    dashboard_test.go:696: dashboard=cmode/snapmirror.json, panel=Volume relationship count by relationship health, legend placement want=bottom got=right val {
                  "displayMode": "hidden",
                  "placement": "right",
                  "values": [
                    "value"
                  ]
                }

```


Patch:
diff --git a/cmd/collectors/ems/ems.go b/cmd/collectors/ems/ems.go
index 439b6af3..9ca5bd54 100644
--- a/cmd/collectors/ems/ems.go
+++ b/cmd/collectors/ems/ems.go
@@ -473,13 +473,6 @@ func (e *Ems) PollData() (map[string]*matrix.Matrix, error) {
 		instanceCount += len(v.GetInstances())
 	}
 
-	e.Logger.Info().
-		Int("instances", instanceCount).
-		Uint64("dataPoints", count).
-		Str("apiTime", apiD.String()).
-		Str("parseTime", parseD.String()).
-		Msg("Collected")
-
 	_ = e.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds())
 	_ = e.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds())
 	_ = e.Metadata.LazySetValueUint64("metrics", "data", count)
diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go
index 6d6738aa..021cacca 100644
--- a/cmd/collectors/rest/plugins/qtree/qtree.go
+++ b/cmd/collectors/rest/plugins/qtree/qtree.go
@@ -90,13 +90,12 @@ func (my *Qtree) Init() error {
 	for _, obj := range quotaMetric {
 		metricName, display, _, _ := util.ParseMetric(obj)
 
-		metric, err := my.data.NewMetricFloat64(metricName)
+		metric, err := my.data.NewMetricFloat64(metricName, display)
 		if err != nil {
 			my.Logger.Error().Stack().Err(err).Msg("add metric")
 			return err
 		}
 
-		metric.SetName(display)
 		my.Logger.Trace().Msgf("added metric: (%s) [%s] %s", metricName, display, metric)
 	}
 
diff --git a/cmd/collectors/rest/plugins/shelf/shelf.go b/cmd/collectors/rest/plugins/shelf/shelf.go
index 2b28b9b4..bc97321b 100644
--- a/cmd/collectors/rest/plugins/shelf/shelf.go
+++ b/cmd/collectors/rest/plugins/shelf/shelf.go
@@ -151,12 +151,11 @@ func (my *Shelf) Init() error {
 				instanceLabels.NewChildS("", display)
 				my.Logger.Debug().Msgf("added instance label: (%s) [%s]", attribute, display)
 			case "float":
-				metric, err := my.data[attribute].NewMetricFloat64(metricName)
+				_, err := my.data[attribute].NewMetricFloat64(metricName, display)
 				if err != nil {
 					my.Logger.Error().Stack().Err(err).Msg("add metric")
 					return err
 				}
-				metric.SetName(display)
 				my.Logger.Debug().Msgf("added metric: (%s) [%s]", attribute, display)
 			}
 		}
diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go
index aec6425d..82531d18 100644
--- a/cmd/collectors/rest/rest.go
+++ b/cmd/collectors/rest/rest.go
@@ -477,7 +477,7 @@ func (r *Rest) HandleResults(result []gjson.Result, prop *prop, isEndPoint bool)
 		for _, metric := range prop.Metrics {
 			metr, ok := mat.GetMetrics()[metric.Name]
 			if !ok {
-				if metr, err = mat.NewMetricFloat64(metric.Name); err != nil {
+				if metr, err = mat.NewMetricFloat64(metric.Name, metric.Label); err != nil {
 					r.Logger.Error().Err(err).
 						Str("name", metric.Name).
 						Msg("NewMetricFloat64")
@@ -485,8 +485,6 @@ func (r *Rest) HandleResults(result []gjson.Result, prop *prop, isEndPoint bool)
 			}
 			f := instanceData.Get(metric.Name)
 			if f.Exists() {
-				metr.SetName(metric.Label)
-
 				var floatValue float64
 				switch metric.MetricType {
 				case "duration":
diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go
index a2ec15a4..7c7ccbd6 100644
--- a/cmd/collectors/restperf/restperf.go
+++ b/cmd/collectors/restperf/restperf.go
@@ -390,13 +390,12 @@ func (r *RestPerf) processWorkLoadCounter() (map[string]*matrix.Matrix, error) {
 		for name, metric := range r.Prop.Metrics {
 			metr, ok := mat.GetMetrics()[name]
 			if !ok {
-				if metr, err = mat.NewMetricFloat64(name); err != nil {
+				if metr, err = mat.NewMetricFloat64(name, metric.Label); err != nil {
 					r.Logger.Error().Err(err).
 						Str("name", name).
 						Msg("NewMetricFloat64")
 				}
 			}
-			metr.SetName(metric.Label)
 			metr.SetExportable(metric.Exportable)
 		}
 
@@ -445,7 +444,7 @@ func (r *RestPerf) processWorkLoadCounter() (map[string]*matrix.Matrix, error) {
 				if m := mat.GetMetric(name); m != nil {
 					continue
 				}
-				if m, err := mat.NewMetricFloat64(name); err != nil {
+				if m, err := mat.NewMetricFloat64(name, "resource_latency"); err != nil {
 					return nil, err
 				} else {
 					r.perfProp.counterInfo[name] = &counter{
@@ -455,7 +454,6 @@ func (r *RestPerf) processWorkLoadCounter() (map[string]*matrix.Matrix, error) {
 						unit:        r.perfProp.counterInfo[service.GetName()].unit,
 						denominator: "ops",
 					}
-					m.SetName("resource_latency")
 					m.SetLabel("resource", resource)
 
 					r.Logger.Debug().Str("name", name).Str("resource", resource).Msg("added workload latency metric")
@@ -482,11 +480,11 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 	r.Logger.Trace().Msg("updating data cache")
 
-	mat := r.Matrix[r.Object]
+	prevMat := r.Matrix[r.Object]
 	// clone matrix without numeric data
-	newData := mat.Clone(false, true, true)
-	newData.Reset()
-	timestamp := newData.GetMetric("timestamp")
+	curMat := prevMat.Clone(false, true, true)
+	curMat.Reset()
+	timestamp := curMat.GetMetric("timestamp")
 	if timestamp == nil {
 		return nil, errs.New(errs.ErrConfig, "missing timestamp metric")
 	}
@@ -584,7 +582,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 					return true
 				}
 
-				if resourceLatency = newData.GetMetric(layer); resourceLatency == nil {
+				if resourceLatency = curMat.GetMetric(layer); resourceLatency == nil {
 					r.Logger.Trace().
 						Str("layer", layer).
 						Msg("Resource-latency metric missing in cache")
@@ -598,7 +596,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 				}
 			}
 
-			instance = newData.GetInstance(instanceKey)
+			instance = curMat.GetInstance(instanceKey)
 
 			if instance == nil {
 				if isWorkloadObject(r.Prop.Query) || isWorkloadDetailObject(r.Prop.Query) {
@@ -684,17 +682,16 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 							isHistogram = false
 							if len(labels) > 0 && strings.Contains(r.perfProp.counterInfo[name].description, "histogram") {
 								key := name + ".bucket"
-								histogramMetric = newData.GetMetric(key)
+								histogramMetric = curMat.GetMetric(key)
 								if histogramMetric != nil {
 									r.Logger.Trace().Str("metric", key).Msg("Updating array metric attributes")
 								} else {
-									histogramMetric, err = newData.NewMetricFloat64(key)
+									histogramMetric, err = curMat.NewMetricFloat64(key, metric.Label)
 									if err != nil {
 										r.Logger.Error().Err(err).Str("key", key).Msg("unable to create histogram metric")
 										continue
 									}
 								}
-								histogramMetric.SetName(metric.Label)
 								histogramMetric.SetArray(true)
 								histogramMetric.SetExportable(metric.Exportable)
 								histogramMetric.SetBuckets(&labels)
@@ -703,15 +700,14 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 							for i, label := range labels {
 								k := name + arrayKeyToken + label
-								metr, ok := newData.GetMetrics()[k]
+								metr, ok := curMat.GetMetrics()[k]
 								if !ok {
-									if metr, err = newData.NewMetricFloat64(k); err != nil {
+									if metr, err = curMat.NewMetricFloat64(k, metric.Label); err != nil {
 										r.Logger.Error().Err(err).
 											Str("name", k).
 											Msg("NewMetricFloat64")
 										continue
 									}
-									metr.SetName(metric.Label)
 									if x := strings.Split(label, arrayKeyToken); len(x) == 2 {
 										metr.SetLabel("metric", x[0])
 										metr.SetLabel("submetric", x[1])
@@ -749,16 +745,15 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 								}
 							}
 						} else {
-							metr, ok := newData.GetMetrics()[name]
+							metr, ok := curMat.GetMetrics()[name]
 							if !ok {
-								if metr, err = newData.NewMetricFloat64(name); err != nil {
+								if metr, err = curMat.NewMetricFloat64(name, metric.Label); err != nil {
 									r.Logger.Error().Err(err).
 										Str("name", name).
 										Int("instIndex", instIndex).
 										Msg("NewMetricFloat64")
 								}
 							}
-							metr.SetName(metric.Label)
 							metr.SetExportable(metric.Exportable)
 							if c, err := strconv.ParseFloat(f.value, 64); err == nil {
 								if err = metr.SetValueFloat64(instance, c); err != nil {
@@ -789,7 +784,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 					r.Logger.Warn().Str("counter", name).Msg("Counter is missing or unable to parse.")
 				}
 			}
-			if err = newData.GetMetric("timestamp").SetValueFloat64(instance, ts); err != nil {
+			if err = curMat.GetMetric("timestamp").SetValueFloat64(instance, ts); err != nil {
 				r.Logger.Error().Err(err).Msg("Failed to set timestamp")
 			}
 
@@ -799,7 +794,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	}
 
 	if isWorkloadDetailObject(r.Prop.Query) {
-		if err := r.getParentOpsCounters(newData); err != nil {
+		if err := r.getParentOpsCounters(curMat); err != nil {
 			// no point to continue as we can't calculate the other counters
 			return nil, err
 		}
@@ -814,7 +809,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	// skip calculating from delta if no data from previous poll
 	if r.perfProp.isCacheEmpty {
 		r.Logger.Debug().Msg("skip postprocessing until next poll (previous cache empty)")
-		r.Matrix[r.Object] = newData
+		r.Matrix[r.Object] = curMat
 		r.perfProp.isCacheEmpty = false
 		return nil, nil
 	}
@@ -824,15 +819,15 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	r.Logger.Debug().Msg("starting delta calculations from previous cache")
 
 	// cache raw data for next poll
-	cachedData := newData.Clone(true, true, true)
+	cachedData := curMat.Clone(true, true, true)
 
-	orderedNonDenominatorMetrics := make([]matrix.Metric, 0, len(newData.GetMetrics()))
+	orderedNonDenominatorMetrics := make([]matrix.Metric, 0, len(curMat.GetMetrics()))
 	orderedNonDenominatorKeys := make([]string, 0, len(orderedNonDenominatorMetrics))
 
-	orderedDenominatorMetrics := make([]matrix.Metric, 0, len(newData.GetMetrics()))
+	orderedDenominatorMetrics := make([]matrix.Metric, 0, len(curMat.GetMetrics()))
 	orderedDenominatorKeys := make([]string, 0, len(orderedDenominatorMetrics))
 
-	for key, metric := range newData.GetMetrics() {
+	for key, metric := range curMat.GetMetrics() {
 		if metric.GetName() != "timestamp" {
 			counter := r.counterLookup(metric, key)
 			if counter != nil {
@@ -857,7 +852,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 	// calculate timestamp delta first since many counters require it for postprocessing.
 	// Timestamp has "raw" property, so it isn't post-processed automatically
-	if _, err = timestamp.Delta(mat.GetMetric("timestamp"), r.Logger); err != nil {
+	if _, err = timestamp.Delta(prevMat.GetMetric("timestamp"), prevMat, curMat, r.Logger); err != nil {
 		r.Logger.Error().Err(err).Msg("(timestamp) calculate delta:")
 	}
 
@@ -883,7 +878,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 		}
 
 		// all other properties - first calculate delta
-		if skips, err = metric.Delta(mat.GetMetric(key), r.Logger); err != nil {
+		if skips, err = metric.Delta(prevMat.GetMetric(key), prevMat, curMat, r.Logger); err != nil {
 			r.Logger.Error().Err(err).Str("key", key).Msg("Calculate delta")
 			continue
 		}
@@ -905,7 +900,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 		// For the next two properties we need base counters
 		// We assume that delta of base counters is already calculated
-		if base = newData.GetMetric(counter.denominator); base == nil {
+		if base = curMat.GetMetric(counter.denominator); base == nil {
 			r.Logger.Warn().
 				Str("key", key).
 				Str("property", property).
@@ -981,7 +976,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	}
 
 	calcD := time.Since(calcStart)
-	_ = r.Metadata.LazySetValueUint64("instances", "data", uint64(len(newData.GetInstances())))
+	_ = r.Metadata.LazySetValueUint64("instances", "data", uint64(len(curMat.GetInstances())))
 	_ = r.Metadata.LazySetValueInt64("calc_time", "data", calcD.Microseconds())
 	_ = r.Metadata.LazySetValueUint64("skips", "data", uint64(totalSkips))
 
@@ -989,7 +984,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	r.Matrix[r.Object] = cachedData
 
 	newDataMap := make(map[string]*matrix.Matrix)
-	newDataMap[r.Object] = newData
+	newDataMap[r.Object] = curMat
 	return newDataMap, nil
 }
 
diff --git a/cmd/collectors/simple/simple.go b/cmd/collectors/simple/simple.go
index 4ee1eaa0..9dc18404 100644
--- a/cmd/collectors/simple/simple.go
+++ b/cmd/collectors/simple/simple.go
@@ -54,8 +54,7 @@ func (n *NodeMon) Init(a *collector.AbstractCollector) error {
 
 func (n *NodeMon) loadMetrics(counters *node.Node) error {
 	var (
-		metric matrix.Metric
-		err    error
+		err error
 	)
 
 	n.Logger.Debug().Msg("initializing metric cache")
@@ -70,10 +69,9 @@ func (n *NodeMon) loadMetrics(counters *node.Node) error {
 		dtype := "int64"
 		n.Logger.Trace().Msgf("handling (%s) (%s) dtype=%s", name, display, dtype)
 
-		if metric, err = mat.NewMetricType(name, dtype); err != nil {
+		if _, err = mat.NewMetricType(name, dtype, display); err != nil {
 			return err
 		}
-		metric.SetName(display)
 		n.Logger.Debug().Msgf("(%s) added metric (%s)", name, display)
 	}
 
diff --git a/cmd/collectors/storagegrid/storagegrid.go b/cmd/collectors/storagegrid/storagegrid.go
index 76420710..1e1ceb00 100644
--- a/cmd/collectors/storagegrid/storagegrid.go
+++ b/cmd/collectors/storagegrid/storagegrid.go
@@ -249,7 +249,7 @@ func (s *StorageGrid) handleResults(result []gjson.Result) uint64 {
 		for _, metric := range s.Props.Metrics {
 			metr, ok := mat.GetMetrics()[metric.Name]
 			if !ok {
-				if metr, err = mat.NewMetricFloat64(metric.Name); err != nil {
+				if metr, err = mat.NewMetricFloat64(metric.Name, metric.Label); err != nil {
 					s.Logger.Error().Err(err).
 						Str("name", metric.Name).
 						Msg("NewMetricFloat64")
@@ -257,8 +257,6 @@ func (s *StorageGrid) handleResults(result []gjson.Result) uint64 {
 			}
 			f := instanceData.Get(metric.Name)
 			if f.Exists() {
-				metr.SetName(metric.Label)
-
 				var floatValue float64
 				switch metric.MetricType {
 				case "":
diff --git a/cmd/collectors/unix/main.go b/cmd/collectors/unix/main.go
index 0adba1f7..f7913751 100644
--- a/cmd/collectors/unix/main.go
+++ b/cmd/collectors/unix/main.go
@@ -218,10 +218,9 @@ func (u *Unix) loadMetrics(counters *node.Node) error {
 		// counter is scalar metric
 		if _, has := _Metrics[name]; has {
 
-			if metric, err = mat.NewMetricType(name, dtype); err != nil {
+			if _, err = mat.NewMetricType(name, dtype, display); err != nil {
 				return err
 			}
-			metric.SetName(display)
 			u.Logger.Debug().Msgf("(%s) added metric (%s)", name, display)
 
 			// counter is histogram
@@ -251,10 +250,9 @@ func (u *Unix) loadMetrics(counters *node.Node) error {
 					continue
 				}
 
-				if metric, err = mat.NewMetricType(name+"."+label, dtype); err != nil {
+				if metric, err = mat.NewMetricType(name+"."+label, dtype, name); err != nil {
 					return err
 				}
-				metric.SetName(name)
 				metric.SetLabel("metric", ldisplay)
 				u.histogramLabels[name] = append(u.histogramLabels[name], label)
 			}
diff --git a/cmd/collectors/zapi/collector/parsers.go b/cmd/collectors/zapi/collector/parsers.go
index d92f1a67..cf146ba6 100644
--- a/cmd/collectors/zapi/collector/parsers.go
+++ b/cmd/collectors/zapi/collector/parsers.go
@@ -79,7 +79,6 @@ func (z *Zapi) HandleCounter(path []string, content string) string {
 	var (
 		name, display, key    string
 		splitValues, fullPath []string
-		metric                matrix.Metric
 		err                   error
 	)
 
@@ -94,7 +93,6 @@ func (z *Zapi) HandleCounter(path []string, content string) string {
 
 	name = strings.TrimSpace(strings.TrimLeft(name, "^"))
 
-	//full_path = append(path[1:], name)
 	fullPath = append(path, name)
 	key = strings.Join(fullPath, ".")
 
@@ -115,16 +113,16 @@ func (z *Zapi) HandleCounter(path []string, content string) string {
 	} else {
 		// use user-defined metric type
 		if t := z.Params.GetChildContentS("metric_type"); t != "" {
-			metric, err = mat.NewMetricType(key, t)
+			_, err = mat.NewMetricType(key, t, display)
 			// use uint64 as default, since nearly all ZAPI counters are unsigned
 		} else {
-			metric, err = mat.NewMetricUint64(key)
+			_, err = mat.NewMetricUint64(key, display)
 		}
 		if err != nil {
-			z.Logger.Error().Stack().Err(err).Msgf("add as metric [%s]: %v", key, display)
+			z.Logger.Error().Err(err).Str("key", key).Str("display", display).Msg("Failed to add metric")
 		} else {
-			metric.SetName(display)
-			z.Logger.Trace().Msgf("%sadd as metric (%s) [%s]%s => %v", color.Blue, key, display, color.End, fullPath)
+			z.Logger.Trace().Str("key", key).Str("display", display).Strs("fullPath", fullPath).
+				Msg("Add metric")
 		}
 	}
 
diff --git a/cmd/collectors/zapi/plugins/qtree/qtree.go b/cmd/collectors/zapi/plugins/qtree/qtree.go
index 0dc1e4f0..e8b97337 100644
--- a/cmd/collectors/zapi/plugins/qtree/qtree.go
+++ b/cmd/collectors/zapi/plugins/qtree/qtree.go
@@ -79,13 +79,12 @@ func (my *Qtree) Init() error {
 
 		metricName, display, _, _ := util.ParseMetric(obj)
 
-		metric, err := my.data.NewMetricFloat64(metricName)
+		metric, err := my.data.NewMetricFloat64(metricName, display)
 		if err != nil {
 			my.Logger.Error().Stack().Err(err).Msg("add metric")
 			return err
 		}
 
-		metric.SetName(display)
 		my.Logger.Debug().Msgf("added metric: (%s) [%s] %s", metricName, display, metric)
 	}
 
diff --git a/cmd/collectors/zapi/plugins/sensor/sensor.go b/cmd/collectors/zapi/plugins/sensor/sensor.go
index 0e6d348d..be053d9d 100644
--- a/cmd/collectors/zapi/plugins/sensor/sensor.go
+++ b/cmd/collectors/zapi/plugins/sensor/sensor.go
@@ -1,6 +1,7 @@
 /*
  * Copyright NetApp Inc, 2021 All rights reserved
  */
+
 package sensor
 
 import (
diff --git a/cmd/collectors/zapi/plugins/shelf/shelf.go b/cmd/collectors/zapi/plugins/shelf/shelf.go
index 8c12ca18..89169c88 100644
--- a/cmd/collectors/zapi/plugins/shelf/shelf.go
+++ b/cmd/collectors/zapi/plugins/shelf/shelf.go
@@ -126,12 +126,11 @@ func (my *Shelf) Init() error {
 					instanceLabels.NewChildS("", display)
 					my.Logger.Debug().Msgf("added instance label: (%s) (%s) [%s]", attribute, x.GetNameS(), display)
 				case "float":
-					metric, err := my.data[attribute].NewMetricFloat64(metricName)
+					_, err := my.data[attribute].NewMetricFloat64(metricName, display)
 					if err != nil {
 						my.Logger.Error().Stack().Err(err).Msg("add metric")
 						return err
 					}
-					metric.SetName(display)
 					my.Logger.Debug().Msgf("added metric: (%s) (%s) [%s]", attribute, x.GetNameS(), display)
 				}
 			}
diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go
index 7b3a018c..40e02931 100644
--- a/cmd/collectors/zapiperf/zapiperf.go
+++ b/cmd/collectors/zapiperf/zapiperf.go
@@ -196,12 +196,12 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	)
 
 	z.Logger.Trace().Msg("updating data cache")
-	m := z.Matrix[z.Object]
+	prevMat := z.Matrix[z.Object]
 	// clone matrix without numeric data
-	newData := m.Clone(false, true, true)
-	newData.Reset()
+	curMat := prevMat.Clone(false, true, true)
+	curMat.Reset()
 
-	timestamp := newData.GetMetric("timestamp")
+	timestamp := curMat.GetMetric("timestamp")
 	if timestamp == nil {
 		return nil, errs.New(errs.ErrConfig, "missing timestamp metric") // @TODO errconfig??
 	}
@@ -226,13 +226,13 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		} else {
 			instanceKeys = make([]string, 0)
 			for _, layer := range resourceMap.GetAllChildNamesS() {
-				for key := range m.GetInstances() {
+				for key := range prevMat.GetInstances() {
 					instanceKeys = append(instanceKeys, key+"."+layer)
 				}
 			}
 		}
 	} else {
-		instanceKeys = newData.GetInstanceKeys()
+		instanceKeys = curMat.GetInstanceKeys()
 	}
 
 	// build ZAPI request
@@ -340,7 +340,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 					continue
 				}
 
-				if resourceLatency = newData.GetMetric(layer); resourceLatency == nil {
+				if resourceLatency = curMat.GetMetric(layer); resourceLatency == nil {
 					z.Logger.Warn().
 						Str("layer", layer).
 						Msg("Resource-latency metric missing in cache")
@@ -357,7 +357,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 				continue
 			}
 
-			instance := newData.GetInstance(key)
+			instance := curMat.GetInstance(key)
 			if instance == nil {
 				z.Logger.Debug().
 					Str("key", key).
@@ -427,7 +427,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 					}
 
 					for i, label := range labels {
-						if metric := newData.GetMetric(name + "." + label); metric != nil {
+						if metric := curMat.GetMetric(name + "." + label); metric != nil {
 							if err = metric.SetValueString(instance, values[i]); err != nil {
 								z.Logger.Error().
 									Stack().
@@ -452,7 +452,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 								Str("label", label).
 								Str("value", value).
 								Int("instIndex", instIndex).
-								Msg("Histogram name.label not in cache")
+								Msg("Histogram name. Label not in cache")
 						}
 					}
 					continue
@@ -486,7 +486,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 				}
 
 				// store as scalar metric
-				if metric := newData.GetMetric(name); metric != nil {
+				if metric := curMat.GetMetric(name); metric != nil {
 					if err = metric.SetValueString(instance, value); err != nil {
 						z.Logger.Error().
 							Err(err).
@@ -521,7 +521,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		Msg("Collected data points in batch polls")
 
 	if z.Query == objWorkloadDetail || z.Query == objWorkloadDetailVolume {
-		if rd, pd, err := z.getParentOpsCounters(newData, keyName); err == nil {
+		if rd, pd, err := z.getParentOpsCounters(curMat, keyName); err == nil {
 			apiT += rd
 			parseT += pd
 		} else {
@@ -540,7 +540,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	// skip calculating from delta if no data from previous poll
 	if z.isCacheEmpty {
 		z.Logger.Debug().Msg("skip postprocessing until next poll (previous cache empty)")
-		z.Matrix[z.Object] = newData
+		z.Matrix[z.Object] = curMat
 		z.isCacheEmpty = false
 		return nil, nil
 	}
@@ -550,19 +550,19 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	z.Logger.Debug().Msg("starting delta calculations from previous cache")
 
 	// cache raw data for next poll
-	cachedData := newData.Clone(true, true, true) // @TODO implement copy data
+	cachedData := curMat.Clone(true, true, true) // @TODO implement copy data
 
 	// order metrics, such that those requiring base counters are processed last
-	orderedMetrics := make([]matrix.Metric, 0, len(newData.GetMetrics()))
+	orderedMetrics := make([]matrix.Metric, 0, len(curMat.GetMetrics()))
 	orderedKeys := make([]string, 0, len(orderedMetrics))
 
-	for key, metric := range newData.GetMetrics() {
+	for key, metric := range curMat.GetMetrics() {
 		if metric.GetComment() == "" { // does not require base counter
 			orderedMetrics = append(orderedMetrics, metric)
 			orderedKeys = append(orderedKeys, key)
 		}
 	}
-	for key, metric := range newData.GetMetrics() {
+	for key, metric := range curMat.GetMetrics() {
 		if metric.GetComment() != "" { // requires base counter
 			orderedMetrics = append(orderedMetrics, metric)
 			orderedKeys = append(orderedKeys, key)
@@ -571,7 +571,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 	// calculate timestamp delta first since many counters require it for postprocessing.
 	// Timestamp has "raw" property, so it isn't post-processed automatically
-	if _, err = timestamp.Delta(m.GetMetric("timestamp"), z.Logger); err != nil {
+	if _, err = timestamp.Delta(prevMat.GetMetric("timestamp"), prevMat, curMat, z.Logger); err != nil {
 		z.Logger.Error().Err(err).Msg("(timestamp) calculate delta:")
 		// @TODO terminate since other counters will be incorrect
 	}
@@ -590,7 +590,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		}
 
 		// all other properties - first calculate delta
-		if skips, err = metric.Delta(m.GetMetric(key), z.Logger); err != nil {
+		if skips, err = metric.Delta(prevMat.GetMetric(key), prevMat, curMat, z.Logger); err != nil {
 			z.Logger.Error().Err(err).Str("key", key).Msg("Calculate delta")
 			continue
 		}
@@ -613,7 +613,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		// For the next two properties we need base counters
 		// We assume that delta of base counters is already calculated
 		// (name of base counter is stored as Comment)
-		if base = newData.GetMetric(metric.GetComment()); base == nil {
+		if base = curMat.GetMetric(metric.GetComment()); base == nil {
 			z.Logger.Warn().
 				Str("key", key).
 				Str("property", property).
@@ -683,7 +683,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	z.Matrix[z.Object] = cachedData
 
 	newDataMap := make(map[string]*matrix.Matrix)
-	newDataMap[z.Object] = newData
+	newDataMap[z.Object] = curMat
 	return newDataMap, nil
 }
 
@@ -1031,10 +1031,9 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 						oldMetrics.Remove(name)
 						continue
 					}
-					if m, err := mat.NewMetricFloat64(name); err != nil {
+					if m, err := mat.NewMetricFloat64(name, "resource_latency"); err != nil {
 						return nil, err
 					} else {
-						m.SetName("resource_latency")
 						m.SetLabel("resource", resource)
 						m.SetProperty(service.GetProperty())
 						// base counter is the ops of the same resource
@@ -1175,13 +1174,12 @@ func (z *ZapiPerf) addCounter(counter *node.Node, name, display string, enabled
 			if histogramMetric != nil {
 				z.Logger.Trace().Str("metric", key).Msg("Updating array metric attributes")
 			} else {
-				histogramMetric, err = mat.NewMetricFloat64(key)
+				histogramMetric, err = mat.NewMetricFloat64(key, display)
 				if err != nil {
 					z.Logger.Error().Err(err).Str("key", key).Msg("unable to create histogram metric")
 					return ""
 				}
 			}
-			histogramMetric.SetName(display)
 			histogramMetric.SetProperty(property)
 			histogramMetric.SetComment(baseKey)
 			histogramMetric.SetExportable(enabled)
@@ -1197,14 +1195,13 @@ func (z *ZapiPerf) addCounter(counter *node.Node, name, display string, enabled
 
 			if m = mat.GetMetric(key); m != nil {
 				z.Logger.Trace().Msgf("updating array metric [%s] attributes", key)
-			} else if m, err = mat.NewMetricFloat64(key); err == nil {
+			} else if m, err = mat.NewMetricFloat64(key, display); err == nil {
 				z.Logger.Trace().Msgf("%s+[%s] added array metric (%s), element with label (%s)%s", color.Pink, name, display, label, color.End)
 			} else {
 				z.Logger.Error().Err(err).Msgf("add array metric element [%s]: ", key)
 				return ""
 			}
 
-			m.SetName(display)
 			m.SetProperty(property)
 			m.SetComment(baseKey)
 			m.SetExportable(enabled)
@@ -1231,7 +1228,7 @@ func (z *ZapiPerf) addCounter(counter *node.Node, name, display string, enabled
 		var m matrix.Metric
 		if m = mat.GetMetric(name); m != nil {
 			z.Logger.Trace().Msgf("updating scalar metric [%s] attributes", name)
-		} else if m, err = mat.NewMetricFloat64(name); err == nil {
+		} else if m, err = mat.NewMetricFloat64(name, display); err == nil {
 			z.Logger.Trace().Msgf("%s+[%s] added scalar metric (%s)%s", color.Cyan, name, display, color.End)
 		} else {
 			z.Logger.Error().Err(err).Msgf("add scalar metric [%s]", name)
@@ -1239,7 +1236,6 @@ func (z *ZapiPerf) addCounter(counter *node.Node, name, display string, enabled
 		}
 
 		z.scalarCounters = append(z.scalarCounters, name)
-		m.SetName(display)
 		m.SetProperty(property)
 		m.SetComment(baseCounter)
 		m.SetExportable(enabled)
diff --git a/cmd/poller/plugin/metricagent/metric_agent.go b/cmd/poller/plugin/metricagent/metric_agent.go
index 63fe74d0..146300f0 100644
--- a/cmd/poller/plugin/metricagent/metric_agent.go
+++ b/cmd/poller/plugin/metricagent/metric_agent.go
@@ -62,10 +62,9 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error {
 
 	// map values for compute_metric mapping rules
 	for _, r := range a.computeMetricRules {
-
-		if metric = m.GetMetric(r.metric); metric == nil {
+		if metric = a.getMetric(m, r.metric); metric == nil {
 			if metric, err = m.NewMetricFloat64(r.metric); err != nil {
-				a.Logger.Error().Stack().Err(err).Str("new metric", r.metric).Msg("computeMetrics: failed to create metric")
+				a.Logger.Error().Err(err).Str("metric", r.metric).Msg("Failed to create metric")
 				return err
 			} else {
 				metric.SetProperty("compute_metric mapping")
@@ -76,7 +75,7 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error {
 			var result float64
 
 			// Parse first operand and store in result for further processing
-			if firstMetricVal = m.GetMetric(r.metricNames[0]); firstMetricVal != nil {
+			if firstMetricVal = a.getMetric(m, r.metricNames[0]); firstMetricVal != nil {
 				if val, ok := firstMetricVal.GetValueFloat64(instance); ok {
 					result = val
 				} else {
@@ -92,7 +91,7 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error {
 				if value, err := strconv.Atoi(r.metricNames[i]); err == nil {
 					v = float64(value)
 				} else {
-					metricVal = m.GetMetric(r.metricNames[i])
+					metricVal = a.getMetric(m, r.metricNames[i])
 					if metricVal != nil {
 						v, _ = metricVal.GetValueFloat64(instance)
 					} else {
@@ -112,16 +111,17 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error {
 					if v != 0 {
 						result /= v
 					} else {
-						a.Logger.Error().
-							Str("operation", r.operation).
-							Msg("Division by zero operation")
+						a.Logger.Error().Str("operation", r.operation).Msg("Division by zero operation")
+					}
+				case "PERCENT":
+					if v != 0 {
+						result = (result / v) * 100
+					} else {
+						a.Logger.Error().Str("operation", r.operation).Msg("Division by zero operation")
 					}
 				default:
-					a.Logger.Warn().
-						Str("operation", r.operation).
-						Msg("Unknown operation")
+					a.Logger.Warn().Str("operation", r.operation).Msg("Unknown operation")
 				}
-
 			}
 
 			_ = metric.SetValueFloat64(instance, result)
@@ -130,3 +130,11 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error {
 	}
 	return nil
 }
+
+func (a *MetricAgent) getMetric(m *matrix.Matrix, name string) matrix.Metric {
+	metric := m.DisplayMetric(name)
+	if metric != nil {
+		return metric
+	}
+	return m.GetMetric(name)
+}
diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go
index fe568ce6..60e9500d 100644
--- a/cmd/tools/grafana/grafana.go
+++ b/cmd/tools/grafana/grafana.go
@@ -513,7 +513,8 @@ func importFiles(dir string, folder *Folder) {
 			fmt.Printf("error importing [%s] to folder [%s] - the server replied with [%s]\n", file.Name(), folder.name, status)
 			if code == 412 {
 				fmt.Printf("That means you are trying to overwrite an existing dashboard.\n\n")
-				fmt.Printf("If you want to overwrite run with the --overwrite flag or choose a different Grafana folder with the --serverfolder flag\n\n")
+				fmt.Printf("If you want to overwrite, run with the --overwrite flag or choose a different Grafana folder with the --serverfolder flag.\n")
+				fmt.Printf("NOTE: If the dashboard already exists, --overwrite will import the changes and increment the dashboard version number.\n\n")
 			}
 			fmt.Printf("Server response follows:\n")
 			for k, v := range result {
@@ -945,19 +946,8 @@ grafana export --addr my.grafana.server:3000 --serverfolder server_folder --dire
 }
 
 func init() {
-	Cmd.AddCommand(importCmd, exportCmd)
-
-	Cmd.PersistentFlags().StringVar(&opts.config, "config", "./harvest.yml", "harvest config file path")
-	Cmd.PersistentFlags().StringVarP(&opts.addr, "addr", "a", "http://127.0.0.1:3000", "Address of Grafana server (IP, FQDN or hostname)")
-	Cmd.PersistentFlags().StringVarP(&opts.token, "token", "t", "", "API token issued by Grafana server for authentication")
-	Cmd.PersistentFlags().StringVarP(&opts.prefix, "prefix", "p", "", "Use global metric prefix in queries")
-	Cmd.PersistentFlags().StringVarP(&opts.datasource, "datasource", "s", grafanaDataSource, "Grafana datasource for the dashboards")
-	Cmd.PersistentFlags().BoolVarP(&opts.variable, "variable", "v", false, "Use datasource as variable, overrides: --datasource")
-	Cmd.PersistentFlags().BoolVarP(&opts.useHTTPS, "https", "S", false, "Use HTTPS")
-	Cmd.PersistentFlags().BoolVarP(&opts.overwrite, "overwrite", "o", false, "Overwrite existing dashboard with same title")
-	Cmd.PersistentFlags().BoolVarP(&opts.useInsecureTLS, "insecure", "k", false, "Allow insecure server connections when using SSL")
-	Cmd.PersistentFlags().StringVarP(&opts.serverfolder.name, "serverfolder", "f", "", "Grafana folder name for dashboards")
-	Cmd.PersistentFlags().StringVarP(&opts.dir, "directory", "d", "", "When importing, import dashboards from this local directory.\nWhen exporting, local directory to write dashboards to")
+	Cmd.AddCommand(importCmd, exportCmd, metricsCmd)
+	addFlags(importCmd, exportCmd)
 
 	importCmd.PersistentFlags().StringSliceVar(&opts.labels, "labels", nil,
 		"For each label, create a variable and add as chained query to other variables")
@@ -965,6 +955,26 @@ func init() {
 		"Modify the dashboards to add multi-select dropdowns for each variable")
 	_ = importCmd.PersistentFlags().MarkHidden("multi")
 
-	_ = Cmd.MarkPersistentFlagRequired("serverfolder")
-	_ = Cmd.MarkPersistentFlagRequired("directory")
+	metricsCmd.PersistentFlags().StringVarP(&opts.dir, "directory", "d",
+		"", "local directory that contains dashboards (searched recursively).")
+
+}
+
+func addFlags(commands ...*cobra.Command) {
+	for _, command := range commands {
+		cmd := command
+		cmd.PersistentFlags().StringVar(&opts.config, "config", "./harvest.yml", "harvest config file path")
+		cmd.PersistentFlags().StringVarP(&opts.addr, "addr", "a", "http://127.0.0.1:3000", "Address of Grafana server (IP, FQDN or hostname)")
+		cmd.PersistentFlags().StringVarP(&opts.token, "token", "t", "", "API token issued by Grafana server for authentication")
+		cmd.PersistentFlags().StringVarP(&opts.prefix, "prefix", "p", "", "Use global metric prefix in queries")
+		cmd.PersistentFlags().StringVarP(&opts.datasource, "datasource", "s", grafanaDataSource, "Grafana datasource for the dashboards")
+		cmd.PersistentFlags().BoolVarP(&opts.variable, "variable", "v", false, "Use datasource as variable, overrides: --datasource")
+		cmd.PersistentFlags().BoolVarP(&opts.useHTTPS, "https", "S", false, "Use HTTPS")
+		cmd.PersistentFlags().BoolVarP(&opts.overwrite, "overwrite", "o", false, "Overwrite existing dashboard with same title")
+		cmd.PersistentFlags().BoolVarP(&opts.useInsecureTLS, "insecure", "k", false, "Allow insecure server connections when using SSL")
+		cmd.PersistentFlags().StringVarP(&opts.serverfolder.name, "serverfolder", "f", "", "Grafana folder name for dashboards")
+		cmd.PersistentFlags().StringVarP(&opts.dir, "directory", "d", "", "When importing, import dashboards from this local directory.\nWhen exporting, local directory to write dashboards to")
+		_ = cmd.MarkPersistentFlagRequired("serverfolder")
+		_ = cmd.MarkPersistentFlagRequired("directory")
+	}
 }
diff --git a/cmd/tools/grafana/metrics.go b/cmd/tools/grafana/metrics.go
new file mode 100644
index 00000000..d6a37155
--- /dev/null
+++ b/cmd/tools/grafana/metrics.go
@@ -0,0 +1,179 @@
+package grafana
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/tidwall/gjson"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+var varRe = regexp.MustCompile(`\$(\w+)`)
+var metricRe = regexp.MustCompile(`(\w+)\{`)
+
+var metricsCmd = &cobra.Command{
+	Use:   "metrics",
+	Short: "print dashboard metrics",
+	Run:   doMetrics,
+}
+
+func doMetrics(_ *cobra.Command, _ []string) {
+	adjustOptions()
+	validateImport()
+	visitDashboards(opts.dir, func(path string, data []byte) {
+		visitExpressionsAndQueries(path, data)
+	})
+}
+
+type exprP struct {
+	path string
+	expr string
+	vars []string
+}
+
+func visitExpressionsAndQueries(path string, data []byte) {
+	// collect all expressions
+	expressions := make([]exprP, 0)
+	gjson.GetBytes(data, "panels").ForEach(func(key, value gjson.Result) bool {
+		doTarget("", key, value, func(path string, expr string, format string) {
+			expressions = append(expressions, newExpr(path, expr))
+		})
+		value.Get("panels").ForEach(func(key2, value2 gjson.Result) bool {
+			pathPrefix := fmt.Sprintf("panels[%d].", key.Int())
+			doTarget(pathPrefix, key2, value2, func(path string, expr string, format string) {
+				expressions = append(expressions, newExpr(path, expr))
+			})
+			return true
+		})
+		return true
+	})
+
+	metricsSeen := make(map[string]struct{})
+	for _, expr := range expressions {
+		allMatches := metricRe.FindAllStringSubmatch(expr.expr, -1)
+		for _, match := range allMatches {
+			m := match[1]
+			if len(m) == 0 {
+				continue
+			}
+			metricsSeen[m] = struct{}{}
+		}
+	}
+
+	// collect all variables
+	variables := allVariables(data)
+	for _, v := range variables {
+		allMatches := metricRe.FindAllStringSubmatch(v.query, -1)
+		for _, match := range allMatches {
+			m := match[1]
+			if len(m) == 0 {
+				continue
+			}
+			metricsSeen[m] = struct{}{}
+		}
+	}
+
+	fmt.Printf("%s\n", shortPath(path))
+	metrics := setToList(metricsSeen)
+	for _, metric := range metrics {
+		fmt.Printf("- %s\n", metric)
+	}
+	fmt.Println()
+}
+
+func setToList(seen map[string]struct{}) []string {
+	list := make([]string, 0)
+	for k := range seen {
+		list = append(list, k)
+	}
+	sort.Strings(list)
+	return list
+}
+
+type variable struct {
+	name  string
+	kind  string
+	query string
+	path  string
+}
+
+func allVariables(data []byte) []variable {
+	variables := make([]variable, 0)
+	gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool {
+		// The datasource variable can be ignored
+		if value.Get("type").String() == "datasource" {
+			return true
+		}
+
+		variables = append(variables, variable{
+			name:  value.Get("name").String(),
+			kind:  value.Get("type").String(),
+			query: value.Get("query.query").String(),
+			path:  key.String(),
+		})
+		return true
+	})
+	return variables
+}
+
+func newExpr(path string, expr string) exprP {
+	allMatches := varRe.FindAllStringSubmatch(expr, -1)
+	vars := make([]string, 0, len(allMatches))
+	for _, match := range allMatches {
+		vars = append(vars, match[1])
+	}
+	return exprP{
+		path: path,
+		expr: expr,
+		vars: vars,
+	}
+}
+
+func doTarget(pathPrefix string, key gjson.Result, value gjson.Result,
+	exprFunc func(path string, expr string, format string)) {
+	kind := value.Get("type").String()
+	if kind == "row" {
+		return
+	}
+	path := fmt.Sprintf("%spanels[%d]", pathPrefix, key.Int())
+	targetsSlice := value.Get("targets").Array()
+	for i, targetN := range targetsSlice {
+		expr := targetN.Get("expr").String()
+		pathWithTarget := path + ".targets[" + strconv.Itoa(i) + "]"
+		exprFunc(pathWithTarget, expr, kind)
+	}
+}
+
+func visitDashboards(dir string, eachDash func(path string, data []byte)) {
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if strings.Contains(path, "influxdb") {
+			return nil
+		}
+		if err != nil {
+			log.Fatal("failed to read directory:", err)
+		}
+		ext := filepath.Ext(path)
+		if ext != ".json" {
+			return nil
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			log.Fatalf("failed to read dashboards path=%s err=%v", path, err)
+		}
+		eachDash(path, data)
+		return nil
+	})
+	if err != nil {
+		log.Fatal("failed to read dashboards:", err)
+	}
+}
+
+func shortPath(dashPath string) string {
+	splits := strings.Split(dashPath, string(filepath.Separator))
+	return strings.Join(splits[len(splits)-2:], string(filepath.Separator))
+}
diff --git a/conf/rest/9.10.0/aggr.yaml b/conf/rest/9.10.0/aggr.yaml
index 1a6dbd55..067367e1 100644
--- a/conf/rest/9.10.0/aggr.yaml
+++ b/conf/rest/9.10.0/aggr.yaml
@@ -59,8 +59,7 @@ plugins:
         - total_physical_used SUBTRACT space.efficiency.logical_used space.efficiency.savings
         - physical_used_wo_snapshots SUBTRACT space.efficiency_without_snapshots.logical_used space.efficiency_without_snapshots.savings
         - physical_used_wo_snapshots_flexclones SUBTRACT space.efficiency_without_snapshots_flexclones.logical_used space.efficiency_without_snapshots_flexclones.savings
-        - space_used_percent DIVIDE space.block_storage.used space.block_storage.size
-        - space_used_percent MULTIPLY space_used_percent 100
+        - space_used_percent PERCENT space.block_storage.used space.block_storage.size
 
 
 export_options:
diff --git a/conf/rest/9.12.0/aggr.yaml b/conf/rest/9.12.0/aggr.yaml
index d1e6d61f..a9ff56fc 100644
--- a/conf/rest/9.12.0/aggr.yaml
+++ b/conf/rest/9.12.0/aggr.yaml
@@ -70,8 +70,7 @@ plugins:
         - total_physical_used SUBTRACT space.efficiency.logical_used space.efficiency.savings
         - physical_used_wo_snapshots SUBTRACT space.efficiency_without_snapshots.logical_used space.efficiency_without_snapshots.savings
         - physical_used_wo_snapshots_flexclones SUBTRACT space.efficiency_without_snapshots_flexclones.logical_used space.efficiency_without_snapshots_flexclones.savings
-        - space_used_percent DIVIDE space.block_storage.used space.block_storage.size
-        - space_used_percent MULTIPLY space_used_percent 100
+        - space_used_percent PERCENT space.block_storage.used space.block_storage.size
 
 export_options:
   instance_keys:
diff --git a/conf/rest/9.12.0/volume.yaml b/conf/rest/9.12.0/volume.yaml
index 06c2c38f..606fbba4 100644
--- a/conf/rest/9.12.0/volume.yaml
+++ b/conf/rest/9.12.0/volume.yaml
@@ -66,6 +66,9 @@ plugins:
   - Volume:
       schedule:
         - data: 900s  # should be multiple of data poll duration
+  - MetricAgent:
+      compute_metric:
+        - inode_used_percent PERCENT inode_files_used inode_files_total
   - LabelAgent:
       value_to_num:
         - new_status state online online `0`
diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml
index e337b8d0..8ee6b6d6 100644
--- a/conf/zapi/cdot/9.8.0/volume.yaml
+++ b/conf/zapi/cdot/9.8.0/volume.yaml
@@ -67,10 +67,13 @@ counters:
     - ^encrypt                              => isEncrypted
 
 plugins:
-  Volume:
-    schedule:
-      - data: 900s  # should be multiple of data poll duration
-  LabelAgent:
+  - Volume:
+      schedule:
+        - data: 900s  # should be multiple of data poll duration
+  - MetricAgent:
+      compute_metric:
+        - inode_used_percent PERCENT inode_files_used inode_files_total
+  - LabelAgent:
     # metric label zapi_value rest_value `default_value`
     value_to_num:
       - new_status state online online `0`
@@ -92,7 +95,7 @@ plugins:
 #      - volume `MDV_aud_.+`
     replace:
       - style style `flexgroup_constituent` `flexgroup`
-  Aggregator:
+  - Aggregator:
     - volume<style=flexgroup>volume node,svm,aggr,style
 
 export_options:
diff --git a/docs/plugins.md b/docs/plugins.md
index a133d450..e4de680b 100644
--- a/docs/plugins.md
+++ b/docs/plugins.md
@@ -555,7 +555,8 @@ plugins:
       - raid_disk_count ADD block_storage.primary.disk_count block_storage.hybrid_cache.disk_count
 ```
 
-**Note:** Metrics for creating new metric should use name defined in left side of =>
+**Note:** Metric names used to create new metrics can come from the left or right side of the rename operator (`=>`)
+**Note:** The metric agent currently does not work for histogram or array metrics.
 
 ## compute_metric
 
@@ -565,7 +566,7 @@ mathematical operation.
 You can provide a numeric value or a metric name with an operation. The plugin will use the provided number or fetch the
 value of a given metric, perform the requested mathematical operation, and store the result in new custom metric.
 
-Currently, we support these operations: ADD SUBTRACT MULTIPLY DIVIDE
+Currently, we support these operations: ADD SUBTRACT MULTIPLY DIVIDE PERCENT
 
 Rule syntax:
 
@@ -621,3 +622,11 @@ compute_metric:
 # value of metric "transmission_rate" would be division of metric value of transfer.bytes_transferred by metric value of transfer.total_duration.
 # transmission_rate = transfer.bytes_transferred / transfer.total_duration
 ```
+
+```yaml
+compute_metric:
+  - inode_used_percent PERCENT inode_files_used inode_files_total
+# a new metric named "inode_used_percent" will be created by dividing the metric "inode_files_used" by 
+#  "inode_files_total" and multiplying the result by 100.
+# inode_used_percent = inode_files_used / inode_files_total * 100
+```
diff --git a/grafana/dashboards/cmode/cdot.json b/grafana/dashboards/cmode/cdot.json
index bb83f9e4..42086e6f 100644
--- a/grafana/dashboards/cmode/cdot.json
+++ b/grafana/dashboards/cmode/cdot.json
@@ -1649,7 +1649,8 @@
             "legend": {
               "calcs": [
                 "mean",
-                "lastNotNull"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -1670,7 +1671,7 @@
               "refId": "A"
             }
           ],
-          "title": "Top Average Throughput by Volumes ",
+          "title": "Top Average Throughput by Volumes",
           "transformations": [],
           "type": "timeseries"
         },
diff --git a/grafana/dashboards/cmode/cluster.json b/grafana/dashboards/cmode/cluster.json
index 0ac7362f..3cfea907 100644
--- a/grafana/dashboards/cmode/cluster.json
+++ b/grafana/dashboards/cmode/cluster.json
@@ -2889,9 +2889,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -2980,9 +2980,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/lun.json b/grafana/dashboards/cmode/lun.json
index 19fecf0d..407b4195 100644
--- a/grafana/dashboards/cmode/lun.json
+++ b/grafana/dashboards/cmode/lun.json
@@ -3299,7 +3299,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -3387,7 +3389,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -3850,9 +3854,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -3944,9 +3948,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4038,9 +4042,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4145,9 +4149,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4239,9 +4243,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -4375,9 +4379,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/node.json b/grafana/dashboards/cmode/node.json
index c32f520c..fa871f3b 100644
--- a/grafana/dashboards/cmode/node.json
+++ b/grafana/dashboards/cmode/node.json
@@ -2796,9 +2796,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -2891,9 +2891,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -2987,9 +2987,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "max",
                 "mean",
-                "last"
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/power.json b/grafana/dashboards/cmode/power.json
index 00f58710..982604e7 100644
--- a/grafana/dashboards/cmode/power.json
+++ b/grafana/dashboards/cmode/power.json
@@ -562,9 +562,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -669,9 +669,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -760,9 +760,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -850,9 +850,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -941,9 +941,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/grafana/dashboards/cmode/qtree.json b/grafana/dashboards/cmode/qtree.json
index 1b1f42f0..cf7130f0 100644
--- a/grafana/dashboards/cmode/qtree.json
+++ b/grafana/dashboards/cmode/qtree.json
@@ -519,8 +519,9 @@
             "legend": {
               "calcs": [
                 "min",
-                "max",
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
@@ -605,8 +606,9 @@
             "legend": {
               "calcs": [
                 "min",
-                "max",
-                "lastNotNull"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/shelf.json b/grafana/dashboards/cmode/shelf.json
index a604e0a9..501e3b9a 100644
--- a/grafana/dashboards/cmode/shelf.json
+++ b/grafana/dashboards/cmode/shelf.json
@@ -1440,9 +1440,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1531,9 +1531,9 @@
       "options": {
         "legend": {
           "calcs": [
-            "max",
             "mean",
-            "last"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/grafana/dashboards/cmode/snapmirror.json b/grafana/dashboards/cmode/snapmirror.json
index 0571b4db..0f4dc4d5 100644
--- a/grafana/dashboards/cmode/snapmirror.json
+++ b/grafana/dashboards/cmode/snapmirror.json
@@ -1094,9 +1094,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1191,9 +1192,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1298,9 +1300,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1404,9 +1407,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1508,9 +1512,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1605,9 +1610,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1711,9 +1717,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1817,9 +1824,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -1939,9 +1947,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -2035,9 +2044,10 @@
       "options": {
         "legend": {
           "calcs": [
+            "min",
+            "mean",
             "lastNotNull",
-            "max",
-            "min"
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/grafana/dashboards/cmode/svm.json b/grafana/dashboards/cmode/svm.json
index afc3fb75..abe6f6c0 100644
--- a/grafana/dashboards/cmode/svm.json
+++ b/grafana/dashboards/cmode/svm.json
@@ -6269,7 +6269,9 @@
           "options": {
             "legend": {
               "calcs": [
-                "mean"
+                "mean",
+                "lastNotNull",
+                "max"
               ],
               "displayMode": "table",
               "placement": "bottom"
diff --git a/grafana/dashboards/cmode/volume.json b/grafana/dashboards/cmode/volume.json
index 768d1745..c9fe8002 100644
--- a/grafana/dashboards/cmode/volume.json
+++ b/grafana/dashboards/cmode/volume.json
@@ -436,7 +436,8 @@
         "legend": {
           "calcs": [
             "mean",
-            "lastNotNull"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
@@ -526,7 +527,8 @@
         "legend": {
           "calcs": [
             "mean",
-            "lastNotNull"
+            "lastNotNull",
+            "max"
           ],
           "displayMode": "table",
           "placement": "bottom"
diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go
index 61437bc9..60eb9d80 100644
--- a/pkg/matrix/matrix.go
+++ b/pkg/matrix/matrix.go
@@ -20,14 +20,15 @@ import (
 )
 
 type Matrix struct {
-	UUID          string
-	Object        string
-	Identifier    string
-	globalLabels  *dict.Dict
-	instances     map[string]*Instance
-	metrics       map[string]Metric
-	exportOptions *node.Node
-	exportable    bool
+	UUID           string
+	Object         string
+	Identifier     string
+	globalLabels   *dict.Dict
+	instances      map[string]*Instance
+	metrics        map[string]Metric // ONTAP metric name => metric (in templates, this is left side)
+	displayMetrics map[string]Metric // display name of metric to => metric (in templates, this is right side)
+	exportOptions  *node.Node
+	exportable     bool
 }
 
 func New(uuid, object string, identifier string) *Matrix {
@@ -35,6 +36,7 @@ func New(uuid, object string, identifier string) *Matrix {
 	me.globalLabels = dict.New()
 	me.instances = make(map[string]*Instance, 0)
 	me.metrics = make(map[string]Metric, 0)
+	me.displayMetrics = make(map[string]Metric, 0)
 	me.exportable = true
 	return &me
 }
@@ -78,7 +80,9 @@ func (m *Matrix) Clone(withData, withMetrics, withInstances bool) *Matrix {
 
 	if withMetrics {
 		for key, metric := range m.GetMetrics() {
-			clone.metrics[key] = metric.Clone(withData)
+			c := metric.Clone(withData)
+			clone.metrics[key] = c
+			clone.displayMetrics[c.GetName()] = c
 		}
 	}
 
@@ -93,6 +97,13 @@ func (m *Matrix) Reset() {
 	}
 }
 
+func (m *Matrix) DisplayMetric(name string) Metric {
+	if metric, has := m.displayMetrics[name]; has {
+		return metric
+	}
+	return nil
+}
+
 func (m *Matrix) GetMetric(key string) Metric {
 	if metric, has := m.metrics[key]; has {
 		return metric
@@ -104,53 +115,58 @@ func (m *Matrix) GetMetrics() map[string]Metric {
 	return m.metrics
 }
 
-func (m *Matrix) NewMetricInt64(key string) (Metric, error) {
-	metric := &MetricInt64{AbstractMetric: &AbstractMetric{name: key, dtype: "int64", exportable: true}}
+func (m *Matrix) NewMetricInt64(key string, display ...string) (Metric, error) {
+	metric := &MetricInt64{AbstractMetric: newAbstract(key, "int64", display...)}
 	return metric, m.addMetric(key, metric)
 }
 
-func (m *Matrix) NewMetricUint8(key string) (Metric, error) {
-	metric := &MetricUint8{AbstractMetric: &AbstractMetric{name: key, dtype: "uint8", exportable: true}}
+func (m *Matrix) NewMetricUint8(key string, display ...string) (Metric, error) {
+	metric := &MetricUint8{AbstractMetric: newAbstract(key, "uint8", display...)}
 	return metric, m.addMetric(key, metric)
 }
 
-func (m *Matrix) NewMetricUint64(key string) (Metric, error) {
-	metric := &MetricUint64{AbstractMetric: &AbstractMetric{name: key, dtype: "uint64", exportable: true}}
+func (m *Matrix) NewMetricUint64(key string, display ...string) (Metric, error) {
+	metric := &MetricUint64{AbstractMetric: newAbstract(key, "uint64", display...)}
 	return metric, m.addMetric(key, metric)
 }
 
-func (m *Matrix) NewMetricFloat64(key string) (Metric, error) {
-	metric := &MetricFloat64{AbstractMetric: &AbstractMetric{name: key, dtype: "float64", exportable: true}}
+func (m *Matrix) NewMetricFloat64(key string, display ...string) (Metric, error) {
+	metric := &MetricFloat64{AbstractMetric: newAbstract(key, "float64", display...)}
 	return metric, m.addMetric(key, metric)
 }
 
-func (m *Matrix) NewMetricType(key, dtype string) (Metric, error) {
+func (m *Matrix) NewMetricType(key string, dataType string, display ...string) (Metric, error) {
 
-	switch dtype {
+	switch dataType {
 	case "int64":
-		return m.NewMetricInt64(key)
+		return m.NewMetricInt64(key, display...)
 	case "uint8":
-		return m.NewMetricUint8(key)
+		return m.NewMetricUint8(key, display...)
 	case "uint64":
-		return m.NewMetricUint64(key)
+		return m.NewMetricUint64(key, display...)
 	case "float64":
-		return m.NewMetricFloat64(key)
+		return m.NewMetricFloat64(key, display...)
 	default:
-		return nil, errs.New(ErrInvalidDtype, dtype)
+		return nil, errs.New(ErrInvalidDtype, dataType)
 	}
 }
 
-func (m *Matrix) ChangeMetricType(key, dtype string) (Metric, error) {
-	m.RemoveMetric(key)
-	return m.NewMetricType(key, dtype)
+func newAbstract(key string, dataType string, display ...string) *AbstractMetric {
+	name := key
+	if len(display) > 0 {
+		name = display[0]
+	}
+	return &AbstractMetric{name: name, dtype: dataType, exportable: true}
 }
 
 func (m *Matrix) addMetric(key string, metric Metric) error {
-	if _, has := m.metrics[key]; has {
+	if _, has := m.metrics[key]; has { // Fail if a metric with the same key already exists
 		return errs.New(ErrDuplicateMetricKey, key)
 	}
+	// Histograms and arrays don't support display metrics yet, last write wins
 	metric.Reset(len(m.instances))
 	m.metrics[key] = metric
+	m.displayMetrics[metric.GetName()] = metric
 	return nil
 }
 
@@ -164,7 +180,8 @@ func (m *Matrix) RemoveExceptMetric(key string) {
 		return
 	}
 	m.metrics = make(map[string]Metric)
-	m.metrics[key] = prev
+	m.displayMetrics = make(map[string]Metric)
+	_ = m.addMetric(key, prev)
 }
 
 func (m *Matrix) GetInstance(key string) *Instance {
diff --git a/pkg/matrix/metric.go b/pkg/matrix/metric.go
index fcba68d6..70bfe822 100644
--- a/pkg/matrix/metric.go
+++ b/pkg/matrix/metric.go
@@ -21,7 +21,6 @@ type Metric interface {
 	// methods related to metric attributes
 
 	GetName() string
-	SetName(string)
 	GetType() string
 	SetLabel(string, string)
 	SetLabels(*dict.Dict)
@@ -80,7 +79,7 @@ type Metric interface {
 	// methods for doing vector arithmetics
 	// currently only supported for float64!
 
-	Delta(Metric, *logging.Logger) (int, error)
+	Delta(Metric, *Matrix, *Matrix, *logging.Logger) (int, error)
 	Divide(Metric, *logging.Logger) (int, error)
 	DivideWithThreshold(Metric, int, *logging.Logger) (int, error)
 	MultiplyByScalar(uint, *logging.Logger) (int, error)
@@ -128,10 +127,6 @@ func (m *AbstractMetric) GetName() string {
 	return m.name
 }
 
-func (m *AbstractMetric) SetName(name string) {
-	m.name = name
-}
-
 func (m *AbstractMetric) IsExportable() bool {
 	return m.exportable
 }
@@ -218,7 +213,7 @@ func (m *AbstractMetric) SetValueNAN(i *Instance) {
 	m.record[i.index] = false
 }
 
-func (m *AbstractMetric) Delta(Metric, *logging.Logger) (int, error) {
+func (m *AbstractMetric) Delta(Metric, *Matrix, *Matrix, *logging.Logger) (int, error) {
 	return 0, errs.New(errs.ErrImplement, m.dtype)
 }
 
diff --git a/pkg/matrix/metric_float64.go b/pkg/matrix/metric_float64.go
index d97cc421..7d401674 100644
--- a/pkg/matrix/metric_float64.go
+++ b/pkg/matrix/metric_float64.go
@@ -161,31 +161,42 @@ func (m *MetricFloat64) GetValuesFloat64() []float64 {
 	return m.values
 }
 
-func (m *MetricFloat64) Delta(s Metric, logger *logging.Logger) (int, error) {
+func (m *MetricFloat64) Delta(prevMetric Metric, prevMat *Matrix, curMat *Matrix, logger *logging.Logger) (int, error) {
 	var skips int
-	prevRaw := s.GetValuesFloat64()
-	sRecord := s.GetRecords()
-	if len(m.values) != len(prevRaw) {
-		return 0, errs.New(ErrUnequalVectors, fmt.Sprintf("minuend=%d, subtrahend=%d", len(m.values), len(prevRaw)))
-	}
-	for i := range m.values {
-		if m.record[i] && sRecord[i] {
-			curRaw := m.values[i]
-			m.values[i] -= prevRaw[i]
-			// Sometimes ONTAP sends spurious zeroes or values less than the previous poll.
-			// Detect and don't publish negative deltas or the subsequent poll will show a large spike.
-			isInvalidZero := (curRaw == 0 || prevRaw[i] == 0) && m.values[i] != 0
-			isNegative := m.values[i] < 0
-			if isInvalidZero || isNegative {
-				m.record[i] = false
-				skips++
-				logger.Trace().
-					Str("metric", m.GetName()).
-					Float64("currentRaw", curRaw).
-					Float64("previousRaw", prevRaw[i]).
-					Int("instIndex", i).
-					Msg("Negative cooked value")
+	prevRaw := prevMetric.GetValuesFloat64()
+	prevRecord := prevMetric.GetRecords()
+	for key, currInstance := range curMat.GetInstances() {
+		// check if this instance key exists in previous matrix
+		prevInstance := prevMat.GetInstance(key)
+		currIndex := currInstance.index
+		curRaw := m.values[currIndex]
+		if prevInstance != nil {
+			prevIndex := prevInstance.index
+			if m.record[currIndex] && prevRecord[prevIndex] {
+				m.values[currIndex] -= prevRaw[prevIndex]
+				// Sometimes ONTAP sends spurious zeroes or values less than the previous poll.
+				// Detect and don't publish negative deltas or the subsequent poll will show a large spike.
+				isInvalidZero := (curRaw == 0 || prevRaw[prevIndex] == 0) && m.values[prevIndex] != 0
+				isNegative := m.values[currIndex] < 0
+				if isInvalidZero || isNegative {
+					m.record[currIndex] = false
+					skips++
+					logger.Trace().
+						Str("metric", m.GetName()).
+						Float64("currentRaw", curRaw).
+						Float64("previousRaw", prevRaw[prevIndex]).
+						Str("instKey", key).
+						Msg("Negative cooked value")
+				}
 			}
+		} else {
+			m.record[currIndex] = false
+			skips++
+			logger.Trace().
+				Str("metric", m.GetName()).
+				Float64("currentRaw", curRaw).
+				Str("instKey", key).
+				Msg("New instance added")
 		}
 	}
 	return skips, nil


Test patch:
diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go
index 8226a6b9..aba22cb5 100644
--- a/cmd/tools/grafana/dashboard_test.go
+++ b/cmd/tools/grafana/dashboard_test.go
@@ -3,9 +3,6 @@ package grafana
 import (
 	"fmt"
 	"github.com/tidwall/gjson"
-	"log"
-	"os"
-	"path/filepath"
 	"regexp"
 	"sort"
 	"strconv"
@@ -33,27 +30,6 @@ func checkDashboardForDatasource(t *testing.T, path string, data []byte) {
 	})
 }
 
-func visitDashboards(dir string, eachDash func(path string, data []byte)) {
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			log.Fatal("failed to read directory:", err)
-		}
-		ext := filepath.Ext(path)
-		if ext != ".json" {
-			return nil
-		}
-		data, err := os.ReadFile(path)
-		if err != nil {
-			log.Fatalf("failed to read dashboards path=%s err=%v", path, err)
-		}
-		eachDash(path, data)
-		return nil
-	})
-	if err != nil {
-		log.Fatal("failed to read dashboards:", err)
-	}
-}
-
 func TestUnitsAndExprMatch(t *testing.T) {
 	mt := newMetricsTable()
 	dir := "../../../grafana/dashboards/cmode"
@@ -281,11 +257,6 @@ func doPanel(pathPrefix string, key gjson.Result, value gjson.Result, mt *metric
 	return true
 }
 
-func shortPath(dashPath string) string {
-	splits := strings.Split(dashPath, string(filepath.Separator))
-	return strings.Join(splits[len(splits)-2:], string(filepath.Separator))
-}
-
 func unitForExpr(e expression, overrides []override, defaultUnit string,
 	valueToName map[string]string, numExpressions int) string {
 
@@ -475,12 +446,6 @@ func TestTopKRange(t *testing.T) {
 	})
 }
 
-type exprP struct {
-	path string
-	expr string
-	vars []string
-}
-
 func checkTopKRange(t *testing.T, path string, data []byte) {
 	// collect all expressions
 	expressions := make([]exprP, 0)
@@ -531,62 +496,6 @@ func checkTopKRange(t *testing.T, path string, data []byte) {
 	}
 }
 
-func doTarget(pathPrefix string, key gjson.Result, value gjson.Result,
-	exprFunc func(path string, expr string, format string)) {
-	kind := value.Get("type").String()
-	if kind == "row" {
-		return
-	}
-	path := fmt.Sprintf("%spanels[%d]", pathPrefix, key.Int())
-	targetsSlice := value.Get("targets").Array()
-	for i, targetN := range targetsSlice {
-		expr := targetN.Get("expr").String()
-		pathWithTarget := path + ".targets[" + strconv.Itoa(i) + "]"
-		exprFunc(pathWithTarget, expr, kind)
-	}
-}
-
-var varRe = regexp.MustCompile(`\$(\w+)`)
-
-func newExpr(path string, expr string) exprP {
-	allMatches := varRe.FindAllStringSubmatch(expr, -1)
-	vars := make([]string, 0, len(allMatches))
-	for _, match := range allMatches {
-		vars = append(vars, match[1])
-	}
-	return exprP{
-		path: path,
-		expr: expr,
-		vars: vars,
-	}
-}
-
-type variable struct {
-	name  string
-	kind  string
-	query string
-	path  string
-}
-
-func allVariables(data []byte) []variable {
-	variables := make([]variable, 0)
-	gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool {
-		// The datasource variable can be ignored
-		if value.Get("type").String() == "datasource" {
-			return true
-		}
-
-		variables = append(variables, variable{
-			name:  value.Get("name").String(),
-			kind:  value.Get("type").String(),
-			query: value.Get("query.query").String(),
-			path:  key.String(),
-		})
-		return true
-	})
-	return variables
-}
-
 func TestOnlyHighlightsExpanded(t *testing.T) {
 	dir := "../../../grafana/dashboards/cmode"
 
@@ -642,3 +551,71 @@ func visitAllPanels(data []byte, handle func(path string, key gjson.Result, valu
 		return true
 	})
 }
+
+func TestLegends(t *testing.T) {
+	dir := "../../../grafana/dashboards/cmode"
+	visitDashboards(dir, func(path string, data []byte) {
+		checkLegends(t, path, data)
+	})
+}
+
+func checkLegends(t *testing.T, path string, data []byte) {
+	// collect all legends
+	dashPath := shortPath(path)
+
+	visitAllPanels(data, func(path string, key, value gjson.Result) {
+		doLegends(t, value, dashPath)
+	})
+}
+
+func doLegends(t *testing.T, value gjson.Result, dashPath string) {
+	wantDisplayMode := "table"
+	wantPlacement := "bottom"
+
+	kind := value.Get("type").String()
+	if kind == "row" || kind == "piechart" {
+		return
+	}
+	optionsData := value.Get("options")
+	if legendData := optionsData.Get("legend"); legendData.Exists() {
+		legendDisplayMode := legendData.Get("displayMode").String()
+		legendPlacementData := legendData.Get("placement").String()
+		title := value.Get("title").String()
+		calcsData := legendData.Get("calcs").Array()
+		var calcsSlice []string
+		for _, result := range calcsData {
+			calcsSlice = append(calcsSlice, result.String())
+		}
+		checkLegendCalculations(t, calcsSlice, dashPath, title)
+
+		// Skip hidden legends
+		if legendDisplayMode == "hidden" {
+			return
+		}
+		if legendDisplayMode != wantDisplayMode {
+			t.Errorf(`dashboard=%s, panel="%s", display mode want=%s got=%s val %v`, dashPath, title, wantDisplayMode, legendDisplayMode, legendData)
+		}
+
+		if legendPlacementData != wantPlacement {
+			t.Errorf(`dashboard=%s, panel="%s", legend placement want=%s got=%s val %v`, dashPath, title, wantPlacement, legendPlacementData, legendData)
+		}
+	}
+}
+
+func checkLegendCalculations(t *testing.T, gotLegendCalculations []string, dashPath, title string) {
+	wantLegendNoMin := strings.Join([]string{"mean", "lastNotNull", "max"}, ",")
+	wantLegendWithMin := "min," + wantLegendNoMin
+	got := strings.Join(gotLegendCalculations, ",")
+	if strings.Contains(got, "sum") {
+		return
+	}
+	if strings.Contains(got, "min") {
+		if got != wantLegendWithMin {
+			t.Errorf(`dashboard=%s, panel="%s", got=[%s] want=[%s]`, dashPath, title, got, wantLegendWithMin)
+		}
+	} else {
+		if got != wantLegendNoMin {
+			t.Errorf(`dashboard=%s, panel="%s", got=[%s] want=[%s]`, dashPath, title, got, wantLegendNoMin)
+		}
+	}
+}
diff --git a/pkg/matrix/metric_float64_test.go b/pkg/matrix/metric_float64_test.go
index 6e04c3f6..3c1ae0fe 100644
--- a/pkg/matrix/metric_float64_test.go
+++ b/pkg/matrix/metric_float64_test.go
@@ -6,19 +6,52 @@ import (
 	"testing"
 )
 
-func setupMatrix(previousRaw float64, currentRaw float64) (*Matrix, *Matrix) {
+type matrixOp string
+
+const (
+	sameInstance      = matrixOp("sameInstance")
+	deleteInstance    = matrixOp("deleteInstance")
+	addInstance       = matrixOp("addInstance")
+	addDeleteInstance = matrixOp("addDeleteInstance")
+)
+
+type instNames struct {
+	prev []string
+	cur  []string
+}
+
+var instanceNames = map[matrixOp]instNames{
+	sameInstance: {
+		prev: []string{"A"},
+		cur:  []string{"A"},
+	},
+	deleteInstance: {
+		prev: []string{"A", "B"},
+		cur:  []string{"A"},
+	},
+	addInstance: {
+		prev: []string{"A"},
+		cur:  []string{"A", "B"},
+	},
+	addDeleteInstance: {
+		prev: []string{"A"},
+		cur:  []string{"C"},
+	},
+}
+
+func setupMatrix(previousRaw float64, currentRaw float64, mop matrixOp) (*Matrix, *Matrix) {
 	m := New("Test", "test", "test")
 	speed, _ := m.NewMetricFloat64("speed")
-	instanceNames := []string{"A"}
-	for _, instanceName := range instanceNames {
+	names := instanceNames[mop]
+	for _, instanceName := range names.prev {
 		instance, _ := m.NewInstance(instanceName)
 		_ = speed.SetValueFloat64(instance, previousRaw)
 	}
 
 	m1 := New("Test", "test", "test")
 	speed1, _ := m1.NewMetricFloat64("speed")
-	instanceNames = []string{"A"}
-	for _, instanceName := range instanceNames {
+
+	for _, instanceName := range names.cur {
 		instance, _ := m1.NewInstance(instanceName)
 		_ = speed1.SetValueFloat64(instance, currentRaw)
 	}
@@ -29,26 +62,63 @@ type test struct {
 	name      string
 	curRaw    float64
 	prevRaw   float64
-	cooked    float64
+	cooked    []float64
 	threshold int
 	scalar    uint
 	skips     int
-	record    bool
+	record    []bool
 }
 
 func TestMetricFloat64_Delta(t *testing.T) {
+	testDelta(t, sameInstance)
+	testDelta(t, deleteInstance)
+
+	tests2 := []test{
+		{curRaw: 10, prevRaw: 10, cooked: []float64{0, 10}, skips: 1, record: []bool{true, false}, name: "no increase AddInstance"},
+		{curRaw: 20, prevRaw: 10, cooked: []float64{10, 20}, skips: 1, record: []bool{true, false}, name: "normal increase AddInstance"},
+		{curRaw: 10, prevRaw: 20, cooked: []float64{-10, 10}, skips: 2, record: []bool{false, false}, name: "bug negative AddInstance"},
+		{curRaw: 10, prevRaw: 0, cooked: []float64{10, 10}, skips: 2, record: []bool{false, false}, name: "bug zeroPrev AddInstance"},
+		{curRaw: 0, prevRaw: 10, cooked: []float64{-10, 0}, skips: 2, record: []bool{false, false}, name: "bug zeroCur AddInstance"},
+	}
+
+	for _, tt := range tests2 {
+		t.Run(tt.name, func(t *testing.T) {
+			previous, current := setupMatrix(tt.prevRaw, tt.curRaw, addInstance)
+			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), previous, current, logging.Get())
+			matrixTest(t, tt, current, skips, err)
+		})
+	}
+
+	tests3 := []test{
+		{curRaw: 10, prevRaw: 10, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "no increase AddDeleteInstance"},
+		{curRaw: 20, prevRaw: 10, cooked: []float64{20}, skips: 1, record: []bool{false}, name: "normal increase AddDeleteInstance"},
+		{curRaw: 10, prevRaw: 20, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "bug negative AddDeleteInstance"},
+		{curRaw: 10, prevRaw: 0, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "bug zeroPrev AddDeleteInstance"},
+		{curRaw: 0, prevRaw: 10, cooked: []float64{0}, skips: 1, record: []bool{false}, name: "bug zeroCur AddDeleteInstance"},
+	}
+
+	for _, tt := range tests3 {
+		t.Run(tt.name, func(t *testing.T) {
+			previous, current := setupMatrix(tt.prevRaw, tt.curRaw, addDeleteInstance)
+			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), previous, current, logging.Get())
+			matrixTest(t, tt, current, skips, err)
+		})
+	}
+}
+
+func testDelta(t *testing.T, op matrixOp) {
 	tests := []test{
-		{curRaw: 10, prevRaw: 10, cooked: 0, skips: 0, record: true, name: "no increase"},
-		{curRaw: 20, prevRaw: 10, cooked: 10, skips: 0, record: true, name: "normal increase"},
-		{curRaw: 10, prevRaw: 20, cooked: -10, skips: 1, record: false, name: "bug negative"},
-		{curRaw: 10, prevRaw: 0, cooked: 10, skips: 1, record: false, name: "bug zeroPrev"},
-		{curRaw: 0, prevRaw: 10, cooked: -10, skips: 1, record: false, name: "bug zeroCur"},
+		{curRaw: 10, prevRaw: 10, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "no increase"},
+		{curRaw: 20, prevRaw: 10, cooked: []float64{10}, skips: 0, record: []bool{true}, name: "normal increase"},
+		{curRaw: 10, prevRaw: 20, cooked: []float64{-10}, skips: 1, record: []bool{false}, name: "bug negative"},
+		{curRaw: 10, prevRaw: 0, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "bug zeroPrev"},
+		{curRaw: 0, prevRaw: 10, cooked: []float64{-10}, skips: 1, record: []bool{false}, name: "bug zeroCur"},
 	}
 
 	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			previous, current := setupMatrix(tt.prevRaw, tt.curRaw)
-			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), logging.Get())
+		t.Run(tt.name+"_"+string(op), func(t *testing.T) {
+			previous, current := setupMatrix(tt.prevRaw, tt.curRaw, op)
+			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), previous, current, logging.Get())
 			matrixTest(t, tt, current, skips, err)
 		})
 	}
@@ -56,17 +126,17 @@ func TestMetricFloat64_Delta(t *testing.T) {
 
 func TestMetricFloat64_Divide(t *testing.T) {
 	tests := []test{
-		{prevRaw: 20, curRaw: 10, cooked: 2, skips: 0, record: true, name: "normal"},
-		{prevRaw: -20, curRaw: 10, cooked: -2, skips: 1, record: false, name: "bug negative num"},
-		{prevRaw: 20, curRaw: -10, cooked: -2, skips: 1, record: false, name: "bug negative den"},
-		{prevRaw: -20, curRaw: -10, cooked: 2, skips: 1, record: false, name: "bug negative both"},
-		{prevRaw: 20, curRaw: 0, cooked: 0, skips: 0, record: true, name: "allow zero den"},
-		{prevRaw: 0, curRaw: 0, cooked: 0, skips: 0, record: true, name: "allow zero both"},
+		{prevRaw: 20, curRaw: 10, cooked: []float64{2}, skips: 0, record: []bool{true}, name: "normal"},
+		{prevRaw: -20, curRaw: 10, cooked: []float64{-2}, skips: 1, record: []bool{false}, name: "bug negative num"},
+		{prevRaw: 20, curRaw: -10, cooked: []float64{-2}, skips: 1, record: []bool{false}, name: "bug negative den"},
+		{prevRaw: -20, curRaw: -10, cooked: []float64{2}, skips: 1, record: []bool{false}, name: "bug negative both"},
+		{prevRaw: 20, curRaw: 0, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "allow zero den"},
+		{prevRaw: 0, curRaw: 0, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "allow zero both"},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw)
+			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw, sameInstance)
 			skips, err := numerator.GetMetric("speed").Divide(denominator.GetMetric("speed"), logging.Get())
 			matrixTest(t, tt, numerator, skips, err)
 		})
@@ -75,17 +145,17 @@ func TestMetricFloat64_Divide(t *testing.T) {
 
 func TestMetricFloat64_DivideWithThreshold(t *testing.T) {
 	tests := []test{
-		{prevRaw: 20, curRaw: 10, threshold: 5, cooked: 2, skips: 0, record: true, name: "normal"},
-		{prevRaw: 20, curRaw: 10, threshold: 15, cooked: 0, skips: 0, record: true, name: "normal < threshold"},
-		{prevRaw: 20, curRaw: -10, threshold: 5, cooked: 0, skips: 1, record: false, name: "bug negative den"},
-		{prevRaw: -20, curRaw: -10, threshold: 5, cooked: 0, skips: 1, record: false, name: "bug negative both"},
-		{prevRaw: 20, curRaw: 0, threshold: 0, cooked: math.Inf(1), skips: 0, record: true, name: "allow no threshold"},
-		{prevRaw: 0, curRaw: 0, threshold: 5, cooked: 0, skips: 0, record: true, name: "allow zero both"},
+		{prevRaw: 20, curRaw: 10, threshold: 5, cooked: []float64{2}, skips: 0, record: []bool{true}, name: "normal"},
+		{prevRaw: 20, curRaw: 10, threshold: 15, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "normal < threshold"},
+		{prevRaw: 20, curRaw: -10, threshold: 5, cooked: []float64{0}, skips: 1, record: []bool{false}, name: "bug negative den"},
+		{prevRaw: -20, curRaw: -10, threshold: 5, cooked: []float64{0}, skips: 1, record: []bool{false}, name: "bug negative both"},
+		{prevRaw: 20, curRaw: 0, threshold: 0, cooked: []float64{math.Inf(1)}, skips: 0, record: []bool{true}, name: "allow no threshold"},
+		{prevRaw: 0, curRaw: 0, threshold: 5, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "allow zero both"},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw)
+			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw, sameInstance)
 			skips, err := numerator.GetMetric("speed").
 				DivideWithThreshold(denominator.GetMetric("speed"), tt.threshold, logging.Get())
 			matrixTest(t, tt, numerator, skips, err)
@@ -95,13 +165,13 @@ func TestMetricFloat64_DivideWithThreshold(t *testing.T) {
 
 func TestMetricFloat64_MultiplyByScalar(t *testing.T) {
 	tests := []test{
-		{prevRaw: 10, curRaw: 10, scalar: 5, cooked: 50, skips: 0, record: true, name: "normal"},
-		{prevRaw: 10, curRaw: -10, scalar: 5, cooked: -50, skips: 1, record: false, name: "bug negative"},
+		{prevRaw: 10, curRaw: 10, scalar: 5, cooked: []float64{50}, skips: 0, record: []bool{true}, name: "normal"},
+		{prevRaw: 10, curRaw: -10, scalar: 5, cooked: []float64{-50}, skips: 1, record: []bool{false}, name: "bug negative"},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, current := setupMatrix(tt.prevRaw, tt.curRaw)
+			_, current := setupMatrix(tt.prevRaw, tt.curRaw, sameInstance)
 			skips, err := current.GetMetric("speed").MultiplyByScalar(tt.scalar, logging.Get())
 			matrixTest(t, tt, current, skips, err)
 		})
@@ -116,13 +186,18 @@ func matrixTest(t *testing.T, tt test, cur *Matrix, skips int, err error) {
 	if skips != tt.skips {
 		t.Errorf("skips expected = %d, got %d", tt.skips, skips)
 	}
-	cooked := cur.GetMetric("speed").GetValuesFloat64()[0]
-	if cooked != tt.cooked {
-		t.Errorf("cooked expected = %v, got %v", tt.cooked, cooked)
+	cooked := cur.GetMetric("speed").GetValuesFloat64()
+	for i := range cooked {
+		if cooked[i] != tt.cooked[i] {
+			t.Errorf("cooked expected = %v, got %v", tt.cooked, cooked)
+		}
 	}
-	record := cur.GetMetric("speed").GetRecords()[0]
-	if record != tt.record {
-		t.Errorf("record expected = %t, got %t", tt.record, record)
+
+	record := cur.GetMetric("speed").GetRecords()
+	for i := range record {
+		if record[i] != tt.record[i] {
+			t.Errorf("record expected = %t, got %t", tt.record, record)
+		}
 	}
 }