Repository: NetApp/harvest
Pull request #1571 (2022-11-30 16:13:01)

feat: perf collector delta calculation handling

cgrinds: nice job, especially on the new tests. I made some minor refactoring changes in #1575 take a look and see what you think


Patch:
diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go
index a2ec15a4..4c77d05c 100644
--- a/cmd/collectors/restperf/restperf.go
+++ b/cmd/collectors/restperf/restperf.go
@@ -482,11 +482,11 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 	r.Logger.Trace().Msg("updating data cache")
 
-	mat := r.Matrix[r.Object]
+	prevMat := r.Matrix[r.Object]
 	// clone matrix without numeric data
-	newData := mat.Clone(false, true, true)
-	newData.Reset()
-	timestamp := newData.GetMetric("timestamp")
+	curMat := prevMat.Clone(false, true, true)
+	curMat.Reset()
+	timestamp := curMat.GetMetric("timestamp")
 	if timestamp == nil {
 		return nil, errs.New(errs.ErrConfig, "missing timestamp metric")
 	}
@@ -584,7 +584,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 					return true
 				}
 
-				if resourceLatency = newData.GetMetric(layer); resourceLatency == nil {
+				if resourceLatency = curMat.GetMetric(layer); resourceLatency == nil {
 					r.Logger.Trace().
 						Str("layer", layer).
 						Msg("Resource-latency metric missing in cache")
@@ -598,7 +598,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 				}
 			}
 
-			instance = newData.GetInstance(instanceKey)
+			instance = curMat.GetInstance(instanceKey)
 
 			if instance == nil {
 				if isWorkloadObject(r.Prop.Query) || isWorkloadDetailObject(r.Prop.Query) {
@@ -684,11 +684,11 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 							isHistogram = false
 							if len(labels) > 0 && strings.Contains(r.perfProp.counterInfo[name].description, "histogram") {
 								key := name + ".bucket"
-								histogramMetric = newData.GetMetric(key)
+								histogramMetric = curMat.GetMetric(key)
 								if histogramMetric != nil {
 									r.Logger.Trace().Str("metric", key).Msg("Updating array metric attributes")
 								} else {
-									histogramMetric, err = newData.NewMetricFloat64(key)
+									histogramMetric, err = curMat.NewMetricFloat64(key)
 									if err != nil {
 										r.Logger.Error().Err(err).Str("key", key).Msg("unable to create histogram metric")
 										continue
@@ -703,9 +703,9 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 							for i, label := range labels {
 								k := name + arrayKeyToken + label
-								metr, ok := newData.GetMetrics()[k]
+								metr, ok := curMat.GetMetrics()[k]
 								if !ok {
-									if metr, err = newData.NewMetricFloat64(k); err != nil {
+									if metr, err = curMat.NewMetricFloat64(k); err != nil {
 										r.Logger.Error().Err(err).
 											Str("name", k).
 											Msg("NewMetricFloat64")
@@ -749,9 +749,9 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 								}
 							}
 						} else {
-							metr, ok := newData.GetMetrics()[name]
+							metr, ok := curMat.GetMetrics()[name]
 							if !ok {
-								if metr, err = newData.NewMetricFloat64(name); err != nil {
+								if metr, err = curMat.NewMetricFloat64(name); err != nil {
 									r.Logger.Error().Err(err).
 										Str("name", name).
 										Int("instIndex", instIndex).
@@ -789,7 +789,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 					r.Logger.Warn().Str("counter", name).Msg("Counter is missing or unable to parse.")
 				}
 			}
-			if err = newData.GetMetric("timestamp").SetValueFloat64(instance, ts); err != nil {
+			if err = curMat.GetMetric("timestamp").SetValueFloat64(instance, ts); err != nil {
 				r.Logger.Error().Err(err).Msg("Failed to set timestamp")
 			}
 
@@ -799,7 +799,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	}
 
 	if isWorkloadDetailObject(r.Prop.Query) {
-		if err := r.getParentOpsCounters(newData); err != nil {
+		if err := r.getParentOpsCounters(curMat); err != nil {
 			// no point to continue as we can't calculate the other counters
 			return nil, err
 		}
@@ -814,7 +814,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	// skip calculating from delta if no data from previous poll
 	if r.perfProp.isCacheEmpty {
 		r.Logger.Debug().Msg("skip postprocessing until next poll (previous cache empty)")
-		r.Matrix[r.Object] = newData
+		r.Matrix[r.Object] = curMat
 		r.perfProp.isCacheEmpty = false
 		return nil, nil
 	}
@@ -824,15 +824,15 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	r.Logger.Debug().Msg("starting delta calculations from previous cache")
 
 	// cache raw data for next poll
-	cachedData := newData.Clone(true, true, true)
+	cachedData := curMat.Clone(true, true, true)
 
-	orderedNonDenominatorMetrics := make([]matrix.Metric, 0, len(newData.GetMetrics()))
+	orderedNonDenominatorMetrics := make([]matrix.Metric, 0, len(curMat.GetMetrics()))
 	orderedNonDenominatorKeys := make([]string, 0, len(orderedNonDenominatorMetrics))
 
-	orderedDenominatorMetrics := make([]matrix.Metric, 0, len(newData.GetMetrics()))
+	orderedDenominatorMetrics := make([]matrix.Metric, 0, len(curMat.GetMetrics()))
 	orderedDenominatorKeys := make([]string, 0, len(orderedDenominatorMetrics))
 
-	for key, metric := range newData.GetMetrics() {
+	for key, metric := range curMat.GetMetrics() {
 		if metric.GetName() != "timestamp" {
 			counter := r.counterLookup(metric, key)
 			if counter != nil {
@@ -857,7 +857,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 	// calculate timestamp delta first since many counters require it for postprocessing.
 	// Timestamp has "raw" property, so it isn't post-processed automatically
-	if _, err = timestamp.Delta(mat.GetMetric("timestamp"), r.Logger); err != nil {
+	if _, err = timestamp.Delta(prevMat.GetMetric("timestamp"), prevMat, curMat, r.Logger); err != nil {
 		r.Logger.Error().Err(err).Msg("(timestamp) calculate delta:")
 	}
 
@@ -883,7 +883,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 		}
 
 		// all other properties - first calculate delta
-		if skips, err = metric.Delta(mat.GetMetric(key), r.Logger); err != nil {
+		if skips, err = metric.Delta(prevMat.GetMetric(key), prevMat, curMat, r.Logger); err != nil {
 			r.Logger.Error().Err(err).Str("key", key).Msg("Calculate delta")
 			continue
 		}
@@ -905,7 +905,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 		// For the next two properties we need base counters
 		// We assume that delta of base counters is already calculated
-		if base = newData.GetMetric(counter.denominator); base == nil {
+		if base = curMat.GetMetric(counter.denominator); base == nil {
 			r.Logger.Warn().
 				Str("key", key).
 				Str("property", property).
@@ -981,7 +981,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	}
 
 	calcD := time.Since(calcStart)
-	_ = r.Metadata.LazySetValueUint64("instances", "data", uint64(len(newData.GetInstances())))
+	_ = r.Metadata.LazySetValueUint64("instances", "data", uint64(len(curMat.GetInstances())))
 	_ = r.Metadata.LazySetValueInt64("calc_time", "data", calcD.Microseconds())
 	_ = r.Metadata.LazySetValueUint64("skips", "data", uint64(totalSkips))
 
@@ -989,7 +989,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	r.Matrix[r.Object] = cachedData
 
 	newDataMap := make(map[string]*matrix.Matrix)
-	newDataMap[r.Object] = newData
+	newDataMap[r.Object] = curMat
 	return newDataMap, nil
 }
 
diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go
index 7b3a018c..8368bf48 100644
--- a/cmd/collectors/zapiperf/zapiperf.go
+++ b/cmd/collectors/zapiperf/zapiperf.go
@@ -196,12 +196,12 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	)
 
 	z.Logger.Trace().Msg("updating data cache")
-	m := z.Matrix[z.Object]
+	prevMat := z.Matrix[z.Object]
 	// clone matrix without numeric data
-	newData := m.Clone(false, true, true)
-	newData.Reset()
+	curMat := prevMat.Clone(false, true, true)
+	curMat.Reset()
 
-	timestamp := newData.GetMetric("timestamp")
+	timestamp := curMat.GetMetric("timestamp")
 	if timestamp == nil {
 		return nil, errs.New(errs.ErrConfig, "missing timestamp metric") // @TODO errconfig??
 	}
@@ -226,13 +226,13 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		} else {
 			instanceKeys = make([]string, 0)
 			for _, layer := range resourceMap.GetAllChildNamesS() {
-				for key := range m.GetInstances() {
+				for key := range prevMat.GetInstances() {
 					instanceKeys = append(instanceKeys, key+"."+layer)
 				}
 			}
 		}
 	} else {
-		instanceKeys = newData.GetInstanceKeys()
+		instanceKeys = curMat.GetInstanceKeys()
 	}
 
 	// build ZAPI request
@@ -340,7 +340,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 					continue
 				}
 
-				if resourceLatency = newData.GetMetric(layer); resourceLatency == nil {
+				if resourceLatency = curMat.GetMetric(layer); resourceLatency == nil {
 					z.Logger.Warn().
 						Str("layer", layer).
 						Msg("Resource-latency metric missing in cache")
@@ -357,7 +357,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 				continue
 			}
 
-			instance := newData.GetInstance(key)
+			instance := curMat.GetInstance(key)
 			if instance == nil {
 				z.Logger.Debug().
 					Str("key", key).
@@ -427,7 +427,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 					}
 
 					for i, label := range labels {
-						if metric := newData.GetMetric(name + "." + label); metric != nil {
+						if metric := curMat.GetMetric(name + "." + label); metric != nil {
 							if err = metric.SetValueString(instance, values[i]); err != nil {
 								z.Logger.Error().
 									Stack().
@@ -452,7 +452,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 								Str("label", label).
 								Str("value", value).
 								Int("instIndex", instIndex).
-								Msg("Histogram name.label not in cache")
+								Msg("Histogram name. Label not in cache")
 						}
 					}
 					continue
@@ -486,7 +486,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 				}
 
 				// store as scalar metric
-				if metric := newData.GetMetric(name); metric != nil {
+				if metric := curMat.GetMetric(name); metric != nil {
 					if err = metric.SetValueString(instance, value); err != nil {
 						z.Logger.Error().
 							Err(err).
@@ -521,7 +521,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		Msg("Collected data points in batch polls")
 
 	if z.Query == objWorkloadDetail || z.Query == objWorkloadDetailVolume {
-		if rd, pd, err := z.getParentOpsCounters(newData, keyName); err == nil {
+		if rd, pd, err := z.getParentOpsCounters(curMat, keyName); err == nil {
 			apiT += rd
 			parseT += pd
 		} else {
@@ -540,7 +540,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	// skip calculating from delta if no data from previous poll
 	if z.isCacheEmpty {
 		z.Logger.Debug().Msg("skip postprocessing until next poll (previous cache empty)")
-		z.Matrix[z.Object] = newData
+		z.Matrix[z.Object] = curMat
 		z.isCacheEmpty = false
 		return nil, nil
 	}
@@ -550,19 +550,19 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	z.Logger.Debug().Msg("starting delta calculations from previous cache")
 
 	// cache raw data for next poll
-	cachedData := newData.Clone(true, true, true) // @TODO implement copy data
+	cachedData := curMat.Clone(true, true, true) // @TODO implement copy data
 
 	// order metrics, such that those requiring base counters are processed last
-	orderedMetrics := make([]matrix.Metric, 0, len(newData.GetMetrics()))
+	orderedMetrics := make([]matrix.Metric, 0, len(curMat.GetMetrics()))
 	orderedKeys := make([]string, 0, len(orderedMetrics))
 
-	for key, metric := range newData.GetMetrics() {
+	for key, metric := range curMat.GetMetrics() {
 		if metric.GetComment() == "" { // does not require base counter
 			orderedMetrics = append(orderedMetrics, metric)
 			orderedKeys = append(orderedKeys, key)
 		}
 	}
-	for key, metric := range newData.GetMetrics() {
+	for key, metric := range curMat.GetMetrics() {
 		if metric.GetComment() != "" { // requires base counter
 			orderedMetrics = append(orderedMetrics, metric)
 			orderedKeys = append(orderedKeys, key)
@@ -571,7 +571,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 
 	// calculate timestamp delta first since many counters require it for postprocessing.
 	// Timestamp has "raw" property, so it isn't post-processed automatically
-	if _, err = timestamp.Delta(m.GetMetric("timestamp"), z.Logger); err != nil {
+	if _, err = timestamp.Delta(prevMat.GetMetric("timestamp"), prevMat, curMat, z.Logger); err != nil {
 		z.Logger.Error().Err(err).Msg("(timestamp) calculate delta:")
 		// @TODO terminate since other counters will be incorrect
 	}
@@ -590,7 +590,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		}
 
 		// all other properties - first calculate delta
-		if skips, err = metric.Delta(m.GetMetric(key), z.Logger); err != nil {
+		if skips, err = metric.Delta(prevMat.GetMetric(key), prevMat, curMat, z.Logger); err != nil {
 			z.Logger.Error().Err(err).Str("key", key).Msg("Calculate delta")
 			continue
 		}
@@ -613,7 +613,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		// For the next two properties we need base counters
 		// We assume that delta of base counters is already calculated
 		// (name of base counter is stored as Comment)
-		if base = newData.GetMetric(metric.GetComment()); base == nil {
+		if base = curMat.GetMetric(metric.GetComment()); base == nil {
 			z.Logger.Warn().
 				Str("key", key).
 				Str("property", property).
@@ -683,7 +683,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 	z.Matrix[z.Object] = cachedData
 
 	newDataMap := make(map[string]*matrix.Matrix)
-	newDataMap[z.Object] = newData
+	newDataMap[z.Object] = curMat
 	return newDataMap, nil
 }
 
diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go
index fe568ce6..3a501c59 100644
--- a/cmd/tools/grafana/grafana.go
+++ b/cmd/tools/grafana/grafana.go
@@ -513,7 +513,8 @@ func importFiles(dir string, folder *Folder) {
 			fmt.Printf("error importing [%s] to folder [%s] - the server replied with [%s]\n", file.Name(), folder.name, status)
 			if code == 412 {
 				fmt.Printf("That means you are trying to overwrite an existing dashboard.\n\n")
-				fmt.Printf("If you want to overwrite run with the --overwrite flag or choose a different Grafana folder with the --serverfolder flag\n\n")
+				fmt.Printf("If you want to overwrite, run with the --overwrite flag or choose a different Grafana folder with the --serverfolder flag.\n")
+				fmt.Printf("NOTE: If the dashboard already exists, --overwrite will import the changes and increment the dashboard version number.\n\n")
 			}
 			fmt.Printf("Server response follows:\n")
 			for k, v := range result {
diff --git a/pkg/matrix/metric.go b/pkg/matrix/metric.go
index fcba68d6..f39357df 100644
--- a/pkg/matrix/metric.go
+++ b/pkg/matrix/metric.go
@@ -80,7 +80,7 @@ type Metric interface {
 	// methods for doing vector arithmetics
 	// currently only supported for float64!
 
-	Delta(Metric, *logging.Logger) (int, error)
+	Delta(Metric, *Matrix, *Matrix, *logging.Logger) (int, error)
 	Divide(Metric, *logging.Logger) (int, error)
 	DivideWithThreshold(Metric, int, *logging.Logger) (int, error)
 	MultiplyByScalar(uint, *logging.Logger) (int, error)
@@ -218,7 +218,7 @@ func (m *AbstractMetric) SetValueNAN(i *Instance) {
 	m.record[i.index] = false
 }
 
-func (m *AbstractMetric) Delta(Metric, *logging.Logger) (int, error) {
+func (m *AbstractMetric) Delta(Metric, *Matrix, *Matrix, *logging.Logger) (int, error) {
 	return 0, errs.New(errs.ErrImplement, m.dtype)
 }
 
diff --git a/pkg/matrix/metric_float64.go b/pkg/matrix/metric_float64.go
index d97cc421..7d401674 100644
--- a/pkg/matrix/metric_float64.go
+++ b/pkg/matrix/metric_float64.go
@@ -161,31 +161,42 @@ func (m *MetricFloat64) GetValuesFloat64() []float64 {
 	return m.values
 }
 
-func (m *MetricFloat64) Delta(s Metric, logger *logging.Logger) (int, error) {
+func (m *MetricFloat64) Delta(prevMetric Metric, prevMat *Matrix, curMat *Matrix, logger *logging.Logger) (int, error) {
 	var skips int
-	prevRaw := s.GetValuesFloat64()
-	sRecord := s.GetRecords()
-	if len(m.values) != len(prevRaw) {
-		return 0, errs.New(ErrUnequalVectors, fmt.Sprintf("minuend=%d, subtrahend=%d", len(m.values), len(prevRaw)))
-	}
-	for i := range m.values {
-		if m.record[i] && sRecord[i] {
-			curRaw := m.values[i]
-			m.values[i] -= prevRaw[i]
-			// Sometimes ONTAP sends spurious zeroes or values less than the previous poll.
-			// Detect and don't publish negative deltas or the subsequent poll will show a large spike.
-			isInvalidZero := (curRaw == 0 || prevRaw[i] == 0) && m.values[i] != 0
-			isNegative := m.values[i] < 0
-			if isInvalidZero || isNegative {
-				m.record[i] = false
-				skips++
-				logger.Trace().
-					Str("metric", m.GetName()).
-					Float64("currentRaw", curRaw).
-					Float64("previousRaw", prevRaw[i]).
-					Int("instIndex", i).
-					Msg("Negative cooked value")
+	prevRaw := prevMetric.GetValuesFloat64()
+	prevRecord := prevMetric.GetRecords()
+	for key, currInstance := range curMat.GetInstances() {
+		// check if this instance key exists in previous matrix
+		prevInstance := prevMat.GetInstance(key)
+		currIndex := currInstance.index
+		curRaw := m.values[currIndex]
+		if prevInstance != nil {
+			prevIndex := prevInstance.index
+			if m.record[currIndex] && prevRecord[prevIndex] {
+				m.values[currIndex] -= prevRaw[prevIndex]
+				// Sometimes ONTAP sends spurious zeroes or values less than the previous poll.
+				// Detect and don't publish negative deltas or the subsequent poll will show a large spike.
+				isInvalidZero := (curRaw == 0 || prevRaw[prevIndex] == 0) && m.values[prevIndex] != 0
+				isNegative := m.values[currIndex] < 0
+				if isInvalidZero || isNegative {
+					m.record[currIndex] = false
+					skips++
+					logger.Trace().
+						Str("metric", m.GetName()).
+						Float64("currentRaw", curRaw).
+						Float64("previousRaw", prevRaw[prevIndex]).
+						Str("instKey", key).
+						Msg("Negative cooked value")
+				}
 			}
+		} else {
+			m.record[currIndex] = false
+			skips++
+			logger.Trace().
+				Str("metric", m.GetName()).
+				Float64("currentRaw", curRaw).
+				Str("instKey", key).
+				Msg("New instance added")
 		}
 	}
 	return skips, nil


Test patch:
diff --git a/pkg/matrix/metric_float64_test.go b/pkg/matrix/metric_float64_test.go
index 6e04c3f6..3c1ae0fe 100644
--- a/pkg/matrix/metric_float64_test.go
+++ b/pkg/matrix/metric_float64_test.go
@@ -6,19 +6,52 @@ import (
 	"testing"
 )
 
-func setupMatrix(previousRaw float64, currentRaw float64) (*Matrix, *Matrix) {
+type matrixOp string
+
+const (
+	sameInstance      = matrixOp("sameInstance")
+	deleteInstance    = matrixOp("deleteInstance")
+	addInstance       = matrixOp("addInstance")
+	addDeleteInstance = matrixOp("addDeleteInstance")
+)
+
+type instNames struct {
+	prev []string
+	cur  []string
+}
+
+var instanceNames = map[matrixOp]instNames{
+	sameInstance: {
+		prev: []string{"A"},
+		cur:  []string{"A"},
+	},
+	deleteInstance: {
+		prev: []string{"A", "B"},
+		cur:  []string{"A"},
+	},
+	addInstance: {
+		prev: []string{"A"},
+		cur:  []string{"A", "B"},
+	},
+	addDeleteInstance: {
+		prev: []string{"A"},
+		cur:  []string{"C"},
+	},
+}
+
+func setupMatrix(previousRaw float64, currentRaw float64, mop matrixOp) (*Matrix, *Matrix) {
 	m := New("Test", "test", "test")
 	speed, _ := m.NewMetricFloat64("speed")
-	instanceNames := []string{"A"}
-	for _, instanceName := range instanceNames {
+	names := instanceNames[mop]
+	for _, instanceName := range names.prev {
 		instance, _ := m.NewInstance(instanceName)
 		_ = speed.SetValueFloat64(instance, previousRaw)
 	}
 
 	m1 := New("Test", "test", "test")
 	speed1, _ := m1.NewMetricFloat64("speed")
-	instanceNames = []string{"A"}
-	for _, instanceName := range instanceNames {
+
+	for _, instanceName := range names.cur {
 		instance, _ := m1.NewInstance(instanceName)
 		_ = speed1.SetValueFloat64(instance, currentRaw)
 	}
@@ -29,26 +62,63 @@ type test struct {
 	name      string
 	curRaw    float64
 	prevRaw   float64
-	cooked    float64
+	cooked    []float64
 	threshold int
 	scalar    uint
 	skips     int
-	record    bool
+	record    []bool
 }
 
 func TestMetricFloat64_Delta(t *testing.T) {
+	testDelta(t, sameInstance)
+	testDelta(t, deleteInstance)
+
+	tests2 := []test{
+		{curRaw: 10, prevRaw: 10, cooked: []float64{0, 10}, skips: 1, record: []bool{true, false}, name: "no increase AddInstance"},
+		{curRaw: 20, prevRaw: 10, cooked: []float64{10, 20}, skips: 1, record: []bool{true, false}, name: "normal increase AddInstance"},
+		{curRaw: 10, prevRaw: 20, cooked: []float64{-10, 10}, skips: 2, record: []bool{false, false}, name: "bug negative AddInstance"},
+		{curRaw: 10, prevRaw: 0, cooked: []float64{10, 10}, skips: 2, record: []bool{false, false}, name: "bug zeroPrev AddInstance"},
+		{curRaw: 0, prevRaw: 10, cooked: []float64{-10, 0}, skips: 2, record: []bool{false, false}, name: "bug zeroCur AddInstance"},
+	}
+
+	for _, tt := range tests2 {
+		t.Run(tt.name, func(t *testing.T) {
+			previous, current := setupMatrix(tt.prevRaw, tt.curRaw, addInstance)
+			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), previous, current, logging.Get())
+			matrixTest(t, tt, current, skips, err)
+		})
+	}
+
+	tests3 := []test{
+		{curRaw: 10, prevRaw: 10, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "no increase AddDeleteInstance"},
+		{curRaw: 20, prevRaw: 10, cooked: []float64{20}, skips: 1, record: []bool{false}, name: "normal increase AddDeleteInstance"},
+		{curRaw: 10, prevRaw: 20, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "bug negative AddDeleteInstance"},
+		{curRaw: 10, prevRaw: 0, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "bug zeroPrev AddDeleteInstance"},
+		{curRaw: 0, prevRaw: 10, cooked: []float64{0}, skips: 1, record: []bool{false}, name: "bug zeroCur AddDeleteInstance"},
+	}
+
+	for _, tt := range tests3 {
+		t.Run(tt.name, func(t *testing.T) {
+			previous, current := setupMatrix(tt.prevRaw, tt.curRaw, addDeleteInstance)
+			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), previous, current, logging.Get())
+			matrixTest(t, tt, current, skips, err)
+		})
+	}
+}
+
+func testDelta(t *testing.T, op matrixOp) {
 	tests := []test{
-		{curRaw: 10, prevRaw: 10, cooked: 0, skips: 0, record: true, name: "no increase"},
-		{curRaw: 20, prevRaw: 10, cooked: 10, skips: 0, record: true, name: "normal increase"},
-		{curRaw: 10, prevRaw: 20, cooked: -10, skips: 1, record: false, name: "bug negative"},
-		{curRaw: 10, prevRaw: 0, cooked: 10, skips: 1, record: false, name: "bug zeroPrev"},
-		{curRaw: 0, prevRaw: 10, cooked: -10, skips: 1, record: false, name: "bug zeroCur"},
+		{curRaw: 10, prevRaw: 10, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "no increase"},
+		{curRaw: 20, prevRaw: 10, cooked: []float64{10}, skips: 0, record: []bool{true}, name: "normal increase"},
+		{curRaw: 10, prevRaw: 20, cooked: []float64{-10}, skips: 1, record: []bool{false}, name: "bug negative"},
+		{curRaw: 10, prevRaw: 0, cooked: []float64{10}, skips: 1, record: []bool{false}, name: "bug zeroPrev"},
+		{curRaw: 0, prevRaw: 10, cooked: []float64{-10}, skips: 1, record: []bool{false}, name: "bug zeroCur"},
 	}
 
 	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			previous, current := setupMatrix(tt.prevRaw, tt.curRaw)
-			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), logging.Get())
+		t.Run(tt.name+"_"+string(op), func(t *testing.T) {
+			previous, current := setupMatrix(tt.prevRaw, tt.curRaw, op)
+			skips, err := current.GetMetric("speed").Delta(previous.GetMetric("speed"), previous, current, logging.Get())
 			matrixTest(t, tt, current, skips, err)
 		})
 	}
@@ -56,17 +126,17 @@ func TestMetricFloat64_Delta(t *testing.T) {
 
 func TestMetricFloat64_Divide(t *testing.T) {
 	tests := []test{
-		{prevRaw: 20, curRaw: 10, cooked: 2, skips: 0, record: true, name: "normal"},
-		{prevRaw: -20, curRaw: 10, cooked: -2, skips: 1, record: false, name: "bug negative num"},
-		{prevRaw: 20, curRaw: -10, cooked: -2, skips: 1, record: false, name: "bug negative den"},
-		{prevRaw: -20, curRaw: -10, cooked: 2, skips: 1, record: false, name: "bug negative both"},
-		{prevRaw: 20, curRaw: 0, cooked: 0, skips: 0, record: true, name: "allow zero den"},
-		{prevRaw: 0, curRaw: 0, cooked: 0, skips: 0, record: true, name: "allow zero both"},
+		{prevRaw: 20, curRaw: 10, cooked: []float64{2}, skips: 0, record: []bool{true}, name: "normal"},
+		{prevRaw: -20, curRaw: 10, cooked: []float64{-2}, skips: 1, record: []bool{false}, name: "bug negative num"},
+		{prevRaw: 20, curRaw: -10, cooked: []float64{-2}, skips: 1, record: []bool{false}, name: "bug negative den"},
+		{prevRaw: -20, curRaw: -10, cooked: []float64{2}, skips: 1, record: []bool{false}, name: "bug negative both"},
+		{prevRaw: 20, curRaw: 0, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "allow zero den"},
+		{prevRaw: 0, curRaw: 0, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "allow zero both"},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw)
+			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw, sameInstance)
 			skips, err := numerator.GetMetric("speed").Divide(denominator.GetMetric("speed"), logging.Get())
 			matrixTest(t, tt, numerator, skips, err)
 		})
@@ -75,17 +145,17 @@ func TestMetricFloat64_Divide(t *testing.T) {
 
 func TestMetricFloat64_DivideWithThreshold(t *testing.T) {
 	tests := []test{
-		{prevRaw: 20, curRaw: 10, threshold: 5, cooked: 2, skips: 0, record: true, name: "normal"},
-		{prevRaw: 20, curRaw: 10, threshold: 15, cooked: 0, skips: 0, record: true, name: "normal < threshold"},
-		{prevRaw: 20, curRaw: -10, threshold: 5, cooked: 0, skips: 1, record: false, name: "bug negative den"},
-		{prevRaw: -20, curRaw: -10, threshold: 5, cooked: 0, skips: 1, record: false, name: "bug negative both"},
-		{prevRaw: 20, curRaw: 0, threshold: 0, cooked: math.Inf(1), skips: 0, record: true, name: "allow no threshold"},
-		{prevRaw: 0, curRaw: 0, threshold: 5, cooked: 0, skips: 0, record: true, name: "allow zero both"},
+		{prevRaw: 20, curRaw: 10, threshold: 5, cooked: []float64{2}, skips: 0, record: []bool{true}, name: "normal"},
+		{prevRaw: 20, curRaw: 10, threshold: 15, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "normal < threshold"},
+		{prevRaw: 20, curRaw: -10, threshold: 5, cooked: []float64{0}, skips: 1, record: []bool{false}, name: "bug negative den"},
+		{prevRaw: -20, curRaw: -10, threshold: 5, cooked: []float64{0}, skips: 1, record: []bool{false}, name: "bug negative both"},
+		{prevRaw: 20, curRaw: 0, threshold: 0, cooked: []float64{math.Inf(1)}, skips: 0, record: []bool{true}, name: "allow no threshold"},
+		{prevRaw: 0, curRaw: 0, threshold: 5, cooked: []float64{0}, skips: 0, record: []bool{true}, name: "allow zero both"},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw)
+			numerator, denominator := setupMatrix(tt.prevRaw, tt.curRaw, sameInstance)
 			skips, err := numerator.GetMetric("speed").
 				DivideWithThreshold(denominator.GetMetric("speed"), tt.threshold, logging.Get())
 			matrixTest(t, tt, numerator, skips, err)
@@ -95,13 +165,13 @@ func TestMetricFloat64_DivideWithThreshold(t *testing.T) {
 
 func TestMetricFloat64_MultiplyByScalar(t *testing.T) {
 	tests := []test{
-		{prevRaw: 10, curRaw: 10, scalar: 5, cooked: 50, skips: 0, record: true, name: "normal"},
-		{prevRaw: 10, curRaw: -10, scalar: 5, cooked: -50, skips: 1, record: false, name: "bug negative"},
+		{prevRaw: 10, curRaw: 10, scalar: 5, cooked: []float64{50}, skips: 0, record: []bool{true}, name: "normal"},
+		{prevRaw: 10, curRaw: -10, scalar: 5, cooked: []float64{-50}, skips: 1, record: []bool{false}, name: "bug negative"},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, current := setupMatrix(tt.prevRaw, tt.curRaw)
+			_, current := setupMatrix(tt.prevRaw, tt.curRaw, sameInstance)
 			skips, err := current.GetMetric("speed").MultiplyByScalar(tt.scalar, logging.Get())
 			matrixTest(t, tt, current, skips, err)
 		})
@@ -116,13 +186,18 @@ func matrixTest(t *testing.T, tt test, cur *Matrix, skips int, err error) {
 	if skips != tt.skips {
 		t.Errorf("skips expected = %d, got %d", tt.skips, skips)
 	}
-	cooked := cur.GetMetric("speed").GetValuesFloat64()[0]
-	if cooked != tt.cooked {
-		t.Errorf("cooked expected = %v, got %v", tt.cooked, cooked)
+	cooked := cur.GetMetric("speed").GetValuesFloat64()
+	for i := range cooked {
+		if cooked[i] != tt.cooked[i] {
+			t.Errorf("cooked expected = %v, got %v", tt.cooked, cooked)
+		}
 	}
-	record := cur.GetMetric("speed").GetRecords()[0]
-	if record != tt.record {
-		t.Errorf("record expected = %t, got %t", tt.record, record)
+
+	record := cur.GetMetric("speed").GetRecords()
+	for i := range record {
+		if record[i] != tt.record[i] {
+			t.Errorf("record expected = %t, got %t", tt.record, record)
+		}
 	}
 }