Repository: flant/shell-operator
Pull request #215 (2020-11-24 12:46:02)

feat: expire action to explicitly expire all metrics in group
- fix: expire all metrics in group before apply new state

nabokihms: LGTM!

Patch:
diff --git a/Dockerfile b/Dockerfile
index 090e525..06ff377 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-# Pre-build libjq.
+# Prebuilt libjq.
 FROM --platform=${TARGETPLATFORM:-linux/amd64} flant/jq:b6be13d5-musl as libjq
 
 # Go builder.
diff --git a/METRICS.md b/METRICS.md
index ab5a483..1b78f5f 100644
--- a/METRICS.md
+++ b/METRICS.md
@@ -39,50 +39,71 @@ Hooks can export metrics by writing a set of operations in JSON format into $MET
 Operation to register a counter and increase its value:
 
 ```json
-{"name":"metric_name","add":1,"labels":{"label1":"value1"}}
+{"name":"metric_name","action":"add","value":1,"labels":{"label1":"value1"}}
 ```
 
 Operation to register a gauge and set its value:
 
 ```json
-{"name":"metric_name","set":33,"labels":{"label1":"value1"}}
+{"name":"metric_name","action":"set","value":33,"labels":{"label1":"value1"}}
 ```
 
-Labels are not required, but Shell-operator adds a `hook` label.
+Labels are not required, but Shell-operator adds a `hook` label with a path to a hook script relative to hooks directory.
 
-Several metrics can be expored at once. For example, this script will create 2 metrics:
+Several metrics can be exported at once. For example, this script will create 2 metrics:
 
 ```
-echo '{"name":"hook_metric_count","add":1,"labels":{"label1":"value1"}}' >> $METRICS_PATH
-echo '{"name":"hook_metrics_items","add":1,"labels":{"label1":"value1"}}' >> $METRICS_PATH
+echo '{"name":"hook_metric_count","action":"add","value":1,"labels":{"label1":"value1"}}' >> $METRICS_PATH
+echo '{"name":"hook_metrics_items","action":"add","value":1,"labels":{"label1":"value1"}}' >> $METRICS_PATH
 ```
 
 The metric name is used as-is, so several hooks can export same metric name. It is responsibility of hooks‘ developer to maintain consistent label cardinality.
 
-There is no operation to delete metrics and no ttl mechanism yet. If hook set value for metric, this value is exported until the process ends.
+There are fields "add" and "set" that can be used as shortcuts for action and value. This feature may be deprecated in future releases.
+
+```
+{"name":"metric_name","add":1,"labels":{"label1":"value1"}}
+```
+
+Note that there is no mechanism to expire this kind of metrics except the shell-operator restart. It is the default behavior of prometheus-client.
+
+### Grouped metrics
+
+The common cause to expire a metric is a removed object. It means that the object is no longer in the snapshot, and the hook can't identify the metric that should be expired.
 
-To expire metrics, you can use a "group" field. When Shell-operator receives operations with "group", it expires previous metrics with the same group and apply new values. This grouping works across hooks and label values.
+To solve this, use the "group" field in metric operations. When Shell-operator receives operations with the "group" field, it expires previous metrics with the same group and applies new metric values. This grouping works across hooks and label values.
+
+```
+echo '{"group":"group1", "name":"hook_metric_count",  "action":"add", "value":1, "labels":{"label1":"value1"}}' >> $METRICS_PATH
+echo '{"group":"group1", "name":"hook_metrics_items", "action":"add", "value":1, "labels":{"label1":"value1"}}' >> $METRICS_PATH
+```
+
+To expire all metrics in a group, use action "expire":
+
+```
+{"group":"group_name_1", "action":"expire"}
+```
 
-For example:
+### Example
 
 `hook1.sh` returns these metrics:
 
 ```
-echo '{"group":"hook1", "name":"hook_metric","add":1,"labels":{"kind":"pod"}}' >> $METRICS_PATH
-echo '{"group":"hook1", "name":"hook_metric","add":1,"labels":{"kind":"replicaset"}}' >> $METRICS_PATH
-echo '{"group":"hook1", "name":"hook_metric","add":1,"labels":{"kind":"deployment"}}' >> $METRICS_PATH
-echo '{"group":"hook1", "name":"hook1_special_metric","set":12,"labels":{"label1":"value1"}}' >> $METRICS_PATH
-echo '{"group":"hook1", "name":"common_metric","set":300,"labels":{"source":"source3"}}' >> $METRICS_PATH
-echo '{"name":"common_metric","set":100,"labels":{"source":"source1"}}' >> $METRICS_PATH
+echo '{"group":"hook1", "name":"hook_metric", "action":"add", "value":1, "labels":{"kind":"pod"}}' >> $METRICS_PATH
+echo '{"group":"hook1", "name":"hook_metric", "action":"add", "value":1, "labels":{"kind":"replicaset"}}' >> $METRICS_PATH
+echo '{"group":"hook1", "name":"hook_metric", "action":"add", "value":1, "labels":{"kind":"deployment"}}' >> $METRICS_PATH
+echo '{"group":"hook1", "name":"hook1_special_metric", "action":"set", "value":12, "labels":{"label1":"value1"}}' >> $METRICS_PATH
+echo '{"group":"hook1", "name":"common_metric", "action":"set", "value":300, "labels":{"source":"source3"}}' >> $METRICS_PATH
+echo '{"name":"common_metric", "action":"set", "value":100, "labels":{"source":"source1"}}' >> $METRICS_PATH
 ```
 
 `hook2.sh` returns these metrics:
 
 ```
-echo '{"group":"hook2", "name":"hook_metric","add":1,"labels":{"kind":"configmap"}}' >> $METRICS_PATH
-echo '{"group":"hook2", "name":"hook_metric","add":1,"labels":{"kind":"secret"}}' >> $METRICS_PATH
-echo '{"group":"hook2", "name":"hook2_special_metric","set":42}' >> $METRICS_PATH
-echo '{"name":"common_metric","set":200,"labels":{"source":"source2"}}' >> $METRICS_PATH
+echo '{"group":"hook2", "name":"hook_metric","action":"add", "value":1, "labels":{"kind":"configmap"}}' >> $METRICS_PATH
+echo '{"group":"hook2", "name":"hook_metric","action":"add", "value":1, "labels":{"kind":"secret"}}' >> $METRICS_PATH
+echo '{"group":"hook2", "name":"hook2_special_metric", "action":"set", "value":42}' >> $METRICS_PATH
+echo '{"name":"common_metric", "action":"set", "value":200, "labels":{"source":"source2"}}' >> $METRICS_PATH
 ```
 
 Prometheus scrapes these metrics:
@@ -90,28 +111,28 @@ Prometheus scrapes these metrics:
 ```
 # HELP hook_metric hook_metric
 # TYPE hook_metric counter
-hook_metric{hook="hook1.sh", kind="pod"} 1
-hook_metric{hook="hook1.sh", kind="replicaset"} 1
-hook_metric{hook="hook1.sh", kind="deployment"} 1
-hook_metric{hook="hook2.sh", kind="configmap"} 1
-hook_metric{hook="hook2.sh", kind="secret"} 1
-# HELP hook2_special_metric hook2_special_metric
-# TYPE hook1_special_metric gauge
-hook1_special_metric{hook="hook1.sh", label1="value1"} 12
-# HELP hook2_special_metric hook2_special_metric
-# TYPE hook2_special_metric gauge
-hook2_special_metric{hook="hook2.sh"} 42
-# HELP common_metric common_metric
-# TYPE common_metric gauge
-common_metric{hook="hook1.sh", source="source1"} 100
-common_metric{hook="hook1.sh", source="source3"} 300
-common_metric{hook="hook2.sh", source="source2"} 200
+hook_metric{hook="hook1.sh", kind="pod"} 1 -------------------+---------- group:hook1
+hook_metric{hook="hook1.sh", kind="replicaset"} 1 ------------+
+hook_metric{hook="hook1.sh", kind="deployment"} 1 ------------+
+hook_metric{hook="hook2.sh", kind="configmap"} 1  ------------|-------+-- group:hook2
+hook_metric{hook="hook2.sh", kind="secret"} 1 ----------------|-------+
+# HELP hook1_special_metric hook1_special_metric              |       |
+# TYPE hook1_special_metric gauge                             |       |
+hook1_special_metric{hook="hook1.sh", label1="value1"} 12 ----+       |
+# HELP hook2_special_metric hook2_special_metric              |       |
+# TYPE hook2_special_metric gauge                             |       |
+hook2_special_metric{hook="hook2.sh"} 42 ---------------------|-------'
+# HELP common_metric common_metric                            |
+# TYPE common_metric gauge                                    |
+common_metric{hook="hook1.sh", source="source3"} 300 ---------'
+common_metric{hook="hook1.sh", source="source1"} 100 ---------------+---- no group
+common_metric{hook="hook2.sh", source="source2"} 200 ---------------'
 ```
 
 On next execution of `hook1.sh` values for `hook_metric{kind="replicaset"}`, `hook_metric{kind="deployment"}`, `common_metric{source="source3"}` and `hook1_special_metric` are expired and hook returns only one metric:
 
 ```
-echo '{"group":"hook1", "name":"hook_metric","add":1,"labels":{"kind":"pod"}}' >> $METRICS_PATH
+echo '{"group":"hook1", "name":"hook_metric", "action":"add", "value":1, "labels":{"kind":"pod"}}' >> $METRICS_PATH
 ```
 
 Shell-operator expires previous values for group "hook1" and updates value for `hook_metric{hook="hook1.sh", kind="pod"}`. Values for group `hook2` and `common_metric` without group are left intact. Now Prometheus scrapes these metrics:
@@ -119,14 +140,32 @@ Shell-operator expires previous values for group "hook1" and updates value for `
 ```
 # HELP hook_metric hook_metric
 # TYPE hook_metric counter
-hook_metric{hook="hook1.sh", kind="pod"} 2
-hook_metric{hook="hook2.sh", kind="configmap"} 1
-hook_metric{hook="hook2.sh", kind="secret"} 1
-# HELP hook2_special_metric hook2_special_metric
-# TYPE hook2_special_metric gauge
-hook2_special_metric{hook="hook2.sh"} 42
+hook_metric{hook="hook1.sh", kind="pod"} 2 --------------- group:hook1
+hook_metric{hook="hook2.sh", kind="configmap"} 1 ----+---- group:hook2
+hook_metric{hook="hook2.sh", kind="secret"} 1 -------+
+# HELP hook2_special_metric hook2_special_metric     |
+# TYPE hook2_special_metric gauge                    |
+hook2_special_metric{hook="hook2.sh"} 42 ------------'
+# HELP common_metric common_metric
+# TYPE common_metric gauge
+common_metric{hook="hook1.sh", source="source1"} 100 --+-- no group
+common_metric{hook="hook2.sh", source="source2"} 200 --'
+```
+
+Next execution of `hook2.sh` expires all metrics in group 'hook2':
+
+```
+echo '{"group":"hook2", "action":"expire"}' >> $METRICS_PATH
+```
+
+Shell-operator expires previous values for group "hook2" but leaves `common_metrics` for "hook2.sh" as is. Now Prometheus scrapes these metrics:
+
+```
+# HELP hook_metric hook_metric
+# TYPE hook_metric counter
+hook_metric{hook="hook1.sh", kind="pod"} 2 --------------- group:hook1
 # HELP common_metric common_metric
 # TYPE common_metric gauge
-common_metric{hook="hook1.sh", source="source1"} 100
-common_metric{hook="hook2.sh", source="source2"} 200
+common_metric{hook="hook1.sh", source="source1"} 100 --+-- no group
+common_metric{hook="hook2.sh", source="source2"} 200 --'
 ```
diff --git a/pkg/metric_storage/metric_storage.go b/pkg/metric_storage/metric_storage.go
index 0cbf913..1f4eae9 100644
--- a/pkg/metric_storage/metric_storage.go
+++ b/pkg/metric_storage/metric_storage.go
@@ -332,7 +332,12 @@ func (m *MetricStorage) SendBatch(ops []operation.MetricOperation, labels map[st
 		return nil
 	}
 
-	// Group ops by 'Group'.
+	err := operation.ValidateOperations(ops)
+	if err != nil {
+		return err
+	}
+
+	// Group operations by 'Group' value.
 	var groupedOps = make(map[string][]operation.MetricOperation)
 	var nonGroupedOps = make([]operation.MetricOperation, 0)
 
@@ -347,14 +352,13 @@ func (m *MetricStorage) SendBatch(ops []operation.MetricOperation, labels map[st
 		groupedOps[op.Group] = append(groupedOps[op.Group], op)
 	}
 
-	// Apply metric operations for each group
+	// Expire each group and apply new metric operations.
 	for group, ops := range groupedOps {
-		// clean group
 		m.ApplyGroupOperations(group, ops, labels)
 	}
 
-	// backward compatibility — send metrics without cleaning
-	err := m.SendBatchV0(nonGroupedOps, labels)
+	// Send non-grouped metrics.
+	err = m.SendBatchV0(nonGroupedOps, labels)
 	if err != nil {
 		return err
 	}
@@ -375,27 +379,23 @@ func (m *MetricStorage) ApplyOperation(op operation.MetricOperation, commonLabel
 	}
 }
 
+// ApplyGroupOperations set metrics for group to a new state defined by ops.
 func (m *MetricStorage) ApplyGroupOperations(group string, ops []operation.MetricOperation, commonLabels map[string]string) {
-	// Gather labels to clear absent metrics by label values
-	var metricLabels = make(map[string][]map[string]string)
-	for _, op := range ops {
-		if _, ok := metricLabels[op.Name]; !ok {
-			metricLabels[op.Name] = make([]map[string]string, 0)
-		}
-		metricLabels[op.Name] = append(metricLabels[op.Name], MergeLabels(op.Labels, commonLabels))
-	}
-	m.GroupedVault.ClearMissingMetrics(group, metricLabels)
+	// Implicitly expire all metrics for group.
+	m.GroupedVault.ExpireGroupMetrics(group)
 
-	// Apply metric operations
+	// Apply metric operations one-by-one.
 	for _, op := range ops {
-		labels := MergeLabels(op.Labels, commonLabels)
-		if op.Add != nil {
-			m.GroupedVault.CounterAdd(group, op.Name, *op.Add, labels)
+		if op.Action == "expire" {
+			m.GroupedVault.ExpireGroupMetrics(group)
 			continue
 		}
-		if op.Set != nil {
-			m.GroupedVault.GaugeSet(group, op.Name, *op.Set, labels)
-			continue
+		labels := MergeLabels(op.Labels, commonLabels)
+		if op.Action == "add" && op.Value != nil {
+			m.GroupedVault.CounterAdd(group, op.Name, *op.Value, labels)
+		}
+		if op.Action == "set" && op.Value != nil {
+			m.GroupedVault.GaugeSet(group, op.Name, *op.Value, labels)
 		}
 	}
 }
diff --git a/pkg/metric_storage/operation/operation.go b/pkg/metric_storage/operation/operation.go
index ef0e052..a6161ea 100644
--- a/pkg/metric_storage/operation/operation.go
+++ b/pkg/metric_storage/operation/operation.go
@@ -6,14 +6,48 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
 )
 
 type MetricOperation struct {
 	Name   string            `json:"name"`
-	Add    *float64          `json:"add,omitempty"`
-	Set    *float64          `json:"set,omitempty"`
+	Add    *float64          `json:"add,omitempty"` // shortcut for action=add value=num
+	Set    *float64          `json:"set,omitempty"` // shortcut for action=set value=num
+	Value  *float64          `json:"value,omitempty"`
 	Labels map[string]string `json:"labels"`
 	Group  string            `json:"group,omitempty"`
+	Action string            `json:"action,omitempty"`
+}
+
+func (m MetricOperation) String() string {
+	parts := []string{}
+
+	if m.Group != "" {
+		parts = append(parts, "group="+m.Group)
+	}
+	if m.Name != "" {
+		parts = append(parts, "name="+m.Name)
+	}
+	if m.Action != "" {
+		parts = append(parts, "action="+m.Action)
+	}
+
+	if m.Value != nil {
+		parts = append(parts, fmt.Sprintf("value=%f", *m.Value))
+	}
+	if m.Set != nil {
+		parts = append(parts, fmt.Sprintf("set=%f", *m.Set))
+	}
+	if m.Add != nil {
+		parts = append(parts, fmt.Sprintf("add=%f", *m.Add))
+	}
+	if m.Labels != nil {
+		parts = append(parts, fmt.Sprintf("labels=%+v", m.Labels))
+	}
+
+	return "[" + strings.Join(parts, ", ") + "]"
 }
 
 func MetricOperationsFromReader(r io.Reader) ([]MetricOperation, error) {
@@ -28,6 +62,16 @@ func MetricOperationsFromReader(r io.Reader) ([]MetricOperation, error) {
 			return nil, err
 		}
 
+		// shortcut transforms
+		if metricOperation.Set != nil && metricOperation.Add == nil {
+			metricOperation.Action = "set"
+			metricOperation.Value = metricOperation.Set
+		}
+		if metricOperation.Add != nil && metricOperation.Set == nil {
+			metricOperation.Action = "add"
+			metricOperation.Value = metricOperation.Add
+		}
+
 		operations = append(operations, metricOperation)
 	}
 
@@ -49,3 +93,54 @@ func MetricOperationsFromFile(filePath string) ([]MetricOperation, error) {
 	}
 	return MetricOperationsFromBytes(data)
 }
+
+func ValidateOperations(ops []MetricOperation) error {
+	var opsErrs *multierror.Error
+
+	for _, op := range ops {
+		err := ValidateMetricOperation(op)
+		if err != nil {
+			opsErrs = multierror.Append(opsErrs, err)
+		}
+	}
+
+	return opsErrs.ErrorOrNil()
+}
+
+func ValidateMetricOperation(op MetricOperation) error {
+	var opErrs *multierror.Error
+
+	if op.Action == "" {
+		opErrs = multierror.Append(opErrs, fmt.Errorf("one of: 'action', 'set' or 'add' is required: %s", op))
+	}
+
+	if op.Group == "" {
+		if op.Action != "set" && op.Action != "add" {
+			opErrs = multierror.Append(opErrs, fmt.Errorf("unsupported action '%s': %s", op.Action, op))
+		}
+	} else {
+		if op.Action != "expire" && op.Action != "set" && op.Action != "add" {
+			opErrs = multierror.Append(opErrs, fmt.Errorf("unsupported action '%s': %s", op.Action, op))
+		}
+	}
+
+	if op.Name == "" && op.Group == "" {
+		opErrs = multierror.Append(opErrs, fmt.Errorf("'name' is required: %s", op))
+	}
+	if op.Name == "" && op.Group != "" && op.Action != "expire" {
+		opErrs = multierror.Append(opErrs, fmt.Errorf("'name' is required when action is not 'expire': %s", op))
+	}
+
+	if op.Action == "set" && op.Value == nil {
+		opErrs = multierror.Append(opErrs, fmt.Errorf("'value' is required for action 'set': %s", op))
+	}
+	if op.Action == "add" && op.Value == nil {
+		opErrs = multierror.Append(opErrs, fmt.Errorf("'value' is required for action 'add': %s", op))
+	}
+
+	if op.Set != nil && op.Add != nil {
+		opErrs = multierror.Append(opErrs, fmt.Errorf("'set' and 'add' are mutual exclusive: %s", op))
+	}
+
+	return opErrs.ErrorOrNil()
+}
diff --git a/pkg/metric_storage/vault/collector.go b/pkg/metric_storage/vault/collector.go
index 88b2bd5..864d472 100644
--- a/pkg/metric_storage/vault/collector.go
+++ b/pkg/metric_storage/vault/collector.go
@@ -16,7 +16,7 @@ type ConstMetricCollector interface {
 	Type() string
 	LabelNames() []string
 	Name() string
-	ClearMissingMetrics(group string, labelSet []map[string]string)
+	ExpireGroupMetrics(group string)
 }
 
 var (
@@ -103,21 +103,13 @@ func (c *ConstCounterCollector) Name() string {
 	return c.name
 }
 
-// ClearMissingMetrics deletes metrics from collection with unmatched hashes of labelValues.
-func (c *ConstCounterCollector) ClearMissingMetrics(group string, labelSet []map[string]string) {
+// ExpireGroupMetrics deletes all metrics from collection with matched group.
+func (c *ConstCounterCollector) ExpireGroupMetrics(group string) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 
-	hashedOps := make(map[uint64]struct{})
-	for _, labels := range labelSet {
-		hashedOps[HashLabelValues(LabelValues(labels, c.labelNames))] = struct{}{}
-	}
-
 	for hash, m := range c.collection {
-		if m.Group != group {
-			continue
-		}
-		if _, ok := hashedOps[hash]; !ok {
+		if m.Group == group {
 			delete(c.collection, hash)
 		}
 	}
@@ -187,27 +179,13 @@ func (c *ConstGaugeCollector) Name() string {
 	return c.name
 }
 
-// ClearGroup removes all metrics for a group
-// TODO ClearGroup
-
-// Clear removes all metrics from this collector
-//TODO
-
-// ClearMissingMetrics deletes metrics from collection with unmatched hashes of labelValues.
-func (c *ConstGaugeCollector) ClearMissingMetrics(group string, labelSet []map[string]string) {
+// ExpireGroupMetrics deletes all metrics from collection with matched group.
+func (c *ConstGaugeCollector) ExpireGroupMetrics(group string) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 
-	hashedOps := make(map[uint64]struct{})
-	for _, labels := range labelSet {
-		hashedOps[HashLabelValues(LabelValues(labels, c.labelNames))] = struct{}{}
-	}
-
 	for hash, m := range c.collection {
-		if m.Group != group {
-			continue
-		}
-		if _, ok := hashedOps[hash]; !ok {
+		if m.Group == group {
 			delete(c.collection, hash)
 		}
 	}
diff --git a/pkg/metric_storage/vault/vault.go b/pkg/metric_storage/vault/vault.go
index 75f0d6f..50a7381 100644
--- a/pkg/metric_storage/vault/vault.go
+++ b/pkg/metric_storage/vault/vault.go
@@ -22,14 +22,12 @@ func NewGroupedVault() *GroupedVault {
 	}
 }
 
-// ClearMissingMetrics takes each collector in collector by its name and clear missing metrics by group.
-func (v *GroupedVault) ClearMissingMetrics(group string, metricLabels map[string][]map[string]string) {
+// ClearAllMetrics takes each collector in collectors and clear all metrics by group.
+func (v *GroupedVault) ExpireGroupMetrics(group string) {
 	v.mtx.Lock()
 	defer v.mtx.Unlock()
-	for name, collector := range v.collectors {
-		if labelSet, ok := metricLabels[name]; ok {
-			collector.ClearMissingMetrics(group, labelSet)
-		}
+	for _, collector := range v.collectors {
+		collector.ExpireGroupMetrics(group)
 	}
 }
 


Test patch:
diff --git a/pkg/metric_storage/operation/operation_test.go b/pkg/metric_storage/operation/operation_test.go
new file mode 100644
index 0000000..88685c8
--- /dev/null
+++ b/pkg/metric_storage/operation/operation_test.go
@@ -0,0 +1,54 @@
+package operation
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+)
+
+func Test_ValidateMetricOperations(t *testing.T) {
+	//g := NewWithT(t)
+	tests := []struct {
+		name     string
+		op       string
+		expected bool
+	}{
+		{
+			"simple",
+			`{"group":"azaza", "action":"expire"}`,
+			true,
+		},
+		{
+			"action set",
+			`{"name":"metric_1", "action":"set", "value":1}`,
+			true,
+		},
+		{
+			"set shortcut",
+			`{"name":"metric_1", "set":1}`,
+			true,
+		},
+		{
+			"invalid",
+			`{"group":"azaza", "action":"expired"}`,
+			false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			ops, err := MetricOperationsFromBytes([]byte(tt.op))
+			g.Expect(err).ShouldNot(HaveOccurred())
+
+			err = ValidateOperations(ops)
+			if tt.expected {
+				g.Expect(err).ShouldNot(HaveOccurred())
+			} else {
+				//t.Logf("expected error: %v", err)
+				g.Expect(err).Should(HaveOccurred())
+			}
+		})
+	}
+}
diff --git a/pkg/metric_storage/vault/vault_test.go b/pkg/metric_storage/vault/vault_test.go
index 8e06f12..8779550 100644
--- a/pkg/metric_storage/vault/vault_test.go
+++ b/pkg/metric_storage/vault/vault_test.go
@@ -33,11 +33,7 @@ metric_total{lbl="val"} 1
 	err := promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expect), "metric_total")
 	g.Expect(err).ShouldNot(HaveOccurred())
 
-	v.ClearMissingMetrics("group1", map[string][]map[string]string{
-		"metric_total": {
-			{"lbl": "qwe"},
-		},
-	})
+	v.ExpireGroupMetrics("group1")
 
 	g.Expect(buf.String()).ShouldNot(ContainSubstring("error"), "error occurred in log: %s", buf.String())
 
@@ -74,14 +70,7 @@ metric_total{lbl="val222"} 1
 	err = promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expect), "metric_total")
 	g.Expect(err).ShouldNot(HaveOccurred())
 
-	v.ClearMissingMetrics("group1", map[string][]map[string]string{
-		"metric_total": {
-			{"lbl": "val"},
-		},
-		"metric2_total": {
-			{"lbl222": "val222"},
-		},
-	})
+	v.ExpireGroupMetrics("group1")
 	v.CounterAdd("group1", "metric_total", 1.0, map[string]string{"lbl": "val"})
 	v.CounterAdd("group2", "metric2_total", 1.0, map[string]string{"lbl": "val222"})
 
@@ -98,18 +87,11 @@ metric2_total{lbl="val222"} 1
 	err = promtest.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expect), "metric_total", "metric2_total")
 	g.Expect(err).ShouldNot(HaveOccurred())
 
-	v.ClearMissingMetrics("group1", map[string][]map[string]string{
-		"metric2_total": {
-			{"lbl222": "val222"},
-		},
-	})
+	v.ExpireGroupMetrics("group1")
 	v.CounterAdd("group2", "metric2_total", 1.0, map[string]string{"lbl": "val222"})
 	g.Expect(buf.String()).ShouldNot(ContainSubstring("error"), "error occurred in log: %s", buf.String())
 	// Expect metric_total is updated and metric2_total is updated and metric_total left as is
 	expect = `
-# HELP metric_total metric_total
-# TYPE metric_total counter
-metric_total{lbl="val"} 1
 # HELP metric2_total metric2_total
 # TYPE metric2_total counter
 metric2_total{lbl="val222"} 2