Repository: VictoriaMetrics/VictoriaMetrics
Pull request #4063 (2023-04-04 09:52:18)

app/vmctl: add multiple filters defined in `--vm-native-filter-match` flag to discovered metric names
This PR fixes the issue when the user defines the `--vm-native-filter-match`, vmctl tries to export data only with metric names but ignores user-defined filters. It happens because vmctl discovers metric names via /api/v1/series with a user-defined filter, but after vmctl uses metric names for build requests to the export API  `match[]` param only include metric name without user-defined filters.

Related issue:  https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4062

Signed-off-by: dmitryk-dk [d.kozlov@victoriametrics.com](mailto:d.kozlov@victoriametrics.com)

codecov[bot]: ## [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4063?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
Patch coverage: **`88.46`**% and project coverage change: **`-0.04`** :warning:
> Comparison is base [(`19b189e`)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/19b189e9b7238f1093b03535575229908d385abf?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 61.14% compared to head [(`c213942`)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4063?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 61.10%.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #4063      +/-   ##
==========================================
- Coverage   61.14%   61.10%   -0.04%     
==========================================
  Files         323      323              
  Lines       58637    58663      +26     
==========================================
- Hits        35852    35848       -4     
- Misses      20705    20728      +23     
- Partials     2080     2087       +7     
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4063?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Coverage Δ | |
|---|---|---|
| [app/vmctl/vm\_native.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4063?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtY3RsL3ZtX25hdGl2ZS5nbw==) | `56.77% <88.46%> (+3.92%)` | :arrow_up: |

... and [6 files with indirect coverage changes](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4063/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)

Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics). Have a feature suggestion? [Share it here.](https://app.codecov.io/gh/feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics)

</details>

[:umbrella: View full report in Codecov by Sentry](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/4063?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).   
:loudspeaker: Do you have feedback about the report comment? [Let us know in this issue](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).


Patch:
diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go
index 999fd3b63..da76d4a05 100644
--- a/app/vmctl/vm_native.go
+++ b/app/vmctl/vm_native.go
@@ -14,6 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/cheggaaa/pb/v3"
 )
 
@@ -232,6 +233,13 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	// any error breaks the import
 	for s := range metrics {
+
+		match, err := buildMatchWithFilter(p.filter.Match, s)
+		if err != nil {
+			logger.Errorf("failed to build export filters: %s", err)
+			continue
+		}
+
 		for _, times := range ranges {
 			select {
 			case <-ctx.Done():
@@ -239,7 +247,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			case infErr := <-errCh:
 				return fmt.Errorf("native error: %s", infErr)
 			case filterCh <- native.Filter{
-				Match:     fmt.Sprintf("{%s=%q}", nameLabel, s),
+				Match:     match,
 				TimeStart: times[0].Format(time.RFC3339),
 				TimeEnd:   times[1].Format(time.RFC3339),
 			}:
@@ -303,3 +311,13 @@ func byteCountSI(b int64) string {
 	return fmt.Sprintf("%.1f %cB",
 		float64(b)/float64(div), "kMGTPE"[exp])
 }
+
+func buildMatchWithFilter(filter string, metricName string) (string, error) {
+	labels, err := promutils.NewLabelsFromString(filter)
+	if err != nil {
+		return "", err
+	}
+	labels.Set("__name__", metricName)
+
+	return labels.String(), nil
+}
diff --git a/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts b/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts
index 9fdb97a55..5a1981fb3 100644
--- a/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts
+++ b/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts
@@ -89,6 +89,7 @@ export const useFetchQuery = ({
 
         const isHideQuery = hideQuery?.includes(counter - 1);
         if (isHideQuery) {
+          setQueryErrors(prev => [...prev, ""]);
           counter++;
           continue;
         }
@@ -105,6 +106,7 @@ export const useFetchQuery = ({
           }
 
           const isHistogramResult = isDisplayChart && isHistogramData(resp.data.result);
+          if (resp.data.result.length) setIsHistogram(isHistogramResult);
           if (isHistogramResult) seriesLimit = Infinity;
           const freeTempSize = seriesLimit - tempData.length;
           resp.data.result.slice(0, freeTempSize).forEach((d: MetricBase) => {
@@ -122,8 +124,6 @@ export const useFetchQuery = ({
 
       const limitText = `Showing ${seriesLimit} series out of ${totalLength} series due to performance reasons. Please narrow down the query, so it returns less series`;
       setWarning(totalLength > seriesLimit ? limitText : "");
-
-      setIsHistogram(isDisplayChart && isHistogramData(tempData));
       isDisplayChart ? setGraphData(tempData as MetricResult[]) : setLiveData(tempData as InstantMetricResult[]);
       setTraces(tempTraces);
     } catch (e) {
diff --git a/app/vmui/packages/vmui/src/utils/uplot/heatmap.ts b/app/vmui/packages/vmui/src/utils/uplot/heatmap.ts
index fbe8f1e77..0f03a11c5 100644
--- a/app/vmui/packages/vmui/src/utils/uplot/heatmap.ts
+++ b/app/vmui/packages/vmui/src/utils/uplot/heatmap.ts
@@ -149,7 +149,7 @@ export const normalizeData = (buckets: MetricResult[], isHistogram?: boolean): M
   const vmBuckets = convertPrometheusToVictoriaMetrics(sortedBuckets);
   const allValues = vmBuckets.map(b => b.values).flat();
 
-  return vmBuckets.map(bucket => {
+  const result = vmBuckets.map(bucket => {
     const values = bucket.values.map((v) => {
       const totalHits = allValues
         .filter(av => av[0] === v[0])
@@ -160,4 +160,6 @@ export const normalizeData = (buckets: MetricResult[], isHistogram?: boolean): M
 
     return { ...bucket, values };
   }) as MetricResult[];
+
+  return result.filter(r => !r.values.every(v => v[1] === "0"));
 };
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ab015c629..0ccfaa4e6 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -46,6 +46,7 @@ created by v1.90.0 or newer versions. The solution is to upgrade to v1.90.0 or n
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth.html):  suppress [proxy protocol](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt) parsing errors in case of `EOF`. Usually, the error is caused by health checks and is not a sign of an actual error.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix displaying errors for each query. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3987).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): fix snapshot not being deleted in case of error during backup. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2055).
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmgent.html) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).
 * BUGFIX: allow using dashes and dots in environment variables names referred in config files via `%{ENV-VAR.SYNTAX}`. See [these docs](https://docs.victoriametrics.com/#environment-variables) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3999).
 * BUGFIX: return back query performance scalability on hosts with big number of CPU cores. The scalability has been reduced in [v1.86.0](https://docs.victoriametrics.com/CHANGELOG.html#v1860). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly convert [VictoriaMetrics historgram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to Prometheus histogram buckets when VictoriaMetrics histogram contain zero buckets. Previously these buckets were ignored, and this could lead to missing Prometheus histogram buckets after the conversion. Thanks to @zklapow for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4021).
@@ -54,6 +55,7 @@ created by v1.90.0 or newer versions. The solution is to upgrade to v1.90.0 or n
 * BUGFIX: properly support comma-separated filters inside [retention filters](https://docs.victoriametrics.com/#retention-filters). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3915).
 * BUGFIX: verify response code when fetching configuration files via HTTP. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4034).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): replace empty labels with "" instead of "<no value>" during templating, as Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4012).
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): properly pass multiple filters from `--vm-native-filter-match` command-line flag to the data source. Previously filters from `--vm-native-filter-match` were only used to discover the metric names, and the metric names like `__name__="metric_name"` has been taken into account, while the remaining filters were ignored. For example `--vm-native-src-addr={foo="bar",baz="abc"}` may found `metric_name{foo="bar",baz="abc"}` and filter was treated as `--vm-native-src-addr={__name__="metrics_name"}`, e.g. `foo="bar",baz="abc"` filter was ignored. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4062). 
 
 
 ## [v1.89.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.89.1)
diff --git a/lib/promutils/labels.go b/lib/promutils/labels.go
index f22c1552f..b4f9216eb 100644
--- a/lib/promutils/labels.go
+++ b/lib/promutils/labels.go
@@ -188,6 +188,22 @@ func (x *Labels) Get(name string) string {
 	return ""
 }
 
+// Set label value for label with given name
+// If the label with the given name doesn't exist, it adds as the new label
+func (x *Labels) Set(name, value string) {
+	if name == "" || value == "" {
+		return
+	}
+	labels := x.GetLabels()
+	for i, label := range labels {
+		if label.Name == name {
+			labels[i].Value = value
+			return
+		}
+	}
+	x.Add(name, value)
+}
+
 // InternStrings interns all the strings used in x labels.
 func (x *Labels) InternStrings() {
 	labels := x.GetLabels()
@@ -306,7 +322,7 @@ func MustNewLabelsFromString(metricWithLabels string) *Labels {
 
 // NewLabelsFromString creates labels from s, which can have the form `metric{labels}`.
 //
-// This function must be used only in tests
+// This function must be used only in non performance-critical code, since it allocates too much
 func NewLabelsFromString(metricWithLabels string) (*Labels, error) {
 	stripDummyMetric := false
 	if strings.HasPrefix(metricWithLabels, "{") {


Test patch:
diff --git a/app/vmctl/vm_native_test.go b/app/vmctl/vm_native_test.go
index 82038e352..173972c0f 100644
--- a/app/vmctl/vm_native_test.go
+++ b/app/vmctl/vm_native_test.go
@@ -294,3 +294,68 @@ func deleteSeries(name, value string) (int, error) {
 	}
 	return vmstorage.DeleteSeries(nil, []*storage.TagFilters{tfs})
 }
+
+func Test_buildMatchWithFilter(t *testing.T) {
+	tests := []struct {
+		name       string
+		filter     string
+		metricName string
+		want       string
+		wantErr    bool
+	}{
+		{
+			name:       "parsed metric with label",
+			filter:     `{__name__="http_request_count_total",cluster="kube1"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster="kube1"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "metric name with label",
+			filter:     `http_request_count_total{cluster="kube1"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster="kube1"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "parsed metric with regexp value",
+			filter:     `{__name__="http_request_count_total",cluster~="kube.*"}`,
+			metricName: "http_request_count_total",
+			want:       `{__name__="http_request_count_total",cluster~="kube.*"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "only label with regexp",
+			filter:     `{cluster~=".*"}`,
+			metricName: "http_request_count_total",
+			want:       `{cluster~=".*",__name__="http_request_count_total"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "many labels in filter with regexp",
+			filter:     `{cluster~=".*",job!=""}`,
+			metricName: "http_request_count_total",
+			want:       `{cluster~=".*",job!="",__name__="http_request_count_total"}`,
+			wantErr:    false,
+		},
+		{
+			name:       "match with error",
+			filter:     `{cluster=~".*"}`,
+			metricName: "http_request_count_total",
+			want:       ``,
+			wantErr:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := buildMatchWithFilter(tt.filter, tt.metricName)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("buildMatchWithFilter() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("buildMatchWithFilter() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/lib/promutils/labels_test.go b/lib/promutils/labels_test.go
index 83e79040c..18bdfee4f 100644
--- a/lib/promutils/labels_test.go
+++ b/lib/promutils/labels_test.go
@@ -175,3 +175,22 @@ func TestLabelsRemoveLabelsWithDoubleUnderscorePrefix(t *testing.T) {
 	f(`{__meta_foo="bar",a="b",__name__="foo",__vm_filepath="aa"}`, `{a="b"}`)
 	f(`{__meta_foo="bdffr",foo="bar",__meta_xxx="basd"}`, `{foo="bar"}`)
 }
+
+func TestLabels_Set(t *testing.T) {
+	f := func(metric, name, value, resultExpected string) {
+		t.Helper()
+		labels := MustNewLabelsFromString(metric)
+		labels.Set(name, value)
+		result := labels.String()
+		if result != resultExpected {
+			t.Fatalf("unexpected result of RemoveLabelsWithDoubleUnderscorePrefix;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+	f(`{}`, ``, ``, `{}`)
+	f(`{foo="bar"}`, `bar`, `baz`, `{foo="bar",bar="baz"}`)
+	f(`{__meta_foo="bar",a="b",__name__="foo",__vm_filepath="aa"}`, `__name__`, `bar`, `{__meta_foo="bar",a="b",__name__="bar",__vm_filepath="aa"}`)
+	f(`{__meta_foo="bdffr",foo="bar",__meta_xxx="basd"}`, `__name__`, `baz`, `{__meta_foo="bdffr",foo="bar",__meta_xxx="basd",__name__="baz"}`)
+	f(`http_request_total{a="b"}`, `__name__`, `metric`, `{__name__="metric",a="b"}`)
+	f(`http_request_total{a="b"}`, `a`, `c`, `{__name__="http_request_total",a="c"}`)
+	f(`http_request_total{a="b"}`, `ip`, `127.0.0.1`, `{__name__="http_request_total",a="b",ip="127.0.0.1"}`)
+}