Repository: VictoriaMetrics/VictoriaMetrics
Pull request #5406 (2023-11-28 12:04:22)

lib/streamaggr: properly reference slice with labels
by limiting slice capacity. It must fix issues with slice modification, in case of append new slice will be allocated, instead of modifying refrenced slice https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5402


Patch:
diff --git a/README.md b/README.md
index fbb4a05d7..8ac1bfec8 100644
--- a/README.md
+++ b/README.md
@@ -507,10 +507,15 @@ See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be
 
 ## How to send data from DataDog agent
 
-VictoriaMetrics accepts data from [DataDog agent](https://docs.datadoghq.com/agent/) 
-or [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/) 
-via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) 
-at `/datadog/api/v1/series` path.
+VictoriaMetrics accepts data in the following protocols:
+* [DataDog agent](https://docs.datadoghq.com/agent/)
+* [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/)
+* [DataDog Lambda Extension](https://docs.datadoghq.com/serverless/libraries_integrations/extension/)
+
+Via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) at the following path:
+* `/datadog/api/v1/series`
+* `/datadog/api/v2/series`
+* `/datadog/api/beta/sketches`
 
 ### Sending metrics to VictoriaMetrics
 
@@ -582,6 +587,19 @@ additional_endpoints:
 
 </div>
 
+### Send via Serverless DataDog plugin
+
+Disable logs (logs ingestion is not supported by Victoria Metrics) and set a custom endpoint in serverless.yaml
+```
+custom:
+  datadog:
+    enableDDLogs: false             # Disabled not supported DD logs
+    apiKey: fakekey                 # Set any key, otherwise plugin fails
+provider:
+  environment:
+    DD_DD_URL: <<vm-url>>/datadog   # Victoria Metrics endpoint for DataDog
+```
+
 ### Send via cURL
 
 See how to send data to VictoriaMetrics via 
@@ -2564,7 +2582,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -csvTrimTimestamp duration
      Trim timestamps when importing csv data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -datadog.maxInsertRequestSize size
-     The maximum size in bytes of a single DataDog POST request to /api/v1/series
+     The maximum size in bytes of a single DataDog POST request to /api/v1/series, /api/v2/series, /api/beta/sketches
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
   -datadog.sanitizeMetricName
      Sanitize metric names for the ingested DataDog data to comply with DataDog behaviour described at https://docs.datadoghq.com/metrics/custom_metrics/#naming-custom-metrics (default true)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b32eabd4d..e3969e1fe 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -37,6 +37,7 @@ The sandbox cluster installation is running under the constant load generated by
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): prevent from `FATAL: cannot flush metainfo` panic when [`-remoteWrite.multitenantURL`](https://docs.victoriametrics.com/vmagent.html#multitenancy) command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5357).
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly decode zstd-encoded data blocks received via [VictoriaMetrics remote_write protocol](https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol). See [this issue comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5301#issuecomment-1815871992).
+* BUGFIX:  [vmagent](https://docs.victoriametrics.com/vmagent.html): properly add new labels at `output_relabel_configs` during [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html). Previously this could lead to corrupted labels in output samples. Thanks to @ChengChung for providing [detailed report for this bug](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5402).
 
 
 ## [v1.95.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.95.1)
diff --git a/docs/README.md b/docs/README.md
index 3338cbb42..16a884c09 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -510,10 +510,15 @@ See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be
 
 ## How to send data from DataDog agent
 
-VictoriaMetrics accepts data from [DataDog agent](https://docs.datadoghq.com/agent/) 
-or [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/) 
-via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) 
-at `/datadog/api/v1/series` path.
+VictoriaMetrics accepts data in the following protocols:
+* [DataDog agent](https://docs.datadoghq.com/agent/)
+* [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/)
+* [DataDog Lambda Extension](https://docs.datadoghq.com/serverless/libraries_integrations/extension/)
+
+Via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) at the following path:
+* `/datadog/api/v1/series`
+* `/datadog/api/v2/series`
+* `/datadog/api/beta/sketches`
 
 ### Sending metrics to VictoriaMetrics
 
@@ -585,6 +590,19 @@ additional_endpoints:
 
 </div>
 
+### Send via Serverless DataDog plugin
+
+Disable logs (logs ingestion is not supported by Victoria Metrics) and set a custom endpoint in serverless.yaml
+```
+custom:
+  datadog:
+    enableDDLogs: false             # Disabled not supported DD logs
+    apiKey: fakekey                 # Set any key, otherwise plugin fails
+provider:
+  environment:
+    DD_DD_URL: <<vm-url>>/datadog   # Victoria Metrics endpoint for DataDog
+```
+
 ### Send via cURL
 
 See how to send data to VictoriaMetrics via 
@@ -2045,6 +2063,7 @@ These metrics can be scraped via [vmagent](https://docs.victoriametrics.com/vmag
 Alternatively, single-node VictoriaMetrics can self-scrape the metrics when `-selfScrapeInterval` command-line flag is 
 set to duration greater than 0. For example, `-selfScrapeInterval=10s` would enable self-scraping of `/metrics` page 
 with 10 seconds interval.
+
 _Please note, never use loadbalancer address for scraping metrics. All monitored components should be scraped directly by their address._
 
 Official Grafana dashboards available for [single-node](https://grafana.com/grafana/dashboards/10229-victoriametrics/) 
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index a89841f33..d11d9f385 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -518,13 +518,12 @@ See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be
 
 ## How to send data from DataDog agent
 
-VictoriaMetrics accepts data from:
+VictoriaMetrics accepts data in the following protocols:
 * [DataDog agent](https://docs.datadoghq.com/agent/)
 * [DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/)
 * [DataDog Lambda Extension](https://docs.datadoghq.com/serverless/libraries_integrations/extension/)
 
-via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) 
-at:
+Via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) at the following path:
 * `/datadog/api/v1/series`
 * `/datadog/api/v2/series`
 * `/datadog/api/beta/sketches`
@@ -560,10 +559,10 @@ dd_url: http://victoriametrics:8428/datadog
 
 </div>
 
-vmagent also can accept DataDog metrics format. Depending on where vmagent will forward data,
+vmagent also can accept Datadog metrics format. Depending on where vmagent will forward data, 
 pick [single-node or cluster URL](https://docs.victoriametrics.com/url-examples.html#datadog) formats.
 
-### Sending metrics to DataDog and VictoriaMetrics
+### Sending metrics to Datadog and VictoriaMetrics
  
 DataDog allows configuring [Dual Shipping](https://docs.datadoghq.com/agent/guide/dual-shipping/) for metrics 
 sending via ENV variable `DD_ADDITIONAL_ENDPOINTS` or via configuration file `additional_endpoints`.
@@ -2072,6 +2071,7 @@ These metrics can be scraped via [vmagent](https://docs.victoriametrics.com/vmag
 Alternatively, single-node VictoriaMetrics can self-scrape the metrics when `-selfScrapeInterval` command-line flag is 
 set to duration greater than 0. For example, `-selfScrapeInterval=10s` would enable self-scraping of `/metrics` page 
 with 10 seconds interval.
+
 _Please note, never use loadbalancer address for scraping metrics. All monitored components should be scraped directly by their address._
 
 Official Grafana dashboards available for [single-node](https://grafana.com/grafana/dashboards/10229-victoriametrics/) 
diff --git a/docs/operator/resources/vmrule.md b/docs/operator/resources/vmrule.md
index 1dd121bb1..20bfc4ec7 100644
--- a/docs/operator/resources/vmrule.md
+++ b/docs/operator/resources/vmrule.md
@@ -104,10 +104,10 @@ metadata:
   name: vmrule-recording-example
 spec:
   groups:
-    - name: vmalert
+    - name: vmrule_recording_groupname
       interval: 1m
       rules:
-        - alert: vmalert config reload error
+        - record: vm_http_request_errors_total:sum_by_cluster_namespace_job:rate:5m
           expr: |-
             sum by (cluster, namespace, job) (
               rate(vm_http_request_errors_total[5m])
diff --git a/docs/operator/vars.md b/docs/operator/vars.md
index e789effc8..f837664e9 100644
--- a/docs/operator/vars.md
+++ b/docs/operator/vars.md
@@ -10,7 +10,7 @@ menu:
 
 <!-- this doc autogenerated - don't edit it manually -->
 # Auto Generated vars for package config 
- updated at Mon Nov 27 08:54:12 UTC 2023 
+ updated at Tue Nov 28 15:49:38 UTC 2023 
 
 
 | varible name | variable default value | variable required | variable description |
diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index e2d5d68da..f1e0a6ce7 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -70,11 +70,15 @@ func (prc *parsedRelabelConfig) String() string {
 //
 // It returns DebugStep list - one entry per each applied relabeling step.
 func (pcs *ParsedConfigs) ApplyDebug(labels []prompbmarshal.Label) ([]prompbmarshal.Label, []DebugStep) {
-	labels, dss := pcs.applyInternal(labels, 0, true)
+	// Protect from overwriting labels between len(labels) and cap(labels) by limiting labels capacity to its length.
+	labels, dss := pcs.applyInternal(labels[:len(labels):len(labels)], 0, true)
 	return labels, dss
 }
 
 // Apply applies pcs to labels starting from the labelsOffset.
+//
+// Apply() may add additional labels after the len(labels), so make sure it doesn't corrupt in-use labels
+// stored between len(labels) and cap(labels).
 func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label {
 	labels, _ = pcs.applyInternal(labels, labelsOffset, false)
 	return labels
diff --git a/lib/streamaggr/streamaggr.go b/lib/streamaggr/streamaggr.go
index 705709dc7..c3d7bacdd 100644
--- a/lib/streamaggr/streamaggr.go
+++ b/lib/streamaggr/streamaggr.go
@@ -471,22 +471,30 @@ func (a *aggregator) flush() {
 
 		tss := ctx.tss
 
-		// Apply output relabeling
-		if a.outputRelabeling != nil {
-			dst := tss[:0]
-			for _, ts := range tss {
-				ts.Labels = a.outputRelabeling.Apply(ts.Labels, 0)
-				if len(ts.Labels) == 0 {
-					// The metric has been deleted by the relabeling
-					continue
-				}
-				dst = append(dst, ts)
-			}
-			tss = dst
+		if a.outputRelabeling == nil {
+			// Fast path - push the output metrics.
+			a.pushFunc(tss)
+			continue
 		}
 
-		// Push the output metrics.
-		a.pushFunc(tss)
+		// Slower path - apply output relabeling and then push the output metrics.
+		auxLabels := promutils.GetLabels()
+		dstLabels := auxLabels.Labels[:0]
+		dst := tss[:0]
+		for _, ts := range tss {
+			dstLabelsLen := len(dstLabels)
+			dstLabels = append(dstLabels, ts.Labels...)
+			dstLabels = a.outputRelabeling.Apply(dstLabels, dstLabelsLen)
+			if len(dstLabels) == dstLabelsLen {
+				// The metric has been deleted by the relabeling
+				continue
+			}
+			ts.Labels = dstLabels[dstLabelsLen:]
+			dst = append(dst, ts)
+		}
+		a.pushFunc(dst)
+		auxLabels.Labels = dstLabels
+		promutils.PutLabels(auxLabels)
 	}
 }
 


Test patch:
diff --git a/lib/streamaggr/streamaggr_test.go b/lib/streamaggr/streamaggr_test.go
index 91ced47e1..33924c8e2 100644
--- a/lib/streamaggr/streamaggr_test.go
+++ b/lib/streamaggr/streamaggr_test.go
@@ -686,6 +686,35 @@ cpu_usage{cpu="2"} 90
 cpu_usage:1m_without_cpu_quantiles{quantile="0.5"} 13.3
 cpu_usage:1m_without_cpu_quantiles{quantile="1"} 90
 `, "1111111")
+
+	// append additional label
+	f(`
+- interval: 1m
+  without: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+  output_relabel_configs:
+  - action: replace_all
+    source_labels: [__name__]
+    regex: ":|_"
+    replacement: "-"
+    target_label: __name__
+  - action: drop
+    source_labels: [de]
+    regex: fg
+  - target_label: new_label
+    replacement: must_keep_metric_name
+`, `
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`, `bar-1m-without-abc-count-samples{new_label="must_keep_metric_name"} 1
+bar-1m-without-abc-count-series{new_label="must_keep_metric_name"} 1
+bar-1m-without-abc-sum-samples{new_label="must_keep_metric_name"} 5
+foo-1m-without-abc-count-samples{new_label="must_keep_metric_name"} 2
+foo-1m-without-abc-count-series{new_label="must_keep_metric_name"} 1
+foo-1m-without-abc-sum-samples{new_label="must_keep_metric_name"} 12.5
+`, "1111")
 }
 
 func TestAggregatorsWithDedupInterval(t *testing.T) {