Repository: VictoriaMetrics/VictoriaMetrics
Pull request #5398 (2023-11-27 12:26:16)

vmalert: automatically add `exported_` prefix for original evaluation…
…result labels if they are conflicted with external or reserved labels, previously they were overridden

address https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5161



codecov[bot]: ## [Codecov](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5398?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) Report
Attention: `1 lines` in your changes are missing coverage. Please review.
> Comparison is base [(`41f7940`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/41f7940f973404a7b93f446f2f9055842d72c103?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 60.09% compared to head [(`72ff68a`)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5398?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) 60.14%.
> Report is 4 commits behind head on master.

| [Files](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5398?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) | Patch % | Lines |
|---|---|---|
| [app/vmalert/rule/alerting.go](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5398?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics#diff-YXBwL3ZtYWxlcnQvcnVsZS9hbGVydGluZy5nbw==) | 91.66% | [1 Missing :warning: ](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5398?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #5398      +/-   ##
==========================================
+ Coverage   60.09%   60.14%   +0.05%     
==========================================
  Files         401      401              
  Lines       75241    75302      +61     
==========================================
+ Hits        45215    45292      +77     
+ Misses      27509    27494      -15     
+ Partials     2517     2516       -1     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/5398?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=VictoriaMetrics).


Patch:
diff --git a/app/vmalert/rule/alerting.go b/app/vmalert/rule/alerting.go
index e10405250..7ca4427eb 100644
--- a/app/vmalert/rule/alerting.go
+++ b/app/vmalert/rule/alerting.go
@@ -237,11 +237,30 @@ type labelSet struct {
 	origin map[string]string
 	// processed labels includes origin labels
 	// plus extra labels (group labels, service labels like alertNameLabel).
-	// in case of conflicts, extra labels are preferred.
+	// in case of key conflicts, origin labels are renamed with prefix `exported_` and extra labels are preferred.
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5161
 	// used as labels attached to notifier.Alert and ALERTS series written to remote storage.
 	processed map[string]string
 }
 
+// add adds a value v with key k to origin and processed label sets.
+// On k conflicts in processed set, the passed v is preferred.
+// On k conflicts in origin set, the original value is preferred and copied
+// to processed with `exported_%k` key. The copy happens only if passed v isn't equal to origin[k] value.
+func (ls *labelSet) add(k, v string) {
+	ls.processed[k] = v
+	ov, ok := ls.origin[k]
+	if !ok {
+		ls.origin[k] = v
+		return
+	}
+	if ov != v {
+		// copy value only if v and ov are different
+		key := fmt.Sprintf("exported_%s", k)
+		ls.processed[key] = ov
+	}
+}
+
 // toLabels converts labels from given Metric
 // to labelSet which contains original and processed labels.
 func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
@@ -267,24 +286,14 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		return nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
 	for k, v := range extraLabels {
-		ls.processed[k] = v
-		if _, ok := ls.origin[k]; !ok {
-			ls.origin[k] = v
-		}
+		ls.add(k, v)
 	}
-
 	// set additional labels to identify group and rule name
 	if ar.Name != "" {
-		ls.processed[alertNameLabel] = ar.Name
-		if _, ok := ls.origin[alertNameLabel]; !ok {
-			ls.origin[alertNameLabel] = ar.Name
-		}
+		ls.add(alertNameLabel, ar.Name)
 	}
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
-		ls.processed[alertGroupNameLabel] = ar.GroupName
-		if _, ok := ls.origin[alertGroupNameLabel]; !ok {
-			ls.origin[alertGroupNameLabel] = ar.GroupName
-		}
+		ls.add(alertGroupNameLabel, ar.GroupName)
 	}
 	return ls, nil
 }
@@ -414,8 +423,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		}
 		h := hash(ls.processed)
 		if _, ok := updated[h]; ok {
-			// duplicate may be caused by extra labels
-			// conflicting with the metric labels
+			// duplicate may be caused the removal of `__name__` label
 			curState.Err = fmt.Errorf("labels %v: %w", ls.processed, errDuplicate)
 			return nil, curState.Err
 		}
diff --git a/app/vmalert/rule/recording.go b/app/vmalert/rule/recording.go
index 08a69a8fe..c015dfe06 100644
--- a/app/vmalert/rule/recording.go
+++ b/app/vmalert/rule/recording.go
@@ -194,6 +194,9 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSer
 	labels["__name__"] = rr.Name
 	// override existing labels with configured ones
 	for k, v := range rr.Labels {
+		if _, ok := labels[k]; ok && labels[k] != v {
+			labels[fmt.Sprintf("exported_%s", k)] = labels[k]
+		}
 		labels[k] = v
 	}
 	return newTimeSeries(m.Values, m.Timestamps, labels)
@@ -203,7 +206,7 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSer
 func (rr *RecordingRule) updateWith(r Rule) error {
 	nr, ok := r.(*RecordingRule)
 	if !ok {
-		return fmt.Errorf("BUG: attempt to update recroding rule with wrong type %#v", r)
+		return fmt.Errorf("BUG: attempt to update recording rule with wrong type %#v", r)
 	}
 	rr.Expr = nr.Expr
 	rr.Labels = nr.Labels
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 1e21d45c6..611d8f6c6 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -44,6 +44,7 @@ The sandbox cluster installation is running under the constant load generated by
 * BUGFIX: `vminsert`: properly accept samples via [OpenTelemetry data ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) when these samples have no [resource attributes](https://opentelemetry.io/docs/instrumentation/go/resources/). Previously such samples were silently skipped.
 * BUGFIX: `vmstorage`: added missing `-inmemoryDataFlushInterval` command-line flag, which was missing in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) after implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337) in [v1.85.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.0).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): check `-external.url` schema when starting vmalert, must be `http` or `https`. Before, alertmanager could reject alert notifications if `-external.url` contained no or wrong schema.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): automatically add `exported_` prefix for original evaluation result label if it's conflicted with external or reserved one, previously it was overridden. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5161).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle queries, which wrap [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions) with multiple arguments without explicitly specified lookbehind window in square brackets into [aggregate functions](https://docs.victoriametrics.com/MetricsQL.html#aggregate-functions). For example, `sum(quantile_over_time(0.5, process_resident_memory_bytes))` was resulting to `expecting at least 2 args to ...; got 1 args` error. Thanks to @atykhyy for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5414).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): retry on import errors in `vm-native` mode. Before, retries happened only on writes into a network connection between source and destination. But errors returned by server after all the data was transmitted were logged, but not retried.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with [AWS IRSA authorization](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). Previously role chaining was not supported. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3822) for details.
diff --git a/docs/guides/guide-vmanomaly-vmalert.md b/docs/guides/guide-vmanomaly-vmalert.md
index c551fc366..196219309 100644
--- a/docs/guides/guide-vmanomaly-vmalert.md
+++ b/docs/guides/guide-vmanomaly-vmalert.md
@@ -13,12 +13,13 @@ aliases:
 **Prerequisites**
 - *vmanomaly* is a part of enterprise package. You can get license key [here](https://victoriametrics.com/products/enterprise/trial) to try this tutorial.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) (v.1.93.2)
-  -  [vmalert](https://docs.victoriametrics.com/vmalert.html) (v.1.93.2)
-  -  [vmagent](https://docs.victoriametrics.com/vmagent.html) (v.1.93.2)
+  -  [VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) (v.1.96.0)
+  -  [vmalert](https://docs.victoriametrics.com/vmalert.html) (v.1.96.0)
+  -  [vmagent](https://docs.victoriametrics.com/vmagent.html) (v.1.96.0)
   
   If you're unfamiliar with the listed components, please read [QuickStart](https://docs.victoriametrics.com/Quick-Start.html) first.
-- It is assumed that you are familiar with [Grafana](https://grafana.com/)(v.9.3.1) and [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/).
+- It is assumed that you are familiar with [Grafana](https://grafana.com/)(v.10.2.1) and [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/).
+
 ## 1. What is vmanomaly?
 *VictoriaMetrics Anomaly Detection* ([vmanomaly](https://docs.victoriametrics.com/vmanomaly.html)) is a service that continuously scans time series stored in VictoriaMetrics and detects unexpected changes within data patterns in real-time. It does so by utilizing user-configurable machine learning models.
 
@@ -90,7 +91,7 @@ ______________________________
 
 ## 5. vmanomaly configuration and parameter description
 **Parameter description**:
-There are 4 main sections in config file:
+There are 4 required sections in config file:
 
 `scheduler` - defines how often to run and make inferences, as well as what timerange to use to train the model. 
 
@@ -113,9 +114,9 @@ Let's look into parameters in each section:
     Here is the previous 14 days of data to put into the model training. 
 
 * `model`
-    * `class` - what model to run. You can use your own model or choose from built-in models: Seasonal Trend Decomposition, Facebook Prophet, ZScore, Rolling Quantile, Holt-Winters and ARIMA.
+    * `class` - what model to run. You can use your own model or choose from built-in models: Seasonal Trend Decomposition, Facebook Prophet, ZScore, Rolling Quantile, Holt-Winters, Isolation Forest and ARIMA.  Here we use Facebook Prophet (`model.prophet.ProphetModel`).
     
-    Here we use Facebook Prophet with default parameters (`model.prophet.ProphetModel`). You can put parameters that are available in their [docs](https://facebook.github.io/prophet/docs/quick_start.html).
+    * `args` - Model specific parameters, represented as YAML dictionary in a simple `key: value` form. For example, you can use parameters that are available in [FB Prophet](https://facebook.github.io/prophet/docs/quick_start.html).
     
 * `reader`
   * `datasource_url` - Data source. An HTTP endpoint that serves `/api/v1/query_range`.
@@ -139,7 +140,8 @@ scheduler:
 
 model:
   class: "model.prophet.ProphetModel"
-  interval_width: 0.98
+  args:
+    interval_width: 0.98
 
 reader:
   datasource_url: "http://victoriametrics:8428/"
@@ -264,7 +266,6 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 
 <div class="with-copy" markdown="1">
 
-{% raw %}
 ``` yaml
 services:
   vmagent:
@@ -286,7 +287,7 @@ services:
   
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.93.2
+    image: victoriametrics/victoria-metrics:v1.96.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -309,7 +310,7 @@ services:
   
   grafana:
     container_name: grafana
-    image: grafana/grafana-oss:9.3.1
+    image: grafana/grafana-oss:10.2.1
     depends_on:
       - "victoriametrics"
     ports:
@@ -346,7 +347,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial:v1.5.0
+    image: us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial:v1.7.2
     depends_on:
       - "victoriametrics"
     ports:
@@ -379,7 +380,6 @@ volumes:
 networks:
   vm_net:
 ```
-{% endraw %}
 
 </div>
 
diff --git a/docs/keyConcepts.md b/docs/keyConcepts.md
index 520eb0c37..94bb2edbc 100644
--- a/docs/keyConcepts.md
+++ b/docs/keyConcepts.md
@@ -80,7 +80,9 @@ See [these docs](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinal
 #### Raw samples
 
 Every unique time series may consist of an arbitrary number of `(value, timestamp)` data points (aka `raw samples`) sorted by `timestamp`.
-The `value` is a [double-precision floating-point number](https://en.wikipedia.org/wiki/Double-precision_floating-point_format).
+VictoriaMetrics stores all the `values` as [float64](https://en.wikipedia.org/wiki/Double-precision_floating-point_format) values 
+with [extra compression](https://faun.pub/victoriametrics-achieving-better-compression-for-time-series-data-than-gorilla-317bc1f95932) applied.
+This guarantees precision correctness for values with up to 12 significant decimal digits ([-2^54 ... 2^54-1]).
 The `timestamp` is a [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) with millisecond precision.
 
 Below is an example of a single raw sample
diff --git a/docs/vmalert.md b/docs/vmalert.md
index a427e6a0f..65128ae13 100644
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@@ -100,6 +100,8 @@ See the full list of configuration flags in [configuration](#configuration) sect
 
 If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
 to specify different `-external.label` command-line flags in order to define which `vmalert` generated rules or alerts.
+If rule result metrics have label that conflict with `-external.label`, `vmalert` will automatically rename
+it with prefix `exported_`.
 
 Configuration for [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
 and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very
@@ -896,33 +898,6 @@ max(vmalert_alerting_rules_last_evaluation_series_fetched) by(group, alertname)
 See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4039).
 This feature is available only if vmalert is using VictoriaMetrics v1.90 or higher as a datasource.
 
-### Series with the same labelset
-
-vmalert can produce the following error message during rules evaluation:
-```
-result contains metrics with the same labelset after applying rule labels
-```
-
-The error means there is a collision between [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series)
-after applying extra labels to result.
-
-For example, a rule with `expr: foo > 0` returns two distinct time series in response:
-```
-foo{bar="baz"} 1
-foo{bar="qux"} 2
-```
-
-If user configures `-external.label=bar=baz` cmd-line flag to enforce
-adding `bar="baz"` label-value pair, then time series won't be distinct anymore:
-```
-foo{bar="baz"} 1
-foo{bar="baz"} 2 # 'bar' label was overriden by `-external.label=bar=baz
-```
-
-The same issue can be caused by collision of configured `labels` on [Group](#groups) or [Rule](#rules) levels.
-To fix it one should avoid collisions by carefully picking label overrides in configuration.
-
-
 ## Security
 
 See general recommendations regarding security [here](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#security).
diff --git a/docs/vmanomaly.md b/docs/vmanomaly.md
index 152d2b3ea..17f10f355 100644
--- a/docs/vmanomaly.md
+++ b/docs/vmanomaly.md
@@ -126,43 +126,67 @@ optionally preserving labels).
 
  
 ## Usage
-The vmanomaly accepts only one parameter -- config file path:
+> Starting from v1.5.0, vmanomaly requires a license key to run. You can obtain a trial license key [here](https://victoriametrics.com/products/enterprise/trial/).
 
-```sh
-python3 vmanomaly.py config_zscore.yaml
-```
-or
-```sh
-python3 -m vmanomaly config_zscore.yaml
-```
+> See [Getting started guide](https://docs.victoriametrics.com/guides/guide-vmanomaly-vmalert.html).
 
-It is also possible to split up config into multiple files, just list them all in the command line:
+### Config file
+There are 4 required sections in config file:
 
-```sh
-python3 -m vmanomaly model_prophet.yaml io_csv.yaml scheduler_oneoff.yaml
+* `scheduler` - defines how often to run and make inferences, as well as what timerange to use to train the model. 
+* `model` - specific model parameters and configurations, 
+* `reader` - how to read data and where it is located
+* `writer` - where and how to write the generated output.
+
+[`monitoring`](#monitoring) - defines how to monitor work of *vmanomaly* service. This config section is *optional*.
+
+#### Config example
+Here is an example of config file that will run FB Prophet model, that will be retrained every 2 hours on 14 days of previous data. It will generate inference (including `anomaly_score` metric) every 1 minute.
+
+
+You need to put your datasource urls to use it:
+
+```yaml
+scheduler:
+  infer_every: "1m"
+  fit_every: "2h"
+  fit_window: "14d"
+
+model:
+  class: "model.prophet.ProphetModel"
+  args:
+    interval_width: 0.98
+
+reader:
+  datasource_url: [YOUR_DATASOURCE_URL] #Example: "http://victoriametrics:8428/"
+  queries:
+    cache: "sum(rate(vm_cache_entries))"
+
+writer:
+  datasource_url: [YOUR_DATASOURCE_URL] # Example: "http://victoriametrics:8428/"
 ```
 
 ### Monitoring
 
-vmanomaly can be monitored by using push or pull approach.
+*vmanomaly* can be monitored by using push or pull approach.
 It can push metrics to VictoriaMetrics or expose metrics in Prometheus exposition format.
 
 #### Push approach
 
-vmanomaly can push metrics to VictoriaMetrics single-node or cluster version.
+*vmanomaly* can push metrics to VictoriaMetrics single-node or cluster version.
 In order to enable push approach, specify `push` section in config file:
 
 ```yaml
 monitoring:
    push:
-      url: "http://victoriametrics:8428/"
+      url: [YOUR_DATASOURCE_URL] #Example: "http://victoriametrics:8428/"
       extra_labels:
          job: "vmanomaly-push"
 ```
 
 #### Pull approach
 
-vmanomaly can export internal metrics in Prometheus exposition format at `/metrics` page.
+*vmanomaly* can export internal metrics in Prometheus exposition format at `/metrics` page.
 These metrics can be scraped via [vmagent](https://docs.victoriametrics.com/vmagent.html) or Prometheus.
 
 In order to enable pull approach, specify `pull` section in config file:
@@ -176,10 +200,30 @@ monitoring:
 
 This will expose metrics at `http://0.0.0.0:8080/metrics` page.
 
-### Licensing
+### Run vmanomaly Docker Container
 
-Starting from v1.5.0 vmanomaly requires a license key to run. You can obtain a trial license
-key [here](https://victoriametrics.com/products/enterprise/trial/).
+To use *vmanomaly* you need to pull docker image:
+
+```sh
+docker pull us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial:latest
+```
+
+You can put a tag on it for your convinience:
+
+```sh
+docker image tag us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial vmanomaly
+```
+Here is an example of how to run *vmanomaly* docker container with [license file](#licensing):
+
+```sh
+docker run -it --net [YOUR_NETWORK] \
+               -v [YOUR_LICENSE_FILE_PATH]:/license.txt \
+               -v [YOUR_CONFIG_FILE_PATH]:/config.yml \
+               vmanomaly /config.yml \
+               --license-file=/license.txt
+```
+
+### Licensing
 
 The license key can be passed via the following command-line flags:
 ```
@@ -194,10 +238,7 @@ The license key can be passed via the following command-line flags:
                         verification offline.
 ```
 
-Usage example:
-```
-python3 -m vmanomaly --license-file /path/to/license_file.yaml config.yaml
-```
+
 
 In order to make it easier to monitor the license expiration date, the following metrics are exposed(see
 [Monitoring](#monitoring) section for details on how to scrape them):
@@ -212,7 +253,7 @@ vm_license_expires_in_seconds 4.886608e+06
 ```
 
 Example alerts for [vmalert](https://docs.victoriametrics.com/vmalert.html):
-{% raw %}
+
 ```yaml
 groups:
   - name: vm-license
@@ -236,4 +277,4 @@ groups:
           description: "{{ $labels.instance }} of job {{ $labels.job }} license expires in {{ $value | humanizeDuration }}. 
             Please make sure to update the license before it expires."
 ```
-{% endraw %}
+


Test patch:
diff --git a/app/vmalert/rule/alerting_test.go b/app/vmalert/rule/alerting_test.go
index 91d90e31e..3e5e3503f 100644
--- a/app/vmalert/rule/alerting_test.go
+++ b/app/vmalert/rule/alerting_test.go
@@ -768,14 +768,16 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 	ar.q = fq
 
 	// successful attempt
+	// label `job` will be overridden by rule extra label, the original value will be reserved by "exported_job"
 	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
+	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
 	_, err := ar.exec(context.TODO(), time.Now(), 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// label `job` will collide with rule extra label and will make both time series equal
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "baz"))
+	// label `__name__` will be omitted and get duplicated results here
+	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo_1", "job", "bar"))
 	_, err = ar.exec(context.TODO(), time.Now(), 0)
 	if !errors.Is(err, errDuplicate) {
 		t.Fatalf("expected to have %s error; got %s", errDuplicate, err)
@@ -899,20 +901,22 @@ func TestAlertingRule_Template(t *testing.T) {
 				metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
 			},
 			map[uint64]*notifier.Alert{
-				hash(map[string]string{alertNameLabel: "override label", "instance": "foo"}): {
+				hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
 					Labels: map[string]string{
-						alertNameLabel: "override label",
-						"instance":     "foo",
+						alertNameLabel:       "override label",
+						"exported_alertname": "override",
+						"instance":           "foo",
 					},
 					Annotations: map[string]string{
 						"summary":     `first: Too high connection number for "foo"`,
 						"description": `override: It is 2 connections for "foo"`,
 					},
 				},
-				hash(map[string]string{alertNameLabel: "override label", "instance": "bar"}): {
+				hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "bar"}): {
 					Labels: map[string]string{
-						alertNameLabel: "override label",
-						"instance":     "bar",
+						alertNameLabel:       "override label",
+						"exported_alertname": "override",
+						"instance":           "bar",
 					},
 					Annotations: map[string]string{
 						"summary":     `second: Too high connection number for "bar"`,
@@ -941,14 +945,18 @@ func TestAlertingRule_Template(t *testing.T) {
 			},
 			map[uint64]*notifier.Alert{
 				hash(map[string]string{
-					alertNameLabel:      "OriginLabels",
-					alertGroupNameLabel: "Testing",
-					"instance":          "foo",
+					alertNameLabel:        "OriginLabels",
+					"exported_alertname":  "originAlertname",
+					alertGroupNameLabel:   "Testing",
+					"exported_alertgroup": "originGroupname",
+					"instance":            "foo",
 				}): {
 					Labels: map[string]string{
-						alertNameLabel:      "OriginLabels",
-						alertGroupNameLabel: "Testing",
-						"instance":          "foo",
+						alertNameLabel:        "OriginLabels",
+						"exported_alertname":  "originAlertname",
+						alertGroupNameLabel:   "Testing",
+						"exported_alertgroup": "originGroupname",
+						"instance":            "foo",
 					},
 					Annotations: map[string]string{
 						"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
@@ -1092,3 +1100,54 @@ func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.
 	rule.KeepFiringFor = keepFiringFor
 	return rule
 }
+
+func TestAlertingRule_ToLabels(t *testing.T) {
+	metric := datasource.Metric{
+		Labels: []datasource.Label{
+			{Name: "instance", Value: "0.0.0.0:8800"},
+			{Name: "group", Value: "vmalert"},
+			{Name: "alertname", Value: "ConfigurationReloadFailure"},
+		},
+		Values:     []float64{1},
+		Timestamps: []int64{time.Now().UnixNano()},
+	}
+
+	ar := &AlertingRule{
+		Labels: map[string]string{
+			"instance": "override", // this should override instance with new value
+			"group":    "vmalert",  // this shouldn't have effect since value in metric is equal
+		},
+		Expr:      "sum(vmalert_alerting_rules_error) by(instance, group, alertname) > 0",
+		Name:      "AlertingRulesError",
+		GroupName: "vmalert",
+	}
+
+	expectedOriginLabels := map[string]string{
+		"instance":   "0.0.0.0:8800",
+		"group":      "vmalert",
+		"alertname":  "ConfigurationReloadFailure",
+		"alertgroup": "vmalert",
+	}
+
+	expectedProcessedLabels := map[string]string{
+		"instance":           "override",
+		"exported_instance":  "0.0.0.0:8800",
+		"alertname":          "AlertingRulesError",
+		"exported_alertname": "ConfigurationReloadFailure",
+		"group":              "vmalert",
+		"alertgroup":         "vmalert",
+	}
+
+	ls, err := ar.toLabels(metric, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if !reflect.DeepEqual(ls.origin, expectedOriginLabels) {
+		t.Errorf("origin labels mismatch, got: %v, want: %v", ls.origin, expectedOriginLabels)
+	}
+
+	if !reflect.DeepEqual(ls.processed, expectedProcessedLabels) {
+		t.Errorf("processed labels mismatch, got: %v, want: %v", ls.processed, expectedProcessedLabels)
+	}
+}
diff --git a/app/vmalert/rule/recording_test.go b/app/vmalert/rule/recording_test.go
index 65b391f19..019d50fc0 100644
--- a/app/vmalert/rule/recording_test.go
+++ b/app/vmalert/rule/recording_test.go
@@ -61,7 +61,7 @@ func TestRecordingRule_Exec(t *testing.T) {
 			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
 			},
 			[]prompbmarshal.TimeSeries{
 				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
@@ -70,9 +70,10 @@ func TestRecordingRule_Exec(t *testing.T) {
 					"source":   "test",
 				}),
 				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "job:foo",
-					"job":      "bar",
-					"source":   "test",
+					"__name__":        "job:foo",
+					"job":             "bar",
+					"source":          "test",
+					"exported_source": "origin",
 				}),
 			},
 		},
@@ -254,10 +255,7 @@ func TestRecordingRule_ExecNegative(t *testing.T) {
 	fq.Add(metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "bar"))
 
 	_, err = rr.exec(context.TODO(), time.Now(), 0)
-	if err == nil {
-		t.Fatalf("expected to get err; got nil")
-	}
-	if !strings.Contains(err.Error(), errDuplicate.Error()) {
-		t.Fatalf("expected to get err %q; got %q insterad", errDuplicate, err)
+	if err != nil {
+		t.Fatal(err)
 	}
 }