Repository: VictoriaMetrics/VictoriaMetrics
Pull request #3186 (2022-09-30 08:21:39)

docs/vmagent.md: fix typo
fix typo: 'SUGHUP'


Patch:
diff --git a/Makefile b/Makefile
index b6ba009a6..5d020de7a 100644
--- a/Makefile
+++ b/Makefile
@@ -323,15 +323,7 @@ install-golint:
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
-	errcheck -exclude=errcheck_excludes.txt ./app/vmctl/...
+	errcheck -exclude=errcheck_excludes.txt ./app/...
 
 install-errcheck:
 	which errcheck || go install github.com/kisielk/errcheck@latest
diff --git a/app/victoria-metrics/self_scraper.go b/app/victoria-metrics/self_scraper.go
index fb544a009..e8540eddd 100644
--- a/app/victoria-metrics/self_scraper.go
+++ b/app/victoria-metrics/self_scraper.go
@@ -85,7 +85,9 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr.Timestamp = currentTimestamp
 			mr.Value = r.Value
 		}
-		vmstorage.AddRows(mrs)
+		if err := vmstorage.AddRows(mrs); err != nil {
+			logger.Errorf("cannot store self-scraped metrics: %s", err)
+		}
 	}
 }
 
diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index 51cc9e71d..e7c25dde9 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -142,6 +142,10 @@ While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx
 
 By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`.
 
+[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) supports writing data to multiple tenants
+specified via special labels - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels).
+This allows specifying tenant ids via [relabeling](#relabeling) and writing multitenant data to a single `-remoteWrite.url=http://<vminsert-addr>/insert/multitenant/api/v1/write`.
+
 [Multitenancy](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) support is enabled when `-remoteWrite.multitenantURL` command-line flag is set. In this case `vmagent` accepts multitenant data at `http://vmagent:8429/insert/<accountID>/...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert/<accountID>/prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants.
 
 If `-remoteWrite.multitenantURL` command-line flag is set and `vmagent` is configured to scrape Prometheus-compatible targets (e.g. if `-promscrape.config` command-line flag is set)
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index 13724ae9b..b020eada7 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -1158,7 +1158,7 @@ is the following:
 # password and password_file are mutually exclusive.
 basic_auth:
   [ username: <string> ]
-  [ password: <secret> ]
+  [ password: <string> ]
   [ password_file: <string> ]
 
 # Optional `Authorization` header configuration.
@@ -1177,10 +1177,41 @@ authorization:
 tls_config:
   [ <tls_config> ]
 
+# Configures Bearer authentication token via string
+bearer_token: <string>
+# or by passing path to the file with token.
+bearer_token_file: <string>
+
+# Configures OAuth 2.0 authentication
+# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2
+oauth2:
+  [ <oauth2_config> ]
+  
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all requests to notifiers
+# For example:
+#  headers:
+#    - "CustomHeader: foo"
+#    - "CustomHeader2: bar"
+headers:
+  [ <string>, ...]
+
 # List of labeled statically configured Notifiers.
+#
+# Each list of targets may be additionally instructed with
+# authorization params. Target's authorization params will
+# inherit params from global authorization params if there
+# are no conflicts.
 static_configs:
-  targets:
-    [ - '<host>' ]
+  [ - targets: ]
+      [ - '<host>' ]
+      [ oauth2 ]
+      [ basic_auth ]
+      [ authorization ]
+      [ tls_config ]
+      [ bearer_token ]
+      [ bearer_token_file ]
+      [ headers ]
 
 # List of Consul service discovery configurations.
 # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go
index 1fb41ef17..f7b7e02ba 100644
--- a/app/vmalert/notifier/config.go
+++ b/app/vmalert/notifier/config.go
@@ -68,6 +68,8 @@ type Config struct {
 //	[ - '<host>' ]
 type StaticConfig struct {
 	Targets []string `yaml:"targets"`
+	// HTTPClientConfig contains HTTP configuration for the Targets
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
diff --git a/app/vmalert/notifier/config_watcher.go b/app/vmalert/notifier/config_watcher.go
index 297eb06ba..f70fed7c5 100644
--- a/app/vmalert/notifier/config_watcher.go
+++ b/app/vmalert/notifier/config_watcher.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 )
@@ -161,12 +162,13 @@ func (cw *configWatcher) start() error {
 	if len(cw.cfg.StaticConfigs) > 0 {
 		var targets []Target
 		for _, cfg := range cw.cfg.StaticConfigs {
+			httpCfg := mergeHTTPClientConfigs(cw.cfg.HTTPClientConfig, cfg.HTTPClientConfig)
 			for _, target := range cfg.Targets {
 				address, labels, err := parseLabels(target, nil, cw.cfg)
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %s", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, cw.cfg.HTTPClientConfig, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %s", address, err)
 				}
@@ -252,3 +254,30 @@ func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
 	cw.targets[key] = targets
 	cw.targetsMu.Unlock()
 }
+
+// mergeHTTPClientConfigs merges fields between child and parent params
+// by populating child from parent params if they're missing.
+func mergeHTTPClientConfigs(parent, child promauth.HTTPClientConfig) promauth.HTTPClientConfig {
+	if child.Authorization == nil {
+		child.Authorization = parent.Authorization
+	}
+	if child.BasicAuth == nil {
+		child.BasicAuth = parent.BasicAuth
+	}
+	if child.BearerToken == nil {
+		child.BearerToken = parent.BearerToken
+	}
+	if child.BearerTokenFile == "" {
+		child.BearerTokenFile = parent.BearerTokenFile
+	}
+	if child.OAuth2 == nil {
+		child.OAuth2 = parent.OAuth2
+	}
+	if child.TLSConfig == nil {
+		child.TLSConfig = parent.TLSConfig
+	}
+	if child.Headers == nil {
+		child.Headers = parent.Headers
+	}
+	return child
+}
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index b5eabc002..6ab93bd4e 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -19,6 +19,14 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 **Update note 2:** [vmalert](https://docs.victoriametrics.com/vmalert.html) changes default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval.
 
+**Update note 3:** `vm_account_id` and `vm_project_id` labels must be passed to tcp-based `Graphite`, `InfluxDB` and `OpenTSDB` endpoints
+at [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) instead of undocumented
+`VictoriaMetrics_AccountID` and `VictoriaMetrics_ProjectID` labels when writing samples to the needed tenant.
+See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) for details.
+
+* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): support specifying tenant ids via `vm_account_id` and `vm_project_id` labels. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): improve [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) performance by up to 3x for non-trivial `regex` values such as `([^:]+):.+`, which can be used for extracting a `host` part from `host:port` label value.
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): improve performance by up to 4x for queries containing non-trivial `regex` filters such as `{path=~"/foo/.+|/bar"}`.
 * FEATURE: sanitize metric names for data ingested via [DataDog protocol](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) according to [DataDog metric naming](https://docs.datadoghq.com/metrics/custom_metrics/#naming-custom-metrics). The behaviour can be disabled by passing `-datadog.sanitizeMetricName=false` command-line flag. Thanks to @PerGon for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3105).
 * FEATURE: add `-usePromCompatibleNaming` command-line flag to [vmagent](https://docs.victoriametrics.com/vmagent.html), to single-node VictoriaMetrics and to `vminsert` component of VictoriaMetrics cluster. This flag can be used for normalizing the ingested metric names and label names to [Prometheus-compatible form](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). If this flag is set, then all the chars unsupported by Prometheus are replaced with `_` chars in metric names and labels of the ingested samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113).
 * FEATURE: accept whitespace in metric names and tags ingested via [Graphite plaintext protocol](https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) according to [the specs](https://graphite.readthedocs.io/en/latest/tags.html). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3102).
@@ -31,10 +39,12 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `debug` mode to the alerting rule settings for printing additional information into logs during evaluation. See `debug` param in [alerting rule config](https://docs.victoriametrics.com/vmalert.html#alerting-rules).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add experimental feature for displaying last 10 states of the rule (recording or alerting) evaluation. The state is available on the Rule page, which can be opened by clicking on `Details` link next to Rule's name on the `/groups` page.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow using extra labels in annotiations. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3013).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): allow configuring authorization params per list of targets in vmalert's notifier config for `static_configs`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2690).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): minimize the time needed for reading large responses from scrape targets in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). This should reduce scrape durations for such targets as [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) running in a big Kubernetes cluster.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [sort_by_label_numeric](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric) and [sort_by_label_numeric_desc](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric_desc) functions for [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html) of input time series by the specified labels. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2938).
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html) and [vmrestore](https://docs.victoriametrics.com/vmrestore.html): retry GCS operations for up to 3 minutes on temporary failures. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3147).
 * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway.html): add ability to extract JWT authorization token from non-standard HTTP header by passing it via `-auth.httpHeader` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3054).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `__meta_ec2_region` label for [ec2_sd_config](https://docs.victoriametrics.com/sd_configs.html#ec2_sd_configs) in the same way as [Prometheus 2.39 does](https://github.com/prometheus/prometheus/pull/11326).
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly encode query params for aws signed requests, use `%20` instead of `+` as api requires. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3171).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly parse relabel config when regex ending with escaped `$`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3131).
diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md
index b73d0be77..ffe7aeabb 100644
--- a/docs/Cluster-VictoriaMetrics.md
+++ b/docs/Cluster-VictoriaMetrics.md
@@ -43,7 +43,9 @@ It increases cluster availability, and simplifies cluster maintenance as well as
 
 VictoriaMetrics cluster supports multiple isolated tenants (aka namespaces).
 Tenants are identified by `accountID` or `accountID:projectID`, which are put inside request urls.
-See [these docs](#url-format) for details. Some facts about tenants in VictoriaMetrics:
+See [these docs](#url-format) for details.
+
+Some facts about tenants in VictoriaMetrics:
 
 - Each `accountID` and `projectID` is identified by an arbitrary 32-bit integer in the range `[0 .. 2^32)`.
 If `projectID` is missing, then it is automatically assigned to `0`. It is expected that other information about tenants
@@ -59,6 +61,30 @@ when different tenants have different amounts of data and different query load.
 
 - VictoriaMetrics doesn't support querying multiple tenants in a single request.
 
+See also [multitenancy via labels](#multitenancy-via-labels).
+
+
+## Multitenancy via labels
+
+`vminsert` can accept data from multiple [tenants](#multitenancy) via a special `multitenant` endpoints `http://vminsert:8480/insert/multitenant/<suffix>`,
+where `<suffix>` can be replaced with any supported suffix for data ingestion from [this list](#url-format).
+In this case the account id and project id are obtained from optional `vm_account_id` and `vm_project_id` labels of the incoming samples.
+If `vm_account_id` or `vm_project_id` labels are missing or invalid, then the corresponding `accountID` or `projectID` is set to 0.
+These labels are automatically removed from samples before forwarding them to `vmstorage`.
+For example, if the following samples are written into `http://vminsert:8480/insert/multitenant/prometheus/api/v1/write`:
+
+```
+http_requests_total{path="/foo",vm_account_id="42"} 12
+http_requests_total{path="/bar",vm_account_id="7",vm_project_id="9"} 34
+```
+
+Then the `http_requests_total{path="/foo"} 12` would be stored in the tenant `accountID=42, projectID=0`,
+while the `http_requests_total{path="/bar"} 34` would be stored in the tenant `accountID=7, projectID=9`.
+
+The `vm_account_id` and `vm_project_id` labels are extracted after applying the [relabeling](https://docs.victoriametrics.com/relabeling.html)
+set via `-relabelConfig` command-line flag, so these labels can be set at this stage.
+
+
 ## Binaries
 
 Compiled binaries for the cluster version are available in the `assets` section of the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
@@ -218,7 +244,9 @@ See [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html
 
 - URLs for data ingestion: `http://<vminsert>:8480/insert/<accountID>/<suffix>`, where:
   - `<accountID>` is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). It is possible to set it as `accountID:projectID`,
-    where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`.
+    where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details.
+    The `<accountID>` can be set to `multitenant` string, e.g. `http://<vminsert>:8480/insert/multitenant/<suffix>`. Such urls accept data from multiple tenants
+    specified via `vm_account_id` and `vm_project_id` labels. See [multitenancy via labels](#multitenancy-via-labels) for more details.
   - `<suffix>` may have the following values:
     - `prometheus` and `prometheus/api/v1/write` - for inserting data with [Prometheus remote write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
     - `datadog/api/v1/series` - for inserting data with [DataDog submit metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent) for details.
diff --git a/docs/sd_configs.md b/docs/sd_configs.md
index c834deda0..c9f4a8e32 100644
--- a/docs/sd_configs.md
+++ b/docs/sd_configs.md
@@ -473,6 +473,7 @@ The following meta labels are available on discovered targets during [relabeling
 * `__meta_ec2_private_ip`: the private IP address of the instance, if present
 * `__meta_ec2_public_dns_name`: the public DNS name of the instance, if available
 * `__meta_ec2_public_ip`: the public IP address of the instance, if available
+* `__meta_ec2_region`: EC2 region for the discovered instance
 * `__meta_ec2_subnet_id`: comma separated list of subnets IDs in which the instance is running, if available
 * `__meta_ec2_tag_<tagkey>`: each tag value of the instance
 * `__meta_ec2_vpc_id`: the ID of the VPC in which the instance is running, if available
diff --git a/docs/url-examples.md b/docs/url-examples.md
index 35e610765..9f0ac3e57 100644
--- a/docs/url-examples.md
+++ b/docs/url-examples.md
@@ -604,7 +604,7 @@ Cluster version of VictoriaMetrics:
 <div class="with-copy" markdown="1">
 
 ```console
-echo "put foo.bar.baz `date +%s` 123  tag1=value1 tag2=value2 VictoriaMetrics_AccountID=0" | nc -N http://<vminsert> 4242
+echo "put foo.bar.baz `date +%s` 123  tag1=value1 tag2=value2" | nc -N http://<vminsert> 4242
 ```
 
 </div>
@@ -651,14 +651,12 @@ Cluster version of VictoriaMetrics:
 <div class="with-copy" markdown="1">
 
 ```console
-echo "foo.bar.baz;tag1=value1;tag2=value2;VictoriaMetrics_AccountID=42 123 `date +%s`" | nc -N http://<vminsert> 2003
+echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N http://<vminsert> 2003
 ```
 
 </div>
 
 Additional information:
 
-`VictoriaMetrics_AccountID=42` - [tenant ID](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) in cluster version of VictoriaMetrics
-
 * [How to send Graphite data to VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
 * [Multitenancy in cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy)
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 7ef86bad4..c2d58fa6a 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -80,7 +80,7 @@ Pass `-help` to `vmagent` in order to see [the full list of supported command-li
 
 `vmagent` supports multiple approaches for reloading configs from updated config files such as `-promscrape.config`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`:
 
-* Sending `SUGHUP` signal to `vmagent` process:
+* Sending `SIGHUP` signal to `vmagent` process:
 
   ```console
   kill -SIGHUP `pidof vmagent`
@@ -146,6 +146,10 @@ While `vmagent` can accept data in several supported protocols (OpenTSDB, Influx
 
 By default `vmagent` collects the data without tenant identifiers and routes it to the configured `-remoteWrite.url`.
 
+[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) supports writing data to multiple tenants
+specified via special labels - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels).
+This allows specifying tenant ids via [relabeling](#relabeling) and writing multitenant data to a single `-remoteWrite.url=http://<vminsert-addr>/insert/multitenant/api/v1/write`.
+
 [Multitenancy](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) support is enabled when `-remoteWrite.multitenantURL` command-line flag is set. In this case `vmagent` accepts multitenant data at `http://vmagent:8429/insert/<accountID>/...` in the same way as cluster version of VictoriaMetrics does according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format) and routes it to `<-remoteWrite.multitenantURL>/insert/<accountID>/prometheus/api/v1/write`. If multiple `-remoteWrite.multitenantURL` command-line options are set, then `vmagent` replicates the collected data across all the configured urls. This allows using a single `vmagent` instance in front of VictoriaMetrics clusters for processing the data from all the tenants.
 
 If `-remoteWrite.multitenantURL` command-line flag is set and `vmagent` is configured to scrape Prometheus-compatible targets (e.g. if `-promscrape.config` command-line flag is set)
diff --git a/docs/vmalert.md b/docs/vmalert.md
index 4ee7fd44d..528728524 100644
--- a/docs/vmalert.md
+++ b/docs/vmalert.md
@@ -1162,7 +1162,7 @@ is the following:
 # password and password_file are mutually exclusive.
 basic_auth:
   [ username: <string> ]
-  [ password: <secret> ]
+  [ password: <string> ]
   [ password_file: <string> ]
 
 # Optional `Authorization` header configuration.
@@ -1181,10 +1181,41 @@ authorization:
 tls_config:
   [ <tls_config> ]
 
+# Configures Bearer authentication token via string
+bearer_token: <string>
+# or by passing path to the file with token.
+bearer_token_file: <string>
+
+# Configures OAuth 2.0 authentication
+# see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2
+oauth2:
+  [ <oauth2_config> ]
+  
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all requests to notifiers
+# For example:
+#  headers:
+#    - "CustomHeader: foo"
+#    - "CustomHeader2: bar"
+headers:
+  [ <string>, ...]
+
 # List of labeled statically configured Notifiers.
+#
+# Each list of targets may be additionally instructed with
+# authorization params. Target's authorization params will
+# inherit params from global authorization params if there
+# are no conflicts.
 static_configs:
-  targets:
-    [ - '<host>' ]
+  [ - targets: ]
+      [ - '<host>' ]
+      [ oauth2 ]
+      [ basic_auth ]
+      [ authorization ]
+      [ tls_config ]
+      [ bearer_token ]
+      [ bearer_token_file ]
+      [ headers ]
 
 # List of Consul service discovery configurations.
 # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
diff --git a/lib/auth/auth.go b/lib/auth/auth.go
index 6d27943e3..38e235448 100644
--- a/lib/auth/auth.go
+++ b/lib/auth/auth.go
@@ -14,30 +14,54 @@ type Token struct {
 
 // String returns string representation of t.
 func (t *Token) String() string {
+	if t == nil {
+		return "multitenant"
+	}
 	if t.ProjectID == 0 {
 		return fmt.Sprintf("%d", t.AccountID)
 	}
 	return fmt.Sprintf("%d:%d", t.AccountID, t.ProjectID)
 }
 
-// NewToken returns new Token for the given authToken
+// NewToken returns new Token for the given authToken.
+//
+// If authToken == "multitenant", then nil Token is returned.
 func NewToken(authToken string) (*Token, error) {
+	if authToken == "multitenant" {
+		return nil, nil
+	}
+	var t Token
+	if err := t.Init(authToken); err != nil {
+		return nil, err
+	}
+	return &t, nil
+}
+
+// Init initializes t from authToken.
+func (t *Token) Init(authToken string) error {
 	tmp := strings.Split(authToken, ":")
 	if len(tmp) > 2 {
-		return nil, fmt.Errorf("unexpected number of items in authToken %q; got %d; want 1 or 2", authToken, len(tmp))
+		return fmt.Errorf("unexpected number of items in authToken %q; got %d; want 1 or 2", authToken, len(tmp))
 	}
-	var at Token
-	accountID, err := strconv.ParseUint(tmp[0], 10, 32)
+	n, err := strconv.ParseUint(tmp[0], 10, 32)
 	if err != nil {
-		return nil, fmt.Errorf("cannot parse accountID from %q: %w", tmp[0], err)
+		return fmt.Errorf("cannot parse accountID from %q: %w", tmp[0], err)
 	}
-	at.AccountID = uint32(accountID)
+	accountID := uint32(n)
+	projectID := uint32(0)
 	if len(tmp) > 1 {
-		projectID, err := strconv.ParseUint(tmp[1], 10, 32)
+		n, err := strconv.ParseUint(tmp[1], 10, 32)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse projectID from %q: %w", tmp[1], err)
+			return fmt.Errorf("cannot parse projectID from %q: %w", tmp[1], err)
 		}
-		at.ProjectID = uint32(projectID)
+		projectID = uint32(n)
 	}
-	return &at, nil
+	t.Set(accountID, projectID)
+	return nil
+}
+
+// Set sets accountID and projectID for the t.
+func (t *Token) Set(accountID, projectID uint32) {
+	t.AccountID = accountID
+	t.ProjectID = projectID
 }
diff --git a/lib/awsapi/config.go b/lib/awsapi/config.go
index 706f87526..fb97be0af 100644
--- a/lib/awsapi/config.go
+++ b/lib/awsapi/config.go
@@ -53,11 +53,9 @@ func NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey,
 		defaultAccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
 		defaultSecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
 	}
-	cfg.service = service
 	if cfg.service == "" {
 		cfg.service = "aps"
 	}
-	cfg.region = region
 	if cfg.region == "" {
 		r, err := getDefaultRegion(cfg.client)
 		if err != nil {
@@ -75,8 +73,6 @@ func NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey,
 		return nil, fmt.Errorf("roleARN is missing for AWS_WEB_IDENTITY_TOKEN_FILE=%q; set it via env var AWS_ROLE_ARN", cfg.webTokenPath)
 	}
 	// explicitly set credentials has priority over env variables
-	cfg.defaultAccessKey = os.Getenv("AWS_ACCESS_KEY_ID")
-	cfg.defaultSecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
 	if len(accessKey) > 0 {
 		cfg.defaultAccessKey = accessKey
 	}
@@ -90,6 +86,11 @@ func NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey,
 	return cfg, nil
 }
 
+// GetRegion returns region for the given cfg.
+func (cfg *Config) GetRegion() string {
+	return cfg.region
+}
+
 // GetEC2APIResponse performs EC2 API request with ghe given action.
 //
 // filtersQueryString must contain an optional percent-encoded query string for aws filters.
diff --git a/lib/bytesutil/fast_string_matcher.go b/lib/bytesutil/fast_string_matcher.go
new file mode 100644
index 000000000..40658e499
--- /dev/null
+++ b/lib/bytesutil/fast_string_matcher.go
@@ -0,0 +1,48 @@
+package bytesutil
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// FastStringMatcher implements fast matcher for strings.
+//
+// It caches string match results and returns them back on the next calls
+// without calling the matchFunc, which may be expensive.
+type FastStringMatcher struct {
+	m    atomic.Value
+	mLen uint64
+
+	matchFunc func(s string) bool
+}
+
+// NewFastStringMatcher creates new matcher, which applies matchFunc to strings passed to Match()
+//
+// matchFunc must return the same result for the same input.
+func NewFastStringMatcher(matchFunc func(s string) bool) *FastStringMatcher {
+	var fsm FastStringMatcher
+	fsm.m.Store(&sync.Map{})
+	fsm.matchFunc = matchFunc
+	return &fsm
+}
+
+// Match applies matchFunc to s and returns the result.
+func (fsm *FastStringMatcher) Match(s string) bool {
+	m := fsm.m.Load().(*sync.Map)
+	v, ok := m.Load(s)
+	if ok {
+		// Fast path - s match result is found in the cache.
+		bp := v.(*bool)
+		return *bp
+	}
+	// Slow path - run matchFunc for s and store the result in the cache.
+	b := fsm.matchFunc(s)
+	bp := &b
+	m.Store(s, bp)
+	n := atomic.AddUint64(&fsm.mLen, 1)
+	if n > 100e3 {
+		atomic.StoreUint64(&fsm.mLen, 0)
+		fsm.m.Store(&sync.Map{})
+	}
+	return b
+}
diff --git a/lib/httpserver/path.go b/lib/httpserver/path.go
index 833c0a6b4..3743341e6 100644
--- a/lib/httpserver/path.go
+++ b/lib/httpserver/path.go
@@ -22,6 +22,8 @@ func ParsePath(path string) (*Path, error) {
 	//
 	// - prefix must contain `select`, `insert` or `delete`.
 	// - authToken contains `accountID[:projectID]`, where projectID is optional.
+	//   authToken may also contain `multitenant` string. In this case the accountID and projectID
+	//   are obtained from vm_account_id and vm_project_id labels of the ingested samples.
 	// - suffix contains arbitrary suffix.
 	//
 	// prefix must be used for the routing to the appropriate service
@@ -29,14 +31,14 @@ func ParsePath(path string) (*Path, error) {
 	s := skipPrefixSlashes(path)
 	n := strings.IndexByte(s, '/')
 	if n < 0 {
-		return nil, fmt.Errorf("cannot find {prefix}")
+		return nil, fmt.Errorf("cannot find {prefix} in %q; expecting /{prefix}/{authToken}/{suffix} format", path)
 	}
 	prefix := s[:n]
 
 	s = skipPrefixSlashes(s[n+1:])
 	n = strings.IndexByte(s, '/')
 	if n < 0 {
-		return nil, fmt.Errorf("cannot find {authToken}")
+		return nil, fmt.Errorf("cannot find {authToken} in %q; expecting /{prefix}/{authToken}/{suffix} format", path)
 	}
 	authToken := s[:n]
 
diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go
index 346e0320a..eee9f002c 100644
--- a/lib/promrelabel/config.go
+++ b/lib/promrelabel/config.go
@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -346,7 +347,7 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) {
 			return nil, fmt.Errorf("`labels` config cannot be applied to `action=%s`; it is applied only to `action=graphite`", action)
 		}
 	}
-	return &parsedRelabelConfig{
+	prc := &parsedRelabelConfig{
 		SourceLabels:  sourceLabels,
 		Separator:     separator,
 		TargetLabel:   targetLabel,
@@ -365,7 +366,10 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) {
 		hasCaptureGroupInTargetLabel:   strings.Contains(targetLabel, "$"),
 		hasCaptureGroupInReplacement:   strings.Contains(replacement, "$"),
 		hasLabelReferenceInReplacement: strings.Contains(replacement, "{{"),
-	}, nil
+	}
+	prc.stringReplacer = bytesutil.NewFastStringTransformer(prc.replaceFullStringSlow)
+	prc.submatchReplacer = bytesutil.NewFastStringTransformer(prc.replaceStringSubmatchesSlow)
+	return prc, nil
 }
 
 func isDefaultRegex(expr string) bool {
diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go
index b8d293aa8..6d55f3b64 100644
--- a/lib/promrelabel/relabel.go
+++ b/lib/promrelabel/relabel.go
@@ -35,6 +35,9 @@ type parsedRelabelConfig struct {
 	hasCaptureGroupInTargetLabel   bool
 	hasCaptureGroupInReplacement   bool
 	hasLabelReferenceInReplacement bool
+
+	stringReplacer   *bytesutil.FastStringTransformer
+	submatchReplacer *bytesutil.FastStringTransformer
 }
 
 // String returns human-readable representation for prc.
@@ -169,7 +172,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 		bb := relabelBufPool.Get()
 		for _, gl := range prc.graphiteLabelRules {
 			bb.B = gl.grt.Expand(bb.B[:0], gm.a)
-			valueStr := string(bb.B)
+			valueStr := bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
 			labels = setLabelValue(labels, labelsOffset, gl.targetLabel, valueStr)
 		}
 		relabelBufPool.Put(bb)
@@ -182,7 +185,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 		if prc.hasLabelReferenceInReplacement {
 			// Fill {{labelName}} references in the replacement
 			bb.B = fillLabelReferences(bb.B[:0], replacement, labels[labelsOffset:])
-			replacement = string(bb.B)
+			replacement = bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
 		}
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
 		if prc.RegexAnchored == defaultRegexForRelabelConfig && !prc.hasCaptureGroupInTargetLabel {
@@ -190,7 +193,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 				// Fast path for the rule that copies source label values to destination:
 				// - source_labels: [...]
 				//   target_label: foobar
-				valueStr := string(bb.B)
+				valueStr := bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
 				relabelBufPool.Put(bb)
 				return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
 			}
@@ -203,23 +206,29 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 				return labels
 			}
 		}
-		if re := prc.regex; re.HasPrefix() && !re.MatchString(bytesutil.ToUnsafeString(bb.B)) {
+		sourceStr := bytesutil.ToUnsafeString(bb.B)
+		if !prc.regex.MatchString(sourceStr) {
 			// Fast path - regexp mismatch.
 			relabelBufPool.Put(bb)
 			return labels
 		}
-		match := prc.RegexAnchored.FindSubmatchIndex(bb.B)
-		if match == nil {
-			// Fast path - nothing to replace.
-			relabelBufPool.Put(bb)
-			return labels
+		var valueStr string
+		if replacement == prc.Replacement {
+			// Fast path - the replacement wasn't modified, so it is safe calling stringReplacer.Transform.
+			valueStr = prc.stringReplacer.Transform(sourceStr)
+		} else {
+			// Slow path - the replacement has been modified, so the valueStr must be calculated
+			// from scratch based on the new replacement value.
+			match := prc.RegexAnchored.FindSubmatchIndex(bb.B)
+			valueStr = prc.expandCaptureGroups(replacement, sourceStr, match)
 		}
-		sourceStr := bytesutil.ToUnsafeString(bb.B)
 		nameStr := prc.TargetLabel
 		if prc.hasCaptureGroupInTargetLabel {
+			// Slow path - target_label contains regex capture groups, so the target_label
+			// must be calculated from the regex match.
+			match := prc.RegexAnchored.FindSubmatchIndex(bb.B)
 			nameStr = prc.expandCaptureGroups(nameStr, sourceStr, match)
 		}
-		valueStr := prc.expandCaptureGroups(replacement, sourceStr, match)
 		relabelBufPool.Put(bb)
 		return setLabelValue(labels, labelsOffset, nameStr, valueStr)
 	case "replace_all":
@@ -227,10 +236,10 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 		// and store the result at `target_label`
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		sourceStr := string(bb.B)
+		sourceStr := bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
 		relabelBufPool.Put(bb)
-		valueStr, ok := prc.replaceStringSubmatches(sourceStr, prc.Replacement, prc.hasCaptureGroupInReplacement)
-		if ok {
+		valueStr := prc.replaceStringSubmatchesFast(sourceStr)
+		if valueStr != sourceStr {
 			labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
 		}
 		return labels
@@ -305,8 +314,8 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 	case "labelmap":
 		// Replace label names with the `replacement` if they match `regex`
 		for _, label := range src {
-			labelName, ok := prc.replaceFullString(label.Name, prc.Replacement, prc.hasCaptureGroupInReplacement)
-			if ok {
+			labelName := prc.replaceFullStringFast(label.Name)
+			if labelName != label.Name {
 				labels = setLabelValue(labels, labelsOffset, labelName, label.Value)
 			}
 		}
@@ -315,7 +324,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 		// Replace all the occurrences of `regex` at label names with `replacement`
 		for i := range src {
 			label := &src[i]
-			label.Name, _ = prc.replaceStringSubmatches(label.Name, prc.Replacement, prc.hasCaptureGroupInReplacement)
+			label.Name = prc.replaceStringSubmatchesFast(label.Name)
 		}
 		return labels
 	case "labeldrop":
@@ -341,7 +350,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 	case "uppercase":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		valueStr := string(bb.B)
+		valueStr := bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
 		relabelBufPool.Put(bb)
 		valueStr = strings.ToUpper(valueStr)
 		labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
@@ -349,7 +358,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 	case "lowercase":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
-		valueStr := string(bb.B)
+		valueStr := bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
 		relabelBufPool.Put(bb)
 		valueStr = strings.ToLower(valueStr)
 		labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
@@ -360,16 +369,23 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset
 	}
 }
 
-func (prc *parsedRelabelConfig) replaceFullString(s, replacement string, hasCaptureGroupInReplacement bool) (string, bool) {
+// replaceFullStringFast replaces s with the replacement if s matches '^regex$'.
+//
+// s is returned as is if it doesn't match '^regex$'.
+func (prc *parsedRelabelConfig) replaceFullStringFast(s string) string {
 	prefix, complete := prc.regexOriginal.LiteralPrefix()
-	if complete && !hasCaptureGroupInReplacement {
+	replacement := prc.Replacement
+	if complete && !prc.hasCaptureGroupInReplacement {
 		if s == prefix {
-			return replacement, true
+			// Fast path - s matches literal regex
+			return replacement
 		}
-		return s, false
+		// Fast path - s doesn't match literal regex
+		return s
 	}
 	if !strings.HasPrefix(s, prefix) {
-		return s, false
+		// Fast path - s doesn't match literal prefix from regex
+		return s
 	}
 	if replacement == "$1" {
 		// Fast path for commonly used rule for deleting label prefixes such as:
@@ -383,50 +399,55 @@ func (prc *parsedRelabelConfig) replaceFullString(s, replacement string, hasCapt
 			reSuffix := reStr[len(prefix):]
 			switch reSuffix {
 			case "(.*)":
-				return suffix, true
+				return suffix
 			case "(.+)":
 				if len(suffix) > 0 {
-					return suffix, true
+					return suffix
 				}
-				return s, false
+				return s
 			}
 		}
 	}
-	if re := prc.regex; re.HasPrefix() && !re.MatchString(s) {
+	if !prc.regex.MatchString(s) {
 		// Fast path - regex mismatch
-		return s, false
+		return s
 	}
+	// Slow path - handle the rest of cases.
+	return prc.stringReplacer.Transform(s)
+}
+
+// replaceFullStringSlow replaces s with the replacement if s matches '^regex$'.
+//
+// s is returned as is if it doesn't match '^regex$'.
+func (prc *parsedRelabelConfig) replaceFullStringSlow(s string) string {
 	// Slow path - regexp processing
 	match := prc.RegexAnchored.FindStringSubmatchIndex(s)
 	if match == nil {
-		return s, false
+		return s
 	}
-	bb := relabelBufPool.Get()
-	bb.B = prc.RegexAnchored.ExpandString(bb.B[:0], replacement, s, match)
-	result := string(bb.B)
-	relabelBufPool.Put(bb)
-	return result, true
+	return prc.expandCaptureGroups(prc.Replacement, s, match)
 }
 
-func (prc *parsedRelabelConfig) replaceStringSubmatches(s, replacement string, hasCaptureGroupInReplacement bool) (string, bool) {
-	re := prc.regexOriginal
-	prefix, complete := re.LiteralPrefix()
-	if complete && !hasCaptureGroupInReplacement {
-		if !strings.Contains(s, prefix) {
-			return s, false
-		}
-		return strings.ReplaceAll(s, prefix, replacement), true
-	}
-	if !re.MatchString(s) {
-		return s, false
+// replaceStringSubmatchesFast replaces all the regex matches with the replacement in s.
+func (prc *parsedRelabelConfig) replaceStringSubmatchesFast(s string) string {
+	prefix, complete := prc.regexOriginal.LiteralPrefix()
+	if complete && !prc.hasCaptureGroupInReplacement && !strings.Contains(s, prefix) {
+		// Fast path - zero regex matches in s.
+		return s
 	}
-	return re.ReplaceAllString(s, replacement), true
+	// Slow path - replace all the regex matches in s with the replacement.
+	return prc.submatchReplacer.Transform(s)
+}
+
+// replaceStringSubmatchesSlow replaces all the regex matches with the replacement in s.
+func (prc *parsedRelabelConfig) replaceStringSubmatchesSlow(s string) string {
+	return prc.regexOriginal.ReplaceAllString(s, prc.Replacement)
 }
 
 func (prc *parsedRelabelConfig) expandCaptureGroups(template, source string, match []int) string {
 	bb := relabelBufPool.Get()
 	bb.B = prc.RegexAnchored.ExpandString(bb.B[:0], template, source, match)
-	s := string(bb.B)
+	s := bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
 	relabelBufPool.Put(bb)
 	return s
 }
diff --git a/lib/promscrape/discovery/ec2/instance.go b/lib/promscrape/discovery/ec2/instance.go
index 9cacb2506..e00f1fb6e 100644
--- a/lib/promscrape/discovery/ec2/instance.go
+++ b/lib/promscrape/discovery/ec2/instance.go
@@ -16,10 +16,11 @@ func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) {
 		return nil, err
 	}
 	azMap := getAZMap(cfg)
+	region := cfg.awsConfig.GetRegion()
 	var ms []map[string]string
 	for _, r := range rs {
 		for _, inst := range r.InstanceSet.Items {
-			ms = inst.appendTargetLabels(ms, r.OwnerID, cfg.port, azMap)
+			ms = inst.appendTargetLabels(ms, r.OwnerID, region, cfg.port, azMap)
 		}
 	}
 	return ms, nil
@@ -134,7 +135,7 @@ func parseInstancesResponse(data []byte) (*InstancesResponse, error) {
 	return &v, nil
 }
 
-func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID string, port int, azMap map[string]string) []map[string]string {
+func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID, region string, port int, azMap map[string]string) []map[string]string {
 	if len(inst.PrivateIPAddress) == 0 {
 		// Cannot scrape instance without private IP address
 		return ms
@@ -157,6 +158,7 @@ func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID string,
 		"__meta_ec2_private_ip":           inst.PrivateIPAddress,
 		"__meta_ec2_public_dns_name":      inst.PublicDNSName,
 		"__meta_ec2_public_ip":            inst.PublicIPAddress,
+		"__meta_ec2_region":               region,
 		"__meta_ec2_vpc_id":               inst.VPCID,
 	}
 	if len(inst.VPCID) > 0 {
diff --git a/lib/regexutil/promregex.go b/lib/regexutil/promregex.go
index 96e39ec96..cf196b8cd 100644
--- a/lib/regexutil/promregex.go
+++ b/lib/regexutil/promregex.go
@@ -3,6 +3,8 @@ package regexutil
 import (
 	"regexp"
 	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 // PromRegex implements an optimized string matching for Prometheus-like regex.
@@ -13,6 +15,8 @@ import (
 // - alternate strings such as "foo|bar|baz"
 // - prefix match such as "foo.*" or "foo.+"
 // - substring match such as ".*foo.*" or ".+bar.+"
+//
+// The rest of regexps are also optimized by returning cached match results for the same input strings.
 type PromRegex struct {
 	// prefix contains literal prefix for regex.
 	// For example, prefix="foo" for regex="foo(a|b)"
@@ -32,9 +36,8 @@ type PromRegex struct {
 	// For example, orValues contain ["foo","bar","baz"] for regex suffix="foo|bar|baz"
 	orValues []string
 
-	// reSuffix contains an anchored regexp built from suffix:
-	// "^(?:suffix)$"
-	reSuffix *regexp.Regexp
+	// reSuffixMatcher contains fast matcher for "^suffix$"
+	reSuffixMatcher *bytesutil.FastStringMatcher
 }
 
 // NewPromRegex returns PromRegex for the given expr.
@@ -50,25 +53,18 @@ func NewPromRegex(expr string) (*PromRegex, error) {
 	// Anchor suffix to the beginning and the end of the matching string.
 	suffixExpr := "^(?:" + suffix + ")$"
 	reSuffix := regexp.MustCompile(suffixExpr)
+	reSuffixMatcher := bytesutil.NewFastStringMatcher(reSuffix.MatchString)
 	pr := &PromRegex{
-		prefix:        prefix,
-		suffix:        suffix,
-		substrDotStar: substrDotStar,
-		substrDotPlus: substrDotPlus,
-		orValues:      orValues,
-		reSuffix:      reSuffix,
+		prefix:          prefix,
+		suffix:          suffix,
+		substrDotStar:   substrDotStar,
+		substrDotPlus:   substrDotPlus,
+		orValues:        orValues,
+		reSuffixMatcher: reSuffixMatcher,
 	}
 	return pr, nil
 }
 
-// HasPrefix returns true if pr contains non-empty literal prefix.
-//
-// For example, if pr is "foo(bar|baz)", then the prefix is "foo",
-// so HasPrefix() returns true.
-func (pr *PromRegex) HasPrefix() bool {
-	return len(pr.prefix) > 0
-}
-
 // MatchString retruns true if s matches pr.
 //
 // The pr is automatically anchored to the beginning and to the end
@@ -106,7 +102,7 @@ func (pr *PromRegex) MatchString(s string) bool {
 		return len(s) > 0
 	}
 	// Fall back to slow path by matching the original regexp.
-	return pr.reSuffix.MatchString(s)
+	return pr.reSuffixMatcher.Match(s)
 }
 
 func getSubstringLiteral(expr, prefixSuffix string) string {
diff --git a/lib/storage/tag_filters.go b/lib/storage/tag_filters.go
index 99c8ee171..bc7492590 100644
--- a/lib/storage/tag_filters.go
+++ b/lib/storage/tag_filters.go
@@ -588,13 +588,21 @@ func getOptimizedReMatchFunc(reMatch func(b []byte) bool, expr string) (func(b [
 	if err != nil {
 		logger.Panicf("BUG: unexpected error when parsing verified expr=%q: %s", expr, err)
 	}
-	if matchFunc, literalSuffix, reCost := getOptimizedReMatchFuncExt(reMatch, sre); matchFunc != nil {
+	// Prepare fast string matcher for reMatch.
+	fsm := bytesutil.NewFastStringMatcher(func(s string) bool {
+		return reMatch(bytesutil.ToUnsafeBytes(s))
+	})
+	reMatchFast := func(b []byte) bool {
+		return fsm.Match(bytesutil.ToUnsafeString(b))
+	}
+
+	if matchFunc, literalSuffix, reCost := getOptimizedReMatchFuncExt(reMatchFast, sre); matchFunc != nil {
 		// Found optimized function for matching the expr.
 		suffixUnescaped := tagCharsReverseRegexpEscaper.Replace(literalSuffix)
 		return matchFunc, suffixUnescaped, reCost
 	}
-	// Fall back to un-optimized reMatch.
-	return reMatch, "", reMatchCost
+	// Fall back to reMatchFast.
+	return reMatchFast, "", reMatchCost
 }
 
 // These cost values are used for sorting tag filters in ascending order or the required CPU time for execution.
diff --git a/lib/tenantmetrics/counter_map.go b/lib/tenantmetrics/counter_map.go
index 710986b80..3ab106679 100644
--- a/lib/tenantmetrics/counter_map.go
+++ b/lib/tenantmetrics/counter_map.go
@@ -39,6 +39,13 @@ func (cm *CounterMap) Get(at *auth.Token) *metrics.Counter {
 	return cm.GetByTenant(key)
 }
 
+// MultiAdd adds multiple values grouped by auth.Token
+func (cm *CounterMap) MultiAdd(perTenantValues map[auth.Token]int) {
+	for token, value := range perTenantValues {
+		cm.Get(&token).Add(value)
+	}
+}
+
 // GetByTenant returns counter for the given key.
 func (cm *CounterMap) GetByTenant(key TenantID) *metrics.Counter {
 	m := cm.m.Load().(map[TenantID]*metrics.Counter)


Test patch:
diff --git a/app/victoria-metrics/main_test.go b/app/victoria-metrics/main_test.go
index 805a5567a..d5f27b0a4 100644
--- a/app/victoria-metrics/main_test.go
+++ b/app/victoria-metrics/main_test.go
@@ -138,7 +138,7 @@ func setUp() {
 		if err != nil {
 			return false
 		}
-		resp.Body.Close()
+		_ = resp.Body.Close()
 		return resp.StatusCode == 200
 	}
 	if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil {
@@ -337,7 +337,9 @@ func tcpWrite(t *testing.T, address string, data string) {
 	s := newSuite(t)
 	conn, err := net.Dial("tcp", address)
 	s.noError(err)
-	defer conn.Close()
+	defer func() {
+		_ = conn.Close()
+	}()
 	n, err := conn.Write([]byte(data))
 	s.noError(err)
 	s.equalInt(n, len(data))
@@ -348,7 +350,9 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	s.equalInt(resp.StatusCode, 200)
 	var rows []Metric
 	for dec := json.NewDecoder(resp.Body); dec.More(); {
@@ -363,7 +367,9 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	s.equalInt(resp.StatusCode, 200)
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
diff --git a/app/vmalert/notifier/config_watcher_test.go b/app/vmalert/notifier/config_watcher_test.go
index 8b70df842..61550a626 100644
--- a/app/vmalert/notifier/config_watcher_test.go
+++ b/app/vmalert/notifier/config_watcher_test.go
@@ -8,6 +8,8 @@ import (
 	"os"
 	"sync"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 func TestConfigWatcherReload(t *testing.T) {
@@ -298,3 +300,20 @@ func newFakeConsulServer() *httptest.Server {
 
 	return httptest.NewServer(mux)
 }
+
+func TestMergeHTTPClientConfigs(t *testing.T) {
+	cfg1 := promauth.HTTPClientConfig{Headers: []string{"Header:Foo"}}
+	cfg2 := promauth.HTTPClientConfig{BasicAuth: &promauth.BasicAuthConfig{
+		Username: "foo",
+		Password: promauth.NewSecret("bar"),
+	}}
+
+	result := mergeHTTPClientConfigs(cfg1, cfg2)
+
+	if result.Headers == nil {
+		t.Fatalf("expected Headers to be inherited")
+	}
+	if result.BasicAuth == nil {
+		t.Fatalf("expected BasicAuth tp be present")
+	}
+}
diff --git a/app/vmalert/notifier/testdata/static.good.yaml b/app/vmalert/notifier/testdata/static.good.yaml
index 99cb98eb2..1d95d252d 100644
--- a/app/vmalert/notifier/testdata/static.good.yaml
+++ b/app/vmalert/notifier/testdata/static.good.yaml
@@ -1,7 +1,21 @@
+headers:
+  - 'CustomHeader: foo'
+
 static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
+    basic_auth:
+      username: foo
+      password: bar
+
+  - targets:
+      - localhost:9096
+      - localhost:9097
+    basic_auth:
+      username: foo
+      password: baz
+
 alert_relabel_configs:
   - target_label: "foo"
-    replacement: "aaa"
\ No newline at end of file
+    replacement: "aaa"
diff --git a/lib/auth/auth_test.go b/lib/auth/auth_test.go
index aa2b4b4f9..eddf1c6fe 100644
--- a/lib/auth/auth_test.go
+++ b/lib/auth/auth_test.go
@@ -26,6 +26,8 @@ func TestNewTokenSuccess(t *testing.T) {
 	f("1:4294967295", "1:4294967295")
 	// max uint32 accountID and projectID
 	f("4294967295:4294967295", "4294967295:4294967295")
+	// multitenant
+	f("multitenant", "multitenant")
 }
 
 func TestNewTokenFailure(t *testing.T) {
diff --git a/lib/bytesutil/fast_string_matcher_test.go b/lib/bytesutil/fast_string_matcher_test.go
new file mode 100644
index 000000000..7ea7925ce
--- /dev/null
+++ b/lib/bytesutil/fast_string_matcher_test.go
@@ -0,0 +1,25 @@
+package bytesutil
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFastStringMatcher(t *testing.T) {
+	fsm := NewFastStringMatcher(func(s string) bool {
+		return strings.HasPrefix(s, "foo")
+	})
+	f := func(s string, resultExpected bool) {
+		t.Helper()
+		for i := 0; i < 10; i++ {
+			result := fsm.Match(s)
+			if result != resultExpected {
+				t.Fatalf("unexpected result for Match(%q) at iteration %d; got %v; want %v", s, i, result, resultExpected)
+			}
+		}
+	}
+	f("", false)
+	f("foo", true)
+	f("a_b-C", false)
+	f("foobar", true)
+}
diff --git a/lib/bytesutil/fast_string_matcher_timing_test.go b/lib/bytesutil/fast_string_matcher_timing_test.go
new file mode 100644
index 000000000..7ccd51eed
--- /dev/null
+++ b/lib/bytesutil/fast_string_matcher_timing_test.go
@@ -0,0 +1,33 @@
+package bytesutil
+
+import (
+	"strings"
+	"sync/atomic"
+	"testing"
+)
+
+func BenchmarkFastStringMatcher(b *testing.B) {
+	for _, s := range []string{"", "foo", "foo-bar-baz", "http_requests_total"} {
+		b.Run(s, func(b *testing.B) {
+			benchmarkFastStringMatcher(b, s)
+		})
+	}
+}
+
+func benchmarkFastStringMatcher(b *testing.B, s string) {
+	fsm := NewFastStringMatcher(func(s string) bool {
+		return strings.HasPrefix(s, "foo")
+	})
+	b.ReportAllocs()
+	b.SetBytes(1)
+	b.RunParallel(func(pb *testing.PB) {
+		n := uint64(0)
+		for pb.Next() {
+			v := fsm.Match(s)
+			if v {
+				n++
+			}
+		}
+		atomic.AddUint64(&GlobalSink, n)
+	})
+}
diff --git a/lib/bytesutil/fast_string_transformer_timing_test.go b/lib/bytesutil/fast_string_transformer_timing_test.go
index ed2324fb1..b23fdbc05 100644
--- a/lib/bytesutil/fast_string_transformer_timing_test.go
+++ b/lib/bytesutil/fast_string_transformer_timing_test.go
@@ -2,6 +2,7 @@ package bytesutil
 
 import (
 	"strings"
+	"sync/atomic"
 	"testing"
 )
 
@@ -18,11 +19,13 @@ func benchmarkFastStringTransformer(b *testing.B, s string) {
 	b.ReportAllocs()
 	b.SetBytes(1)
 	b.RunParallel(func(pb *testing.PB) {
+		n := uint64(0)
 		for pb.Next() {
 			sTransformed := fst.Transform(s)
-			GlobalSink += len(sTransformed)
+			n += uint64(len(sTransformed))
 		}
+		atomic.AddUint64(&GlobalSink, n)
 	})
 }
 
-var GlobalSink int
+var GlobalSink uint64
diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go
index 18066c1e2..dcc689b27 100644
--- a/lib/promrelabel/config_test.go
+++ b/lib/promrelabel/config_test.go
@@ -161,6 +161,12 @@ func TestParseRelabelConfigsSuccess(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
+		if pcs != nil {
+			for _, prc := range pcs.prcs {
+				prc.stringReplacer = nil
+				prc.submatchReplacer = nil
+			}
+		}
 		if !reflect.DeepEqual(pcs, pcsExpected) {
 			t.Fatalf("unexpected pcs; got\n%#v\nwant\n%#v", pcs, pcsExpected)
 		}
diff --git a/lib/promrelabel/relabel_timing_test.go b/lib/promrelabel/relabel_timing_test.go
index eaba87047..df588db3b 100644
--- a/lib/promrelabel/relabel_timing_test.go
+++ b/lib/promrelabel/relabel_timing_test.go
@@ -270,22 +270,149 @@ func BenchmarkMatchRegexOrValuesMismatchUnoptimized(b *testing.B) {
 }
 
 func BenchmarkApplyRelabelConfigs(b *testing.B) {
+	b.Run("kubernetes", func(b *testing.B) {
+		// See https://github.com/Arnoways/prometheus/blob/d521933053bdf68d252e365da9376706d04addcc/model/relabel/relabel_test.go#L505
+		pcs := mustParseRelabelConfigs(`
+- source_labels:
+  - __meta_kubernetes_pod_container_port_name
+  regex: .*-metrics
+  action: keep
+- source_labels:
+  - __meta_kubernetes_pod_label_name
+  action: drop
+  regex: ""
+- source_labels:
+  - __meta_kubernetes_pod_phase
+  regex: Succeeded|Failed
+  action: drop
+- source_labels:
+  - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+  regex: "false"
+  action: drop
+- source_labels:
+  - __meta_kubernetes_pod_annotation_prometheus_io_scheme
+  target_label: __scheme__
+  regex: (https?)
+  replacement: $1
+  action: replace
+- source_labels:
+  - __meta_kubernetes_pod_annotation_prometheus_io_path
+  target_label: __metrics_path__
+  regex: (.+)
+  replacement: $1
+  action: replace
+- source_labels:
+  - __address__
+  - __meta_kubernetes_pod_annotation_prometheus_io_port
+  target_label: __address__
+  regex: (.+?)(\:\d+)?;(\d+)
+  replacement: $1:$3
+  action: replace
+- regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
+  replacement: __param_$1
+  action: labelmap
+- regex: __meta_kubernetes_pod_label_prometheus_io_label_(.+)
+  action: labelmap
+- regex: __meta_kubernetes_pod_annotation_prometheus_io_label_(.+)
+  action: labelmap
+- source_labels:
+  - __meta_kubernetes_namespace
+  - __meta_kubernetes_pod_label_name
+  separator: /
+  target_label: job
+  replacement: $1
+  action: replace
+- source_labels:
+  - __meta_kubernetes_namespace
+  target_label: namespace
+  action: replace
+- source_labels:
+  - __meta_kubernetes_pod_name
+  target_label: pod
+  action: replace
+- source_labels:
+  - __meta_kubernetes_pod_container_name
+  target_label: container
+  action: replace
+- source_labels:
+  - __meta_kubernetes_pod_name
+  - __meta_kubernetes_pod_container_name
+  - __meta_kubernetes_pod_container_port_name
+  separator: ':'
+  target_label: instance
+  action: replace
+- target_label: cluster
+  replacement: dev-us-central-0
+- source_labels:
+  - __meta_kubernetes_namespace
+  regex: hosted-grafana
+  action: drop
+- source_labels:
+  - __address__
+  target_label: __tmp_hash
+  modulus: 3
+  action: hashmod
+- source_labels:
+  - __tmp_hash
+  regex: ^0$
+  action: keep
+- regex: __tmp_hash
+  action: labeldrop
+`)
+		labelsOrig := labelsFromStrings(
+			"__address__", "10.132.183.40:80",
+			"__meta_kubernetes_namespace", "loki-boltdb-shipper",
+			"__meta_kubernetes_pod_annotation_promtail_loki_boltdb_shipper_hash", "50523b9759094a144adcec2eae0aa4ad",
+			"__meta_kubernetes_pod_annotationpresent_promtail_loki_boltdb_shipper_hash", "true",
+			"__meta_kubernetes_pod_container_init", "false",
+			"__meta_kubernetes_pod_container_name", "promtail",
+			"__meta_kubernetes_pod_container_port_name", "http-metrics",
+			"__meta_kubernetes_pod_container_port_number", "80",
+			"__meta_kubernetes_pod_container_port_protocol", "TCP",
+			"__meta_kubernetes_pod_controller_kind", "DaemonSet",
+			"__meta_kubernetes_pod_controller_name", "promtail-loki-boltdb-shipper",
+			"__meta_kubernetes_pod_host_ip", "10.128.0.178",
+			"__meta_kubernetes_pod_ip", "10.132.183.40",
+			"__meta_kubernetes_pod_label_controller_revision_hash", "555b77cd7d",
+			"__meta_kubernetes_pod_label_name", "promtail-loki-boltdb-shipper",
+			"__meta_kubernetes_pod_label_pod_template_generation", "45",
+			"__meta_kubernetes_pod_labelpresent_controller_revision_hash", "true",
+			"__meta_kubernetes_pod_labelpresent_name", "true",
+			"__meta_kubernetes_pod_labelpresent_pod_template_generation", "true",
+			"__meta_kubernetes_pod_name", "promtail-loki-boltdb-shipper-jgtr7",
+			"__meta_kubernetes_pod_node_name", "gke-dev-us-central-0-main-n2s8-2-14d53341-9hkr",
+			"__meta_kubernetes_pod_phase", "Running",
+			"__meta_kubernetes_pod_ready", "true",
+			"__meta_kubernetes_pod_uid", "4c586419-7f6c-448d-aeec-ca4fa5b05e60",
+			"__metrics_path__", "/metrics",
+			"__scheme__", "http",
+			"__scrape_interval__", "15s",
+			"__scrape_timeout__", "10s",
+			"job", "kubernetes-pods",
+		)
+		b.ReportAllocs()
+		b.SetBytes(1)
+		b.RunParallel(func(pb *testing.PB) {
+			var labels []prompbmarshal.Label
+			for pb.Next() {
+				labels = append(labels[:0], labelsOrig...)
+				labels = pcs.Apply(labels, 0, false)
+				if len(labels) != 0 {
+					panic(fmt.Errorf("BUG: expecting empty labels"))
+				}
+			}
+		})
+	})
 	b.Run("replace-label-copy", func(b *testing.B) {
 		pcs := mustParseRelabelConfigs(`
 - action: replace
   source_labels: [id]
   target_label: __name__
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -294,7 +421,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 				labels = append(labels[:0], labelsOrig...)
 				labels = pcs.Apply(labels, 0, true)
 				if len(labels) != len(labelsOrig) {
-					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labels))
+					panic(fmt.Errorf("unexpected number of labels; got %d; want %d; labels:\n%#v", len(labels), len(labelsOrig), labelsOrig))
 				}
 				if labels[0].Name != "__name__" {
 					panic(fmt.Errorf("unexpected label name; got %q; want %q", labels[0].Name, "__name__"))
@@ -317,16 +444,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   target_label: __name__
   replacement: foobar
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -358,12 +479,9 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   target_label: aaa
   replacement: foobar
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -396,16 +514,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   target_label: id
   regex: "(foobar)-.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -438,16 +550,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   target_label: id
   regex: "(foobar)-.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -479,16 +585,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   source_labels: ["non-existing-label"]
   regex: "(foobar)-.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -520,16 +620,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   source_labels: [id]
   regex: "yes"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "yes",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "yes",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -549,16 +643,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   source_labels: [id]
   regex: "(foobar)-.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -578,16 +666,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   source_labels: ["non-existing-label"]
   regex: "(foobar)-.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -607,16 +689,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   source_labels: [id]
   regex: "yes"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "yes",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "yes",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -648,16 +724,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   source_labels: [id]
   regex: "(foobar)-.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -688,16 +758,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labeldrop
   regex: "non-existing-label"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -728,16 +792,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labeldrop
   regex: id
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -762,16 +820,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labeldrop
   regex: "id.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -796,16 +848,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labeldrop
   regex: ".*id.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -830,16 +876,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labelkeep
   regex: "non-existing-label"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -858,16 +898,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labelkeep
   regex: id
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -892,16 +926,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labelkeep
   regex: "id.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -926,16 +954,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labelkeep
   regex: ".*id.*"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -960,12 +982,9 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labelmap
   regex: "a(.*)"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "foo",
-				Value: "bar",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"foo", "bar",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -990,12 +1009,9 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labelmap
   regex: "a(.*)"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "aabc",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"aabc", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -1026,12 +1042,9 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
 - action: labelmap
   regex: "(.*)bc"
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "aabc",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"aabc", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -1064,16 +1077,10 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) {
   target_label: id
   modulus: 23
 `)
-		labelsOrig := []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "metric",
-			},
-			{
-				Name:  "id",
-				Value: "foobar-random-string-here",
-			},
-		}
+		labelsOrig := labelsFromStrings(
+			"__name__", "metric",
+			"id", "foobar-random-string-here",
+		)
 		b.ReportAllocs()
 		b.SetBytes(1)
 		b.RunParallel(func(pb *testing.PB) {
@@ -1108,3 +1115,15 @@ func mustParseRelabelConfigs(config string) *ParsedConfigs {
 	}
 	return pcs
 }
+
+func labelsFromStrings(ss ...string) []prompbmarshal.Label {
+	labelsLen := len(ss) / 2
+	labels := make([]prompbmarshal.Label, 0, labelsLen)
+	for i := 0; i < len(ss); i += 2 {
+		labels = append(labels, prompbmarshal.Label{
+			Name:  ss[i],
+			Value: ss[i+1],
+		})
+	}
+	return labels
+}
diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go
index 7a728b428..7f256f90f 100644
--- a/lib/promscrape/config_test.go
+++ b/lib/promscrape/config_test.go
@@ -674,8 +674,9 @@ scrape_config_files:
 }
 
 func resetNonEssentialFields(sws []*ScrapeWork) {
-	for i := range sws {
-		sws[i].OriginalLabels = nil
+	for _, sw := range sws {
+		sw.OriginalLabels = nil
+		sw.MetricRelabelConfigs = nil
 	}
 }
 
@@ -1446,10 +1447,6 @@ scrape_configs:
 			},
 			AuthConfig:      &promauth.Config{},
 			ProxyAuthConfig: &promauth.Config{},
-			MetricRelabelConfigs: mustParseRelabelConfigs(`
-- source_labels: [foo]
-  target_label: abc
-`),
 			jobNameOriginal: "foo",
 		},
 	})
@@ -1847,8 +1844,10 @@ func TestScrapeConfigClone(t *testing.T) {
 	f := func(sc *ScrapeConfig) {
 		t.Helper()
 		scCopy := sc.clone()
-		if !reflect.DeepEqual(sc, scCopy) {
-			t.Fatalf("unexpected result after unmarshalJSON() for JSON:\n%s", sc.marshalJSON())
+		scJSON := sc.marshalJSON()
+		scCopyJSON := scCopy.marshalJSON()
+		if !reflect.DeepEqual(scJSON, scCopyJSON) {
+			t.Fatalf("unexpected cloned result:\ngot\n%s\nwant\n%s", scCopyJSON, scJSON)
 		}
 	}
 
diff --git a/lib/promscrape/discovery/ec2/instance_test.go b/lib/promscrape/discovery/ec2/instance_test.go
index a8441a96c..b6667db89 100644
--- a/lib/promscrape/discovery/ec2/instance_test.go
+++ b/lib/promscrape/discovery/ec2/instance_test.go
@@ -238,7 +238,7 @@ func TestParseInstancesResponse(t *testing.T) {
 	ownerID := rs.OwnerID
 	port := 423
 	inst := rs.InstanceSet.Items[0]
-	labelss := inst.appendTargetLabels(nil, ownerID, port, map[string]string{
+	labelss := inst.appendTargetLabels(nil, ownerID, "region-a", port, map[string]string{
 		"eu-west-2c": "foobar-zone",
 	})
 	var sortedLabelss [][]prompbmarshal.Label
@@ -263,6 +263,7 @@ func TestParseInstancesResponse(t *testing.T) {
 			"__meta_ec2_private_ip":           "172.31.11.152",
 			"__meta_ec2_public_dns_name":      "ec2-3-8-232-141.eu-west-2.compute.amazonaws.com",
 			"__meta_ec2_public_ip":            "3.8.232.141",
+			"__meta_ec2_region":               "region-a",
 			"__meta_ec2_subnet_id":            ",subnet-57044c3e,",
 			"__meta_ec2_tag_foo":              "bar",
 			"__meta_ec2_vpc_id":               "vpc-f1eaad99",